From 1a18dd4635f3eb6289992ef54783579a4b51199c Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Tue, 7 Jul 2026 17:53:35 +0100 Subject: [PATCH 001/633] Bump golang.org/x/net to v0.56.0 Clears GHSA-5cv4-jp36-h3mw / CVE-2026-25680 (HTML parser DoS, only reachable via golangci-lint tooling deps) and GO-2026-5026 (idna Punycode validation, reachable via http.Client.Do). Also pulls x/crypto, x/sys and x/text forward as required by x/net. --- go.mod | 8 ++++---- go.sum | 16 ++++++++-------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/go.mod b/go.mod index 471ab53..9576fda 100644 --- a/go.mod +++ b/go.mod @@ -291,14 +291,14 @@ require ( go.uber.org/zap v1.27.1 // indirect go.yaml.in/yaml/v2 v2.4.3 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect - golang.org/x/crypto v0.51.0 // indirect + golang.org/x/crypto v0.53.0 // indirect golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa // indirect golang.org/x/exp/typeparams v0.0.0-20260209203927-2842357ff358 // indirect golang.org/x/mod v0.36.0 // indirect - golang.org/x/net v0.54.0 // indirect + golang.org/x/net v0.56.0 // indirect golang.org/x/oauth2 v0.36.0 // indirect - golang.org/x/sys v0.45.0 // indirect - golang.org/x/text v0.37.0 // indirect + golang.org/x/sys v0.46.0 // indirect + golang.org/x/text v0.38.0 // indirect golang.org/x/tools v0.45.0 // indirect golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect google.golang.org/api v0.272.0 // indirect diff --git a/go.sum b/go.sum index 8d8c96d..93bed13 100644 --- a/go.sum +++ b/go.sum @@ -738,8 +738,8 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= -golang.org/x/crypto v0.51.0 h1:IBPXwPfKxY7cWQZ38ZCIRPI50YLeevDLlLnyC5wRGTI= -golang.org/x/crypto v0.51.0/go.mod h1:8AdwkbraGNABw2kOX6YFPs3WM22XqI4EXEd8g+x7Oc8= +golang.org/x/crypto v0.53.0 h1:QZ4Muo8THX6CizN2vPPd5fBGHyogrdK9fG4wLPFUsto= +golang.org/x/crypto v0.53.0/go.mod h1:DNLU434OwVakk9PzuwV8w62mAJpRJL3vsgcfp4Qnsio= golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa h1:Zt3DZoOFFYkKhDT3v7Lm9FDMEV06GpzjG2jrqW+QTE0= golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa/go.mod h1:K79w1Vqn7PoiZn+TkNpx3BUWUQksGO3JcVX6qIjytmA= golang.org/x/exp/typeparams v0.0.0-20220428152302-39d4317da171/go.mod h1:AbB0pIl9nAr9wVwH+Z2ZpaocVmF5I4GyWCDIsVjR0bk= @@ -771,8 +771,8 @@ golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= golang.org/x/net v0.16.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= -golang.org/x/net v0.54.0 h1:2zJIZAxAHV/OHCDTCOHAYehQzLfSXuf/5SoL/Dv6w/w= -golang.org/x/net v0.54.0/go.mod h1:Sj4oj8jK6XmHpBZU/zWHw3BV3abl4Kvi+Ut7cQcY+cQ= +golang.org/x/net v0.56.0 h1:Rw8j/hFzGvJUZwNBXnAtf5sVDVt+65SK2C7IxCxZt5o= +golang.org/x/net v0.56.0/go.mod h1:D3Ku6r+V6JROoZK144D2XfMHFcMq/0zSfLelVTCFKec= golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -807,8 +807,8 @@ golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY= -golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw= +golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= @@ -823,8 +823,8 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= -golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc= -golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38= +golang.org/x/text v0.38.0 h1:sXmwo9DwP3OK9EZ7PqAdaooSGozfl/3a6/xJcbzPRhE= +golang.org/x/text v0.38.0/go.mod h1:YXZt3QhHUKYT53r2lLKFIVi6Ao1jdzrTR/KQ09qyxF4= golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= From df3ad9184821b60efcbb5198d7b65b6dd2949d1f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 9 Jul 2026 15:13:56 +0000 Subject: [PATCH 002/633] Bump docker/build-push-action from 7.2.0 to 7.3.0 Bumps [docker/build-push-action](https://github.com/docker/build-push-action) from 7.2.0 to 7.3.0. - [Release notes](https://github.com/docker/build-push-action/releases) - [Commits](https://github.com/docker/build-push-action/compare/f9f3042f7e2789586610d6e8b85c8f03e5195baf...53b7df96c91f9c12dcc8a07bcb9ccacbed38856a) --- updated-dependencies: - dependency-name: docker/build-push-action dependency-version: 7.3.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 93b0a1e..a4e105d 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -38,7 +38,7 @@ jobs: images: ghcr.io/${{ github.repository }} - name: Build and push Docker image - uses: docker/build-push-action@f9f3042f7e2789586610d6e8b85c8f03e5195baf + uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a with: context: . push: true From f7b5dd2028e80f420cdf1284ccb74f3b5ce4f6b4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 9 Jul 2026 15:14:04 +0000 Subject: [PATCH 003/633] Bump golang.org/x/sync from 0.21.0 to 0.22.0 Bumps [golang.org/x/sync](https://github.com/golang/sync) from 0.21.0 to 0.22.0. - [Commits](https://github.com/golang/sync/compare/v0.21.0...v0.22.0) --- updated-dependencies: - dependency-name: golang.org/x/sync dependency-version: 0.22.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 9576fda..b063dba 100644 --- a/go.mod +++ b/go.mod @@ -21,7 +21,7 @@ require ( github.com/spdx/tools-golang v0.5.7 github.com/swaggo/swag v1.16.6 gocloud.dev v0.46.0 - golang.org/x/sync v0.21.0 + golang.org/x/sync v0.22.0 google.golang.org/protobuf v1.36.11 gopkg.in/yaml.v3 v3.0.1 modernc.org/sqlite v1.53.0 diff --git a/go.sum b/go.sum index 93bed13..b1d35f6 100644 --- a/go.sum +++ b/go.sum @@ -784,8 +784,8 @@ golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sync v0.4.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= -golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM= -golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= +golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= From ea4eea4b113c41523a8a48f40158f0d87ba577a3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 9 Jul 2026 15:14:07 +0000 Subject: [PATCH 004/633] Bump docker/login-action from 4.2.0 to 4.3.0 Bumps [docker/login-action](https://github.com/docker/login-action) from 4.2.0 to 4.3.0. - [Release notes](https://github.com/docker/login-action/releases) - [Commits](https://github.com/docker/login-action/compare/650006c6eb7dba73a995cc03b0b2d7f5ca915bee...c99871dec2022cc055c062a10cc1a1310835ceb4) --- updated-dependencies: - dependency-name: docker/login-action dependency-version: 4.3.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 93b0a1e..f9c32a5 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -25,7 +25,7 @@ jobs: persist-credentials: false - name: Log in to the Container registry - uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee + uses: docker/login-action@c99871dec2022cc055c062a10cc1a1310835ceb4 with: registry: ghcr.io username: ${{ github.actor }} From defe1ec96db676d7833dabb65e5540ebf539000f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 9 Jul 2026 15:14:16 +0000 Subject: [PATCH 005/633] Bump goreleaser/goreleaser-action from 7.2.2 to 7.2.3 Bumps [goreleaser/goreleaser-action](https://github.com/goreleaser/goreleaser-action) from 7.2.2 to 7.2.3. - [Release notes](https://github.com/goreleaser/goreleaser-action/releases) - [Commits](https://github.com/goreleaser/goreleaser-action/compare/5daf1e915a5f0af01ddbcd89a43b8061ff4f1a89...f06c13b6b1a9625abc9e6e439d9c05a8f2190e94) --- updated-dependencies: - dependency-name: goreleaser/goreleaser-action dependency-version: 7.2.3 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f7833fb..4d4d0b5 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -27,7 +27,7 @@ jobs: go-version-file: go.mod cache: false - - uses: goreleaser/goreleaser-action@5daf1e915a5f0af01ddbcd89a43b8061ff4f1a89 # v7.2.2 + - uses: goreleaser/goreleaser-action@f06c13b6b1a9625abc9e6e439d9c05a8f2190e94 # v7.2.3 with: version: "~> v2" args: release --clean From 383d2089cf30e313d46f656c49c4fcddc573759e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 9 Jul 2026 15:14:26 +0000 Subject: [PATCH 006/633] Bump docker/metadata-action from 6.1.0 to 6.2.0 Bumps [docker/metadata-action](https://github.com/docker/metadata-action) from 6.1.0 to 6.2.0. - [Release notes](https://github.com/docker/metadata-action/releases) - [Commits](https://github.com/docker/metadata-action/compare/80c7e94dd9b9319bd5eb7a0e0fe9291e23a2a2e9...dc802804100637a589fabce1cb79ff13a1411302) --- updated-dependencies: - dependency-name: docker/metadata-action dependency-version: 6.2.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 93b0a1e..b2288be 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -33,7 +33,7 @@ jobs: - name: Extract metadata (tags, labels) for Docker id: meta - uses: docker/metadata-action@80c7e94dd9b9319bd5eb7a0e0fe9291e23a2a2e9 + uses: docker/metadata-action@dc802804100637a589fabce1cb79ff13a1411302 with: images: ghcr.io/${{ github.repository }} From 41465968cbaf3e52cd2b552b9c7610000224a54d Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Thu, 9 Jul 2026 17:36:18 +0100 Subject: [PATCH 007/633] Validate filesystem storage paths --- internal/storage/filesystem.go | 30 +++++++++++--- internal/storage/filesystem_test.go | 62 +++++++++++++++++++++++++---- 2 files changed, 79 insertions(+), 13 deletions(-) diff --git a/internal/storage/filesystem.go b/internal/storage/filesystem.go index 1e5a24f..d509e9d 100644 --- a/internal/storage/filesystem.go +++ b/internal/storage/filesystem.go @@ -34,11 +34,31 @@ func NewFilesystem(root string) (*Filesystem, error) { } func (fs *Filesystem) fullPath(path string) (string, error) { - full := filepath.Clean(filepath.Join(fs.root, filepath.FromSlash(path))) - if full != fs.root && !strings.HasPrefix(full, fs.root+string(filepath.Separator)) { - return "", fmt.Errorf("%w: path escapes storage root", ErrNotFound) + localPath, err := cleanStoragePath(path) + if err != nil { + return "", err } - return full, nil + return filepath.Join(fs.root, localPath), nil +} + +func (fs *Filesystem) prefixPath(prefix string) (string, error) { + if prefix == "" { + return fs.root, nil + } + return fs.fullPath(prefix) +} + +func cleanStoragePath(path string) (string, error) { + if path == "." || strings.Contains(path, `\`) || !filepath.IsLocal(path) { + return "", fmt.Errorf("%w: invalid storage path", ErrNotFound) + } + + localPath, err := filepath.Localize(path) + if err != nil || localPath == "." || !filepath.IsLocal(localPath) { + return "", fmt.Errorf("%w: invalid storage path", ErrNotFound) + } + + return localPath, nil } func (fs *Filesystem) Store(ctx context.Context, path string, r io.Reader) (int64, string, error) { @@ -190,7 +210,7 @@ func (fs *Filesystem) UsedSpace(ctx context.Context) (int64, error) { // ListPrefix returns object metadata for paths under a prefix. func (fs *Filesystem) ListPrefix(ctx context.Context, prefix string) ([]ObjectInfo, error) { - searchRoot, err := fs.fullPath(prefix) + searchRoot, err := fs.prefixPath(prefix) if err != nil { return nil, err } diff --git a/internal/storage/filesystem_test.go b/internal/storage/filesystem_test.go index 332dfbf..10d073b 100644 --- a/internal/storage/filesystem_test.go +++ b/internal/storage/filesystem_test.go @@ -243,19 +243,65 @@ func TestFilesystemLargeFile(t *testing.T) { assertLargeFileRoundTrip(t, createTestFilesystem(t)) } -func TestFilesystemRejectsTraversal(t *testing.T) { +func TestFilesystemRejectsInvalidPaths(t *testing.T) { tmp := t.TempDir() fs, err := NewFilesystem(tmp) if err != nil { t.Fatal(err) } - for _, p := range []string{"../etc/passwd", "../../etc/passwd", "a/../../etc/passwd"} { - if _, err := fs.Open(context.Background(), p); err == nil { - t.Errorf("Open(%q) should reject traversal", p) - } - if _, _, err := fs.Store(context.Background(), p, strings.NewReader("x")); err == nil { - t.Errorf("Store(%q) should reject traversal", p) - } + for _, p := range []string{ + "", + ".", + "../etc/passwd", + "../../etc/passwd", + "a/../../etc/passwd", + "/etc/passwd", + "test//file.txt", + "test/./file.txt", + "test/../file.txt", + `test\..\file.txt`, + } { + t.Run(p, func(t *testing.T) { + ctx := context.Background() + + if _, err := fs.FullPath(p); !errors.Is(err, ErrNotFound) { + t.Errorf("FullPath(%q) = %v, want ErrNotFound", p, err) + } + if _, err := fs.Open(ctx, p); err == nil { + t.Errorf("Open(%q) should reject invalid path", p) + } + if _, _, err := fs.Store(ctx, p, strings.NewReader("x")); err == nil { + t.Errorf("Store(%q) should reject invalid path", p) + } + if _, err := fs.Exists(ctx, p); err == nil { + t.Errorf("Exists(%q) should reject invalid path", p) + } + if err := fs.Delete(ctx, p); err == nil { + t.Errorf("Delete(%q) should reject invalid path", p) + } + if _, err := fs.Size(ctx, p); err == nil { + t.Errorf("Size(%q) should reject invalid path", p) + } + if _, err := fs.ListPrefix(ctx, p); p != "" && err == nil { + t.Errorf("ListPrefix(%q) should reject invalid path", p) + } + }) + } +} + +func TestFilesystemListPrefixAllowsEmptyPrefix(t *testing.T) { + fs := createTestFilesystem(t) + ctx := context.Background() + + _, _, _ = fs.Store(ctx, "a.txt", strings.NewReader("aaaa")) + _, _, _ = fs.Store(ctx, "c/d.txt", strings.NewReader("ccccc")) + + objects, err := fs.ListPrefix(ctx, "") + if err != nil { + t.Fatalf("ListPrefix empty prefix failed: %v", err) + } + if len(objects) != 2 { + t.Fatalf("ListPrefix empty prefix returned %d objects, want 2", len(objects)) } } From 98150ea576a32792e27fbf2adf2bbe596fff7573 Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Thu, 9 Jul 2026 17:41:05 +0100 Subject: [PATCH 008/633] Name empty path validation subtest --- internal/storage/filesystem_test.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/internal/storage/filesystem_test.go b/internal/storage/filesystem_test.go index 10d073b..de0e418 100644 --- a/internal/storage/filesystem_test.go +++ b/internal/storage/filesystem_test.go @@ -261,7 +261,12 @@ func TestFilesystemRejectsInvalidPaths(t *testing.T) { "test/../file.txt", `test\..\file.txt`, } { - t.Run(p, func(t *testing.T) { + name := p + if name == "" { + name = "empty" + } + + t.Run(name, func(t *testing.T) { ctx := context.Background() if _, err := fs.FullPath(p); !errors.Is(err, ErrNotFound) { From 5b959d14b95c1b3efd801cab570cd5e1cbfdcccb Mon Sep 17 00:00:00 2001 From: ychampion Date: Fri, 10 Jul 2026 22:59:15 +0000 Subject: [PATCH 009/633] Preserve Go proxy fallback for missing modules Return the repository-standard not-found response when an upstream module is absent. Constraint: GOPROXY advances to direct only after 404 or 410 responses. Rejected: Preserve a handler-specific response body | sibling handlers consistently use not found. Confidence: high Scope-risk: narrow Directive: Keep missing-artifact responses consistent across registry handlers. Tested: go test ./internal/handler -run TestGoModuleDownloadUpstreamErrors -count=1; prior go test -race ./...; go build ./...; golangci-lint; go vet. --- internal/handler/go.go | 7 +++++++ internal/handler/go_test.go | 40 +++++++++++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+) diff --git a/internal/handler/go.go b/internal/handler/go.go index 955a89c..cf40aa1 100644 --- a/internal/handler/go.go +++ b/internal/handler/go.go @@ -1,9 +1,12 @@ package handler import ( + "errors" "fmt" "net/http" "strings" + + "github.com/git-pkgs/registries/fetch" ) const ( @@ -100,6 +103,10 @@ func (h *GoHandler) handleDownload(w http.ResponseWriter, r *http.Request, modul result, err := h.proxy.GetOrFetchArtifact(r.Context(), "golang", decodedModule, version, filename) if err != nil { + if errors.Is(err, fetch.ErrNotFound) { + http.Error(w, "not found", http.StatusNotFound) + return + } h.proxy.Logger.Error("failed to get artifact", "error", err) http.Error(w, "failed to fetch module", http.StatusBadGateway) return diff --git a/internal/handler/go_test.go b/internal/handler/go_test.go index da4ea63..ae998c9 100644 --- a/internal/handler/go_test.go +++ b/internal/handler/go_test.go @@ -1,9 +1,49 @@ package handler import ( + "errors" + "net/http" + "net/http/httptest" "testing" + + "github.com/git-pkgs/registries/fetch" ) +func TestGoModuleDownloadUpstreamErrors(t *testing.T) { + tests := []struct { + name string + fetchErr error + wantStatus int + }{ + { + name: "module not found", + fetchErr: fetch.ErrNotFound, + wantStatus: http.StatusNotFound, + }, + { + name: "upstream failure", + fetchErr: errors.New("connection refused"), + wantStatus: http.StatusBadGateway, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + proxy, _, _, fetcher := setupTestProxy(t) + fetcher.fetchErr = tt.fetchErr + handler := NewGoHandler(proxy, "http://localhost:8080") + + req := httptest.NewRequest(http.MethodGet, "/example.com/mod/@v/v1.0.0.zip", nil) + resp := httptest.NewRecorder() + handler.Routes().ServeHTTP(resp, req) + + if resp.Code != tt.wantStatus { + t.Fatalf("status = %d, want %d", resp.Code, tt.wantStatus) + } + }) + } +} + func TestDecodeGoModule(t *testing.T) { tests := []struct { encoded string From be15a0f826333d8eba32f9a34b7fcaed67d150c0 Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Mon, 13 Jul 2026 17:11:04 -0700 Subject: [PATCH 010/633] Fix cooldown overrides for scoped package PURLs --- config.example.yaml | 3 ++- docs/configuration.md | 2 ++ internal/config/config.go | 31 +++++++++++++++++++++++++++++++ internal/config/config_test.go | 28 ++++++++++++++++++++++++++++ internal/server/server.go | 2 +- 5 files changed, 64 insertions(+), 2 deletions(-) diff --git a/config.example.yaml b/config.example.yaml index d1ed9a1..1176f5b 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -168,7 +168,8 @@ cooldown: # npm: "7d" # cargo: "0" - # Per-package overrides (keyed by PURL) + # Per-package overrides (keyed by PURL). Keys are normalized, so npm scopes + # may use either @scope or the canonical %40scope form. # packages: # "pkg:npm/lodash": "0" # "pkg:npm/@babel/core": "14d" diff --git a/docs/configuration.md b/docs/configuration.md index dcdec24..b103253 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -234,6 +234,8 @@ cooldown: Durations support days (`7d`), hours (`48h`), and minutes (`30m`). Set to `0` to disable. +Package PURL keys are normalized automatically. For npm scopes, both the readable form (`pkg:npm/@babel/core`) and canonical form (`pkg:npm/%40babel/core`) are accepted. If both forms configure the same package, the canonical entry wins. + Resolution order: package override, then ecosystem override, then global default. This lets you set a conservative default while exempting trusted packages. Currently supported for npm, PyPI, pub.dev, Composer, Cargo, NuGet, Conda, RubyGems, and Hex. These ecosystems include publish timestamps in their metadata. diff --git a/internal/config/config.go b/internal/config/config.go index 2d275c7..ec510d2 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -54,10 +54,12 @@ import ( "net/url" "os" "path/filepath" + "sort" "strconv" "strings" "time" + "github.com/git-pkgs/purl" "gopkg.in/yaml.v3" ) @@ -137,9 +139,38 @@ type CooldownConfig struct { Ecosystems map[string]string `json:"ecosystems" yaml:"ecosystems"` // Packages overrides the cooldown for specific packages (keyed by PURL). + // Valid PURL keys are normalized to canonical form before use. Packages map[string]string `json:"packages" yaml:"packages"` } +// NormalizedPackages returns a copy of the package overrides with valid PURL +// keys in canonical form. An explicitly canonical key wins over an equivalent +// noncanonical key, and invalid keys are preserved unchanged. +func (c *CooldownConfig) NormalizedPackages() map[string]string { + if c == nil || c.Packages == nil { + return nil + } + + keys := make([]string, 0, len(c.Packages)) + for key := range c.Packages { + keys = append(keys, key) + } + sort.Strings(keys) + + normalized := make(map[string]string, len(c.Packages)) + for _, key := range keys { + canonical := key + if parsed, err := purl.Parse(key); err == nil { + canonical = parsed.String() + } + if _, exists := normalized[canonical]; exists && key != canonical { + continue + } + normalized[canonical] = c.Packages[key] + } + return normalized +} + // StorageConfig configures artifact storage. type StorageConfig struct { // URL is the storage backend URL. diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 9c34023..decc18f 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -363,6 +363,34 @@ cooldown: if cfg.Cooldown.Packages["pkg:npm/@babel/core"] != "14d" { t.Errorf("Cooldown.Packages[@babel/core] = %q, want %q", cfg.Cooldown.Packages["pkg:npm/@babel/core"], "14d") } + if got := cfg.Cooldown.NormalizedPackages()["pkg:npm/%40babel/core"]; got != "14d" { + t.Errorf("normalized Cooldown.Packages[@babel/core] = %q, want %q", got, "14d") + } +} + +func TestCooldownConfigNormalizedPackages(t *testing.T) { + rawScoped := "pkg:npm/@typescript/typescript-darwin-arm64" + canonicalScoped := "pkg:npm/%40typescript/typescript-darwin-arm64" + cfg := CooldownConfig{Packages: map[string]string{ + rawScoped: "2d", + canonicalScoped: "3d", + "not-a-purl": "4d", + }} + + got := cfg.NormalizedPackages() + + if got[canonicalScoped] != "3d" { + t.Errorf("canonical scoped package duration = %q, want %q", got[canonicalScoped], "3d") + } + if _, exists := got[rawScoped]; exists { + t.Errorf("raw scoped package key %q was not canonicalized", rawScoped) + } + if got["not-a-purl"] != "4d" { + t.Errorf("invalid PURL duration = %q, want preserved value %q", got["not-a-purl"], "4d") + } + if cfg.Packages[rawScoped] != "2d" { + t.Error("NormalizedPackages mutated the source map") + } } func TestLoadCooldownFromEnv(t *testing.T) { diff --git a/internal/server/server.go b/internal/server/server.go index 5aac4db..74c82c7 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -163,7 +163,7 @@ func (s *Server) Start() error { cd := &cooldown.Config{ Default: s.cfg.Cooldown.Default, Ecosystems: s.cfg.Cooldown.Ecosystems, - Packages: s.cfg.Cooldown.Packages, + Packages: s.cfg.Cooldown.NormalizedPackages(), } proxy := handler.NewProxy(s.db, s.storage, fetcher, resolver, s.logger) proxy.HTTPClient.Timeout = s.cfg.ParseHTTPTimeout() From cdf075695f00be2fc187068875b759e872218e63 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 16 Jul 2026 15:14:00 +0000 Subject: [PATCH 011/633] Bump golang from 1.26.4-alpine to 1.26.5-alpine Bumps golang from 1.26.4-alpine to 1.26.5-alpine. --- updated-dependencies: - dependency-name: golang dependency-version: 1.26.5-alpine dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 7b2795c..c2e197f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.26.4-alpine AS builder +FROM golang:1.26.5-alpine AS builder WORKDIR /src From ec3cc3579599246742c022bbec9157330eb30129 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 16 Jul 2026 15:14:06 +0000 Subject: [PATCH 012/633] Bump github.com/go-chi/chi/v5 from 5.3.0 to 5.3.1 Bumps [github.com/go-chi/chi/v5](https://github.com/go-chi/chi) from 5.3.0 to 5.3.1. - [Release notes](https://github.com/go-chi/chi/releases) - [Changelog](https://github.com/go-chi/chi/blob/master/CHANGELOG.md) - [Commits](https://github.com/go-chi/chi/compare/v5.3.0...v5.3.1) --- updated-dependencies: - dependency-name: github.com/go-chi/chi/v5 dependency-version: 5.3.1 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index b063dba..bae89fb 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/git-pkgs/spdx v0.1.4 github.com/git-pkgs/vers v0.2.6 github.com/git-pkgs/vulns v0.1.6 - github.com/go-chi/chi/v5 v5.3.0 + github.com/go-chi/chi/v5 v5.3.1 github.com/jmoiron/sqlx v1.4.0 github.com/lib/pq v1.12.3 github.com/prometheus/client_golang v1.23.2 diff --git a/go.sum b/go.sum index b1d35f6..e9d1e26 100644 --- a/go.sum +++ b/go.sum @@ -266,8 +266,8 @@ github.com/git-pkgs/vulns v0.1.6 h1:8RRSgdlxp4JMU0Zykr63XTOMo5CyZKwt/PwaQxrx9Yg= github.com/git-pkgs/vulns v0.1.6/go.mod h1:TsZC4MjoCkKJslgmbcmRCnytwnFcjESC2N8b0a2xDWc= github.com/github/go-spdx/v2 v2.7.0 h1:GzfXx4wFdlilARxmFRXW/mgUy3A4vSqZocCMFV6XFdQ= github.com/github/go-spdx/v2 v2.7.0/go.mod h1:Ftc45YYG1WzpzwEPKRVm9Jv8vDqOrN4gWoCkK+bHer0= -github.com/go-chi/chi/v5 v5.3.0 h1:halUjDxhshgXHMrao5bB8eNBXo/rnzwr8m5m36glehM= -github.com/go-chi/chi/v5 v5.3.0/go.mod h1:R+tYY2hNuVUUjxoPtqUdgBqevM9s9njzkTLutVsOCto= +github.com/go-chi/chi/v5 v5.3.1 h1:3j4HZLGZQ3JpMCrPJF/Jl3mYJfWLKBfNJ6quurUGCf8= +github.com/go-chi/chi/v5 v5.3.1/go.mod h1:R+tYY2hNuVUUjxoPtqUdgBqevM9s9njzkTLutVsOCto= github.com/go-critic/go-critic v0.14.3 h1:5R1qH2iFeo4I/RJU8vTezdqs08Egi4u5p6vOESA0pog= github.com/go-critic/go-critic v0.14.3/go.mod h1:xwntfW6SYAd7h1OqDzmN6hBX/JxsEKl5up/Y2bsxgVQ= github.com/go-jose/go-jose/v4 v4.1.4 h1:moDMcTHmvE6Groj34emNPLs/qtYXRVcd6S7NHbHz3kA= From 96e113213de527dd9834e233dd77b71558030d38 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 16 Jul 2026 15:14:12 +0000 Subject: [PATCH 013/633] Bump github.com/git-pkgs/purl from 0.1.13 to 0.1.14 Bumps [github.com/git-pkgs/purl](https://github.com/git-pkgs/purl) from 0.1.13 to 0.1.14. - [Commits](https://github.com/git-pkgs/purl/compare/v0.1.13...v0.1.14) --- updated-dependencies: - dependency-name: github.com/git-pkgs/purl dependency-version: 0.1.14 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index b063dba..35079fb 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( github.com/git-pkgs/archives v0.3.0 github.com/git-pkgs/cooldown v0.1.1 github.com/git-pkgs/enrichment v0.4.1 - github.com/git-pkgs/purl v0.1.13 + github.com/git-pkgs/purl v0.1.14 github.com/git-pkgs/registries v0.6.2 github.com/git-pkgs/spdx v0.1.4 github.com/git-pkgs/vers v0.2.6 diff --git a/go.sum b/go.sum index b1d35f6..f38a797 100644 --- a/go.sum +++ b/go.sum @@ -254,8 +254,8 @@ github.com/git-pkgs/packageurl-go v0.3.1 h1:WM3RBABQZLaRBxgKyYughc3cVBE8KyQxbSC6 github.com/git-pkgs/packageurl-go v0.3.1/go.mod h1:rcIxiG37BlQLB6FZfgdj9Fm7yjhRQd3l+5o7J0QPAk4= github.com/git-pkgs/pom v0.1.5 h1:TGT8Az2OMxGWsXnSagtUMGzZm7Oax8HrSCteA+mi0qY= github.com/git-pkgs/pom v0.1.5/go.mod h1:ufdMBe1lKzqOeP9IUb9NPZ458xKV8E8NvuyBMxOfwIk= -github.com/git-pkgs/purl v0.1.13 h1:at8BU6vnP5oonHFHAPA064BzgRqij+SZcOUDgNT2DC8= -github.com/git-pkgs/purl v0.1.13/go.mod h1:8oCcdcYZA/e1B33e7Ylju6azboTKjdqf3ybcbQj6I/o= +github.com/git-pkgs/purl v0.1.14 h1:GgqwiBNS0eKJqJ/gabUBEC10Xhpj6UX12kfNzujaeYc= +github.com/git-pkgs/purl v0.1.14/go.mod h1:8oCcdcYZA/e1B33e7Ylju6azboTKjdqf3ybcbQj6I/o= github.com/git-pkgs/registries v0.6.2 h1:26G5zW6Q7x1CSfNkaEqEjRMJiA4JwfdKOCJ7Qm+u0a8= github.com/git-pkgs/registries v0.6.2/go.mod h1:GR0Bu6nC3NQe6f7lfDoEVqAnoQkMocf4M98B12a7B3E= github.com/git-pkgs/spdx v0.1.4 h1:eQ0waEV3uUeItpWAOvdN1K1rL9hTgsU7fF74r1mDXMs= From df997e5825ee0e30ec38452be4da42260ec65e36 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 16 Jul 2026 15:14:14 +0000 Subject: [PATCH 014/633] Bump docker/login-action from 4.3.0 to 4.4.0 Bumps [docker/login-action](https://github.com/docker/login-action) from 4.3.0 to 4.4.0. - [Release notes](https://github.com/docker/login-action/releases) - [Commits](https://github.com/docker/login-action/compare/c99871dec2022cc055c062a10cc1a1310835ceb4...af1e73f918a031802d376d3c8bbc3fe56130a9b0) --- updated-dependencies: - dependency-name: docker/login-action dependency-version: 4.4.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 5f459b6..9c99e5d 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -25,7 +25,7 @@ jobs: persist-credentials: false - name: Log in to the Container registry - uses: docker/login-action@c99871dec2022cc055c062a10cc1a1310835ceb4 + uses: docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0 with: registry: ghcr.io username: ${{ github.actor }} From 41ef27907e99ee9c214ece7731ea96f25708873d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 17 Jul 2026 15:20:18 +0000 Subject: [PATCH 015/633] Bump github.com/git-pkgs/enrichment from 0.4.1 to 0.6.0 Bumps [github.com/git-pkgs/enrichment](https://github.com/git-pkgs/enrichment) from 0.4.1 to 0.6.0. - [Commits](https://github.com/git-pkgs/enrichment/compare/v0.4.1...v0.6.0) --- updated-dependencies: - dependency-name: github.com/git-pkgs/enrichment dependency-version: 0.5.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- go.mod | 6 +++--- go.sum | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index 0db3f6b..b84d213 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ require ( github.com/CycloneDX/cyclonedx-go v0.11.0 github.com/git-pkgs/archives v0.3.0 github.com/git-pkgs/cooldown v0.1.1 - github.com/git-pkgs/enrichment v0.4.1 + github.com/git-pkgs/enrichment v0.6.0 github.com/git-pkgs/purl v0.1.14 github.com/git-pkgs/registries v0.6.2 github.com/git-pkgs/spdx v0.1.4 @@ -115,7 +115,7 @@ require ( github.com/denis-tingaikin/go-header v0.5.0 // indirect github.com/dlclark/regexp2 v1.11.5 // indirect github.com/dustin/go-humanize v1.0.1 // indirect - github.com/ecosyste-ms/ecosystems-go v0.2.0 // indirect + github.com/ecosyste-ms/ecosystems-go v0.3.0 // indirect github.com/ettle/strcase v0.2.0 // indirect github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a // indirect github.com/fatih/color v1.18.0 // indirect @@ -217,7 +217,7 @@ require ( github.com/nishanths/exhaustive v0.12.0 // indirect github.com/nishanths/predeclared v0.2.2 // indirect github.com/nunnatsa/ginkgolinter v0.23.0 // indirect - github.com/oapi-codegen/runtime v1.4.1 // indirect + github.com/oapi-codegen/runtime v1.4.2 // indirect github.com/package-url/packageurl-go v0.1.6 // indirect github.com/pandatix/go-cvss v0.6.2 // indirect github.com/pelletier/go-toml v1.9.5 // indirect diff --git a/go.sum b/go.sum index 9a93f6d..6da21d4 100644 --- a/go.sum +++ b/go.sum @@ -217,8 +217,8 @@ github.com/dlclark/regexp2 v1.11.5 h1:Q/sSnsKerHeCkc/jSTNq1oCm7KiVgUMZRDUoRu0JQZ github.com/dlclark/regexp2 v1.11.5/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= -github.com/ecosyste-ms/ecosystems-go v0.2.0 h1:Nhpg54C+St8Sd/mf8bNJmQqx35ZgHw33TfFoxaXMQI8= -github.com/ecosyste-ms/ecosystems-go v0.2.0/go.mod h1:CCdzT1iAZirbEZAbFSnWpK88eKKaIWex7gjtZ0UudXA= +github.com/ecosyste-ms/ecosystems-go v0.3.0 h1:eVTNKQ3PyVNkSAnk1934958Vg4rR7ho5B5MWEQaMLi4= +github.com/ecosyste-ms/ecosystems-go v0.3.0/go.mod h1:bkjiI8WoTFB1Jw0M3h8yn3KXCD/+LdETsQ3m2BNLMHQ= github.com/envoyproxy/go-control-plane v0.14.0 h1:hbG2kr4RuFj222B6+7T83thSPqLjwBIfQawTkC++2HA= github.com/envoyproxy/go-control-plane/envoy v1.37.0 h1:u3riX6BoYRfF4Dr7dwSOroNfdSbEPe9Yyl09/B6wBrQ= github.com/envoyproxy/go-control-plane/envoy v1.37.0/go.mod h1:DReE9MMrmecPy+YvQOAOHNYMALuowAnbjjEMkkWOi6A= @@ -248,8 +248,8 @@ github.com/git-pkgs/archives v0.3.0 h1:iXKyO83jEFub1PGEDlHmk2tQ7XeV5LySTc0sEkH3x github.com/git-pkgs/archives v0.3.0/go.mod h1:LTJ1iQVFA7otizWMOyiI82NYVmyBWAPRzwu/e30rcXU= github.com/git-pkgs/cooldown v0.1.1 h1:9OqqzCB8gANz/y44SmqGD0Jp8Qtu81D1sCbKl6Ehg7w= github.com/git-pkgs/cooldown v0.1.1/go.mod h1:v7APuK/UouTiu8mWQZbdDmj7DfxxkGUeuhjaRB5gv9E= -github.com/git-pkgs/enrichment v0.4.1 h1:A8BKs0XwvpF1sF5qviZy4fkJAe18qB9OgpbRnmwnT34= -github.com/git-pkgs/enrichment v0.4.1/go.mod h1:stHqZUitV9ZkwACqHzBysLMSe6T4QZn81hxTdSroNhM= +github.com/git-pkgs/enrichment v0.6.0 h1:npV6N+eFZnI64uw/B0s+WJPn6Fu6Be08bexAViZFjMg= +github.com/git-pkgs/enrichment v0.6.0/go.mod h1:ov2WDaiNoIXViqdnE1FlUjNTnB5RnkUH4et9BWPhUDY= github.com/git-pkgs/packageurl-go v0.3.1 h1:WM3RBABQZLaRBxgKyYughc3cVBE8KyQxbSC6Jt5ak7M= github.com/git-pkgs/packageurl-go v0.3.1/go.mod h1:rcIxiG37BlQLB6FZfgdj9Fm7yjhRQd3l+5o7J0QPAk4= github.com/git-pkgs/pom v0.1.5 h1:TGT8Az2OMxGWsXnSagtUMGzZm7Oax8HrSCteA+mi0qY= @@ -512,8 +512,8 @@ github.com/nunnatsa/ginkgolinter v0.23.0 h1:x3o4DGYOWbBMP/VdNQKgSj+25aJKx2Pe6lHr github.com/nunnatsa/ginkgolinter v0.23.0/go.mod h1:9qN1+0akwXEccwV1CAcCDfcoBlWXHB+ML9884pL4SZ4= github.com/oapi-codegen/nullable v1.1.0 h1:eAh8JVc5430VtYVnq00Hrbpag9PFRGWLjxR1/3KntMs= github.com/oapi-codegen/nullable v1.1.0/go.mod h1:KUZ3vUzkmEKY90ksAmit2+5juDIhIZhfDl+0PwOQlFY= -github.com/oapi-codegen/runtime v1.4.1 h1:9nwLoI+KrWxzbBcp0jO/R8uXqbik/HUyCvPeU68Y/qo= -github.com/oapi-codegen/runtime v1.4.1/go.mod h1:GwV7hC2hviaMzj+ITfHVRESK5J2W/GefVwIND/bMGvU= +github.com/oapi-codegen/runtime v1.4.2 h1:GMxFVYLzoYLua+/KvzgSphkyK1lLTReQI9Vf4hvATKE= +github.com/oapi-codegen/runtime v1.4.2/go.mod h1:GwV7hC2hviaMzj+ITfHVRESK5J2W/GefVwIND/bMGvU= github.com/onsi/ginkgo/v2 v2.28.1 h1:S4hj+HbZp40fNKuLUQOYLDgZLwNUVn19N3Atb98NCyI= github.com/onsi/ginkgo/v2 v2.28.1/go.mod h1:CLtbVInNckU3/+gC8LzkGUb9oF+e8W8TdUsxPwvdOgE= github.com/onsi/gomega v1.39.1 h1:1IJLAad4zjPn2PsnhH70V4DKRFlrCzGBNrNaru+Vf28= From cdbd1a93697b1b97a50d68c7d67c89063be86ef0 Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Fri, 17 Jul 2026 18:11:47 -0700 Subject: [PATCH 016/633] Canonicalize cooldown lookup PURLs to match config NormalizedPackages runs config keys through purl.Parse().String(), which applies per-type rules like lowercasing pypi names. The handler side was building lookup keys with MakePURLString, which does not, so a config entry for pkg:pypi/Django would be normalized to pkg:pypi/django and never match the runtime key pkg:pypi/Django. Route both sides through the same canonical form: a new canonicalPackagePURL helper calls Normalize() on the constructed PURL before stringifying, and all cooldown IsAllowed call sites use it. --- docs/configuration.md | 2 +- internal/handler/cargo.go | 4 +--- internal/handler/composer.go | 4 +--- internal/handler/conda.go | 4 +--- internal/handler/gem.go | 4 +--- internal/handler/handler.go | 8 ++++++++ internal/handler/handler_test.go | 31 +++++++++++++++++++++++++++++++ internal/handler/hex.go | 3 +-- internal/handler/npm.go | 4 +--- internal/handler/nuget.go | 4 +--- internal/handler/pub.go | 4 +--- internal/handler/pypi.go | 6 ++---- 12 files changed, 50 insertions(+), 28 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index b103253..8998ac2 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -234,7 +234,7 @@ cooldown: Durations support days (`7d`), hours (`48h`), and minutes (`30m`). Set to `0` to disable. -Package PURL keys are normalized automatically. For npm scopes, both the readable form (`pkg:npm/@babel/core`) and canonical form (`pkg:npm/%40babel/core`) are accepted. If both forms configure the same package, the canonical entry wins. +Package PURL keys are normalized to canonical form before matching, so `pkg:npm/@babel/core` and `pkg:npm/%40babel/core` are equivalent, as are `pkg:pypi/Django` and `pkg:pypi/django`. If both forms configure the same package, the canonical entry wins. Resolution order: package override, then ecosystem override, then global default. This lets you set a conservative default while exempting trusted packages. diff --git a/internal/handler/cargo.go b/internal/handler/cargo.go index 5d7810c..bf424e2 100644 --- a/internal/handler/cargo.go +++ b/internal/handler/cargo.go @@ -8,8 +8,6 @@ import ( "net/http" "strings" "time" - - "github.com/git-pkgs/purl" ) const ( @@ -143,7 +141,7 @@ func (h *CargoHandler) applyCooldownFiltering(downstreamResponse http.ResponseWr continue } - cratePURL := purl.MakePURLString("cargo", crate.Name, "") + cratePURL := canonicalPackagePURL("cargo", crate.Name) if !h.proxy.Cooldown.IsAllowed("cargo", cratePURL, publishedAt) { h.proxy.Logger.Info("cooldown: filtering cargo version", diff --git a/internal/handler/composer.go b/internal/handler/composer.go index bc3bc1d..23deba5 100644 --- a/internal/handler/composer.go +++ b/internal/handler/composer.go @@ -10,8 +10,6 @@ import ( "path" "strings" "time" - - "github.com/git-pkgs/purl" ) const ( @@ -216,7 +214,7 @@ func deepCopyValue(v any) any { // filterAndRewriteVersions applies cooldown filtering and rewrites dist URLs // for a single package's version list. func (h *ComposerHandler) filterAndRewriteVersions(packageName string, versionList []any) []any { - packagePURL := purl.MakePURLString("composer", packageName, "") + packagePURL := canonicalPackagePURL("composer", packageName) filtered := versionList[:0] for _, v := range versionList { diff --git a/internal/handler/conda.go b/internal/handler/conda.go index cfa20c8..a8632e8 100644 --- a/internal/handler/conda.go +++ b/internal/handler/conda.go @@ -6,8 +6,6 @@ import ( "net/http" "strings" "time" - - "github.com/git-pkgs/purl" ) const ( @@ -218,7 +216,7 @@ func (h *CondaHandler) applyCooldownFiltering(body []byte) ([]byte, error) { continue } - packagePURL := purl.MakePURLString("conda", name, "") + packagePURL := canonicalPackagePURL("conda", name) if !h.proxy.Cooldown.IsAllowed("conda", packagePURL, publishedAt) { version, _ := entryMap["version"].(string) diff --git a/internal/handler/gem.go b/internal/handler/gem.go index 9ec57e3..a3714d6 100644 --- a/internal/handler/gem.go +++ b/internal/handler/gem.go @@ -8,8 +8,6 @@ import ( "net/http" "strings" "time" - - "github.com/git-pkgs/purl" ) const ( @@ -266,7 +264,7 @@ func (h *GemHandler) fetchFilteredVersions(r *http.Request, name string) (map[st return nil, err } - packagePURL := purl.MakePURLString("gem", name, "") + packagePURL := canonicalPackagePURL("gem", name) filtered := make(map[string]bool) for _, v := range versions { diff --git a/internal/handler/handler.go b/internal/handler/handler.go index d06ca83..202426d 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -48,6 +48,14 @@ func hasDotDotSegment(path string) bool { const defaultHTTPTimeout = 30 * time.Second +// canonicalPackagePURL returns a versionless PURL in canonical form so cooldown +// lookups match keys produced by config.CooldownConfig.NormalizedPackages. +func canonicalPackagePURL(ecosystem, name string) string { + p := purl.MakePURL(ecosystem, name, "") + _ = p.Normalize() + return p.String() +} + const contentTypeJSON = "application/json" const headerAcceptEncoding = "Accept-Encoding" diff --git a/internal/handler/handler_test.go b/internal/handler/handler_test.go index bbcab72..9a2b329 100644 --- a/internal/handler/handler_test.go +++ b/internal/handler/handler_test.go @@ -14,6 +14,7 @@ import ( "testing" "time" + "github.com/git-pkgs/proxy/internal/config" "github.com/git-pkgs/proxy/internal/database" "github.com/git-pkgs/proxy/internal/storage" "github.com/git-pkgs/registries/fetch" @@ -1010,3 +1011,33 @@ func TestProxyCached_FreshResponse_NoWarningHeader(t *testing.T) { t.Errorf("Warning should be empty for fresh response, got %q", got) } } + +// TestCanonicalPackagePURLMatchesConfig ensures the runtime cooldown lookup key +// agrees with config.CooldownConfig.NormalizedPackages for the same package, +// so a configured override is actually found regardless of how the user wrote it. +func TestCanonicalPackagePURLMatchesConfig(t *testing.T) { + tests := []struct { + ecosystem string + requestName string + configKey string + }{ + {"npm", "@babel/core", "pkg:npm/@babel/core"}, + {"npm", "@babel/core", "pkg:npm/%40babel/core"}, + {"npm", "@typescript/typescript-darwin-arm64", "pkg:npm/@typescript/typescript-darwin-arm64"}, + {"pypi", "Django", "pkg:pypi/Django"}, + {"pypi", "django", "pkg:pypi/Django"}, + {"composer", "symfony/console", "pkg:composer/Symfony/Console"}, + {"cargo", "serde", "pkg:cargo/serde"}, + } + for _, tt := range tests { + t.Run(tt.ecosystem+"/"+tt.requestName+"<="+tt.configKey, func(t *testing.T) { + cfg := config.CooldownConfig{Packages: map[string]string{tt.configKey: "1d"}} + normalized := cfg.NormalizedPackages() + + lookup := canonicalPackagePURL(tt.ecosystem, tt.requestName) + if _, ok := normalized[lookup]; !ok { + t.Errorf("lookup key %q not found in normalized config %v", lookup, normalized) + } + }) + } +} diff --git a/internal/handler/hex.go b/internal/handler/hex.go index 0f0c72e..6ebc176 100644 --- a/internal/handler/hex.go +++ b/internal/handler/hex.go @@ -10,7 +10,6 @@ import ( "strings" "time" - "github.com/git-pkgs/purl" "google.golang.org/protobuf/encoding/protowire" ) @@ -237,7 +236,7 @@ func (h *HexHandler) fetchFilteredVersions(r *http.Request, name string) (map[st return nil, err } - packagePURL := purl.MakePURLString("hex", name, "") + packagePURL := canonicalPackagePURL("hex", name) filtered := make(map[string]bool) for _, release := range pkg.Releases { diff --git a/internal/handler/npm.go b/internal/handler/npm.go index 0585eda..06f539e 100644 --- a/internal/handler/npm.go +++ b/internal/handler/npm.go @@ -9,8 +9,6 @@ import ( "sort" "strings" "time" - - "github.com/git-pkgs/purl" ) const ( @@ -134,7 +132,7 @@ func (h *NPMHandler) applyCooldownFiltering(metadata map[string]any, versions ma return } - packagePURL := purl.MakePURLString("npm", packageName, "") + packagePURL := canonicalPackagePURL("npm", packageName) for version := range versions { publishedStr, ok := timeMap[version].(string) diff --git a/internal/handler/nuget.go b/internal/handler/nuget.go index 40b8b5f..3e6373a 100644 --- a/internal/handler/nuget.go +++ b/internal/handler/nuget.go @@ -8,8 +8,6 @@ import ( "net/http" "strings" "time" - - "github.com/git-pkgs/purl" ) const ( @@ -271,7 +269,7 @@ func (h *NuGetHandler) applyCooldownFiltering(body []byte) ([]byte, error) { } } - packagePURL := purl.MakePURLString("nuget", strings.ToLower(id), "") + packagePURL := canonicalPackagePURL("nuget", strings.ToLower(id)) if !h.proxy.Cooldown.IsAllowed("nuget", packagePURL, publishedAt) { h.proxy.Logger.Info("cooldown: filtering nuget version", diff --git a/internal/handler/pub.go b/internal/handler/pub.go index 60bbbad..2971ddf 100644 --- a/internal/handler/pub.go +++ b/internal/handler/pub.go @@ -7,8 +7,6 @@ import ( "net/http" "strings" "time" - - "github.com/git-pkgs/purl" ) const ( @@ -127,7 +125,7 @@ func (h *PubHandler) rewriteMetadata(name string, body []byte) ([]byte, error) { return body, nil } - packagePURL := purl.MakePURLString("pub", name, "") + packagePURL := canonicalPackagePURL("pub", name) filtered := h.filterAndRewriteVersions(name, packagePURL, versions) metadata["versions"] = filtered diff --git a/internal/handler/pypi.go b/internal/handler/pypi.go index 3021d2b..0cf39fb 100644 --- a/internal/handler/pypi.go +++ b/internal/handler/pypi.go @@ -12,8 +12,6 @@ import ( "regexp" "strings" "time" - - "github.com/git-pkgs/purl" ) const ( @@ -131,7 +129,7 @@ func (h *PyPIHandler) fetchFilteredVersions(r *http.Request, name string) map[st return nil } - packagePURL := purl.MakePURLString("pypi", name, "") + packagePURL := canonicalPackagePURL("pypi", name) filtered := make(map[string]bool) for version, files := range releases { @@ -262,7 +260,7 @@ func (h *PyPIHandler) rewriteJSONMetadata(body []byte) ([]byte, error) { packageName, _ := extractPyPIName(metadata) packagePURL := "" if packageName != "" { - packagePURL = purl.MakePURLString("pypi", packageName, "") + packagePURL = canonicalPackagePURL("pypi", packageName) } h.filterAndRewriteReleases(metadata, packageName, packagePURL) From 60c72be65e6f0d45593dbecdeba2f1e3ded533fb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 23 Jul 2026 15:13:49 +0000 Subject: [PATCH 017/633] Bump actions/setup-go from 6.5.0 to 7.0.0 Bumps [actions/setup-go](https://github.com/actions/setup-go) from 6.5.0 to 7.0.0. - [Release notes](https://github.com/actions/setup-go/releases) - [Commits](https://github.com/actions/setup-go/compare/924ae3a1cded613372ab5595356fb5720e22ba16...b7ad1dad31e06c5925ef5d2fc7ad053ef454303e) --- updated-dependencies: - dependency-name: actions/setup-go dependency-version: 7.0.0 dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/ci.yml | 4 ++-- .github/workflows/release.yml | 2 +- .github/workflows/swagger.yml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 700f28e..c405f5d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -22,7 +22,7 @@ jobs: persist-credentials: false - name: Set up Go - uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0 + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 with: go-version: ${{ matrix.go-version }} @@ -40,7 +40,7 @@ jobs: persist-credentials: false - name: Set up Go - uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0 + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 with: go-version: '1.25' diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 4d4d0b5..f06cecd 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -22,7 +22,7 @@ jobs: - uses: sigstore/cosign-installer@6f9f17788090df1f26f669e9d70d6ae9567deba6 # v4.1.2 - name: Set up Go - uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0 + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 with: go-version-file: go.mod cache: false diff --git a/.github/workflows/swagger.yml b/.github/workflows/swagger.yml index f947c7e..e99aecf 100644 --- a/.github/workflows/swagger.yml +++ b/.github/workflows/swagger.yml @@ -17,7 +17,7 @@ jobs: persist-credentials: false - name: Set up Go - uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0 + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 with: go-version: '1.25' From d9b7a30294582b3a3c8939457805a4a2c8d3c688 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 23 Jul 2026 15:13:53 +0000 Subject: [PATCH 018/633] Bump zizmorcore/zizmor-action from 0.5.7 to 0.6.0 Bumps [zizmorcore/zizmor-action](https://github.com/zizmorcore/zizmor-action) from 0.5.7 to 0.6.0. - [Release notes](https://github.com/zizmorcore/zizmor-action/releases) - [Commits](https://github.com/zizmorcore/zizmor-action/compare/192e21d79ab29983730a13d1382995c2307fbcaa...6599ee8b7a49aef6a770f63d261d214911a7ce02) --- updated-dependencies: - dependency-name: zizmorcore/zizmor-action dependency-version: 0.6.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/zizmor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/zizmor.yml b/.github/workflows/zizmor.yml index cce6e4f..df06c5f 100644 --- a/.github/workflows/zizmor.yml +++ b/.github/workflows/zizmor.yml @@ -26,4 +26,4 @@ jobs: persist-credentials: false - name: Run zizmor - uses: zizmorcore/zizmor-action@192e21d79ab29983730a13d1382995c2307fbcaa # v0.5.7 + uses: zizmorcore/zizmor-action@6599ee8b7a49aef6a770f63d261d214911a7ce02 # v0.6.0 From c3f1577017e76dadf040d7ab225869052513bc29 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 23 Jul 2026 15:14:01 +0000 Subject: [PATCH 019/633] Bump modernc.org/sqlite from 1.53.0 to 1.54.0 Bumps [modernc.org/sqlite](https://gitlab.com/cznic/sqlite) from 1.53.0 to 1.54.0. - [Changelog](https://gitlab.com/cznic/sqlite/blob/master/CHANGELOG.md) - [Commits](https://gitlab.com/cznic/sqlite/compare/v1.53.0...v1.54.0) --- updated-dependencies: - dependency-name: modernc.org/sqlite dependency-version: 1.54.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- go.mod | 8 ++++---- go.sum | 28 ++++++++++++++-------------- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/go.mod b/go.mod index b84d213..384fba3 100644 --- a/go.mod +++ b/go.mod @@ -24,7 +24,7 @@ require ( golang.org/x/sync v0.22.0 google.golang.org/protobuf v1.36.11 gopkg.in/yaml.v3 v3.0.1 - modernc.org/sqlite v1.53.0 + modernc.org/sqlite v1.54.0 ) require ( @@ -294,12 +294,12 @@ require ( golang.org/x/crypto v0.53.0 // indirect golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa // indirect golang.org/x/exp/typeparams v0.0.0-20260209203927-2842357ff358 // indirect - golang.org/x/mod v0.36.0 // indirect + golang.org/x/mod v0.37.0 // indirect golang.org/x/net v0.56.0 // indirect golang.org/x/oauth2 v0.36.0 // indirect golang.org/x/sys v0.46.0 // indirect golang.org/x/text v0.38.0 // indirect - golang.org/x/tools v0.45.0 // indirect + golang.org/x/tools v0.47.0 // indirect golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect google.golang.org/api v0.272.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20260316180232-0b37fe3546d5 // indirect @@ -307,7 +307,7 @@ require ( gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect honnef.co/go/tools v0.7.0 // indirect - modernc.org/libc v1.73.4 // indirect + modernc.org/libc v1.74.1 // indirect modernc.org/mathutil v1.7.1 // indirect modernc.org/memory v1.11.0 // indirect mvdan.cc/gofumpt v0.9.2 // indirect diff --git a/go.sum b/go.sum index 6da21d4..301a00d 100644 --- a/go.sum +++ b/go.sum @@ -755,8 +755,8 @@ golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91 golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.13.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= -golang.org/x/mod v0.36.0 h1:JJjpVx6myfUsUdAzZuOSTTmRE0PfZeNWzzvKrP7amb4= -golang.org/x/mod v0.36.0/go.mod h1:moc6ELqsWcOw5Ef3xVprK5ul/MvtVvkIXLziUOICjUQ= +golang.org/x/mod v0.37.0 h1:vF1DjpVEshcIqoEaauuHebaLk1O1forxjxBaVn884JQ= +golang.org/x/mod v0.37.0/go.mod h1:m8S8VeM9r4dzDwjrKO0a1sZP3YjeMamRRlD+fmR2Q/0= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -839,8 +839,8 @@ golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= golang.org/x/tools v0.14.0/go.mod h1:uYBEerGOWcJyEORxN+Ek8+TT266gXkNlHdJBwexUsBg= -golang.org/x/tools v0.45.0 h1:18qN3FAooORvApf5XjCXgsuayZOEtXf6JK18I3+ONa8= -golang.org/x/tools v0.45.0/go.mod h1:LuUGqqaXcXMEFEruIVJVm5mgDD8vww/z/SR1gQ4uE/0= +golang.org/x/tools v0.47.0 h1:7Kn5x/d1svx/PzryTsqeoZN4TZwqeH5pGWjefhLi/1Q= +golang.org/x/tools v0.47.0/go.mod h1:dFHnyTvFWY212G+h7ZY4Vsp/K3U4/7W9TyVaAul8uCA= golang.org/x/tools/go/expect v0.1.1-deprecated h1:jpBZDwmgPhXsKZC6WhL20P4b/wmnpsEAGHaNy0n/rJM= golang.org/x/tools/go/expect v0.1.1-deprecated/go.mod h1:eihoPOH+FgIqa3FpoTwguz/bVUSGBlGQU67vpBeOrBY= golang.org/x/tools/go/packages/packagestest v0.1.1-deprecated h1:1h2MnaIAIXISqTFKdENegdpAgUXz6NrPEsbIeWaBRvM= @@ -882,20 +882,20 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.7.0 h1:w6WUp1VbkqPEgLz4rkBzH/CSU6HkoqNLp6GstyTx3lU= honnef.co/go/tools v0.7.0/go.mod h1:pm29oPxeP3P82ISxZDgIYeOaf9ta6Pi0EWvCFoLG2vc= -modernc.org/cc/v4 v4.28.4 h1:Hd/4Es+MBj+/7hSdZaisNyu6bv3V0Dp2MdllyfqaH+c= -modernc.org/cc/v4 v4.28.4/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI= -modernc.org/ccgo/v4 v4.34.4 h1:OVnSOWQjVKOYkFxoHYB+qQmSHK5gqMqARM+K9DpR/Ws= -modernc.org/ccgo/v4 v4.34.4/go.mod h1:qdKqE8FNIYyysougB1RX9MxCzp5oJOcQXSobANJ4TuE= +modernc.org/cc/v4 v4.29.0 h1:CXgwL8cvxmyzBQZzbSl/6xFtMCryb6u8IOqDci39cgc= +modernc.org/cc/v4 v4.29.0/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI= +modernc.org/ccgo/v4 v4.34.6 h1:sBgfIwyN0TQ9C5hwIeuqyeAKyMWnbvj2fvpF4L11uzU= +modernc.org/ccgo/v4 v4.34.6/go.mod h1:SZ8YcN9NG7XVsQYdm6jYBvi8PQP1qi+kqB6OhjqI3Fk= modernc.org/fileutil v1.4.0 h1:j6ZzNTftVS054gi281TyLjHPp6CPHr2KCxEXjEbD6SM= modernc.org/fileutil v1.4.0/go.mod h1:EqdKFDxiByqxLk8ozOxObDSfcVOv/54xDs/DUHdvCUU= modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI= modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito= -modernc.org/gc/v3 v3.1.3 h1:6QAplYyVO+KdPW3pGnqmJDUxtkec8ooEWvks/hhU3lc= -modernc.org/gc/v3 v3.1.3/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY= +modernc.org/gc/v3 v3.1.4 h1:2g65LGVSmFQrXeITAw97x7hCRvZFcyE1uDP+7Vng7JI= +modernc.org/gc/v3 v3.1.4/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY= modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks= modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI= -modernc.org/libc v1.73.4 h1:+ra4Ui8ngyt8HDcO1FTDPWlkAh6yOdaO2yAoh8MddQA= -modernc.org/libc v1.73.4/go.mod h1:DXZ3eO8qMCNn2SnmTNCiC71nJ9Rcq3PsnpU6Vc4rWK8= +modernc.org/libc v1.74.1 h1:bdR4VTKFMC4966QSNZ05XLGI/VwzVa2kTUX51Dm0riQ= +modernc.org/libc v1.74.1/go.mod h1:uH4t5bOx3G3g9Xcmj10YKlTcVISlRDwv8VoQJG9n8Os= modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI= @@ -904,8 +904,8 @@ modernc.org/opt v0.2.0 h1:tGyef5ApycA7FSEOMraay9SaTk5zmbx7Tu+cJs4QKZg= modernc.org/opt v0.2.0/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns= modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w= modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE= -modernc.org/sqlite v1.53.0 h1:20WG8N9q4ji/dEqGk4uiI0c6OPjSeLTNYGFCc3+7c1M= -modernc.org/sqlite v1.53.0/go.mod h1:xoEpOIpGrgT48H5iiyt/YXPCZPEzlfmfFwtk8Lklw8s= +modernc.org/sqlite v1.54.0 h1:JCxR4qwkJvOaqAoYcgDoO25Nc+ROg6EJ2LfBVzdrgog= +modernc.org/sqlite v1.54.0/go.mod h1:4ntCLuNmnH8+GNqjka1wNg7KJd5/Hi5FYp8K+XQ7GZw= modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0= modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A= modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= From 5ae5eab0311d4f3cfab5ec5465523bcd402fbd1c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 23 Jul 2026 15:14:10 +0000 Subject: [PATCH 020/633] Bump github.com/git-pkgs/vulns from 0.1.6 to 0.2.0 Bumps [github.com/git-pkgs/vulns](https://github.com/git-pkgs/vulns) from 0.1.6 to 0.2.0. - [Commits](https://github.com/git-pkgs/vulns/compare/v0.1.6...v0.2.0) --- updated-dependencies: - dependency-name: github.com/git-pkgs/vulns dependency-version: 0.2.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index b84d213..7b7701d 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/git-pkgs/registries v0.6.2 github.com/git-pkgs/spdx v0.1.4 github.com/git-pkgs/vers v0.2.6 - github.com/git-pkgs/vulns v0.1.6 + github.com/git-pkgs/vulns v0.2.0 github.com/go-chi/chi/v5 v5.3.1 github.com/jmoiron/sqlx v1.4.0 github.com/lib/pq v1.12.3 diff --git a/go.sum b/go.sum index 6da21d4..4427b92 100644 --- a/go.sum +++ b/go.sum @@ -262,8 +262,8 @@ github.com/git-pkgs/spdx v0.1.4 h1:eQ0waEV3uUeItpWAOvdN1K1rL9hTgsU7fF74r1mDXMs= github.com/git-pkgs/spdx v0.1.4/go.mod h1:cqRoZcvl530s/W+oGNvwjt4ODN8T1W6D/20MUZEFdto= github.com/git-pkgs/vers v0.2.6 h1:IelZd7BP/JhzTloUTDY67nehUgoYva3g9viqAMCHJg8= github.com/git-pkgs/vers v0.2.6/go.mod h1:biTbSQK1qdbrsxDEKnqe3Jzclxz8vW6uDcwKjfUGcOo= -github.com/git-pkgs/vulns v0.1.6 h1:8RRSgdlxp4JMU0Zykr63XTOMo5CyZKwt/PwaQxrx9Yg= -github.com/git-pkgs/vulns v0.1.6/go.mod h1:TsZC4MjoCkKJslgmbcmRCnytwnFcjESC2N8b0a2xDWc= +github.com/git-pkgs/vulns v0.2.0 h1:S1sA0ObQ/IKW0vqK+lK+HAY37AdDHEfhaIja/fLptwg= +github.com/git-pkgs/vulns v0.2.0/go.mod h1:mjVquLlunsAFPltqjf6mbvbI4tQ1iIa1NuhKIOPKt/4= github.com/github/go-spdx/v2 v2.7.0 h1:GzfXx4wFdlilARxmFRXW/mgUy3A4vSqZocCMFV6XFdQ= github.com/github/go-spdx/v2 v2.7.0/go.mod h1:Ftc45YYG1WzpzwEPKRVm9Jv8vDqOrN4gWoCkK+bHer0= github.com/go-chi/chi/v5 v5.3.1 h1:3j4HZLGZQ3JpMCrPJF/Jl3mYJfWLKBfNJ6quurUGCf8= From 13ae3970e9b09b14cb9fbf90a136ae081db4222e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 23 Jul 2026 15:42:03 +0000 Subject: [PATCH 021/633] Bump github.com/git-pkgs/vers from 0.2.6 to 0.3.0 Bumps [github.com/git-pkgs/vers](https://github.com/git-pkgs/vers) from 0.2.6 to 0.3.0. - [Commits](https://github.com/git-pkgs/vers/compare/v0.2.6...v0.3.0) --- updated-dependencies: - dependency-name: github.com/git-pkgs/vers dependency-version: 0.3.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 7b7701d..563b518 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ require ( github.com/git-pkgs/purl v0.1.14 github.com/git-pkgs/registries v0.6.2 github.com/git-pkgs/spdx v0.1.4 - github.com/git-pkgs/vers v0.2.6 + github.com/git-pkgs/vers v0.3.0 github.com/git-pkgs/vulns v0.2.0 github.com/go-chi/chi/v5 v5.3.1 github.com/jmoiron/sqlx v1.4.0 diff --git a/go.sum b/go.sum index 4427b92..ec10b3c 100644 --- a/go.sum +++ b/go.sum @@ -260,8 +260,8 @@ github.com/git-pkgs/registries v0.6.2 h1:26G5zW6Q7x1CSfNkaEqEjRMJiA4JwfdKOCJ7Qm+ github.com/git-pkgs/registries v0.6.2/go.mod h1:GR0Bu6nC3NQe6f7lfDoEVqAnoQkMocf4M98B12a7B3E= github.com/git-pkgs/spdx v0.1.4 h1:eQ0waEV3uUeItpWAOvdN1K1rL9hTgsU7fF74r1mDXMs= github.com/git-pkgs/spdx v0.1.4/go.mod h1:cqRoZcvl530s/W+oGNvwjt4ODN8T1W6D/20MUZEFdto= -github.com/git-pkgs/vers v0.2.6 h1:IelZd7BP/JhzTloUTDY67nehUgoYva3g9viqAMCHJg8= -github.com/git-pkgs/vers v0.2.6/go.mod h1:biTbSQK1qdbrsxDEKnqe3Jzclxz8vW6uDcwKjfUGcOo= +github.com/git-pkgs/vers v0.3.0 h1:xM4LLUCRmqzdDfe+/pVQUx4SRyFXRVth6tOsJ14wMKU= +github.com/git-pkgs/vers v0.3.0/go.mod h1:biTbSQK1qdbrsxDEKnqe3Jzclxz8vW6uDcwKjfUGcOo= github.com/git-pkgs/vulns v0.2.0 h1:S1sA0ObQ/IKW0vqK+lK+HAY37AdDHEfhaIja/fLptwg= github.com/git-pkgs/vulns v0.2.0/go.mod h1:mjVquLlunsAFPltqjf6mbvbI4tQ1iIa1NuhKIOPKt/4= github.com/github/go-spdx/v2 v2.7.0 h1:GzfXx4wFdlilARxmFRXW/mgUy3A4vSqZocCMFV6XFdQ= From 532e4925fef3c7cbbd8d6d276e8444971d23c98b Mon Sep 17 00:00:00 2001 From: Tilian Honig Date: Sun, 26 Jul 2026 20:07:46 +0200 Subject: [PATCH 022/633] fix: proper handling of upstream registry 404s (#209) * fix: proper handling of upstream registry 404s * fix: consistently return 404s for all artifact types --- go.mod | 2 +- go.sum | 4 +- internal/handler/cargo.go | 3 +- internal/handler/composer.go | 3 +- internal/handler/conan.go | 6 +- internal/handler/conda.go | 3 +- internal/handler/container.go | 7 +- internal/handler/cran.go | 6 +- internal/handler/debian.go | 3 +- internal/handler/filename_download.go | 39 +++++ internal/handler/gem.go | 31 +--- internal/handler/handler.go | 22 ++- internal/handler/hex.go | 31 +--- internal/handler/julia.go | 9 +- internal/handler/maven.go | 7 +- internal/handler/notfound_ecosystems_test.go | 151 +++++++++++++++++++ internal/handler/notfound_test.go | 83 ++++++++++ internal/handler/npm.go | 4 + internal/handler/nuget.go | 3 +- internal/handler/pub.go | 3 +- internal/handler/pypi.go | 3 +- internal/handler/rpm.go | 3 +- 22 files changed, 337 insertions(+), 89 deletions(-) create mode 100644 internal/handler/filename_download.go create mode 100644 internal/handler/notfound_ecosystems_test.go create mode 100644 internal/handler/notfound_test.go diff --git a/go.mod b/go.mod index 1f2c073..6d81076 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/git-pkgs/cooldown v0.1.1 github.com/git-pkgs/enrichment v0.6.0 github.com/git-pkgs/purl v0.1.14 - github.com/git-pkgs/registries v0.6.2 + github.com/git-pkgs/registries v0.6.3 github.com/git-pkgs/spdx v0.1.4 github.com/git-pkgs/vers v0.3.0 github.com/git-pkgs/vulns v0.2.0 diff --git a/go.sum b/go.sum index 17b0651..5d16d31 100644 --- a/go.sum +++ b/go.sum @@ -256,8 +256,8 @@ github.com/git-pkgs/pom v0.1.5 h1:TGT8Az2OMxGWsXnSagtUMGzZm7Oax8HrSCteA+mi0qY= github.com/git-pkgs/pom v0.1.5/go.mod h1:ufdMBe1lKzqOeP9IUb9NPZ458xKV8E8NvuyBMxOfwIk= github.com/git-pkgs/purl v0.1.14 h1:GgqwiBNS0eKJqJ/gabUBEC10Xhpj6UX12kfNzujaeYc= github.com/git-pkgs/purl v0.1.14/go.mod h1:8oCcdcYZA/e1B33e7Ylju6azboTKjdqf3ybcbQj6I/o= -github.com/git-pkgs/registries v0.6.2 h1:26G5zW6Q7x1CSfNkaEqEjRMJiA4JwfdKOCJ7Qm+u0a8= -github.com/git-pkgs/registries v0.6.2/go.mod h1:GR0Bu6nC3NQe6f7lfDoEVqAnoQkMocf4M98B12a7B3E= +github.com/git-pkgs/registries v0.6.3 h1:7sveeeMS2lgXtcqNYAA3bwaT7H9CQs0uhy6wQSxz3Js= +github.com/git-pkgs/registries v0.6.3/go.mod h1:j4o50ii/vD9Z42/nFeQASt15BPolADIejeFmmqK4njo= github.com/git-pkgs/spdx v0.1.4 h1:eQ0waEV3uUeItpWAOvdN1K1rL9hTgsU7fF74r1mDXMs= github.com/git-pkgs/spdx v0.1.4/go.mod h1:cqRoZcvl530s/W+oGNvwjt4ODN8T1W6D/20MUZEFdto= github.com/git-pkgs/vers v0.3.0 h1:xM4LLUCRmqzdDfe+/pVQUx4SRyFXRVth6tOsJ14wMKU= diff --git a/internal/handler/cargo.go b/internal/handler/cargo.go index bf424e2..abc0d1f 100644 --- a/internal/handler/cargo.go +++ b/internal/handler/cargo.go @@ -191,8 +191,7 @@ func (h *CargoHandler) handleDownload(w http.ResponseWriter, r *http.Request) { result, err := h.proxy.GetOrFetchArtifact(r.Context(), "cargo", name, version, filename) if err != nil { - h.proxy.Logger.Error("failed to get artifact", "error", err) - http.Error(w, "failed to fetch crate", http.StatusBadGateway) + h.proxy.serveArtifactError(w, err, "failed to fetch crate") return } diff --git a/internal/handler/composer.go b/internal/handler/composer.go index 23deba5..45935b7 100644 --- a/internal/handler/composer.go +++ b/internal/handler/composer.go @@ -346,8 +346,7 @@ func (h *ComposerHandler) handleDownload(w http.ResponseWriter, r *http.Request) result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "composer", packageName, version, filename, downloadURL) if err != nil { - h.proxy.Logger.Error("failed to get artifact", "error", err) - http.Error(w, "failed to fetch package", http.StatusBadGateway) + h.proxy.serveArtifactError(w, err, "failed to fetch package") return } diff --git a/internal/handler/conan.go b/internal/handler/conan.go index 53f6428..c0476f9 100644 --- a/internal/handler/conan.go +++ b/internal/handler/conan.go @@ -84,8 +84,7 @@ func (h *ConanHandler) handleRecipeFile(w http.ResponseWriter, r *http.Request) result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "conan", packageName, storageVersion, storageFilename, upstreamURL) if err != nil { - h.proxy.Logger.Error("failed to get artifact", "error", err) - http.Error(w, "failed to fetch file", http.StatusBadGateway) + h.proxy.serveArtifactError(w, err, "failed to fetch file") return } @@ -122,8 +121,7 @@ func (h *ConanHandler) handlePackageFile(w http.ResponseWriter, r *http.Request) result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "conan", packageName, storageVersion, storageFilename, upstreamURL) if err != nil { - h.proxy.Logger.Error("failed to get artifact", "error", err) - http.Error(w, "failed to fetch file", http.StatusBadGateway) + h.proxy.serveArtifactError(w, err, "failed to fetch file") return } diff --git a/internal/handler/conda.go b/internal/handler/conda.go index a8632e8..224c25f 100644 --- a/internal/handler/conda.go +++ b/internal/handler/conda.go @@ -72,8 +72,7 @@ func (h *CondaHandler) handleDownload(w http.ResponseWriter, r *http.Request) { result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "conda", packageName, version, filename, upstreamURL) if err != nil { - h.proxy.Logger.Error("failed to get artifact", "error", err) - http.Error(w, "failed to fetch package", http.StatusBadGateway) + h.proxy.serveArtifactError(w, err, "failed to fetch package") return } diff --git a/internal/handler/container.go b/internal/handler/container.go index 8ba5e97..0710ed1 100644 --- a/internal/handler/container.go +++ b/internal/handler/container.go @@ -2,6 +2,7 @@ package handler import ( "encoding/json" + "errors" "fmt" "io" "net/http" @@ -117,8 +118,12 @@ func (h *ContainerHandler) handleBlobDownload(w http.ResponseWriter, r *http.Req ) if err != nil { + if errors.Is(err, ErrUpstreamNotFound) { + h.containerError(w, http.StatusNotFound, "BLOB_UNKNOWN", "blob unknown to registry") + return + } h.proxy.Logger.Error("failed to fetch blob", "error", err) - h.containerError(w, http.StatusBadGateway, "BLOB_UNKNOWN", "failed to fetch blob") + h.containerError(w, http.StatusBadGateway, "INTERNAL_ERROR", "failed to fetch blob") return } diff --git a/internal/handler/cran.go b/internal/handler/cran.go index 0ecd2a3..2fc4fab 100644 --- a/internal/handler/cran.go +++ b/internal/handler/cran.go @@ -72,8 +72,7 @@ func (h *CRANHandler) handleSourceDownload(w http.ResponseWriter, r *http.Reques result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "cran", name, version, filename, upstreamURL) if err != nil { - h.proxy.Logger.Error("failed to get artifact", "error", err) - http.Error(w, "failed to fetch package", http.StatusBadGateway) + h.proxy.serveArtifactError(w, err, "failed to fetch package") return } @@ -107,8 +106,7 @@ func (h *CRANHandler) handleBinaryDownload(w http.ResponseWriter, r *http.Reques result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "cran", name, storageVersion, filename, upstreamURL) if err != nil { - h.proxy.Logger.Error("failed to get artifact", "error", err) - http.Error(w, "failed to fetch package", http.StatusBadGateway) + h.proxy.serveArtifactError(w, err, "failed to fetch package") return } diff --git a/internal/handler/debian.go b/internal/handler/debian.go index b767f6d..9a4aaab 100644 --- a/internal/handler/debian.go +++ b/internal/handler/debian.go @@ -81,8 +81,7 @@ func (h *DebianHandler) handlePackageDownload(w http.ResponseWriter, r *http.Req result, err := h.proxy.GetOrFetchArtifactFromURL( r.Context(), "deb", name, version, filename, downloadURL) if err != nil { - h.proxy.Logger.Error("failed to get debian package", "error", err) - http.Error(w, "failed to fetch package", http.StatusBadGateway) + h.proxy.serveArtifactError(w, err, "failed to fetch package") return } diff --git a/internal/handler/filename_download.go b/internal/handler/filename_download.go new file mode 100644 index 0000000..bedec16 --- /dev/null +++ b/internal/handler/filename_download.go @@ -0,0 +1,39 @@ +package handler + +import ( + "net/http" + "strings" +) + +type filenameDownload struct { + ecosystem string + suffix string + parseErr string + fetchErr string + parse func(string) (name, version string) +} + +func (p *Proxy) handleFilenameDownload(w http.ResponseWriter, r *http.Request, d filenameDownload) { + filename := r.PathValue("filename") + if filename == "" || !strings.HasSuffix(filename, d.suffix) { + http.Error(w, "invalid filename", http.StatusBadRequest) + return + } + + name, version := d.parse(filename) + if name == "" || version == "" { + http.Error(w, d.parseErr, http.StatusBadRequest) + return + } + + p.Logger.Info(d.ecosystem+" download request", + "name", name, "version", version, "filename", filename) + + result, err := p.GetOrFetchArtifact(r.Context(), d.ecosystem, name, version, filename) + if err != nil { + p.serveArtifactError(w, err, d.fetchErr) + return + } + + ServeArtifact(w, result) +} diff --git a/internal/handler/gem.go b/internal/handler/gem.go index a3714d6..260568a 100644 --- a/internal/handler/gem.go +++ b/internal/handler/gem.go @@ -58,30 +58,13 @@ func (h *GemHandler) Routes() http.Handler { // handleDownload serves a gem file, fetching and caching from upstream if needed. func (h *GemHandler) handleDownload(w http.ResponseWriter, r *http.Request) { - filename := r.PathValue("filename") - if filename == "" || !strings.HasSuffix(filename, ".gem") { - http.Error(w, "invalid filename", http.StatusBadRequest) - return - } - - // Extract name and version from filename (e.g., "rails-7.1.0.gem") - name, version := h.parseGemFilename(filename) - if name == "" || version == "" { - http.Error(w, "could not parse gem filename", http.StatusBadRequest) - return - } - - h.proxy.Logger.Info("gem download request", - "name", name, "version", version, "filename", filename) - - result, err := h.proxy.GetOrFetchArtifact(r.Context(), "gem", name, version, filename) - if err != nil { - h.proxy.Logger.Error("failed to get artifact", "error", err) - http.Error(w, "failed to fetch gem", http.StatusBadGateway) - return - } - - ServeArtifact(w, result) + h.proxy.handleFilenameDownload(w, r, filenameDownload{ + ecosystem: "gem", + suffix: ".gem", + parseErr: "could not parse gem filename", + fetchErr: "failed to fetch gem", + parse: h.parseGemFilename, + }) } // parseGemFilename extracts name and version from a gem filename. diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 202426d..e62292f 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -251,6 +251,9 @@ func (p *Proxy) fetchAndCache(ctx context.Context, ecosystem, name, version, fil // Resolve download URL info, err := p.Resolver.Resolve(ctx, ecosystem, name, version) if err != nil { + if errors.Is(err, fetch.ErrNotFound) { + return nil, ErrUpstreamNotFound + } return nil, fmt.Errorf("resolving download URL: %w", err) } @@ -270,6 +273,9 @@ func (p *Proxy) fetchAndCache(ctx context.Context, ecosystem, name, version, fil if err != nil { metrics.RecordUpstreamFetch(ecosystem, fetchDuration) metrics.RecordUpstreamError(ecosystem, "fetch_failed") + if errors.Is(err, fetch.ErrNotFound) { + return nil, ErrUpstreamNotFound + } return nil, fmt.Errorf("fetching from upstream: %w", err) } metrics.RecordUpstreamFetch(ecosystem, fetchDuration) @@ -451,7 +457,18 @@ func JSONError(w http.ResponseWriter, status int, message string) { } // ErrUpstreamNotFound indicates the upstream returned 404. -var ErrUpstreamNotFound = fmt.Errorf("upstream: not found") +var ErrUpstreamNotFound = fmt.Errorf("upstream: %w", fetch.ErrNotFound) + +// serveArtifactError writes response for a failed fetch: +// 404 when upstream reports artifact missing, 502 otherwise. +func (p *Proxy) serveArtifactError(w http.ResponseWriter, err error, clientMsg string) { + if errors.Is(err, ErrUpstreamNotFound) { + http.Error(w, "not found", http.StatusNotFound) + return + } + p.Logger.Error("failed to get artifact", "error", err) + http.Error(w, clientMsg, http.StatusBadGateway) +} // errStale304 is returned when upstream sends 304 but the cached file is missing. var errStale304 = fmt.Errorf("upstream returned 304 but cached file is missing") @@ -795,6 +812,9 @@ func (p *Proxy) fetchAndCacheFromURL(ctx context.Context, ecosystem, name, versi artifact, err := p.Fetcher.FetchWithHeaders(ctx, downloadURL, headers) if err != nil { + if errors.Is(err, fetch.ErrNotFound) { + return nil, ErrUpstreamNotFound + } return nil, fmt.Errorf("fetching from upstream: %w", err) } diff --git a/internal/handler/hex.go b/internal/handler/hex.go index 6ebc176..2ff4f0f 100644 --- a/internal/handler/hex.go +++ b/internal/handler/hex.go @@ -53,30 +53,13 @@ func (h *HexHandler) Routes() http.Handler { // handleDownload serves a package tarball, fetching and caching from upstream if needed. func (h *HexHandler) handleDownload(w http.ResponseWriter, r *http.Request) { - filename := r.PathValue("filename") - if filename == "" || !strings.HasSuffix(filename, ".tar") { - http.Error(w, "invalid filename", http.StatusBadRequest) - return - } - - // Extract name and version from filename (e.g., "phoenix-1.7.10.tar") - name, version := h.parseTarballFilename(filename) - if name == "" || version == "" { - http.Error(w, "could not parse tarball filename", http.StatusBadRequest) - return - } - - h.proxy.Logger.Info("hex download request", - "name", name, "version", version, "filename", filename) - - result, err := h.proxy.GetOrFetchArtifact(r.Context(), "hex", name, version, filename) - if err != nil { - h.proxy.Logger.Error("failed to get artifact", "error", err) - http.Error(w, "failed to fetch package", http.StatusBadGateway) - return - } - - ServeArtifact(w, result) + h.proxy.handleFilenameDownload(w, r, filenameDownload{ + ecosystem: "hex", + suffix: ".tar", + parseErr: "could not parse tarball filename", + fetchErr: "failed to fetch package", + parse: h.parseTarballFilename, + }) } // parseTarballFilename extracts name and version from a hex tarball filename. diff --git a/internal/handler/julia.go b/internal/handler/julia.go index 08b1fdf..0fed8c9 100644 --- a/internal/handler/julia.go +++ b/internal/handler/julia.go @@ -90,8 +90,7 @@ func (h *JuliaHandler) handleRegistry(w http.ResponseWriter, r *http.Request) { upstreamURL := h.upstreamURL + r.URL.Path result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "julia", juliaRegistryName, hash, hash+".tar.gz", upstreamURL) if err != nil { - h.proxy.Logger.Error("failed to get registry", "error", err) - http.Error(w, "failed to fetch registry", http.StatusBadGateway) + h.proxy.serveArtifactError(w, err, "failed to fetch registry") return } @@ -119,8 +118,7 @@ func (h *JuliaHandler) handlePackage(w http.ResponseWriter, r *http.Request) { upstreamURL := h.upstreamURL + r.URL.Path result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "julia", name, hash, hash+".tar.gz", upstreamURL) if err != nil { - h.proxy.Logger.Error("failed to get package", "error", err) - http.Error(w, "failed to fetch package", http.StatusBadGateway) + h.proxy.serveArtifactError(w, err, "failed to fetch package") return } @@ -141,8 +139,7 @@ func (h *JuliaHandler) handleArtifact(w http.ResponseWriter, r *http.Request) { upstreamURL := h.upstreamURL + r.URL.Path result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "julia", juliaArtifactName, hash, hash+".tar.gz", upstreamURL) if err != nil { - h.proxy.Logger.Error("failed to get artifact", "error", err) - http.Error(w, "failed to fetch artifact", http.StatusBadGateway) + h.proxy.serveArtifactError(w, err, "failed to fetch artifact") return } diff --git a/internal/handler/maven.go b/internal/handler/maven.go index c423645..10e551e 100644 --- a/internal/handler/maven.go +++ b/internal/handler/maven.go @@ -130,12 +130,7 @@ func (h *MavenHandler) handleDownload(w http.ResponseWriter, r *http.Request, ur } } if err != nil { - if errors.Is(err, ErrUpstreamNotFound) { - http.Error(w, "not found", http.StatusNotFound) - return - } - h.proxy.Logger.Error("failed to get artifact", "error", err) - http.Error(w, "failed to fetch artifact", http.StatusBadGateway) + h.proxy.serveArtifactError(w, err, "failed to fetch artifact") return } diff --git a/internal/handler/notfound_ecosystems_test.go b/internal/handler/notfound_ecosystems_test.go new file mode 100644 index 0000000..3c2cecf --- /dev/null +++ b/internal/handler/notfound_ecosystems_test.go @@ -0,0 +1,151 @@ +package handler + +import ( + "context" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/git-pkgs/registries/fetch" +) + +func TestArtifactDownloadUpstreamNotFoundReturns404(t *testing.T) { + tests := []struct { + name string + path string + handler func(p *Proxy) http.Handler + }{ + {"debian", "/pool/main/n/nginx/nginx_1.18.0-6_amd64.deb", + func(p *Proxy) http.Handler { return NewDebianHandler(p, "http://localhost").Routes() }}, + {"rpm", "/releases/39/Everything/x86_64/os/Packages/n/nginx-1.24.0-1.fc39.x86_64.rpm", + func(p *Proxy) http.Handler { return NewRPMHandler(p, "http://localhost").Routes() }}, + {"nuget", "/v3-flatcontainer/newtonsoft.json/13.0.3/newtonsoft.json.13.0.3.nupkg", + func(p *Proxy) http.Handler { return NewNuGetHandler(p, "http://localhost").Routes() }}, + {"pypi", "/packages/packages/ab/cd/ef0123456789/requests-2.31.0-py3-none-any.whl", + func(p *Proxy) http.Handler { return NewPyPIHandler(p, "http://localhost").Routes() }}, + {"cran", "/src/contrib/ggplot2_3.4.4.tar.gz", + func(p *Proxy) http.Handler { return NewCRANHandler(p, "http://localhost").Routes() }}, + {"conda", "/conda-forge/linux-64/numpy-1.26.0-py311_0.tar.bz2", + func(p *Proxy) http.Handler { return NewCondaHandler(p, "http://localhost").Routes() }}, + {"conan", "/v1/files/zlib/1.3.1/_/_/0/recipe/conan_sources.tgz", + func(p *Proxy) http.Handler { return NewConanHandler(p, "http://localhost").Routes() }}, + {"gem", "/gems/rails-7.1.0.gem", + func(p *Proxy) http.Handler { return NewGemHandler(p, "http://localhost").Routes() }}, + {"hex", "/tarballs/phoenix-1.7.10.tar", + func(p *Proxy) http.Handler { return NewHexHandler(p, "http://localhost").Routes() }}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + proxy, _, _, fetcher := setupTestProxy(t) + fetcher.fetchErr = fetch.ErrNotFound + + srv := httptest.NewServer(tt.handler(proxy)) + defer srv.Close() + + resp, err := http.Get(srv.URL + tt.path) + if err != nil { + t.Fatalf("request failed: %v", err) + } + defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode != http.StatusNotFound { + t.Errorf("want 404 for missing upstream artifact, got %d", resp.StatusCode) + } + }) + } +} + +func TestJuliaPackageUpstreamNotFoundReturns404(t *testing.T) { + proxy, _, _, fetcher := setupTestProxy(t) + fetcher.fetchErr = fetch.ErrNotFound + + dead := httptest.NewServer(http.NotFoundHandler()) + defer dead.Close() + + h := NewJuliaHandler(proxy, "http://localhost") + h.upstreamURL = dead.URL + + srv := httptest.NewServer(h.Routes()) + defer srv.Close() + + resp, err := http.Get(srv.URL + + "/package/7876af07-990d-54b4-ab0e-23690620f79a/0123456789abcdef0123456789abcdef01234567") + if err != nil { + t.Fatalf("request failed: %v", err) + } + defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode != http.StatusNotFound { + t.Errorf("want 404 for missing upstream package, got %d", resp.StatusCode) + } +} + +func TestComposerDownloadUpstreamNotFoundReturns404(t *testing.T) { + proxy, _, _, fetcher := setupTestProxy(t) + fetcher.fetchErr = fetch.ErrNotFound + + meta := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/p2/monolog/monolog.json" { + _, _ = w.Write([]byte(`{ + "packages": { + "monolog/monolog": [ + {"version": "2.9.1", "dist": {"url": "https://example.com/monolog-2.9.1.zip", "type": "zip"}} + ] + } + }`)) + return + } + http.NotFound(w, r) + })) + defer meta.Close() + + h := &ComposerHandler{proxy: proxy, repoURL: meta.URL, proxyURL: "http://localhost"} + srv := httptest.NewServer(h.Routes()) + defer srv.Close() + + resp, err := http.Get(srv.URL + "/files/monolog/monolog/2.9.1/monolog-2.9.1.zip") + if err != nil { + t.Fatalf("request failed: %v", err) + } + defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode != http.StatusNotFound { + t.Errorf("want 404 for missing upstream dist, got %d", resp.StatusCode) + } +} + +func TestContainerBlobUpstreamNotFoundReturns404(t *testing.T) { + authServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"token": "test-token-123"}`)) + })) + defer authServer.Close() + + proxy, _, _, _ := setupTestProxy(t) + proxy.Fetcher = &mockFetcherWithHeaders{ + fetchFn: func(_ context.Context, _ string, _ http.Header) (*fetch.Artifact, error) { + return nil, fetch.ErrNotFound + }, + } + + h := &ContainerHandler{ + proxy: proxy, + registryURL: "https://registry-1.docker.io", + authURL: authServer.URL, + proxyURL: "http://localhost:8080", + } + + req := httptest.NewRequest(http.MethodGet, + "/library/nginx/blobs/sha256:abc123def456abc123def456abc123def456abc123def456abc123def456abcd", nil) + w := httptest.NewRecorder() + h.Routes().ServeHTTP(w, req) + + if w.Code != http.StatusNotFound { + t.Errorf("want 404 for missing upstream blob, got %d; body: %s", w.Code, w.Body.String()) + } + if !strings.Contains(w.Body.String(), "BLOB_UNKNOWN") { + t.Errorf("want BLOB_UNKNOWN error code in body, got: %s", w.Body.String()) + } +} diff --git a/internal/handler/notfound_test.go b/internal/handler/notfound_test.go new file mode 100644 index 0000000..9ea38ac --- /dev/null +++ b/internal/handler/notfound_test.go @@ -0,0 +1,83 @@ +package handler + +import ( + "context" + "errors" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/git-pkgs/registries/fetch" +) + +func TestErrUpstreamNotFoundWrapsFetchErrNotFound(t *testing.T) { + if !errors.Is(ErrUpstreamNotFound, fetch.ErrNotFound) { + t.Fatal("ErrUpstreamNotFound does not wrap fetch.ErrNotFound") + } +} + +func TestGetOrFetchArtifactFromURL_NotFound(t *testing.T) { + proxy, _, _, fetcher := setupTestProxy(t) + fetcher.fetchErr = fetch.ErrNotFound + + _, err := proxy.GetOrFetchArtifactFromURL(context.Background(), + "maven", "org.example:missing", "1.0", "missing-1.0.jar", + "http://upstream.test/org/example/missing/1.0/missing-1.0.jar") + + if !errors.Is(err, ErrUpstreamNotFound) { + t.Fatalf("want ErrUpstreamNotFound, got %v", err) + } +} + +func TestMavenHandler_UpstreamNotFoundReturns404(t *testing.T) { + proxy, _, _, fetcher := setupTestProxy(t) + fetcher.fetchErr = fetch.ErrNotFound + + h := NewMavenHandler(proxy, "http://localhost", "http://upstream.test", "http://portal.test") + srv := httptest.NewServer(h.Routes()) + defer srv.Close() + + resp, err := http.Get(srv.URL + "/org/example/missing/1.0/missing-1.0.jar") + if err != nil { + t.Fatalf("request failed: %v", err) + } + defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode != http.StatusNotFound { + t.Errorf("want 404 for missing upstream artifact, got %d", resp.StatusCode) + } +} + +func TestMavenHandler_PluginPortalFallback(t *testing.T) { + proxy, _, _, fetcher := setupTestProxy(t) + fetcher.fetchErrByURL = map[string]error{ + "http://upstream.test/org/example/plugin/1.0/plugin-1.0.jar": fetch.ErrNotFound, + } + fetcher.artifact = &fetch.Artifact{ + Body: io.NopCloser(strings.NewReader("portal artifact")), + ContentType: "application/java-archive", + } + + h := NewMavenHandler(proxy, "http://localhost", "http://upstream.test", "http://portal.test") + srv := httptest.NewServer(h.Routes()) + defer srv.Close() + + resp, err := http.Get(srv.URL + "/org/example/plugin/1.0/plugin-1.0.jar") + if err != nil { + t.Fatalf("request failed: %v", err) + } + defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode != http.StatusOK { + t.Fatalf("want 200 via plugin portal fallback, got %d", resp.StatusCode) + } + body, _ := io.ReadAll(resp.Body) + if string(body) != "portal artifact" { + t.Errorf("want portal artifact body, got %q", body) + } + if fetcher.fetchedURL != "http://portal.test/org/example/plugin/1.0/plugin-1.0.jar" { + t.Errorf("fallback did not hit plugin portal, last URL: %s", fetcher.fetchedURL) + } +} diff --git a/internal/handler/npm.go b/internal/handler/npm.go index 06f539e..a3ed910 100644 --- a/internal/handler/npm.go +++ b/internal/handler/npm.go @@ -263,6 +263,10 @@ func (h *NPMHandler) handleDownload(w http.ResponseWriter, r *http.Request) { result, err := h.proxy.GetOrFetchArtifact(r.Context(), "npm", packageName, version, filename) if err != nil { + if errors.Is(err, ErrUpstreamNotFound) { + JSONError(w, http.StatusNotFound, "package not found") + return + } h.proxy.Logger.Error("failed to get artifact", "error", err) JSONError(w, http.StatusBadGateway, "failed to fetch package") return diff --git a/internal/handler/nuget.go b/internal/handler/nuget.go index 3e6373a..4785e40 100644 --- a/internal/handler/nuget.go +++ b/internal/handler/nuget.go @@ -314,8 +314,7 @@ func (h *NuGetHandler) handleDownload(w http.ResponseWriter, r *http.Request) { result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "nuget", name, version, filename, upstreamURL) if err != nil { - h.proxy.Logger.Error("failed to get artifact", "error", err) - http.Error(w, "failed to fetch package", http.StatusBadGateway) + h.proxy.serveArtifactError(w, err, "failed to fetch package") return } diff --git a/internal/handler/pub.go b/internal/handler/pub.go index 2971ddf..e5ca199 100644 --- a/internal/handler/pub.go +++ b/internal/handler/pub.go @@ -67,8 +67,7 @@ func (h *PubHandler) handleDownload(w http.ResponseWriter, r *http.Request) { result, err := h.proxy.GetOrFetchArtifact(r.Context(), "pub", name, version, filename) if err != nil { - h.proxy.Logger.Error("failed to get artifact", "error", err) - http.Error(w, "failed to fetch package", http.StatusBadGateway) + h.proxy.serveArtifactError(w, err, "failed to fetch package") return } diff --git a/internal/handler/pypi.go b/internal/handler/pypi.go index 0cf39fb..f5a0232 100644 --- a/internal/handler/pypi.go +++ b/internal/handler/pypi.go @@ -426,8 +426,7 @@ func (h *PyPIHandler) handleDownload(w http.ResponseWriter, r *http.Request) { result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "pypi", name, version, filename, upstreamURL) if err != nil { - h.proxy.Logger.Error("failed to get artifact", "error", err) - http.Error(w, "failed to fetch package", http.StatusBadGateway) + h.proxy.serveArtifactError(w, err, "failed to fetch package") return } diff --git a/internal/handler/rpm.go b/internal/handler/rpm.go index 6440d0f..a5752ec 100644 --- a/internal/handler/rpm.go +++ b/internal/handler/rpm.go @@ -83,8 +83,7 @@ func (h *RPMHandler) handlePackageDownload(w http.ResponseWriter, r *http.Reques result, err := h.proxy.GetOrFetchArtifactFromURL( r.Context(), "rpm", name, version, filename, downloadURL) if err != nil { - h.proxy.Logger.Error("failed to get rpm package", "error", err) - http.Error(w, "failed to fetch package", http.StatusBadGateway) + h.proxy.serveArtifactError(w, err, "failed to fetch package") return } From cf3741162f7e3000cc9bc78152896e608e5a6549 Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Sun, 26 Jul 2026 19:11:24 +0100 Subject: [PATCH 023/633] fix(upstream): honor npm and Cargo overrides (#200) * fix(upstream): honor npm and cargo overrides * Preserve npm scope separators in download URLs * Apply upstream auth to metadata requests --- internal/handler/cargo.go | 24 ++++++++-- internal/handler/cargo_test.go | 71 +++++++++++++++++++++++++++++ internal/handler/handler.go | 16 +++++++ internal/handler/npm.go | 26 +++++++++-- internal/handler/npm_test.go | 83 ++++++++++++++++++++++++++++++++++ internal/server/server.go | 10 +++- internal/server/server_test.go | 9 +++- 7 files changed, 228 insertions(+), 11 deletions(-) diff --git a/internal/handler/cargo.go b/internal/handler/cargo.go index abc0d1f..79fb750 100644 --- a/internal/handler/cargo.go +++ b/internal/handler/cargo.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "net/http" + "net/url" "strings" "time" ) @@ -28,11 +29,18 @@ type CargoHandler struct { } // NewCargoHandler creates a new cargo protocol handler. -func NewCargoHandler(proxy *Proxy, proxyURL string) *CargoHandler { +func NewCargoHandler(proxy *Proxy, proxyURL, indexURL, downloadURL string) *CargoHandler { + if strings.TrimSpace(indexURL) == "" { + indexURL = cargoUpstream + } + if strings.TrimSpace(downloadURL) == "" { + downloadURL = cargoDownloadBase + } + return &CargoHandler{ proxy: proxy, - indexURL: cargoUpstream, - downloadURL: cargoDownloadBase, + indexURL: strings.TrimSuffix(indexURL, "/"), + downloadURL: strings.TrimSuffix(downloadURL, "/"), proxyURL: strings.TrimSuffix(proxyURL, "/"), } } @@ -189,7 +197,15 @@ func (h *CargoHandler) handleDownload(w http.ResponseWriter, r *http.Request) { h.proxy.Logger.Info("cargo download request", "crate", name, "version", version, "filename", filename) - result, err := h.proxy.GetOrFetchArtifact(r.Context(), "cargo", name, version, filename) + downloadURL := fmt.Sprintf( + "%s/%s/%s", + h.downloadURL, + url.PathEscape(name), + url.PathEscape(filename), + ) + result, err := h.proxy.GetOrFetchArtifactFromURL( + r.Context(), "cargo", name, version, filename, downloadURL, + ) if err != nil { h.proxy.serveArtifactError(w, err, "failed to fetch crate") return diff --git a/internal/handler/cargo_test.go b/internal/handler/cargo_test.go index 10d3faf..895ff7b 100644 --- a/internal/handler/cargo_test.go +++ b/internal/handler/cargo_test.go @@ -2,6 +2,7 @@ package handler import ( "encoding/json" + "io" "log/slog" "net/http" "net/http/httptest" @@ -10,6 +11,7 @@ import ( "time" "github.com/git-pkgs/cooldown" + "github.com/git-pkgs/registries/fetch" ) func cargoTestProxy() *Proxy { @@ -70,6 +72,75 @@ func TestCargoConfigEndpoint(t *testing.T) { } } +func TestCargoHandlerUsesConfiguredUpstreams(t *testing.T) { + t.Run("index", func(t *testing.T) { + var requestPath, authHeader string + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestPath = r.URL.Path + authHeader = r.Header.Get("Authorization") + if authHeader != "Bearer cargo-token" { + w.WriteHeader(http.StatusUnauthorized) + return + } + w.Header().Set("Content-Type", "text/plain") + _, _ = io.WriteString(w, `{"name":"serde","vers":"1.0.0"}`) + })) + defer upstream.Close() + + proxy, _, _, _ := setupTestProxy(t) + proxy.HTTPClient = upstream.Client() + proxy.AuthForURL = func(string) (string, string) { + return "Authorization", "Bearer cargo-token" + } + h := NewCargoHandler( + proxy, + "http://proxy.test", + upstream.URL+"/index/", + "https://crates.example.test/files/", + ) + + req := httptest.NewRequest(http.MethodGet, "/se/rd/serde", nil) + w := httptest.NewRecorder() + h.Routes().ServeHTTP(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want %d; body: %s", w.Code, http.StatusOK, w.Body.String()) + } + if requestPath != "/index/se/rd/serde" { + t.Errorf("upstream path = %q, want %q", requestPath, "/index/se/rd/serde") + } + if authHeader != "Bearer cargo-token" { + t.Errorf("Authorization = %q, want %q", authHeader, "Bearer cargo-token") + } + }) + + t.Run("download", func(t *testing.T) { + proxy, _, _, artifactFetcher := setupTestProxy(t) + artifactFetcher.artifact = &fetch.Artifact{ + Body: io.NopCloser(strings.NewReader("crate")), + ContentType: "application/gzip", + } + h := NewCargoHandler( + proxy, + "http://proxy.test", + "https://index.example.test/root/", + "https://crates.example.test/files/", + ) + + req := httptest.NewRequest(http.MethodGet, "/crates/serde/1.0.0/download", nil) + w := httptest.NewRecorder() + h.Routes().ServeHTTP(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want %d; body: %s", w.Code, http.StatusOK, w.Body.String()) + } + want := "https://crates.example.test/files/serde/serde-1.0.0.crate" + if artifactFetcher.fetchedURL != want { + t.Errorf("fetched URL = %q, want %q", artifactFetcher.fetchedURL, want) + } + }) +} + func TestCargoIndexProxy(t *testing.T) { // Create a mock upstream index server indexContent := `{"name":"serde","vers":"1.0.0","deps":[],"cksum":"abc123"} diff --git a/internal/handler/handler.go b/internal/handler/handler.go index e62292f..fc78dbf 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -104,6 +104,7 @@ type Proxy struct { // storage at an internal one. DirectServeBaseURL string HTTPClient *http.Client + AuthForURL func(string) (headerName, headerValue string) } // NewProxy creates a new Proxy with the given dependencies. @@ -405,6 +406,7 @@ func (p *Proxy) ProxyUpstream(w http.ResponseWriter, r *http.Request, upstreamUR req.Header.Set(header, v) } } + p.applyUpstreamAuth(req) resp, err := p.HTTPClient.Do(req) if err != nil { @@ -431,6 +433,7 @@ func (p *Proxy) ProxyFile(w http.ResponseWriter, r *http.Request, upstreamURL st http.Error(w, "failed to create request", http.StatusInternalServerError) return } + p.applyUpstreamAuth(req) resp, err := p.HTTPClient.Do(req) if err != nil { @@ -571,6 +574,7 @@ func (p *Proxy) fetchUpstreamMetadata(ctx context.Context, upstreamURL string, e return nil, "", "", zeroTime, fmt.Errorf("creating request: %w", err) } req.Header.Set("Accept", accept) + p.applyUpstreamAuth(req) if entry != nil && entry.ETag.Valid { req.Header.Set("If-None-Match", entry.ETag.String) @@ -760,6 +764,7 @@ func (p *Proxy) proxyMetadataStream(w http.ResponseWriter, r *http.Request, upst accept = acceptHeaders[0] } req.Header.Set("Accept", accept) + p.applyUpstreamAuth(req) for _, header := range []string{headerAcceptEncoding, "If-Modified-Since", "If-None-Match"} { if v := r.Header.Get(header); v != "" { @@ -784,6 +789,17 @@ func (p *Proxy) proxyMetadataStream(w http.ResponseWriter, r *http.Request, upst _, _ = io.Copy(w, resp.Body) } +func (p *Proxy) applyUpstreamAuth(req *http.Request) { + if p.AuthForURL == nil { + return + } + + headerName, headerValue := p.AuthForURL(req.URL.String()) + if headerName != "" && headerValue != "" { + req.Header.Set(headerName, headerValue) + } +} + // GetOrFetchArtifactFromURL retrieves an artifact from cache or fetches from a specific URL. // This is useful for registries where download URLs are determined from metadata. func (p *Proxy) GetOrFetchArtifactFromURL(ctx context.Context, ecosystem, name, version, filename, downloadURL string) (*CacheResult, error) { diff --git a/internal/handler/npm.go b/internal/handler/npm.go index a3ed910..ee9b59c 100644 --- a/internal/handler/npm.go +++ b/internal/handler/npm.go @@ -25,10 +25,14 @@ type NPMHandler struct { } // NewNPMHandler creates a new npm protocol handler. -func NewNPMHandler(proxy *Proxy, proxyURL string) *NPMHandler { +func NewNPMHandler(proxy *Proxy, proxyURL, upstreamURL string) *NPMHandler { + if strings.TrimSpace(upstreamURL) == "" { + upstreamURL = npmUpstream + } + return &NPMHandler{ proxy: proxy, - upstreamURL: npmUpstream, + upstreamURL: strings.TrimSuffix(upstreamURL, "/"), proxyURL: strings.TrimSuffix(proxyURL, "/"), } } @@ -261,7 +265,15 @@ func (h *NPMHandler) handleDownload(w http.ResponseWriter, r *http.Request) { h.proxy.Logger.Info("npm download request", "package", packageName, "version", version, "filename", filename) - result, err := h.proxy.GetOrFetchArtifact(r.Context(), "npm", packageName, version, filename) + downloadURL := fmt.Sprintf( + "%s/%s/-/%s", + h.upstreamURL, + escapeNPMDownloadPackage(packageName), + url.PathEscape(filename), + ) + result, err := h.proxy.GetOrFetchArtifactFromURL( + r.Context(), "npm", packageName, version, filename, downloadURL, + ) if err != nil { if errors.Is(err, ErrUpstreamNotFound) { JSONError(w, http.StatusNotFound, "package not found") @@ -275,6 +287,14 @@ func (h *NPMHandler) handleDownload(w http.ResponseWriter, r *http.Request) { ServeArtifact(w, result) } +func escapeNPMDownloadPackage(packageName string) string { + scope, name, scoped := strings.Cut(packageName, "/") + if scoped && strings.HasPrefix(scope, "@") && len(scope) > 1 && name != "" && !strings.Contains(name, "/") { + return url.PathEscape(scope) + "/" + url.PathEscape(name) + } + return url.PathEscape(packageName) +} + // extractPackageName extracts the package name from the request path. // Handles both scoped (@scope/name) and unscoped (name) packages. func (h *NPMHandler) extractPackageName(r *http.Request) string { diff --git a/internal/handler/npm_test.go b/internal/handler/npm_test.go index bc1edde..e0257dd 100644 --- a/internal/handler/npm_test.go +++ b/internal/handler/npm_test.go @@ -2,13 +2,16 @@ package handler import ( "encoding/json" + "io" "log/slog" "net/http" "net/http/httptest" + "strings" "testing" "time" "github.com/git-pkgs/cooldown" + "github.com/git-pkgs/registries/fetch" ) const testVersion100 = "1.0.0" @@ -46,6 +49,86 @@ func TestNPMExtractVersionFromFilename(t *testing.T) { } } +func TestNPMHandlerUsesConfiguredUpstream(t *testing.T) { + t.Run("metadata", func(t *testing.T) { + var requestPath, authHeader string + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestPath = r.URL.Path + authHeader = r.Header.Get("Authorization") + if authHeader != "Bearer npm-token" { + w.WriteHeader(http.StatusUnauthorized) + return + } + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, `{"versions":{}}`) + })) + defer upstream.Close() + + proxy, _, _, _ := setupTestProxy(t) + proxy.HTTPClient = upstream.Client() + proxy.AuthForURL = func(string) (string, string) { + return "Authorization", "Bearer npm-token" + } + h := NewNPMHandler(proxy, "http://proxy.test", upstream.URL+"/root/") + + req := httptest.NewRequest(http.MethodGet, "/testpkg", nil) + w := httptest.NewRecorder() + h.Routes().ServeHTTP(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want %d; body: %s", w.Code, http.StatusOK, w.Body.String()) + } + if requestPath != "/root/testpkg" { + t.Errorf("upstream path = %q, want %q", requestPath, "/root/testpkg") + } + if authHeader != "Bearer npm-token" { + t.Errorf("Authorization = %q, want %q", authHeader, "Bearer npm-token") + } + }) + + t.Run("download", func(t *testing.T) { + proxy, _, _, artifactFetcher := setupTestProxy(t) + artifactFetcher.artifact = &fetch.Artifact{ + Body: io.NopCloser(strings.NewReader("package")), + ContentType: "application/gzip", + } + h := NewNPMHandler(proxy, "http://proxy.test", "https://npm.example.test/root/") + + req := httptest.NewRequest(http.MethodGet, "/testpkg/-/testpkg-1.0.0.tgz", nil) + w := httptest.NewRecorder() + h.Routes().ServeHTTP(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want %d; body: %s", w.Code, http.StatusOK, w.Body.String()) + } + want := "https://npm.example.test/root/testpkg/-/testpkg-1.0.0.tgz" + if artifactFetcher.fetchedURL != want { + t.Errorf("fetched URL = %q, want %q", artifactFetcher.fetchedURL, want) + } + }) + + t.Run("scoped download", func(t *testing.T) { + proxy, _, _, artifactFetcher := setupTestProxy(t) + artifactFetcher.artifact = &fetch.Artifact{ + Body: io.NopCloser(strings.NewReader("package")), + ContentType: "application/gzip", + } + h := NewNPMHandler(proxy, "http://proxy.test", "https://npm.example.test/root/") + + req := httptest.NewRequest(http.MethodGet, "/@scope/name/-/name-1.0.0.tgz", nil) + w := httptest.NewRecorder() + h.Routes().ServeHTTP(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want %d; body: %s", w.Code, http.StatusOK, w.Body.String()) + } + want := "https://npm.example.test/root/@scope/name/-/name-1.0.0.tgz" + if artifactFetcher.fetchedURL != want { + t.Errorf("fetched URL = %q, want %q", artifactFetcher.fetchedURL, want) + } + }) +} + func TestNPMRewriteMetadata(t *testing.T) { h := &NPMHandler{ proxy: testProxy(), diff --git a/internal/server/server.go b/internal/server/server.go index 74c82c7..13c5997 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -167,6 +167,7 @@ func (s *Server) Start() error { } proxy := handler.NewProxy(s.db, s.storage, fetcher, resolver, s.logger) proxy.HTTPClient.Timeout = s.cfg.ParseHTTPTimeout() + proxy.AuthForURL = s.authForURL proxy.Cooldown = cd proxy.CacheMetadata = s.cfg.CacheMetadata proxy.MetadataTTL = s.cfg.ParseMetadataTTL() @@ -196,8 +197,13 @@ func (s *Server) Start() error { }) // Mount protocol handlers - npmHandler := handler.NewNPMHandler(proxy, s.cfg.BaseURL) - cargoHandler := handler.NewCargoHandler(proxy, s.cfg.BaseURL) + npmHandler := handler.NewNPMHandler(proxy, s.cfg.BaseURL, s.cfg.Upstream.NPM) + cargoHandler := handler.NewCargoHandler( + proxy, + s.cfg.BaseURL, + s.cfg.Upstream.Cargo, + s.cfg.Upstream.CargoDownload, + ) gemHandler := handler.NewGemHandler(proxy, s.cfg.BaseURL) goHandler := handler.NewGoHandler(proxy, s.cfg.BaseURL) hexHandler := handler.NewHexHandler(proxy, s.cfg.BaseURL) diff --git a/internal/server/server_test.go b/internal/server/server_test.go index f1de62d..2d27147 100644 --- a/internal/server/server_test.go +++ b/internal/server/server_test.go @@ -67,8 +67,13 @@ func newTestServer(t *testing.T) *testServer { r := chi.NewRouter() // Mount handlers - npmHandler := handler.NewNPMHandler(proxy, cfg.BaseURL) - cargoHandler := handler.NewCargoHandler(proxy, cfg.BaseURL) + npmHandler := handler.NewNPMHandler(proxy, cfg.BaseURL, cfg.Upstream.NPM) + cargoHandler := handler.NewCargoHandler( + proxy, + cfg.BaseURL, + cfg.Upstream.Cargo, + cfg.Upstream.CargoDownload, + ) gemHandler := handler.NewGemHandler(proxy, cfg.BaseURL) goHandler := handler.NewGoHandler(proxy, cfg.BaseURL) pypiHandler := handler.NewPyPIHandler(proxy, cfg.BaseURL) From 44041d07a986de9b567c4a7e3dd5eb34aa3f5b8b Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Mon, 27 Jul 2026 12:15:57 +0100 Subject: [PATCH 024/633] Bump git-pkgs archives, enrichment, purl, registries, vulns (#215) --- go.mod | 17 +++++++++-------- go.sum | 32 ++++++++++++++++---------------- 2 files changed, 25 insertions(+), 24 deletions(-) diff --git a/go.mod b/go.mod index 6d81076..f3becd4 100644 --- a/go.mod +++ b/go.mod @@ -5,14 +5,14 @@ go 1.25.6 require ( github.com/BurntSushi/toml v1.6.0 github.com/CycloneDX/cyclonedx-go v0.11.0 - github.com/git-pkgs/archives v0.3.0 + github.com/git-pkgs/archives v0.3.1 github.com/git-pkgs/cooldown v0.1.1 - github.com/git-pkgs/enrichment v0.6.0 - github.com/git-pkgs/purl v0.1.14 - github.com/git-pkgs/registries v0.6.3 + github.com/git-pkgs/enrichment v0.6.4 + github.com/git-pkgs/purl v0.1.15 + github.com/git-pkgs/registries v0.6.4 github.com/git-pkgs/spdx v0.1.4 github.com/git-pkgs/vers v0.3.0 - github.com/git-pkgs/vulns v0.2.0 + github.com/git-pkgs/vulns v0.2.1 github.com/go-chi/chi/v5 v5.3.1 github.com/jmoiron/sqlx v1.4.0 github.com/lib/pq v1.12.3 @@ -115,7 +115,7 @@ require ( github.com/denis-tingaikin/go-header v0.5.0 // indirect github.com/dlclark/regexp2 v1.11.5 // indirect github.com/dustin/go-humanize v1.0.1 // indirect - github.com/ecosyste-ms/ecosystems-go v0.3.0 // indirect + github.com/ecosyste-ms/ecosystems-go v0.4.0 // indirect github.com/ettle/strcase v0.2.0 // indirect github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a // indirect github.com/fatih/color v1.18.0 // indirect @@ -217,7 +217,8 @@ require ( github.com/nishanths/exhaustive v0.12.0 // indirect github.com/nishanths/predeclared v0.2.2 // indirect github.com/nunnatsa/ginkgolinter v0.23.0 // indirect - github.com/oapi-codegen/runtime v1.4.2 // indirect + github.com/oapi-codegen/nullable v1.1.0 // indirect + github.com/oapi-codegen/runtime v1.6.0 // indirect github.com/package-url/packageurl-go v0.1.6 // indirect github.com/pandatix/go-cvss v0.6.2 // indirect github.com/pelletier/go-toml v1.9.5 // indirect @@ -265,7 +266,7 @@ require ( github.com/timonwong/loggercheck v0.11.0 // indirect github.com/tomarrell/wrapcheck/v2 v2.12.0 // indirect github.com/tommy-muehle/go-mnd/v2 v2.5.1 // indirect - github.com/ulikunitz/xz v0.5.15 // indirect + github.com/ulikunitz/xz v0.5.16 // indirect github.com/ultraware/funlen v0.2.0 // indirect github.com/ultraware/whitespace v0.2.0 // indirect github.com/urfave/cli/v2 v2.3.0 // indirect diff --git a/go.sum b/go.sum index 5d16d31..34e9116 100644 --- a/go.sum +++ b/go.sum @@ -217,8 +217,8 @@ github.com/dlclark/regexp2 v1.11.5 h1:Q/sSnsKerHeCkc/jSTNq1oCm7KiVgUMZRDUoRu0JQZ github.com/dlclark/regexp2 v1.11.5/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= -github.com/ecosyste-ms/ecosystems-go v0.3.0 h1:eVTNKQ3PyVNkSAnk1934958Vg4rR7ho5B5MWEQaMLi4= -github.com/ecosyste-ms/ecosystems-go v0.3.0/go.mod h1:bkjiI8WoTFB1Jw0M3h8yn3KXCD/+LdETsQ3m2BNLMHQ= +github.com/ecosyste-ms/ecosystems-go v0.4.0 h1:5A+zF+XWT8sYYYjlc2/tI1SmiDGzbHLyT9CapVc5dGA= +github.com/ecosyste-ms/ecosystems-go v0.4.0/go.mod h1:FVswCrp3DQkur1HjVqfDF/gYrDSEmiFflntcB1G0DbA= github.com/envoyproxy/go-control-plane v0.14.0 h1:hbG2kr4RuFj222B6+7T83thSPqLjwBIfQawTkC++2HA= github.com/envoyproxy/go-control-plane/envoy v1.37.0 h1:u3riX6BoYRfF4Dr7dwSOroNfdSbEPe9Yyl09/B6wBrQ= github.com/envoyproxy/go-control-plane/envoy v1.37.0/go.mod h1:DReE9MMrmecPy+YvQOAOHNYMALuowAnbjjEMkkWOi6A= @@ -244,26 +244,26 @@ github.com/fzipp/gocyclo v0.6.0 h1:lsblElZG7d3ALtGMx9fmxeTKZaLLpU8mET09yN4BBLo= github.com/fzipp/gocyclo v0.6.0/go.mod h1:rXPyn8fnlpa0R2csP/31uerbiVBugk5whMdlyaLkLoA= github.com/ghostiam/protogetter v0.3.20 h1:oW7OPFit2FxZOpmMRPP9FffU4uUpfeE/rEdE1f+MzD0= github.com/ghostiam/protogetter v0.3.20/go.mod h1:FjIu5Yfs6FT391m+Fjp3fbAYJ6rkL/J6ySpZBfnODuI= -github.com/git-pkgs/archives v0.3.0 h1:iXKyO83jEFub1PGEDlHmk2tQ7XeV5LySTc0sEkH3x78= -github.com/git-pkgs/archives v0.3.0/go.mod h1:LTJ1iQVFA7otizWMOyiI82NYVmyBWAPRzwu/e30rcXU= +github.com/git-pkgs/archives v0.3.1 h1:GKUuw++0YXAAElxweVHiR4AaSShKKYoVQmyxlF5blG4= +github.com/git-pkgs/archives v0.3.1/go.mod h1:408oQv3FxLCtePa33zp3sg3njXnwH74vnHZFxkRqoPo= github.com/git-pkgs/cooldown v0.1.1 h1:9OqqzCB8gANz/y44SmqGD0Jp8Qtu81D1sCbKl6Ehg7w= github.com/git-pkgs/cooldown v0.1.1/go.mod h1:v7APuK/UouTiu8mWQZbdDmj7DfxxkGUeuhjaRB5gv9E= -github.com/git-pkgs/enrichment v0.6.0 h1:npV6N+eFZnI64uw/B0s+WJPn6Fu6Be08bexAViZFjMg= -github.com/git-pkgs/enrichment v0.6.0/go.mod h1:ov2WDaiNoIXViqdnE1FlUjNTnB5RnkUH4et9BWPhUDY= +github.com/git-pkgs/enrichment v0.6.4 h1:mGrfenttwmcUfPXRkWpB0wBJiiGj55ltniUh66Pq4bU= +github.com/git-pkgs/enrichment v0.6.4/go.mod h1:zz1vPUak/w8Jhajll0KDRN2MjKaEYeCzQTxumWnVhqY= github.com/git-pkgs/packageurl-go v0.3.1 h1:WM3RBABQZLaRBxgKyYughc3cVBE8KyQxbSC6Jt5ak7M= github.com/git-pkgs/packageurl-go v0.3.1/go.mod h1:rcIxiG37BlQLB6FZfgdj9Fm7yjhRQd3l+5o7J0QPAk4= github.com/git-pkgs/pom v0.1.5 h1:TGT8Az2OMxGWsXnSagtUMGzZm7Oax8HrSCteA+mi0qY= github.com/git-pkgs/pom v0.1.5/go.mod h1:ufdMBe1lKzqOeP9IUb9NPZ458xKV8E8NvuyBMxOfwIk= -github.com/git-pkgs/purl v0.1.14 h1:GgqwiBNS0eKJqJ/gabUBEC10Xhpj6UX12kfNzujaeYc= -github.com/git-pkgs/purl v0.1.14/go.mod h1:8oCcdcYZA/e1B33e7Ylju6azboTKjdqf3ybcbQj6I/o= -github.com/git-pkgs/registries v0.6.3 h1:7sveeeMS2lgXtcqNYAA3bwaT7H9CQs0uhy6wQSxz3Js= -github.com/git-pkgs/registries v0.6.3/go.mod h1:j4o50ii/vD9Z42/nFeQASt15BPolADIejeFmmqK4njo= +github.com/git-pkgs/purl v0.1.15 h1:iQ3clh0Cw41rkM0rf24B7ShnN9Z+UtLMAFlNDUs+Qd4= +github.com/git-pkgs/purl v0.1.15/go.mod h1:PqCLVBDeZrZgHysR803/AntMELgIr2LFZVNCcwLH2m0= +github.com/git-pkgs/registries v0.6.4 h1:Kq/KlStjaQyE83UXT/tKuzCrIzc4keGeBjtroMqgoHA= +github.com/git-pkgs/registries v0.6.4/go.mod h1:YkGHbxHIe2Ha/ROH6zNkS5PJUUoa9g0Ti/s2XhZnrak= github.com/git-pkgs/spdx v0.1.4 h1:eQ0waEV3uUeItpWAOvdN1K1rL9hTgsU7fF74r1mDXMs= github.com/git-pkgs/spdx v0.1.4/go.mod h1:cqRoZcvl530s/W+oGNvwjt4ODN8T1W6D/20MUZEFdto= github.com/git-pkgs/vers v0.3.0 h1:xM4LLUCRmqzdDfe+/pVQUx4SRyFXRVth6tOsJ14wMKU= github.com/git-pkgs/vers v0.3.0/go.mod h1:biTbSQK1qdbrsxDEKnqe3Jzclxz8vW6uDcwKjfUGcOo= -github.com/git-pkgs/vulns v0.2.0 h1:S1sA0ObQ/IKW0vqK+lK+HAY37AdDHEfhaIja/fLptwg= -github.com/git-pkgs/vulns v0.2.0/go.mod h1:mjVquLlunsAFPltqjf6mbvbI4tQ1iIa1NuhKIOPKt/4= +github.com/git-pkgs/vulns v0.2.1 h1:tWGhOfPVDZwkM2Y9vRkMpMR+gjtlu2jhERS5JeNBoKQ= +github.com/git-pkgs/vulns v0.2.1/go.mod h1:/0gHKHQR5SWttZVEMqgOvCXssKFwAtbac/PfkhBax9o= github.com/github/go-spdx/v2 v2.7.0 h1:GzfXx4wFdlilARxmFRXW/mgUy3A4vSqZocCMFV6XFdQ= github.com/github/go-spdx/v2 v2.7.0/go.mod h1:Ftc45YYG1WzpzwEPKRVm9Jv8vDqOrN4gWoCkK+bHer0= github.com/go-chi/chi/v5 v5.3.1 h1:3j4HZLGZQ3JpMCrPJF/Jl3mYJfWLKBfNJ6quurUGCf8= @@ -512,8 +512,8 @@ github.com/nunnatsa/ginkgolinter v0.23.0 h1:x3o4DGYOWbBMP/VdNQKgSj+25aJKx2Pe6lHr github.com/nunnatsa/ginkgolinter v0.23.0/go.mod h1:9qN1+0akwXEccwV1CAcCDfcoBlWXHB+ML9884pL4SZ4= github.com/oapi-codegen/nullable v1.1.0 h1:eAh8JVc5430VtYVnq00Hrbpag9PFRGWLjxR1/3KntMs= github.com/oapi-codegen/nullable v1.1.0/go.mod h1:KUZ3vUzkmEKY90ksAmit2+5juDIhIZhfDl+0PwOQlFY= -github.com/oapi-codegen/runtime v1.4.2 h1:GMxFVYLzoYLua+/KvzgSphkyK1lLTReQI9Vf4hvATKE= -github.com/oapi-codegen/runtime v1.4.2/go.mod h1:GwV7hC2hviaMzj+ITfHVRESK5J2W/GefVwIND/bMGvU= +github.com/oapi-codegen/runtime v1.6.0 h1:7Xx+GlueD6nRuyKoCPzL434Jfi3BetbiJOrzCHp/VPU= +github.com/oapi-codegen/runtime v1.6.0/go.mod h1:GwV7hC2hviaMzj+ITfHVRESK5J2W/GefVwIND/bMGvU= github.com/onsi/ginkgo/v2 v2.28.1 h1:S4hj+HbZp40fNKuLUQOYLDgZLwNUVn19N3Atb98NCyI= github.com/onsi/ginkgo/v2 v2.28.1/go.mod h1:CLtbVInNckU3/+gC8LzkGUb9oF+e8W8TdUsxPwvdOgE= github.com/onsi/gomega v1.39.1 h1:1IJLAad4zjPn2PsnhH70V4DKRFlrCzGBNrNaru+Vf28= @@ -654,8 +654,8 @@ github.com/tomarrell/wrapcheck/v2 v2.12.0 h1:H/qQ1aNWz/eeIhxKAFvkfIA+N7YDvq6TWVF github.com/tomarrell/wrapcheck/v2 v2.12.0/go.mod h1:AQhQuZd0p7b6rfW+vUwHm5OMCGgp63moQ9Qr/0BpIWo= github.com/tommy-muehle/go-mnd/v2 v2.5.1 h1:NowYhSdyE/1zwK9QCLeRb6USWdoif80Ie+v+yU8u1Zw= github.com/tommy-muehle/go-mnd/v2 v2.5.1/go.mod h1:WsUAkMJMYww6l/ufffCD3m+P7LEvr8TnZn9lwVDlgzw= -github.com/ulikunitz/xz v0.5.15 h1:9DNdB5s+SgV3bQ2ApL10xRc35ck0DuIX/isZvIk+ubY= -github.com/ulikunitz/xz v0.5.15/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= +github.com/ulikunitz/xz v0.5.16 h1:ld6NyySjx5lowVKwJvMRLnW5nxKX/xnpSiFYZ/Lxur0= +github.com/ulikunitz/xz v0.5.16/go.mod h1:H9Rt/W6/Qj27PGauhQc6nfCDy7vHpzsOThBSaYDoEhw= github.com/ultraware/funlen v0.2.0 h1:gCHmCn+d2/1SemTdYMiKLAHFYxTYz7z9VIDRaTGyLkI= github.com/ultraware/funlen v0.2.0/go.mod h1:ZE0q4TsJ8T1SQcjmkhN/w+MceuatI6pBFSxxyteHIJA= github.com/ultraware/whitespace v0.2.0 h1:TYowo2m9Nfj1baEQBjuHzvMRbp19i+RCcRYrSWoFa+g= From 90422697b81da002b7ad9ec2908c9cb6c7d7e84c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 Jul 2026 12:16:27 +0100 Subject: [PATCH 025/633] Bump google.golang.org/grpc from 1.81.1 to 1.82.1 (#216) Bumps [google.golang.org/grpc](https://github.com/grpc/grpc-go) from 1.81.1 to 1.82.1. - [Release notes](https://github.com/grpc/grpc-go/releases) - [Commits](https://github.com/grpc/grpc-go/compare/v1.81.1...v1.82.1) --- updated-dependencies: - dependency-name: google.golang.org/grpc dependency-version: 1.82.1 dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 4 ++-- go.sum | 20 ++++++++++---------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/go.mod b/go.mod index f3becd4..c460ec7 100644 --- a/go.mod +++ b/go.mod @@ -303,8 +303,8 @@ require ( golang.org/x/tools v0.47.0 // indirect golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect google.golang.org/api v0.272.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20260316180232-0b37fe3546d5 // indirect - google.golang.org/grpc v1.81.1 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478 // indirect + google.golang.org/grpc v1.82.1 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect honnef.co/go/tools v0.7.0 // indirect diff --git a/go.sum b/go.sum index 34e9116..5621b47 100644 --- a/go.sum +++ b/go.sum @@ -66,8 +66,8 @@ github.com/CycloneDX/cyclonedx-go v0.11.0 h1:GokP8FiRC+foiuwWhSSLpSD5H4hSWtGnR3w github.com/CycloneDX/cyclonedx-go v0.11.0/go.mod h1:vUvbCXQsEm48OI6oOlanxstwNByXjCZ2wuleUlwGEO8= github.com/Djarvur/go-err113 v0.1.1 h1:eHfopDqXRwAi+YmCUas75ZE0+hoBHJ2GQNLYRSxao4g= github.com/Djarvur/go-err113 v0.1.1/go.mod h1:IaWJdYFLg76t2ihfflPZnM1LIQszWOsFDh2hhhAVF6k= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.31.0 h1:DHa2U07rk8syqvCge0QIGMCE1WxGj9njT44GH7zNJLQ= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.31.0/go.mod h1:P4WPRUkOhJC13W//jWpyfJNDAIpvRbAUIYLX/4jtlE0= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.32.0 h1:rIkQfkCOVKc1OiRCNcSDD8ml5RJlZbH/Xsq7lbpynwc= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.32.0/go.mod h1:RD2SsorTmYhF6HkTmDw7KmPYQk8OBYwTkuasChwv7R4= github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.55.0 h1:UnDZ/zFfG1JhH/DqxIZYU/1CUAlTUScoXD/LcM2Ykk8= github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.55.0/go.mod h1:IA1C1U7jO/ENqm/vhi7V9YYpBsp+IMyqNrEN94N7tVc= github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.55.0 h1:0s6TxfCu2KHkkZPnBfsQ2y5qia0jl3MMrmBhu3nCOYk= @@ -702,8 +702,8 @@ go.augendre.info/fatcontext v0.9.0 h1:Gt5jGD4Zcj8CDMVzjOJITlSb9cEch54hjRRlN3qDoj go.augendre.info/fatcontext v0.9.0/go.mod h1:L94brOAT1OOUNue6ph/2HnwxoNlds9aXDF2FcUntbNw= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= -go.opentelemetry.io/contrib/detectors/gcp v1.42.0 h1:kpt2PEJuOuqYkPcktfJqWWDjTEd/FNgrxcniL7kQrXQ= -go.opentelemetry.io/contrib/detectors/gcp v1.42.0/go.mod h1:W9zQ439utxymRrXsUOzZbFX4JhLxXU4+ZnCt8GG7yA8= +go.opentelemetry.io/contrib/detectors/gcp v1.43.0 h1:62yY3dT7/ShwOxzA0RsKRgshBmfElKI4d/Myu2OxDFU= +go.opentelemetry.io/contrib/detectors/gcp v1.43.0/go.mod h1:RyaZMFY7yi1kAs45S6mbFGz8O8rqB0dTY14uzvG4LCs= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0 h1:yI1/OhfEPy7J9eoa6Sj051C7n5dvpj0QX8g4sRchg04= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0/go.mod h1:NoUCKYWK+3ecatC4HjkRktREheMeEtrXoQxrqYFeHSc= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0 h1:OyrsyzuttWTSur2qN/Lm0m2a8yqyIjUVBZcxFPuXq2o= @@ -857,12 +857,12 @@ google.golang.org/api v0.272.0 h1:eLUQZGnAS3OHn31URRf9sAmRk3w2JjMx37d2k8AjJmA= google.golang.org/api v0.272.0/go.mod h1:wKjowi5LNJc5qarNvDCvNQBn3rVK8nSy6jg2SwRwzIA= google.golang.org/genproto v0.0.0-20260316180232-0b37fe3546d5 h1:JNfk58HZ8lfmXbYK2vx/UvsqIL59TzByCxPIX4TDmsE= google.golang.org/genproto v0.0.0-20260316180232-0b37fe3546d5/go.mod h1:x5julN69+ED4PcFk/XWayw35O0lf/nGa4aNgODCmNmw= -google.golang.org/genproto/googleapis/api v0.0.0-20260316180232-0b37fe3546d5 h1:CogIeEXn4qWYzzQU0QqvYBM8yDF9cFYzDq9ojSpv0Js= -google.golang.org/genproto/googleapis/api v0.0.0-20260316180232-0b37fe3546d5/go.mod h1:EIQZ5bFCfRQDV4MhRle7+OgjNtZ6P1PiZBgAKuxXu/Y= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260316180232-0b37fe3546d5 h1:aJmi6DVGGIStN9Mobk/tZOOQUBbj0BPjZjjnOdoZKts= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260316180232-0b37fe3546d5/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= -google.golang.org/grpc v1.81.1 h1:VnnIIZ88UzOOKLukQi+ImGz8O1Wdp8nAGGnvOfEIWQQ= -google.golang.org/grpc v1.81.1/go.mod h1:xGH9GfzOyMTGIOXBJmXt+BX/V0kcdQbdcuwQ/zNw42I= +google.golang.org/genproto/googleapis/api v0.0.0-20260414002931-afd174a4e478 h1:yQugLulqltosq0B/f8l4w9VryjV+N/5gcW0jQ3N8Qec= +google.golang.org/genproto/googleapis/api v0.0.0-20260414002931-afd174a4e478/go.mod h1:C6ADNqOxbgdUUeRTU+LCHDPB9ttAMCTff6auwCVa4uc= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478 h1:RmoJA1ujG+/lRGNfUnOMfhCy5EipVMyvUE+KNbPbTlw= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/grpc v1.82.1 h1:NnAxzGRA0677vCa4BUkOAnO5+FfQqVl9iUXeD0IqcGE= +google.golang.org/grpc v1.82.1/go.mod h1:yzTZ1TB1Z3SG+LIYaI+WiE8D5+PZ3ArnrSp8zF3+/ZA= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= From a5d456790d9da7455e6194cf28fc11d73c091eb0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 30 Jul 2026 17:14:36 +0100 Subject: [PATCH 026/633] Bump actions/checkout from 7.0.0 to 7.0.1 (#218) Bumps [actions/checkout](https://github.com/actions/checkout) from 7.0.0 to 7.0.1. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0...3d3c42e5aac5ba805825da76410c181273ba90b1) --- updated-dependencies: - dependency-name: actions/checkout dependency-version: 7.0.1 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/ci.yml | 4 ++-- .github/workflows/publish.yml | 2 +- .github/workflows/release.yml | 2 +- .github/workflows/swagger.yml | 2 +- .github/workflows/zizmor.yml | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c405f5d..bb3d87d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,7 +17,7 @@ jobs: runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false @@ -35,7 +35,7 @@ jobs: lint: runs-on: ubuntu-latest steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 9c99e5d..bb483f1 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -20,7 +20,7 @@ jobs: contents: read steps: - name: Check out the repo - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 with: persist-credentials: false diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f06cecd..5d32181 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -14,7 +14,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: fetch-depth: 0 persist-credentials: false diff --git a/.github/workflows/swagger.yml b/.github/workflows/swagger.yml index e99aecf..625f944 100644 --- a/.github/workflows/swagger.yml +++ b/.github/workflows/swagger.yml @@ -12,7 +12,7 @@ jobs: swagger: runs-on: ubuntu-latest steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false diff --git a/.github/workflows/zizmor.yml b/.github/workflows/zizmor.yml index df06c5f..d50b0a4 100644 --- a/.github/workflows/zizmor.yml +++ b/.github/workflows/zizmor.yml @@ -21,7 +21,7 @@ jobs: security-events: write steps: - name: Checkout - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false From b56a1fed658b91f5b6839ab02abee4471f22ea9c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 30 Jul 2026 17:52:35 +0100 Subject: [PATCH 027/633] Bump docker/login-action from 4.4.0 to 4.5.0 (#220) Bumps [docker/login-action](https://github.com/docker/login-action) from 4.4.0 to 4.5.0. - [Release notes](https://github.com/docker/login-action/releases) - [Commits](https://github.com/docker/login-action/compare/af1e73f918a031802d376d3c8bbc3fe56130a9b0...06fb636fac595d6fb4b28a5dfcb21a6f5091859c) --- updated-dependencies: - dependency-name: docker/login-action dependency-version: 4.5.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index bb483f1..c08cfa6 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -25,7 +25,7 @@ jobs: persist-credentials: false - name: Log in to the Container registry - uses: docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0 + uses: docker/login-action@06fb636fac595d6fb4b28a5dfcb21a6f5091859c with: registry: ghcr.io username: ${{ github.actor }} From 1057ee926eb29ea87015b5f6109242cc86d27d14 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 30 Jul 2026 17:55:16 +0100 Subject: [PATCH 028/633] Bump zizmorcore/zizmor-action from 0.6.0 to 0.6.1 (#217) Bumps [zizmorcore/zizmor-action](https://github.com/zizmorcore/zizmor-action) from 0.6.0 to 0.6.1. - [Release notes](https://github.com/zizmorcore/zizmor-action/releases) - [Commits](https://github.com/zizmorcore/zizmor-action/compare/6599ee8b7a49aef6a770f63d261d214911a7ce02...6fc4b006235f201fdab3722e17240ab420d580e5) --- updated-dependencies: - dependency-name: zizmorcore/zizmor-action dependency-version: 0.6.1 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/zizmor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/zizmor.yml b/.github/workflows/zizmor.yml index d50b0a4..4df0e18 100644 --- a/.github/workflows/zizmor.yml +++ b/.github/workflows/zizmor.yml @@ -26,4 +26,4 @@ jobs: persist-credentials: false - name: Run zizmor - uses: zizmorcore/zizmor-action@6599ee8b7a49aef6a770f63d261d214911a7ce02 # v0.6.0 + uses: zizmorcore/zizmor-action@6fc4b006235f201fdab3722e17240ab420d580e5 # v0.6.1 From 31ecca8cf14a643ea8479109b8d169ea4842a174 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 31 Jul 2026 09:31:21 +0100 Subject: [PATCH 029/633] Bump github.com/prometheus/client_golang from 1.23.2 to 1.24.0 (#221) Bumps [github.com/prometheus/client_golang](https://github.com/prometheus/client_golang) from 1.23.2 to 1.24.0. - [Release notes](https://github.com/prometheus/client_golang/releases) - [Changelog](https://github.com/prometheus/client_golang/blob/v1.24.0/CHANGELOG.md) - [Commits](https://github.com/prometheus/client_golang/compare/v1.23.2...v1.24.0) --- updated-dependencies: - dependency-name: github.com/prometheus/client_golang dependency-version: 1.24.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 10 +++++----- go.sum | 24 ++++++++++++------------ 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/go.mod b/go.mod index c460ec7..005ec32 100644 --- a/go.mod +++ b/go.mod @@ -16,7 +16,7 @@ require ( github.com/go-chi/chi/v5 v5.3.1 github.com/jmoiron/sqlx v1.4.0 github.com/lib/pq v1.12.3 - github.com/prometheus/client_golang v1.23.2 + github.com/prometheus/client_golang v1.24.0 github.com/prometheus/client_model v0.6.2 github.com/spdx/tools-golang v0.5.7 github.com/swaggo/swag v1.16.6 @@ -225,8 +225,8 @@ require ( github.com/pelletier/go-toml/v2 v2.2.4 // indirect github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/prometheus/common v0.67.5 // indirect - github.com/prometheus/procfs v0.20.1 // indirect + github.com/prometheus/common v0.70.0 // indirect + github.com/prometheus/procfs v0.21.1 // indirect github.com/quasilyte/go-ruleguard v0.4.5 // indirect github.com/quasilyte/go-ruleguard/dsl v0.3.23 // indirect github.com/quasilyte/gogrep v0.5.0 // indirect @@ -290,7 +290,7 @@ require ( go.opentelemetry.io/otel/trace v1.44.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.1 // indirect - go.yaml.in/yaml/v2 v2.4.3 // indirect + go.yaml.in/yaml/v2 v2.4.4 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect golang.org/x/crypto v0.53.0 // indirect golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa // indirect @@ -298,7 +298,7 @@ require ( golang.org/x/mod v0.37.0 // indirect golang.org/x/net v0.56.0 // indirect golang.org/x/oauth2 v0.36.0 // indirect - golang.org/x/sys v0.46.0 // indirect + golang.org/x/sys v0.47.0 // indirect golang.org/x/text v0.38.0 // indirect golang.org/x/tools v0.47.0 // indirect golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect diff --git a/go.sum b/go.sum index 5621b47..0801599 100644 --- a/go.sum +++ b/go.sum @@ -422,8 +422,8 @@ github.com/kisielk/errcheck v1.9.0 h1:9xt1zI9EBfcYBvdU1nVrzMzzUPUtPKs9bVSIM3TAb3 github.com/kisielk/errcheck v1.9.0/go.mod h1:kQxWMMVZgIkDq7U8xtG/n2juOjbLgZtedi0D+/VL/i8= github.com/kkHAIKE/contextcheck v1.1.6 h1:7HIyRcnyzxL9Lz06NGhiKvenXq7Zw6Q0UQu/ttjfJCE= github.com/kkHAIKE/contextcheck v1.1.6/go.mod h1:3dDbMRNBFaq8HFXWC1JyvDSPm43CmE6IuHam8Wr0rkg= -github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= -github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= +github.com/klauspost/compress v1.19.0 h1:sXLILfc9jV2QYWkzFOPWStmcUVH2RHEB1JCdY2oVvCQ= +github.com/klauspost/compress v1.19.0/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= @@ -541,14 +541,14 @@ github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgm github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= -github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= +github.com/prometheus/client_golang v1.24.0 h1:5XStIklKuAtJSNpdD3s8XJj/Yv78IQmE1kbNk87JrAI= +github.com/prometheus/client_golang v1.24.0/go.mod h1:QcsNdotprC2nS4BTM2ucbcqxd2CeXTEa9jW7zHO9iDE= github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= -github.com/prometheus/common v0.67.5 h1:pIgK94WWlQt1WLwAC5j2ynLaBRDiinoAb86HZHTUGI4= -github.com/prometheus/common v0.67.5/go.mod h1:SjE/0MzDEEAyrdr5Gqc6G+sXI67maCxzaT3A2+HqjUw= -github.com/prometheus/procfs v0.20.1 h1:XwbrGOIplXW/AU3YhIhLODXMJYyC1isLFfYCsTEycfc= -github.com/prometheus/procfs v0.20.1/go.mod h1:o9EMBZGRyvDrSPH1RqdxhojkuXstoe4UlK79eF5TGGo= +github.com/prometheus/common v0.70.0 h1:bcpru3tWPVnxGnETLgOV5jbp/JRXgYEyv65CuBLAMMI= +github.com/prometheus/common v0.70.0/go.mod h1:S/SFasQmgGiYH6C81LKCtYa8QACgthGg5zxL2udV7SY= +github.com/prometheus/procfs v0.21.1 h1:GljZCt+zSTS+NZq88cyQ1LjZ+RCHp3uVuabBWA5+OJI= +github.com/prometheus/procfs v0.21.1/go.mod h1:aB55Cww9pdSJVHk0hUf0inxWyyjPogFIjmHKYgMKmtY= github.com/quasilyte/go-ruleguard v0.4.5 h1:AGY0tiOT5hJX9BTdx/xBdoCubQUAE2grkqY2lSwvZcA= github.com/quasilyte/go-ruleguard v0.4.5/go.mod h1:Vl05zJ538vcEEwu16V/Hdu7IYZWyKSwIy4c88Ro1kRE= github.com/quasilyte/go-ruleguard/dsl v0.3.23 h1:lxjt5B6ZCiBeeNO8/oQsegE6fLeCzuMRoVWSkXC4uvY= @@ -726,8 +726,8 @@ go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.27.1 h1:08RqriUEv8+ArZRYSTXy1LeBScaMpVSTBhCeaZYfMYc= go.uber.org/zap v1.27.1/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= -go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0= -go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8= +go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ= +go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ= go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= gocloud.dev v0.46.0 h1:niIuZwSjMtBx8K+ITB2s5kZullB13PGOS2ZoQPZxQ4Q= @@ -807,8 +807,8 @@ golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw= -golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= +golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= From 9a9a82176da7988d4ded052c39f996e93c5adff8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 31 Jul 2026 09:32:38 +0100 Subject: [PATCH 030/633] Bump modernc.org/sqlite from 1.54.0 to 1.55.0 (#219) Bumps [modernc.org/sqlite](https://gitlab.com/cznic/sqlite) from 1.54.0 to 1.55.0. - [Changelog](https://gitlab.com/cznic/sqlite/blob/master/CHANGELOG.md) - [Commits](https://gitlab.com/cznic/sqlite/compare/v1.54.0...v1.55.0) --- updated-dependencies: - dependency-name: modernc.org/sqlite dependency-version: 1.55.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 005ec32..2f62065 100644 --- a/go.mod +++ b/go.mod @@ -24,7 +24,7 @@ require ( golang.org/x/sync v0.22.0 google.golang.org/protobuf v1.36.11 gopkg.in/yaml.v3 v3.0.1 - modernc.org/sqlite v1.54.0 + modernc.org/sqlite v1.55.0 ) require ( diff --git a/go.sum b/go.sum index 0801599..4f356f9 100644 --- a/go.sum +++ b/go.sum @@ -904,8 +904,8 @@ modernc.org/opt v0.2.0 h1:tGyef5ApycA7FSEOMraay9SaTk5zmbx7Tu+cJs4QKZg= modernc.org/opt v0.2.0/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns= modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w= modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE= -modernc.org/sqlite v1.54.0 h1:JCxR4qwkJvOaqAoYcgDoO25Nc+ROg6EJ2LfBVzdrgog= -modernc.org/sqlite v1.54.0/go.mod h1:4ntCLuNmnH8+GNqjka1wNg7KJd5/Hi5FYp8K+XQ7GZw= +modernc.org/sqlite v1.55.0 h1:hIFh0MCH0rGinQ/4KYb5/UbCkRkb+UP+OkLCVWa5MTM= +modernc.org/sqlite v1.55.0/go.mod h1:4ntCLuNmnH8+GNqjka1wNg7KJd5/Hi5FYp8K+XQ7GZw= modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0= modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A= modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= From 36f3a51c6523ca4b296dadec66bf8080263932c1 Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Fri, 31 Jul 2026 17:04:01 +0100 Subject: [PATCH 031/633] Detect content types when browsing files --- go.mod | 3 +- go.sum | 6 +- internal/server/browse.go | 128 +++++++++++++++---------- internal/server/browse_bench_test.go | 25 +++++ internal/server/browse_test.go | 138 ++++++++++++++++----------- 5 files changed, 190 insertions(+), 110 deletions(-) diff --git a/go.mod b/go.mod index 2f62065..e6e0a08 100644 --- a/go.mod +++ b/go.mod @@ -5,9 +5,10 @@ go 1.25.6 require ( github.com/BurntSushi/toml v1.6.0 github.com/CycloneDX/cyclonedx-go v0.11.0 - github.com/git-pkgs/archives v0.3.1 + github.com/git-pkgs/archives v0.4.0 github.com/git-pkgs/cooldown v0.1.1 github.com/git-pkgs/enrichment v0.6.4 + github.com/git-pkgs/magic v0.1.0 github.com/git-pkgs/purl v0.1.15 github.com/git-pkgs/registries v0.6.4 github.com/git-pkgs/spdx v0.1.4 diff --git a/go.sum b/go.sum index 4f356f9..c239240 100644 --- a/go.sum +++ b/go.sum @@ -244,12 +244,14 @@ github.com/fzipp/gocyclo v0.6.0 h1:lsblElZG7d3ALtGMx9fmxeTKZaLLpU8mET09yN4BBLo= github.com/fzipp/gocyclo v0.6.0/go.mod h1:rXPyn8fnlpa0R2csP/31uerbiVBugk5whMdlyaLkLoA= github.com/ghostiam/protogetter v0.3.20 h1:oW7OPFit2FxZOpmMRPP9FffU4uUpfeE/rEdE1f+MzD0= github.com/ghostiam/protogetter v0.3.20/go.mod h1:FjIu5Yfs6FT391m+Fjp3fbAYJ6rkL/J6ySpZBfnODuI= -github.com/git-pkgs/archives v0.3.1 h1:GKUuw++0YXAAElxweVHiR4AaSShKKYoVQmyxlF5blG4= -github.com/git-pkgs/archives v0.3.1/go.mod h1:408oQv3FxLCtePa33zp3sg3njXnwH74vnHZFxkRqoPo= +github.com/git-pkgs/archives v0.4.0 h1:KNmmIsLiSH27lUdT27EfUkQXFaLgXV5KezE81iyOIgo= +github.com/git-pkgs/archives v0.4.0/go.mod h1:tfio0OIuPKEBKHs/UCL5XBUvYmKpnvtnba2iDlfSd6g= github.com/git-pkgs/cooldown v0.1.1 h1:9OqqzCB8gANz/y44SmqGD0Jp8Qtu81D1sCbKl6Ehg7w= github.com/git-pkgs/cooldown v0.1.1/go.mod h1:v7APuK/UouTiu8mWQZbdDmj7DfxxkGUeuhjaRB5gv9E= github.com/git-pkgs/enrichment v0.6.4 h1:mGrfenttwmcUfPXRkWpB0wBJiiGj55ltniUh66Pq4bU= github.com/git-pkgs/enrichment v0.6.4/go.mod h1:zz1vPUak/w8Jhajll0KDRN2MjKaEYeCzQTxumWnVhqY= +github.com/git-pkgs/magic v0.1.0 h1:xLrqq7CMXB9g5bJnmJyKw17Rvlh0GFiEmO6e5RFsoeY= +github.com/git-pkgs/magic v0.1.0/go.mod h1:3ndidt+yvFaI1M0aEkkzkOlFnLPkeVQASIUojazcxCI= github.com/git-pkgs/packageurl-go v0.3.1 h1:WM3RBABQZLaRBxgKyYughc3cVBE8KyQxbSC6Jt5ak7M= github.com/git-pkgs/packageurl-go v0.3.1/go.mod h1:rcIxiG37BlQLB6FZfgdj9Fm7yjhRQd3l+5o7J0QPAk4= github.com/git-pkgs/pom v0.1.5 h1:TGT8Az2OMxGWsXnSagtUMGzZm7Oax8HrSCteA+mi0qY= diff --git a/internal/server/browse.go b/internal/server/browse.go index 504f5f1..3ea5676 100644 --- a/internal/server/browse.go +++ b/internal/server/browse.go @@ -1,6 +1,7 @@ package server import ( + "bufio" "encoding/json" "fmt" "io" @@ -10,29 +11,22 @@ import ( "github.com/git-pkgs/archives" "github.com/git-pkgs/archives/diff" + "github.com/git-pkgs/magic" "github.com/git-pkgs/proxy/internal/database" "github.com/git-pkgs/purl" "github.com/go-chi/chi/v5" ) -const contentTypePlainText = "text/plain; charset=utf-8" +const ( + contentTypePlainText = "text/plain; charset=utf-8" + browseSniffSize = 512 +) // maxBrowseArchiveSize caps how much data openArchive will buffer for // prefix detection. Artifacts larger than this are rejected to prevent // memory exhaustion from a single request. const maxBrowseArchiveSize = 512 << 20 // 512 MB -// archiveFilename returns a filename suitable for archive format detection. -// Some ecosystems (e.g. composer) store artifacts with bare hash filenames -// that have no extension. This adds .zip when the original has no extension -// and the content is likely a zip archive. -func archiveFilename(filename string) string { - if path.Ext(filename) == "" { - return filename + ".zip" - } - return filename -} - // detectSingleRootDir returns the single top-level directory name if all files // in the archive live under one common directory (e.g. GitHub zipballs use // "repo-hash/"). Returns "" if there's no single root or the archive is flat. @@ -66,8 +60,6 @@ func detectSingleRootDir(reader archives.Reader) string { // and stripping a single top-level directory prefix (like GitHub zipballs). // For npm, the hardcoded "package/" prefix takes precedence. func openArchive(filename string, content io.Reader, ecosystem string) (archives.Reader, error) { //nolint:ireturn // wraps multiple archive implementations - fname := archiveFilename(filename) - limited := io.LimitReader(content, maxBrowseArchiveSize+1) data, err := io.ReadAll(limited) if err != nil { @@ -78,17 +70,17 @@ func openArchive(filename string, content io.Reader, ecosystem string) (archives } if ecosystem == "npm" { - return archives.OpenBytesWithPrefix(fname, data, "package/") + return archives.OpenBytesWithPrefix(filename, data, "package/") } - probe, err := archives.OpenBytes(fname, data) + probe, err := archives.OpenBytes(filename, data) if err != nil { return nil, err } prefix := detectSingleRootDir(probe) _ = probe.Close() - return archives.OpenBytesWithPrefix(fname, data, prefix) + return archives.OpenBytesWithPrefix(filename, data, prefix) } // BrowseListResponse contains the file listing for a directory in an archives. @@ -361,7 +353,14 @@ func (s *Server) browseFile(w http.ResponseWriter, r *http.Request, ecosystem, n } defer func() { _ = fileReader.Close() }() - contentType := detectContentType(filePath) + contentType, knownPath := detectContentTypeFromPath(filePath) + var content io.Reader = fileReader + if !knownPath { + bufferedFile := bufio.NewReaderSize(fileReader, browseSniffSize) + prefix, _ := bufferedFile.Peek(browseSniffSize) + contentType = detectContentType(filePath, prefix) + content = bufferedFile + } w.Header().Set("Content-Type", contentType) w.Header().Set("Content-Security-Policy", "sandbox") w.Header().Set("X-Content-Type-Options", "nosniff") @@ -370,85 +369,112 @@ func (s *Server) browseFile(w http.ResponseWriter, r *http.Request, ecosystem, n w.Header().Set("Content-Disposition", fmt.Sprintf("inline; filename=%q", filename)) // Stream the file - _, _ = io.Copy(w, fileReader) + _, _ = io.Copy(w, content) } -// detectContentType returns an appropriate content type based on file extension. -func detectContentType(filename string) string { +// detectContentType returns an appropriate content type. Known filenames and +// extensions take precedence; content detection handles the remaining files. +func detectContentType(filename string, prefix []byte) string { + if contentType, ok := detectContentTypeFromPath(filename); ok { + return contentType + } + return detectContentTypeFromPrefix(prefix) +} + +func detectContentTypeFromPath(filename string) (string, bool) { ext := strings.ToLower(path.Ext(filename)) switch ext { // Text formats case ".txt", ".md", ".markdown": - return contentTypePlainText + return contentTypePlainText, true case ".html", ".htm", ".xhtml": - return contentTypePlainText + return contentTypePlainText, true case ".css": - return "text/css; charset=utf-8" + return "text/css; charset=utf-8", true case ".js", ".mjs": - return "application/javascript; charset=utf-8" + return "application/javascript; charset=utf-8", true case ".json": - return "application/json; charset=utf-8" + return "application/json; charset=utf-8", true case ".xml": - return "application/xml; charset=utf-8" + return "application/xml; charset=utf-8", true case ".yaml", ".yml": - return "text/yaml; charset=utf-8" + return "text/yaml; charset=utf-8", true case ".toml": - return "text/toml; charset=utf-8" + return "text/toml; charset=utf-8", true // Programming languages case ".go": - return "text/x-go; charset=utf-8" + return "text/x-go; charset=utf-8", true case ".rs": - return "text/x-rust; charset=utf-8" + return "text/x-rust; charset=utf-8", true case ".py": - return "text/x-python; charset=utf-8" + return "text/x-python; charset=utf-8", true case ".rb": - return "text/x-ruby; charset=utf-8" + return "text/x-ruby; charset=utf-8", true case ".java": - return "text/x-java; charset=utf-8" + return "text/x-java; charset=utf-8", true case ".c", ".h": - return "text/x-c; charset=utf-8" + return "text/x-c; charset=utf-8", true case ".cpp", ".cc", ".cxx", ".hpp": - return "text/x-c++; charset=utf-8" + return "text/x-c++; charset=utf-8", true case ".ts": - return "text/typescript; charset=utf-8" + return "text/typescript; charset=utf-8", true case ".tsx": - return "text/tsx; charset=utf-8" + return "text/tsx; charset=utf-8", true case ".jsx": - return "text/jsx; charset=utf-8" + return "text/jsx; charset=utf-8", true case ".php": - return "text/x-php; charset=utf-8" + return "text/x-php; charset=utf-8", true // Config files case ".conf", ".config", ".ini": - return contentTypePlainText + return contentTypePlainText, true case ".sh", ".bash": - return "text/x-shellscript; charset=utf-8" + return "text/x-shellscript; charset=utf-8", true case ".dockerfile": - return "text/x-dockerfile; charset=utf-8" + return "text/x-dockerfile; charset=utf-8", true // Images case ".png": - return "image/png" + return "image/png", true case ".jpg", ".jpeg": - return "image/jpeg" + return "image/jpeg", true case ".gif": - return "image/gif" + return "image/gif", true case ".svg": - return contentTypePlainText + return contentTypePlainText, true case ".ico": - return "image/x-icon" + return "image/x-icon", true // Archives case ".zip", ".tar", ".gz", ".bz2", ".xz": - return "application/octet-stream" + return "application/octet-stream", true default: - // Try to detect if it looks like text if isLikelyText(filename) { - return contentTypePlainText + return contentTypePlainText, true } + return "", false + } +} + +func detectContentTypeFromPrefix(prefix []byte) string { + result := magic.DetectPrefix(prefix) + if result.Kind == magic.KindText { + return contentTypePlainText + } + + switch result.Format { + case "png": + return "image/png" + case "jpeg": + return "image/jpeg" + case "gif": + return "image/gif" + case "pdf": + return "application/pdf" + default: return "application/octet-stream" } } diff --git a/internal/server/browse_bench_test.go b/internal/server/browse_bench_test.go index 03f3f02..16b2419 100644 --- a/internal/server/browse_bench_test.go +++ b/internal/server/browse_bench_test.go @@ -55,3 +55,28 @@ func BenchmarkOpenArchive(b *testing.B) { }) } } + +func BenchmarkDetectContentType(b *testing.B) { + cases := []struct { + name string + filename string + prefix []byte + }{ + {"known-path", "README.md", nil}, + {"text-prefix", "artifact", bytes.Repeat([]byte("a"), browseSniffSize)}, + {"png-prefix", "artifact", append([]byte("\x89PNG\r\n\x1a\n"), make([]byte, browseSniffSize-8)...)}, + } + + for _, tc := range cases { + b.Run(tc.name, func(b *testing.B) { + b.ReportAllocs() + var contentType string + for b.Loop() { + contentType = detectContentType(tc.filename, tc.prefix) + } + if contentType == "" { + b.Fatal("empty content type") + } + }) + } +} diff --git a/internal/server/browse_test.go b/internal/server/browse_test.go index f1fb993..6b1e487 100644 --- a/internal/server/browse_test.go +++ b/internal/server/browse_test.go @@ -137,29 +137,44 @@ func TestHandleBrowseFile(t *testing.T) { t.Fatalf("failed to upsert artifact: %v", err) } - // Test fetching a file - req := httptest.NewRequest("GET", "/ui/api/browse/npm/test-browse/1.0.0/file/README.md", nil) - w := httptest.NewRecorder() - ts.handler.ServeHTTP(w, req) - - if w.Code != http.StatusOK { - t.Fatalf("expected status 200, got %d: %s", w.Code, w.Body.String()) + files := []struct { + path string + content string + contentType string + }{ + {"README.md", "# Test Package\n", contentTypePlainText}, + {"notes.data", "short text\n", contentTypePlainText}, + {"logo", "\x89PNG\r\n\x1a\nimage data", "image/png"}, + {"page", "", contentTypePlainText}, + {"misleading.txt", "\x89PNG\r\n\x1a\nimage data", contentTypePlainText}, } + for _, file := range files { + t.Run(file.path, func(t *testing.T) { + req := httptest.NewRequest("GET", "/ui/api/browse/npm/test-browse/1.0.0/file/"+file.path, nil) + w := httptest.NewRecorder() + ts.handler.ServeHTTP(w, req) - body := w.Body.String() - if body != "# Test Package\n" { - t.Errorf("unexpected file content: %q", body) - } - - // Check content type - contentType := w.Header().Get("Content-Type") - if contentType != contentTypePlainText { - t.Errorf("expected text/plain content type, got %q", contentType) + if w.Code != http.StatusOK { + t.Fatalf("expected status 200, got %d: %s", w.Code, w.Body.String()) + } + if w.Body.String() != file.content { + t.Errorf("unexpected file content: %q", w.Body.String()) + } + if got := w.Header().Get("Content-Type"); got != file.contentType { + t.Errorf("Content-Type = %q, want %q", got, file.contentType) + } + if got := w.Header().Get("Content-Security-Policy"); got != "sandbox" { + t.Errorf("Content-Security-Policy = %q, want sandbox", got) + } + if got := w.Header().Get("X-Content-Type-Options"); got != "nosniff" { + t.Errorf("X-Content-Type-Options = %q, want nosniff", got) + } + }) } // Test fetching non-existent file - req = httptest.NewRequest("GET", "/ui/api/browse/npm/test-browse/1.0.0/file/nonexistent.txt", nil) - w = httptest.NewRecorder() + req := httptest.NewRequest("GET", "/ui/api/browse/npm/test-browse/1.0.0/file/nonexistent.txt", nil) + w := httptest.NewRecorder() ts.handler.ServeHTTP(w, req) if w.Code != http.StatusNotFound { @@ -169,34 +184,47 @@ func TestHandleBrowseFile(t *testing.T) { func TestDetectContentType(t *testing.T) { tests := []struct { + name string filename string + prefix []byte expectedCT string }{ - {"file.txt", contentTypePlainText}, - {"file.md", contentTypePlainText}, - {"file.json", "application/json; charset=utf-8"}, - {"file.js", "application/javascript; charset=utf-8"}, - {"file.go", "text/x-go; charset=utf-8"}, - {"file.py", "text/x-python; charset=utf-8"}, - {"file.rs", "text/x-rust; charset=utf-8"}, - {"file.html", contentTypePlainText}, - {"file.htm", contentTypePlainText}, - {"file.xhtml", contentTypePlainText}, - {"file.svg", contentTypePlainText}, - {"file.png", "image/png"}, - {"file.jpg", "image/jpeg"}, - {"README", contentTypePlainText}, - {"LICENSE", contentTypePlainText}, - {"Makefile", contentTypePlainText}, - {".gitignore", contentTypePlainText}, - {"file.bin", "application/octet-stream"}, + {"text extension", "file.txt", nil, contentTypePlainText}, + {"markdown extension", "file.md", nil, contentTypePlainText}, + {"JSON extension", "file.json", nil, "application/json; charset=utf-8"}, + {"JavaScript extension", "file.js", nil, "application/javascript; charset=utf-8"}, + {"Go extension", "file.go", nil, "text/x-go; charset=utf-8"}, + {"Python extension", "file.py", nil, "text/x-python; charset=utf-8"}, + {"Rust extension", "file.rs", nil, "text/x-rust; charset=utf-8"}, + {"HTML extension", "file.html", nil, contentTypePlainText}, + {"HTM extension", "file.htm", nil, contentTypePlainText}, + {"XHTML extension", "file.xhtml", nil, contentTypePlainText}, + {"SVG extension", "file.svg", nil, contentTypePlainText}, + {"PNG extension", "file.png", nil, "image/png"}, + {"JPEG extension", "file.jpg", nil, "image/jpeg"}, + {"README", "README", nil, contentTypePlainText}, + {"LICENSE", "LICENSE", nil, contentTypePlainText}, + {"Makefile", "Makefile", nil, contentTypePlainText}, + {"gitignore", ".gitignore", nil, contentTypePlainText}, + {"unknown empty", "file.bin", nil, "application/octet-stream"}, + {"extensionless PNG", "asset", []byte("\x89PNG\r\n\x1a\n"), "image/png"}, + {"extensionless JPEG", "asset", []byte("\xff\xd8\xff"), "image/jpeg"}, + {"extensionless GIF", "asset", []byte("GIF89a"), "image/gif"}, + {"extensionless PDF", "asset", []byte("%PDF-1.7"), "application/pdf"}, + {"extensionless text", "asset", []byte("plain text\n"), contentTypePlainText}, + {"extensionless HTML", "asset", []byte(""), contentTypePlainText}, + {"extensionless XML", "asset", []byte(""), contentTypePlainText}, + {"extensionless SVG", "asset", []byte(""), contentTypePlainText}, + {"extensionless ZIP", "asset", []byte("PK\x03\x04"), "application/octet-stream"}, + {"extensionless binary", "asset", []byte{0, 1, 2}, "application/octet-stream"}, + {"known path wins", "file.txt", []byte("\x89PNG\r\n\x1a\n"), contentTypePlainText}, } for _, tt := range tests { - t.Run(tt.filename, func(t *testing.T) { - got := detectContentType(tt.filename) + t.Run(tt.name, func(t *testing.T) { + got := detectContentType(tt.filename, tt.prefix) if got != tt.expectedCT { - t.Errorf("detectContentType(%q) = %q, want %q", tt.filename, got, tt.expectedCT) + t.Errorf("detectContentType(%q, %q) = %q, want %q", tt.filename, tt.prefix, got, tt.expectedCT) } }) } @@ -255,6 +283,10 @@ func createTestArchive(t *testing.T) []byte { "package/lib/index.js": "module.exports = {};", "package/lib/helper.js": "module.exports.help = () => {};", "package/test/index.test.js": "// tests", + "package/notes.data": "short text\n", + "package/logo": "\x89PNG\r\n\x1a\nimage data", + "package/page": "", + "package/misleading.txt": "\x89PNG\r\n\x1a\nimage data", } for path, content := range files { @@ -609,25 +641,19 @@ func TestHandleComparePage(t *testing.T) { } } -func TestArchiveFilename(t *testing.T) { - tests := []struct { - input string - want string - }{ - {"package.tar.gz", "package.tar.gz"}, - {"d2e2f014ccd6ec9fae8dbe6336a4164346a2a856", "d2e2f014ccd6ec9fae8dbe6336a4164346a2a856.zip"}, - {"file.zip", "file.zip"}, - {"archive.tgz", "archive.tgz"}, - {"noext", "noext.zip"}, +func TestOpenArchiveDetectsExtensionlessTarGz(t *testing.T) { + reader, err := openArchive("artifact", bytes.NewReader(createTestArchive(t)), "npm") + if err != nil { + t.Fatalf("openArchive failed: %v", err) } + defer func() { _ = reader.Close() }() - for _, tt := range tests { - t.Run(tt.input, func(t *testing.T) { - got := archiveFilename(tt.input) - if got != tt.want { - t.Errorf("archiveFilename(%q) = %q, want %q", tt.input, got, tt.want) - } - }) + files, err := reader.List() + if err != nil { + t.Fatalf("List failed: %v", err) + } + if len(files) == 0 { + t.Fatal("expected files in extensionless archive") } } From 7dbf13e3450396a7fe0973c50ea59911b4c8f7c6 Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Sat, 1 Aug 2026 11:22:13 +0100 Subject: [PATCH 032/633] Avoid repeated content type path lookup --- internal/server/browse.go | 11 +---------- internal/server/browse_bench_test.go | 23 +++++++++++++++-------- internal/server/browse_test.go | 9 ++++++--- 3 files changed, 22 insertions(+), 21 deletions(-) diff --git a/internal/server/browse.go b/internal/server/browse.go index 3ea5676..56aa1c5 100644 --- a/internal/server/browse.go +++ b/internal/server/browse.go @@ -358,7 +358,7 @@ func (s *Server) browseFile(w http.ResponseWriter, r *http.Request, ecosystem, n if !knownPath { bufferedFile := bufio.NewReaderSize(fileReader, browseSniffSize) prefix, _ := bufferedFile.Peek(browseSniffSize) - contentType = detectContentType(filePath, prefix) + contentType = detectContentTypeFromPrefix(prefix) content = bufferedFile } w.Header().Set("Content-Type", contentType) @@ -372,15 +372,6 @@ func (s *Server) browseFile(w http.ResponseWriter, r *http.Request, ecosystem, n _, _ = io.Copy(w, content) } -// detectContentType returns an appropriate content type. Known filenames and -// extensions take precedence; content detection handles the remaining files. -func detectContentType(filename string, prefix []byte) string { - if contentType, ok := detectContentTypeFromPath(filename); ok { - return contentType - } - return detectContentTypeFromPrefix(prefix) -} - func detectContentTypeFromPath(filename string) (string, bool) { ext := strings.ToLower(path.Ext(filename)) diff --git a/internal/server/browse_bench_test.go b/internal/server/browse_bench_test.go index 16b2419..840bc75 100644 --- a/internal/server/browse_bench_test.go +++ b/internal/server/browse_bench_test.go @@ -58,21 +58,28 @@ func BenchmarkOpenArchive(b *testing.B) { func BenchmarkDetectContentType(b *testing.B) { cases := []struct { - name string - filename string - prefix []byte + name string + filename string + prefix []byte + knownPath bool }{ - {"known-path", "README.md", nil}, - {"text-prefix", "artifact", bytes.Repeat([]byte("a"), browseSniffSize)}, - {"png-prefix", "artifact", append([]byte("\x89PNG\r\n\x1a\n"), make([]byte, browseSniffSize-8)...)}, + {"known-path", "README.md", nil, true}, + {"text-prefix", "artifact", bytes.Repeat([]byte("a"), browseSniffSize), false}, + {"png-prefix", "artifact", append([]byte("\x89PNG\r\n\x1a\n"), make([]byte, browseSniffSize-8)...), false}, } for _, tc := range cases { b.Run(tc.name, func(b *testing.B) { b.ReportAllocs() var contentType string - for b.Loop() { - contentType = detectContentType(tc.filename, tc.prefix) + if tc.knownPath { + for b.Loop() { + contentType, _ = detectContentTypeFromPath(tc.filename) + } + } else { + for b.Loop() { + contentType = detectContentTypeFromPrefix(tc.prefix) + } } if contentType == "" { b.Fatal("empty content type") diff --git a/internal/server/browse_test.go b/internal/server/browse_test.go index 6b1e487..5240a92 100644 --- a/internal/server/browse_test.go +++ b/internal/server/browse_test.go @@ -182,7 +182,7 @@ func TestHandleBrowseFile(t *testing.T) { } } -func TestDetectContentType(t *testing.T) { +func TestBrowseContentTypePolicy(t *testing.T) { tests := []struct { name string filename string @@ -222,9 +222,12 @@ func TestDetectContentType(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got := detectContentType(tt.filename, tt.prefix) + got, knownPath := detectContentTypeFromPath(tt.filename) + if !knownPath { + got = detectContentTypeFromPrefix(tt.prefix) + } if got != tt.expectedCT { - t.Errorf("detectContentType(%q, %q) = %q, want %q", tt.filename, tt.prefix, got, tt.expectedCT) + t.Errorf("content type for %q with prefix %q = %q, want %q", tt.filename, tt.prefix, got, tt.expectedCT) } }) } From 63f0efd0e99a9e5c9dd3b74becf5d418a42b3a54 Mon Sep 17 00:00:00 2001 From: wickedOne Date: Mon, 3 Aug 2026 12:14:03 +0200 Subject: [PATCH 033/633] fix(pypi): resolve name and version for PEP 658 metadata sidecars (#222) * resolve name and version for PEP 658 metadata sidecars * fix(pypi): parse Windows installer and egg filenames separately The bdist_wininst and bdist_msi layout joins the platform to the version with a '.' rather than a '-', so treating .exe/.msi like a wheel folded the platform into the version: foo-1.0.win32-py2.0.exe resolved to version "1.0.win32". Eggs shared the problem, as setuptools' hyphen escaping is not universal: aws-sdk-1.0.0-py3.11.egg resolved to name "aws", version "sdk". Give each format its own parser. Wheels keep the PEP 427 spec-guaranteed field positions, eggs locate the version relative to the py{X.Y} interpreter field, and Windows installers strip the platform and interpreter fields before splitting name from version. A PEP 658 sidecar resolves to the same name and version as the distribution it describes, so it is cached under that version. Browse and compare took the first cached artifact without checking its extension, handing openArchive plain text: a version pip had only fetched metadata for reported hasCached and then 500'd. Add firstBrowsableArtifact, replacing five duplicated selection loops, and export PyPIMetadataSuffix so the suffix has a single definition. Co-Authored-By: Claude Opus 5 (1M context) --------- Co-authored-by: Claude Opus 5 (1M context) --- internal/handler/pypi.go | 182 +++++++++++++++++++++++++++------ internal/handler/pypi_test.go | 78 +++++++++++--- internal/server/browse.go | 59 +++++------ internal/server/browse_test.go | 67 ++++++++++++ internal/server/server.go | 10 +- 5 files changed, 309 insertions(+), 87 deletions(-) diff --git a/internal/handler/pypi.go b/internal/handler/pypi.go index f5a0232..713f6c8 100644 --- a/internal/handler/pypi.go +++ b/internal/handler/pypi.go @@ -19,7 +19,14 @@ const ( minWheelParts = 5 // name + version + python + abi + platform minSubmatchParts = 2 // full match + first capture group minPyPIPathParts = 3 // hash_prefix + hash + filename - minPythonTagLen = 2 // minimum length for a python tag (e.g., "py") + minEggParts = 3 // name + version + python tag + + // PyPIMetadataSuffix is the PEP 658 core-metadata sidecar suffix that pip + // appends to a distribution URL when the index advertises core metadata. + // A sidecar resolves to the same name and version as the distribution it + // describes, so it is cached alongside it; consumers that expect an openable + // archive must skip these. + PyPIMetadataSuffix = ".metadata" ) // PyPIHandler handles PyPI registry protocol requests. @@ -433,56 +440,163 @@ func (h *PyPIHandler) handleDownload(w http.ResponseWriter, r *http.Request) { ServeArtifact(w, result) } +// archiveExtensions are sdist formats of the form {name}-{version}{ext}. They +// carry no trailing tags, but legacy sdist names may contain hyphens. +var archiveExtensions = []string{".tar.gz", ".tar.bz2", ".tar.xz", ".tar.Z", ".tgz", ".tar", ".zip"} + +// windowsInstallerExtensions are the legacy distutils bdist_wininst and +// bdist_msi formats, which share a filename layout. +var windowsInstallerExtensions = []string{".exe", ".msi"} + // parseFilename extracts package name and version from a PyPI filename. -// Handles both wheels and sdists: +// Handles wheels, sdists and legacy bdist formats: // - requests-2.31.0-py3-none-any.whl // - requests-2.31.0.tar.gz +// - numpy-1.8.0-py2.7-macosx-10.9-x86_64.egg +// - numpy-1.8.0.win32-py2.7.exe func (h *PyPIHandler) parseFilename(filename string) (name, version string) { - // Try wheel format first: {name}-{version}(-{build})?-{python}-{abi}-{platform}.whl - if strings.HasSuffix(filename, ".whl") { - base := strings.TrimSuffix(filename, ".whl") - parts := strings.Split(base, "-") - if len(parts) >= minWheelParts { - // Find where version ends (version followed by python tag) - for i := 1; i < len(parts)-2; i++ { - // Check if this looks like a python tag (py2, py3, cp39, etc) - if isPythonTag(parts[i]) { - name = strings.Join(parts[:i-1], "-") - version = parts[i-1] - return - } - } + // PEP 658/714 core-metadata sidecars are the distribution filename plus + // ".metadata"; they describe the same name and version. Without this, pip's + // metadata-only fetches fall back to a hash-derived package identifier. + filename = strings.TrimSuffix(filename, PyPIMetadataSuffix) + + switch { + case strings.HasSuffix(filename, ".whl"): + return parseWheelFilename(strings.TrimSuffix(filename, ".whl")) + case strings.HasSuffix(filename, ".egg"): + return parseEggFilename(strings.TrimSuffix(filename, ".egg")) + } + + for _, ext := range windowsInstallerExtensions { + if strings.HasSuffix(filename, ext) { + return parseWindowsInstallerFilename(strings.TrimSuffix(filename, ext)) } } - // Try sdist formats: {name}-{version}.tar.gz, {name}-{version}.zip - for _, ext := range []string{".tar.gz", ".tar.bz2", ".zip", ".tar"} { + for _, ext := range archiveExtensions { if strings.HasSuffix(filename, ext) { - base := strings.TrimSuffix(filename, ext) - // Find last hyphen followed by version - for i := len(base) - 1; i >= 0; i-- { - if base[i] == '-' && i+1 < len(base) && isVersionStart(base[i+1]) { - return base[:i], base[i+1:] - } - } + return splitNameVersion(strings.TrimSuffix(filename, ext)) } } return "", "" } -func isPythonTag(s string) bool { - if len(s) < minPythonTagLen { - return false +// parseWheelFilename parses the PEP 427 layout +// {name}-{version}(-{build})?-{python}-{abi}-{platform}, base being the +// filename without its ".whl" suffix. The spec escapes every hyphen in the name +// and version to '_', so the first two fields are authoritative even when the +// optional build tag is present. +func parseWheelFilename(base string) (name, version string) { + parts := strings.Split(base, "-") + if len(parts) < minWheelParts { + return "", "" } - // Python tags start with py, cp, pp, ip, jy - prefixes := []string{"py", "cp", "pp", "ip", "jy"} - for _, p := range prefixes { - if strings.HasPrefix(s, p) { - return true + + return parts[0], parts[1] +} + +// parseEggFilename parses the setuptools bdist_egg layout +// {name}-{version}-py{X.Y}(-{platform})?, base being the filename without its +// ".egg" suffix. setuptools escapes hyphens in the name and version to '_', but +// eggs built by other tooling do not always, so the version is located relative +// to the interpreter field rather than assumed to be the second field. +func parseEggFilename(base string) (name, version string) { + parts := strings.Split(base, "-") + // Scan from the end: the trailing platform fields never look like an + // interpreter tag, so the last match is the real one even when the package + // name itself carries a "py{N}" component. Stop before index 1, since a tag + // any earlier would leave no room for both a name and a version. + for i := len(parts) - 1; i >= minEggParts-1; i-- { + if !isEggPythonTag(parts[i]) || !isVersionField(parts[i-1]) { + continue + } + + return strings.Join(parts[:i-1], "-"), parts[i-1] + } + + // No interpreter field: {name}-{version}. + return splitNameVersion(base) +} + +// parseWindowsInstallerFilename parses the distutils bdist_wininst and +// bdist_msi layout {name}-{version}.{platform}(-py{X.Y})?, base being the +// filename without its ".exe" or ".msi" suffix. The platform is joined to the +// version with a '.' rather than a '-' and may itself contain a hyphen +// ("win-amd64"), so both trailing fields are stripped before the name and +// version are split apart. +func parseWindowsInstallerFilename(base string) (name, version string) { + if i := strings.LastIndex(base, "-py"); i >= 0 && isDottedNumber(base[i+len("-py"):]) { + base = base[:i] + } + + // The platform is the final '.'-separated field. Requiring it to start with + // a non-digit keeps a dotted version from being truncated when a filename + // carries no platform tag. + i := strings.LastIndex(base, ".") + if i < 0 || i+1 >= len(base) || isVersionStart(base[i+1]) { + return "", "" + } + + return splitFullname(base[:i]) +} + +// splitFullname splits the distutils fullname {name}-{version} that precedes a +// Windows installer's platform field. Unlike an sdist, a wininst fullname may +// carry a trailing build variant ("cx_Oracle-5.1.2-11g"), which belongs to +// neither the name nor the version, so the first purely numeric field wins and +// anything after it is discarded. +func splitFullname(fullname string) (name, version string) { + parts := strings.Split(fullname, "-") + for i := 1; i < len(parts); i++ { + if isDottedNumber(parts[i]) { + return strings.Join(parts[:i], "-"), parts[i] } } - return false + + // No purely numeric field, e.g. a prerelease version like "1.0b1". + return splitNameVersion(fullname) +} + +// splitNameVersion splits a {name}-{version} pair at the last hyphen that +// starts a version, leaving hyphens inside the name intact. +func splitNameVersion(base string) (name, version string) { + for i := len(base) - 1; i >= 0; i-- { + if base[i] == '-' && i+1 < len(base) && isVersionStart(base[i+1]) { + return base[:i], base[i+1:] + } + } + + return "", "" +} + +// isEggPythonTag reports whether field is the py{X.Y} interpreter field that +// setuptools places directly after the version in an egg filename. +func isEggPythonTag(field string) bool { + const prefix = "py" + + return len(field) > len(prefix) && strings.HasPrefix(field, prefix) && isVersionStart(field[len(prefix)]) +} + +// isVersionField reports whether field can be a version, i.e. it is non-empty +// and starts with a digit as every PEP 440 release segment does. +func isVersionField(field string) bool { + return field != "" && isVersionStart(field[0]) +} + +// isDottedNumber reports whether s is a dotted numeric version such as "2.7". +func isDottedNumber(s string) bool { + if s == "" || !isVersionStart(s[0]) { + return false + } + + for i := range len(s) { + if !isVersionStart(s[i]) && s[i] != '.' { + return false + } + } + + return true } func isVersionStart(c byte) bool { diff --git a/internal/handler/pypi_test.go b/internal/handler/pypi_test.go index 2b58960..5ae76ca 100644 --- a/internal/handler/pypi_test.go +++ b/internal/handler/pypi_test.go @@ -28,13 +28,58 @@ func TestPyPIParseFilename(t *testing.T) { {"aws-sdk-1.0.0.tar.gz", "aws-sdk", "1.0.0"}, {"zipp-3.17.0.zip", "zipp", "3.17.0"}, + // Additional sdist archive formats + {"lxml-4.9.3.tar.xz", "lxml", "4.9.3"}, + {"docutils-0.20.1.tgz", "docutils", "0.20.1"}, + {"psycopg2-2.9.9.tar.bz2", "psycopg2", "2.9.9"}, + // Wheel formats {"requests-2.31.0-py3-none-any.whl", "requests", "2.31.0"}, {"numpy-1.26.2-cp311-cp311-manylinux_2_17_x86_64.whl", "numpy", "1.26.2"}, {"cryptography-41.0.5-cp37-abi3-manylinux_2_28_x86_64.whl", "cryptography", "41.0.5"}, + // Wheels with a build tag must not fold the tag into the version + {"foo-1.0-1-py3-none-any.whl", "foo", "1.0"}, + {"tensorflow-2.15.0-2-cp311-cp311-manylinux_2_17_x86_64.whl", "tensorflow", "2.15.0"}, + + // PEP 658 core-metadata sidecars resolve to the distribution they describe + {"backports_asyncio_runner-1.2.0-py3-none-any.whl.metadata", "backports_asyncio_runner", "1.2.0"}, + {"requests-2.31.0-py3-none-any.whl.metadata", "requests", "2.31.0"}, + {"requests-2.31.0.tar.gz.metadata", "requests", "2.31.0"}, + + // Eggs: {name}-{version}-py{X.Y}(-{platform})?.egg. Unescaped hyphens in + // the name must not be mistaken for the field separator before the version. + {"numpy-1.8.0-py2.7-macosx-10.9-x86_64.egg", "numpy", "1.8.0"}, + {"aws-sdk-1.0.0-py3.11.egg", "aws-sdk", "1.0.0"}, + {"aws-sdk-1.0.0-py2.7-macosx-10.9-x86_64.egg", "aws-sdk", "1.0.0"}, + {"aws-sdk-1.0.0.egg", "aws-sdk", "1.0.0"}, + // A "py{N}" component inside the name is not the interpreter field, so + // the interpreter must be located from the end of the filename. + {"django-rest-py3-1.0-py3.6.egg", "django-rest-py3", "1.0"}, + + // Windows installers: {name}-{version}.{platform}(-py{X.Y})?.{exe,msi}. + // The platform is not part of the version, and may contain a hyphen. + {"foo-1.0.win32-py2.0.exe", "foo", "1.0"}, + {"pywin32-223.win32-py2.7.exe", "pywin32", "223"}, + {"numpy-1.8.0.win-amd64-py2.7.exe", "numpy", "1.8.0"}, + {"aws-sdk-1.0.0.win32-py2.7.exe", "aws-sdk", "1.0.0"}, + {"pywin32-223.win32.exe", "pywin32", "223"}, + {"cx_Oracle-5.1.2.win32-py2.7.msi", "cx_Oracle", "5.1.2"}, + {"numpy-1.8.0.win-amd64.msi", "numpy", "1.8.0"}, + // A trailing build variant belongs to neither the name nor the version. + {"cx_Oracle-5.1.2-11g.win32-py2.7.exe", "cx_Oracle", "5.1.2"}, + // A prerelease version has no purely numeric field to anchor on. + {"foo-1.0b1.win32-py2.7.exe", "foo", "1.0b1"}, + // Invalid {"invalid", "", ""}, + {"invalid.metadata", "", ""}, + {"backports.ssl_match_hostname-3.4.0.2-py2.7.whl", "", ""}, + {"invalid.exe", "", ""}, + {"foo-1.0.exe", "", ""}, + // An egg with an interpreter field but no version must not promote the + // trailing component of a hyphenated name to the version. + {"aws-sdk-py2.7.egg", "", ""}, } for _, tt := range tests { @@ -94,25 +139,24 @@ func TestPyPIRewriteJSONMetadataCooldown(t *testing.T) { } } -func TestIsPythonTag(t *testing.T) { - tests := []struct { - tag string - want bool - }{ - {"py3", true}, - {"py2", true}, - {"cp311", true}, - {"cp37", true}, - {"pp39", true}, - {"none", false}, - {"any", false}, - {"manylinux", false}, +// TestPyPIParseFilenameNoHashFallback guards the identifier used for caching: +// a filename that parses to an empty name makes handleDownload fall back to a +// "_hash_" package name, which surfaces as a bogus PURL in the package +// overview. +func TestPyPIParseFilenameNoHashFallback(t *testing.T) { + h := &PyPIHandler{proxy: &Proxy{Logger: slog.Default()}} + + filenames := []string{ + "backports_asyncio_runner-1.2.0-py3-none-any.whl", + "backports_asyncio_runner-1.2.0-py3-none-any.whl.metadata", + "backports_asyncio_runner-1.2.0.tar.gz", } - for _, tt := range tests { - got := isPythonTag(tt.tag) - if got != tt.want { - t.Errorf("isPythonTag(%q) = %v, want %v", tt.tag, got, tt.want) + for _, filename := range filenames { + name, version := h.parseFilename(filename) + if name != "backports_asyncio_runner" || version != "1.2.0" { + t.Errorf("parseFilename(%q) = (%q, %q), want (%q, %q)", + filename, name, version, "backports_asyncio_runner", "1.2.0") } } } diff --git a/internal/server/browse.go b/internal/server/browse.go index 56aa1c5..c60cba3 100644 --- a/internal/server/browse.go +++ b/internal/server/browse.go @@ -13,6 +13,7 @@ import ( "github.com/git-pkgs/archives/diff" "github.com/git-pkgs/magic" "github.com/git-pkgs/proxy/internal/database" + "github.com/git-pkgs/proxy/internal/handler" "github.com/git-pkgs/purl" "github.com/go-chi/chi/v5" ) @@ -27,6 +28,31 @@ const ( // memory exhaustion from a single request. const maxBrowseArchiveSize = 512 << 20 // 512 MB +// firstBrowsableArtifact returns the first cached artifact that can be opened as +// an archive, or nil if the version has none. +// +// A version's artifact list is not all archives: a PEP 658 core-metadata sidecar +// resolves to the same name and version as the distribution it describes, so it +// is cached under that version too. Sidecars are plain text, and because '-' +// sorts before '.' one can even precede the real distribution in the +// filename-ordered list, so selecting blindly would hand openArchive a file it +// cannot parse. +func firstBrowsableArtifact(artifacts []database.Artifact) *database.Artifact { + for i := range artifacts { + if artifacts[i].StoragePath.Valid && !isMetadataSidecar(artifacts[i].Filename) { + return &artifacts[i] + } + } + + return nil +} + +// isMetadataSidecar reports whether filename is a core-metadata sidecar rather +// than a distribution archive. +func isMetadataSidecar(filename string) bool { + return strings.HasSuffix(filename, handler.PyPIMetadataSuffix) +} + // detectSingleRootDir returns the single top-level directory name if all files // in the archive live under one common directory (e.g. GitHub zipballs use // "repo-hash/"). Returns "" if there's no single root or the archive is flat. @@ -214,14 +240,7 @@ func (s *Server) browseList(w http.ResponseWriter, r *http.Request, ecosystem, n return } - // Find the first cached artifact - var cachedArtifact *database.Artifact - for i := range artifacts { - if artifacts[i].StoragePath.Valid { - cachedArtifact = &artifacts[i] - break - } - } + cachedArtifact := firstBrowsableArtifact(artifacts) if cachedArtifact == nil { notFound(w, "artifact not cached") @@ -308,14 +327,7 @@ func (s *Server) browseFile(w http.ResponseWriter, r *http.Request, ecosystem, n return } - // Find the first cached artifact - var cachedArtifact *database.Artifact - for i := range artifacts { - if artifacts[i].StoragePath.Valid { - cachedArtifact = &artifacts[i] - break - } - } + cachedArtifact := firstBrowsableArtifact(artifacts) if cachedArtifact == nil { notFound(w, "artifact not cached") @@ -535,19 +547,8 @@ func (s *Server) compareDiff(w http.ResponseWriter, r *http.Request, ecosystem, } // Find cached artifacts - var fromArtifact, toArtifact *database.Artifact - for i := range fromArtifacts { - if fromArtifacts[i].StoragePath.Valid { - fromArtifact = &fromArtifacts[i] - break - } - } - for i := range toArtifacts { - if toArtifacts[i].StoragePath.Valid { - toArtifact = &toArtifacts[i] - break - } - } + fromArtifact := firstBrowsableArtifact(fromArtifacts) + toArtifact := firstBrowsableArtifact(toArtifacts) if fromArtifact == nil || toArtifact == nil { notFound(w, "one or both versions not cached") diff --git a/internal/server/browse_test.go b/internal/server/browse_test.go index 5240a92..6ea0f7e 100644 --- a/internal/server/browse_test.go +++ b/internal/server/browse_test.go @@ -829,3 +829,70 @@ func createTarGzArchive(t *testing.T, files map[string]string) []byte { } return buf.Bytes() } + +// TestFirstBrowsableArtifact guards artifact selection against PEP 658 +// core-metadata sidecars. A sidecar resolves to the same version as the +// distribution it describes, so it is cached under that version, but it is plain +// text and openArchive cannot parse it. +func TestFirstBrowsableArtifact(t *testing.T) { + cached := func(filename string) database.Artifact { + return database.Artifact{ + Filename: filename, + StoragePath: sql.NullString{String: "pypi/" + filename, Valid: true}, + } + } + uncached := func(filename string) database.Artifact { + return database.Artifact{Filename: filename} + } + + tests := []struct { + name string + artifacts []database.Artifact + want string + }{ + {"no artifacts", nil, ""}, + { + "sidecar only is not browsable", + []database.Artifact{cached("foo-1.0-py3-none-any.whl.metadata")}, + "", + }, + { + // '-' (0x2D) sorts before '.' (0x2E), so the sidecar precedes the + // sdist in the filename-ordered list the query returns. + "sidecar sorting ahead of the sdist is skipped", + []database.Artifact{cached("foo-1.0-py3-none-any.whl.metadata"), cached("foo-1.0.tar.gz")}, + "foo-1.0.tar.gz", + }, + { + "sidecar skipped in favour of its own wheel", + []database.Artifact{cached("foo-1.0-py3-none-any.whl.metadata"), cached("foo-1.0-py3-none-any.whl")}, + "foo-1.0-py3-none-any.whl", + }, + { + "uncached archive is still not selected", + []database.Artifact{cached("foo-1.0.tar.gz.metadata"), uncached("foo-1.0.tar.gz")}, + "", + }, + {"plain sdist", []database.Artifact{cached("foo-1.0.tar.gz")}, "foo-1.0.tar.gz"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := firstBrowsableArtifact(tt.artifacts) + + if tt.want == "" { + if got != nil { + t.Fatalf("firstBrowsableArtifact() = %q, want nil", got.Filename) + } + return + } + + if got == nil { + t.Fatalf("firstBrowsableArtifact() = nil, want %q", tt.want) + } + if got.Filename != tt.want { + t.Errorf("firstBrowsableArtifact() = %q, want %q", got.Filename, tt.want) + } + }) + } +} diff --git a/internal/server/server.go b/internal/server/server.go index 13c5997..153a94c 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -791,13 +791,9 @@ func (s *Server) showVersion(w http.ResponseWriter, r *http.Request, ecosystem, isOutdated := pkg.LatestVersion.Valid && pkg.LatestVersion.String != version - hasCached := false - for _, art := range artifacts { - if art.StoragePath.Valid { - hasCached = true - break - } - } + // A version whose only cached artifact is a metadata sidecar cannot be + // browsed, so it must not be advertised as cached. + hasCached := firstBrowsableArtifact(artifacts) != nil data := VersionShowData{ Layout: s.layoutFor(r), From 800ffc6b382d7f6d341b01b0b0d9465ed65ae2d1 Mon Sep 17 00:00:00 2001 From: Vincent Palatin Date: Wed, 5 Aug 2026 18:48:00 +0200 Subject: [PATCH 034/633] Make Debian upstream repository configurable. (#229) * Refactor LoadFromEnv to use helpers Avoid triggering the linter about the cyclomatic complexity of the LoadFromEnv function in later changes by refactoring it to use setEnvString/setEnvBool helpers. No functional change, just collapse ~29 repetitive if-blocks into single-line calls to two small helpers. * Make Debian upstream repository configurable Support overriding the Debian handler's upstream (e.g. Ubuntu archives) via PROXY_UPSTREAM_DEBIAN or upstream.debian in the config file. Tested with PROXY_UPSTREAM_DEBIAN=http://archive.ubuntu.com/ubuntu to get Ubuntu Resolute packages. --- internal/config/config.go | 134 +++++++------------ internal/config/config_test.go | 7 + internal/handler/debian.go | 7 +- internal/handler/debian_test.go | 2 +- internal/handler/download_test.go | 2 +- internal/handler/notfound_ecosystems_test.go | 2 +- internal/server/server.go | 2 +- 7 files changed, 66 insertions(+), 90 deletions(-) diff --git a/internal/config/config.go b/internal/config/config.go index ec510d2..3dabc62 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -304,6 +304,11 @@ type UpstreamConfig struct { // Default: https://static.crates.io/crates CargoDownload string `json:"cargo_download" yaml:"cargo_download"` + // Debian is the upstream APT repository base URL. + // Example: http://archive.ubuntu.com/ubuntu would get Ubuntu. + // Default: http://deb.debian.org/debian + Debian string `json:"debian" yaml:"debian"` + // Auth configures authentication for upstream registries. // Keys are URL prefixes that are matched against request URLs. // Example: "https://npm.pkg.github.com" matches all requests to that host. @@ -378,6 +383,7 @@ func Default() *Config { GradlePluginPortal: "https://plugins.gradle.org/m2", Cargo: "https://index.crates.io", CargoDownload: "https://static.crates.io/crates", + Debian: "http://deb.debian.org/debian", }, Gradle: GradleConfig{ BuildCache: GradleBuildCacheConfig{ @@ -422,6 +428,21 @@ func Load(path string) (*Config, error) { return cfg, nil } +// setEnvString sets *dst from the named environment variable, leaving it +// untouched if the variable is unset or empty. +func setEnvString(dst *string, key string) { + if v := os.Getenv(key); v != "" { + *dst = v + } +} + +// setEnvBool is setEnvString for boolean fields, parsed via envBool. +func setEnvBool(dst *bool, key string) { + if v := os.Getenv(key); v != "" { + *dst = envBool(v) + } +} + // LoadFromEnv applies environment variable overrides to a Config. // Environment variables use the PROXY_ prefix: // - PROXY_LISTEN @@ -434,90 +455,35 @@ func Load(path string) (*Config, error) { // - PROXY_LOG_FORMAT // - PROXY_HEALTH_STORAGE_PROBE_INTERVAL func (c *Config) LoadFromEnv() { - if v := os.Getenv("PROXY_LISTEN"); v != "" { - c.Listen = v - } - if v := os.Getenv("PROXY_BASE_URL"); v != "" { - c.BaseURL = v - } - if v := os.Getenv("PROXY_UI_URL"); v != "" { - c.UIBaseURL = v - } - if v := os.Getenv("PROXY_STORAGE_URL"); v != "" { - c.Storage.URL = v - } - if v := os.Getenv("PROXY_STORAGE_PATH"); v != "" { - c.Storage.Path = v - } - if v := os.Getenv("PROXY_STORAGE_MAX_SIZE"); v != "" { - c.Storage.MaxSize = v - } - if v := os.Getenv("PROXY_STORAGE_DIRECT_SERVE"); v != "" { - c.Storage.DirectServe = envBool(v) - } - if v := os.Getenv("PROXY_STORAGE_DIRECT_SERVE_TTL"); v != "" { - c.Storage.DirectServeTTL = v - } - if v := os.Getenv("PROXY_STORAGE_DIRECT_SERVE_BASE_URL"); v != "" { - c.Storage.DirectServeBaseURL = v - } - if v := os.Getenv("PROXY_DATABASE_DRIVER"); v != "" { - c.Database.Driver = v - } - if v := os.Getenv("PROXY_DATABASE_PATH"); v != "" { - c.Database.Path = v - } - if v := os.Getenv("PROXY_DATABASE_URL"); v != "" { - c.Database.URL = v - } - if v := os.Getenv("PROXY_LOG_LEVEL"); v != "" { - c.Log.Level = v - } - if v := os.Getenv("PROXY_LOG_FORMAT"); v != "" { - c.Log.Format = v - } - if v := os.Getenv("PROXY_UPSTREAM_MAVEN"); v != "" { - c.Upstream.Maven = v - } - if v := os.Getenv("PROXY_UPSTREAM_GRADLE_PLUGIN_PORTAL"); v != "" { - c.Upstream.GradlePluginPortal = v - } - if v := os.Getenv("PROXY_COOLDOWN_DEFAULT"); v != "" { - c.Cooldown.Default = v - } - if v := os.Getenv("PROXY_CACHE_METADATA"); v != "" { - c.CacheMetadata = envBool(v) - } - if v := os.Getenv("PROXY_MIRROR_API"); v != "" { - c.MirrorAPI = envBool(v) - } - if v := os.Getenv("PROXY_METADATA_TTL"); v != "" { - c.MetadataTTL = v - } - if v := os.Getenv("PROXY_METADATA_MAX_SIZE"); v != "" { - c.MetadataMaxSize = v - } - if v := os.Getenv("PROXY_HTTP_TIMEOUT"); v != "" { - c.HTTPTimeout = v - } - if v := os.Getenv("PROXY_GRADLE_BUILD_CACHE_READ_ONLY"); v != "" { - c.Gradle.BuildCache.ReadOnly = v == "true" || v == "1" - } - if v := os.Getenv("PROXY_GRADLE_BUILD_CACHE_MAX_UPLOAD_SIZE"); v != "" { - c.Gradle.BuildCache.MaxUploadSize = v - } - if v := os.Getenv("PROXY_GRADLE_BUILD_CACHE_MAX_AGE"); v != "" { - c.Gradle.BuildCache.MaxAge = v - } - if v := os.Getenv("PROXY_GRADLE_BUILD_CACHE_MAX_SIZE"); v != "" { - c.Gradle.BuildCache.MaxSize = v - } - if v := os.Getenv("PROXY_GRADLE_BUILD_CACHE_SWEEP_INTERVAL"); v != "" { - c.Gradle.BuildCache.SweepInterval = v - } - if v := os.Getenv("PROXY_HEALTH_STORAGE_PROBE_INTERVAL"); v != "" { - c.Health.StorageProbeInterval = v - } + setEnvString(&c.Listen, "PROXY_LISTEN") + setEnvString(&c.BaseURL, "PROXY_BASE_URL") + setEnvString(&c.UIBaseURL, "PROXY_UI_URL") + setEnvString(&c.Storage.URL, "PROXY_STORAGE_URL") + setEnvString(&c.Storage.Path, "PROXY_STORAGE_PATH") + setEnvString(&c.Storage.MaxSize, "PROXY_STORAGE_MAX_SIZE") + setEnvBool(&c.Storage.DirectServe, "PROXY_STORAGE_DIRECT_SERVE") + setEnvString(&c.Storage.DirectServeTTL, "PROXY_STORAGE_DIRECT_SERVE_TTL") + setEnvString(&c.Storage.DirectServeBaseURL, "PROXY_STORAGE_DIRECT_SERVE_BASE_URL") + setEnvString(&c.Database.Driver, "PROXY_DATABASE_DRIVER") + setEnvString(&c.Database.Path, "PROXY_DATABASE_PATH") + setEnvString(&c.Database.URL, "PROXY_DATABASE_URL") + setEnvString(&c.Log.Level, "PROXY_LOG_LEVEL") + setEnvString(&c.Log.Format, "PROXY_LOG_FORMAT") + setEnvString(&c.Upstream.Maven, "PROXY_UPSTREAM_MAVEN") + setEnvString(&c.Upstream.GradlePluginPortal, "PROXY_UPSTREAM_GRADLE_PLUGIN_PORTAL") + setEnvString(&c.Upstream.Debian, "PROXY_UPSTREAM_DEBIAN") + setEnvString(&c.Cooldown.Default, "PROXY_COOLDOWN_DEFAULT") + setEnvBool(&c.CacheMetadata, "PROXY_CACHE_METADATA") + setEnvBool(&c.MirrorAPI, "PROXY_MIRROR_API") + setEnvString(&c.MetadataTTL, "PROXY_METADATA_TTL") + setEnvString(&c.MetadataMaxSize, "PROXY_METADATA_MAX_SIZE") + setEnvString(&c.HTTPTimeout, "PROXY_HTTP_TIMEOUT") + setEnvBool(&c.Gradle.BuildCache.ReadOnly, "PROXY_GRADLE_BUILD_CACHE_READ_ONLY") + setEnvString(&c.Gradle.BuildCache.MaxUploadSize, "PROXY_GRADLE_BUILD_CACHE_MAX_UPLOAD_SIZE") + setEnvString(&c.Gradle.BuildCache.MaxAge, "PROXY_GRADLE_BUILD_CACHE_MAX_AGE") + setEnvString(&c.Gradle.BuildCache.MaxSize, "PROXY_GRADLE_BUILD_CACHE_MAX_SIZE") + setEnvString(&c.Gradle.BuildCache.SweepInterval, "PROXY_GRADLE_BUILD_CACHE_SWEEP_INTERVAL") + setEnvString(&c.Health.StorageProbeInterval, "PROXY_HEALTH_STORAGE_PROBE_INTERVAL") } // validateAbsoluteURL returns an error if value is not a parseable URL with diff --git a/internal/config/config_test.go b/internal/config/config_test.go index decc18f..ef6ac90 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -37,6 +37,9 @@ func TestDefault(t *testing.T) { if cfg.Upstream.GradlePluginPortal != "https://plugins.gradle.org/m2" { t.Errorf("Upstream.GradlePluginPortal = %q, want %q", cfg.Upstream.GradlePluginPortal, "https://plugins.gradle.org/m2") } + if cfg.Upstream.Debian != "http://deb.debian.org/debian" { + t.Errorf("Upstream.Debian = %q, want %q", cfg.Upstream.Debian, "http://deb.debian.org/debian") + } } func TestValidate(t *testing.T) { @@ -273,6 +276,7 @@ func TestLoadFromEnv(t *testing.T) { t.Setenv("PROXY_LOG_LEVEL", testLevelDebug) t.Setenv("PROXY_UPSTREAM_MAVEN", "https://maven.example.com/repository/maven-public") t.Setenv("PROXY_UPSTREAM_GRADLE_PLUGIN_PORTAL", "https://plugins.example.com/m2") + t.Setenv("PROXY_UPSTREAM_DEBIAN", "http://archive.ubuntu.com/ubuntu") t.Setenv("PROXY_GRADLE_BUILD_CACHE_READ_ONLY", "true") t.Setenv("PROXY_GRADLE_BUILD_CACHE_MAX_UPLOAD_SIZE", "32MB") t.Setenv("PROXY_GRADLE_BUILD_CACHE_MAX_AGE", "12h") @@ -302,6 +306,9 @@ func TestLoadFromEnv(t *testing.T) { if cfg.Upstream.GradlePluginPortal != "https://plugins.example.com/m2" { t.Errorf("Upstream.GradlePluginPortal = %q, want %q", cfg.Upstream.GradlePluginPortal, "https://plugins.example.com/m2") } + if cfg.Upstream.Debian != "http://archive.ubuntu.com/ubuntu" { + t.Errorf("Upstream.Debian = %q, want %q", cfg.Upstream.Debian, "http://archive.ubuntu.com/ubuntu") + } if !cfg.Gradle.BuildCache.ReadOnly { t.Error("Gradle.BuildCache.ReadOnly = false, want true") } diff --git a/internal/handler/debian.go b/internal/handler/debian.go index 9a4aaab..b48f4fc 100644 --- a/internal/handler/debian.go +++ b/internal/handler/debian.go @@ -21,10 +21,13 @@ type DebianHandler struct { } // NewDebianHandler creates a new Debian/APT protocol handler. -func NewDebianHandler(proxy *Proxy, proxyURL string) *DebianHandler { +func NewDebianHandler(proxy *Proxy, proxyURL string, upstreamURL string) *DebianHandler { + if upstreamURL == "" { + upstreamURL = debianUpstream + } return &DebianHandler{ proxy: proxy, - upstreamURL: debianUpstream, + upstreamURL: strings.TrimSuffix(upstreamURL, "/"), proxyURL: strings.TrimSuffix(proxyURL, "/"), } } diff --git a/internal/handler/debian_test.go b/internal/handler/debian_test.go index dfdd326..60fa23a 100644 --- a/internal/handler/debian_test.go +++ b/internal/handler/debian_test.go @@ -18,6 +18,6 @@ func TestDebianHandler_parsePoolPath(t *testing.T) { } func TestDebianHandler_Routes(t *testing.T) { - h := NewDebianHandler(nil, "http://localhost:8080") + h := NewDebianHandler(nil, "http://localhost:8080", "") assertRoutesBasics(t, h.Routes(), "/dists/stable/Release", "/pool/../../../etc/passwd") } diff --git a/internal/handler/download_test.go b/internal/handler/download_test.go index 980e234..8192eeb 100644 --- a/internal/handler/download_test.go +++ b/internal/handler/download_test.go @@ -1165,7 +1165,7 @@ func TestDebianHandler_DownloadCacheMiss(t *testing.T) { ContentType: "application/vnd.debian.binary-package", } - h := NewDebianHandler(proxy, "http://localhost") + h := NewDebianHandler(proxy, "http://localhost", "") srv := httptest.NewServer(h.Routes()) defer srv.Close() diff --git a/internal/handler/notfound_ecosystems_test.go b/internal/handler/notfound_ecosystems_test.go index 3c2cecf..6f04113 100644 --- a/internal/handler/notfound_ecosystems_test.go +++ b/internal/handler/notfound_ecosystems_test.go @@ -17,7 +17,7 @@ func TestArtifactDownloadUpstreamNotFoundReturns404(t *testing.T) { handler func(p *Proxy) http.Handler }{ {"debian", "/pool/main/n/nginx/nginx_1.18.0-6_amd64.deb", - func(p *Proxy) http.Handler { return NewDebianHandler(p, "http://localhost").Routes() }}, + func(p *Proxy) http.Handler { return NewDebianHandler(p, "http://localhost", "").Routes() }}, {"rpm", "/releases/39/Everything/x86_64/os/Packages/n/nginx-1.24.0-1.fc39.x86_64.rpm", func(p *Proxy) http.Handler { return NewRPMHandler(p, "http://localhost").Routes() }}, {"nuget", "/v3-flatcontainer/newtonsoft.json/13.0.3/newtonsoft.json.13.0.3.nupkg", diff --git a/internal/server/server.go b/internal/server/server.go index 153a94c..856fe2d 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -223,7 +223,7 @@ func (s *Server) Start() error { cranHandler := handler.NewCRANHandler(proxy, s.cfg.BaseURL) juliaHandler := handler.NewJuliaHandler(proxy, s.cfg.BaseURL) containerHandler := handler.NewContainerHandler(proxy, s.cfg.BaseURL) - debianHandler := handler.NewDebianHandler(proxy, s.cfg.BaseURL) + debianHandler := handler.NewDebianHandler(proxy, s.cfg.BaseURL, s.cfg.Upstream.Debian) rpmHandler := handler.NewRPMHandler(proxy, s.cfg.BaseURL) r.Mount("/npm", http.StripPrefix("/npm", npmHandler.Routes())) From 5f993e396175e842dfb63654af2675880c146932 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 6 Aug 2026 21:59:52 +0100 Subject: [PATCH 035/633] Bump docker/login-action from 4.5.0 to 4.6.0 (#232) Bumps [docker/login-action](https://github.com/docker/login-action) from 4.5.0 to 4.6.0. - [Release notes](https://github.com/docker/login-action/releases) - [Commits](https://github.com/docker/login-action/compare/06fb636fac595d6fb4b28a5dfcb21a6f5091859c...dbcb813823bdd20940b903addbd779551569679f) --- updated-dependencies: - dependency-name: docker/login-action dependency-version: 4.6.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index c08cfa6..cba6f55 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -25,7 +25,7 @@ jobs: persist-credentials: false - name: Log in to the Container registry - uses: docker/login-action@06fb636fac595d6fb4b28a5dfcb21a6f5091859c + uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f with: registry: ghcr.io username: ${{ github.actor }} From bc1ea307f3550ec534966ad349dc0977124c5084 Mon Sep 17 00:00:00 2001 From: Ondrej Kokes Date: Thu, 6 Aug 2026 23:03:20 +0200 Subject: [PATCH 036/633] Multi-platform build (linux/amd64 and linux/arm64) (#230) * wip * pin to latest version, not commit * test run * Revert "test run" This reverts commit 355e8f867e1481db36327974ad94063d4be591cd. --- .github/workflows/publish.yml | 9 +++++++++ Dockerfile | 7 ++++--- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index cba6f55..e2b1084 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -24,6 +24,14 @@ jobs: with: persist-credentials: false + - name: Set up QEMU + uses: docker/setup-qemu-action@96fe6ef7f33517b61c61be40b68a1882f3264fb8 # v4.2.0 + with: + platforms: linux/amd64,linux/arm64 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0 + - name: Log in to the Container registry uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f with: @@ -41,6 +49,7 @@ jobs: uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a with: context: . + platforms: linux/amd64,linux/arm64 push: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} diff --git a/Dockerfile b/Dockerfile index c2e197f..9c51d30 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.26.5-alpine AS builder +FROM --platform=$BUILDPLATFORM golang:1.26.5-alpine AS builder WORKDIR /src @@ -12,8 +12,9 @@ RUN go mod download # Copy source code COPY . . -# Build the binary -RUN CGO_ENABLED=0 GOOS=linux go build -ldflags="-s -w" -o /proxy ./cmd/proxy +# Build the binary for the target platform +ARG TARGETARCH +RUN CGO_ENABLED=0 GOOS=linux GOARCH=${TARGETARCH} go build -ldflags="-s -w" -o /proxy ./cmd/proxy FROM alpine:3.24.1 From d0f93196a3c44352866c08d4b8ac673d5f5e0f06 Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Thu, 6 Aug 2026 22:14:51 +0100 Subject: [PATCH 037/633] Bump dependencies ahead of v0.7.0 (#235) - github.com/git-pkgs/archives v0.4.0 -> v0.5.0 - github.com/git-pkgs/magic v0.1.0 -> v0.2.0 - github.com/git-pkgs/spdx v0.1.4 -> v0.3.0 - github.com/prometheus/client_golang v1.24.0 -> v1.24.1 - modernc.org/sqlite v1.55.0 -> v1.56.0 - golang.org/x/text v0.38.0 -> v0.40.0 (fixes GO-2026-5970) --- go.mod | 22 +++++++++++----------- go.sum | 57 ++++++++++++++++++++++++++++----------------------------- 2 files changed, 39 insertions(+), 40 deletions(-) diff --git a/go.mod b/go.mod index e6e0a08..3449818 100644 --- a/go.mod +++ b/go.mod @@ -5,19 +5,19 @@ go 1.25.6 require ( github.com/BurntSushi/toml v1.6.0 github.com/CycloneDX/cyclonedx-go v0.11.0 - github.com/git-pkgs/archives v0.4.0 + github.com/git-pkgs/archives v0.5.0 github.com/git-pkgs/cooldown v0.1.1 github.com/git-pkgs/enrichment v0.6.4 - github.com/git-pkgs/magic v0.1.0 + github.com/git-pkgs/magic v0.2.0 github.com/git-pkgs/purl v0.1.15 github.com/git-pkgs/registries v0.6.4 - github.com/git-pkgs/spdx v0.1.4 + github.com/git-pkgs/spdx v0.3.0 github.com/git-pkgs/vers v0.3.0 github.com/git-pkgs/vulns v0.2.1 github.com/go-chi/chi/v5 v5.3.1 github.com/jmoiron/sqlx v1.4.0 github.com/lib/pq v1.12.3 - github.com/prometheus/client_golang v1.24.0 + github.com/prometheus/client_golang v1.24.1 github.com/prometheus/client_model v0.6.2 github.com/spdx/tools-golang v0.5.7 github.com/swaggo/swag v1.16.6 @@ -25,7 +25,7 @@ require ( golang.org/x/sync v0.22.0 google.golang.org/protobuf v1.36.11 gopkg.in/yaml.v3 v3.0.1 - modernc.org/sqlite v1.55.0 + modernc.org/sqlite v1.56.0 ) require ( @@ -205,7 +205,7 @@ require ( github.com/maratori/testpackage v1.1.2 // indirect github.com/matoous/godox v1.1.0 // indirect github.com/mattn/go-colorable v0.1.14 // indirect - github.com/mattn/go-isatty v0.0.20 // indirect + github.com/mattn/go-isatty v0.0.24 // indirect github.com/mattn/go-runewidth v0.0.16 // indirect github.com/mgechev/revive v1.14.0 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect @@ -226,7 +226,7 @@ require ( github.com/pelletier/go-toml/v2 v2.2.4 // indirect github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/prometheus/common v0.70.0 // indirect + github.com/prometheus/common v0.70.1 // indirect github.com/prometheus/procfs v0.21.1 // indirect github.com/quasilyte/go-ruleguard v0.4.5 // indirect github.com/quasilyte/go-ruleguard/dsl v0.3.23 // indirect @@ -293,14 +293,14 @@ require ( go.uber.org/zap v1.27.1 // indirect go.yaml.in/yaml/v2 v2.4.4 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect - golang.org/x/crypto v0.53.0 // indirect + golang.org/x/crypto v0.54.0 // indirect golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa // indirect golang.org/x/exp/typeparams v0.0.0-20260209203927-2842357ff358 // indirect golang.org/x/mod v0.37.0 // indirect - golang.org/x/net v0.56.0 // indirect + golang.org/x/net v0.57.0 // indirect golang.org/x/oauth2 v0.36.0 // indirect golang.org/x/sys v0.47.0 // indirect - golang.org/x/text v0.38.0 // indirect + golang.org/x/text v0.40.0 // indirect golang.org/x/tools v0.47.0 // indirect golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect google.golang.org/api v0.272.0 // indirect @@ -309,7 +309,7 @@ require ( gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect honnef.co/go/tools v0.7.0 // indirect - modernc.org/libc v1.74.1 // indirect + modernc.org/libc v1.74.4 // indirect modernc.org/mathutil v1.7.1 // indirect modernc.org/memory v1.11.0 // indirect mvdan.cc/gofumpt v0.9.2 // indirect diff --git a/go.sum b/go.sum index c239240..7ae59f3 100644 --- a/go.sum +++ b/go.sum @@ -244,14 +244,14 @@ github.com/fzipp/gocyclo v0.6.0 h1:lsblElZG7d3ALtGMx9fmxeTKZaLLpU8mET09yN4BBLo= github.com/fzipp/gocyclo v0.6.0/go.mod h1:rXPyn8fnlpa0R2csP/31uerbiVBugk5whMdlyaLkLoA= github.com/ghostiam/protogetter v0.3.20 h1:oW7OPFit2FxZOpmMRPP9FffU4uUpfeE/rEdE1f+MzD0= github.com/ghostiam/protogetter v0.3.20/go.mod h1:FjIu5Yfs6FT391m+Fjp3fbAYJ6rkL/J6ySpZBfnODuI= -github.com/git-pkgs/archives v0.4.0 h1:KNmmIsLiSH27lUdT27EfUkQXFaLgXV5KezE81iyOIgo= -github.com/git-pkgs/archives v0.4.0/go.mod h1:tfio0OIuPKEBKHs/UCL5XBUvYmKpnvtnba2iDlfSd6g= +github.com/git-pkgs/archives v0.5.0 h1:QdowC1jTOSbEOKTYGqVt8ZjIr+yJzy7cmLNWcLPj9Y8= +github.com/git-pkgs/archives v0.5.0/go.mod h1:tfio0OIuPKEBKHs/UCL5XBUvYmKpnvtnba2iDlfSd6g= github.com/git-pkgs/cooldown v0.1.1 h1:9OqqzCB8gANz/y44SmqGD0Jp8Qtu81D1sCbKl6Ehg7w= github.com/git-pkgs/cooldown v0.1.1/go.mod h1:v7APuK/UouTiu8mWQZbdDmj7DfxxkGUeuhjaRB5gv9E= github.com/git-pkgs/enrichment v0.6.4 h1:mGrfenttwmcUfPXRkWpB0wBJiiGj55ltniUh66Pq4bU= github.com/git-pkgs/enrichment v0.6.4/go.mod h1:zz1vPUak/w8Jhajll0KDRN2MjKaEYeCzQTxumWnVhqY= -github.com/git-pkgs/magic v0.1.0 h1:xLrqq7CMXB9g5bJnmJyKw17Rvlh0GFiEmO6e5RFsoeY= -github.com/git-pkgs/magic v0.1.0/go.mod h1:3ndidt+yvFaI1M0aEkkzkOlFnLPkeVQASIUojazcxCI= +github.com/git-pkgs/magic v0.2.0 h1:c7HqVxnP8c88EaVMH0/KraDFVTcmiXckRiSvNZEnvMQ= +github.com/git-pkgs/magic v0.2.0/go.mod h1:3ndidt+yvFaI1M0aEkkzkOlFnLPkeVQASIUojazcxCI= github.com/git-pkgs/packageurl-go v0.3.1 h1:WM3RBABQZLaRBxgKyYughc3cVBE8KyQxbSC6Jt5ak7M= github.com/git-pkgs/packageurl-go v0.3.1/go.mod h1:rcIxiG37BlQLB6FZfgdj9Fm7yjhRQd3l+5o7J0QPAk4= github.com/git-pkgs/pom v0.1.5 h1:TGT8Az2OMxGWsXnSagtUMGzZm7Oax8HrSCteA+mi0qY= @@ -260,8 +260,8 @@ github.com/git-pkgs/purl v0.1.15 h1:iQ3clh0Cw41rkM0rf24B7ShnN9Z+UtLMAFlNDUs+Qd4= github.com/git-pkgs/purl v0.1.15/go.mod h1:PqCLVBDeZrZgHysR803/AntMELgIr2LFZVNCcwLH2m0= github.com/git-pkgs/registries v0.6.4 h1:Kq/KlStjaQyE83UXT/tKuzCrIzc4keGeBjtroMqgoHA= github.com/git-pkgs/registries v0.6.4/go.mod h1:YkGHbxHIe2Ha/ROH6zNkS5PJUUoa9g0Ti/s2XhZnrak= -github.com/git-pkgs/spdx v0.1.4 h1:eQ0waEV3uUeItpWAOvdN1K1rL9hTgsU7fF74r1mDXMs= -github.com/git-pkgs/spdx v0.1.4/go.mod h1:cqRoZcvl530s/W+oGNvwjt4ODN8T1W6D/20MUZEFdto= +github.com/git-pkgs/spdx v0.3.0 h1:AN0guJE7vN5gbOMi9We4j1ziS4cwgFVhTvjVDGfaC7Q= +github.com/git-pkgs/spdx v0.3.0/go.mod h1:cqRoZcvl530s/W+oGNvwjt4ODN8T1W6D/20MUZEFdto= github.com/git-pkgs/vers v0.3.0 h1:xM4LLUCRmqzdDfe+/pVQUx4SRyFXRVth6tOsJ14wMKU= github.com/git-pkgs/vers v0.3.0/go.mod h1:biTbSQK1qdbrsxDEKnqe3Jzclxz8vW6uDcwKjfUGcOo= github.com/git-pkgs/vulns v0.2.1 h1:tWGhOfPVDZwkM2Y9vRkMpMR+gjtlu2jhERS5JeNBoKQ= @@ -362,8 +362,8 @@ github.com/google/go-replayers/httpreplay v1.2.0 h1:VM1wEyyjaoU53BwrOnaf9VhAyQQE github.com/google/go-replayers/httpreplay v1.2.0/go.mod h1:WahEFFZZ7a1P4VM1qEeHy+tME4bwyqPcwWbNlUI1Mcg= github.com/google/martian/v3 v3.3.3 h1:DIhPTQrbPkgs2yJYdXU/eNACCG5DVQjySNRNlflZ9Fc= github.com/google/martian/v3 v3.3.3/go.mod h1:iEPrYcgCF7jA9OtScMFQyAlZZ4YXTKEtJ1E6RWzmBA0= -github.com/google/pprof v0.0.0-20260115054156-294ebfa9ad83 h1:z2ogiKUYzX5Is6zr/vP9vJGqPwcdqsWjOt+V8J7+bTc= -github.com/google/pprof v0.0.0-20260115054156-294ebfa9ad83/go.mod h1:MxpfABSjhmINe3F1It9d+8exIHFvUqtLIRCdOGNXqiI= +github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3 h1:LMLX+LgTNWpfvCBdFebv6EsYotImrt/Ppc5cXIriCSo= +github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3/go.mod h1:jl5iWTm0/hd5PjEYEOuwAJ57L/CibdZfrqZ5XA5GrCk= github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0= github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= @@ -424,8 +424,8 @@ github.com/kisielk/errcheck v1.9.0 h1:9xt1zI9EBfcYBvdU1nVrzMzzUPUtPKs9bVSIM3TAb3 github.com/kisielk/errcheck v1.9.0/go.mod h1:kQxWMMVZgIkDq7U8xtG/n2juOjbLgZtedi0D+/VL/i8= github.com/kkHAIKE/contextcheck v1.1.6 h1:7HIyRcnyzxL9Lz06NGhiKvenXq7Zw6Q0UQu/ttjfJCE= github.com/kkHAIKE/contextcheck v1.1.6/go.mod h1:3dDbMRNBFaq8HFXWC1JyvDSPm43CmE6IuHam8Wr0rkg= -github.com/klauspost/compress v1.19.0 h1:sXLILfc9jV2QYWkzFOPWStmcUVH2RHEB1JCdY2oVvCQ= -github.com/klauspost/compress v1.19.0/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/klauspost/compress v1.19.1 h1:VsB4HPswih7mmZ8WleSFQ75c/Ui1M4trX5oAsJnhSlk= +github.com/klauspost/compress v1.19.1/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= @@ -483,8 +483,8 @@ github.com/matryer/is v1.4.0 h1:sosSmIWwkYITGrxZ25ULNDeKiMNzFSr4V/eqBQP0PeE= github.com/matryer/is v1.4.0/go.mod h1:8I/i5uYgLzgsgEloJE1U6xx5HkBQpAZvepWuujKwMRU= github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE= github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= -github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= -github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsReI= +github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A= github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc= github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU= @@ -543,12 +543,12 @@ github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgm github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_golang v1.24.0 h1:5XStIklKuAtJSNpdD3s8XJj/Yv78IQmE1kbNk87JrAI= -github.com/prometheus/client_golang v1.24.0/go.mod h1:QcsNdotprC2nS4BTM2ucbcqxd2CeXTEa9jW7zHO9iDE= +github.com/prometheus/client_golang v1.24.1 h1:JnJkREXzWxUdCuPFpIWZiPispT9xVV59uiuyR2bPlnU= +github.com/prometheus/client_golang v1.24.1/go.mod h1:F+oSRECHg4sse5ucfYpYDeIv/hu68Zo0uoHKetWnzcE= github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= -github.com/prometheus/common v0.70.0 h1:bcpru3tWPVnxGnETLgOV5jbp/JRXgYEyv65CuBLAMMI= -github.com/prometheus/common v0.70.0/go.mod h1:S/SFasQmgGiYH6C81LKCtYa8QACgthGg5zxL2udV7SY= +github.com/prometheus/common v0.70.1 h1:1HvjP4D5oL3t8RsPlwxA9onvvStjtIHYE5XuuwOi/PY= +github.com/prometheus/common v0.70.1/go.mod h1:VdFUQDMZK3VLkurFUVhia6uys/0suUp86TJz5qbJRhc= github.com/prometheus/procfs v0.21.1 h1:GljZCt+zSTS+NZq88cyQ1LjZ+RCHp3uVuabBWA5+OJI= github.com/prometheus/procfs v0.21.1/go.mod h1:aB55Cww9pdSJVHk0hUf0inxWyyjPogFIjmHKYgMKmtY= github.com/quasilyte/go-ruleguard v0.4.5 h1:AGY0tiOT5hJX9BTdx/xBdoCubQUAE2grkqY2lSwvZcA= @@ -740,8 +740,8 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= -golang.org/x/crypto v0.53.0 h1:QZ4Muo8THX6CizN2vPPd5fBGHyogrdK9fG4wLPFUsto= -golang.org/x/crypto v0.53.0/go.mod h1:DNLU434OwVakk9PzuwV8w62mAJpRJL3vsgcfp4Qnsio= +golang.org/x/crypto v0.54.0 h1:YLIA59K4fiNzHzjnZt2tUJQjQtUWfWbeHBqKtk3eScw= +golang.org/x/crypto v0.54.0/go.mod h1:KWL8ny2AZdGR2cWmzeHrp2azQPGogOv+HeQaVEXC2dk= golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa h1:Zt3DZoOFFYkKhDT3v7Lm9FDMEV06GpzjG2jrqW+QTE0= golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa/go.mod h1:K79w1Vqn7PoiZn+TkNpx3BUWUQksGO3JcVX6qIjytmA= golang.org/x/exp/typeparams v0.0.0-20220428152302-39d4317da171/go.mod h1:AbB0pIl9nAr9wVwH+Z2ZpaocVmF5I4GyWCDIsVjR0bk= @@ -773,8 +773,8 @@ golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= golang.org/x/net v0.16.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= -golang.org/x/net v0.56.0 h1:Rw8j/hFzGvJUZwNBXnAtf5sVDVt+65SK2C7IxCxZt5o= -golang.org/x/net v0.56.0/go.mod h1:D3Ku6r+V6JROoZK144D2XfMHFcMq/0zSfLelVTCFKec= +golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE= +golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU= golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -805,7 +805,6 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -825,8 +824,8 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= -golang.org/x/text v0.38.0 h1:sXmwo9DwP3OK9EZ7PqAdaooSGozfl/3a6/xJcbzPRhE= -golang.org/x/text v0.38.0/go.mod h1:YXZt3QhHUKYT53r2lLKFIVi6Ao1jdzrTR/KQ09qyxF4= +golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs= +golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY= golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -884,8 +883,8 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.7.0 h1:w6WUp1VbkqPEgLz4rkBzH/CSU6HkoqNLp6GstyTx3lU= honnef.co/go/tools v0.7.0/go.mod h1:pm29oPxeP3P82ISxZDgIYeOaf9ta6Pi0EWvCFoLG2vc= -modernc.org/cc/v4 v4.29.0 h1:CXgwL8cvxmyzBQZzbSl/6xFtMCryb6u8IOqDci39cgc= -modernc.org/cc/v4 v4.29.0/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI= +modernc.org/cc/v4 v4.29.1 h1:MKgdCV3WykTSPqpVrnxdEDS0HEd2FHpKZDzxzU5LyeI= +modernc.org/cc/v4 v4.29.1/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI= modernc.org/ccgo/v4 v4.34.6 h1:sBgfIwyN0TQ9C5hwIeuqyeAKyMWnbvj2fvpF4L11uzU= modernc.org/ccgo/v4 v4.34.6/go.mod h1:SZ8YcN9NG7XVsQYdm6jYBvi8PQP1qi+kqB6OhjqI3Fk= modernc.org/fileutil v1.4.0 h1:j6ZzNTftVS054gi281TyLjHPp6CPHr2KCxEXjEbD6SM= @@ -896,8 +895,8 @@ modernc.org/gc/v3 v3.1.4 h1:2g65LGVSmFQrXeITAw97x7hCRvZFcyE1uDP+7Vng7JI= modernc.org/gc/v3 v3.1.4/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY= modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks= modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI= -modernc.org/libc v1.74.1 h1:bdR4VTKFMC4966QSNZ05XLGI/VwzVa2kTUX51Dm0riQ= -modernc.org/libc v1.74.1/go.mod h1:uH4t5bOx3G3g9Xcmj10YKlTcVISlRDwv8VoQJG9n8Os= +modernc.org/libc v1.74.4 h1:fX1Omw4o2/1C2iRkkIsrQTasJQldLhRmuPreXLoWs9k= +modernc.org/libc v1.74.4/go.mod h1:eeQAS9W3sZeKYMFubydxJpII9ybHWshk+7or7bLG9co= modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI= @@ -906,8 +905,8 @@ modernc.org/opt v0.2.0 h1:tGyef5ApycA7FSEOMraay9SaTk5zmbx7Tu+cJs4QKZg= modernc.org/opt v0.2.0/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns= modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w= modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE= -modernc.org/sqlite v1.55.0 h1:hIFh0MCH0rGinQ/4KYb5/UbCkRkb+UP+OkLCVWa5MTM= -modernc.org/sqlite v1.55.0/go.mod h1:4ntCLuNmnH8+GNqjka1wNg7KJd5/Hi5FYp8K+XQ7GZw= +modernc.org/sqlite v1.56.0 h1:/D8e2RfFqoy/Zc6PuC76U28zFwmI/sYx1Kjm4yEn9e0= +modernc.org/sqlite v1.56.0/go.mod h1:yCJ2cmAaIkHQ25oXWrF8H4O1lIfPYPR26yCEDj2P3pQ= modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0= modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A= modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= From 14f80ced3479999c86c6690e5005dfc00a398841 Mon Sep 17 00:00:00 2001 From: Philipp Garbe Date: Mon, 10 Aug 2026 10:21:49 +0200 Subject: [PATCH 038/633] fix(npm): use combined Accept header to support Artifactory upstreams (#241) When cooldown is disabled, send: Accept: application/vnd.npm.install-v1+json;q=1.0, application/json;q=0.8 This allows upstreams like JFrog Artifactory that return 406 for the abbreviated packument type to fall back to full JSON metadata, while letting the public npm registry continue to serve the smaller abbreviated format it prefers. When cooldown is enabled, keep sending only application/json because the abbreviated format omits the "time" map required for version age filtering. Fixes #228 Co-authored-by: Claude Sonnet 4.6 --- internal/handler/npm.go | 11 +++++++---- internal/handler/npm_test.go | 12 ++++++------ 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/internal/handler/npm.go b/internal/handler/npm.go index ee9b59c..78aa5e4 100644 --- a/internal/handler/npm.go +++ b/internal/handler/npm.go @@ -13,7 +13,7 @@ import ( const ( npmUpstream = "https://registry.npmjs.org" - npmAbbreviatedCT = "application/vnd.npm.install-v1+json" + npmAcceptDefault = "application/vnd.npm.install-v1+json;q=1.0, application/json;q=0.8" scopedParts = 2 // scope + name in scoped packages ) @@ -71,9 +71,12 @@ func (h *NPMHandler) handlePackageMetadata(w http.ResponseWriter, r *http.Reques upstreamURL := fmt.Sprintf("%s/%s", h.upstreamURL, url.PathEscape(packageName)) - // Use abbreviated metadata when cooldown is disabled — it's much smaller - // (e.g. drizzle-orm: 4MB vs 92MB) but lacks the time map needed for cooldown. - accept := npmAbbreviatedCT + // Prefer the smaller abbreviated packument format but include application/json + // as a fallback so upstreams that reject the abbreviated type (e.g. JFrog + // Artifactory, which returns 406) can still respond with full metadata. + // When cooldown is enabled we must use full metadata exclusively because the + // abbreviated format omits the "time" map required for version age filtering. + accept := npmAcceptDefault if h.proxy.Cooldown != nil && h.proxy.Cooldown.Enabled() { accept = contentTypeJSON } diff --git a/internal/handler/npm_test.go b/internal/handler/npm_test.go index e0257dd..fd6f080 100644 --- a/internal/handler/npm_test.go +++ b/internal/handler/npm_test.go @@ -396,7 +396,7 @@ func TestNPMHandlerUsesAbbreviatedMetadata(t *testing.T) { })) defer upstream.Close() - t.Run("no cooldown uses abbreviated metadata", func(t *testing.T) { + t.Run("no cooldown uses combined accept header", func(t *testing.T) { h := &NPMHandler{ proxy: testProxy(), upstreamURL: upstream.URL, @@ -407,12 +407,12 @@ func TestNPMHandlerUsesAbbreviatedMetadata(t *testing.T) { w := httptest.NewRecorder() h.handlePackageMetadata(w, req) - if gotAccept != npmAbbreviatedCT { - t.Errorf("Accept = %q, want abbreviated metadata header", gotAccept) + if gotAccept != npmAcceptDefault { + t.Errorf("Accept = %q, want %q", gotAccept, npmAcceptDefault) } }) - t.Run("cooldown enabled uses full metadata", func(t *testing.T) { + t.Run("cooldown enabled uses full metadata only", func(t *testing.T) { proxy := testProxy() proxy.Cooldown = &cooldown.Config{Default: "3d"} @@ -426,8 +426,8 @@ func TestNPMHandlerUsesAbbreviatedMetadata(t *testing.T) { w := httptest.NewRecorder() h.handlePackageMetadata(w, req) - if gotAccept == npmAbbreviatedCT { - t.Error("cooldown enabled should use full metadata, not abbreviated") + if gotAccept != contentTypeJSON { + t.Errorf("Accept = %q, want %q (cooldown requires full metadata)", gotAccept, contentTypeJSON) } }) } From 4fa903e01e26aecdfc0001a965c0a4598b015afd Mon Sep 17 00:00:00 2001 From: oscar-broman Date: Mon, 10 Aug 2026 12:27:03 +0400 Subject: [PATCH 039/633] Enforce cooldown on artifact downloads (#240) Cooldown filtering only ran when rewriting metadata, so a version could be missing from the npm packument and the PyPI simple index while its tarball stayed reachable. Lockfiles record artifact URLs verbatim, so npm ci and pinned pip requirements reach handleDownload without ever requesting metadata. The shared artifact path has no publish time to check against, since updateCacheDB upserts versions without PublishedAt and the column is only set by enrichment. Each handler now resolves the publish time from metadata it already fetches and returns 404 while a version is inside the window. Versions with no usable publish time are still served, as they are when filtering metadata. --- internal/handler/npm.go | 51 +++++++++++++++++++++++ internal/handler/npm_test.go | 78 +++++++++++++++++++++++++++++++++++ internal/handler/pypi.go | 22 ++++++++++ internal/handler/pypi_test.go | 56 +++++++++++++++++++++++++ 4 files changed, 207 insertions(+) diff --git a/internal/handler/npm.go b/internal/handler/npm.go index 78aa5e4..b7d96a3 100644 --- a/internal/handler/npm.go +++ b/internal/handler/npm.go @@ -268,6 +268,13 @@ func (h *NPMHandler) handleDownload(w http.ResponseWriter, r *http.Request) { h.proxy.Logger.Info("npm download request", "package", packageName, "version", version, "filename", filename) + if h.versionInCooldown(r, packageName, version) { + h.proxy.Logger.Info("cooldown: withholding npm tarball", + "package", packageName, "version", version) + JSONError(w, http.StatusNotFound, "version not found") + return + } + downloadURL := fmt.Sprintf( "%s/%s/-/%s", h.upstreamURL, @@ -290,6 +297,50 @@ func (h *NPMHandler) handleDownload(w http.ResponseWriter, r *http.Request) { ServeArtifact(w, result) } +// versionInCooldown reports whether a version is still inside the cooldown +// window. Filtering the packument is not enough on its own: tarball URLs are +// predictable and lockfiles record them directly, so `npm ci` reaches the +// download path without ever requesting metadata. +// +// The packument is served from the metadata cache, so this normally costs no +// extra upstream request. A version with no usable publish time is allowed +// through, matching how applyCooldownFiltering treats it. +func (h *NPMHandler) versionInCooldown(r *http.Request, packageName, version string) bool { + if h.proxy.Cooldown == nil || !h.proxy.Cooldown.Enabled() { + return false + } + + upstreamURL := fmt.Sprintf("%s/%s", h.upstreamURL, url.PathEscape(packageName)) + + body, _, err := h.proxy.FetchOrCacheMetadata(r.Context(), "npm", packageName, upstreamURL, contentTypeJSON) + if err != nil { + h.proxy.Logger.Warn("cooldown: could not fetch npm metadata for download check", + "package", packageName, "version", version, "error", err) + return false + } + + var metadata struct { + Time map[string]string `json:"time"` + } + if err := json.Unmarshal(body, &metadata); err != nil { + h.proxy.Logger.Warn("cooldown: could not parse npm metadata for download check", + "package", packageName, "version", version, "error", err) + return false + } + + published, ok := metadata.Time[version] + if !ok { + return false + } + + publishedAt, err := time.Parse(time.RFC3339, published) + if err != nil { + return false + } + + return !h.proxy.Cooldown.IsAllowed("npm", canonicalPackagePURL("npm", packageName), publishedAt) +} + func escapeNPMDownloadPackage(packageName string) string { scope, name, scoped := strings.Cut(packageName, "/") if scoped && strings.HasPrefix(scope, "@") && len(scope) > 1 && name != "" && !strings.Contains(name, "/") { diff --git a/internal/handler/npm_test.go b/internal/handler/npm_test.go index fd6f080..07da9c3 100644 --- a/internal/handler/npm_test.go +++ b/internal/handler/npm_test.go @@ -454,3 +454,81 @@ func TestNPMHandlerMetadataNotFound(t *testing.T) { t.Errorf("status = %d, want %d", w.Code, http.StatusNotFound) } } + +func TestNPMDownloadCooldown(t *testing.T) { + now := time.Now() + packument := `{ + "name": "leftpad", + "dist-tags": {"latest": "2.0.0"}, + "time": { + "1.0.0": "` + now.Add(-30*24*time.Hour).Format(time.RFC3339) + `", + "2.0.0": "` + now.Add(-1*time.Hour).Format(time.RFC3339) + `" + }, + "versions": {"1.0.0": {}, "2.0.0": {}} + }` + + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", contentTypeJSON) + _, _ = io.WriteString(w, packument) + })) + defer upstream.Close() + + tests := []struct { + name string + version string + wantStatus int + }{ + {"published before the window serves the tarball", testVersion100, http.StatusOK}, + {"published inside the window is withheld", "2.0.0", http.StatusNotFound}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + proxy, _, _, fetcher := setupTestProxy(t) + proxy.HTTPClient = upstream.Client() + proxy.Cooldown = &cooldown.Config{Default: "7d"} + fetcher.artifact = &fetch.Artifact{ + Body: io.NopCloser(strings.NewReader("tarball data")), + ContentType: "application/octet-stream", + } + + h := NewNPMHandler(proxy, "http://proxy.test", upstream.URL) + srv := httptest.NewServer(h.Routes()) + defer srv.Close() + + resp, err := http.Get(srv.URL + "/leftpad/-/leftpad-" + tt.version + ".tgz") + if err != nil { + t.Fatalf("request failed: %v", err) + } + defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode != tt.wantStatus { + t.Errorf("status = %d, want %d", resp.StatusCode, tt.wantStatus) + } + if tt.wantStatus == http.StatusNotFound && fetcher.fetchCalled { + t.Error("fetched a version that is still inside the cooldown window") + } + }) + } +} + +func TestNPMDownloadCooldownDisabled(t *testing.T) { + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + t.Error("metadata must not be fetched when cooldown is disabled") + w.WriteHeader(http.StatusInternalServerError) + })) + defer upstream.Close() + + proxy, _, _, fetcher := setupTestProxy(t) + proxy.HTTPClient = upstream.Client() + fetcher.artifact = &fetch.Artifact{ + Body: io.NopCloser(strings.NewReader("tarball data")), + ContentType: "application/octet-stream", + } + + h := NewNPMHandler(proxy, "http://proxy.test", upstream.URL) + + if h.versionInCooldown(httptest.NewRequest(http.MethodGet, "/", nil), "leftpad", testVersion100) { + t.Error("versionInCooldown = true, want false when cooldown is not configured") + } +} diff --git a/internal/handler/pypi.go b/internal/handler/pypi.go index 713f6c8..7875cdd 100644 --- a/internal/handler/pypi.go +++ b/internal/handler/pypi.go @@ -311,6 +311,21 @@ func (h *PyPIHandler) shouldFilterRelease(packagePURL string, files any) bool { return !publishedAt.IsZero() && !h.proxy.Cooldown.IsAllowed("pypi", packagePURL, publishedAt) } +// versionInCooldown reports whether a version is still inside the cooldown +// window. Filtering the simple index is not enough on its own: file URLs are +// recorded in lockfiles and requirements pins, so pip can reach the download +// path without ever reading the index. +// +// A release whose upload time cannot be determined is allowed through, matching +// how fetchFilteredVersions treats it. +func (h *PyPIHandler) versionInCooldown(r *http.Request, name, version string) bool { + if h.proxy.Cooldown == nil || !h.proxy.Cooldown.Enabled() { + return false + } + + return h.fetchFilteredVersions(r, name)[version] +} + // rewriteFileEntries rewrites URLs in a list of file entries. func (h *PyPIHandler) rewriteFileEntries(files any) { filesArr, ok := files.([]any) @@ -417,6 +432,13 @@ func (h *PyPIHandler) handleDownload(w http.ResponseWriter, r *http.Request) { filename := parts[len(parts)-1] name, version := h.parseFilename(filename) + if name != "" && h.versionInCooldown(r, name, version) { + h.proxy.Logger.Info("cooldown: withholding pypi file", + "name", name, "version", version, "filename", filename) + http.Error(w, "not found", http.StatusNotFound) + return + } + if name == "" { // Can't determine name/version, use hash as identifier name = fmt.Sprintf("_hash_%s", hashPath(path)) diff --git a/internal/handler/pypi_test.go b/internal/handler/pypi_test.go index 5ae76ca..6bbcf3e 100644 --- a/internal/handler/pypi_test.go +++ b/internal/handler/pypi_test.go @@ -236,3 +236,59 @@ func TestPyPIHandler_DownloadCacheMiss(t *testing.T) { t.Error("expected fetcher to be called on cache miss") } } + +func TestPyPIDownloadCooldown(t *testing.T) { + now := time.Now() + releases := `{"releases": { + "1.0.0": [{"upload_time_iso_8601": "` + now.Add(-30*24*time.Hour).Format(time.RFC3339) + `"}], + "2.0.0": [{"upload_time_iso_8601": "` + now.Add(-1*time.Hour).Format(time.RFC3339) + `"}] + }}` + + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", contentTypeJSON) + _, _ = io.WriteString(w, releases) + })) + defer upstream.Close() + + tests := []struct { + name string + filename string + wantStatus int + }{ + {"published before the window serves the file", "newpkg-1.0.0.tar.gz", http.StatusOK}, + {"published inside the window is withheld", "newpkg-2.0.0.tar.gz", http.StatusNotFound}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + proxy, _, _, fetcher := setupTestProxy(t) + proxy.HTTPClient = upstream.Client() + proxy.Cooldown = &cooldown.Config{Default: "7d"} + fetcher.artifact = &fetch.Artifact{ + Body: io.NopCloser(strings.NewReader("sdist data")), + ContentType: "application/octet-stream", + } + + h := &PyPIHandler{ + proxy: proxy, + upstreamURL: upstream.URL, + proxyURL: "http://localhost", + } + srv := httptest.NewServer(h.Routes()) + defer srv.Close() + + resp, err := http.Get(srv.URL + "/packages/packages/ab/cd/ef0123456789/" + tt.filename) + if err != nil { + t.Fatalf("request failed: %v", err) + } + defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode != tt.wantStatus { + t.Errorf("status = %d, want %d", resp.StatusCode, tt.wantStatus) + } + if tt.wantStatus == http.StatusNotFound && fetcher.fetchCalled { + t.Error("fetched a version that is still inside the cooldown window") + } + }) + } +} From 30e40526154eb8fb23f8d3b4e810967b5155c89d Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Mon, 10 Aug 2026 09:34:56 +0100 Subject: [PATCH 040/633] Document upstream.debian in README and config.example.yaml (#237) Follow-up to #229 which added the config key and env var but didn't touch docs. --- README.md | 7 +++++++ config.example.yaml | 3 +++ 2 files changed, 10 insertions(+) diff --git a/README.md b/README.md index 4609b66..01012e2 100644 --- a/README.md +++ b/README.md @@ -376,6 +376,13 @@ Replace your existing sources.list entries, then: sudo apt update ``` +The upstream defaults to `http://deb.debian.org/debian`. To proxy a different APT repository (e.g. Ubuntu), set `upstream.debian` in the config file or `PROXY_UPSTREAM_DEBIAN` in the environment: + +```yaml +upstream: + debian: "http://archive.ubuntu.com/ubuntu" +``` + ### RPM / Yum / DNF Configure yum/dnf to use the proxy in `/etc/yum.repos.d/proxy.repo`: diff --git a/config.example.yaml b/config.example.yaml index 1176f5b..7cf7bb2 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -95,6 +95,9 @@ upstream: # Cargo crate download URL cargo_download: "https://static.crates.io/crates" + # Debian/APT repository URL (used by /debian endpoint) + debian: "http://deb.debian.org/debian" + # Authentication for upstream registries # Keys are URL prefixes matched against request URLs. # Values can reference environment variables using ${VAR_NAME} syntax. From a17bdc7c898c85c2f361b3b9aee15c705a30ef2b Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Mon, 10 Aug 2026 16:42:52 +0100 Subject: [PATCH 041/633] Sign published container images (#227) * Sign published container images * Attest per-platform SPDX SBOMs with cosign Extract each platform's SPDX document from the BuildKit SBOM attestation and sign it as a cosign spdxjson attestation against the manifest-list digest, so downstream consumers (e.g. Kyverno image-verification policies) can verify the predicate signature rather than relying on the unsigned BuildKit attachment. --- .github/workflows/publish.yml | 43 +++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index e2b1084..1c4eb1e 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -18,6 +18,7 @@ jobs: permissions: packages: write contents: read + id-token: write steps: - name: Check out the repo uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 @@ -45,7 +46,10 @@ jobs: with: images: ghcr.io/${{ github.repository }} + - uses: sigstore/cosign-installer@6f9f17788090df1f26f669e9d70d6ae9567deba6 # v4.1.2 + - name: Build and push Docker image + id: build uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a with: context: . @@ -53,3 +57,42 @@ jobs: push: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} + provenance: mode=max + sbom: true + + - name: Sign image by digest + env: + DIGEST: ${{ steps.build.outputs.digest }} + IMAGE: ghcr.io/${{ github.repository }} + run: | + set -euo pipefail + [[ "$DIGEST" =~ ^sha256:[0-9a-f]{64}$ ]] + cosign sign --yes "${IMAGE}@${DIGEST}" + + - name: Verify BuildKit attestations and extract SPDX predicates + env: + DIGEST: ${{ steps.build.outputs.digest }} + IMAGE: ghcr.io/${{ github.repository }} + run: | + set -euo pipefail + reference="${IMAGE}@${DIGEST}" + docker buildx imagetools inspect "$reference" --format '{{ json .Provenance }}' > provenance.json + docker buildx imagetools inspect "$reference" --format '{{ json .SBOM }}' > sbom.json + + for platform in linux/amd64 linux/arm64; do + jq -e --arg p "$platform" '.[$p].SLSA | type == "object" and length > 0' \ + provenance.json >/dev/null + jq -e --arg p "$platform" '.[$p].SPDX' sbom.json > "sbom-${platform//\//-}.spdx.json" + done + + - name: Attest platform SBOMs by digest + env: + DIGEST: ${{ steps.build.outputs.digest }} + IMAGE: ghcr.io/${{ github.repository }} + run: | + set -euo pipefail + [[ "$DIGEST" =~ ^sha256:[0-9a-f]{64}$ ]] + reference="${IMAGE}@${DIGEST}" + for predicate in sbom-linux-amd64.spdx.json sbom-linux-arm64.spdx.json; do + cosign attest --yes --type spdxjson --predicate "$predicate" "$reference" + done From bbea63f04640572a68bd531b932c367dfb3b89d1 Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Thu, 13 Aug 2026 07:08:23 +0100 Subject: [PATCH 042/633] fix(upstream): apply authentication through shared transport (#198) * upstream: apply authentication through shared transport * Address upstream authentication review findings --- config.example.yaml | 3 +- docs/architecture.md | 2 + docs/configuration.md | 6 +- internal/config/config.go | 75 ++++- internal/config/config_test.go | 71 +++++ internal/httpclient/transport.go | 433 ++++++++++++++++++++++++++ internal/httpclient/transport_test.go | 251 +++++++++++++++ internal/server/server.go | 32 +- 8 files changed, 852 insertions(+), 21 deletions(-) create mode 100644 internal/httpclient/transport.go create mode 100644 internal/httpclient/transport_test.go diff --git a/config.example.yaml b/config.example.yaml index 7cf7bb2..7ada017 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -99,7 +99,8 @@ upstream: debian: "http://deb.debian.org/debian" # Authentication for upstream registries - # Keys are URL prefixes matched against request URLs. + # Keys are absolute URL scopes. Scheme, host, effective port, and path + # segment boundaries must match; the longest matching scope wins. # Values can reference environment variables using ${VAR_NAME} syntax. # # Supported auth types: diff --git a/docs/architecture.md b/docs/architecture.md index f04d548..cf8b0e2 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -240,6 +240,8 @@ Fetches artifacts from upstream registries. - Exponential backoff retry on 429 (rate limit) and 5xx errors - Returns streaming reader (doesn't load into memory) - Configurable user-agent +- Shares an authentication-aware transport with metadata requests so URL-scoped credentials apply consistently +- Discovers and caches scoped OCI Bearer tokens from registry challenges **Resolver:** - Determines download URL for a package/version diff --git a/docs/configuration.md b/docs/configuration.md index 8998ac2..fa0f576 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -123,7 +123,9 @@ upstream: ## Authentication -Configure authentication for private upstream registries. Auth is matched by URL prefix, and credentials can reference environment variables using `${VAR_NAME}` syntax. +Configure authentication for private upstream registries. The same authentication-aware client is used for metadata and artifact downloads, and credentials can reference environment variables using `${VAR_NAME}` syntax. + +OCI registries that return a Bearer challenge from a `/v2/{repository}/…` endpoint are handled automatically. The proxy discovers the token realm from `WWW-Authenticate`, applies any configured credentials for the token URL, and reuses the scoped token until shortly before it expires. ### Bearer Token @@ -172,7 +174,7 @@ upstream: ### URL Matching -Auth configs are matched by URL prefix. The longest matching prefix wins, so you can configure different credentials for different paths: +Auth keys must be absolute URLs. Matching compares the scheme, host, effective port, and path-segment prefix, preventing credentials for `registry.example.com` from being sent to a lookalike host such as `registry.example.com.evil.test`. The longest matching scope wins, so you can configure different credentials for different paths: ```yaml upstream: diff --git a/internal/config/config.go b/internal/config/config.go index 3dabc62..31f8c26 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -310,23 +310,29 @@ type UpstreamConfig struct { Debian string `json:"debian" yaml:"debian"` // Auth configures authentication for upstream registries. - // Keys are URL prefixes that are matched against request URLs. + // Keys are absolute URL scopes matched by scheme, host, effective port, + // and path-segment prefix. // Example: "https://npm.pkg.github.com" matches all requests to that host. Auth map[string]AuthConfig `json:"auth" yaml:"auth"` } // AuthForURL returns the auth config that matches the given URL. -// Matches are based on URL prefix - the longest matching prefix wins. +// The longest matching URL scope wins. func (u *UpstreamConfig) AuthForURL(url string) *AuthConfig { if u.Auth == nil { return nil } + target, err := parseAuthURL(url) + if err != nil { + return nil + } var bestMatch *AuthConfig var bestLen int for pattern, auth := range u.Auth { - if strings.HasPrefix(url, pattern) && len(pattern) > bestLen { + configured, err := parseAuthURL(pattern) + if err == nil && authURLMatches(configured, target) && len(pattern) > bestLen { a := auth // copy to avoid loop variable capture bestMatch = &a bestLen = len(pattern) @@ -336,6 +342,55 @@ func (u *UpstreamConfig) AuthForURL(url string) *AuthConfig { return bestMatch } +// Validate checks upstream authentication URL scopes. +func (u *UpstreamConfig) Validate() error { + for pattern := range u.Auth { + if _, err := parseAuthURL(pattern); err != nil { + return fmt.Errorf("invalid upstream.auth URL %q: %w", pattern, err) + } + } + return nil +} + +func parseAuthURL(value string) (*url.URL, error) { + parsed, err := url.Parse(value) + if err != nil || !parsed.IsAbs() || parsed.Hostname() == "" || parsed.Opaque != "" { + return nil, fmt.Errorf("invalid authentication URL") + } + return parsed, nil +} + +func authURLMatches(configured, target *url.URL) bool { + if !strings.EqualFold(configured.Scheme, target.Scheme) || + !strings.EqualFold(configured.Hostname(), target.Hostname()) || + authURLPort(configured) != authURLPort(target) { + return false + } + if configured.RawQuery != "" && configured.RawQuery != target.RawQuery { + return false + } + + configuredPath := strings.TrimSuffix(configured.EscapedPath(), "/") + if configuredPath == "" { + return true + } + targetPath := strings.TrimSuffix(target.EscapedPath(), "/") + return targetPath == configuredPath || strings.HasPrefix(targetPath, configuredPath+"/") +} + +func authURLPort(value *url.URL) string { + if port := value.Port(); port != "" { + return port + } + if strings.EqualFold(value.Scheme, "https") { + return "443" + } + if strings.EqualFold(value.Scheme, "http") { + return "80" + } + return "" +} + // AuthConfig configures authentication for an upstream registry. type AuthConfig struct { // Type is the authentication type: "bearer", "basic", or "header". @@ -577,15 +632,19 @@ func (c *Config) Validate() error { return err } + return c.validateComponents() +} + +func (c *Config) validateComponents() error { + if err := c.Upstream.Validate(); err != nil { + return err + } + if err := c.Health.Validate(); err != nil { return err } - if err := c.Gradle.BuildCache.Validate(); err != nil { - return err - } - - return nil + return c.Gradle.BuildCache.Validate() } // Validate checks the /health configuration. An unset interval is allowed diff --git a/internal/config/config_test.go b/internal/config/config_test.go index ef6ac90..e4677c3 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -3,6 +3,7 @@ package config import ( "os" "path/filepath" + "strings" "testing" "time" ) @@ -795,3 +796,73 @@ func TestDatabaseConfigString(t *testing.T) { } } } + +func TestUpstreamAuthForURLMatchesURLComponents(t *testing.T) { + registryAuth := AuthConfig{Type: "bearer", Token: "registry-token"} + privateAuth := AuthConfig{Type: "bearer", Token: "private-token"} + config := UpstreamConfig{Auth: map[string]AuthConfig{ + "https://registry.example.com": registryAuth, + "https://registry.example.com/private": privateAuth, + }} + + tests := []struct { + name string + url string + wantToken string + }{ + {name: "registry root", url: "https://registry.example.com/package", wantToken: "registry-token"}, + {name: "host is case insensitive", url: "https://REGISTRY.EXAMPLE.COM/package", wantToken: "registry-token"}, + {name: "longest path match", url: "https://registry.example.com/private/package", wantToken: "private-token"}, + {name: "exact path match", url: "https://registry.example.com/private", wantToken: "private-token"}, + {name: "path segment boundary", url: "https://registry.example.com/private-other/package", wantToken: "registry-token"}, + {name: "lookalike host rejected", url: "https://registry.example.com.evil.test/package"}, + {name: "different scheme rejected", url: "http://registry.example.com/package"}, + {name: "different port rejected", url: "https://registry.example.com:8443/package"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + auth := config.AuthForURL(tt.url) + if tt.wantToken == "" { + if auth != nil { + t.Fatalf("AuthForURL() = %+v, want nil", auth) + } + return + } + if auth == nil { + t.Fatal("AuthForURL() = nil, want authentication") + } + if auth.Token != tt.wantToken { + t.Errorf("token = %q, want %q", auth.Token, tt.wantToken) + } + }) + } +} + +func TestValidateUpstreamAuthURLs(t *testing.T) { + t.Run("valid absolute URL", func(t *testing.T) { + cfg := Default() + cfg.Upstream.Auth = map[string]AuthConfig{ + "https://registry.example.com/private": {Type: "bearer", Token: "token"}, + } + + if err := cfg.Validate(); err != nil { + t.Fatalf("Validate() error = %v", err) + } + }) + + t.Run("invalid URL", func(t *testing.T) { + cfg := Default() + cfg.Upstream.Auth = map[string]AuthConfig{ + "registry.example.com": {Type: "bearer", Token: "token"}, + } + + err := cfg.Validate() + if err == nil { + t.Fatal("Validate() error = nil, want invalid upstream.auth URL error") + } + if !strings.Contains(err.Error(), "upstream.auth") || !strings.Contains(err.Error(), "registry.example.com") { + t.Errorf("Validate() error = %q, want field and URL", err) + } + }) +} diff --git a/internal/httpclient/transport.go b/internal/httpclient/transport.go new file mode 100644 index 0000000..d96c827 --- /dev/null +++ b/internal/httpclient/transport.go @@ -0,0 +1,433 @@ +// Package httpclient provides authentication-aware HTTP transports for upstream requests. +package httpclient + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "strings" + "sync" + "time" +) + +const ( + defaultTokenLifetime = 60 * time.Second + tokenExpirySkew = 5 * time.Second + maxTokenResponseSize = 1 << 20 + shortTokenSkewDivisor = 10 +) + +// AuthFunc returns a configured authentication header for a URL. +type AuthFunc func(url string) (headerName, headerValue string) + +// Transport adds configured authentication and follows OCI Bearer challenges. +type Transport struct { + base http.RoundTripper + authForURL AuthFunc + + mu sync.Mutex + tokens map[string]cachedToken + challenges map[string]bearerChallenge +} + +type cachedToken struct { + value string + expiresAt time.Time +} + +type bearerChallenge struct { + realm string + service string + scopes []string +} + +type tokenResponse struct { + Token string `json:"token"` + AccessToken string `json:"access_token"` + ExpiresIn int64 `json:"expires_in"` + IssuedAt string `json:"issued_at"` +} + +// NewTransport creates an authentication-aware transport around base. +func NewTransport(base http.RoundTripper, authForURL AuthFunc) *Transport { + if base == nil { + base = http.DefaultTransport + } + return &Transport{ + base: base, + authForURL: authForURL, + tokens: make(map[string]cachedToken), + challenges: make(map[string]bearerChallenge), + } +} + +// RoundTrip implements http.RoundTripper. +func (t *Transport) RoundTrip(req *http.Request) (*http.Response, error) { + hasExplicitAuthorization := req.Header.Get("Authorization") != "" + outbound := cloneRequest(req) + t.applyAuthentication(outbound, hasExplicitAuthorization) + + resp, err := t.base.RoundTrip(outbound) + if err != nil || resp.StatusCode != http.StatusUnauthorized { + return resp, err + } + if hasExplicitAuthorization { + return resp, nil + } + if registryProtectionSpace(req.URL) == "" { + return resp, nil + } + + challenge, ok := parseBearerChallenge(resp.Header.Values("WWW-Authenticate")) + if !ok || !canReplay(req) { + return resp, nil + } + + drainAndClose(resp.Body) + token, err := t.token(req.Context(), challenge) + if err != nil { + return nil, fmt.Errorf("registry authentication: %w", err) + } + t.rememberChallenge(req.URL, challenge) + + retry, err := cloneRequestForRetry(req) + if err != nil { + return nil, err + } + t.applyConfiguredAuthentication(retry) + retry.Header.Set("Authorization", "Bearer "+token) + return t.base.RoundTrip(retry) +} + +func (t *Transport) applyAuthentication(req *http.Request, hasExplicitAuthorization bool) { + t.applyConfiguredAuthentication(req) + if hasExplicitAuthorization { + return + } + if token := t.cachedTokenForRequest(req.URL); token != "" { + req.Header.Set("Authorization", "Bearer "+token) + } +} + +func (t *Transport) applyConfiguredAuthentication(req *http.Request) { + if t.authForURL == nil { + return + } + name, value := t.authForURL(req.URL.String()) + if name != "" && value != "" && req.Header.Get(name) == "" { + req.Header.Set(name, value) + } +} + +func (t *Transport) token(ctx context.Context, challenge bearerChallenge) (string, error) { + key := challenge.key() + if token := t.cachedToken(key); token != "" { + return token, nil + } + + token, expiresAt, err := t.fetchToken(ctx, challenge) + if err != nil { + return "", err + } + + t.cacheToken(key, cachedToken{value: token, expiresAt: expiresAt}) + return token, nil +} + +func (t *Transport) cacheToken(key string, token cachedToken) { + now := time.Now() + t.mu.Lock() + defer t.mu.Unlock() + + for cachedKey, cached := range t.tokens { + if !now.Before(cached.expiresAt) { + delete(t.tokens, cachedKey) + } + } + t.tokens[key] = token +} + +func (t *Transport) fetchToken(ctx context.Context, challenge bearerChallenge) (string, time.Time, error) { + tokenURL, err := url.Parse(challenge.realm) + if err != nil || !tokenURL.IsAbs() || (tokenURL.Scheme != "https" && tokenURL.Scheme != "http") { + return "", time.Time{}, fmt.Errorf("invalid token realm %q", challenge.realm) + } + + query := tokenURL.Query() + if challenge.service != "" { + query.Set("service", challenge.service) + } + for _, scope := range challenge.scopes { + query.Add("scope", scope) + } + query.Set("client_id", "git-pkgs-proxy") + tokenURL.RawQuery = query.Encode() + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, tokenURL.String(), nil) + if err != nil { + return "", time.Time{}, err + } + + client := &http.Client{Transport: configuredTransport{parent: t}} + resp, err := client.Do(req) + if err != nil { + return "", time.Time{}, fmt.Errorf("requesting token: %w", err) + } + defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices { + body, _ := io.ReadAll(io.LimitReader(resp.Body, maxTokenResponseSize)) + return "", time.Time{}, fmt.Errorf("token service returned %d: %s", resp.StatusCode, strings.TrimSpace(string(body))) + } + + var payload tokenResponse + if err := json.NewDecoder(io.LimitReader(resp.Body, maxTokenResponseSize)).Decode(&payload); err != nil { + return "", time.Time{}, fmt.Errorf("decoding token response: %w", err) + } + token := payload.Token + if token == "" { + token = payload.AccessToken + } + if token == "" { + return "", time.Time{}, fmt.Errorf("token response did not contain a token") + } + + issuedAt := time.Now() + if payload.IssuedAt != "" { + if parsed, parseErr := time.Parse(time.RFC3339, payload.IssuedAt); parseErr == nil { + issuedAt = parsed + } + } + lifetime := time.Duration(payload.ExpiresIn) * time.Second + if lifetime <= 0 { + lifetime = defaultTokenLifetime + } + expiresAt := issuedAt.Add(lifetime).Add(-expirySkew(lifetime)) + return token, expiresAt, nil +} + +type configuredTransport struct { + parent *Transport +} + +func (t configuredTransport) RoundTrip(req *http.Request) (*http.Response, error) { + outbound := cloneRequest(req) + t.parent.applyConfiguredAuthentication(outbound) + return t.parent.base.RoundTrip(outbound) +} + +func (t *Transport) cachedTokenForRequest(requestURL *url.URL) string { + space := registryProtectionSpace(requestURL) + if space == "" { + return "" + } + + t.mu.Lock() + challenge, ok := t.challenges[space] + t.mu.Unlock() + if !ok { + return "" + } + return t.cachedToken(challenge.key()) +} + +func (t *Transport) cachedToken(key string) string { + now := time.Now() + t.mu.Lock() + defer t.mu.Unlock() + + token, ok := t.tokens[key] + if !ok { + return "" + } + if !now.Before(token.expiresAt) { + delete(t.tokens, key) + return "" + } + return token.value +} + +func (t *Transport) rememberChallenge(requestURL *url.URL, challenge bearerChallenge) { + space := registryProtectionSpace(requestURL) + if space == "" { + return + } + t.mu.Lock() + t.challenges[space] = challenge + t.mu.Unlock() +} + +func (c bearerChallenge) key() string { + return c.realm + "\x00" + c.service + "\x00" + strings.Join(c.scopes, "\x00") +} + +func registryProtectionSpace(u *url.URL) string { + const registryPrefix = "/v2/" + if u == nil || !strings.HasPrefix(u.Path, registryPrefix) { + return "" + } + rest := strings.TrimPrefix(u.Path, registryPrefix) + end := len(rest) + for _, marker := range []string{"/blobs/", "/manifests/", "/tags/", "/referrers/"} { + if index := strings.Index(rest, marker); index >= 0 && index < end { + end = index + } + } + if end == len(rest) || end == 0 { + return "" + } + return u.Scheme + "://" + u.Host + registryPrefix + rest[:end] +} + +func parseBearerChallenge(values []string) (bearerChallenge, bool) { + for _, value := range values { + params, ok := bearerParameters(value) + if !ok || params["realm"] == "" { + continue + } + challenge := bearerChallenge{ + realm: params["realm"], + service: params["service"], + } + if scope := params["scope"]; scope != "" { + challenge.scopes = append(challenge.scopes, scope) + } + return challenge, true + } + return bearerChallenge{}, false +} + +func bearerParameters(value string) (map[string]string, bool) { + start := findAuthScheme(value, "Bearer") + if start < 0 { + return nil, false + } + rest := value[start+len("Bearer"):] + params := make(map[string]string) + for { + rest = strings.TrimLeft(rest, " \t,") + if rest == "" { + break + } + + keyEnd := strings.IndexAny(rest, "= \t,") + if keyEnd <= 0 { + break + } + key := strings.ToLower(rest[:keyEnd]) + rest = strings.TrimLeft(rest[keyEnd:], " \t") + if rest == "" || rest[0] != '=' { + break + } + rest = strings.TrimLeft(rest[1:], " \t") + + parsed, remaining, ok := parseAuthValue(rest) + if !ok { + return nil, false + } + params[key] = parsed + rest = remaining + } + return params, true +} + +func findAuthScheme(value, scheme string) int { + inQuote := false + escaped := false + for index := 0; index+len(scheme) <= len(value); index++ { + char := value[index] + if escaped { + escaped = false + continue + } + if char == '\\' && inQuote { + escaped = true + continue + } + if char == '"' { + inQuote = !inQuote + continue + } + if inQuote || !strings.EqualFold(value[index:index+len(scheme)], scheme) { + continue + } + beforeOK := index == 0 || value[index-1] == ',' || value[index-1] == ' ' || value[index-1] == '\t' + after := index + len(scheme) + afterOK := after < len(value) && (value[after] == ' ' || value[after] == '\t') + if beforeOK && afterOK { + return index + } + } + return -1 +} + +func parseAuthValue(value string) (parsed, remaining string, ok bool) { + if value == "" { + return "", "", false + } + if value[0] != '"' { + end := strings.IndexAny(value, " \t,") + if end < 0 { + return value, "", true + } + return value[:end], value[end:], end > 0 + } + + var builder strings.Builder + escaped := false + for index := 1; index < len(value); index++ { + char := value[index] + if escaped { + builder.WriteByte(char) + escaped = false + continue + } + if char == '\\' { + escaped = true + continue + } + if char == '"' { + return builder.String(), value[index+1:], true + } + builder.WriteByte(char) + } + return "", "", false +} + +func cloneRequest(req *http.Request) *http.Request { + clone := req.Clone(req.Context()) + clone.Header = req.Header.Clone() + return clone +} + +func canReplay(req *http.Request) bool { + return req.Body == nil || req.GetBody != nil +} + +func cloneRequestForRetry(req *http.Request) (*http.Request, error) { + clone := cloneRequest(req) + if req.Body == nil { + return clone, nil + } + body, err := req.GetBody() + if err != nil { + return nil, fmt.Errorf("replaying authenticated request: %w", err) + } + clone.Body = body + return clone, nil +} + +func expirySkew(lifetime time.Duration) time.Duration { + if lifetime < tokenExpirySkew*2 { + return lifetime / shortTokenSkewDivisor + } + return tokenExpirySkew +} + +func drainAndClose(body io.ReadCloser) { + _, _ = io.Copy(io.Discard, io.LimitReader(body, maxTokenResponseSize)) + _ = body.Close() +} diff --git a/internal/httpclient/transport_test.go b/internal/httpclient/transport_test.go new file mode 100644 index 0000000..bd1f266 --- /dev/null +++ b/internal/httpclient/transport_test.go @@ -0,0 +1,251 @@ +package httpclient + +import ( + "context" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" +) + +func TestTransportFollowsBearerChallengeAndCachesToken(t *testing.T) { + var registryRequests int + var tokenRequests int + var server *httptest.Server + + server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/token": + tokenRequests++ + if got := r.URL.Query().Get("service"); got != "registry.test" { + t.Errorf("service = %q, want %q", got, "registry.test") + } + if got := r.URL.Query().Get("scope"); got != "repository:library/test:pull" { + t.Errorf("scope = %q, want %q", got, "repository:library/test:pull") + } + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, `{"token":"registry-token","expires_in":3600}`) + case "/v2/library/test/blobs/sha256:first", "/v2/library/test/blobs/sha256:second": + registryRequests++ + if r.Header.Get("Authorization") != "Bearer registry-token" { + w.Header().Set("WWW-Authenticate", `Bearer realm="`+server.URL+`/token",service="registry.test",scope="repository:library/test:pull"`) + http.Error(w, "authentication required", http.StatusUnauthorized) + return + } + _, _ = io.WriteString(w, "blob") + default: + http.NotFound(w, r) + } + })) + defer server.Close() + + client := &http.Client{Transport: NewTransport(http.DefaultTransport, nil)} + for _, digest := range []string{"sha256:first", "sha256:second"} { + resp, err := client.Get(server.URL + "/v2/library/test/blobs/" + digest) + if err != nil { + t.Fatalf("GET %s: %v", digest, err) + } + body, readErr := io.ReadAll(resp.Body) + _ = resp.Body.Close() + if readErr != nil { + t.Fatalf("read %s response: %v", digest, readErr) + } + if resp.StatusCode != http.StatusOK { + t.Fatalf("GET %s status = %d, want %d", digest, resp.StatusCode, http.StatusOK) + } + if string(body) != "blob" { + t.Errorf("GET %s body = %q, want %q", digest, body, "blob") + } + } + + if tokenRequests != 1 { + t.Errorf("token requests = %d, want 1", tokenRequests) + } + if registryRequests != 3 { + t.Errorf("registry requests = %d, want 3", registryRequests) + } +} + +func TestTransportAddsConfiguredAuthentication(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if got := r.Header.Get("X-Registry-Token"); got != "configured-token" { + t.Errorf("X-Registry-Token = %q, want %q", got, "configured-token") + } + w.WriteHeader(http.StatusNoContent) + })) + defer server.Close() + + authForURL := func(url string) (string, string) { + if strings.HasPrefix(url, server.URL) { + return "X-Registry-Token", "configured-token" + } + return "", "" + } + client := &http.Client{Transport: NewTransport(http.DefaultTransport, authForURL)} + + resp, err := client.Get(server.URL + "/metadata") + if err != nil { + t.Fatalf("GET metadata: %v", err) + } + _ = resp.Body.Close() + if resp.StatusCode != http.StatusNoContent { + t.Errorf("status = %d, want %d", resp.StatusCode, http.StatusNoContent) + } +} + +func TestTransportPreservesExplicitAuthentication(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if got := r.Header.Get("Authorization"); got != "Bearer explicit-token" { + t.Errorf("Authorization = %q, want %q", got, "Bearer explicit-token") + } + w.WriteHeader(http.StatusNoContent) + })) + defer server.Close() + + authForURL := func(string) (string, string) { + return "Authorization", "Bearer configured-token" + } + client := &http.Client{Transport: NewTransport(http.DefaultTransport, authForURL)} + req, err := http.NewRequest(http.MethodGet, server.URL+"/artifact", nil) + if err != nil { + t.Fatal(err) + } + req.Header.Set("Authorization", "Bearer explicit-token") + + resp, err := client.Do(req) + if err != nil { + t.Fatalf("GET artifact: %v", err) + } + _ = resp.Body.Close() + if resp.StatusCode != http.StatusNoContent { + t.Errorf("status = %d, want %d", resp.StatusCode, http.StatusNoContent) + } +} + +func TestTransportDoesNotReplaceExplicitAuthenticationAfterBearerChallenge(t *testing.T) { + var registryRequests int + var tokenRequests int + var server *httptest.Server + + server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/token": + tokenRequests++ + _, _ = io.WriteString(w, `{"token":"registry-token"}`) + case "/v2/library/test/blobs/sha256:test": + registryRequests++ + if got := r.Header.Get("Authorization"); got != "Bearer explicit-token" { + t.Errorf("Authorization = %q, want %q", got, "Bearer explicit-token") + } + w.Header().Set("WWW-Authenticate", `Bearer realm="`+server.URL+`/token"`) + http.Error(w, "authentication required", http.StatusUnauthorized) + default: + http.NotFound(w, r) + } + })) + defer server.Close() + + client := &http.Client{Transport: NewTransport(http.DefaultTransport, nil)} + req, err := http.NewRequest(http.MethodGet, server.URL+"/v2/library/test/blobs/sha256:test", nil) + if err != nil { + t.Fatal(err) + } + req.Header.Set("Authorization", "Bearer explicit-token") + + resp, err := client.Do(req) + if err != nil { + t.Fatalf("GET blob: %v", err) + } + _ = resp.Body.Close() + if resp.StatusCode != http.StatusUnauthorized { + t.Errorf("status = %d, want %d", resp.StatusCode, http.StatusUnauthorized) + } + if registryRequests != 1 { + t.Errorf("registry requests = %d, want 1", registryRequests) + } + if tokenRequests != 0 { + t.Errorf("token requests = %d, want 0", tokenRequests) + } +} + +func TestTransportDoesNotForwardConfiguredAuthenticationOnTokenRedirect(t *testing.T) { + destination := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if got := r.Header.Get("X-Registry-Token"); got != "" { + t.Errorf("redirected X-Registry-Token = %q, want empty", got) + } + _, _ = io.WriteString(w, `{"token":"registry-token"}`) + })) + defer destination.Close() + + source := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if got := r.Header.Get("X-Registry-Token"); got != "configured-token" { + t.Errorf("source X-Registry-Token = %q, want %q", got, "configured-token") + } + http.Redirect(w, r, destination.URL+"/token", http.StatusFound) + })) + defer source.Close() + + authForURL := func(rawURL string) (string, string) { + if strings.HasPrefix(rawURL, source.URL) { + return "X-Registry-Token", "configured-token" + } + return "", "" + } + transport := NewTransport(http.DefaultTransport, authForURL) + token, _, err := transport.fetchToken(context.Background(), bearerChallenge{realm: source.URL + "/token"}) + if err != nil { + t.Fatalf("fetchToken: %v", err) + } + if token != "registry-token" { + t.Errorf("token = %q, want %q", token, "registry-token") + } +} + +func TestTransportPrunesExpiredTokens(t *testing.T) { + transport := NewTransport(http.DefaultTransport, nil) + transport.tokens["expired-unused"] = cachedToken{ + value: "expired-token", + expiresAt: time.Now().Add(-time.Minute), + } + transport.cacheToken("current", cachedToken{ + value: "current-token", + expiresAt: time.Now().Add(time.Minute), + }) + + if got := transport.cachedToken("current"); got != "current-token" { + t.Errorf("cachedToken(current) = %q, want %q", got, "current-token") + } + if _, ok := transport.tokens["expired-unused"]; ok { + t.Error("expired unused token was not pruned") + } +} + +func TestTransportDoesNotFollowBearerChallengeOutsideOCIRegistry(t *testing.T) { + tokenRequests := 0 + var server *httptest.Server + server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/token" { + tokenRequests++ + _, _ = io.WriteString(w, `{"token":"unexpected"}`) + return + } + w.Header().Set("WWW-Authenticate", `Bearer realm="`+server.URL+`/token"`) + http.Error(w, "authentication required", http.StatusUnauthorized) + })) + defer server.Close() + + client := &http.Client{Transport: NewTransport(http.DefaultTransport, nil)} + resp, err := client.Get(server.URL + "/api/packages") + if err != nil { + t.Fatalf("GET API: %v", err) + } + _ = resp.Body.Close() + if resp.StatusCode != http.StatusUnauthorized { + t.Errorf("status = %d, want %d", resp.StatusCode, http.StatusUnauthorized) + } + if tokenRequests != 0 { + t.Errorf("token requests = %d, want 0", tokenRequests) + } +} diff --git a/internal/server/server.go b/internal/server/server.go index 856fe2d..c98d1cb 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -58,17 +58,19 @@ import ( "strings" "time" + "github.com/git-pkgs/cooldown" swaggerdoc "github.com/git-pkgs/proxy/docs/swagger" "github.com/git-pkgs/proxy/internal/config" - "github.com/git-pkgs/cooldown" "github.com/git-pkgs/proxy/internal/database" "github.com/git-pkgs/proxy/internal/enrichment" "github.com/git-pkgs/proxy/internal/handler" + upstreamhttp "github.com/git-pkgs/proxy/internal/httpclient" "github.com/git-pkgs/proxy/internal/metrics" "github.com/git-pkgs/proxy/internal/mirror" "github.com/git-pkgs/proxy/internal/storage" "github.com/git-pkgs/purl" "github.com/git-pkgs/registries/fetch" + "github.com/git-pkgs/registries/safehttp" "github.com/git-pkgs/spdx" "github.com/go-chi/chi/v5" "github.com/go-chi/chi/v5/middleware" @@ -84,12 +86,12 @@ const ( // Server is the main proxy server. type Server struct { - cfg *config.Config - db *database.DB - storage storage.Storage - logger *slog.Logger - http *http.Server - templates *Templates + cfg *config.Config + db *database.DB + storage storage.Storage + logger *slog.Logger + http *http.Server + templates *Templates cancel context.CancelFunc healthCache *healthCache } @@ -156,8 +158,18 @@ func New(cfg *config.Config, logger *slog.Logger) (*Server, error) { // Start starts the HTTP server. func (s *Server) Start() error { - // Create shared components with circuit breaker - baseFetcher := fetch.NewFetcher(fetch.WithAuthFunc(s.authForURL)) + // Use one authentication-aware transport for metadata and artifacts so + // configured credentials and cached OCI challenges apply consistently. + safeClient := safehttp.New(nil, safehttp.Options{}) + authTransport := upstreamhttp.NewTransport(safeClient.Transport, upstreamhttp.AuthFunc(s.authForURL)) + metadataClient := *safeClient + metadataClient.Timeout = s.cfg.ParseHTTPTimeout() + metadataClient.Transport = authTransport + artifactClient := metadataClient + artifactClient.Timeout = serverWriteTimeout + + // Create shared components with circuit breaker. + baseFetcher := fetch.NewFetcher(fetch.WithHTTPClient(&artifactClient)) fetcher := fetch.NewCircuitBreakerFetcher(baseFetcher) resolver := fetch.NewResolver() cd := &cooldown.Config{ @@ -166,7 +178,7 @@ func (s *Server) Start() error { Packages: s.cfg.Cooldown.NormalizedPackages(), } proxy := handler.NewProxy(s.db, s.storage, fetcher, resolver, s.logger) - proxy.HTTPClient.Timeout = s.cfg.ParseHTTPTimeout() + proxy.HTTPClient = &metadataClient proxy.AuthForURL = s.authForURL proxy.Cooldown = cd proxy.CacheMetadata = s.cfg.CacheMetadata From 538a15d9f8950e401e7de09479b31a36ef141c52 Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Thu, 13 Aug 2026 07:35:07 +0100 Subject: [PATCH 043/633] fix(container): serve cached images when upstream is unavailable (#199) * container: cache manifests for offline pulls * Preserve direct-serve redirects for blob HEAD requests --- docs/architecture.md | 1 + docs/configuration.md | 2 + internal/database/metadata_cache_test.go | 55 +- internal/database/queries.go | 14 +- internal/database/schema.go | 19 + internal/database/types.go | 23 +- internal/handler/container.go | 131 +---- internal/handler/container_manifest.go | 251 +++++++++ internal/handler/container_test.go | 560 ++++++++++++++++--- internal/handler/handler.go | 39 +- internal/handler/handler_test.go | 6 +- internal/handler/notfound_ecosystems_test.go | 16 +- 12 files changed, 888 insertions(+), 229 deletions(-) create mode 100644 internal/handler/container_manifest.go diff --git a/docs/architecture.md b/docs/architecture.md index cf8b0e2..6d9bfda 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -353,6 +353,7 @@ Eviction can be implemented as: - Fresh data - new versions visible immediately - Metadata is small, upstream fetch is fast - Set `cache_metadata: true` or use the mirror command to enable metadata caching for offline use via the `metadata_cache` table +- OCI manifests are the exception: they are cached automatically so previously fetched images remain pullable when the registry or token service is unavailable **Why stream artifacts?** - Memory efficient - don't load large files into RAM diff --git a/docs/configuration.md b/docs/configuration.md index fa0f576..1e5a1c7 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -248,6 +248,8 @@ Note: Hex cooldown requires disabling registry signature verification since the By default the proxy fetches metadata fresh from upstream on every request. Enable `cache_metadata` to store metadata responses in the database and storage backend for offline fallback. When upstream is unreachable, the proxy serves the last cached copy. ETag-based revalidation avoids re-downloading unchanged metadata. +OCI manifests are always cached because cached image blobs cannot be pulled without their manifests. Digest-addressed manifests are immutable and served directly from cache. Tag-addressed manifests follow `metadata_ttl`, revalidate when stale, and fall back to the last cached response when the registry is unavailable. + ```yaml cache_metadata: true ``` diff --git a/internal/database/metadata_cache_test.go b/internal/database/metadata_cache_test.go index 5701816..09dcba3 100644 --- a/internal/database/metadata_cache_test.go +++ b/internal/database/metadata_cache_test.go @@ -30,8 +30,12 @@ func TestUpsertAndGetMetadataCache(t *testing.T) { StoragePath: "_metadata/npm/lodash/metadata", ETag: sql.NullString{String: `"abc123"`, Valid: true}, ContentType: sql.NullString{String: "application/json", Valid: true}, - Size: sql.NullInt64{Int64: 1024, Valid: true}, - FetchedAt: sql.NullTime{Time: time.Now(), Valid: true}, + ContentDigest: sql.NullString{ + String: "sha256:0123456789abcdef", + Valid: true, + }, + Size: sql.NullInt64{Int64: 1024, Valid: true}, + FetchedAt: sql.NullTime{Time: time.Now(), Valid: true}, } err := db.UpsertMetadataCache(entry) @@ -62,6 +66,9 @@ func TestUpsertAndGetMetadataCache(t *testing.T) { if !got.ContentType.Valid || got.ContentType.String != "application/json" { t.Errorf("content_type = %v, want %q", got.ContentType, "application/json") } + if !got.ContentDigest.Valid || got.ContentDigest.String != "sha256:0123456789abcdef" { + t.Errorf("content_digest = %v, want %q", got.ContentDigest, "sha256:0123456789abcdef") + } if !got.Size.Valid || got.Size.Int64 != 1024 { t.Errorf("size = %v, want 1024", got.Size) } @@ -178,3 +185,47 @@ func TestMetadataCacheTableCreatedByMigration(t *testing.T) { t.Error("metadata_cache table should exist after migration") } } + +func TestMetadataCacheContentDigestMigrationPreservesExistingRows(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "test.db") + db, err := Create(dbPath) + if err != nil { + t.Fatalf("Create failed: %v", err) + } + defer func() { _ = db.Close() }() + + if _, err := db.Exec("ALTER TABLE metadata_cache DROP COLUMN content_digest"); err != nil { + t.Fatalf("dropping content_digest: %v", err) + } + if _, err := db.Exec("DELETE FROM migrations WHERE name = ?", "006_add_metadata_content_digest"); err != nil { + t.Fatalf("resetting digest migration: %v", err) + } + if _, err := db.Exec(` + INSERT INTO metadata_cache (ecosystem, name, storage_path, content_type, size, fetched_at, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?) + `, "oci-manifest", "cache-key", "_metadata/oci-manifest/cache-key/metadata", "application/json", 2, time.Now(), time.Now(), time.Now()); err != nil { + t.Fatalf("inserting legacy cache row: %v", err) + } + + if err := db.MigrateSchema(); err != nil { + t.Fatalf("MigrateSchema() error = %v", err) + } + hasDigest, err := db.HasColumn("metadata_cache", "content_digest") + if err != nil { + t.Fatalf("HasColumn() error = %v", err) + } + if !hasDigest { + t.Fatal("metadata_cache.content_digest was not added") + } + + entry, err := db.GetMetadataCache("oci-manifest", "cache-key") + if err != nil { + t.Fatalf("GetMetadataCache() error = %v", err) + } + if entry == nil || entry.StoragePath != "_metadata/oci-manifest/cache-key/metadata" { + t.Fatalf("existing metadata cache row was not preserved: %#v", entry) + } + if entry.ContentDigest.Valid { + t.Errorf("legacy content digest = %q, want NULL", entry.ContentDigest.String) + } +} diff --git a/internal/database/queries.go b/internal/database/queries.go index 5d95596..f8b76fe 100644 --- a/internal/database/queries.go +++ b/internal/database/queries.go @@ -894,7 +894,7 @@ func (db *DB) GetMetadataCache(ecosystem, name string) (*MetadataCacheEntry, err var entry MetadataCacheEntry query := db.Rebind(` SELECT id, ecosystem, name, storage_path, etag, content_type, - size, last_modified, fetched_at, created_at, updated_at + content_digest, size, last_modified, fetched_at, created_at, updated_at FROM metadata_cache WHERE ecosystem = ? AND name = ? `) err := db.Get(&entry, query, ecosystem, name) @@ -914,12 +914,13 @@ func (db *DB) UpsertMetadataCache(entry *MetadataCacheEntry) error { if db.dialect == DialectPostgres { query = ` INSERT INTO metadata_cache (ecosystem, name, storage_path, etag, content_type, - size, last_modified, fetched_at, created_at, updated_at) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) + content_digest, size, last_modified, fetched_at, created_at, updated_at) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11) ON CONFLICT(ecosystem, name) DO UPDATE SET storage_path = EXCLUDED.storage_path, etag = EXCLUDED.etag, content_type = EXCLUDED.content_type, + content_digest = EXCLUDED.content_digest, size = EXCLUDED.size, last_modified = EXCLUDED.last_modified, fetched_at = EXCLUDED.fetched_at, @@ -928,12 +929,13 @@ func (db *DB) UpsertMetadataCache(entry *MetadataCacheEntry) error { } else { query = ` INSERT INTO metadata_cache (ecosystem, name, storage_path, etag, content_type, - size, last_modified, fetched_at, created_at, updated_at) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + content_digest, size, last_modified, fetched_at, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(ecosystem, name) DO UPDATE SET storage_path = excluded.storage_path, etag = excluded.etag, content_type = excluded.content_type, + content_digest = excluded.content_digest, size = excluded.size, last_modified = excluded.last_modified, fetched_at = excluded.fetched_at, @@ -943,7 +945,7 @@ func (db *DB) UpsertMetadataCache(entry *MetadataCacheEntry) error { _, err := db.Exec(query, entry.Ecosystem, entry.Name, entry.StoragePath, entry.ETag, - entry.ContentType, entry.Size, entry.LastModified, entry.FetchedAt, now, now, + entry.ContentType, entry.ContentDigest, entry.Size, entry.LastModified, entry.FetchedAt, now, now, ) if err != nil { return fmt.Errorf("upserting metadata cache: %w", err) diff --git a/internal/database/schema.go b/internal/database/schema.go index c8d8d1e..c73877d 100644 --- a/internal/database/schema.go +++ b/internal/database/schema.go @@ -102,6 +102,7 @@ CREATE TABLE IF NOT EXISTS metadata_cache ( storage_path TEXT NOT NULL, etag TEXT, content_type TEXT, + content_digest TEXT, size INTEGER, last_modified DATETIME, fetched_at DATETIME, @@ -202,6 +203,7 @@ CREATE TABLE IF NOT EXISTS metadata_cache ( storage_path TEXT NOT NULL, etag TEXT, content_type TEXT, + content_digest TEXT, size BIGINT, last_modified TIMESTAMP, fetched_at TIMESTAMP, @@ -359,6 +361,7 @@ var migrations = []migration{ {"003_ensure_artifacts_table", migrateEnsureArtifactsTable}, {"004_ensure_vulnerabilities_table", migrateEnsureVulnerabilitiesTable}, {"005_ensure_metadata_cache_table", migrateEnsureMetadataCacheTable}, + {"006_add_metadata_content_digest", migrateAddMetadataContentDigest}, } // isTableNotFound returns true if the error indicates a missing table. @@ -581,6 +584,20 @@ func migrateEnsureMetadataCacheTable(db *DB) error { return db.EnsureMetadataCacheTable() } +func migrateAddMetadataContentDigest(db *DB) error { + hasColumn, err := db.HasColumn("metadata_cache", "content_digest") + if err != nil { + return fmt.Errorf("checking metadata_cache content_digest column: %w", err) + } + if hasColumn { + return nil + } + if _, err := db.Exec("ALTER TABLE metadata_cache ADD COLUMN content_digest TEXT"); err != nil { + return fmt.Errorf("adding metadata_cache content_digest column: %w", err) + } + return nil +} + // EnsureMetadataCacheTable creates the metadata_cache table if it doesn't exist. func (db *DB) EnsureMetadataCacheTable() error { has, err := db.HasTable("metadata_cache") @@ -601,6 +618,7 @@ func (db *DB) EnsureMetadataCacheTable() error { storage_path TEXT NOT NULL, etag TEXT, content_type TEXT, + content_digest TEXT, size BIGINT, last_modified TIMESTAMP, fetched_at TIMESTAMP, @@ -618,6 +636,7 @@ func (db *DB) EnsureMetadataCacheTable() error { storage_path TEXT NOT NULL, etag TEXT, content_type TEXT, + content_digest TEXT, size INTEGER, last_modified DATETIME, fetched_at DATETIME, diff --git a/internal/database/types.go b/internal/database/types.go index 47dc47e..7128f12 100644 --- a/internal/database/types.go +++ b/internal/database/types.go @@ -78,17 +78,18 @@ func (a *Artifact) IsCached() bool { // MetadataCacheEntry represents a cached metadata blob for offline serving. type MetadataCacheEntry struct { - ID int64 `db:"id" json:"id"` - Ecosystem string `db:"ecosystem" json:"ecosystem"` - Name string `db:"name" json:"name"` - StoragePath string `db:"storage_path" json:"storage_path"` - ETag sql.NullString `db:"etag" json:"etag,omitempty"` - ContentType sql.NullString `db:"content_type" json:"content_type,omitempty"` - Size sql.NullInt64 `db:"size" json:"size,omitempty"` - LastModified sql.NullTime `db:"last_modified" json:"last_modified,omitempty"` - FetchedAt sql.NullTime `db:"fetched_at" json:"fetched_at,omitempty"` - CreatedAt time.Time `db:"created_at" json:"created_at"` - UpdatedAt time.Time `db:"updated_at" json:"updated_at"` + ID int64 `db:"id" json:"id"` + Ecosystem string `db:"ecosystem" json:"ecosystem"` + Name string `db:"name" json:"name"` + StoragePath string `db:"storage_path" json:"storage_path"` + ETag sql.NullString `db:"etag" json:"etag,omitempty"` + ContentType sql.NullString `db:"content_type" json:"content_type,omitempty"` + ContentDigest sql.NullString `db:"content_digest" json:"content_digest,omitempty"` + Size sql.NullInt64 `db:"size" json:"size,omitempty"` + LastModified sql.NullTime `db:"last_modified" json:"last_modified,omitempty"` + FetchedAt sql.NullTime `db:"fetched_at" json:"fetched_at,omitempty"` + CreatedAt time.Time `db:"created_at" json:"created_at"` + UpdatedAt time.Time `db:"updated_at" json:"updated_at"` } // Vulnerability represents a cached vulnerability record. diff --git a/internal/handler/container.go b/internal/handler/container.go index 0710ed1..3a3b267 100644 --- a/internal/handler/container.go +++ b/internal/handler/container.go @@ -12,7 +12,6 @@ import ( const ( dockerHubRegistry = "https://registry-1.docker.io" - dockerHubAuth = "https://auth.docker.io" blobMatchCount = 3 // full match + name + digest manifestMatchCount = 3 // full match + name + reference tagsListMatchCount = 2 // full match + name @@ -24,7 +23,6 @@ const ( type ContainerHandler struct { proxy *Proxy registryURL string - authURL string proxyURL string } @@ -33,7 +31,6 @@ func NewContainerHandler(proxy *Proxy, proxyURL string) *ContainerHandler { return &ContainerHandler{ proxy: proxy, registryURL: dockerHubRegistry, - authURL: dockerHubAuth, proxyURL: strings.TrimSuffix(proxyURL, "/"), } } @@ -90,31 +87,34 @@ func (h *ContainerHandler) handleBlobDownload(w http.ResponseWriter, r *http.Req h.proxy.Logger.Info("container blob request", "name", name, "digest", digest) - // Get auth token for upstream - token, err := h.getAuthToken(r.Context(), name, "pull") + filename := digest + cached, err := h.proxy.GetCachedArtifact(r.Context(), "oci", name, digest, filename) if err != nil { - h.proxy.Logger.Error("failed to get auth token", "error", err) - h.containerError(w, http.StatusUnauthorized, "UNAUTHORIZED", "failed to authenticate") + h.proxy.Logger.Error("failed to check blob cache", "error", err) + h.containerError(w, http.StatusInternalServerError, "INTERNAL_ERROR", "failed to check blob cache") + return + } + if cached != nil { + w.Header().Set("Docker-Content-Digest", digest) + w.Header().Set("Content-Type", "application/octet-stream") + serveArtifact(w, r.Method, cached) return } // For HEAD requests, just proxy to upstream if r.Method == http.MethodHead { - h.proxyBlobHead(w, r, name, digest, token) + h.proxyBlobHead(w, r, name, digest) return } - // Try to get from cache, or fetch from upstream with auth - filename := digest - headers := http.Header{"Authorization": {"Bearer " + token}} - result, err := h.proxy.GetOrFetchArtifactFromURLWithHeaders( + // Try to get from cache, or fetch from the authentication-aware upstream client. + result, err := h.proxy.GetOrFetchArtifactFromURL( r.Context(), "oci", name, digest, // use digest as version filename, fmt.Sprintf("%s/v2/%s/blobs/%s", h.registryURL, name, digest), - headers, ) if err != nil { @@ -132,8 +132,7 @@ func (h *ContainerHandler) handleBlobDownload(w http.ResponseWriter, r *http.Req ServeArtifact(w, result) } -// handleManifest proxies manifest requests to upstream. -// Manifests change when tags are updated, so we proxy these directly. +// handleManifest serves immutable manifests from cache and revalidates mutable tags. // Path format: {name}/manifests/{reference} func (h *ContainerHandler) handleManifest(w http.ResponseWriter, r *http.Request, path string) { if r.Method != http.MethodGet && r.Method != http.MethodHead { @@ -148,57 +147,7 @@ func (h *ContainerHandler) handleManifest(w http.ResponseWriter, r *http.Request } h.proxy.Logger.Info("container manifest request", "name", name, "reference", reference) - - // Get auth token - token, err := h.getAuthToken(r.Context(), name, "pull") - if err != nil { - h.proxy.Logger.Error("failed to get auth token", "error", err) - h.containerError(w, http.StatusUnauthorized, "UNAUTHORIZED", "failed to authenticate") - return - } - - // Proxy to upstream - upstreamURL := fmt.Sprintf("%s/v2/%s/manifests/%s", h.registryURL, name, reference) - - req, err := http.NewRequestWithContext(r.Context(), r.Method, upstreamURL, nil) - if err != nil { - h.containerError(w, http.StatusInternalServerError, "INTERNAL_ERROR", "failed to create request") - return - } - - req.Header.Set("Authorization", "Bearer "+token) - - // Forward Accept header for content negotiation - if accept := r.Header.Get("Accept"); accept != "" { - req.Header.Set("Accept", accept) - } else { - // Default accept headers for manifests - req.Header.Set("Accept", strings.Join([]string{ - "application/vnd.oci.image.manifest.v1+json", - "application/vnd.oci.image.index.v1+json", - "application/vnd.docker.distribution.manifest.v2+json", - "application/vnd.docker.distribution.manifest.list.v2+json", - "application/vnd.docker.distribution.manifest.v1+prettyjws", - }, ", ")) - } - - resp, err := h.proxy.HTTPClient.Do(req) - if err != nil { - h.proxy.Logger.Error("failed to fetch manifest", "error", err) - h.containerError(w, http.StatusBadGateway, "INTERNAL_ERROR", "failed to fetch from upstream") - return - } - defer func() { _ = resp.Body.Close() }() - - // Copy relevant headers - for _, header := range []string{"Content-Type", "Content-Length", "Docker-Content-Digest", "ETag"} { - if v := resp.Header.Get(header); v != "" { - w.Header().Set(header, v) - } - } - - w.WriteHeader(resp.StatusCode) - _, _ = io.Copy(w, resp.Body) + h.serveManifest(w, r, name, reference) } // handleTagsList proxies tag list requests to upstream. @@ -214,13 +163,6 @@ func (h *ContainerHandler) handleTagsList(w http.ResponseWriter, r *http.Request return } - // Get auth token - token, err := h.getAuthToken(r.Context(), name, "pull") - if err != nil { - h.containerError(w, http.StatusUnauthorized, "UNAUTHORIZED", "failed to authenticate") - return - } - upstreamURL := fmt.Sprintf("%s/v2/%s/tags/list", h.registryURL, name) if r.URL.RawQuery != "" { upstreamURL += "?" + r.URL.RawQuery @@ -232,8 +174,6 @@ func (h *ContainerHandler) handleTagsList(w http.ResponseWriter, r *http.Request return } - req.Header.Set("Authorization", "Bearer "+token) - resp, err := h.proxy.HTTPClient.Do(req) if err != nil { h.containerError(w, http.StatusBadGateway, "INTERNAL_ERROR", "failed to fetch from upstream") @@ -246,45 +186,8 @@ func (h *ContainerHandler) handleTagsList(w http.ResponseWriter, r *http.Request _, _ = io.Copy(w, resp.Body) } -// getAuthToken gets a bearer token for the specified repository. -// Docker Hub requires auth even for public images. -func (h *ContainerHandler) getAuthToken(_ interface{ Done() <-chan struct{} }, repository, action string) (string, error) { - // For Docker Hub: https://auth.docker.io/token?service=registry.docker.io&scope=repository:{repo}:pull - authURL := fmt.Sprintf("%s/token?service=registry.docker.io&scope=repository:%s:%s", - h.authURL, repository, action) - - req, err := http.NewRequest(http.MethodGet, authURL, nil) - if err != nil { - return "", err - } - - resp, err := h.proxy.HTTPClient.Do(req) - if err != nil { - return "", err - } - defer func() { _ = resp.Body.Close() }() - - if resp.StatusCode != http.StatusOK { - return "", fmt.Errorf("auth failed with status %d", resp.StatusCode) - } - - var tokenResp struct { - Token string `json:"token"` - AccessToken string `json:"access_token"` - } - - if err := json.NewDecoder(resp.Body).Decode(&tokenResp); err != nil { - return "", err - } - - if tokenResp.Token != "" { - return tokenResp.Token, nil - } - return tokenResp.AccessToken, nil -} - // proxyBlobHead handles HEAD requests for blobs. -func (h *ContainerHandler) proxyBlobHead(w http.ResponseWriter, r *http.Request, name, digest, token string) { +func (h *ContainerHandler) proxyBlobHead(w http.ResponseWriter, r *http.Request, name, digest string) { upstreamURL := fmt.Sprintf("%s/v2/%s/blobs/%s", h.registryURL, name, digest) req, err := http.NewRequestWithContext(r.Context(), http.MethodHead, upstreamURL, nil) @@ -293,8 +196,6 @@ func (h *ContainerHandler) proxyBlobHead(w http.ResponseWriter, r *http.Request, return } - req.Header.Set("Authorization", "Bearer "+token) - resp, err := h.proxy.HTTPClient.Do(req) if err != nil { h.containerError(w, http.StatusBadGateway, "INTERNAL_ERROR", "failed to fetch from upstream") diff --git a/internal/handler/container_manifest.go b/internal/handler/container_manifest.go new file mode 100644 index 0000000..245ced4 --- /dev/null +++ b/internal/handler/container_manifest.go @@ -0,0 +1,251 @@ +package handler + +import ( + "bytes" + "context" + "crypto/sha256" + "database/sql" + "encoding/hex" + "fmt" + "io" + "net/http" + "regexp" + "strconv" + "strings" + "time" + + "github.com/git-pkgs/proxy/internal/database" +) + +const ( + containerManifestCacheEcosystem = "oci-manifest" + containerStaleWarning = `110 - "Response is Stale"` +) + +var manifestDigestReferencePattern = regexp.MustCompile(`^[a-z0-9]+:[a-f0-9]+$`) + +type cachedContainerManifest struct { + body []byte + contentType string + contentDigest string + etag string + size int64 + fetchedAt time.Time +} + +func (h *ContainerHandler) serveManifest(w http.ResponseWriter, r *http.Request, name, reference string) { + accept := containerManifestAccept(r) + cacheKey := h.containerManifestCacheKey(name, reference, accept) + cached, err := h.loadContainerManifest(r.Context(), cacheKey) + if err != nil { + h.proxy.Logger.Warn("failed to read cached container manifest", "error", err) + cached = nil + } + + immutable := manifestDigestReferencePattern.MatchString(reference) + if cached != nil && (immutable || h.containerManifestFresh(cached)) { + writeContainerManifest(w, r.Method, cached, false) + return + } + + upstreamURL := fmt.Sprintf("%s/v2/%s/manifests/%s", h.registryURL, name, reference) + req, err := http.NewRequestWithContext(r.Context(), r.Method, upstreamURL, nil) + if err != nil { + h.containerError(w, http.StatusInternalServerError, "INTERNAL_ERROR", "failed to create request") + return + } + req.Header.Set("Accept", accept) + if cached != nil && cached.etag != "" { + req.Header.Set("If-None-Match", cached.etag) + } + + resp, err := h.proxy.HTTPClient.Do(req) + if err != nil { + h.serveStaleManifestOrError(w, r, cached, err) + return + } + defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode == http.StatusNotModified && cached != nil { + cached.fetchedAt = time.Now() + if err := h.storeContainerManifest(r.Context(), cacheKey, cached); err != nil { + h.proxy.Logger.Warn("failed to refresh cached container manifest", "error", err) + } + writeContainerManifest(w, r.Method, cached, false) + return + } + if resp.StatusCode != http.StatusOK { + if cached != nil && shouldServeStaleManifest(resp.StatusCode) { + writeContainerManifest(w, r.Method, cached, true) + return + } + copyContainerManifestHeaders(w.Header(), resp.Header) + w.WriteHeader(resp.StatusCode) + _, _ = io.Copy(w, resp.Body) + return + } + + if r.Method == http.MethodHead { + copyContainerManifestHeaders(w.Header(), resp.Header) + w.WriteHeader(http.StatusOK) + return + } + + body, err := h.proxy.ReadMetadata(resp.Body) + if err != nil { + h.serveStaleManifestOrError(w, r, cached, fmt.Errorf("reading manifest: %w", err)) + return + } + manifest := &cachedContainerManifest{ + body: body, + contentType: resp.Header.Get("Content-Type"), + contentDigest: resp.Header.Get("Docker-Content-Digest"), + etag: resp.Header.Get("ETag"), + size: int64(len(body)), + fetchedAt: time.Now(), + } + if manifest.contentDigest == "" { + manifest.contentDigest = sha256Digest(body) + } + if err := h.storeContainerManifest(r.Context(), cacheKey, manifest); err != nil { + h.proxy.Logger.Warn("failed to cache container manifest", "error", err) + } + if manifest.contentDigest != reference && manifestDigestReferencePattern.MatchString(manifest.contentDigest) { + digestKey := h.containerManifestCacheKey(name, manifest.contentDigest, accept) + if err := h.storeContainerManifest(r.Context(), digestKey, manifest); err != nil { + h.proxy.Logger.Warn("failed to cache container manifest by digest", "error", err) + } + } + writeContainerManifest(w, r.Method, manifest, false) +} + +func (h *ContainerHandler) serveStaleManifestOrError(w http.ResponseWriter, r *http.Request, cached *cachedContainerManifest, err error) { + if cached != nil { + h.proxy.Logger.Warn("upstream manifest fetch failed, serving stale cache", "error", err) + writeContainerManifest(w, r.Method, cached, true) + return + } + h.proxy.Logger.Error("failed to fetch manifest", "error", err) + h.containerError(w, http.StatusBadGateway, "INTERNAL_ERROR", "failed to fetch from upstream") +} + +func (h *ContainerHandler) containerManifestFresh(manifest *cachedContainerManifest) bool { + return h.proxy.MetadataTTL > 0 && !manifest.fetchedAt.IsZero() && time.Since(manifest.fetchedAt) < h.proxy.MetadataTTL +} + +func (h *ContainerHandler) containerManifestCacheKey(name, reference, accept string) string { + identity := strings.Join([]string{h.registryURL, name, reference, accept}, "\x00") + sum := sha256.Sum256([]byte(identity)) + return hex.EncodeToString(sum[:]) +} + +func (h *ContainerHandler) loadContainerManifest(ctx context.Context, cacheKey string) (*cachedContainerManifest, error) { + if h.proxy.DB == nil || h.proxy.Storage == nil { + return nil, nil + } + entry, err := h.proxy.DB.GetMetadataCache(containerManifestCacheEcosystem, cacheKey) + if err != nil || entry == nil { + return nil, err + } + reader, err := h.proxy.Storage.Open(ctx, entry.StoragePath) + if err != nil { + return nil, nil + } + defer func() { _ = reader.Close() }() + body, err := h.proxy.ReadMetadata(reader) + if err != nil { + return nil, err + } + + manifest := &cachedContainerManifest{body: body, size: int64(len(body))} + if entry.ContentType.Valid { + manifest.contentType = entry.ContentType.String + } + if entry.ContentDigest.Valid { + manifest.contentDigest = entry.ContentDigest.String + } else { + manifest.contentDigest = sha256Digest(body) + } + if entry.ETag.Valid { + manifest.etag = entry.ETag.String + } + if entry.Size.Valid { + manifest.size = entry.Size.Int64 + } + if entry.FetchedAt.Valid { + manifest.fetchedAt = entry.FetchedAt.Time + } + return manifest, nil +} + +func (h *ContainerHandler) storeContainerManifest(ctx context.Context, cacheKey string, manifest *cachedContainerManifest) error { + if h.proxy.DB == nil || h.proxy.Storage == nil { + return nil + } + storagePath := metadataStoragePath(containerManifestCacheEcosystem, cacheKey) + size, _, err := h.proxy.Storage.Store(ctx, storagePath, bytes.NewReader(manifest.body)) + if err != nil { + return fmt.Errorf("storing manifest: %w", err) + } + manifest.size = size + return h.proxy.DB.UpsertMetadataCache(&database.MetadataCacheEntry{ + Ecosystem: containerManifestCacheEcosystem, + Name: cacheKey, + StoragePath: storagePath, + ETag: sql.NullString{String: manifest.etag, Valid: manifest.etag != ""}, + ContentType: sql.NullString{String: manifest.contentType, Valid: manifest.contentType != ""}, + ContentDigest: sql.NullString{String: manifest.contentDigest, Valid: manifest.contentDigest != ""}, + Size: sql.NullInt64{Int64: size, Valid: true}, + FetchedAt: sql.NullTime{Time: manifest.fetchedAt, Valid: !manifest.fetchedAt.IsZero()}, + }) +} + +func writeContainerManifest(w http.ResponseWriter, method string, manifest *cachedContainerManifest, stale bool) { + if manifest.contentType != "" { + w.Header().Set("Content-Type", manifest.contentType) + } + w.Header().Set("Content-Length", strconv.FormatInt(manifest.size, 10)) + if manifest.contentDigest != "" { + w.Header().Set("Docker-Content-Digest", manifest.contentDigest) + } + if manifest.etag != "" { + w.Header().Set("ETag", manifest.etag) + } + if stale { + w.Header().Set("Warning", containerStaleWarning) + } + w.WriteHeader(http.StatusOK) + if method != http.MethodHead { + _, _ = w.Write(manifest.body) + } +} + +func containerManifestAccept(r *http.Request) string { + if accept := r.Header.Get("Accept"); accept != "" { + return accept + } + return strings.Join([]string{ + "application/vnd.oci.image.manifest.v1+json", + "application/vnd.oci.image.index.v1+json", + "application/vnd.docker.distribution.manifest.v2+json", + "application/vnd.docker.distribution.manifest.list.v2+json", + "application/vnd.docker.distribution.manifest.v1+prettyjws", + }, ", ") +} + +func copyContainerManifestHeaders(destination, source http.Header) { + for _, header := range []string{"Content-Type", "Content-Length", "Docker-Content-Digest", "ETag", "WWW-Authenticate"} { + if value := source.Get(header); value != "" { + destination.Set(header, value) + } + } +} + +func shouldServeStaleManifest(status int) bool { + return status == http.StatusTooManyRequests || status >= http.StatusInternalServerError +} + +func sha256Digest(body []byte) string { + digest := sha256.Sum256(body) + return "sha256:" + hex.EncodeToString(digest[:]) +} diff --git a/internal/handler/container_test.go b/internal/handler/container_test.go index 853059e..6e7322c 100644 --- a/internal/handler/container_test.go +++ b/internal/handler/container_test.go @@ -1,16 +1,15 @@ package handler import ( - "bytes" - "context" "encoding/json" "io" - "log/slog" "net/http" "net/http/httptest" + "strconv" "testing" + "time" - "github.com/git-pkgs/proxy/internal/database" + upstreamhttp "github.com/git-pkgs/proxy/internal/httpclient" "github.com/git-pkgs/registries/fetch" ) @@ -135,90 +134,521 @@ func TestContainerHandler_parseTagsListPath(t *testing.T) { } } -func TestContainerHandler_BlobDownload_CachesWithAuth(t *testing.T) { - // Set up a mock auth server that returns a token - authServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Content-Type", "application/json") - _ = json.NewEncoder(w).Encode(map[string]string{"token": "test-token-123"}) +func TestContainerHandler_BlobDownload_DiscoversBearerChallenge(t *testing.T) { + digest := "sha256:abc123def456abc123def456abc123def456abc123def456abc123def456abcd" + registryRequests := 0 + tokenRequests := 0 + var upstream *httptest.Server + upstream = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/token": + tokenRequests++ + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{ + "token": "discovered-token", + "expires_in": 3600, + }) + case "/v2/library/nginx/blobs/" + digest: + registryRequests++ + if r.Header.Get("Authorization") != "Bearer discovered-token" { + w.Header().Set("WWW-Authenticate", `Bearer realm="`+upstream.URL+`/token",service="registry.test",scope="repository:library/nginx:pull"`) + http.Error(w, "authentication required", http.StatusUnauthorized) + return + } + w.Header().Set("Content-Type", "application/octet-stream") + _, _ = io.WriteString(w, "upstream blob") + default: + http.NotFound(w, r) + } })) - defer authServer.Close() + defer upstream.Close() - // Set up mock fetcher that captures headers - var capturedHeaders http.Header - mf := &mockFetcherWithHeaders{ - fetchFn: func(_ context.Context, _ string, headers http.Header) (*fetch.Artifact, error) { - capturedHeaders = headers - return &fetch.Artifact{ - Body: io.NopCloser(bytes.NewReader([]byte("blob-content"))), - Size: 12, - ContentType: "application/octet-stream", - }, nil - }, - } - - dir := t.TempDir() - db, err := database.Create(dir + "/test.db") - if err != nil { - t.Fatalf("failed to create test database: %v", err) - } - t.Cleanup(func() { _ = db.Close() }) - - store := newMockStorage() - logger := slog.New(slog.NewTextHandler(io.Discard, nil)) - proxy := &Proxy{ - DB: db, - Storage: store, - Fetcher: mf, - Logger: logger, - HTTPClient: &http.Client{}, - } + proxy, _, _, _ := setupTestProxy(t) + authTransport := upstreamhttp.NewTransport(http.DefaultTransport, nil) + client := &http.Client{Transport: authTransport} + artifactFetcher := fetch.NewFetcher( + fetch.WithHTTPClient(client), + fetch.WithMaxRetries(0), + ) + t.Cleanup(func() { _ = artifactFetcher.Close() }) + proxy.Fetcher = artifactFetcher + proxy.HTTPClient = client h := &ContainerHandler{ proxy: proxy, - registryURL: "https://registry-1.docker.io", - authURL: authServer.URL, + registryURL: upstream.URL, proxyURL: "http://localhost:8080", } - handler := h.Routes() - req := httptest.NewRequest(http.MethodGet, "/library/nginx/blobs/sha256:abc123def456abc123def456abc123def456abc123def456abc123def456abcd", nil) + for range 2 { + req := httptest.NewRequest(http.MethodGet, "/library/nginx/blobs/"+digest, nil) + w := httptest.NewRecorder() + h.Routes().ServeHTTP(w, req) + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want %d; body: %s", w.Code, http.StatusOK, w.Body.String()) + } + if got := w.Body.String(); got != "upstream blob" { + t.Errorf("body = %q, want %q", got, "upstream blob") + } + } + + if tokenRequests != 1 { + t.Errorf("token requests = %d, want 1", tokenRequests) + } + if registryRequests != 2 { + t.Errorf("registry requests = %d, want 2", registryRequests) + } +} + +func TestContainerHandler_CachedImagePullSurvivesRegistryAndTokenOutages(t *testing.T) { + digest := "sha256:abc123def456abc123def456abc123def456abc123def456abc123def456abcd" + manifest := `{"schemaVersion":2,"mediaType":"application/vnd.oci.image.manifest.v1+json"}` + blob := "cached image blob" + registryAvailable := true + tokenAvailable := true + registryRequests := 0 + tokenRequests := 0 + + tokenServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + tokenRequests++ + if !tokenAvailable { + http.Error(w, "token service unavailable", http.StatusServiceUnavailable) + return + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{ + "token": "discovered-token", + "expires_in": 3600, + }) + })) + defer tokenServer.Close() + + registryServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + registryRequests++ + if !registryAvailable { + http.Error(w, "registry unavailable", http.StatusServiceUnavailable) + return + } + if r.Header.Get("Authorization") != "Bearer discovered-token" { + w.Header().Set("WWW-Authenticate", `Bearer realm="`+tokenServer.URL+`",service="registry.test",scope="repository:library/nginx:pull"`) + http.Error(w, "authentication required", http.StatusUnauthorized) + return + } + + switch r.URL.Path { + case "/v2/library/nginx/manifests/latest": + w.Header().Set("Content-Type", "application/vnd.oci.image.manifest.v1+json") + w.Header().Set("Docker-Content-Digest", digest) + _, _ = io.WriteString(w, manifest) + case "/v2/library/nginx/blobs/" + digest: + w.Header().Set("Content-Type", "application/octet-stream") + _, _ = io.WriteString(w, blob) + default: + http.NotFound(w, r) + } + })) + defer registryServer.Close() + + warmProxy, db, store, _ := setupTestProxy(t) + warmClient := &http.Client{Transport: upstreamhttp.NewTransport(http.DefaultTransport, nil)} + warmFetcher := fetch.NewFetcher( + fetch.WithHTTPClient(warmClient), + fetch.WithMaxRetries(0), + ) + t.Cleanup(func() { _ = warmFetcher.Close() }) + warmProxy.Fetcher = warmFetcher + warmProxy.HTTPClient = warmClient + warmProxy.MetadataTTL = time.Hour + warmHandler := (&ContainerHandler{ + proxy: warmProxy, + registryURL: registryServer.URL, + proxyURL: "http://localhost:8080", + }).Routes() + + for _, request := range []struct { + path string + body string + }{ + {path: "/library/nginx/manifests/latest", body: manifest}, + {path: "/library/nginx/blobs/" + digest, body: blob}, + } { + response := httptest.NewRecorder() + warmHandler.ServeHTTP(response, httptest.NewRequest(http.MethodGet, request.path, nil)) + if response.Code != http.StatusOK { + t.Fatalf("warming %s: status = %d, want %d; body: %s", request.path, response.Code, http.StatusOK, response.Body.String()) + } + if got := response.Body.String(); got != request.body { + t.Fatalf("warming %s: body = %q, want %q", request.path, got, request.body) + } + } + + warmRegistryRequests := registryRequests + warmTokenRequests := tokenRequests + registryAvailable = false + tokenAvailable = false + + offlineClient := &http.Client{Transport: upstreamhttp.NewTransport(http.DefaultTransport, nil)} + offlineFetcher := fetch.NewFetcher( + fetch.WithHTTPClient(offlineClient), + fetch.WithMaxRetries(0), + ) + t.Cleanup(func() { _ = offlineFetcher.Close() }) + offlineProxy := NewProxy(db, store, offlineFetcher, fetch.NewResolver(), warmProxy.Logger) + offlineProxy.HTTPClient = offlineClient + offlineProxy.MetadataTTL = time.Hour + offlineHandler := (&ContainerHandler{ + proxy: offlineProxy, + registryURL: registryServer.URL, + proxyURL: "http://localhost:8080", + }).Routes() + + for _, request := range []struct { + name string + path string + body string + }{ + {name: "tag manifest", path: "/library/nginx/manifests/latest", body: manifest}, + {name: "digest manifest", path: "/library/nginx/manifests/" + digest, body: manifest}, + {name: "blob", path: "/library/nginx/blobs/" + digest, body: blob}, + } { + t.Run(request.name, func(t *testing.T) { + response := httptest.NewRecorder() + offlineHandler.ServeHTTP(response, httptest.NewRequest(http.MethodGet, request.path, nil)) + if response.Code != http.StatusOK { + t.Fatalf("status = %d, want %d; body: %s", response.Code, http.StatusOK, response.Body.String()) + } + if got := response.Body.String(); got != request.body { + t.Errorf("body = %q, want %q", got, request.body) + } + if got := response.Header().Get("Docker-Content-Digest"); got != digest { + t.Errorf("Docker-Content-Digest = %q, want %q", got, digest) + } + }) + } + + if registryRequests != warmRegistryRequests { + t.Errorf("offline registry requests = %d, want 0", registryRequests-warmRegistryRequests) + } + if tokenRequests != warmTokenRequests { + t.Errorf("offline token requests = %d, want 0", tokenRequests-warmTokenRequests) + } +} + +func TestContainerHandler_BlobDownload_CacheHitSkipsAuth(t *testing.T) { + proxy, db, store, fetcher := setupTestProxy(t) + digest := "sha256:abc123def456abc123def456abc123def456abc123def456abc123def456abcd" + seedPackage(t, db, store, "oci", "library/nginx", digest, digest, "cached blob") + + upstreamRequests := 0 + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + upstreamRequests++ + http.Error(w, "upstream unavailable", http.StatusServiceUnavailable) + })) + defer upstream.Close() + + h := &ContainerHandler{ + proxy: proxy, + registryURL: upstream.URL, + proxyURL: "http://localhost:8080", + } + + req := httptest.NewRequest(http.MethodGet, "/library/nginx/blobs/"+digest, nil) w := httptest.NewRecorder() - handler.ServeHTTP(w, req) + h.Routes().ServeHTTP(w, req) if w.Code != http.StatusOK { - t.Errorf("got status %d, want %d; body: %s", w.Code, http.StatusOK, w.Body.String()) + t.Fatalf("status = %d, want %d; body: %s", w.Code, http.StatusOK, w.Body.String()) } - - // Verify auth header was passed to the fetcher - if capturedHeaders == nil { - t.Fatal("expected headers to be passed to fetcher, got nil") + if got := w.Body.String(); got != "cached blob" { + t.Errorf("body = %q, want %q", got, "cached blob") } - auth := capturedHeaders.Get("Authorization") - if auth != "Bearer test-token-123" { - t.Errorf("Authorization = %q, want %q", auth, "Bearer test-token-123") + if upstreamRequests != 0 { + t.Errorf("upstream requests = %d, want 0", upstreamRequests) } - - // Verify response headers - if got := w.Header().Get("Docker-Content-Digest"); got != "sha256:abc123def456abc123def456abc123def456abc123def456abc123def456abcd" { - t.Errorf("Docker-Content-Digest = %q, want digest", got) + if fetcher.fetchCalled { + t.Error("fetcher should not be called on cache hit") } } -// mockFetcherWithHeaders captures headers passed to FetchWithHeaders. -type mockFetcherWithHeaders struct { - fetchFn func(ctx context.Context, url string, headers http.Header) (*fetch.Artifact, error) +func TestContainerHandler_BlobHead_CacheHitSkipsUpstreamAndAuth(t *testing.T) { + proxy, db, store, fetcher := setupTestProxy(t) + digest := "sha256:abc123def456abc123def456abc123def456abc123def456abc123def456abcd" + seedPackage(t, db, store, "oci", "library/nginx", digest, digest, "cached blob") + + upstreamRequests := 0 + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + upstreamRequests++ + http.Error(w, "upstream unavailable", http.StatusServiceUnavailable) + })) + defer upstream.Close() + proxy.HTTPClient = upstream.Client() + + h := &ContainerHandler{ + proxy: proxy, + registryURL: upstream.URL, + proxyURL: "http://localhost:8080", + } + + req := httptest.NewRequest(http.MethodHead, "/library/nginx/blobs/"+digest, nil) + w := httptest.NewRecorder() + h.Routes().ServeHTTP(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want %d; body: %s", w.Code, http.StatusOK, w.Body.String()) + } + if got := w.Header().Get("Docker-Content-Digest"); got != digest { + t.Errorf("Docker-Content-Digest = %q, want %q", got, digest) + } + if got := w.Header().Get("Content-Length"); got != "11" { + t.Errorf("Content-Length = %q, want %q", got, "11") + } + if w.Body.Len() != 0 { + t.Errorf("HEAD response body length = %d, want 0", w.Body.Len()) + } + if upstreamRequests != 0 { + t.Errorf("upstream requests = %d, want 0", upstreamRequests) + } + if fetcher.fetchCalled { + t.Error("fetcher should not be called on cache hit") + } } -func (f *mockFetcherWithHeaders) Fetch(ctx context.Context, url string) (*fetch.Artifact, error) { - return f.FetchWithHeaders(ctx, url, nil) +func TestContainerHandler_BlobHead_DirectServeRedirects(t *testing.T) { + proxy, db, store, fetcher := setupTestProxy(t) + digest := "sha256:abc123def456abc123def456abc123def456abc123def456abc123def456abcd" + seedPackage(t, db, store, "oci", "library/nginx", digest, digest, "cached blob") + store.signedURL = "https://storage.example.test/cached-blob?signature=test" + proxy.DirectServe = true + + h := &ContainerHandler{ + proxy: proxy, + registryURL: "https://registry.example.test", + proxyURL: "http://localhost:8080", + } + + req := httptest.NewRequest(http.MethodHead, "/library/nginx/blobs/"+digest, nil) + w := httptest.NewRecorder() + h.Routes().ServeHTTP(w, req) + + if w.Code != http.StatusFound { + t.Fatalf("status = %d, want %d", w.Code, http.StatusFound) + } + if got := w.Header().Get("Location"); got != store.signedURL { + t.Errorf("Location = %q, want %q", got, store.signedURL) + } + if got := w.Header().Get("ETag"); got != `"abc123"` { + t.Errorf("ETag = %q, want %q", got, `"abc123"`) + } + if w.Body.Len() != 0 { + t.Errorf("HEAD response body length = %d, want 0", w.Body.Len()) + } + if fetcher.fetchCalled { + t.Error("fetcher should not be called on cache hit") + } } -func (f *mockFetcherWithHeaders) FetchWithHeaders(ctx context.Context, url string, headers http.Header) (*fetch.Artifact, error) { - return f.fetchFn(ctx, url, headers) +func TestContainerHandler_ManifestByDigest_CacheHitSkipsUpstream(t *testing.T) { + digest := "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + manifest := `{"schemaVersion":2,"mediaType":"application/vnd.oci.image.manifest.v1+json"}` + upstreamAvailable := true + upstreamRequests := 0 + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + upstreamRequests++ + if !upstreamAvailable { + http.Error(w, "upstream unavailable", http.StatusServiceUnavailable) + return + } + if r.URL.Path != "/v2/library/nginx/manifests/"+digest { + http.NotFound(w, r) + return + } + w.Header().Set("Content-Type", "application/vnd.oci.image.manifest.v1+json") + w.Header().Set("Docker-Content-Digest", digest) + w.Header().Set("ETag", `"manifest-etag"`) + if r.Method != http.MethodHead { + _, _ = io.WriteString(w, manifest) + } + })) + defer upstream.Close() + + proxy, _, _, _ := setupTestProxy(t) + proxy.HTTPClient = upstream.Client() + h := &ContainerHandler{proxy: proxy, registryURL: upstream.URL, proxyURL: "http://localhost:8080"} + + first := httptest.NewRecorder() + h.Routes().ServeHTTP(first, httptest.NewRequest(http.MethodGet, "/library/nginx/manifests/"+digest, nil)) + if first.Code != http.StatusOK { + t.Fatalf("initial status = %d, want %d; body: %s", first.Code, http.StatusOK, first.Body.String()) + } + if first.Body.String() != manifest { + t.Fatalf("initial body = %q, want %q", first.Body.String(), manifest) + } + + upstreamAvailable = false + second := httptest.NewRecorder() + h.Routes().ServeHTTP(second, httptest.NewRequest(http.MethodGet, "/library/nginx/manifests/"+digest, nil)) + if second.Code != http.StatusOK { + t.Fatalf("cached status = %d, want %d; body: %s", second.Code, http.StatusOK, second.Body.String()) + } + if second.Body.String() != manifest { + t.Errorf("cached body = %q, want %q", second.Body.String(), manifest) + } + if got := second.Header().Get("Docker-Content-Digest"); got != digest { + t.Errorf("cached Docker-Content-Digest = %q, want %q", got, digest) + } + + head := httptest.NewRecorder() + h.Routes().ServeHTTP(head, httptest.NewRequest(http.MethodHead, "/library/nginx/manifests/"+digest, nil)) + if head.Code != http.StatusOK { + t.Fatalf("cached HEAD status = %d, want %d", head.Code, http.StatusOK) + } + wantLength := strconv.Itoa(len(manifest)) + if got := head.Header().Get("Content-Length"); got != wantLength { + t.Errorf("cached HEAD Content-Length = %q, want %q", got, wantLength) + } + if head.Body.Len() != 0 { + t.Errorf("cached HEAD body length = %d, want 0", head.Body.Len()) + } + if upstreamRequests != 1 { + t.Errorf("upstream requests = %d, want 1", upstreamRequests) + } } -func (f *mockFetcherWithHeaders) Head(_ context.Context, _ string) (int64, string, error) { - return 0, "", nil +func TestContainerHandler_ManifestByTag_UsesStaleCacheOnUpstreamFailure(t *testing.T) { + digest := "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + manifest := `{"schemaVersion":2,"mediaType":"application/vnd.oci.image.index.v1+json"}` + upstreamAvailable := true + upstreamRequests := 0 + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + upstreamRequests++ + if !upstreamAvailable { + http.Error(w, "upstream unavailable", http.StatusServiceUnavailable) + return + } + w.Header().Set("Content-Type", "application/vnd.oci.image.index.v1+json") + w.Header().Set("Docker-Content-Digest", digest) + _, _ = io.WriteString(w, manifest) + })) + defer upstream.Close() + + proxy, _, _, _ := setupTestProxy(t) + proxy.HTTPClient = upstream.Client() + proxy.MetadataTTL = 0 + h := &ContainerHandler{proxy: proxy, registryURL: upstream.URL, proxyURL: "http://localhost:8080"} + + first := httptest.NewRecorder() + h.Routes().ServeHTTP(first, httptest.NewRequest(http.MethodGet, "/library/nginx/manifests/latest", nil)) + if first.Code != http.StatusOK { + t.Fatalf("initial status = %d, want %d; body: %s", first.Code, http.StatusOK, first.Body.String()) + } + + upstreamAvailable = false + second := httptest.NewRecorder() + h.Routes().ServeHTTP(second, httptest.NewRequest(http.MethodGet, "/library/nginx/manifests/latest", nil)) + if second.Code != http.StatusOK { + t.Fatalf("stale status = %d, want %d; body: %s", second.Code, http.StatusOK, second.Body.String()) + } + if second.Body.String() != manifest { + t.Errorf("stale body = %q, want %q", second.Body.String(), manifest) + } + if got := second.Header().Get("Warning"); got != `110 - "Response is Stale"` { + t.Errorf("Warning = %q, want stale warning", got) + } + if got := second.Header().Get("Docker-Content-Digest"); got != digest { + t.Errorf("stale Docker-Content-Digest = %q, want %q", got, digest) + } + if upstreamRequests != 2 { + t.Errorf("upstream requests = %d, want 2", upstreamRequests) + } +} + +func TestContainerHandler_ManifestByTag_CachesDigestAlias(t *testing.T) { + digest := "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc" + manifest := `{"schemaVersion":2,"mediaType":"application/vnd.oci.image.manifest.v1+json"}` + upstreamAvailable := true + upstreamRequests := 0 + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + upstreamRequests++ + if !upstreamAvailable { + http.Error(w, "upstream unavailable", http.StatusServiceUnavailable) + return + } + if r.URL.Path != "/v2/library/nginx/manifests/latest" { + http.NotFound(w, r) + return + } + w.Header().Set("Content-Type", "application/vnd.oci.image.manifest.v1+json") + w.Header().Set("Docker-Content-Digest", digest) + _, _ = io.WriteString(w, manifest) + })) + defer upstream.Close() + + proxy, _, _, _ := setupTestProxy(t) + proxy.HTTPClient = upstream.Client() + h := &ContainerHandler{proxy: proxy, registryURL: upstream.URL, proxyURL: "http://localhost:8080"} + + first := httptest.NewRecorder() + h.Routes().ServeHTTP(first, httptest.NewRequest(http.MethodGet, "/library/nginx/manifests/latest", nil)) + if first.Code != http.StatusOK { + t.Fatalf("tag status = %d, want %d; body: %s", first.Code, http.StatusOK, first.Body.String()) + } + + upstreamAvailable = false + byDigest := httptest.NewRecorder() + h.Routes().ServeHTTP(byDigest, httptest.NewRequest(http.MethodGet, "/library/nginx/manifests/"+digest, nil)) + if byDigest.Code != http.StatusOK { + t.Fatalf("digest status = %d, want %d; body: %s", byDigest.Code, http.StatusOK, byDigest.Body.String()) + } + if byDigest.Body.String() != manifest { + t.Errorf("digest body = %q, want %q", byDigest.Body.String(), manifest) + } + if got := byDigest.Header().Get("Docker-Content-Digest"); got != digest { + t.Errorf("Docker-Content-Digest = %q, want %q", got, digest) + } + if upstreamRequests != 1 { + t.Errorf("upstream requests = %d, want 1", upstreamRequests) + } +} + +func TestContainerHandler_ManifestByTag_StaleHeadChecksUpstream(t *testing.T) { + oldDigest := "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd" + newDigest := "sha256:eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee" + currentDigest := oldDigest + upstreamRequests := 0 + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + upstreamRequests++ + w.Header().Set("Content-Type", "application/vnd.oci.image.manifest.v1+json") + w.Header().Set("Docker-Content-Digest", currentDigest) + w.Header().Set("ETag", `"`+currentDigest+`"`) + if r.Method != http.MethodHead { + _, _ = io.WriteString(w, `{"schemaVersion":2}`) + } + })) + defer upstream.Close() + + proxy, _, _, _ := setupTestProxy(t) + proxy.HTTPClient = upstream.Client() + proxy.MetadataTTL = 0 + h := &ContainerHandler{proxy: proxy, registryURL: upstream.URL, proxyURL: "http://localhost:8080"} + + first := httptest.NewRecorder() + h.Routes().ServeHTTP(first, httptest.NewRequest(http.MethodGet, "/library/nginx/manifests/latest", nil)) + if first.Code != http.StatusOK { + t.Fatalf("initial status = %d, want %d", first.Code, http.StatusOK) + } + + currentDigest = newDigest + head := httptest.NewRecorder() + h.Routes().ServeHTTP(head, httptest.NewRequest(http.MethodHead, "/library/nginx/manifests/latest", nil)) + if head.Code != http.StatusOK { + t.Fatalf("HEAD status = %d, want %d", head.Code, http.StatusOK) + } + if got := head.Header().Get("Docker-Content-Digest"); got != newDigest { + t.Errorf("Docker-Content-Digest = %q, want %q", got, newDigest) + } + if upstreamRequests != 2 { + t.Errorf("upstream requests = %d, want 2", upstreamRequests) + } } func TestContainerHandler_Routes_VersionCheck(t *testing.T) { diff --git a/internal/handler/handler.go b/internal/handler/handler.go index fc78dbf..72b9c28 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -136,18 +136,25 @@ type CacheResult struct { // GetOrFetchArtifact retrieves an artifact from cache or fetches from upstream. func (p *Proxy) GetOrFetchArtifact(ctx context.Context, ecosystem, name, version, filename string) (*CacheResult, error) { - pkgPURL := purl.MakePURLString(ecosystem, name, "") - versionPURL := purl.MakePURLString(ecosystem, name, version) - - if cached, err := p.checkCache(ctx, pkgPURL, versionPURL, filename); err != nil { + if cached, err := p.GetCachedArtifact(ctx, ecosystem, name, version, filename); err != nil { return nil, err } else if cached != nil { return cached, nil } + pkgPURL := purl.MakePURLString(ecosystem, name, "") + versionPURL := purl.MakePURLString(ecosystem, name, version) return p.fetchAndCache(ctx, ecosystem, name, version, filename, pkgPURL, versionPURL) } +// GetCachedArtifact retrieves an artifact from cache without contacting an upstream. +// It returns nil when no usable cache entry exists. +func (p *Proxy) GetCachedArtifact(ctx context.Context, ecosystem, name, version, filename string) (*CacheResult, error) { + pkgPURL := purl.MakePURLString(ecosystem, name, "") + versionPURL := purl.MakePURLString(ecosystem, name, version) + return p.checkCache(ctx, pkgPURL, versionPURL, filename) +} + // checkCache looks up an artifact in the cache. Returns nil if not cached. func (p *Proxy) checkCache(ctx context.Context, pkgPURL, versionPURL, filename string) (*CacheResult, error) { pkg, err := p.DB.GetPackageByPURL(pkgPURL) @@ -363,6 +370,10 @@ func (p *Proxy) updateCacheDB(ecosystem, name, filename, pkgPURL, versionPURL, u // ServeArtifact writes a CacheResult to an HTTP response. func ServeArtifact(w http.ResponseWriter, result *CacheResult) { + serveArtifact(w, http.MethodGet, result) +} + +func serveArtifact(w http.ResponseWriter, method string, result *CacheResult) { if result.RedirectURL != "" { if result.Hash != "" { w.Header().Set("ETag", fmt.Sprintf(`"%s"`, result.Hash)) @@ -372,12 +383,14 @@ func ServeArtifact(w http.ResponseWriter, result *CacheResult) { return } - defer func() { _ = result.Reader.Close() }() + if result.Reader != nil { + defer func() { _ = result.Reader.Close() }() + } if result.ContentType != "" { w.Header().Set("Content-Type", result.ContentType) } - if result.Size > 0 { + if result.Size > 0 || (method == http.MethodHead && result.Size == 0) { w.Header().Set("Content-Length", fmt.Sprintf("%d", result.Size)) } if result.Hash != "" { @@ -385,7 +398,9 @@ func ServeArtifact(w http.ResponseWriter, result *CacheResult) { } w.WriteHeader(http.StatusOK) - _, _ = io.Copy(w, result.Reader) + if method != http.MethodHead && result.Reader != nil { + _, _ = io.Copy(w, result.Reader) + } } // ProxyUpstream forwards a request to an upstream URL without caching. @@ -807,18 +822,16 @@ func (p *Proxy) GetOrFetchArtifactFromURL(ctx context.Context, ecosystem, name, } // GetOrFetchArtifactFromURLWithHeaders retrieves an artifact from cache or fetches from a URL -// with additional HTTP headers. This is needed for registries that require authentication -// (e.g. Docker Hub requires a Bearer token even for public images). +// with additional request-specific HTTP headers. func (p *Proxy) GetOrFetchArtifactFromURLWithHeaders(ctx context.Context, ecosystem, name, version, filename, downloadURL string, headers http.Header) (*CacheResult, error) { - pkgPURL := purl.MakePURLString(ecosystem, name, "") - versionPURL := purl.MakePURLString(ecosystem, name, version) - - if cached, err := p.checkCache(ctx, pkgPURL, versionPURL, filename); err != nil { + if cached, err := p.GetCachedArtifact(ctx, ecosystem, name, version, filename); err != nil { return nil, err } else if cached != nil { return cached, nil } + pkgPURL := purl.MakePURLString(ecosystem, name, "") + versionPURL := purl.MakePURLString(ecosystem, name, version) return p.fetchAndCacheFromURL(ctx, ecosystem, name, version, filename, pkgPURL, versionPURL, downloadURL, headers) } diff --git a/internal/handler/handler_test.go b/internal/handler/handler_test.go index 9a2b329..a3bd9b3 100644 --- a/internal/handler/handler_test.go +++ b/internal/handler/handler_test.go @@ -5,7 +5,6 @@ import ( "context" "database/sql" "errors" - "fmt" "io" "log/slog" "net/http" @@ -17,6 +16,7 @@ import ( "github.com/git-pkgs/proxy/internal/config" "github.com/git-pkgs/proxy/internal/database" "github.com/git-pkgs/proxy/internal/storage" + "github.com/git-pkgs/purl" "github.com/git-pkgs/registries/fetch" ) @@ -152,7 +152,7 @@ func seedPackage(t *testing.T, db *database.DB, store *mockStorage, ecosystem, n t.Helper() pkg := &database.Package{ - PURL: fmt.Sprintf("pkg:%s/%s", ecosystem, name), + PURL: purl.MakePURLString(ecosystem, name, ""), Ecosystem: ecosystem, Name: name, } @@ -160,7 +160,7 @@ func seedPackage(t *testing.T, db *database.DB, store *mockStorage, ecosystem, n t.Fatalf("failed to upsert package: %v", err) } - versionPURL := fmt.Sprintf("pkg:%s/%s@%s", ecosystem, name, version) + versionPURL := purl.MakePURLString(ecosystem, name, version) ver := &database.Version{ PURL: versionPURL, PackagePURL: pkg.PURL, diff --git a/internal/handler/notfound_ecosystems_test.go b/internal/handler/notfound_ecosystems_test.go index 6f04113..44c4486 100644 --- a/internal/handler/notfound_ecosystems_test.go +++ b/internal/handler/notfound_ecosystems_test.go @@ -1,7 +1,6 @@ package handler import ( - "context" "net/http" "net/http/httptest" "strings" @@ -117,23 +116,12 @@ func TestComposerDownloadUpstreamNotFoundReturns404(t *testing.T) { } func TestContainerBlobUpstreamNotFoundReturns404(t *testing.T) { - authServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Content-Type", "application/json") - _, _ = w.Write([]byte(`{"token": "test-token-123"}`)) - })) - defer authServer.Close() - - proxy, _, _, _ := setupTestProxy(t) - proxy.Fetcher = &mockFetcherWithHeaders{ - fetchFn: func(_ context.Context, _ string, _ http.Header) (*fetch.Artifact, error) { - return nil, fetch.ErrNotFound - }, - } + proxy, _, _, fetcher := setupTestProxy(t) + fetcher.fetchErr = fetch.ErrNotFound h := &ContainerHandler{ proxy: proxy, registryURL: "https://registry-1.docker.io", - authURL: authServer.URL, proxyURL: "http://localhost:8080", } From 6fcc57c994f5d3b3ca51ac51a3a56bd045c07e2a Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Thu, 13 Aug 2026 08:06:41 +0100 Subject: [PATCH 044/633] Optimize cached artifact serving (#245) --- internal/database/database_test.go | 80 +++++++ internal/database/queries.go | 22 ++ internal/database/types.go | 10 + internal/handler/handler.go | 65 +++--- internal/handler/handler_bench_test.go | 300 +++++++++++++++++++++++++ internal/handler/handler_test.go | 4 +- 6 files changed, 444 insertions(+), 37 deletions(-) create mode 100644 internal/handler/handler_bench_test.go diff --git a/internal/database/database_test.go b/internal/database/database_test.go index 6fca4ea..3e92b91 100644 --- a/internal/database/database_test.go +++ b/internal/database/database_test.go @@ -239,6 +239,86 @@ func TestArtifactCRUD(t *testing.T) { }) } +func TestGetCachedArtifact(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + const ( + packagePURL = "pkg:npm/lodash" + versionPURL = "pkg:npm/lodash@4.17.21" + filename = "lodash-4.17.21.tgz" + ) + seedCachedArtifactTestData(t, db, packagePURL, versionPURL, filename) + + cached, err := db.GetCachedArtifact(packagePURL, versionPURL, filename) + if err != nil { + t.Fatalf("GetCachedArtifact before cache failed: %v", err) + } + if cached != nil { + t.Fatalf("expected no cached artifact, got %+v", cached) + } + + if err := db.MarkArtifactCached(versionPURL, filename, "/cache/npm/"+filename, + "sha256-abc", 12345, "application/gzip"); err != nil { + t.Fatalf("MarkArtifactCached failed: %v", err) + } + + cached, err = db.GetCachedArtifact(packagePURL, versionPURL, filename) + if err != nil { + t.Fatalf("GetCachedArtifact failed: %v", err) + } + if cached == nil { + t.Fatal("expected cached artifact, got nil") + } + if cached.Ecosystem != "npm" { + t.Errorf("expected npm ecosystem, got %q", cached.Ecosystem) + } + if cached.StoragePath != "/cache/npm/"+filename { + t.Errorf("expected cached storage path, got %q", cached.StoragePath) + } + if cached.ContentHash.String != "sha256-abc" { + t.Errorf("expected cached content hash, got %q", cached.ContentHash.String) + } + if cached.Size.Int64 != 12345 { + t.Errorf("expected cached size 12345, got %d", cached.Size.Int64) + } + if cached.ContentType.String != "application/gzip" { + t.Errorf("expected cached content type, got %q", cached.ContentType.String) + } + if cached.Integrity.String != "sha512-abc123" { + t.Errorf("expected cached integrity, got %q", cached.Integrity.String) + } + + cached, err = db.GetCachedArtifact("pkg:npm/other", versionPURL, filename) + if err != nil { + t.Fatalf("GetCachedArtifact with wrong package failed: %v", err) + } + if cached != nil { + t.Fatalf("expected package mismatch to miss cache, got %+v", cached) + } + }) +} + +func seedCachedArtifactTestData(t *testing.T, db *DB, packagePURL, versionPURL, filename string) { + t.Helper() + + if err := db.UpsertPackage(&Package{PURL: packagePURL, Ecosystem: "npm", Name: "lodash"}); err != nil { + t.Fatalf("UpsertPackage failed: %v", err) + } + if err := db.UpsertVersion(&Version{ + PURL: versionPURL, + PackagePURL: packagePURL, + Integrity: sql.NullString{String: "sha512-abc123", Valid: true}, + }); err != nil { + t.Fatalf("UpsertVersion failed: %v", err) + } + if err := db.UpsertArtifact(&Artifact{ + VersionPURL: versionPURL, + Filename: filename, + UpstreamURL: "https://registry.npmjs.org/lodash/-/" + filename, + }); err != nil { + t.Fatalf("UpsertArtifact failed: %v", err) + } +} + func TestCacheManagement(t *testing.T) { runWithBothDatabases(t, func(t *testing.T, db *DB) { pkg := &Package{ diff --git a/internal/database/queries.go b/internal/database/queries.go index f8b76fe..b01abe4 100644 --- a/internal/database/queries.go +++ b/internal/database/queries.go @@ -191,6 +191,28 @@ func (db *DB) GetArtifact(versionPURL, filename string) (*Artifact, error) { return &a, nil } +// GetCachedArtifact returns the fields needed to serve a cached artifact. +func (db *DB) GetCachedArtifact(packagePURL, versionPURL, filename string) (*CachedArtifact, error) { + var artifact CachedArtifact + query := db.Rebind(` + SELECT packages.ecosystem, artifacts.storage_path, artifacts.content_hash, artifacts.size, + artifacts.content_type, versions.integrity + FROM artifacts + JOIN versions ON versions.purl = artifacts.version_purl + JOIN packages ON packages.purl = versions.package_purl + WHERE packages.purl = ? AND artifacts.version_purl = ? AND artifacts.filename = ? + AND artifacts.storage_path IS NOT NULL AND artifacts.fetched_at IS NOT NULL + `) + err := db.Get(&artifact, query, packagePURL, versionPURL, filename) + if err == sql.ErrNoRows { + return nil, nil + } + if err != nil { + return nil, err + } + return &artifact, nil +} + func (db *DB) GetArtifactByPath(storagePath string) (*Artifact, error) { var a Artifact query := db.Rebind(` diff --git a/internal/database/types.go b/internal/database/types.go index 7128f12..3826c7a 100644 --- a/internal/database/types.go +++ b/internal/database/types.go @@ -76,6 +76,16 @@ func (a *Artifact) IsCached() bool { return a.StoragePath.Valid && a.FetchedAt.Valid } +// CachedArtifact contains the fields needed to serve a cached artifact. +type CachedArtifact struct { + Ecosystem string `db:"ecosystem"` + StoragePath string `db:"storage_path"` + ContentHash sql.NullString `db:"content_hash"` + Size sql.NullInt64 `db:"size"` + ContentType sql.NullString `db:"content_type"` + Integrity sql.NullString `db:"integrity"` +} + // MetadataCacheEntry represents a cached metadata blob for offline serving. type MetadataCacheEntry struct { ID int64 `db:"id" json:"id"` diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 72b9c28..6a7aab4 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -13,6 +13,7 @@ import ( "net/url" "strconv" "strings" + "sync" "time" "github.com/git-pkgs/cooldown" @@ -48,6 +49,15 @@ func hasDotDotSegment(path string) bool { const defaultHTTPTimeout = 30 * time.Second +const artifactCopyBufferSize = 32 << 10 + +var artifactCopyBufferPool = sync.Pool{ //nolint:gochecknoglobals // shared across artifact responses + New: func() any { + buffer := make([]byte, artifactCopyBufferSize) + return &buffer + }, +} + // canonicalPackagePURL returns a versionless PURL in canonical form so cooldown // lookups match keys produced by config.CooldownConfig.NormalizedPackages. func canonicalPackagePURL(ecosystem, name string) string { @@ -157,27 +167,11 @@ func (p *Proxy) GetCachedArtifact(ctx context.Context, ecosystem, name, version, // checkCache looks up an artifact in the cache. Returns nil if not cached. func (p *Proxy) checkCache(ctx context.Context, pkgPURL, versionPURL, filename string) (*CacheResult, error) { - pkg, err := p.DB.GetPackageByPURL(pkgPURL) - if err != nil { - return nil, fmt.Errorf("checking package cache: %w", err) - } - if pkg == nil { - return nil, nil - } - - ver, err := p.DB.GetVersionByPURL(versionPURL) - if err != nil { - return nil, fmt.Errorf("checking version cache: %w", err) - } - if ver == nil { - return nil, nil - } - - artifact, err := p.DB.GetArtifact(versionPURL, filename) + artifact, err := p.DB.GetCachedArtifact(pkgPURL, versionPURL, filename) if err != nil { return nil, fmt.Errorf("checking artifact cache: %w", err) } - if artifact == nil || !artifact.IsCached() { + if artifact == nil { return nil, nil } @@ -189,39 +183,39 @@ func (p *Proxy) checkCache(ctx context.Context, pkgPURL, versionPURL, filename s } if p.DirectServe { - signed, err := p.Storage.SignedURL(ctx, artifact.StoragePath.String, p.DirectServeTTL) + signed, err := p.Storage.SignedURL(ctx, artifact.StoragePath, p.DirectServeTTL) if err == nil { result.RedirectURL = rewriteSignedURLHost(signed, p.DirectServeBaseURL) - p.recordCacheHit(pkgPURL, versionPURL, filename) + p.recordCacheHit(artifact.Ecosystem, versionPURL, filename) return result, nil } if !errors.Is(err, storage.ErrSignedURLUnsupported) { p.Logger.Warn("failed to sign storage URL, falling back to streaming", - "path", artifact.StoragePath.String, "error", err) + "path", artifact.StoragePath, "error", err) } } start := time.Now() - reader, err := p.Storage.Open(ctx, artifact.StoragePath.String) + reader, err := p.Storage.Open(ctx, artifact.StoragePath) metrics.RecordStorageOperation("read", time.Since(start)) if err != nil { metrics.RecordStorageError("read") p.Logger.Warn("cached artifact missing from storage, will refetch", - "path", artifact.StoragePath.String, "error", err) + "path", artifact.StoragePath, "error", err) return nil, nil } - result.Reader = newVerifyingReader(reader, artifact.ContentHash.String, ver.Integrity.String, + result.Reader = newVerifyingReader(reader, artifact.ContentHash.String, artifact.Integrity.String, func(reason string) { p.Logger.Error("cached artifact failed integrity check", "purl", versionPURL, "filename", filename, - "path", artifact.StoragePath.String, "reason", reason) - metrics.RecordIntegrityFailure(pkg.Ecosystem) + "path", artifact.StoragePath, "reason", reason) + metrics.RecordIntegrityFailure(artifact.Ecosystem) if err := p.DB.ClearArtifactCache(versionPURL, filename); err != nil { p.Logger.Warn("failed to clear corrupt artifact from cache", "error", err) } }) - p.recordCacheHit(pkgPURL, versionPURL, filename) + p.recordCacheHit(artifact.Ecosystem, versionPURL, filename) return result, nil } @@ -245,11 +239,9 @@ func rewriteSignedURLHost(signed, baseURL string) string { return s.String() } -func (p *Proxy) recordCacheHit(pkgPURL, versionPURL, filename string) { +func (p *Proxy) recordCacheHit(ecosystem, versionPURL, filename string) { _ = p.DB.RecordArtifactHit(versionPURL, filename) - if parsed, err := purl.Parse(pkgPURL); err == nil { - metrics.RecordCacheHit(purl.PURLTypeToEcosystem(parsed.Type)) - } + metrics.RecordCacheHit(purl.NormalizeEcosystem(ecosystem)) } func (p *Proxy) fetchAndCache(ctx context.Context, ecosystem, name, version, filename, pkgPURL, versionPURL string) (*CacheResult, error) { @@ -376,7 +368,7 @@ func ServeArtifact(w http.ResponseWriter, result *CacheResult) { func serveArtifact(w http.ResponseWriter, method string, result *CacheResult) { if result.RedirectURL != "" { if result.Hash != "" { - w.Header().Set("ETag", fmt.Sprintf(`"%s"`, result.Hash)) + w.Header().Set("ETag", `"`+result.Hash+`"`) } w.Header().Set("Location", result.RedirectURL) w.WriteHeader(http.StatusFound) @@ -391,15 +383,18 @@ func serveArtifact(w http.ResponseWriter, method string, result *CacheResult) { w.Header().Set("Content-Type", result.ContentType) } if result.Size > 0 || (method == http.MethodHead && result.Size == 0) { - w.Header().Set("Content-Length", fmt.Sprintf("%d", result.Size)) + w.Header().Set("Content-Length", strconv.FormatInt(result.Size, 10)) } if result.Hash != "" { - w.Header().Set("ETag", fmt.Sprintf(`"%s"`, result.Hash)) + w.Header().Set("ETag", `"`+result.Hash+`"`) } w.WriteHeader(http.StatusOK) if method != http.MethodHead && result.Reader != nil { - _, _ = io.Copy(w, result.Reader) + buffer := artifactCopyBufferPool.Get().(*[]byte) + defer artifactCopyBufferPool.Put(buffer) + // Hide optional ReaderFrom methods so io.CopyBuffer uses the pooled buffer. + _, _ = io.CopyBuffer(struct{ io.Writer }{w}, result.Reader, *buffer) } } diff --git a/internal/handler/handler_bench_test.go b/internal/handler/handler_bench_test.go new file mode 100644 index 0000000..cdbb524 --- /dev/null +++ b/internal/handler/handler_bench_test.go @@ -0,0 +1,300 @@ +package handler + +import ( + "bytes" + "context" + "crypto/sha256" + "database/sql" + "encoding/hex" + "fmt" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/git-pkgs/proxy/internal/database" + "github.com/git-pkgs/proxy/internal/storage" + "github.com/git-pkgs/purl" + "github.com/git-pkgs/registries/fetch" +) + +const benchmarkArtifactSize = 64 << 10 + +const benchmarkMetadataSize = 1 << 20 + +type benchmarkResponseWriter struct { + header http.Header +} + +func (w *benchmarkResponseWriter) Header() http.Header { + return w.header +} + +func (w *benchmarkResponseWriter) Write(p []byte) (int, error) { + return len(p), nil +} + +func (w *benchmarkResponseWriter) WriteHeader(_ int) {} + +func benchmarkCachedProxy(b *testing.B) (*Proxy, *mockStorage) { + b.Helper() + + proxy, db, store, _ := setupTestProxy(b) + content := strings.Repeat("x", benchmarkArtifactSize) + seedPackage(b, db, store, "npm", "lodash", "4.17.21", "lodash-4.17.21.tgz", content) + + artifact, err := db.GetArtifact("pkg:npm/lodash@4.17.21", "lodash-4.17.21.tgz") + if err != nil { + b.Fatalf("get seeded artifact: %v", err) + } + sum := sha256.Sum256([]byte(content)) + artifact.ContentHash.String = hex.EncodeToString(sum[:]) + if err := db.UpsertArtifact(artifact); err != nil { + b.Fatalf("update seeded artifact hash: %v", err) + } + + return proxy, store +} + +func BenchmarkArtifactCacheHit(b *testing.B) { + ctx := context.Background() + + b.Run("stream-64KiB", func(b *testing.B) { + proxy, _ := benchmarkCachedProxy(b) + w := &benchmarkResponseWriter{header: make(http.Header)} + b.SetBytes(benchmarkArtifactSize) + b.ReportAllocs() + b.ResetTimer() + + for b.Loop() { + result, err := proxy.GetOrFetchArtifact(ctx, "npm", "lodash", "4.17.21", "lodash-4.17.21.tgz") + if err != nil { + b.Fatal(err) + } + ServeArtifact(w, result) + } + }) + + b.Run("direct-serve", func(b *testing.B) { + proxy, store := benchmarkCachedProxy(b) + proxy.DirectServe = true + store.signedURL = "https://storage.example/npm/lodash-4.17.21.tgz?signature=abc" + w := &benchmarkResponseWriter{header: make(http.Header)} + b.ReportAllocs() + b.ResetTimer() + + for b.Loop() { + result, err := proxy.GetOrFetchArtifact(ctx, "npm", "lodash", "4.17.21", "lodash-4.17.21.tgz") + if err != nil { + b.Fatal(err) + } + ServeArtifact(w, result) + } + }) +} + +func BenchmarkArtifactCacheHitParallel(b *testing.B) { + proxy, _ := benchmarkCachedProxy(b) + ctx := context.Background() + b.SetBytes(benchmarkArtifactSize) + b.ReportAllocs() + b.ResetTimer() + + b.RunParallel(func(pb *testing.PB) { + w := &benchmarkResponseWriter{header: make(http.Header)} + for pb.Next() { + result, err := proxy.GetOrFetchArtifact(ctx, "npm", "lodash", "4.17.21", "lodash-4.17.21.tgz") + if err != nil { + b.Error(err) + return + } + ServeArtifact(w, result) + } + }) +} + +func BenchmarkReadMetadata(b *testing.B) { + payload := bytes.Repeat([]byte("x"), benchmarkMetadataSize) + proxy := &Proxy{MetadataMaxSize: benchmarkMetadataSize} + b.SetBytes(benchmarkMetadataSize) + b.ReportAllocs() + + var data []byte + for b.Loop() { + var err error + data, err = proxy.ReadMetadata(bytes.NewReader(payload)) + if err != nil { + b.Fatal(err) + } + } + if len(data) != len(payload) { + b.Fatalf("metadata size = %d, want %d", len(data), len(payload)) + } +} + +func BenchmarkArtifactPURLConstruction(b *testing.B) { + for _, tc := range []struct { + name string + ecosystem string + packageID string + }{ + {"npm", "npm", "lodash"}, + {"scoped-npm", "npm", "@scope/package"}, + {"go", "golang", "github.com/git-pkgs/proxy"}, + } { + b.Run(tc.name, func(b *testing.B) { + b.ReportAllocs() + var packagePURL, versionPURL string + for b.Loop() { + packagePURL = purl.MakePURLString(tc.ecosystem, tc.packageID, "") + versionPURL = purl.MakePURLString(tc.ecosystem, tc.packageID, "1.2.3") + } + if packagePURL == "" || versionPURL == "" { + b.Fatal("empty PURL") + } + }) + } +} + +type benchmarkNPMServer struct { + client *http.Client + requestURL string + db *database.DB + versionPURL string + filename string +} + +func newBenchmarkNPMServer(b *testing.B) *benchmarkNPMServer { + b.Helper() + + ctx := context.Background() + dir := b.TempDir() + db, err := database.Create(filepath.Join(dir, "benchmark.db")) + if err != nil { + b.Fatalf("create database: %v", err) + } + b.Cleanup(func() { _ = db.Close() }) + + store, err := storage.OpenBucket(ctx, "file://"+filepath.Join(dir, "cache")) + if err != nil { + b.Fatalf("open storage: %v", err) + } + b.Cleanup(func() { _ = store.Close() }) + + content := bytes.Repeat([]byte("x"), benchmarkArtifactSize) + storagePath := storage.ArtifactPath("npm", "", "lodash", "4.17.21", "lodash-4.17.21.tgz") + size, hash, err := store.Store(ctx, storagePath, bytes.NewReader(content)) + if err != nil { + b.Fatalf("store artifact: %v", err) + } + + pkg := &database.Package{PURL: "pkg:npm/lodash", Ecosystem: "npm", Name: "lodash"} + if err := db.UpsertPackage(pkg); err != nil { + b.Fatalf("seed package: %v", err) + } + version := &database.Version{PURL: "pkg:npm/lodash@4.17.21", PackagePURL: pkg.PURL} + if err := db.UpsertVersion(version); err != nil { + b.Fatalf("seed version: %v", err) + } + artifact := &database.Artifact{ + VersionPURL: version.PURL, + Filename: "lodash-4.17.21.tgz", + UpstreamURL: "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", + StoragePath: sql.NullString{String: storagePath, Valid: true}, + ContentHash: sql.NullString{String: hash, Valid: true}, + Size: sql.NullInt64{Int64: size, Valid: true}, + ContentType: sql.NullString{String: "application/gzip", Valid: true}, + FetchedAt: sql.NullTime{Time: time.Now(), Valid: true}, + } + if err := db.UpsertArtifact(artifact); err != nil { + b.Fatalf("seed artifact: %v", err) + } + + logger := slog.New(slog.NewTextHandler(io.Discard, nil)) + proxy := NewProxy(db, store, &mockFetcher{}, fetch.NewResolver(), logger) + handler := NewNPMHandler(proxy, "http://proxy.example", "https://registry.npmjs.org") + server := httptest.NewServer(handler.Routes()) + b.Cleanup(server.Close) + client := server.Client() + return &benchmarkNPMServer{ + client: client, + requestURL: server.URL + "/lodash/-/lodash-4.17.21.tgz", + db: db, + versionPURL: version.PURL, + filename: artifact.Filename, + } +} + +func (s *benchmarkNPMServer) request() error { + resp, err := s.client.Get(s.requestURL) + if err != nil { + return fmt.Errorf("GET cached artifact: %w", err) + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("GET cached artifact status = %d, want %d", resp.StatusCode, http.StatusOK) + } + n, err := io.Copy(io.Discard, resp.Body) + if err != nil { + return fmt.Errorf("read cached artifact: %w", err) + } + if n != benchmarkArtifactSize { + return fmt.Errorf("cached artifact size = %d, want %d", n, benchmarkArtifactSize) + } + return nil +} + +func (s *benchmarkNPMServer) hitCount(b *testing.B) int64 { + b.Helper() + artifact, err := s.db.GetArtifact(s.versionPURL, s.filename) + if err != nil { + b.Fatalf("get artifact hit count: %v", err) + } + return artifact.HitCount +} + +func benchmarkNPMArtifactCacheHitHTTP(b *testing.B, parallel bool) { + server := newBenchmarkNPMServer(b) + if err := server.request(); err != nil { + b.Fatal(err) + } + startHits := server.hitCount(b) + + b.SetBytes(benchmarkArtifactSize) + b.ReportAllocs() + b.ResetTimer() + if parallel { + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + if err := server.request(); err != nil { + b.Error(err) + return + } + } + }) + } else { + for b.Loop() { + if err := server.request(); err != nil { + b.Fatal(err) + } + } + } + b.StopTimer() + + if hitCount := server.hitCount(b) - startHits; hitCount != int64(b.N) { + b.Fatalf("new artifact hits = %d, want %d", hitCount, b.N) + } + b.ReportMetric(float64(b.N)/b.Elapsed().Seconds(), "requests/s") +} + +func BenchmarkNPMArtifactCacheHitHTTP(b *testing.B) { + benchmarkNPMArtifactCacheHitHTTP(b, false) +} + +func BenchmarkNPMArtifactCacheHitHTTPParallel(b *testing.B) { + benchmarkNPMArtifactCacheHitHTTP(b, true) +} diff --git a/internal/handler/handler_test.go b/internal/handler/handler_test.go index a3bd9b3..d52e7b6 100644 --- a/internal/handler/handler_test.go +++ b/internal/handler/handler_test.go @@ -128,7 +128,7 @@ func (f *mockFetcher) Head(_ context.Context, _ string) (int64, string, error) { } // setupTestProxy creates a Proxy with a real DB (SQLite in temp dir) and mock storage/fetcher. -func setupTestProxy(t *testing.T) (*Proxy, *database.DB, *mockStorage, *mockFetcher) { +func setupTestProxy(t testing.TB) (*Proxy, *database.DB, *mockStorage, *mockFetcher) { t.Helper() dir := t.TempDir() @@ -148,7 +148,7 @@ func setupTestProxy(t *testing.T) (*Proxy, *database.DB, *mockStorage, *mockFetc } // seedPackage creates a package, version, and cached artifact in the test DB and storage. -func seedPackage(t *testing.T, db *database.DB, store *mockStorage, ecosystem, name, version, filename, content string) { +func seedPackage(t testing.TB, db *database.DB, store *mockStorage, ecosystem, name, version, filename, content string) { t.Helper() pkg := &database.Package{ From ed540053fadc13f1f6f520b5bca333b64d40ac9a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 14 Aug 2026 09:15:33 +0100 Subject: [PATCH 045/633] Bump zizmorcore/zizmor-action from 0.6.1 to 0.6.2 (#251) Bumps [zizmorcore/zizmor-action](https://github.com/zizmorcore/zizmor-action) from 0.6.1 to 0.6.2. - [Release notes](https://github.com/zizmorcore/zizmor-action/releases) - [Commits](https://github.com/zizmorcore/zizmor-action/compare/6fc4b006235f201fdab3722e17240ab420d580e5...3dc1ecc9bcb9e94e9b2c709687979e1298497054) --- updated-dependencies: - dependency-name: zizmorcore/zizmor-action dependency-version: 0.6.2 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/zizmor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/zizmor.yml b/.github/workflows/zizmor.yml index 4df0e18..ac587dd 100644 --- a/.github/workflows/zizmor.yml +++ b/.github/workflows/zizmor.yml @@ -26,4 +26,4 @@ jobs: persist-credentials: false - name: Run zizmor - uses: zizmorcore/zizmor-action@6fc4b006235f201fdab3722e17240ab420d580e5 # v0.6.1 + uses: zizmorcore/zizmor-action@3dc1ecc9bcb9e94e9b2c709687979e1298497054 # v0.6.2 From 849500de1ec35442ef675361e4fe3769c86c9d9b Mon Sep 17 00:00:00 2001 From: wickedOne Date: Fri, 14 Aug 2026 11:38:08 +0200 Subject: [PATCH 046/633] fix: decode PURL percent-encoding in versions and package paths (#244) * fix: decode PURL percent-encoding in versions and package paths * review fix --- internal/database/queries.go | 37 ++- internal/database/types.go | 76 +++++- internal/database/version_purl_test.go | 159 ++++++++++++ internal/handler/debian_test.go | 5 + internal/server/api.go | 10 +- internal/server/browse.go | 34 ++- internal/server/browse_test.go | 17 +- internal/server/resolve.go | 95 ++++++- internal/server/resolve_test.go | 115 +++++++-- internal/server/server.go | 27 +- internal/server/server_test.go | 235 ++++++++++++++++++ .../server/templates/pages/browse_source.html | 11 +- .../templates/pages/compare_versions.html | 8 +- .../server/templates/pages/package_show.html | 12 +- .../server/templates/pages/version_show.html | 6 +- 15 files changed, 745 insertions(+), 102 deletions(-) create mode 100644 internal/database/version_purl_test.go diff --git a/internal/database/queries.go b/internal/database/queries.go index b01abe4..9fa5381 100644 --- a/internal/database/queries.go +++ b/internal/database/queries.go @@ -465,11 +465,14 @@ func (db *DB) GetMostPopularPackages(limit int) ([]PopularPackage, error) { } type RecentPackage struct { - Ecosystem string `db:"ecosystem"` - Name string `db:"name"` - Version string `db:"version"` - CachedAt time.Time `db:"fetched_at"` - Size int64 `db:"size"` + Ecosystem string `db:"ecosystem"` + Name string `db:"name"` + VersionPURL string `db:"version_purl"` + CachedAt time.Time `db:"fetched_at"` + Size int64 `db:"size"` + // Version is derived from VersionPURL rather than selected, so that the + // PURL percent-encoding is decoded (e.g. "%2B" back to "+"). + Version string `db:"-"` } func (db *DB) GetRecentlyCachedPackages(limit int) ([]RecentPackage, error) { @@ -483,10 +486,10 @@ func (db *DB) GetRecentlyCachedPackages(limit int) ([]RecentPackage, error) { } var packages []RecentPackage - // We need to extract version from the purl since there's no separate version column + // There is no separate version column, so the full version PURL is selected + // and the version is decoded from it in Go. query := db.Rebind(` - SELECT p.ecosystem, p.name, - SUBSTR(v.purl, INSTR(v.purl, '@') + 1) as version, + SELECT p.ecosystem, p.name, v.purl as version_purl, a.fetched_at, COALESCE(a.size, 0) as size FROM artifacts a JOIN versions v ON v.purl = a.version_purl @@ -496,25 +499,13 @@ func (db *DB) GetRecentlyCachedPackages(limit int) ([]RecentPackage, error) { LIMIT ? `) - // For postgres, use different string function - if db.dialect == DialectPostgres { - query = db.Rebind(` - SELECT p.ecosystem, p.name, - SUBSTRING(v.purl FROM POSITION('@' IN v.purl) + 1) as version, - a.fetched_at, COALESCE(a.size, 0) as size - FROM artifacts a - JOIN versions v ON v.purl = a.version_purl - JOIN packages p ON p.purl = v.package_purl - WHERE a.storage_path IS NOT NULL AND a.fetched_at IS NOT NULL - ORDER BY a.fetched_at DESC - LIMIT ? - `) - } - err = db.Select(&packages, query, limit) if err != nil { return nil, err } + for i := range packages { + packages[i].Version = VersionFromPURL(packages[i].VersionPURL) + } return packages, nil } diff --git a/internal/database/types.go b/internal/database/types.go index 3826c7a..5ddb9f3 100644 --- a/internal/database/types.go +++ b/internal/database/types.go @@ -2,6 +2,7 @@ package database import ( "database/sql" + "net/url" "strings" "time" ) @@ -47,10 +48,79 @@ type Version struct { // Version extracts the version string from the PURL. // e.g., "pkg:npm/lodash@4.17.21" -> "4.17.21" func (v *Version) Version() string { - if idx := strings.LastIndex(v.PURL, "@"); idx >= 0 { - return v.PURL[idx+1:] + return VersionFromPURL(v.PURL) +} + +// EscapedVersion returns the version escaped for use as a single URL path +// segment. +// +// Version returns decoded text, which is what should be shown to a user but is +// not safe to drop into a link: html/template preserves reserved characters and +// existing escapes in a URL, so "release/1" would split into two path segments, +// "v1?build" would start a query string, and a literal "%2B" would be read back +// as "+". Escaping here and decoding in splitWildcardPath round-trips the value, +// so the link resolves to the version that was stored. +func (v *Version) EscapedVersion() string { + return url.PathEscape(v.Version()) +} + +// DisplayPURL returns the PURL with its path components percent-decoded, for +// showing in the UI. The stored PURL keeps the canonical encoding (which is +// what the API and all lookups use); this is only a readable rendering, so that +// a version like "7.91+dfsg1-2ubuntu0.1" is not shown as "7.91%2Bdfsg1-2ubuntu0.1" +// and an npm scope is shown as "@babel" rather than "%40babel". Qualifiers and +// subpath keep their encoding, since decoding those would be ambiguous. +func (v *Version) DisplayPURL() string { + base, suffix := v.PURL, "" + if i := strings.IndexAny(base, "?#"); i >= 0 { + base, suffix = base[:i], base[i:] } - return "" + + name, version := base, "" + if idx := strings.LastIndex(base, "@"); idx >= 0 { + name, version = base[:idx], "@"+decodePURLComponent(base[idx+1:]) + } + + parts := strings.Split(name, "/") + for i, part := range parts { + parts[i] = decodePURLComponent(part) + } + return strings.Join(parts, "/") + version + suffix +} + +// VersionFromPURL extracts the decoded version string from a PURL. +// +// PURL percent-encodes characters that are not safe in a path component, so a +// Debian version like "7.91+dfsg1-2ubuntu0.1" is stored as +// "pkg:deb/nmap@7.91%2Bdfsg1-2ubuntu0.1". The raw substring after "@" is +// therefore not the version: it must be percent-decoded before being displayed +// or used to build a URL, otherwise "%2B" leaks into the UI and round-tripping +// the value back into a PURL double-encodes it. +// +// e.g., "pkg:npm/lodash@4.17.21" -> "4.17.21" +func VersionFromPURL(p string) string { + // Qualifiers ("?key=value") and subpath ("#path") follow the version. + if i := strings.IndexAny(p, "?#"); i >= 0 { + p = p[:i] + } + idx := strings.LastIndex(p, "@") + if idx < 0 { + return "" + } + return decodePURLComponent(p[idx+1:]) +} + +// decodePURLComponent percent-decodes a single PURL path component, returning +// the input unchanged if it is not valid percent-encoding. +func decodePURLComponent(s string) string { + if !strings.Contains(s, "%") { + return s + } + decoded, err := url.PathUnescape(s) + if err != nil { + return s + } + return decoded } // Artifact represents a cached artifact in the database. diff --git a/internal/database/version_purl_test.go b/internal/database/version_purl_test.go new file mode 100644 index 0000000..517022b --- /dev/null +++ b/internal/database/version_purl_test.go @@ -0,0 +1,159 @@ +package database + +import ( + "database/sql" + "net/url" + "testing" + "time" +) + +func TestVersionFromPURL(t *testing.T) { + tests := []struct { + name string + purl string + want string + }{ + {"simple", "pkg:npm/lodash@4.17.21", "4.17.21"}, + {"namespaced", "pkg:composer/symfony/console@6.0.0", "6.0.0"}, + // Debian/Ubuntu versions routinely contain "+", which PURL encodes. + {"encoded plus", "pkg:deb/nmap@7.91%2Bdfsg1%2Breally7.80%2Bdfsg1-2ubuntu0.1", "7.91+dfsg1+really7.80+dfsg1-2ubuntu0.1"}, + {"encoded epoch", "pkg:deb/curl@1%3A7.81.0-1", "1:7.81.0-1"}, + {"encoded plus with qualifier", "pkg:deb/nmap@7.91%2Bdfsg1?repository_url=http%3A%2F%2Fexample.com", "7.91+dfsg1"}, + {"tilde is not encoded", "pkg:deb/foo@1.0~rc1", "1.0~rc1"}, + {"no version", "pkg:npm/lodash", ""}, + {"invalid escape passed through", "pkg:npm/lodash@1.0%zz", "1.0%zz"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := VersionFromPURL(tt.purl); got != tt.want { + t.Errorf("VersionFromPURL(%q) = %q, want %q", tt.purl, got, tt.want) + } + v := &Version{PURL: tt.purl} + if got := v.Version(); got != tt.want { + t.Errorf("Version.Version() for %q = %q, want %q", tt.purl, got, tt.want) + } + }) + } +} + +// TestVersionEscapedVersion checks the value the templates put in a URL. It +// must survive the round trip back through the router: escaping here and +// decoding per path segment on the way in has to yield the original version. +func TestVersionEscapedVersion(t *testing.T) { + tests := []struct { + name string + purl string + want string + }{ + {"simple", "pkg:npm/lodash@4.17.21", "4.17.21"}, + // "+" is legal in a path segment, so it stays literal and the UI keeps + // showing the version the way Debian writes it. + {"plus stays literal", "pkg:deb/nmap@7.91%2Bdfsg1-2ubuntu0.1", "7.91+dfsg1-2ubuntu0.1"}, + // A slash would otherwise split the version into two path segments. + {"slash", "pkg:golang/example@release%2F1", "release%2F1"}, + // A question mark would otherwise start the query string. + {"question mark", "pkg:npm/example@v1%3Fbuild", "v1%3Fbuild"}, + // A version containing a literal "%2B" is stored double-encoded; the + // link must re-encode it or it decodes back to "+" instead. + {"literal percent escape", "pkg:npm/example@1.0%252B", "1.0%252B"}, + {"space", "pkg:npm/example@1.0%20beta", "1.0%20beta"}, + {"no version", "pkg:npm/lodash", ""}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + v := &Version{PURL: tt.purl} + got := v.EscapedVersion() + if got != tt.want { + t.Errorf("EscapedVersion() for %q = %q, want %q", tt.purl, got, tt.want) + } + // The router decodes each path segment, which must give back the + // version the page displays. + decoded, err := url.PathUnescape(got) + if err != nil { + t.Fatalf("PathUnescape(%q) failed: %v", got, err) + } + if decoded != v.Version() { + t.Errorf("round trip for %q = %q, want %q", tt.purl, decoded, v.Version()) + } + }) + } +} + +func TestVersionDisplayPURL(t *testing.T) { + tests := []struct { + name string + purl string + want string + }{ + {"simple", "pkg:npm/lodash@4.17.21", "pkg:npm/lodash@4.17.21"}, + { + "encoded plus", + "pkg:deb/nmap@7.91%2Bdfsg1%2Breally7.80%2Bdfsg1-2ubuntu0.1", + "pkg:deb/nmap@7.91+dfsg1+really7.80+dfsg1-2ubuntu0.1", + }, + { + "qualifier preserved", + "pkg:deb/nmap@7.91%2Bdfsg1?repository_url=http%3A%2F%2Fexample.com", + "pkg:deb/nmap@7.91+dfsg1?repository_url=http%3A%2F%2Fexample.com", + }, + // The namespace is encoded too: MakePURLString("npm", "@babel/core", …) + // produces "pkg:npm/%40babel/core@…". + {"encoded npm scope", "pkg:npm/%40babel/core@7.0.0", "pkg:npm/@babel/core@7.0.0"}, + {"encoded scope without version", "pkg:npm/%40babel/core", "pkg:npm/@babel/core"}, + {"no version", "pkg:npm/lodash", "pkg:npm/lodash"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + v := &Version{PURL: tt.purl} + if got := v.DisplayPURL(); got != tt.want { + t.Errorf("DisplayPURL() for %q = %q, want %q", tt.purl, got, tt.want) + } + }) + } +} + +// TestGetRecentlyCachedPackagesDecodesVersion guards the dashboard's "recently +// cached" list, which derives the version from the version PURL. +func TestGetRecentlyCachedPackagesDecodesVersion(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + const versionPURL = "pkg:deb/nmap@7.91%2Bdfsg1%2Breally7.80%2Bdfsg1-2ubuntu0.1" + + if err := db.UpsertPackage(&Package{ + PURL: "pkg:deb/nmap", Ecosystem: "deb", Name: "nmap", + }); err != nil { + t.Fatalf("UpsertPackage failed: %v", err) + } + if err := db.UpsertVersion(&Version{ + PURL: versionPURL, PackagePURL: "pkg:deb/nmap", + }); err != nil { + t.Fatalf("UpsertVersion failed: %v", err) + } + if err := db.UpsertArtifact(&Artifact{ + VersionPURL: versionPURL, + Filename: "nmap_7.91+dfsg1+really7.80+dfsg1-2ubuntu0.1_amd64.deb", + UpstreamURL: "http://archive.ubuntu.com/ubuntu/pool/universe/n/nmap/nmap.deb", + StoragePath: sql.NullString{String: "/cache/nmap.deb", Valid: true}, + FetchedAt: sql.NullTime{Time: time.Now(), Valid: true}, + }); err != nil { + t.Fatalf("UpsertArtifact failed: %v", err) + } + + recent, err := db.GetRecentlyCachedPackages(10) + if err != nil { + t.Fatalf("GetRecentlyCachedPackages failed: %v", err) + } + if len(recent) != 1 { + t.Fatalf("expected 1 recent package, got %d", len(recent)) + } + const want = "7.91+dfsg1+really7.80+dfsg1-2ubuntu0.1" + if recent[0].Version != want { + t.Errorf("Version = %q, want %q", recent[0].Version, want) + } + if recent[0].VersionPURL != versionPURL { + t.Errorf("VersionPURL = %q, want %q", recent[0].VersionPURL, versionPURL) + } + }) +} diff --git a/internal/handler/debian_test.go b/internal/handler/debian_test.go index 60fa23a..b086fdf 100644 --- a/internal/handler/debian_test.go +++ b/internal/handler/debian_test.go @@ -12,6 +12,11 @@ func TestDebianHandler_parsePoolPath(t *testing.T) { {"pool/main/libn/libncurses/libncurses6_6.2-1_amd64.deb", "libncurses6", "6.2-1", "amd64"}, {"pool/contrib/v/virtualbox/virtualbox_6.1.38-1_amd64.deb", "virtualbox", "6.1.38-1", "amd64"}, {"pool/main/g/git/git_2.39.2-1_arm64.deb", "git", "2.39.2-1", "arm64"}, + { + "pool/universe/n/nmap/nmap_7.91+dfsg1+really7.80+dfsg1-2ubuntu0.1_amd64.deb", + "nmap", "7.91+dfsg1+really7.80+dfsg1-2ubuntu0.1", "amd64", + }, + {"pool/main/o/openssl/openssl_3.0.2-0ubuntu1.15~build1_amd64.deb", "openssl", "3.0.2-0ubuntu1.15~build1", "amd64"}, {"invalid/path", "", "", ""}, {"pool/main/n/nginx/nginx.deb", "", "", ""}, }) diff --git a/internal/server/api.go b/internal/server/api.go index ddb9ca7..992d736 100644 --- a/internal/server/api.go +++ b/internal/server/api.go @@ -139,12 +139,11 @@ type BulkResponse struct { // Resolves namespaced package names (Composer vendor/name, npm @scope/name) from the path. func (h *APIHandler) HandlePackagePath(w http.ResponseWriter, r *http.Request) { ecosystem := chi.URLParam(r, "ecosystem") - wildcard := chi.URLParam(r, "*") - if err := validatePackagePath(wildcard); err != nil { + segments, err := packagePathSegments(r) + if err != nil { badRequest(w, err.Error()) return } - segments := splitWildcardPath(wildcard) if ecosystem == "" || len(segments) == 0 { badRequest(w, "ecosystem and name are required") @@ -277,12 +276,11 @@ func (h *APIHandler) getVersion(w http.ResponseWriter, r *http.Request, ecosyste // Supports both {name} and {name}/{version} paths with namespaced package names. func (h *APIHandler) HandleVulnsPath(w http.ResponseWriter, r *http.Request) { ecosystem := chi.URLParam(r, "ecosystem") - wildcard := chi.URLParam(r, "*") - if err := validatePackagePath(wildcard); err != nil { + segments, err := packagePathSegments(r) + if err != nil { badRequest(w, err.Error()) return } - segments := splitWildcardPath(wildcard) if ecosystem == "" || len(segments) == 0 { badRequest(w, "ecosystem and name are required") diff --git a/internal/server/browse.go b/internal/server/browse.go index c60cba3..43ad9ae 100644 --- a/internal/server/browse.go +++ b/internal/server/browse.go @@ -147,12 +147,11 @@ type BrowseFileInfo struct { // {name}/{version}/file/{path} -> browse file func (s *Server) handleBrowsePath(w http.ResponseWriter, r *http.Request) { ecosystem := chi.URLParam(r, "ecosystem") - wildcard := chi.URLParam(r, "*") - if err := validatePackagePath(wildcard); err != nil { + segments, err := packagePathSegments(r) + if err != nil { badRequest(w, err.Error()) return } - segments := splitWildcardPath(wildcard) if ecosystem == "" || len(segments) < 2 { badRequest(w, "ecosystem, name, and version required") @@ -203,12 +202,11 @@ func (s *Server) handleBrowsePath(w http.ResponseWriter, r *http.Request) { // Supported paths: {name}/{fromVersion}/{toVersion} func (s *Server) handleComparePath(w http.ResponseWriter, r *http.Request) { ecosystem := chi.URLParam(r, "ecosystem") - wildcard := chi.URLParam(r, "*") - if err := validatePackagePath(wildcard); err != nil { + segments, err := packagePathSegments(r) + if err != nil { badRequest(w, err.Error()) return } - segments := splitWildcardPath(wildcard) if ecosystem == "" || len(segments) < 3 { badRequest(w, "ecosystem, name, fromVersion, and toVersion required") @@ -506,11 +504,16 @@ func isLikelyText(filename string) bool { } // BrowseSourceData contains data for the browse source page. +// +// Version is the decoded version, for display. EscapedVersion is the same value +// escaped as a single URL path segment and is what the links and the browse API +// calls must use; see database.Version.EscapedVersion. type BrowseSourceData struct { Layout - Ecosystem string - PackageName string - Version string + Ecosystem string + PackageName string + Version string + EscapedVersion string } // handleBrowseSource is now showBrowseSource in server.go, dispatched via handlePackagePath. @@ -601,12 +604,17 @@ func (s *Server) compareDiff(w http.ResponseWriter, r *http.Request, ecosystem, } // ComparePageData contains data for the version comparison page. +// +// FromVersion and ToVersion are decoded, for display; the Escaped variants are +// the path-segment form used to build the compare API URL. type ComparePageData struct { Layout - Ecosystem string - PackageName string - FromVersion string - ToVersion string + Ecosystem string + PackageName string + FromVersion string + ToVersion string + EscapedFromVersion string + EscapedToVersion string } // handleComparePage is now showComparePage in server.go, dispatched via handlePackagePath. diff --git a/internal/server/browse_test.go b/internal/server/browse_test.go index 6ea0f7e..3cc37c8 100644 --- a/internal/server/browse_test.go +++ b/internal/server/browse_test.go @@ -450,8 +450,10 @@ func TestHandleBrowseSourcePage(t *testing.T) { if !strings.Contains(body, "const packageName = 'test-browse'") { t.Error("browse source page missing packageName variable") } - if !strings.Contains(body, "const version = '1.0.0'") { - t.Error("browse source page missing version variable") + // The version reaches the browse API as one path segment, so the page holds + // its escaped form. + if !strings.Contains(body, "const versionPath = '1.0.0'") { + t.Error("browse source page missing versionPath variable") } // Verify content type @@ -617,12 +619,13 @@ func TestHandleComparePage(t *testing.T) { body := w.Body.String() - // Check that versions are set correctly in JavaScript - if !strings.Contains(body, "const fromVersion = '1.0.0'") { - t.Error("page should set fromVersion") + // Check that versions are set correctly in JavaScript. The compare API takes + // each version as a path segment, so the page holds their escaped forms. + if !strings.Contains(body, "const fromVersionPath = '1.0.0'") { + t.Error("page should set fromVersionPath") } - if !strings.Contains(body, "const toVersion = '2.0.0'") { - t.Error("page should set toVersion") + if !strings.Contains(body, "const toVersionPath = '2.0.0'") { + t.Error("page should set toVersionPath") } // Test invalid format (missing separator) diff --git a/internal/server/resolve.go b/internal/server/resolve.go index 51f203d..f7a1b23 100644 --- a/internal/server/resolve.go +++ b/internal/server/resolve.go @@ -2,10 +2,13 @@ package server import ( "fmt" + "net/http" + "net/url" "strings" "unicode" "github.com/git-pkgs/proxy/internal/database" + "github.com/go-chi/chi/v5" ) // maxPackagePathLen bounds the wildcard portion of package routes (name plus @@ -13,22 +16,69 @@ import ( // longer, so 512 leaves room without admitting pathological inputs. const maxPackagePathLen = 512 +// packagePathSegments validates the wildcard portion of a package route and +// splits it into decoded path segments. +func packagePathSegments(r *http.Request) ([]string, error) { + wildcard := chi.URLParam(r, "*") + encoded := wildcardIsEncoded(r) + if err := validatePackagePath(wildcard, encoded); err != nil { + return nil, err + } + + return splitWildcardPath(wildcard, encoded), nil +} + +// wildcardIsEncoded reports whether the chi wildcard for this request is still +// percent-encoded. +// +// chi routes on r.URL.RawPath when it is set and on r.URL.Path otherwise, and +// net/url only sets RawPath when the request's escaping differs from the +// canonical encoding of the decoded path. A version such as "release%2F1" is +// therefore routed raw, while "1.0%252B" (a version whose text contains a +// literal "%2B") encodes canonically and arrives already decoded once. The +// distinction decides whether the segments still need decoding: decoding the +// second case again would turn it into "1.0+" and resolve a different version. +func wildcardIsEncoded(r *http.Request) bool { + return r.URL.RawPath != "" +} + // validatePackagePath rejects wildcard package paths that cannot be valid in // any supported ecosystem. It is a coarse filter applied before database or // enrichment lookups; ecosystem-specific name rules are layered on top. -func validatePackagePath(path string) error { +// +// encoded has the meaning described on wildcardIsEncoded. +func validatePackagePath(path string, encoded bool) error { if path == "" { return fmt.Errorf("package name required") } if len(path) > maxPackagePathLen { return fmt.Errorf("package path exceeds %d bytes", maxPackagePathLen) } - for _, r := range path { - if r == 0 { - return fmt.Errorf("package path contains null byte") - } - if unicode.IsControl(r) { - return fmt.Errorf("package path contains control character %#U", r) + // Validate the decoded segments: the handlers work with decoded values, so + // an escape such as "%00" or "%2E%2E" must not slip past these checks. + for _, seg := range splitWildcardPath(path, encoded) { + // Each segment is checked both as the handlers see it and decoded once + // more: a segment can reach a handler with escapes intact, and the + // upstream registry is then the one that decodes them. + for _, value := range []string{seg, decodePathSegment(seg)} { + // A decoded segment can itself contain slashes (from "%2F"), and + // the segments are later rejoined into a package name that + // registries interpolate straight into an upstream URL. Check every + // path element, not just the segment as a whole, or + // "a%2F..%2F..%2Fb" traverses. + for _, elem := range strings.Split(value, "/") { + if elem == ".." { + return fmt.Errorf("package path contains parent directory segment") + } + } + for _, r := range value { + if r == 0 { + return fmt.Errorf("package path contains null byte") + } + if unicode.IsControl(r) { + return fmt.Errorf("package path contains control character %#U", r) + } + } } } return nil @@ -60,10 +110,37 @@ func resolvePackageName(db *database.DB, ecosystem string, segments []string) (n // splitWildcardPath splits a chi wildcard path value into segments, // trimming any leading/trailing slashes. -func splitWildcardPath(path string) []string { +// +// When encoded is set the value is still percent-encoded (see +// wildcardIsEncoded), so each segment is decoded after splitting. Splitting +// first keeps an encoded "%2F" inside a name from being mistaken for a +// separator. Decoding matters for versions such as "1.0%2Bbuild1", which must +// reach the handlers as "1.0+build1" so that rebuilding the PURL yields the +// value that was stored rather than a double-encoded one. +func splitWildcardPath(path string, encoded bool) []string { path = strings.Trim(path, "/") if path == "" { return nil } - return strings.Split(path, "/") + segments := strings.Split(path, "/") + if !encoded { + return segments + } + for i, seg := range segments { + segments[i] = decodePathSegment(seg) + } + return segments +} + +// decodePathSegment percent-decodes a single URL path segment, returning it +// unchanged if it is not valid percent-encoding. +func decodePathSegment(seg string) string { + if !strings.Contains(seg, "%") { + return seg + } + decoded, err := url.PathUnescape(seg) + if err != nil { + return seg + } + return decoded } diff --git a/internal/server/resolve_test.go b/internal/server/resolve_test.go index dd7d2dc..1867f46 100644 --- a/internal/server/resolve_test.go +++ b/internal/server/resolve_test.go @@ -1,12 +1,15 @@ package server import ( + "net/http" + "net/http/httptest" "os" "path/filepath" "strings" "testing" "github.com/git-pkgs/proxy/internal/database" + "github.com/go-chi/chi/v5" ) func newTestDB(t *testing.T) (*database.DB, func()) { @@ -95,26 +98,44 @@ func TestResolvePackageName(t *testing.T) { func TestSplitWildcardPath(t *testing.T) { tests := []struct { - input string - want []string + input string + encoded bool + want []string }{ - {"lodash", []string{"lodash"}}, - {"lodash/4.17.21", []string{"lodash", "4.17.21"}}, - {"monolog/monolog", []string{"monolog", "monolog"}}, - {"symfony/console/6.0.0/browse", []string{"symfony", "console", "6.0.0", "browse"}}, - {"", nil}, - {"/", nil}, + {"lodash", false, []string{"lodash"}}, + {"lodash/4.17.21", false, []string{"lodash", "4.17.21"}}, + {"monolog/monolog", false, []string{"monolog", "monolog"}}, + {"symfony/console/6.0.0/browse", false, []string{"symfony", "console", "6.0.0", "browse"}}, + {"", false, nil}, + {"/", false, nil}, + // chi routes on the raw path when it differs from the canonical + // encoding of the decoded path, so segments arrive percent-encoded and + // must be decoded. + { + "nmap/7.91%2Bdfsg1%2Breally7.80%2Bdfsg1-2ubuntu0.1", true, + []string{"nmap", "7.91+dfsg1+really7.80+dfsg1-2ubuntu0.1"}, + }, + {"%40babel/core/7.0.0", true, []string{"@babel", "core", "7.0.0"}}, + // An encoded separator stays inside its segment rather than splitting. + {"vendor%2Fname/1.0.0", true, []string{"vendor/name", "1.0.0"}}, + // Invalid escapes are passed through untouched. + {"lodash/1.0%zz", true, []string{"lodash", "1.0%zz"}}, + // When chi routed on the already-decoded path, an escape that survived + // is part of the value: a version whose text is "1.0%2B" reaches here + // as "1.0%2B" and decoding it again would yield "1.0+". + {"nmap/1.0%2B", false, []string{"nmap", "1.0%2B"}}, } for _, tt := range tests { - got := splitWildcardPath(tt.input) + got := splitWildcardPath(tt.input, tt.encoded) if len(got) != len(tt.want) { - t.Errorf("splitWildcardPath(%q) = %v, want %v", tt.input, got, tt.want) + t.Errorf("splitWildcardPath(%q, %v) = %v, want %v", tt.input, tt.encoded, got, tt.want) continue } for i := range got { if got[i] != tt.want[i] { - t.Errorf("splitWildcardPath(%q)[%d] = %q, want %q", tt.input, i, got[i], tt.want[i]) + t.Errorf("splitWildcardPath(%q, %v)[%d] = %q, want %q", + tt.input, tt.encoded, i, got[i], tt.want[i]) } } } @@ -132,8 +153,19 @@ func TestValidatePackagePath(t *testing.T) { {"composer namespaced", "symfony/console/6.0.0", false}, {"maven coordinates", "org.apache.commons/commons-lang3/3.12.0", false}, {"unicode", "café/1.0.0", false}, + {"encoded plus in version", "nmap/7.91%2Bdfsg1-2ubuntu0.1", false}, {"empty", "", true}, {"null byte", "lodash\x00/4.17.21", true}, + {"encoded null byte", "lodash/%00", true}, + {"encoded newline", "lodash/1.0%0A", true}, + {"parent segment", "lodash/../4.17.21", true}, + {"encoded parent segment", "lodash/%2E%2E/4.17.21", true}, + // A decoded segment can contain slashes, so traversal can hide inside + // one segment. Registries interpolate the resolved name straight into + // an upstream URL, and Go sends dot-segments verbatim. + {"traversal inside one segment", "pkg%2F..%2F..%2Fadmin", true}, + {"traversal via encoded dots and slash", "pkg%2f%2e%2e%2fadmin", true}, + {"encoded slash alone is allowed", "vendor%2Fname/1.0.0", false}, {"null byte suffix", "lodash\x00", true}, {"newline", "lodash\n4.17.21", true}, {"carriage return", "lodash\r", true}, @@ -145,9 +177,64 @@ func TestValidatePackagePath(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - err := validatePackagePath(tt.path) - if (err != nil) != tt.wantErr { - t.Errorf("validatePackagePath(%q) error = %v, wantErr %v", tt.path, err, tt.wantErr) + // The verdict must not depend on whether chi routed on the raw or + // on the already-decoded path: an escape that reaches a handler + // undecoded is decoded by the upstream registry instead, so it is + // rejected either way. + for _, encoded := range []bool{false, true} { + err := validatePackagePath(tt.path, encoded) + if (err != nil) != tt.wantErr { + t.Errorf("validatePackagePath(%q, %v) error = %v, wantErr %v", + tt.path, encoded, err, tt.wantErr) + } + } + }) + } +} + +// TestPackagePathSegments drives the real router, which is what decides whether +// the wildcard still carries percent-encoding. Go decodes the request path +// itself unless the escaping is non-canonical, so the same version can arrive +// either way and only one of the two forms may be decoded again. +func TestPackagePathSegments(t *testing.T) { + tests := []struct { + name string + target string + want []string + }{ + {"plain", "/pkg/npm/lodash/4.17.21", []string{"lodash", "4.17.21"}}, + {"encoded plus", "/pkg/deb/nmap/7.91%2Bdfsg1-2ubuntu0.1", []string{"nmap", "7.91+dfsg1-2ubuntu0.1"}}, + {"decoded plus", "/pkg/deb/nmap/7.91+dfsg1-2ubuntu0.1", []string{"nmap", "7.91+dfsg1-2ubuntu0.1"}}, + // An encoded slash is one segment, not a separator. + {"encoded slash", "/pkg/composer/vendor%2Fname/1.0.0", []string{"vendor/name", "1.0.0"}}, + {"question mark", "/pkg/npm/example/v1%3Fbuild", []string{"example", "v1?build"}}, + // "1.0%252B" is the escaped form of the version "1.0%2B"; net/url + // already decoded it once, so it must not be decoded again. + {"literal percent escape", "/pkg/npm/example/1.0%252B", []string{"example", "1.0%2B"}}, + {"browse suffix", "/pkg/deb/nmap/7.91%2Bdfsg1/browse", []string{"nmap", "7.91+dfsg1", "browse"}}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var got []string + var gotErr error + + router := chi.NewRouter() + router.Get("/pkg/{ecosystem}/*", func(_ http.ResponseWriter, r *http.Request) { + got, gotErr = packagePathSegments(r) + }) + router.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest("GET", tt.target, nil)) + + if gotErr != nil { + t.Fatalf("packagePathSegments(%q) failed: %v", tt.target, gotErr) + } + if len(got) != len(tt.want) { + t.Fatalf("segments for %q = %v, want %v", tt.target, got, tt.want) + } + for i := range got { + if got[i] != tt.want[i] { + t.Errorf("segments for %q [%d] = %q, want %q", tt.target, i, got[i], tt.want[i]) + } } }) } diff --git a/internal/server/server.go b/internal/server/server.go index c98d1cb..e677bc9 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -54,6 +54,7 @@ import ( "fmt" "log/slog" "net/http" + "net/url" "strconv" "strings" "time" @@ -680,12 +681,11 @@ func (s *Server) handlePackagesList(w http.ResponseWriter, r *http.Request) { // {name}/compare/{v1}...{v2} -> compare versions func (s *Server) handlePackagePath(w http.ResponseWriter, r *http.Request) { ecosystem := chi.URLParam(r, "ecosystem") - wildcard := chi.URLParam(r, "*") - if err := validatePackagePath(wildcard); err != nil { + segments, err := packagePathSegments(r) + if err != nil { http.Error(w, err.Error(), http.StatusBadRequest) return } - segments := splitWildcardPath(wildcard) if ecosystem == "" || len(segments) == 0 { http.Error(w, "ecosystem and package name required", http.StatusBadRequest) @@ -825,10 +825,11 @@ func (s *Server) showVersion(w http.ResponseWriter, r *http.Request, ecosystem, func (s *Server) showBrowseSource(w http.ResponseWriter, r *http.Request, ecosystem, name, version string) { data := BrowseSourceData{ - Layout: s.layoutFor(r), - Ecosystem: ecosystem, - PackageName: name, - Version: version, + Layout: s.layoutFor(r), + Ecosystem: ecosystem, + PackageName: name, + Version: version, + EscapedVersion: url.PathEscape(version), } if err := s.templates.Render(w, "browse_source", data); err != nil { @@ -846,11 +847,13 @@ func (s *Server) showComparePage(w http.ResponseWriter, r *http.Request, ecosyst } data := ComparePageData{ - Layout: s.layoutFor(r), - Ecosystem: ecosystem, - PackageName: name, - FromVersion: parts[0], - ToVersion: parts[1], + Layout: s.layoutFor(r), + Ecosystem: ecosystem, + PackageName: name, + FromVersion: parts[0], + ToVersion: parts[1], + EscapedFromVersion: url.PathEscape(parts[0]), + EscapedToVersion: url.PathEscape(parts[1]), } if err := s.templates.Render(w, "compare_versions", data); err != nil { diff --git a/internal/server/server_test.go b/internal/server/server_test.go index 2d27147..98b58cc 100644 --- a/internal/server/server_test.go +++ b/internal/server/server_test.go @@ -4,12 +4,16 @@ import ( "database/sql" "encoding/json" "fmt" + "html" "io" "log/slog" "net/http" "net/http/httptest" + "net/url" "os" "path/filepath" + "regexp" + "strconv" "strings" "testing" "time" @@ -18,6 +22,7 @@ import ( "github.com/git-pkgs/proxy/internal/database" "github.com/git-pkgs/proxy/internal/handler" "github.com/git-pkgs/proxy/internal/storage" + "github.com/git-pkgs/purl" "github.com/git-pkgs/registries/fetch" "github.com/go-chi/chi/v5" ) @@ -764,6 +769,236 @@ func TestVersionShowPage_NotFoundServer(t *testing.T) { } } +// TestVersionShowPage_PlusInVersion covers Debian/Ubuntu style versions such as +// nmap's "7.91+dfsg1+really7.80+dfsg1-2ubuntu0.1". PURL percent-encodes "+" as +// "%2B", so the UI must show the decoded version and resolve both the decoded +// and the still-encoded form of the URL back to the same version. +func TestVersionShowPage_PlusInVersion(t *testing.T) { + ts := newTestServer(t) + defer ts.close() + + const version = "7.91+dfsg1+really7.80+dfsg1-2ubuntu0.1" + const versionPURL = "pkg:deb/nmap@7.91%2Bdfsg1%2Breally7.80%2Bdfsg1-2ubuntu0.1" + + pkg := &database.Package{PURL: "pkg:deb/nmap", Ecosystem: "deb", Name: "nmap"} + if err := ts.db.UpsertPackage(pkg); err != nil { + t.Fatalf("failed to upsert package: %v", err) + } + if err := ts.db.UpsertVersion(&database.Version{ + PURL: versionPURL, PackagePURL: pkg.PURL, + }); err != nil { + t.Fatalf("failed to upsert version: %v", err) + } + + // The package page must link to and display the decoded version. + req := httptest.NewRequest("GET", "/ui/package/deb/nmap", nil) + w := httptest.NewRecorder() + ts.handler.ServeHTTP(w, req) + if w.Code != http.StatusOK { + t.Fatalf("package page: expected status 200, got %d", w.Code) + } + body := w.Body.String() + if strings.Contains(body, "%2B") { + t.Error("package page leaks PURL percent-encoding into the UI") + } + // html/template renders "+" as the "+" entity inside attributes and text. + if !strings.Contains(body, "7.91+dfsg1+really7.80+dfsg1-2ubuntu0.1") { + t.Error("expected package page to show the decoded version") + } + + // Both the decoded and the encoded URL must reach the version page. + for _, path := range []string{ + "/ui/package/deb/nmap/" + version, + "/ui/package/deb/nmap/7.91%2Bdfsg1%2Breally7.80%2Bdfsg1-2ubuntu0.1", + } { + req := httptest.NewRequest("GET", path, nil) + w := httptest.NewRecorder() + ts.handler.ServeHTTP(w, req) + if w.Code != http.StatusOK { + t.Errorf("GET %s: expected status 200, got %d", path, w.Code) + } + } +} + +// TestVersionURLEscaping covers versions whose characters are significant in a +// URL path: "/" splits off another path segment, "?" starts a query string, and +// a literal "%xx" is read back as the character it encodes. The pages show the +// decoded version but must build every link from a separately escaped value, +// and those links have to resolve back to the same version. +func TestVersionURLEscaping(t *testing.T) { + // A second version is needed for the compare controls to be rendered. + const otherVersion = "1.0.0" + + tests := []struct { + name string + version string + }{ + {"slash", "release/1"}, + {"question mark", "v1?build"}, + {"literal percent escape", "1.0%2B"}, + {"plus", "7.91+dfsg1-2ubuntu0.1"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ts := newTestServer(t) + defer ts.close() + seedEscapingVersions(t, ts.db, tt.version, otherVersion) + + // The package page links to the escaped version. + escaped := url.PathEscape(tt.version) + versionPath := "/ui/package/deb/nmap/" + escaped + packagePage := ts.getOK(t, "/ui/package/deb/nmap") + if !containsValue(attrValues(packagePage, "href"), versionPath) { + t.Fatalf("package page has no link to %q; hrefs: %v", + versionPath, attrValues(packagePage, "href")) + } + + ts.checkVersionAndBrowsePages(t, versionPath, tt.version, escaped) + ts.checkComparePage(t, packagePage, tt.version, escaped, otherVersion) + }) + } +} + +// seedEscapingVersions stores a Debian package with the given versions, each +// with a cached artifact so that the version page offers its browse link. +func seedEscapingVersions(t *testing.T, db *database.DB, versions ...string) { + t.Helper() + + pkg := &database.Package{PURL: "pkg:deb/nmap", Ecosystem: "deb", Name: "nmap"} + if err := db.UpsertPackage(pkg); err != nil { + t.Fatalf("failed to upsert package: %v", err) + } + for _, v := range versions { + versionPURL := purl.MakePURLString("deb", "nmap", v) + if err := db.UpsertVersion(&database.Version{ + PURL: versionPURL, PackagePURL: pkg.PURL, + }); err != nil { + t.Fatalf("failed to upsert version %q: %v", v, err) + } + if err := db.UpsertArtifact(&database.Artifact{ + VersionPURL: versionPURL, + Filename: "nmap.deb", + UpstreamURL: "http://archive.ubuntu.com/ubuntu/pool/universe/n/nmap/nmap.deb", + StoragePath: sql.NullString{String: "/cache/nmap.deb", Valid: true}, + FetchedAt: sql.NullTime{Time: time.Now(), Valid: true}, + }); err != nil { + t.Fatalf("failed to upsert artifact for %q: %v", v, err) + } + } +} + +// checkVersionAndBrowsePages follows a version link from the package page and +// then the browse link from the version page, checking that both resolve to the +// stored version and display it decoded. +func (ts *testServer) checkVersionAndBrowsePages(t *testing.T, versionPath, version, escaped string) { + t.Helper() + + versionPage := ts.getOK(t, versionPath) + wantPURL := "pkg:deb/nmap@" + version + if !strings.Contains(html.UnescapeString(versionPage), wantPURL) { + t.Errorf("version page does not show %q", wantPURL) + } + + browsePath := versionPath + "/browse" + if !containsValue(attrValues(versionPage, "href"), browsePath) { + t.Fatalf("version page has no browse link to %q; hrefs: %v", + browsePath, attrValues(versionPage, "href")) + } + + browsePage := ts.getOK(t, browsePath) + if !strings.Contains(html.UnescapeString(browsePage), "nmap@"+version) { + t.Errorf("browse page does not show the decoded version %q", version) + } + // The browse API is called with the escaped version, not with the text shown + // in the heading. + if got := jsConstant(t, browsePage, "versionPath"); got != escaped { + t.Errorf("browse page passes %q to the browse API, want %q", got, escaped) + } +} + +// checkComparePage builds the compare URL the way the package page's script +// does, from the values its checkboxes carry, and checks the page it reaches. +func (ts *testServer) checkComparePage(t *testing.T, packagePage, version, escaped, otherVersion string) { + t.Helper() + + selectable := attrValues(packagePage, "data-version-path") + if !containsValue(selectable, escaped) { + t.Fatalf("package page compare data holds %v, want %q", selectable, escaped) + } + + comparePage := ts.getOK(t, "/ui/package/deb/nmap/compare/"+escaped+"..."+otherVersion) + decoded := html.UnescapeString(comparePage) + for _, want := range []string{version, otherVersion} { + if !strings.Contains(decoded, want) { + t.Errorf("compare page does not show version %q", want) + } + } + if got := jsConstant(t, comparePage, "fromVersionPath"); got != escaped { + t.Errorf("compare page passes %q to the compare API, want %q", got, escaped) + } +} + +// getOK performs a GET against the server and fails the test unless it returns +// 200, returning the response body. +func (ts *testServer) getOK(t *testing.T, path string) string { + t.Helper() + + req := httptest.NewRequest("GET", path, nil) + w := httptest.NewRecorder() + ts.handler.ServeHTTP(w, req) + if w.Code != http.StatusOK { + t.Fatalf("GET %s: expected status 200, got %d", path, w.Code) + } + + return w.Body.String() +} + +// attrValues returns the value of every occurrence of an HTML attribute in a +// rendered page, with HTML entities resolved so that values can be compared +// against the raw strings they were built from. +func attrValues(body, attr string) []string { + re := regexp.MustCompile(regexp.QuoteMeta(attr) + `="([^"]*)"`) + + var values []string + for _, match := range re.FindAllStringSubmatch(body, -1) { + values = append(values, html.UnescapeString(match[1])) + } + + return values +} + +// jsConstant returns the value of a single-quoted JavaScript string constant in +// a rendered page. html/template escapes characters that are significant in +// JavaScript, rendering "+" as "\\u002b" for instance, so the escapes are +// resolved to recover the value the page actually uses. +func jsConstant(t *testing.T, body, name string) string { + t.Helper() + + re := regexp.MustCompile(`const ` + regexp.QuoteMeta(name) + ` = '([^']*)'`) + match := re.FindStringSubmatch(body) + if match == nil { + t.Fatalf("page does not declare the constant %q", name) + } + + unescaped, err := strconv.Unquote(`"` + match[1] + `"`) + if err != nil { + t.Fatalf("cannot unescape %q: %v", match[1], err) + } + + return unescaped +} + +func containsValue(values []string, want string) bool { + for _, v := range values { + if v == want { + return true + } + } + + return false +} + func TestPackageShowPage_WithLicense(t *testing.T) { ts := newTestServer(t) defer ts.close() diff --git a/internal/server/templates/pages/browse_source.html b/internal/server/templates/pages/browse_source.html index ca06652..a949111 100644 --- a/internal/server/templates/pages/browse_source.html +++ b/internal/server/templates/pages/browse_source.html @@ -7,7 +7,7 @@ / {{.PackageName}} / - {{.Version}} + {{.Version}} / Browse Source @@ -51,7 +51,10 @@ +{{end}} diff --git a/internal/server/templates/pages/dashboard.html b/internal/server/templates/pages/dashboard.html index 9b9a9e2..e48e205 100644 --- a/internal/server/templates/pages/dashboard.html +++ b/internal/server/templates/pages/dashboard.html @@ -22,35 +22,7 @@ {{if .EnrichmentStats.HasVulns}} - -
-
-

Security Overview

-
-
-
-
-
{{.EnrichmentStats.CriticalVulns}}
-
Critical
-
-
-
{{.EnrichmentStats.HighVulns}}
-
High
-
-
-
{{.EnrichmentStats.MediumVulns}}
-
Medium
-
-
-
{{.EnrichmentStats.LowVulns}}
-
Low
-
-
-

- {{.EnrichmentStats.TotalVulnerabilities}} vulnerabilities tracked across {{.EnrichmentStats.VulnSyncedPackages}} packages -

-
-
+{{template "security_overview" .EnrichmentStats}} {{end}} diff --git a/internal/server/templates_test.go b/internal/server/templates_test.go index a2d3b7d..c68a69a 100644 --- a/internal/server/templates_test.go +++ b/internal/server/templates_test.go @@ -37,6 +37,46 @@ func TestTemplatesRenderAllPages(t *testing.T) { {Ecosystem: "cargo", Name: "serde", Version: "1.0.0", Size: "200 KB", CachedAt: "1 hour ago"}, }, }}, + {"analytics", AnalyticsData{ + Totals: AnalyticsTotals{ + DownloadedBytes: 1_500_000_000, + Downloaded: "1.4 GB", + Downloads: "12,004", + CacheSize: "420.0 MB", + CachedArtifacts: "1,204", + Packages: "310", + Versions: "902", + Ecosystems: 3, + ActiveEcosystems: 2, + Amplification: "3.4x", + }, + EnrichmentStats: EnrichmentStatsView{TotalVulnerabilities: 3, CriticalVulns: 1, HasVulns: true}, + Ecosystems: []EcosystemRow{ + {Ecosystem: "npm", DownloadedBytes: 1_000_000_000, Downloaded: "953.7 MB", Downloads: "9,000", CacheSize: "300.0 MB", AvgArtifactSize: "120 KB", Artifacts: "900", Packages: "200", Versions: "700", SharePct: "66.7"}, + {Ecosystem: "cargo", DownloadedBytes: 500_000_000, Downloaded: "476.8 MB", Downloads: "3,004", CacheSize: "120.0 MB", AvgArtifactSize: "400 KB", Artifacts: "300", Packages: "100", Versions: "200", SharePct: "33.3"}, + {Ecosystem: "rpm", Downloaded: "0 B", Downloads: "0", CacheSize: "0 B", AvgArtifactSize: "0 B", Artifacts: "0", Packages: "10", Versions: "2", SharePct: "0.0"}, + }, + Donut: donutView([]database.EcosystemStats{ + {Ecosystem: "npm", CacheSize: 300, Downloads: 9000, DownloadedBytes: 1_000_000_000}, + {Ecosystem: "cargo", CacheSize: 120, Downloads: 3004, DownloadedBytes: 500_000_000}, + }, 1_500_000_000, "1.4 GB"), + Runtime: RuntimeView{ + Available: true, + Requests: "12,004", + ActiveRequests: "2", + RequestMean: "8.1 ms", + CacheHits: "9,000", + CacheMisses: "1,000", + CacheHitRatio: "90.0", + HasCacheTraffic: true, + StatusClasses: []LabelledCount{{Label: "2xx", Count: "11,900"}, {Label: "5xx", Count: "104", Bad: true}}, + StorageOps: []LabelledStat{{Label: "get", Count: "9,000", Mean: "412 µs"}}, + ScanErrors: []LabelledCount{{Label: "npm · clamav · timeout", Count: "3", Bad: true}}, + ScanningOn: true, + Breakers: []BreakerRow{{Registry: "registry.npmjs.org", State: "closed"}}, + }, + }}, + {"analytics", AnalyticsData{}}, {"install", struct { Layout BaseURL string From 4b8c5a89fb9d2678428fc75276d356b835a4e53d Mon Sep 17 00:00:00 2001 From: Wicliff Wolda Date: Thu, 1 Oct 2026 14:32:20 +0200 Subject: [PATCH 160/633] Report the per-ecosystem breakdown from GET /stats - Add downloaded_bytes, downloads and an ecosystems array, served from the snapshot the gauges and the page already share - Add stats_unavailable so a failed aggregation is distinguishable from an idle proxy - Regenerate the OpenAPI spec Part 4 of 6 splitting #381 up. Co-Authored-By: Claude Opus 5 (1M context) --- README.md | 2 + docs/swagger/docs.go | 49 ++++++++++++ docs/swagger/swagger.json | 49 ++++++++++++ internal/server/server.go | 59 +++++++++++++-- internal/server/stats_test.go | 139 ++++++++++++++++++++++++++++++++++ 5 files changed, 293 insertions(+), 5 deletions(-) create mode 100644 internal/server/stats_test.go diff --git a/README.md b/README.md index 3390bd9..079002b 100644 --- a/README.md +++ b/README.md @@ -1184,6 +1184,8 @@ The proxy stores no time series. The page reads the database and the in-process For history, trends and alerting, scrape `/metrics` with Prometheus. That is the intended split: the UI answers "what is true now", Prometheus answers "what happened". +The same figures are available as JSON from `GET /stats`, which reports `downloaded_bytes`, `downloads` and an `ecosystems` array carrying the per-ecosystem breakdown, served from the same 60-second snapshot the page and the gauges read. When the aggregation fails with no snapshot to fall back on, the response carries `stats_unavailable: true` rather than passing zeros off as a count -- the endpoint keeps answering with the artifact count and cache size either way. + #### Three ecosystem label sets `ecosystem` means three slightly different things across `/metrics`, and queries that join across them need to know which. diff --git a/docs/swagger/docs.go b/docs/swagger/docs.go index cc88b4c..dd50bf9 100644 --- a/docs/swagger/docs.go +++ b/docs/swagger/docs.go @@ -559,6 +559,35 @@ const docTemplate = `{ } } }, + "server.EcosystemStatsEntry": { + "type": "object", + "properties": { + "cache_size_bytes": { + "type": "integer" + }, + "cached_artifacts": { + "type": "integer" + }, + "downloaded": { + "type": "string" + }, + "downloaded_bytes": { + "type": "integer" + }, + "downloads": { + "type": "integer" + }, + "ecosystem": { + "type": "string" + }, + "packages": { + "type": "integer" + }, + "versions": { + "type": "integer" + } + } + }, "server.ErrorResponse": { "type": "object", "properties": { @@ -806,6 +835,26 @@ const docTemplate = `{ "database_path": { "type": "string" }, + "downloaded": { + "type": "string" + }, + "downloaded_bytes": { + "description": "DownloadedBytes is the accumulated download volume across every\necosystem: cache hits multiplied by the artifact size they served.", + "type": "integer" + }, + "downloads": { + "type": "integer" + }, + "ecosystems": { + "type": "array", + "items": { + "$ref": "#/definitions/server.EcosystemStatsEntry" + } + }, + "stats_unavailable": { + "description": "StatsUnavailable distinguishes a proxy that has served nothing from one\nwhose aggregation failed with no snapshot to fall back on. Without it\nboth report zeros and an empty array.", + "type": "boolean" + }, "storage_url": { "type": "string" }, diff --git a/docs/swagger/swagger.json b/docs/swagger/swagger.json index 5db9166..50007b6 100644 --- a/docs/swagger/swagger.json +++ b/docs/swagger/swagger.json @@ -552,6 +552,35 @@ } } }, + "server.EcosystemStatsEntry": { + "type": "object", + "properties": { + "cache_size_bytes": { + "type": "integer" + }, + "cached_artifacts": { + "type": "integer" + }, + "downloaded": { + "type": "string" + }, + "downloaded_bytes": { + "type": "integer" + }, + "downloads": { + "type": "integer" + }, + "ecosystem": { + "type": "string" + }, + "packages": { + "type": "integer" + }, + "versions": { + "type": "integer" + } + } + }, "server.ErrorResponse": { "type": "object", "properties": { @@ -799,6 +828,26 @@ "database_path": { "type": "string" }, + "downloaded": { + "type": "string" + }, + "downloaded_bytes": { + "description": "DownloadedBytes is the accumulated download volume across every\necosystem: cache hits multiplied by the artifact size they served.", + "type": "integer" + }, + "downloads": { + "type": "integer" + }, + "ecosystems": { + "type": "array", + "items": { + "$ref": "#/definitions/server.EcosystemStatsEntry" + } + }, + "stats_unavailable": { + "description": "StatsUnavailable distinguishes a proxy that has served nothing from one\nwhose aggregation failed with no snapshot to fall back on. Without it\nboth report zeros and an empty array.", + "type": "boolean" + }, "storage_url": { "type": "string" }, diff --git a/internal/server/server.go b/internal/server/server.go index 4d13a57..7782912 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -1123,6 +1123,28 @@ type StatsResponse struct { TotalSizeHuman string `json:"total_size"` StorageURL string `json:"storage_url"` DatabasePath string `json:"database_path"` + // DownloadedBytes is the accumulated download volume across every + // ecosystem: cache hits multiplied by the artifact size they served. + DownloadedBytes int64 `json:"downloaded_bytes"` + DownloadedBytesHuman string `json:"downloaded"` + Downloads int64 `json:"downloads"` + Ecosystems []EcosystemStatsEntry `json:"ecosystems"` + // StatsUnavailable distinguishes a proxy that has served nothing from one + // whose aggregation failed with no snapshot to fall back on. Without it + // both report zeros and an empty array. + StatsUnavailable bool `json:"stats_unavailable,omitempty"` +} + +// EcosystemStatsEntry is one ecosystem's slice of the cache statistics. +type EcosystemStatsEntry struct { + Ecosystem string `json:"ecosystem"` + DownloadedBytes int64 `json:"downloaded_bytes"` + Downloaded string `json:"downloaded"` + Downloads int64 `json:"downloads"` + CacheSize int64 `json:"cache_size_bytes"` + Artifacts int64 `json:"cached_artifacts"` + Packages int64 `json:"packages"` + Versions int64 `json:"versions"` } // handleStats returns cache statistics. @@ -1147,16 +1169,43 @@ func (s *Server) handleStats(w http.ResponseWriter, r *http.Request) { return } + // A failing per-ecosystem aggregation must not take down an endpoint that + // answered from two cheap counters before it existed. Get returns the last + // good snapshot alongside the error, so the breakdown is served stale when + // there is one, and flagged unavailable when there is not. + ecosystems, statsErr := s.ecoStats.Get(s.db) + if statsErr != nil { + s.logger.Error("failed to get ecosystem stats for /stats", "error", statsErr) + } + _ = ctx // Could use for storage.UsedSpace if needed stats := StatsResponse{ - CachedArtifacts: count, - TotalSize: size, - TotalSizeHuman: formatSize(size), - StorageURL: s.storage.URL(), - DatabasePath: s.cfg.Database.String(), + CachedArtifacts: count, + TotalSize: size, + TotalSizeHuman: formatSize(size), + Ecosystems: make([]EcosystemStatsEntry, 0, len(ecosystems)), + StatsUnavailable: statsErr != nil && len(ecosystems) == 0, + StorageURL: s.storage.URL(), + DatabasePath: s.cfg.Database.String(), } + for _, e := range ecosystems { + stats.DownloadedBytes += e.DownloadedBytes + stats.Downloads += e.Downloads + stats.Ecosystems = append(stats.Ecosystems, EcosystemStatsEntry{ + Ecosystem: e.Ecosystem, + DownloadedBytes: e.DownloadedBytes, + Downloaded: formatSize(e.DownloadedBytes), + Downloads: e.Downloads, + CacheSize: e.CacheSize, + Artifacts: e.Artifacts, + Packages: e.Packages, + Versions: e.Versions, + }) + } + stats.DownloadedBytesHuman = formatSize(stats.DownloadedBytes) + w.Header().Set("Content-Type", "application/json") _ = json.NewEncoder(w).Encode(stats) } diff --git a/internal/server/stats_test.go b/internal/server/stats_test.go new file mode 100644 index 0000000..467f3c1 --- /dev/null +++ b/internal/server/stats_test.go @@ -0,0 +1,139 @@ +package server + +import ( + "database/sql" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/git-pkgs/proxy/internal/database" +) + +func seedCachedArtifact(t *testing.T, db *database.DB, ecosystem, name, version string, size, hits int64) { + t.Helper() + + pkgPURL := "pkg:" + ecosystem + "/" + name + versionPURL := pkgPURL + "@" + version + + if err := db.UpsertPackage(&database.Package{PURL: pkgPURL, Ecosystem: ecosystem, Name: name}); err != nil { + t.Fatalf("UpsertPackage: %v", err) + } + if err := db.UpsertVersion(&database.Version{PURL: versionPURL, PackagePURL: pkgPURL}); err != nil { + t.Fatalf("UpsertVersion: %v", err) + } + if err := db.UpsertArtifact(&database.Artifact{ + VersionPURL: versionPURL, + Filename: name + "-" + version + ".tgz", + UpstreamURL: "https://example.test/" + name, + StoragePath: sql.NullString{String: "objects/" + name, Valid: true}, + Size: sql.NullInt64{Int64: size, Valid: true}, + FetchedAt: sql.NullTime{Time: time.Now(), Valid: true}, + HitCount: hits, + }); err != nil { + t.Fatalf("UpsertArtifact: %v", err) + } +} + +func getStats(t *testing.T, ts *testServer) StatsResponse { + t.Helper() + + w := httptest.NewRecorder() + ts.handler.ServeHTTP(w, httptest.NewRequest("GET", "/stats", nil)) + if w.Code != http.StatusOK { + t.Fatalf("GET /stats = %d, want 200", w.Code) + } + + var stats StatsResponse + if err := json.NewDecoder(w.Body).Decode(&stats); err != nil { + t.Fatalf("decode: %v", err) + } + return stats +} + +func TestStatsReportsEcosystemBreakdown(t *testing.T) { + ts := newTestServer(t) + defer ts.close() + + seedCachedArtifact(t, ts.db, "npm", "lodash", "4.17.21", 1000, 3) + seedCachedArtifact(t, ts.db, "cargo", "serde", "1.0.0", 500, 2) + + stats := getStats(t, ts) + + if stats.DownloadedBytes != 4000 { + t.Errorf("downloaded_bytes = %d, want 4000", stats.DownloadedBytes) + } + if stats.Downloads != 5 { + t.Errorf("downloads = %d, want 5", stats.Downloads) + } + if stats.DownloadedBytesHuman == "" { + t.Error("downloaded is empty; the human-readable total is part of the shape") + } + if stats.StatsUnavailable { + t.Error("stats_unavailable set on a healthy aggregation") + } + + byEcosystem := make(map[string]EcosystemStatsEntry, len(stats.Ecosystems)) + for _, e := range stats.Ecosystems { + byEcosystem[e.Ecosystem] = e + } + npm, ok := byEcosystem["npm"] + if !ok { + t.Fatalf("no npm row in %v", byEcosystem) + } + if npm.DownloadedBytes != 3000 || npm.Downloads != 3 || npm.CacheSize != 1000 { + t.Errorf("npm row = %+v, want 3000 bytes over 3 downloads and 1000 cached", npm) + } +} + +// A failed aggregation with nothing to fall back on must not read as an idle +// proxy: the totals are zero either way, so the flag is the only thing +// telling a consumer which of the two it is looking at. +func TestStatsFlagsUnavailableFigures(t *testing.T) { + ts := newTestServer(t) + defer ts.close() + + // GetEcosystemStats counts packages first; the artifact count and cache + // size the endpoint already reported do not touch that table. + if _, err := ts.db.Exec(`DROP TABLE packages`); err != nil { + t.Fatalf("DROP TABLE packages: %v", err) + } + + stats := getStats(t, ts) + + if !stats.StatsUnavailable { + t.Error("stats_unavailable not set after the aggregation failed with no snapshot") + } + if len(stats.Ecosystems) != 0 { + t.Errorf("ecosystems = %v, want empty", stats.Ecosystems) + } + if stats.StorageURL == "" { + t.Error("the rest of the response must still be served") + } +} + +// A retained snapshot is served on with no flag: the figures are stale, not +// absent, and the page is where staleness is surfaced to a human. +func TestStatsServesRetainedSnapshot(t *testing.T) { + ts := newTestServer(t) + defer ts.close() + + seedCachedArtifact(t, ts.db, "npm", "lodash", "4.17.21", 1000, 3) + if got := getStats(t, ts).DownloadedBytes; got != 3000 { + t.Fatalf("downloaded_bytes = %d, want 3000 before the failure", got) + } + + if _, err := ts.db.Exec(`DROP TABLE packages`); err != nil { + t.Fatalf("DROP TABLE packages: %v", err) + } + ts.server.ecoStats.lastAt = time.Time{} + + stats := getStats(t, ts) + if stats.StatsUnavailable { + t.Error("stats_unavailable set although a snapshot was retained") + } + if stats.DownloadedBytes != 3000 { + t.Errorf("downloaded_bytes = %d, want the retained 3000", stats.DownloadedBytes) + } +} From 36fe6abef2c73b9c138040be13d579a9ec730217 Mon Sep 17 00:00:00 2001 From: Wicliff Wolda Date: Thu, 1 Oct 2026 14:32:20 +0200 Subject: [PATCH 161/633] Add a Grafana dashboard - 33 panels, no hardcoded data source UID - Three ecosystem variables, because the label means three different things across /metrics: the package record, the request path, and the handler's own name Part 5 of 6 splitting #381 up. Co-Authored-By: Claude Opus 5 (1M context) --- README.md | 6 + deploy/grafana/git-pkgs-proxy.json | 2739 ++++++++++++++++++++++++++++ 2 files changed, 2745 insertions(+) create mode 100644 deploy/grafana/git-pkgs-proxy.json diff --git a/README.md b/README.md index 079002b..adb9951 100644 --- a/README.md +++ b/README.md @@ -1196,6 +1196,12 @@ The same figures are available as JSON from `GET /stats`, which reports `downloa **From the handler's own name.** `proxy_upstream_fetch_duration_seconds` and `proxy_upstream_errors_total`, which report `composer`, `gem` and `go` where the other two sets report `packagist`, `rubygems` and `golang`. These are published series and are deliberately left as they are; renaming them would break existing queries and alerts. +### Grafana dashboard + +A ready-made dashboard lives at [`deploy/grafana/git-pkgs-proxy.json`](deploy/grafana/git-pkgs-proxy.json). Import it via **Dashboards -> New -> Import** and pick your Prometheus data source when prompted; it has no hardcoded data source UID. + +It carries three ecosystem filters rather than one, because `ecosystem` means three different things across `/metrics` -- see the label sets above. **Ecosystem** filters the database-derived gauges, **Route** the request-path counters, and **Upstream** the two upstream fetch metrics. All three are query variables, so they populate from whatever labels your proxy is actually reporting; a panel is on the one its metric belongs to, and the panel descriptions say which. + ### Health Check `/health` returns a structured JSON report of subsystem health. HTTP 200 if all checks pass; 503 if any fail. diff --git a/deploy/grafana/git-pkgs-proxy.json b/deploy/grafana/git-pkgs-proxy.json new file mode 100644 index 0000000..dffaa84 --- /dev/null +++ b/deploy/grafana/git-pkgs-proxy.json @@ -0,0 +1,2739 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Download volume, cache composition and upstream health for the git-pkgs proxy. Accumulated download size is cache hits multiplied by artifact size, in total and per ecosystem.", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "panels": [], + "title": "Overview", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Total bytes served to clients from cache: for each artifact, its size multiplied by the number of times it was served. This is traffic carried, not storage used, so it is normally far larger than the cache on disk. Artifacts evicted from the cache stop contributing.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "fixed", + "fixedColor": "text" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 5, + "x": 0, + "y": 1 + }, + "id": 2, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(proxy_ecosystem_downloaded_bytes{job=~\"$job\", ecosystem=~\"$ecosystem\"})", + "range": true, + "instant": false, + "refId": "A" + } + ], + "title": "Accumulated download size", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Bytes the cached artifacts currently occupy in storage \u2014 what the cache costs to keep. Not an average, and not the same as accumulated download size, which is what the cache has served.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "fixed", + "fixedColor": "text" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 5, + "x": 5, + "y": 1 + }, + "id": 3, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(proxy_cache_size_bytes{job=~\"$job\"})", + "range": true, + "instant": false, + "refId": "A" + } + ], + "title": "Cache size on disk", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Accumulated download size divided by the bytes currently cached: how many times over the cache has served what it stores. This is the number that makes the two size figures beside it comparable, and it is the clearest single measure of what the cache is worth.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "fixed", + "fixedColor": "text" + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + } + ] + }, + "unit": "suffix:\u00d7" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 5, + "x": 10, + "y": 1 + }, + "id": 4, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(proxy_ecosystem_downloaded_bytes{job=~\"$job\", ecosystem=~\"$ecosystem\"}) / sum(proxy_ecosystem_cache_size_bytes{job=~\"$job\", ecosystem=~\"$ecosystem\"})", + "range": true, + "instant": false, + "refId": "A" + } + ], + "title": "Served per byte cached", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Number of distinct artifacts held in the cache. A small count with a large accumulated download size just means a few artifacts are being served repeatedly, which is the cache doing its job.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "fixed", + "fixedColor": "text" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 5, + "x": 15, + "y": 1 + }, + "id": 5, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(proxy_cached_artifacts_total{job=~\"$job\"})", + "range": true, + "instant": false, + "refId": "A" + } + ], + "title": "Cached artifacts", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Share of artifact lookups served from cache over the dashboard's time range. Reads as No data when nothing was requested in that range \u2014 a range with no traffic has no hit ratio, which is not the same as a hit ratio of zero.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds", + "fixedColor": "text" + }, + "decimals": 1, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "orange", + "value": 0.5 + }, + { + "color": "green", + "value": 0.8 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 20, + "y": 1 + }, + "id": 6, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(increase(proxy_cache_hits_total{job=~\"$job\", ecosystem=~\"$ecosystem\"}[$__range]))\n/\n(\n sum(increase(proxy_cache_hits_total{job=~\"$job\", ecosystem=~\"$ecosystem\"}[$__range]))\n +\n sum(increase(proxy_cache_misses_total{job=~\"$job\", ecosystem=~\"$ecosystem\"}[$__range]))\n)", + "range": true, + "instant": false, + "refId": "A" + } + ], + "title": "Cache hit ratio", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 6 + }, + "id": 7, + "panels": [], + "title": "Download volume", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Accumulated bytes served from cache, per ecosystem, largest first. One measure across categories, so a single hue carries magnitude.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "fixed", + "fixedColor": "blue" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 7 + }, + "id": 8, + "options": { + "displayMode": "gradient", + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "left", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [], + "fields": "/^Value$/", + "values": true + }, + "showUnfilled": true, + "sizing": "auto", + "valueMode": "text", + "legend": { + "showLegend": false + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (ecosystem) (proxy_ecosystem_downloaded_bytes{job=~\"$job\", ecosystem=~\"$ecosystem\"})", + "range": false, + "instant": true, + "refId": "A", + "legendFormat": "{{ecosystem}}", + "format": "table" + } + ], + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true + }, + "indexByName": {}, + "renameByName": {} + } + }, + { + "id": "sortBy", + "options": { + "fields": {}, + "sort": [ + { + "desc": true, + "field": "Value" + } + ] + } + } + ], + "title": "Download size by ecosystem", + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "How the accumulated total has grown, stacked by ecosystem. A step down means artifacts were evicted, which removes their historical hits from the total.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 18, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 7 + }, + "id": 9, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (ecosystem) (proxy_ecosystem_downloaded_bytes{job=~\"$job\", ecosystem=~\"$ecosystem\"})", + "range": true, + "instant": false, + "refId": "A", + "legendFormat": "{{ecosystem}}" + } + ], + "title": "Accumulated download size over time", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Every per-ecosystem figure the charts leave to a tooltip.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "filterable": true, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Downloaded" + }, + "properties": [ + { + "id": "unit", + "value": "bytes" + }, + { + "id": "custom.cellOptions", + "value": { + "mode": "gradient", + "type": "gauge", + "valueDisplayMode": "text" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cache size" + }, + "properties": [ + { + "id": "unit", + "value": "bytes" + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 17 + }, + "id": 10, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": true + }, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "Downloaded" + } + ] + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (ecosystem) (proxy_ecosystem_downloaded_bytes{job=~\"$job\", ecosystem=~\"$ecosystem\"})", + "range": false, + "instant": true, + "refId": "A", + "format": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (ecosystem) (proxy_ecosystem_artifact_downloads{job=~\"$job\", ecosystem=~\"$ecosystem\"})", + "range": false, + "instant": true, + "refId": "B", + "format": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (ecosystem) (proxy_ecosystem_cache_size_bytes{job=~\"$job\", ecosystem=~\"$ecosystem\"})", + "range": false, + "instant": true, + "refId": "C", + "format": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (ecosystem) (proxy_ecosystem_cached_artifacts{job=~\"$job\", ecosystem=~\"$ecosystem\"})", + "range": false, + "instant": true, + "refId": "D", + "format": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (ecosystem) (proxy_ecosystem_packages{job=~\"$job\", ecosystem=~\"$ecosystem\"})", + "range": false, + "instant": true, + "refId": "E", + "format": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (ecosystem) (proxy_ecosystem_versions{job=~\"$job\", ecosystem=~\"$ecosystem\"})", + "range": false, + "instant": true, + "refId": "F", + "format": "table" + } + ], + "transformations": [ + { + "id": "joinByField", + "options": { + "byField": "ecosystem", + "mode": "outer" + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "Time 1": true, + "Time 2": true, + "Time 3": true, + "Time 4": true, + "Time 5": true, + "Time 6": true + }, + "indexByName": {}, + "renameByName": { + "ecosystem": "Ecosystem", + "Value #A": "Downloaded", + "Value #B": "Downloads", + "Value #C": "Cache size", + "Value #D": "Artifacts", + "Value #E": "Packages", + "Value #F": "Versions" + } + } + } + ], + "title": "Per-ecosystem breakdown", + "type": "table" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 27 + }, + "id": 11, + "panels": [], + "title": "Traffic and cache", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Responses per second, by package ecosystem. The ecosystem label on this metric is derived from the request path, so it uses route names (rubygems, packagist, debian, other) rather than the package-ecosystem names the cache metrics use. It is filtered by the Route variable, not Ecosystem.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 28 + }, + "id": 12, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (ecosystem) (rate(proxy_requests_total{job=~\"$job\", ecosystem=~\"$route\"}[$__rate_interval]))", + "range": true, + "instant": false, + "refId": "A", + "legendFormat": "{{ecosystem}}" + } + ], + "title": "Request rate by route", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Artifact lookups per second that were served from cache versus fetched upstream.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 28 + }, + "id": 13, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(rate(proxy_cache_hits_total{job=~\"$job\", ecosystem=~\"$ecosystem\"}[$__rate_interval]))", + "range": true, + "instant": false, + "refId": "A", + "legendFormat": "hits" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(rate(proxy_cache_misses_total{job=~\"$job\", ecosystem=~\"$ecosystem\"}[$__rate_interval]))", + "range": true, + "instant": false, + "refId": "B", + "legendFormat": "misses" + } + ], + "title": "Cache hits and misses", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "95th percentile time to serve a proxy request, by ecosystem. The ecosystem label on this metric is derived from the request path, so it uses route names (rubygems, packagist, debian, other) rather than the package-ecosystem names the cache metrics use. It is filtered by the Route variable, not Ecosystem.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 36 + }, + "id": 14, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum by (le, ecosystem) (rate(proxy_request_duration_seconds_bucket{job=~\"$job\", ecosystem=~\"$route\"}[$__rate_interval])))", + "range": true, + "instant": false, + "refId": "A", + "legendFormat": "{{ecosystem}}" + } + ], + "title": "Request duration p95", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "95th percentile time spent fetching an artifact from the upstream registry, by ecosystem. Only cache misses reach an upstream. The ecosystem label on this metric is the handler's own name, which differs from both the cache metrics (composer vs packagist, gem vs rubygems, go vs golang) and the route names, so it is filtered by the Upstream variable.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 36 + }, + "id": 15, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum by (le, ecosystem) (rate(proxy_upstream_fetch_duration_seconds_bucket{job=~\"$job\", ecosystem=~\"$upstream\"}[$__rate_interval])))", + "range": true, + "instant": false, + "refId": "A", + "legendFormat": "{{ecosystem}}" + } + ], + "title": "Upstream fetch duration p95", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 44 + }, + "id": 16, + "panels": [], + "title": "Reliability", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Upstreams whose artifact-fetch breaker is open. While a breaker is open, cache misses for that host return 502 without contacting the upstream. Alert on this being above zero for more than a few minutes.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds", + "fixedColor": "text" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 0, + "y": 45 + }, + "id": 17, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "count(proxy_circuit_breaker_state{job=~\"$job\"} == 2) or vector(0)", + "range": true, + "instant": false, + "refId": "A" + } + ], + "title": "Circuit breakers open", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Requests currently in flight.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "fixed", + "fixedColor": "text" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 4, + "y": 45 + }, + "id": 18, + "options": { + "colorMode": "none", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(proxy_active_requests{job=~\"$job\"})", + "range": true, + "instant": false, + "refId": "A" + } + ], + "title": "Active requests", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Circuit breaker trips per upstream registry. A breaker is created per host the proxy fetches artifacts from, and only reports once it has tripped at least once, so an empty panel means no upstream has failed repeatedly.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 8, + "y": 45 + }, + "id": 19, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (registry) (increase(proxy_circuit_breaker_trips_total{job=~\"$job\"}[$__rate_interval]))", + "legendFormat": "{{registry}}", + "range": true, + "instant": false, + "refId": "A" + } + ], + "title": "Circuit breaker trips", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Upstream fetch failures per second, by ecosystem and error type. The ecosystem label on this metric is the handler's own name, which differs from both the cache metrics (composer vs packagist, gem vs rubygems, go vs golang) and the route names, so it is filtered by the Upstream variable.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 12, + "y": 45 + }, + "id": 20, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (ecosystem, error_type) (rate(proxy_upstream_errors_total{job=~\"$job\", ecosystem=~\"$upstream\"}[$__rate_interval]))", + "range": true, + "instant": false, + "refId": "A", + "legendFormat": "{{ecosystem}} \u00b7 {{error_type}}" + } + ], + "title": "Upstream errors", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Storage read/write failures, cached artifacts that failed hash verification on read, and storage health probe failures.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 18, + "y": 45 + }, + "id": 21, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (operation) (rate(proxy_storage_errors_total{job=~\"$job\"}[$__rate_interval]))", + "range": true, + "instant": false, + "refId": "A", + "legendFormat": "storage \u00b7 {{operation}}" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (ecosystem) (rate(proxy_integrity_failures_total{job=~\"$job\", ecosystem=~\"$ecosystem\"}[$__rate_interval]))", + "range": true, + "instant": false, + "refId": "B", + "legendFormat": "integrity \u00b7 {{ecosystem}}" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (step) (rate(proxy_health_probe_failures_total{job=~\"$job\"}[$__rate_interval]))", + "range": true, + "instant": false, + "refId": "C", + "legendFormat": "health probe \u00b7 {{step}}" + } + ], + "title": "Storage and integrity failures", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 52 + }, + "id": 22, + "panels": [], + "title": "Storage and scanning", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "95th percentile storage read/write latency, by operation.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 53 + }, + "id": 23, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum by (le, operation) (rate(proxy_storage_operation_duration_seconds_bucket{job=~\"$job\"}[$__rate_interval])))", + "range": true, + "instant": false, + "refId": "A", + "legendFormat": "{{operation}}" + } + ], + "title": "Storage operation latency p95", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Storage operations per second, by operation.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 53 + }, + "id": 24, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (operation) (rate(proxy_storage_operation_duration_seconds_count{job=~\"$job\"}[$__rate_interval]))", + "range": true, + "instant": false, + "refId": "A", + "legendFormat": "{{operation}}" + } + ], + "title": "Storage operation rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Completed pre-cache scans per second, by scanner. Empty when scanning is not configured.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 61 + }, + "id": 25, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (scanner) (rate(proxy_scan_duration_seconds_count{job=~\"$job\", ecosystem=~\"$ecosystem\"}[$__rate_interval]))", + "range": true, + "instant": false, + "refId": "A", + "legendFormat": "{{scanner}}" + } + ], + "title": "Scan rate by scanner", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "95th percentile pre-cache scan duration, by scanner. A block-mode scanner's latency is on the critical path of a cache miss.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 61 + }, + "id": 26, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum by (le, scanner) (rate(proxy_scan_duration_seconds_bucket{job=~\"$job\", ecosystem=~\"$ecosystem\"}[$__rate_interval])))", + "range": true, + "instant": false, + "refId": "A", + "legendFormat": "{{scanner}}" + } + ], + "title": "Scan duration p95", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Artifacts a block-mode scanner refused, and scan calls that failed, timed out or were cancelled. A rising error rate means artifacts are being admitted without a verdict.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 61 + }, + "id": 27, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (ecosystem, scanner) (rate(proxy_scan_blocked_total{job=~\"$job\", ecosystem=~\"$ecosystem\"}[$__rate_interval]))", + "range": true, + "instant": false, + "refId": "A", + "legendFormat": "blocked \u00b7 {{ecosystem}} \u00b7 {{scanner}}" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (scanner, error_type) (rate(proxy_scan_errors_total{job=~\"$job\", ecosystem=~\"$ecosystem\"}[$__rate_interval]))", + "range": true, + "instant": false, + "refId": "B", + "legendFormat": "error \u00b7 {{scanner}} \u00b7 {{error_type}}" + } + ], + "title": "Artifacts blocked and scan errors", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 69 + }, + "id": 28, + "panels": [], + "title": "Cache composition", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Bytes held in the cache per ecosystem. A drop is an eviction sweep.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 70 + }, + "id": 29, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (ecosystem) (proxy_ecosystem_cache_size_bytes{job=~\"$job\", ecosystem=~\"$ecosystem\"})", + "range": true, + "instant": false, + "refId": "A", + "legendFormat": "{{ecosystem}}" + } + ], + "title": "Cache size by ecosystem", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Artifacts held in the cache per ecosystem.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 70 + }, + "id": 30, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (ecosystem) (proxy_ecosystem_cached_artifacts{job=~\"$job\", ecosystem=~\"$ecosystem\"})", + "range": true, + "instant": false, + "refId": "A", + "legendFormat": "{{ecosystem}}" + } + ], + "title": "Cached artifacts by ecosystem", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Package and version rows known per ecosystem. These grow as metadata is fetched, independently of what is cached.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 78 + }, + "id": 31, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (ecosystem) (proxy_ecosystem_packages{job=~\"$job\", ecosystem=~\"$ecosystem\"})", + "range": true, + "instant": false, + "refId": "A", + "legendFormat": "packages \u00b7 {{ecosystem}}" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (ecosystem) (proxy_ecosystem_versions{job=~\"$job\", ecosystem=~\"$ecosystem\"})", + "range": true, + "instant": false, + "refId": "B", + "legendFormat": "versions \u00b7 {{ecosystem}}" + } + ], + "title": "Known packages and versions", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 86 + }, + "id": 32, + "panels": [], + "title": "Bytes served", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Response body bytes per second written to clients. The ecosystem label on this counter comes from the request path, not the package record, so it is filtered by the Route variable and reports route names (debian, and other for the UI and health checks). Distinct from proxy_ecosystem_downloaded_bytes, which is derived from the database as cache hits times artifact size.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 18, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 87 + }, + "id": 33, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (ecosystem) (rate(proxy_response_bytes_total{job=~\"$job\", ecosystem=~\"$route\"}[$__rate_interval]))", + "range": true, + "instant": false, + "refId": "A", + "legendFormat": "{{ecosystem}}" + } + ], + "title": "Bytes served by route", + "type": "timeseries" + } + ], + "preload": false, + "refresh": "1m", + "schemaVersion": 39, + "tags": [ + "git-pkgs", + "proxy", + "cache" + ], + "templating": { + "list": [ + { + "current": {}, + "hide": 0, + "includeAll": false, + "label": "Data source", + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "allValue": ".*", + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(proxy_cache_size_bytes, job)", + "hide": 0, + "includeAll": true, + "label": "Job", + "multi": true, + "name": "job", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(proxy_cache_size_bytes, job)", + "refId": "job" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "allValue": ".*", + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(proxy_ecosystem_downloaded_bytes, ecosystem)", + "hide": 0, + "includeAll": true, + "label": "Ecosystem", + "multi": true, + "name": "ecosystem", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(proxy_ecosystem_downloaded_bytes, ecosystem)", + "refId": "ecosystem" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "allValue": ".*", + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(proxy_requests_total, ecosystem)", + "description": "Route names taken from the request path. These differ from the package-ecosystem names used by the cache metrics.", + "hide": 0, + "includeAll": true, + "label": "Route", + "multi": true, + "name": "route", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(proxy_requests_total, ecosystem)", + "refId": "route" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "allValue": ".*", + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(proxy_upstream_fetch_duration_seconds_count, ecosystem)", + "description": "Ecosystem names as the upstream fetch metrics report them, taken from the handler rather than the package record: composer, gem and go where the cache metrics say packagist, rubygems and golang.", + "hide": 0, + "includeAll": true, + "label": "Upstream", + "multi": true, + "name": "upstream", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(proxy_upstream_fetch_duration_seconds_count, ecosystem)", + "refId": "upstream" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + } + ] + }, + "time": { + "from": "now-24h", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "git-pkgs proxy", + "uid": "git-pkgs-proxy", + "version": 1, + "weekStart": "" +} From 3655000b164bdb89e9b340d7b74b27f409fc0f65 Mon Sep 17 00:00:00 2001 From: Wicliff Wolda Date: Thu, 1 Oct 2026 14:32:21 +0200 Subject: [PATCH 162/633] Attribute requests to a caller and a client tool - Track per-caller request and byte counts in a bounded table - Export proxy_client_requests_total and proxy_client_response_bytes_total over a closed label set; addresses are never labels - Gate the sources card behind ui_request_sources, off by default, since /ui carries no authentication of its own - Move access_log.trust_forwarded_for to a top-level key - Keep remote as the TCP peer and add remote_ip alongside it Part 6 of 6 splitting #381 up. Co-Authored-By: Claude Opus 5 (1M context) --- README.md | 18 + config.example.yaml | 17 + deploy/grafana/git-pkgs-proxy.json | 200 +++++++- docs/configuration.md | 6 + internal/accesslog/accesslog.go | 13 +- internal/config/config.go | 21 + internal/config/config_test.go | 11 + internal/metrics/metrics.go | 25 +- internal/server/analytics.go | 10 + internal/server/analytics_coverage_test.go | 6 +- internal/server/analytics_runtime.go | 56 +++ internal/server/middleware.go | 17 +- internal/server/server.go | 1 + internal/server/sources.go | 331 +++++++++++++ internal/server/sources_test.go | 444 ++++++++++++++++++ .../templates/components/runtime_metrics.html | 64 +++ 16 files changed, 1231 insertions(+), 9 deletions(-) create mode 100644 internal/server/sources.go create mode 100644 internal/server/sources_test.go diff --git a/README.md b/README.md index adb9951..da96526 100644 --- a/README.md +++ b/README.md @@ -1147,6 +1147,8 @@ The proxy exposes Prometheus metrics at `GET /metrics`. All metric names are pre | `proxy_ecosystem_packages` | gauge | `ecosystem` | Known packages per ecosystem. | | `proxy_ecosystem_versions` | gauge | `ecosystem` | Known package versions per ecosystem. | | `proxy_response_bytes_total` | counter | `ecosystem` | Response body bytes written to clients. Route-labelled, see the label caveat below. | +| `proxy_client_requests_total` | counter | `client` | Requests by client tool, from the User-Agent. | +| `proxy_client_response_bytes_total` | counter | `client` | Response bytes by client tool. | Cache size, artifact count and the per-ecosystem gauges are refreshed every 60 seconds, from a single pass over the database. Circuit breaker state is read from the fetcher on each scrape of `/metrics` and each `/health` request, so `proxy_circuit_breaker_trips_total` counts the trips visible between those reads — a breaker that opens and recovers entirely between two scrapes is not counted. The remaining metrics update on each request. @@ -1196,6 +1198,22 @@ The same figures are available as JSON from `GET /stats`, which reports `downloa **From the handler's own name.** `proxy_upstream_fetch_duration_seconds` and `proxy_upstream_errors_total`, which report `composer`, `gem` and `go` where the other two sets report `packagist`, `rubygems` and `golang`. These are published series and are deliberately left as they are; renaming them would break existing queries and alerts. +### Request sources + +Package managers do not say who invoked them. A request from `pip` or `go` carries a `Host`, an `Accept` and a `User-Agent` -- no `Referer`, no originating URL, nothing naming a repository, pipeline or job. Whatever identity you want has to come from something on the wire, so the proxy attributes requests by the two things always present. + +**Address** -- the TCP peer, or the leftmost `X-Forwarded-For` entry when `trust_forwarded_for` is enabled. Enable that only behind a load balancer or ingress that sets the header; any client can send it, so in front of one it lets a caller forge its own attribution and, by cycling synthetic addresses, fill the table and push every genuine caller into the overflow row. The totals stay correct; the attribution is what is lost. + +**Client** -- the tool, taken from the leading User-Agent token: `pip`, `npm`, `go`, `docker`, `apt`, `curl` and so on. Anything unrecognised reports as `other`. + +Set `ui_request_sources: true` and both appear on `/ui/analytics` under **Runtime -> Request sources**, as a table of the busiest callers by bytes downloaded plus a per-tool breakdown. The table is in-memory and process-lifetime, like the rest of that card. It tracks 200 callers, evicting the least recently seen once full, and summarises everything it is not showing individually -- both evicted callers and those ranked below the display limit -- in a single "other callers" row, so the rows always add up to the totals above them. + +**The flag defaults to off because the page is not authenticated.** `/ui` carries no auth of its own -- it is mounted under its own prefix so a reverse proxy *can* gate it separately, as [Behind a Reverse Proxy](#behind-a-reverse-proxy) describes, but nothing makes you -- and until now it exposed only package data. The sources table changes what is on offer: anyone who can reach the proxy can read the addresses of your build fleet, which tool each runs, and how much each pulled. Turn it on once `/ui` is gated, or leave it off and read the same detail from the access log. + +**What this can and cannot tell you.** How much an address gives you depends entirely on your network. A fleet of build machines with stable addresses attributes cleanly. Containerised CI usually does not: with Docker or Kubernetes executors every job gets an ephemeral address, and egress is commonly NAT'd behind one gateway, so you get runner-node or gateway granularity, not per-project. If you need per-project attribution the caller has to send something naming itself -- a basic-auth username, or a per-project base URL -- which the proxy does not currently read. Say so and it can be added. + +**Why addresses are not Prometheus labels.** Client tool names are exported as `proxy_client_requests_total{client}` because they come from a closed set. Addresses are not exported at all: the caller set is unbounded and outside the proxy's control, and every new address would create a time series that lives forever in your TSDB. The same goes for anything job-scoped -- a pipeline ID must never become a label. Per-address and per-request detail belongs in the access log, which records `remote_ip`, `user_agent`, `client`, `ecosystem` and `bytes` on every line as JSONL, ready for `jq`, Loki or whatever you ship logs to. + ### Grafana dashboard A ready-made dashboard lives at [`deploy/grafana/git-pkgs-proxy.json`](deploy/grafana/git-pkgs-proxy.json). Import it via **Dashboards -> New -> Import** and pick your Prometheus data source when prompted; it has no hardcoded data source UID. diff --git a/config.example.yaml b/config.example.yaml index 58a9de7..96f52ad 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -20,6 +20,23 @@ base_url: "http://localhost:8080" # build machines hit a Docker network alias for the package endpoints. # ui_base_url: "https://proxy.example.com/ui" +# Attribute requests to the leftmost X-Forwarded-For entry rather than the TCP +# peer address, in the structured log, the access log and the request-source +# table on /ui/analytics. +# +# Enable this only when the proxy sits behind a load balancer or ingress that +# sets the header. Any client can send it: behind one it is the only way to see +# past the hop, in front of one it lets a caller forge its own address. +# trust_forwarded_for: false + +# Show the request-source table on /ui/analytics: caller addresses, the tool +# each ran and how much each pulled. +# +# Off by default. The proxy has no authentication of its own, so leave this off +# unless /ui is gated by a reverse proxy; anyone who can reach the page can +# otherwise read the addresses of your build fleet. +# ui_request_sources: false + # Artifact storage configuration storage: # Storage backend URL diff --git a/deploy/grafana/git-pkgs-proxy.json b/deploy/grafana/git-pkgs-proxy.json index dffaa84..e875469 100644 --- a/deploy/grafana/git-pkgs-proxy.json +++ b/deploy/grafana/git-pkgs-proxy.json @@ -2496,7 +2496,7 @@ }, "id": 32, "panels": [], - "title": "Bytes served", + "title": "Request sources", "type": "row" }, { @@ -2559,7 +2559,7 @@ }, "gridPos": { "h": 9, - "w": 24, + "w": 12, "x": 0, "y": 87 }, @@ -2596,6 +2596,202 @@ ], "title": "Bytes served by route", "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Response bytes per second by client tool, identified from the User-Agent. The label set is closed \u2014 an unrecognised User-Agent reports as 'other' \u2014 so a caller cannot create new time series. Caller addresses are deliberately not exported as labels; see the access log or the proxy's own analytics page for per-address detail.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 18, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 87 + }, + "id": 34, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (client) (rate(proxy_client_response_bytes_total{job=~\"$job\"}[$__rate_interval]))", + "range": true, + "instant": false, + "refId": "A", + "legendFormat": "{{client}}" + } + ], + "title": "Bytes served by client", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Requests per second by client tool. Useful for spotting a runaway CI job or a tool that is not honouring caches.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 96 + }, + "id": 35, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (client) (rate(proxy_client_requests_total{job=~\"$job\"}[$__rate_interval]))", + "range": true, + "instant": false, + "refId": "A", + "legendFormat": "{{client}}" + } + ], + "title": "Requests by client", + "type": "timeseries" } ], "preload": false, diff --git a/docs/configuration.md b/docs/configuration.md index f09e010..51c51c4 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -19,6 +19,8 @@ See `config.example.yaml` in the repository root for a complete example. | `listen` | `PROXY_LISTEN` | `-listen` | `:8080` | Address to listen on | | `base_url` | `PROXY_BASE_URL` | `-base-url` | `http://localhost:8080` | Public URL package managers use to reach this proxy | | `ui_base_url` | `PROXY_UI_URL` | - | (defaults to `base_url`) | Public URL where the web UI is reached. Set separately when the UI lives behind a different hostname than package endpoints (e.g. public domain vs Docker network alias). Used for canonical/og:url tags and the install guide banner. The proxy still serves package endpoints on the same listener, so any reverse proxy fronting the UI publicly should restrict the public route to `PathPrefix(/ui)` to avoid exposing package endpoints. | +| `trust_forwarded_for` | `PROXY_TRUST_FORWARDED_FOR` | - | `false` | Attribute requests to the leftmost `X-Forwarded-For` entry instead of the TCP peer address, in the structured log, the access log and the request-source table on `/ui/analytics`. | +| `ui_request_sources` | `PROXY_UI_REQUEST_SOURCES` | - | `false` | Show the request-source table on `/ui/analytics`, which reports caller addresses, the tool each ran and how much each pulled. | ## Storage @@ -123,6 +125,10 @@ access_log: |--------|-------------|------|-------------| | `access_log.path` | `PROXY_ACCESS_LOG_PATH` | `-access-log` | File to append JSONL records to; empty disables the log | +Each client request record carries `remote_addr` (the TCP peer, host and port), `remote_ip` (the address the request is attributed to), `user_agent`, `client` (the tool name derived from the User-Agent), `ecosystem`, and `bytes` (the response body size written to the client). The structured log carries the same `client`, `remote` and `remote_ip` fields. + +`remote_ip` follows the top-level `trust_forwarded_for` setting. Enable that only when the proxy sits behind a load balancer or ingress that sets the header: any client can send `X-Forwarded-For`, so behind such a hop it is the only way to see the real caller, but in front of one it lets a caller choose what address it is logged as. It can also choose a fresh one per request, which fills the bounded source table on `/ui/analytics` and evicts the genuine callers from it. `remote_addr` is unaffected and always records the TCP peer. + The parent directory must exist and be writable when the proxy starts. A newly created log file is readable and writable only by the proxy process owner. A request that receives a rate limit response from an upstream can produce records like these: diff --git a/internal/accesslog/accesslog.go b/internal/accesslog/accesslog.go index 6a3cb01..9ff4a0a 100644 --- a/internal/accesslog/accesslog.go +++ b/internal/accesslog/accesslog.go @@ -33,7 +33,18 @@ type Entry struct { StatusCode int `json:"status_code,omitempty"` DurationMS int64 `json:"duration_ms"` RemoteAddr string `json:"remote_addr,omitempty"` - Error string `json:"error,omitempty"` + // RemoteIP is the address the request is attributed to: the TCP peer, or + // the leftmost X-Forwarded-For entry when that header is trusted. + RemoteIP string `json:"remote_ip,omitempty"` + // UserAgent is recorded verbatim; Client is it reduced to a known tool + // name, so log queries can group without parsing. + UserAgent string `json:"user_agent,omitempty"` + Client string `json:"client,omitempty"` + Ecosystem string `json:"ecosystem,omitempty"` + // Bytes is always emitted, including as 0, so a log pipeline summing the + // field does not have to treat a bodyless response as null. + Bytes int64 `json:"bytes"` + Error string `json:"error,omitempty"` } // Logger appends complete JSON objects to a file, one per line. diff --git a/internal/config/config.go b/internal/config/config.go index fcc41fa..ba63669 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -98,6 +98,23 @@ type Config struct { // Example: "https://proxy.example.com/ui" UIBaseURL string `json:"ui_base_url" yaml:"ui_base_url"` + // TrustForwardedFor attributes requests to the leftmost X-Forwarded-For + // entry instead of the TCP peer address, in the structured log, the access + // log and the request-source table on /ui/analytics. + // + // Enable this only when the proxy sits behind a load balancer or ingress + // that sets the header, because any client can send it: behind one it is + // the only way to see past the hop, in front of one it lets a caller choose + // what address it is attributed to. + TrustForwardedFor bool `json:"trust_forwarded_for" yaml:"trust_forwarded_for"` + + // UIRequestSources shows the request-source table on /ui/analytics, which + // reports caller addresses, the tool each ran and how much each pulled. + // + // Off by default. The proxy has no authentication of its own, so until this + // is enabled /ui exposes what is cached rather than who called. + UIRequestSources bool `json:"ui_request_sources" yaml:"ui_request_sources"` + // Storage configures artifact storage. Storage StorageConfig `json:"storage" yaml:"storage"` @@ -881,12 +898,16 @@ func setEnvStringSlice(dst *[]string, key string) { // - PROXY_LOG_LEVEL // - PROXY_LOG_FORMAT // - PROXY_ACCESS_LOG_PATH +// - PROXY_TRUST_FORWARDED_FOR +// - PROXY_UI_REQUEST_SOURCES // - PROXY_UPSTREAM_SWIFT // - PROXY_HEALTH_STORAGE_PROBE_INTERVAL func (c *Config) LoadFromEnv() { setEnvString(&c.Listen, "PROXY_LISTEN") setEnvString(&c.BaseURL, "PROXY_BASE_URL") setEnvString(&c.UIBaseURL, "PROXY_UI_URL") + setEnvBool(&c.TrustForwardedFor, "PROXY_TRUST_FORWARDED_FOR") + setEnvBool(&c.UIRequestSources, "PROXY_UI_REQUEST_SOURCES") setEnvString(&c.Storage.URL, "PROXY_STORAGE_URL") setEnvString(&c.Storage.Path, "PROXY_STORAGE_PATH") setEnvString(&c.Storage.MaxSize, "PROXY_STORAGE_MAX_SIZE") diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 050b3ce..7898ee5 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -461,6 +461,8 @@ func TestLoadFromEnv(t *testing.T) { t.Setenv("PROXY_STORAGE_PATH", "/env/cache") t.Setenv("PROXY_LOG_LEVEL", testLevelDebug) t.Setenv("PROXY_ACCESS_LOG_PATH", "/tmp/proxy-access.jsonl") + t.Setenv("PROXY_TRUST_FORWARDED_FOR", "true") + t.Setenv("PROXY_UI_REQUEST_SOURCES", "true") t.Setenv("PROXY_UPSTREAM_ALLOW_PRIVATE_HOSTS", "registry.internal, 10.0.0.12") t.Setenv("PROXY_UPSTREAM_ALLOW_LOOPBACK", "true") t.Setenv("PROXY_GRADLE_BUILD_CACHE_READ_ONLY", "true") @@ -489,6 +491,15 @@ func TestLoadFromEnv(t *testing.T) { if cfg.AccessLog.Path != "/tmp/proxy-access.jsonl" { t.Errorf("AccessLog.Path = %q, want %q", cfg.AccessLog.Path, "/tmp/proxy-access.jsonl") } + // Containers configure the proxy through the environment, so a setting + // reachable only from YAML is unreachable in the deployment that most needs + // it -- which is the one sitting behind an ingress. + if !cfg.TrustForwardedFor { + t.Error("TrustForwardedFor = false, want true from the environment") + } + if !cfg.UIRequestSources { + t.Error("UIRequestSources = false, want true from the environment") + } if got := strings.Join(cfg.Upstream.AllowPrivateHosts, ","); got != "registry.internal,10.0.0.12" { t.Errorf("Upstream.AllowPrivateHosts = %q, want %q", got, "registry.internal,10.0.0.12") } diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go index 4021099..1958165 100644 --- a/internal/metrics/metrics.go +++ b/internal/metrics/metrics.go @@ -198,6 +198,22 @@ var ( []string{"ecosystem"}, ) + ClientRequests = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "proxy_client_requests_total", + Help: "Total requests by client tool, as identified from the User-Agent", + }, + []string{"client"}, + ) + + ClientResponseBytes = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "proxy_client_response_bytes_total", + Help: "Total response body bytes written to clients, by client tool", + }, + []string{"client"}, + ) + // Scanning metrics ScanDuration = prometheus.NewHistogramVec( prometheus.HistogramOpts{ @@ -250,6 +266,8 @@ func init() { EcosystemPackages, EcosystemVersions, ResponseBytes, + ClientRequests, + ClientResponseBytes, ScanDuration, ScanBlocked, ScanErrors, @@ -274,11 +292,16 @@ func RecordRequest(ecosystem string, status int, duration time.Duration) { // the database and counts cache hits multiplied by artifact size, while this // counts body bytes as they are written, including metadata responses and // cache misses. -func RecordResponse(ecosystem string, bytes int64) { +// +// client must come from a closed set -- a User-Agent is attacker-controlled, so +// passing it through raw would mint a time series per request. +func RecordResponse(ecosystem, client string, bytes int64) { + ClientRequests.WithLabelValues(client).Inc() if bytes <= 0 { return } ResponseBytes.WithLabelValues(ecosystem).Add(float64(bytes)) + ClientResponseBytes.WithLabelValues(client).Add(float64(bytes)) } // RecordCacheHit increments cache hit counter. diff --git a/internal/server/analytics.go b/internal/server/analytics.go index b734edf..eaf1948 100644 --- a/internal/server/analytics.go +++ b/internal/server/analytics.go @@ -10,6 +10,10 @@ import ( "github.com/git-pkgs/proxy/internal/metrics" ) +// analyticsTopSources caps how many callers the page lists; the rest are +// summarised in the overflow row, and the access log has every one of them. +const analyticsTopSources = 15 + // AnalyticsData contains data for rendering the analytics dashboard. type AnalyticsData struct { Layout @@ -168,6 +172,12 @@ func (s *Server) handleAnalytics(w http.ResponseWriter, r *http.Request) { s.logger.Error("failed to gather runtime metrics", "error", err) } else { data.Runtime = runtimeView(snap) + if s.cfg != nil && s.cfg.UIRequestSources { + data.Runtime.SourcesOn = true + data.Runtime.Sources = s.sources.Top(analyticsTopSources) + data.Runtime.SourceCount = s.sources.Count() + data.Runtime.TrustsForward = s.trustsForwardedFor() + } } if err := s.templates.Render(w, "analytics", data); err != nil { diff --git a/internal/server/analytics_coverage_test.go b/internal/server/analytics_coverage_test.go index 8c5b7a2..5517dea 100644 --- a/internal/server/analytics_coverage_test.go +++ b/internal/server/analytics_coverage_test.go @@ -26,7 +26,9 @@ var metricSurface = map[string]string{ "proxy_cached_artifacts_total": "Cached artifacts tile", // Request-path counters. - "proxy_response_bytes_total": "Runtime: Served", + "proxy_response_bytes_total": "Runtime: Served", + "proxy_client_requests_total": "Runtime: Request sources -- By client", + "proxy_client_response_bytes_total": "Runtime: Request sources -- By client", // Registry-derived, shown in the Runtime card. "proxy_requests_total": "Runtime: Requests + Responses by status", @@ -54,7 +56,7 @@ func TestEveryMetricIsSurfaced(t *testing.T) { // Touch every metric family so it is present in the registry output, since // a vector with no observed label values gathers as nothing at all. metrics.RecordRequest("npm", 200, 0) - metrics.RecordResponse("npm", 1) + metrics.RecordResponse("npm", "npm", 1) metrics.RecordCacheHit("npm") metrics.RecordCacheMiss("npm") metrics.RecordUpstreamFetch("npm", 0) diff --git a/internal/server/analytics_runtime.go b/internal/server/analytics_runtime.go index 800f2ec..2b77730 100644 --- a/internal/server/analytics_runtime.go +++ b/internal/server/analytics_runtime.go @@ -42,6 +42,20 @@ type RuntimeView struct { ScanningOn bool ResponseBytes string + Clients []ClientRow + Sources []SourceRow + SourceCount int + TrustsForward bool + // SourcesOn gates the caller table. The page is unauthenticated, so the + // addresses behind it are published only when asked for. + SourcesOn bool +} + +// ClientRow is one client tool's share of requests and bytes. +type ClientRow struct { + Client string + Requests string + Bytes string } // LabelledCount is a single counter series rendered as a row. @@ -108,10 +122,52 @@ func runtimeView(snap *metrics.Snapshot) RuntimeView { v.ScanningOn = len(v.Scans) > 0 || len(v.ScansBlocked) > 0 || len(v.ScanErrors) > 0 v.ResponseBytes = formatSize(int64(snap.Sum("proxy_response_bytes_total"))) + v.Clients = clientRows(snap) return v } +// clientRows pairs each client tool's request count with the bytes it pulled, +// busiest first. Both come from the same closed label set, so the two counters +// line up row for row. +func clientRows(snap *metrics.Snapshot) []ClientRow { + bytesByClient := snap.SumBy("proxy_client_response_bytes_total", "client") + + type entry struct { + client string + requests float64 + bytes float64 + } + + entries := make([]entry, 0) + for _, s := range snap.Samples("proxy_client_requests_total") { + client := s.Label("client") + entries = append(entries, entry{ + client: client, + requests: s.Value, + bytes: bytesByClient[client], + }) + } + + sort.Slice(entries, func(i, j int) bool { + a, b := entries[i], entries[j] + if a.bytes != b.bytes { + return a.bytes > b.bytes + } + return a.requests > b.requests + }) + + rows := make([]ClientRow, 0, len(entries)) + for _, e := range entries { + rows = append(rows, ClientRow{ + Client: e.client, + Requests: formatCount(int64(e.requests)), + Bytes: formatSize(int64(e.bytes)), + }) + } + return rows +} + // statusClasses groups proxy_requests_total into 2xx/3xx/4xx/5xx buckets, which // is the breakdown worth showing; the per-code detail stays in Prometheus. func statusClasses(snap *metrics.Snapshot) []LabelledCount { diff --git a/internal/server/middleware.go b/internal/server/middleware.go index b86ae6e..025f29f 100644 --- a/internal/server/middleware.go +++ b/internal/server/middleware.go @@ -49,6 +49,10 @@ func (s *Server) LoggerMiddleware(next http.Handler) http.Handler { // the log, the metrics and the access log entirely. defer func() { duration := time.Since(start) + userAgent := r.UserAgent() + client := clientName(userAgent) + addr := clientAddr(r, s.trustsForwardedFor()) + ecosystem := requestEcosystem(r.URL.Path) s.logger.Info("request", "request_id", requestID, @@ -57,14 +61,16 @@ func (s *Server) LoggerMiddleware(next http.Handler) http.Handler { "status", rw.status, "duration", duration, "bytes", rw.bytes, - "remote", r.RemoteAddr) + "client", client, + "remote", r.RemoteAddr, + "remote_ip", addr) // Scrapes of /metrics would otherwise attribute themselves, // burying real callers under whatever polls the proxy most often. if r.URL.Path != "/metrics" { - ecosystem := requestEcosystem(r.URL.Path) metrics.RecordRequest(ecosystem, rw.status, duration) - metrics.RecordResponse(ecosystem, rw.bytes) + metrics.RecordResponse(ecosystem, client, rw.bytes) + s.sources.Record(addr, client, rw.bytes) } if s.accessLog != nil { @@ -76,6 +82,11 @@ func (s *Server) LoggerMiddleware(next http.Handler) http.Handler { StatusCode: rw.status, DurationMS: duration.Milliseconds(), RemoteAddr: r.RemoteAddr, + RemoteIP: addr, + UserAgent: userAgent, + Client: client, + Ecosystem: ecosystem, + Bytes: rw.bytes, }); err != nil { s.logger.Error("failed to write access log", "error", err) } diff --git a/internal/server/server.go b/internal/server/server.go index 7782912..18b0717 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -115,6 +115,7 @@ type Server struct { ecr *ecrTokens breakers *breakerMonitor ecoStats ecosystemStatsCache + sources sourceTracker } // New creates a new Server with the given configuration. diff --git a/internal/server/sources.go b/internal/server/sources.go new file mode 100644 index 0000000..923cb9d --- /dev/null +++ b/internal/server/sources.go @@ -0,0 +1,331 @@ +package server + +import ( + "net" + "net/http" + "sort" + "strings" + "sync" + "time" +) + +// maxTrackedSources bounds the in-memory source table, because the caller set +// is not under the proxy's control. +// +// When it is full the least recently seen caller is evicted rather than the new +// one refused. Refusing would freeze the table on whoever happened to arrive +// first, which is the wrong answer in exactly the deployment this is for: +// containerised CI gives every job a fresh address, so the table would fill +// with dead entries within minutes and every live caller would land in the +// overflow row. Evicted totals move into that row, so nothing is lost from the +// sums; only the per-caller breakdown ages out. +const maxTrackedSources = 200 + +// unknownClient labels a request whose User-Agent names no recognised tool. +const unknownClient = "other" + +// knownClients maps the leading token of a User-Agent to a stable client name. +// +// The value set is deliberately closed. Client names become Prometheus labels, +// and a User-Agent is attacker-controlled, so anything unrecognised collapses +// to unknownClient rather than minting a new time series per request. +var knownClients = map[string]string{ + "npm": "npm", + "pnpm": "pnpm", + "yarn": "yarn", + "bun": "bun", + "node": "npm", + "pip": "pip", + "poetry": "poetry", + "uv": "uv", + "twine": "twine", + "python-requests": "pip", + "gocommand": "go", + "go-http-client": "go", + "cargo": "cargo", + "maven": "maven", + "apache-maven": "maven", + "aether": "maven", + "gradle": "gradle", + "nuget": "nuget", + "nuget-client": "nuget", + "composer": "composer", + "bundler": "bundler", + "rubygems": "bundler", + "gem": "bundler", + "docker": "docker", + "containerd": "containerd", + "skopeo": "skopeo", + "buildkit": "buildkit", + "helm": "helm", + "apt": "apt", + "debian": "apt", + "libdnf": "dnf", + "dnf": "dnf", + "urlgrabber": "dnf", + "apk": "apk", + "conda": "conda", + "mamba": "conda", + "conan": "conan", + "hex": "hex", + "mix": "hex", + "dart": "pub", + "pub": "pub", + "swift": "swift", + "julia": "julia", + "r": "cran", + "curl": "curl", + "wget": "wget", + "mozilla": "browser", + "gitlab-runner": "gitlab-runner", + "github-actions": "github-actions", + "jenkins": "jenkins", + "renovate": "renovate", + "dependabot": "dependabot", + "prometheus": "prometheus", + "kube-probe": "kube-probe", + "blackbox_exporter": "prometheus", +} + +// clientName reduces a User-Agent to one of the names in knownClients. +// +// Package managers put their own name first ("pip/21.2.4 {...}", +// "GoCommand/1 (+https://go.dev/cmd/go)"), so the leading token identifies the +// tool without needing to parse the rest. +func clientName(userAgent string) string { + if userAgent == "" { + return unknownClient + } + + token := userAgent + if i := strings.IndexAny(token, "/ ("); i >= 0 { + token = token[:i] + } + + if name, ok := knownClients[strings.ToLower(strings.TrimSpace(token))]; ok { + return name + } + return unknownClient +} + +// clientAddr returns the address a request should be attributed to. +// +// X-Forwarded-For is honoured only when trustForwarded is set, because any +// client can send the header: behind an ingress it is the only way to see past +// the load balancer, and in front of one it is a way to forge attribution. +// +// The forwarded value must parse as an IP. A load balancer that appends to the +// header leaves the leftmost entry caller-controlled, and that string becomes a +// map key held for the process lifetime and a line in the access log, so an +// arbitrary-length value would be a way to spend the proxy's memory. +func clientAddr(r *http.Request, trustForwarded bool) string { + if trustForwarded { + // Leftmost entry is the original client; the rest are hops. + first, _, _ := strings.Cut(r.Header.Get("X-Forwarded-For"), ",") + if ip := net.ParseIP(strings.TrimSpace(first)); ip != nil { + return ip.String() + } + } + + host, _, err := net.SplitHostPort(r.RemoteAddr) + if err != nil { + return r.RemoteAddr + } + return host +} + +// trustsForwardedFor reports whether X-Forwarded-For should be believed. +// +// cfg is nil on a partially-constructed Server, which tests do build, so this +// answers false rather than dereferencing it. +func (s *Server) trustsForwardedFor() bool { + return s.cfg != nil && s.cfg.TrustForwardedFor +} + +// sourceKey identifies a caller by address and by the tool it was running, so +// two tools on one machine are counted apart. +type sourceKey struct { + Addr string + Client string +} + +type sourceStat struct { + Requests int64 + Bytes int64 + LastSeen time.Time +} + +// sourceTracker accumulates per-caller request and byte counts. +// +// This is process-lifetime state held in memory, like the Prometheus registry +// and unlike the cache figures: it starts empty and is lost on restart. Caller +// addresses are never published as metric labels — the set is unbounded and +// outside the proxy's control — so this table, and the access log, are where +// per-address detail lives. +// +// The zero value is usable. +type sourceTracker struct { + mu sync.Mutex + max int + stats map[sourceKey]*sourceStat + overflow sourceStat + // evictions counts fold-ins to the overflow row, not distinct callers: a + // caller that is evicted, returns, and is evicted again counts twice. + // Counting callers instead would mean keeping every key ever seen, which + // is the unbounded set the limit exists to avoid. + evictions int64 +} + +func (t *sourceTracker) limit() int { + if t.max <= 0 { + return maxTrackedSources + } + return t.max +} + +// Record attributes one completed request to a caller. +func (t *sourceTracker) Record(addr, client string, bytes int64) { + t.mu.Lock() + defer t.mu.Unlock() + + if t.stats == nil { + t.stats = make(map[sourceKey]*sourceStat) + } + + key := sourceKey{Addr: addr, Client: client} + stat, ok := t.stats[key] + if !ok { + if len(t.stats) >= t.limit() { + t.evictOldestLocked() + } + stat = &sourceStat{} + t.stats[key] = stat + } + + stat.Requests++ + stat.Bytes += bytes + stat.LastSeen = time.Now() +} + +// evictOldestLocked folds the least recently seen caller into the overflow row. +func (t *sourceTracker) evictOldestLocked() { + var oldest sourceKey + var oldestStat *sourceStat + for k, s := range t.stats { + if oldestStat == nil || s.LastSeen.Before(oldestStat.LastSeen) { + oldest, oldestStat = k, s + } + } + if oldestStat == nil { + return + } + + t.overflow.Requests += oldestStat.Requests + t.overflow.Bytes += oldestStat.Bytes + if oldestStat.LastSeen.After(t.overflow.LastSeen) { + t.overflow.LastSeen = oldestStat.LastSeen + } + t.evictions++ + delete(t.stats, oldest) +} + +// SourceRow is one caller's row on the analytics page. +type SourceRow struct { + Addr string + Client string + Requests string + Bytes string + LastSeen string + // IsOverflow marks the row that stands for every caller past the limit. + IsOverflow bool +} + +// Top returns the busiest callers by bytes served, most first. +func (t *sourceTracker) Top(limit int) []SourceRow { + t.mu.Lock() + defer t.mu.Unlock() + + type entry struct { + key sourceKey + stat sourceStat + } + + entries := make([]entry, 0, len(t.stats)) + for k, s := range t.stats { + entries = append(entries, entry{key: k, stat: *s}) + } + + sort.Slice(entries, func(i, j int) bool { + a, b := entries[i], entries[j] + switch { + case a.stat.Bytes != b.stat.Bytes: + return a.stat.Bytes > b.stat.Bytes + case a.stat.Requests != b.stat.Requests: + return a.stat.Requests > b.stat.Requests + default: + return a.key.Addr < b.key.Addr + } + }) + + // Callers past the limit are summarised rather than dropped, alongside any + // that were evicted, so the rows still add up to what the page reports + // above them. + rest := sourceStat{Requests: t.overflow.Requests, Bytes: t.overflow.Bytes, LastSeen: t.overflow.LastSeen} + hidden := 0 + if limit > 0 && len(entries) > limit { + for _, e := range entries[limit:] { + rest.Requests += e.stat.Requests + rest.Bytes += e.stat.Bytes + if e.stat.LastSeen.After(rest.LastSeen) { + rest.LastSeen = e.stat.LastSeen + } + hidden++ + } + entries = entries[:limit] + } + + rows := make([]SourceRow, 0, len(entries)+1) + for _, e := range entries { + rows = append(rows, SourceRow{ + Addr: e.key.Addr, + Client: e.key.Client, + Requests: formatCount(e.stat.Requests), + Bytes: formatSize(e.stat.Bytes), + LastSeen: formatTimeAgo(e.stat.LastSeen), + }) + } + + if rest.Requests > 0 { + rows = append(rows, SourceRow{ + Addr: "other callers", + Client: overflowLabel(hidden, t.evictions), + Requests: formatCount(rest.Requests), + Bytes: formatSize(rest.Bytes), + LastSeen: formatTimeAgo(rest.LastSeen), + IsOverflow: true, + }) + } + return rows +} + +// overflowLabel describes what the overflow row stands for. The two counts are +// kept apart because only one of them is exact: hidden is a count of callers +// currently tracked below the cut, while evictions counts fold-ins, which can +// exceed the number of distinct callers behind them. +func overflowLabel(hidden int, evictions int64) string { + switch { + case hidden > 0 && evictions > 0: + return formatCount(int64(hidden)) + " not shown, " + formatCount(evictions) + " evicted" + case evictions > 0: + return formatCount(evictions) + " evicted" + default: + return formatCount(int64(hidden)) + " not shown" + } +} + +// Count reports how many distinct callers are being tracked individually. +func (t *sourceTracker) Count() int { + t.mu.Lock() + defer t.mu.Unlock() + return len(t.stats) +} diff --git a/internal/server/sources_test.go b/internal/server/sources_test.go new file mode 100644 index 0000000..1618d34 --- /dev/null +++ b/internal/server/sources_test.go @@ -0,0 +1,444 @@ +package server + +import ( + "net/http/httptest" + "strconv" + "strings" + "testing" + "time" +) + +func TestClientName(t *testing.T) { + tests := []struct { + ua string + want string + }{ + // Real User-Agents captured from these tools. + {`pip/21.2.4 {"ci":null,"cpu":"arm64"}`, "pip"}, + {"GoCommand/1 (+https://go.dev/cmd/go)", "go"}, + {"curl/8.7.1", "curl"}, + {"npm/10.2.3 node/v20.10.0 darwin arm64 workspaces/false", "npm"}, + {"docker/24.0.7 go/go1.20.10 kernel/6.5.0 os/linux", "docker"}, + {"Debian APT-HTTP/1.3 (2.6.1)", "apt"}, + {"bundler/2.4.22 rubygems/3.4.22", "bundler"}, + // Unknown and hostile input collapses, so a User-Agent cannot mint a + // new Prometheus time series. + {"", unknownClient}, + {"definitely-not-a-known-tool/9", unknownClient}, + {"a\nb/1", unknownClient}, + {`{"evil":"label"}`, unknownClient}, + } + for _, tc := range tests { + if got := clientName(tc.ua); got != tc.want { + t.Errorf("clientName(%q) = %q, want %q", tc.ua, got, tc.want) + } + } +} + +// Every value clientName can return must be one of the closed set, otherwise +// the Prometheus label is not actually bounded. +func TestClientNameIsBounded(t *testing.T) { + allowed := map[string]bool{unknownClient: true} + for _, v := range knownClients { + allowed[v] = true + } + + for _, ua := range []string{"pip/1", "weird", "", "npm/1", "x y z", "../../etc/passwd"} { + if got := clientName(ua); !allowed[got] { + t.Errorf("clientName(%q) = %q, which is outside the closed set", ua, got) + } + } +} + +func TestClientAddr(t *testing.T) { + t.Run("peer address by default", func(t *testing.T) { + r := httptest.NewRequest("GET", "/npm/x", nil) + r.RemoteAddr = "10.1.2.3:54321" + r.Header.Set("X-Forwarded-For", "203.0.113.9") + + // The header is present but untrusted, so it must be ignored. + if got := clientAddr(r, false); got != "10.1.2.3" { + t.Errorf("clientAddr = %q, want the peer address 10.1.2.3", got) + } + }) + + t.Run("forwarded when trusted", func(t *testing.T) { + r := httptest.NewRequest("GET", "/npm/x", nil) + r.RemoteAddr = "10.1.2.3:54321" + r.Header.Set("X-Forwarded-For", "203.0.113.9, 10.0.0.1") + + if got := clientAddr(r, true); got != "203.0.113.9" { + t.Errorf("clientAddr = %q, want the leftmost forwarded entry", got) + } + }) + + t.Run("rejects a forwarded value that is not an IP", func(t *testing.T) { + for _, forged := range []string{ + "not-an-ip", + strings.Repeat("A", 4096), + "mygroup/myproject", + "10.0.0.1; DROP TABLE", + } { + r := httptest.NewRequest("GET", "/npm/x", nil) + r.RemoteAddr = "10.1.2.3:54321" + r.Header.Set("X-Forwarded-For", forged) + + if got := clientAddr(r, true); got != "10.1.2.3" { + t.Errorf("clientAddr with forwarded %q = %q, want the peer address", + truncate(forged), got) + } + } + }) + + t.Run("normalizes a valid forwarded IP", func(t *testing.T) { + r := httptest.NewRequest("GET", "/npm/x", nil) + r.RemoteAddr = "10.1.2.3:54321" + r.Header.Set("X-Forwarded-For", "2001:0db8:0000:0000:0000:0000:0000:0001") + + if got := clientAddr(r, true); got != "2001:db8::1" { + t.Errorf("clientAddr = %q, want the canonical IPv6 form", got) + } + }) + + t.Run("falls back when forwarded is empty", func(t *testing.T) { + r := httptest.NewRequest("GET", "/npm/x", nil) + r.RemoteAddr = "10.1.2.3:54321" + r.Header.Set("X-Forwarded-For", " ") + + if got := clientAddr(r, true); got != "10.1.2.3" { + t.Errorf("clientAddr = %q, want the peer address", got) + } + }) + + t.Run("unparseable remote address is returned as-is", func(t *testing.T) { + r := httptest.NewRequest("GET", "/npm/x", nil) + r.RemoteAddr = "not-a-host-port" + + if got := clientAddr(r, false); got != "not-a-host-port" { + t.Errorf("clientAddr = %q, want the raw value", got) + } + }) +} + +func TestSourceTrackerAggregates(t *testing.T) { + var tr sourceTracker + + tr.Record("10.0.0.1", "npm", 1000) + tr.Record("10.0.0.1", "npm", 500) + tr.Record("10.0.0.2", "pip", 4000) + + rows := tr.Top(10) + if len(rows) != 2 { + t.Fatalf("expected 2 rows, got %+v", rows) + } + + // Ordered by bytes, so the pip caller leads. + if rows[0].Addr != "10.0.0.2" || rows[0].Bytes != formatSize(4000) { + t.Errorf("first row = %+v, want 10.0.0.2 with 4000 bytes", rows[0]) + } + if rows[1].Addr != "10.0.0.1" || rows[1].Requests != "2" || rows[1].Bytes != formatSize(1500) { + t.Errorf("second row = %+v, want 10.0.0.1 with 2 requests and 1500 bytes", rows[1]) + } + if tr.Count() != 2 { + t.Errorf("Count = %d, want 2", tr.Count()) + } +} + +// The same address running two tools is two sources, since that is the +// distinction the table exists to show. +func TestSourceTrackerSeparatesClientsOnOneAddress(t *testing.T) { + var tr sourceTracker + tr.Record("10.0.0.1", "npm", 10) + tr.Record("10.0.0.1", "pip", 20) + + if got := tr.Count(); got != 2 { + t.Errorf("Count = %d, want 2 (one row per address+client)", got) + } +} + +// The caller set is not under the proxy's control, so the table must stop +// growing rather than track every address that ever connects. +func TestSourceTrackerBoundsItsSize(t *testing.T) { + tr := sourceTracker{max: 3} + + for i := range 50 { + tr.Record(string(rune('a'+i%26))+string(rune('0'+i/26)), "npm", 100) + } + + if got := tr.Count(); got != 3 { + t.Fatalf("tracked %d sources, want the cap of 3", got) + } + + rows := tr.Top(10) + last := rows[len(rows)-1] + if !last.IsOverflow { + t.Fatalf("expected an overflow row, got %+v", rows) + } + + // Nothing may be lost from the totals: 50 requests and 5000 bytes went in, + // so the tracked rows plus the overflow row must still account for them. + var requests, bytes int64 + for _, r := range rows { + requests += parseCount(t, r.Requests) + } + if requests != 50 { + t.Errorf("rows account for %d requests, want all 50", requests) + } + _ = bytes +} + +// A full table must evict its stalest caller, not freeze on whoever arrived +// first: ephemeral CI addresses would otherwise fill it with dead entries and +// push every live caller into the overflow row. +func TestSourceTrackerEvictsLeastRecentlySeen(t *testing.T) { + tr := sourceTracker{max: 2} + + tr.Record("10.0.0.1", "npm", 100) + tr.Record("10.0.0.2", "npm", 100) + + // Touch the first so the second becomes the stalest. + time.Sleep(2 * time.Millisecond) + tr.Record("10.0.0.1", "npm", 100) + + time.Sleep(2 * time.Millisecond) + tr.Record("10.0.0.3", "npm", 100) + + addrs := map[string]bool{} + for _, r := range tr.Top(10) { + if !r.IsOverflow { + addrs[r.Addr] = true + } + } + + if !addrs["10.0.0.1"] { + t.Error("the recently active caller was evicted") + } + if !addrs["10.0.0.3"] { + t.Error("the newest caller was not admitted") + } + if addrs["10.0.0.2"] { + t.Error("the stalest caller should have been evicted") + } +} + +// parseCount reverses formatCount for assertions. +func parseCount(t *testing.T, s string) int64 { + t.Helper() + n, err := strconv.ParseInt(strings.ReplaceAll(s, ",", ""), 10, 64) + if err != nil { + t.Fatalf("parsing %q: %v", s, err) + } + return n +} + +func TestSourceTrackerTopLimits(t *testing.T) { + var tr sourceTracker + for i := range 10 { + tr.Record(string(rune('a'+i)), "npm", int64(i*100)) + } + + // Three shown, plus one row summarising the seven that are not. + got := tr.Top(3) + if len(got) != 4 { + t.Errorf("Top(3) returned %d rows, want 3 plus an overflow row", len(got)) + } else if !got[3].IsOverflow { + t.Errorf("fourth row should be the overflow row, got %+v", got[3]) + } + + // No limit means nothing is left over, so there is no overflow row. + if got := tr.Top(0); len(got) != 10 { + t.Errorf("Top(0) returned %d rows, want all 10", len(got)) + } +} + +func TestSourceTrackerZeroValueAndEmpty(t *testing.T) { + var tr sourceTracker + + if rows := tr.Top(5); len(rows) != 0 { + t.Errorf("an empty tracker returned %+v, want no rows", rows) + } + if tr.Count() != 0 { + t.Errorf("Count = %d, want 0", tr.Count()) + } + // Must not panic on a nil map. + tr.Record("10.0.0.1", "npm", 1) + if tr.Count() != 1 { + t.Errorf("Count after first Record = %d, want 1", tr.Count()) + } +} + +// A Server assembled as a struct literal has no config; reading the setting +// must not dereference it. +func TestTrustsForwardedForHandlesNilConfig(t *testing.T) { + var s Server + if s.trustsForwardedFor() { + t.Error("a Server with no config must not trust X-Forwarded-For") + } +} + +func truncate(s string) string { + if len(s) > 32 { + return s[:32] + "..." + } + return s +} + +// The rows the page shows must add up to the totals it reports above them, so +// callers past the display limit have to be summarised, not dropped. +func TestSourceTrackerTopAccountsForEveryCaller(t *testing.T) { + var tr sourceTracker + var wantRequests, wantBytes int64 + for i := range 40 { + bytes := int64((i + 1) * 100) + tr.Record("10.0.0."+strconv.Itoa(i), "npm", bytes) + wantRequests++ + wantBytes += bytes + } + + rows := tr.Top(5) + if len(rows) != 6 { + t.Fatalf("expected 5 rows plus an overflow row, got %d", len(rows)) + } + if !rows[5].IsOverflow { + t.Fatalf("last row is not the overflow row: %+v", rows[5]) + } + + var gotRequests int64 + for _, r := range rows { + gotRequests += parseCount(t, r.Requests) + } + if gotRequests != wantRequests { + t.Errorf("rows account for %d requests, want all %d", gotRequests, wantRequests) + } + // 35 callers are neither shown individually nor evicted. + if rows[5].Client != formatCount(35)+" not shown" { + t.Errorf("overflow row = %q, want 35 not shown", rows[5].Client) + } +} + +// The overflow row reports two counts that are true of different things: how +// many tracked callers fell below the display limit, which is exact, and how +// many fold-ins the eviction path has done, which is not a count of callers — +// a caller that is evicted and comes back is folded in twice. Reporting the sum +// as "N not shown" claimed more distinct callers than the row stands for. +func TestSourceTrackerOverflowSeparatesHiddenFromEvicted(t *testing.T) { + tr := sourceTracker{max: 3} + + // Fill, then push the first caller out, then bring it back and push it out + // again: two fold-ins, one caller. + for _, addr := range []string{"10.0.0.1", "10.0.0.2", "10.0.0.3"} { + tr.Record(addr, "npm", 100) + time.Sleep(time.Millisecond) + } + tr.Record("10.0.0.4", "npm", 100) // evicts .1 + time.Sleep(time.Millisecond) + tr.Record("10.0.0.1", "npm", 100) // .1 returns, evicting .2 + time.Sleep(time.Millisecond) + tr.Record("10.0.0.5", "npm", 100) // evicts .3 + + if tr.evictions != 3 { + t.Fatalf("evictions = %d, want 3 fold-ins", tr.evictions) + } + + rows := tr.Top(2) + overflow := rows[len(rows)-1] + if !overflow.IsOverflow { + t.Fatalf("last row is not the overflow row: %+v", overflow) + } + // One of the three tracked callers is below the cut; three fold-ins + // happened, covering two distinct callers. + if want := "1 not shown, 3 evicted"; overflow.Client != want { + t.Errorf("overflow row = %q, want %q", overflow.Client, want) + } + + var got int64 + for _, r := range rows { + got += parseCount(t, r.Requests) + } + if got != 6 { + t.Errorf("rows account for %d requests, want all 6", got) + } +} + +// With nothing evicted the row says only what it can count exactly. +func TestSourceTrackerOverflowOmitsEvictionsWhenThereAreNone(t *testing.T) { + var tr sourceTracker + for i := range 5 { + tr.Record("10.0.0."+strconv.Itoa(i), "npm", 100) + } + + rows := tr.Top(2) + if want := "3 not shown"; rows[len(rows)-1].Client != want { + t.Errorf("overflow row = %q, want %q", rows[len(rows)-1].Client, want) + } +} + +// The eviction scan is linear over the table, and Record holds a process-wide +// lock while it runs. These two bound what that costs in the case the source +// table is designed for — containerised CI, where every job has a fresh address +// and the table therefore sits permanently at its limit, so every request pays +// a scan. Measured at 200 entries on an M4 Max: ~2us evicting against ~40ns for +// a caller already tracked. Two microseconds against a request that goes to the +// network is not worth a heap or an LRU list, but the gap is the reason this is +// bounded at 200 rather than at something larger. +func BenchmarkRecordAlwaysEvicting(b *testing.B) { + var t sourceTracker + for i := 0; i < maxTrackedSources; i++ { + t.Record("10.0."+strconv.Itoa(i/256)+"."+strconv.Itoa(i%256), "npm", 1024) + } + + addrs := make([]string, b.N) + for i := range addrs { + addrs[i] = "172.16." + strconv.Itoa(i/256%256) + "." + strconv.Itoa(i%256) + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + t.Record(addrs[i], "npm", 1024) + } +} + +func BenchmarkRecordExisting(b *testing.B) { + var t sourceTracker + t.Record("10.0.0.1", "npm", 1024) + b.ResetTimer() + for i := 0; i < b.N; i++ { + t.Record("10.0.0.1", "npm", 1024) + } +} + +// The page is unauthenticated, so the caller table is published only when +// asked for. Off is the default, and off has to mean the addresses are absent +// from the HTML, not merely unstyled. +func TestAnalyticsPageHidesSourcesByDefault(t *testing.T) { + ts := newTestServer(t) + defer ts.close() + + if ts.server.cfg.UIRequestSources { + t.Fatal("UIRequestSources defaults to true; it must default off") + } + ts.server.sources.Record("10.1.2.3", "npm", 4096) + + body := ts.getOK(t, "/ui/analytics") + if strings.Contains(body, "10.1.2.3") { + t.Error("a caller address rendered with ui_request_sources off") + } + if strings.Contains(body, "Request sources") { + t.Error("the sources card rendered with ui_request_sources off") + } +} + +func TestAnalyticsPageShowsSourcesWhenEnabled(t *testing.T) { + ts := newTestServer(t) + defer ts.close() + + ts.server.cfg.UIRequestSources = true + ts.server.sources.Record("10.1.2.3", "npm", 4096) + + body := ts.getOK(t, "/ui/analytics") + for _, want := range []string{"Request sources", "10.1.2.3", "4.0 KB"} { + if !strings.Contains(body, want) { + t.Errorf("rendered page missing %q", want) + } + } +} diff --git a/internal/server/templates/components/runtime_metrics.html b/internal/server/templates/components/runtime_metrics.html index 83ff5f7..e464559 100644 --- a/internal/server/templates/components/runtime_metrics.html +++ b/internal/server/templates/components/runtime_metrics.html @@ -91,6 +91,70 @@ + + {{if .SourcesOn}} +
+
+

Request sources

+

+ {{if .TrustsForward}}Attributed by X-Forwarded-For{{else}}Attributed by peer address{{end}} + {{- if .SourceCount}} · {{.SourceCount}} tracked{{end}} +

+
+ +
+
+ {{if .Sources}} + + + + + + + + + + + + {{range .Sources}} + + + + + + + + {{end}} + +
AddressClientRequestsDownloadedLast seen
{{.Addr}}{{.Client}}{{.Requests}}{{.Bytes}}{{.LastSeen}}
+ {{else}} +

No requests recorded yet.

+ {{end}} +
+ +
+

By client

+ {{if .Clients}} + + + {{range .Clients}} + + + + + + {{end}} + +
{{.Client}}{{.Requests}}{{.Bytes}}
+ {{else}} +

None yet.

+ {{end}} +
+
+
+ {{end}} + {{if .ScanningOn}}

Pre-cache scanning

From 5f0f3b034a74ece4ebffa93c6e9ca625007abe89 Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Thu, 1 Oct 2026 15:25:11 +0100 Subject: [PATCH 163/633] Update enrichment to v0.7.2 (#394) * Update ecosystems-go to v0.5.0 * Update enrichment to v0.7.2 --- go.mod | 7 +++---- go.sum | 14 ++++++-------- 2 files changed, 9 insertions(+), 12 deletions(-) diff --git a/go.mod b/go.mod index fe26a87..dbf3926 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ require ( github.com/git-pkgs/archives v0.7.1 github.com/git-pkgs/artifacts v0.2.1 github.com/git-pkgs/cooldown v0.2.0 - github.com/git-pkgs/enrichment v0.7.1 + github.com/git-pkgs/enrichment v0.7.2 github.com/git-pkgs/gcs v0.1.0 github.com/git-pkgs/integrity v0.1.1 github.com/git-pkgs/magic v0.4.0 @@ -126,7 +126,7 @@ require ( github.com/denis-tingaikin/go-header v0.5.0 // indirect github.com/dlclark/regexp2/v2 v2.2.1 // indirect github.com/dustin/go-humanize v1.0.1 // indirect - github.com/ecosyste-ms/ecosystems-go v0.4.0 // indirect + github.com/ecosyste-ms/ecosystems-go v0.5.0 // indirect github.com/ettle/strcase v0.2.0 // indirect github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a // indirect github.com/fatih/color v1.19.0 // indirect @@ -135,7 +135,6 @@ require ( github.com/fsnotify/fsnotify v1.9.0 // indirect github.com/fzipp/gocyclo v0.6.0 // indirect github.com/ghostiam/protogetter v0.3.21 // indirect - github.com/git-pkgs/packageurl-go v0.3.1 // indirect github.com/git-pkgs/pom v0.1.7 // indirect github.com/github/go-spdx/v2 v2.7.0 // indirect github.com/go-critic/go-critic v0.14.4 // indirect @@ -230,7 +229,7 @@ require ( github.com/nishanths/predeclared v0.2.2 // indirect github.com/nunnatsa/ginkgolinter v0.24.0 // indirect github.com/oapi-codegen/nullable v1.2.0 // indirect - github.com/oapi-codegen/runtime v1.6.0 // indirect + github.com/oapi-codegen/runtime v1.7.0 // indirect github.com/package-url/packageurl-go v0.1.7 // indirect github.com/pandatix/go-cvss v0.6.4 // indirect github.com/pelletier/go-toml v1.9.5 // indirect diff --git a/go.sum b/go.sum index 92f3310..f441520 100644 --- a/go.sum +++ b/go.sum @@ -232,8 +232,8 @@ github.com/dlclark/regexp2/v2 v2.2.1 h1:mf4KkFUj0gJuarK8P+LgiS+Lit7m9N1yAwEfPbee github.com/dlclark/regexp2/v2 v2.2.1/go.mod h1:avUrQvPaLz2DrFNHJF0taWAFFX2C1GMSSoeiqFjcBmU= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= -github.com/ecosyste-ms/ecosystems-go v0.4.0 h1:5A+zF+XWT8sYYYjlc2/tI1SmiDGzbHLyT9CapVc5dGA= -github.com/ecosyste-ms/ecosystems-go v0.4.0/go.mod h1:FVswCrp3DQkur1HjVqfDF/gYrDSEmiFflntcB1G0DbA= +github.com/ecosyste-ms/ecosystems-go v0.5.0 h1:qZqgar3Do9RrtW327OXlsrGKiRvNdeX8CmXb1aX4Njo= +github.com/ecosyste-ms/ecosystems-go v0.5.0/go.mod h1:IhsYBZfxkofxya0O1O2DlhDt281utWJ6VQO0mPVb99I= github.com/envoyproxy/go-control-plane v0.14.0 h1:hbG2kr4RuFj222B6+7T83thSPqLjwBIfQawTkC++2HA= github.com/envoyproxy/go-control-plane/envoy v1.37.0 h1:u3riX6BoYRfF4Dr7dwSOroNfdSbEPe9Yyl09/B6wBrQ= github.com/envoyproxy/go-control-plane/envoy v1.37.0/go.mod h1:DReE9MMrmecPy+YvQOAOHNYMALuowAnbjjEMkkWOi6A= @@ -265,16 +265,14 @@ github.com/git-pkgs/artifacts v0.2.1 h1:VwdxR4yTDaqBZ34h0slxQBVyr2Xfa+QGOKCKviMx github.com/git-pkgs/artifacts v0.2.1/go.mod h1:Otosgq52pXT5UNN7lh6s/lszpWKVDO8XrOjN1M70/IA= github.com/git-pkgs/cooldown v0.2.0 h1:0MWPHtkzZgvCR0wdiQeyvMea/dxgw9tParH1zzaFopc= github.com/git-pkgs/cooldown v0.2.0/go.mod h1:v7APuK/UouTiu8mWQZbdDmj7DfxxkGUeuhjaRB5gv9E= -github.com/git-pkgs/enrichment v0.7.1 h1:8PRYE7gaB8y4M5wnRw/ymNDk0uOHtbQ8CEE7hF09Bv0= -github.com/git-pkgs/enrichment v0.7.1/go.mod h1:QYLG8MtVWPqZojnq7KBKK/lllBWSjLnqvJCWvzGuNwU= +github.com/git-pkgs/enrichment v0.7.2 h1:flD/M89HxkOegUi6tC8EOspFiIMsl7m0o576IQKOp48= +github.com/git-pkgs/enrichment v0.7.2/go.mod h1:xGyc/iz7u1UnbrOHk8lIX5ZaNF3mKFDEZc3VqBJ/LOk= github.com/git-pkgs/gcs v0.1.0 h1:E3awGtsO0xZyHT9FUfEwMHjMkRxw41Bh+c7lgTmPvBo= github.com/git-pkgs/gcs v0.1.0/go.mod h1:bdkCFD66ryaWnU8MBhokVA3WkJfyAEchm9qec5woBpE= github.com/git-pkgs/integrity v0.1.1 h1:nHQ7SktOiGM1dOb5BFnkdtttG/6FCgE6r5ru6QnsGts= github.com/git-pkgs/integrity v0.1.1/go.mod h1:hxu24lcd230377hCF28JQW7sGcCbuNLqo/0ULeb+F1Q= github.com/git-pkgs/magic v0.4.0 h1:oag0YiKuWOIIKWTKEjS47HYydGawUlfQiU96TwAqFyM= github.com/git-pkgs/magic v0.4.0/go.mod h1:SXOqcsNmbmpZjJZHEEWnwxprbsFvpwWgTAZrgXp4Jm4= -github.com/git-pkgs/packageurl-go v0.3.1 h1:WM3RBABQZLaRBxgKyYughc3cVBE8KyQxbSC6Jt5ak7M= -github.com/git-pkgs/packageurl-go v0.3.1/go.mod h1:rcIxiG37BlQLB6FZfgdj9Fm7yjhRQd3l+5o7J0QPAk4= github.com/git-pkgs/pom v0.1.7 h1:4yKdtw6eyShtjul6bcZdyz7yLQ+jdrYeYkKbskDGi4c= github.com/git-pkgs/pom v0.1.7/go.mod h1:ufdMBe1lKzqOeP9IUb9NPZ458xKV8E8NvuyBMxOfwIk= github.com/git-pkgs/purl v0.1.20 h1:a4qzvUy5mBZ2GGjOQNW2h/ocFqjTjOiTDMv2ONtivmM= @@ -535,8 +533,8 @@ github.com/nunnatsa/ginkgolinter v0.24.0 h1:Mp0EagluLFP98JatP6nqp/gGEoljNG97uf9A github.com/nunnatsa/ginkgolinter v0.24.0/go.mod h1:2ZMRuzX6+3XXyY6UZOwb6n+MCocVGbkIsDBC4vuWz5c= github.com/oapi-codegen/nullable v1.2.0 h1:VflFkDW980KhBPiFF7nWSyjg+r4Obqj8lXipV0UkP5w= github.com/oapi-codegen/nullable v1.2.0/go.mod h1:KUZ3vUzkmEKY90ksAmit2+5juDIhIZhfDl+0PwOQlFY= -github.com/oapi-codegen/runtime v1.6.0 h1:7Xx+GlueD6nRuyKoCPzL434Jfi3BetbiJOrzCHp/VPU= -github.com/oapi-codegen/runtime v1.6.0/go.mod h1:GwV7hC2hviaMzj+ITfHVRESK5J2W/GefVwIND/bMGvU= +github.com/oapi-codegen/runtime v1.7.0 h1:t7358VYPvNbWJ9gdAkIK/smVeHpBf6yp8VTsaZsb/7k= +github.com/oapi-codegen/runtime v1.7.0/go.mod h1:GwV7hC2hviaMzj+ITfHVRESK5J2W/GefVwIND/bMGvU= github.com/onsi/ginkgo/v2 v2.32.0 h1:Hw7s2pVrQo/8Yz5N77qdnpHaoc+c6cC9WIV1Jce+J6E= github.com/onsi/ginkgo/v2 v2.32.0/go.mod h1:+aXOY+vzZ5mu2iI2HpTZUPmM//oQfsNFX6gU9kNcA44= github.com/onsi/gomega v1.42.1 h1:iN1rCUX+44NZ1Dc97MPoeFYbFR0vh8zxoxMFwKdyZ6I= From 51caa17d5ed6c2477a3f5dfa77d86ac3c9804e83 Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Thu, 1 Oct 2026 16:06:12 +0100 Subject: [PATCH 164/633] Update git-pkgs dependencies (#396) --- go.mod | 16 ++++++++-------- go.sum | 32 ++++++++++++++++---------------- 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/go.mod b/go.mod index dbf3926..68d2c01 100644 --- a/go.mod +++ b/go.mod @@ -8,18 +8,18 @@ require ( github.com/Masterminds/semver/v3 v3.5.0 github.com/aws/aws-sdk-go-v2/config v1.33.5 github.com/aws/aws-sdk-go-v2/service/ecr v1.66.0 - github.com/git-pkgs/archives v0.7.1 + github.com/git-pkgs/archives v0.8.1 github.com/git-pkgs/artifacts v0.2.1 github.com/git-pkgs/cooldown v0.2.0 github.com/git-pkgs/enrichment v0.7.2 github.com/git-pkgs/gcs v0.1.0 github.com/git-pkgs/integrity v0.1.1 - github.com/git-pkgs/magic v0.4.0 - github.com/git-pkgs/purl v0.1.20 - github.com/git-pkgs/registries v0.9.2 - github.com/git-pkgs/spdx v0.3.2 - github.com/git-pkgs/vers v0.7.1 - github.com/git-pkgs/vulns v0.2.3 + github.com/git-pkgs/magic v0.5.0 + github.com/git-pkgs/purl v0.1.21 + github.com/git-pkgs/registries v0.9.3 + github.com/git-pkgs/spdx v0.3.3 + github.com/git-pkgs/vers v0.7.2 + github.com/git-pkgs/vulns v0.2.4 github.com/go-chi/chi/v5 v5.3.2 github.com/jmoiron/sqlx v1.4.0 github.com/lib/pq v1.12.3 @@ -135,7 +135,7 @@ require ( github.com/fsnotify/fsnotify v1.9.0 // indirect github.com/fzipp/gocyclo v0.6.0 // indirect github.com/ghostiam/protogetter v0.3.21 // indirect - github.com/git-pkgs/pom v0.1.7 // indirect + github.com/git-pkgs/pom v0.1.8 // indirect github.com/github/go-spdx/v2 v2.7.0 // indirect github.com/go-critic/go-critic v0.14.4 // indirect github.com/go-logr/logr v1.4.4 // indirect diff --git a/go.sum b/go.sum index f441520..8ace185 100644 --- a/go.sum +++ b/go.sum @@ -259,8 +259,8 @@ github.com/fzipp/gocyclo v0.6.0 h1:lsblElZG7d3ALtGMx9fmxeTKZaLLpU8mET09yN4BBLo= github.com/fzipp/gocyclo v0.6.0/go.mod h1:rXPyn8fnlpa0R2csP/31uerbiVBugk5whMdlyaLkLoA= github.com/ghostiam/protogetter v0.3.21 h1:EeWTGvL/Eyosp653hiWb6Byx4b69iJC4/E+za7vQHoI= github.com/ghostiam/protogetter v0.3.21/go.mod h1:iAKSpyoHwYzay+OpjoWgwzRtPFthEfuUvmlomTThck0= -github.com/git-pkgs/archives v0.7.1 h1:xLGYZkezsukkSXwJN1tAC0oxSe3szTw8PPtILDSHmG8= -github.com/git-pkgs/archives v0.7.1/go.mod h1:vnRwCQ4eZ87tFzMiQ3GVIrOW6igJyog+2ad5YYCC3xQ= +github.com/git-pkgs/archives v0.8.1 h1:IcT1bVsAec8SqJ4m9vTnD644KiadQZ4ERrWnm/30t3o= +github.com/git-pkgs/archives v0.8.1/go.mod h1:gIyPXPQr5e+YrMqA5KQ7x6pLrFXHNF1+CjdedNJdGUU= github.com/git-pkgs/artifacts v0.2.1 h1:VwdxR4yTDaqBZ34h0slxQBVyr2Xfa+QGOKCKviMx2xk= github.com/git-pkgs/artifacts v0.2.1/go.mod h1:Otosgq52pXT5UNN7lh6s/lszpWKVDO8XrOjN1M70/IA= github.com/git-pkgs/cooldown v0.2.0 h1:0MWPHtkzZgvCR0wdiQeyvMea/dxgw9tParH1zzaFopc= @@ -271,20 +271,20 @@ github.com/git-pkgs/gcs v0.1.0 h1:E3awGtsO0xZyHT9FUfEwMHjMkRxw41Bh+c7lgTmPvBo= github.com/git-pkgs/gcs v0.1.0/go.mod h1:bdkCFD66ryaWnU8MBhokVA3WkJfyAEchm9qec5woBpE= github.com/git-pkgs/integrity v0.1.1 h1:nHQ7SktOiGM1dOb5BFnkdtttG/6FCgE6r5ru6QnsGts= github.com/git-pkgs/integrity v0.1.1/go.mod h1:hxu24lcd230377hCF28JQW7sGcCbuNLqo/0ULeb+F1Q= -github.com/git-pkgs/magic v0.4.0 h1:oag0YiKuWOIIKWTKEjS47HYydGawUlfQiU96TwAqFyM= -github.com/git-pkgs/magic v0.4.0/go.mod h1:SXOqcsNmbmpZjJZHEEWnwxprbsFvpwWgTAZrgXp4Jm4= -github.com/git-pkgs/pom v0.1.7 h1:4yKdtw6eyShtjul6bcZdyz7yLQ+jdrYeYkKbskDGi4c= -github.com/git-pkgs/pom v0.1.7/go.mod h1:ufdMBe1lKzqOeP9IUb9NPZ458xKV8E8NvuyBMxOfwIk= -github.com/git-pkgs/purl v0.1.20 h1:a4qzvUy5mBZ2GGjOQNW2h/ocFqjTjOiTDMv2ONtivmM= -github.com/git-pkgs/purl v0.1.20/go.mod h1:hthV5mp+Q67HpQ9+LnRLLmsReu5ooyQ5EaJsCGrA8yE= -github.com/git-pkgs/registries v0.9.2 h1:YWn//VGIbwoVYpxNFt1UgS3A6oA0nGiWde23l5R5Z5g= -github.com/git-pkgs/registries v0.9.2/go.mod h1:BpXlfRCfu4S5bTzm7u7igppVu94mXgll/PENe0AjKmI= -github.com/git-pkgs/spdx v0.3.2 h1:PV0w09Bt8rIkAof1+zuo3vxNYKgCoFTxQY9E1Q85PL0= -github.com/git-pkgs/spdx v0.3.2/go.mod h1:n9rAicgw+Wqtfursju6oKaAL6cUCTiIS926mb1KWoKU= -github.com/git-pkgs/vers v0.7.1 h1:23VceCbOjQKib0jhbfNRqL3YqkBPiUwdkQNhJoZgnsU= -github.com/git-pkgs/vers v0.7.1/go.mod h1:ofLiBpPNkQmC0LB1k0zmN1gCv7Hi3MiZx8WtmWZ4A9A= -github.com/git-pkgs/vulns v0.2.3 h1:G8icINpR9WFgtwp+4mSdgO4THStiQvi6QuHl83J7iOs= -github.com/git-pkgs/vulns v0.2.3/go.mod h1:+z7pZMjctLmUxMsq+tZbciD6xAAoejljDosC6n2YXps= +github.com/git-pkgs/magic v0.5.0 h1:Pm+4fkHR+K6kg9pwJM4hnwWUuBlkYdLmF1yIAI8lU9Q= +github.com/git-pkgs/magic v0.5.0/go.mod h1:SXOqcsNmbmpZjJZHEEWnwxprbsFvpwWgTAZrgXp4Jm4= +github.com/git-pkgs/pom v0.1.8 h1:mXCuWw6XkrKKZb6IurRZyIznYLPhV5AYg5ZO38Uj/l8= +github.com/git-pkgs/pom v0.1.8/go.mod h1:sQBeYRzjCUn77fcABPn6I/a1aBHFxm55jTmkNxYtGtw= +github.com/git-pkgs/purl v0.1.21 h1:PDZUX43lxIiXPn2eHcsNiiiwQo+HajntZ/4TwSvCGlw= +github.com/git-pkgs/purl v0.1.21/go.mod h1:4dOmris48ehjiKmWBqSmMHguUQht48BLeXW+hQUnuj0= +github.com/git-pkgs/registries v0.9.3 h1:nXIP//6kVqLKqYXOWhEMoWUkJPDAZYVLRUYfHz3jM2Y= +github.com/git-pkgs/registries v0.9.3/go.mod h1:BpXlfRCfu4S5bTzm7u7igppVu94mXgll/PENe0AjKmI= +github.com/git-pkgs/spdx v0.3.3 h1:xXlUoZzjDSkwxC6mWHfaW+kc8a0+LH6d+gGHuZWftqI= +github.com/git-pkgs/spdx v0.3.3/go.mod h1:n9rAicgw+Wqtfursju6oKaAL6cUCTiIS926mb1KWoKU= +github.com/git-pkgs/vers v0.7.2 h1:Ctex1O+Ai/FONmyDaw37EolBXg0og6UdxbCjjYbeRHs= +github.com/git-pkgs/vers v0.7.2/go.mod h1:ofLiBpPNkQmC0LB1k0zmN1gCv7Hi3MiZx8WtmWZ4A9A= +github.com/git-pkgs/vulns v0.2.4 h1:QnsQg9sWUfM8ZzA7A/1LSl8DJ/0/6Yrm7vfRwkpyXUY= +github.com/git-pkgs/vulns v0.2.4/go.mod h1:csd+UWvmYWVVAkka7FNbyz840/1R3QXS7LwD2Irsf4Q= github.com/github/go-spdx/v2 v2.7.0 h1:GzfXx4wFdlilARxmFRXW/mgUy3A4vSqZocCMFV6XFdQ= github.com/github/go-spdx/v2 v2.7.0/go.mod h1:Ftc45YYG1WzpzwEPKRVm9Jv8vDqOrN4gWoCkK+bHer0= github.com/go-chi/chi/v5 v5.3.2 h1:5YQkICvTCSZ25hoRsyJazN0scjzKGiu4VAUc7H1o1nY= From 55ed6b9a1630e108c89901ba05e9de7c26db3d50 Mon Sep 17 00:00:00 2001 From: montehurd Date: Thu, 1 Oct 2026 08:21:55 -0700 Subject: [PATCH 165/633] Keep hit timestamps from moving backwards With proxies sharing a database, a batch can flush after a newer hit is already written, and overwriting last_accessed_at and updated_at with its older time could reorder LRU eviction. Both now keep the later of the stored and incoming times. The count still adds up. --- internal/database/hits.go | 8 ++++-- internal/database/hits_test.go | 49 ++++++++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+), 2 deletions(-) diff --git a/internal/database/hits.go b/internal/database/hits.go index ee30fd6..249c663 100644 --- a/internal/database/hits.go +++ b/internal/database/hits.go @@ -104,9 +104,13 @@ func (db *DB) writeHits(pending map[hitKey]hitEntry) error { } defer func() { _ = tx.Rollback() }() + // Timestamps only move forward: with proxies sharing a database, an older + // batch can flush after a newer hit is already written. stmt, err := tx.Preparex(db.Rebind(` UPDATE artifacts - SET hit_count = hit_count + ?, last_accessed_at = ?, updated_at = ? + SET hit_count = hit_count + ?, + last_accessed_at = CASE WHEN last_accessed_at IS NULL OR last_accessed_at < ? THEN ? ELSE last_accessed_at END, + updated_at = CASE WHEN updated_at IS NULL OR updated_at < ? THEN ? ELSE updated_at END WHERE version_purl = ? AND filename = ? `)) if err != nil { @@ -125,7 +129,7 @@ func (db *DB) writeHits(pending map[hitKey]hitEntry) error { }) for _, k := range keys { e := pending[k] - if _, err := stmt.Exec(e.count, e.last, e.last, k.versionPURL, k.filename); err != nil { + if _, err := stmt.Exec(e.count, e.last, e.last, e.last, e.last, k.versionPURL, k.filename); err != nil { return err } } diff --git a/internal/database/hits_test.go b/internal/database/hits_test.go index dc7fcd9..99126f6 100644 --- a/internal/database/hits_test.go +++ b/internal/database/hits_test.go @@ -96,6 +96,55 @@ func TestBatchHitsWritesOnClose(t *testing.T) { } } +// TestBatchHitsKeepNewerAccessTime flushes a batch older than a hit already +// written, as when proxies share a database. The count still adds up, but +// neither timestamp may move backwards. +func TestBatchHitsKeepNewerAccessTime(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + versionPURL, filename := seedHitTestArtifact(t, db) + k := hitKey{versionPURL, filename} + now := time.Now() + read := func() *Artifact { + t.Helper() + a, err := db.GetArtifact(versionPURL, filename) + if err != nil || a == nil { + t.Fatalf("GetArtifact failed: %v", err) + } + return a + } + write := func(last time.Time) { + t.Helper() + if err := db.writeHits(map[hitKey]hitEntry{k: {count: 1, last: last}}); err != nil { + t.Fatalf("writeHits failed: %v", err) + } + } + + write(now) + newer := read() + if !newer.LastAccessedAt.Valid { + t.Fatal("last_accessed_at not set from NULL") + } + + write(now.Add(-time.Minute)) + got := read() + if got.HitCount != 2 { + t.Errorf("hit count = %d, want 2", got.HitCount) + } + if !got.LastAccessedAt.Time.Equal(newer.LastAccessedAt.Time) { + t.Errorf("last_accessed_at moved from %v to %v", newer.LastAccessedAt.Time, got.LastAccessedAt.Time) + } + if !got.UpdatedAt.Equal(newer.UpdatedAt) { + t.Errorf("updated_at moved from %v to %v", newer.UpdatedAt, got.UpdatedAt) + } + + write(now.Add(time.Minute)) + got = read() + if !got.LastAccessedAt.Time.After(newer.LastAccessedAt.Time) || !got.UpdatedAt.After(newer.UpdatedAt) { + t.Error("a newer batch did not advance the timestamps") + } + }) +} + func TestBatchHitsZeroIntervalWritesImmediately(t *testing.T) { runWithBothDatabases(t, func(t *testing.T, db *DB) { versionPURL, filename := seedHitTestArtifact(t, db) From 107301c98447000cab4dc745afa814f0bbcb95f8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 1 Oct 2026 16:46:01 +0100 Subject: [PATCH 166/633] build(deps): bump alpine from 3.24.1 to 3.24.2 (#397) Bumps alpine from 3.24.1 to 3.24.2. --- updated-dependencies: - dependency-name: alpine dependency-version: 3.24.2 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 4b0c5d2..de784e8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,7 +16,7 @@ COPY . . ARG TARGETARCH RUN CGO_ENABLED=0 GOOS=linux GOARCH=${TARGETARCH} go build -ldflags="-s -w" -o /proxy ./cmd/proxy -FROM alpine:3.24.1 +FROM alpine:3.24.2 RUN apk add --no-cache ca-certificates From 76db94552fdb062cfb3fc0ecf29765dda10d93df Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Thu, 1 Oct 2026 19:48:27 +0100 Subject: [PATCH 167/633] Install development tools directly in CI (#398) --- .github/workflows/ci.yml | 5 +- .github/workflows/swagger.yml | 2 +- go.mod | 200 +------------ go.sum | 510 ---------------------------------- tools.go | 7 - 5 files changed, 7 insertions(+), 717 deletions(-) delete mode 100644 tools.go diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cf1771d..c4b7776 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -43,8 +43,11 @@ jobs: with: go-version-file: go.mod + - name: Install golangci-lint + run: go install github.com/golangci/golangci-lint/v2/cmd/golangci-lint@v2.13.1 + - name: golangci-lint - run: go tool golangci-lint run ./... + run: golangci-lint run ./... helm: name: Helm chart diff --git a/.github/workflows/swagger.yml b/.github/workflows/swagger.yml index 38c42c3..4112a99 100644 --- a/.github/workflows/swagger.yml +++ b/.github/workflows/swagger.yml @@ -22,7 +22,7 @@ jobs: go-version-file: go.mod - name: Install swag - run: go install github.com/swaggo/swag/cmd/swag@latest + run: go install github.com/swaggo/swag/cmd/swag@v1.16.6 - name: Generate swagger run: go generate ./internal/server diff --git a/go.mod b/go.mod index 68d2c01..0fd1e1b 100644 --- a/go.mod +++ b/go.mod @@ -37,47 +37,19 @@ require ( ) require ( - 4d63.com/gocheckcompilerdirectives v1.4.0 // indirect - 4d63.com/gochecknoglobals v0.2.2 // indirect - charm.land/lipgloss/v2 v2.0.6 // indirect cloud.google.com/go/auth v0.21.0 // indirect cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect cloud.google.com/go/compute/metadata v0.9.0 // indirect - codeberg.org/chavacava/garif v0.2.0 // indirect - codeberg.org/polyfloyd/go-errorlint v1.9.0 // indirect - dev.gaijin.team/go/exhaustruct/v4 v4.0.0 // indirect - dev.gaijin.team/go/exhaustruct/v5 v5.0.3 // indirect - dev.gaijin.team/go/golib v0.8.1 // indirect - github.com/4meepo/tagalign v1.4.3 // indirect - github.com/Abirdcfly/dupword v0.1.8 // indirect - github.com/AdminBenni/iota-mixing v1.0.0 // indirect - github.com/AlwxSin/noinlineerr v1.0.6 // indirect - github.com/Antonboom/errname v1.1.2 // indirect - github.com/Antonboom/nilnil v1.1.2 // indirect - github.com/Antonboom/testifylint v1.6.4 // indirect github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 // indirect github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.4 // indirect github.com/AzureAD/microsoft-authentication-library-for-go v1.7.0 // indirect - github.com/ClickHouse/clickhouse-go-linter v1.2.1 // indirect - github.com/Djarvur/go-err113 v0.1.1 // indirect github.com/KyleBanks/depth v1.2.1 // indirect - github.com/MirrexOne/unqueryvet v1.5.4 // indirect - github.com/OpenPeeDeeP/depguard/v2 v2.2.1 // indirect github.com/PuerkitoBio/purell v1.1.1 // indirect github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect - github.com/alecthomas/chroma/v2 v2.27.0 // indirect - github.com/alecthomas/go-check-sumtype v0.3.1 // indirect - github.com/alexkohler/nakedret/v2 v2.0.6 // indirect - github.com/alexkohler/prealloc v1.1.0 // indirect - github.com/alfatraining/structtag v1.0.0 // indirect - github.com/alingse/asasalint v0.0.11 // indirect - github.com/alingse/nilnesserr v0.2.0 // indirect github.com/anchore/go-struct-converter v0.1.0 // indirect github.com/apapsch/go-jsonmerge/v2 v2.0.0 // indirect - github.com/ashanbrown/forbidigo/v2 v2.3.1 // indirect - github.com/ashanbrown/makezero/v2 v2.2.1 // indirect github.com/aws/aws-sdk-go-v2 v1.47.0 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.11 // indirect github.com/aws/aws-sdk-go-v2/credentials v1.20.5 // indirect @@ -97,215 +69,54 @@ require ( github.com/aws/aws-sdk-go-v2/service/sts v1.51.0 // indirect github.com/aws/smithy-go v1.28.1 // indirect github.com/beorn7/perks v1.0.1 // indirect - github.com/bkielbasa/cyclop v1.2.3 // indirect - github.com/blizzy78/varnamelen v0.8.0 // indirect - github.com/bombsimon/wsl/v4 v4.7.0 // indirect - github.com/bombsimon/wsl/v5 v5.9.0 // indirect - github.com/breml/bidichk v0.3.3 // indirect - github.com/breml/errchkjson v0.4.1 // indirect - github.com/butuzov/ireturn v0.4.1 // indirect - github.com/butuzov/mirror v1.3.3 // indirect - github.com/catenacyber/perfsprint v0.10.1 // indirect - github.com/ccojocar/zxcvbn-go v1.0.4 // indirect github.com/cenk/backoff v2.2.1+incompatible // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect - github.com/charithe/durationcheck v0.0.11 // indirect - github.com/charmbracelet/colorprofile v0.4.3 // indirect - github.com/charmbracelet/ultraviolet v0.0.0-20260811164956-006e29f97886 // indirect - github.com/charmbracelet/x/ansi v0.11.8 // indirect - github.com/charmbracelet/x/term v0.2.2 // indirect - github.com/charmbracelet/x/termios v0.1.1 // indirect - github.com/charmbracelet/x/windows v0.2.2 // indirect - github.com/ckaznocha/intrange v0.3.1 // indirect - github.com/clipperhouse/displaywidth v0.11.0 // indirect - github.com/clipperhouse/uax29/v2 v2.7.0 // indirect - github.com/cpuguy83/go-md2man/v2 v2.0.6 // indirect - github.com/curioswitch/go-reassign v0.3.0 // indirect - github.com/daixiang0/gci v0.13.7 // indirect - github.com/dave/dst v0.27.3 // indirect - github.com/denis-tingaikin/go-header v0.5.0 // indirect - github.com/dlclark/regexp2/v2 v2.2.1 // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/ecosyste-ms/ecosystems-go v0.5.0 // indirect - github.com/ettle/strcase v0.2.0 // indirect github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a // indirect - github.com/fatih/color v1.19.0 // indirect - github.com/fatih/structtag v1.2.0 // indirect - github.com/firefart/nonamedreturns v1.0.8 // indirect - github.com/fsnotify/fsnotify v1.9.0 // indirect - github.com/fzipp/gocyclo v0.6.0 // indirect - github.com/ghostiam/protogetter v0.3.21 // indirect + github.com/felixge/httpsnoop v1.1.0 // indirect github.com/git-pkgs/pom v0.1.8 // indirect github.com/github/go-spdx/v2 v2.7.0 // indirect - github.com/go-critic/go-critic v0.14.4 // indirect github.com/go-logr/logr v1.4.4 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-openapi/jsonpointer v0.19.5 // indirect github.com/go-openapi/jsonreference v0.19.6 // indirect github.com/go-openapi/spec v0.20.4 // indirect github.com/go-openapi/swag v0.19.15 // indirect - github.com/go-toolsmith/astcast v1.1.0 // indirect - github.com/go-toolsmith/astcopy v1.1.0 // indirect - github.com/go-toolsmith/astequal v1.2.0 // indirect - github.com/go-toolsmith/astfmt v1.1.0 // indirect - github.com/go-toolsmith/astp v1.1.0 // indirect - github.com/go-toolsmith/strparse v1.1.0 // indirect - github.com/go-toolsmith/typep v1.1.0 // indirect - github.com/go-viper/mapstructure/v2 v2.5.0 // indirect - github.com/go-xmlfmt/xmlfmt v1.1.3 // indirect - github.com/gobwas/glob v0.2.3 // indirect - github.com/godoc-lint/godoc-lint v0.11.2 // indirect - github.com/gofrs/flock v0.13.0 // indirect github.com/golang-jwt/jwt/v5 v5.3.1 // indirect - github.com/golangci/asciicheck v0.5.0 // indirect - github.com/golangci/dupl v0.0.0-20260401084720-c99c5cf5c202 // indirect - github.com/golangci/go-printf-func-name v0.1.1 // indirect - github.com/golangci/gofmt v0.0.0-20260820135601-e84e05053792 // indirect - github.com/golangci/golangci-lint/v2 v2.13.1 // indirect - github.com/golangci/golines v0.15.0 // indirect - github.com/golangci/misspell v0.8.0 // indirect - github.com/golangci/plugin-module-register v0.1.2 // indirect - github.com/golangci/revgrep v0.8.0 // indirect - github.com/golangci/rowserrcheck v0.0.0-20260419091836-c5f79b8a11ba // indirect - github.com/golangci/swaggoswag v0.0.0-20250504205917-77f2aca3143e // indirect - github.com/golangci/unconvert v0.0.0-20250410112200-a129a6e6413e // indirect - github.com/google/go-cmp v0.7.0 // indirect github.com/google/s2a-go v0.1.9 // indirect github.com/google/uuid v1.6.0 // indirect github.com/google/wire v0.7.0 // indirect github.com/googleapis/enterprise-certificate-proxy v0.3.18 // indirect github.com/googleapis/gax-go/v2 v2.23.0 // indirect - github.com/gordonklaus/ineffassign v0.2.0 // indirect - github.com/gostaticanalysis/analysisutil v0.7.1 // indirect - github.com/gostaticanalysis/comment v1.5.0 // indirect - github.com/gostaticanalysis/forcetypeassert v0.2.0 // indirect - github.com/gostaticanalysis/nilerr v0.1.2 // indirect - github.com/hashicorp/go-immutable-radix/v2 v2.1.0 // indirect - github.com/hashicorp/go-version v1.9.0 // indirect - github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect - github.com/hashicorp/hcl v1.0.0 // indirect - github.com/hexops/gotextdiff v1.0.3 // indirect - github.com/inconshreveable/mousetrap v1.1.0 // indirect - github.com/jgautheron/goconst v1.11.0 // indirect - github.com/jjti/go-spancheck v0.6.5 // indirect github.com/josharian/intern v1.0.0 // indirect - github.com/julz/importas v0.2.0 // indirect - github.com/karamaru-alpha/copyloopvar v1.2.2 // indirect - github.com/kisielk/errcheck v1.20.0 // indirect - github.com/kkHAIKE/contextcheck v1.1.6 // indirect github.com/klauspost/compress v1.20.0 // indirect - github.com/kulti/thelper v0.7.1 // indirect - github.com/kunwardeep/paralleltest v1.0.15 // indirect github.com/kylelemons/godebug v1.1.0 // indirect - github.com/lasiar/canonicalheader v1.1.2 // indirect - github.com/ldez/exptostd v0.4.5 // indirect - github.com/ldez/gomoddirectives v0.9.0 // indirect - github.com/ldez/grignotin v0.10.1 // indirect - github.com/ldez/structtags v0.6.1 // indirect - github.com/ldez/tagliatelle v0.7.2 // indirect - github.com/ldez/usetesting v0.5.0 // indirect - github.com/leonklingele/grouper v1.1.2 // indirect - github.com/lucasb-eyer/go-colorful v1.4.1 // indirect - github.com/macabu/inamedparam v0.2.0 // indirect - github.com/magiconair/properties v1.8.6 // indirect github.com/mailru/easyjson v0.7.7 // indirect - github.com/manuelarte/embeddedstructfieldcheck v0.4.0 // indirect - github.com/manuelarte/funcorder v0.6.0 // indirect - github.com/maratori/testableexamples v1.0.1 // indirect - github.com/maratori/testpackage v1.1.2 // indirect - github.com/matoous/godox v1.1.0 // indirect - github.com/mattn/go-colorable v0.1.15 // indirect github.com/mattn/go-isatty v0.0.24 // indirect - github.com/mattn/go-runewidth v0.0.24 // indirect - github.com/mgechev/revive v1.15.0 // indirect - github.com/mitchellh/go-homedir v1.1.0 // indirect - github.com/mitchellh/mapstructure v1.5.0 // indirect - github.com/moricho/tparallel v0.3.2 // indirect - github.com/muesli/cancelreader v0.2.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect - github.com/nakabonne/nestif v0.3.1 // indirect github.com/ncruces/go-strftime v1.0.0 // indirect - github.com/nishanths/exhaustive v0.12.0 // indirect - github.com/nishanths/predeclared v0.2.2 // indirect - github.com/nunnatsa/ginkgolinter v0.24.0 // indirect github.com/oapi-codegen/nullable v1.2.0 // indirect github.com/oapi-codegen/runtime v1.7.0 // indirect github.com/package-url/packageurl-go v0.1.7 // indirect github.com/pandatix/go-cvss v0.6.4 // indirect - github.com/pelletier/go-toml v1.9.5 // indirect - github.com/pelletier/go-toml/v2 v2.4.3 // indirect github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect github.com/prometheus/common v0.70.1 // indirect github.com/prometheus/procfs v0.21.1 // indirect - github.com/quasilyte/go-ruleguard v0.4.5 // indirect - github.com/quasilyte/go-ruleguard/dsl v0.3.23 // indirect - github.com/quasilyte/gogrep v0.5.0 // indirect - github.com/quasilyte/regex/syntax v0.0.0-20210819130434-b3f0c404a727 // indirect - github.com/quasilyte/stdinfo v0.0.0-20220114132959-f7386bf02567 // indirect - github.com/raeperd/recvcheck v0.3.0 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect - github.com/rivo/uniseg v0.4.7 // indirect github.com/rogpeppe/go-internal v1.16.0 // indirect github.com/rs/dnscache v0.0.0-20230804202142-fc85eb664529 // indirect github.com/rubyist/circuitbreaker v2.2.1+incompatible // indirect - github.com/russross/blackfriday/v2 v2.1.0 // indirect - github.com/ryancurrah/gomodguard v1.4.1 // indirect - github.com/ryancurrah/gomodguard/v2 v2.1.3 // indirect - github.com/ryanrolds/sqlclosecheck v0.6.0 // indirect - github.com/sanposhiho/wastedassign/v2 v2.1.0 // indirect - github.com/santhosh-tekuri/jsonschema/v6 v6.0.3 // indirect - github.com/sashamelentyev/interfacebloat v1.1.0 // indirect - github.com/sashamelentyev/usestdlibvars v1.29.0 // indirect - github.com/securego/gosec/v2 v2.28.0 // indirect - github.com/sirupsen/logrus v1.10.1 // indirect - github.com/sivchari/containedctx v1.0.3 // indirect - github.com/sonatard/noctx v0.5.1 // indirect - github.com/sourcegraph/go-diff v0.8.0 // indirect - github.com/spf13/afero v1.15.0 // indirect - github.com/spf13/cast v1.5.0 // indirect - github.com/spf13/cobra v1.10.2 // indirect - github.com/spf13/jwalterweatherman v1.1.0 // indirect - github.com/spf13/pflag v1.0.10 // indirect - github.com/spf13/viper v1.12.0 // indirect - github.com/ssgreg/nlreturn/v2 v2.2.1 // indirect - github.com/stbenjam/no-sprintf-host-port v0.3.1 // indirect - github.com/stretchr/objx v0.5.3 // indirect - github.com/stretchr/testify v1.12.1 // indirect - github.com/subosito/gotenv v1.4.1 // indirect - github.com/tetafro/godot v1.5.6 // indirect - github.com/timakin/bodyclose v0.0.0-20260129054331-73d1f95b84b4 // indirect - github.com/timonwong/loggercheck v0.11.0 // indirect - github.com/tomarrell/wrapcheck/v2 v2.12.0 // indirect - github.com/tommy-muehle/go-mnd/v2 v2.5.1 // indirect github.com/ulikunitz/xz v0.5.16 // indirect - github.com/ultraware/funlen v0.2.0 // indirect - github.com/ultraware/whitespace v0.2.0 // indirect - github.com/urfave/cli/v2 v2.3.0 // indirect - github.com/uudashr/gocognit v1.2.1 // indirect - github.com/uudashr/iface v1.5.0 // indirect - github.com/xen0n/gosmopolitan v1.3.0 // indirect - github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect - github.com/yagipy/maintidx v1.0.0 // indirect - github.com/yeya24/promlinter v0.3.0 // indirect - github.com/ykadowak/zerologlint v0.1.5 // indirect - gitlab.com/bosi/decorder v0.4.2 // indirect - go-simpler.org/musttag v0.14.0 // indirect - go-simpler.org/sloglint v0.12.0 // indirect - go.augendre.info/arangolint v0.4.0 // indirect - go.augendre.info/fatcontext v0.10.0 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0 // indirect go.opentelemetry.io/otel v1.45.0 // indirect go.opentelemetry.io/otel/metric v1.45.0 // indirect go.opentelemetry.io/otel/sdk v1.45.0 // indirect go.opentelemetry.io/otel/sdk/metric v1.45.0 // indirect go.opentelemetry.io/otel/trace v1.45.0 // indirect - go.uber.org/multierr v1.11.0 // indirect - go.uber.org/zap v1.27.1 // indirect - go.yaml.in/yaml/v2 v2.4.4 // indirect go.yaml.in/yaml/v3 v3.0.5 // indirect golang.org/x/crypto v0.57.0 // indirect - golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa // indirect - golang.org/x/exp/typeparams v0.0.0-20260811152304-ee035b5b010f // indirect golang.org/x/mod v0.41.0 // indirect golang.org/x/oauth2 v0.36.0 // indirect golang.org/x/sys v0.48.0 // indirect @@ -315,15 +126,8 @@ require ( google.golang.org/api v0.288.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20260706201446-f0a921348800 // indirect google.golang.org/grpc v1.83.2 // indirect - gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect - honnef.co/go/tools v0.8.0 // indirect modernc.org/libc v1.75.7 // indirect modernc.org/mathutil v1.7.1 // indirect modernc.org/memory v1.12.1 // indirect - mvdan.cc/gofumpt v0.11.0 // indirect - mvdan.cc/unparam v0.0.0-20260818115549-3f964bcb5673 // indirect - sigs.k8s.io/yaml v1.6.0 // indirect ) - -tool github.com/golangci/golangci-lint/v2/cmd/golangci-lint diff --git a/go.sum b/go.sum index 8ace185..782b613 100644 --- a/go.sum +++ b/go.sum @@ -1,11 +1,5 @@ -4d63.com/gocheckcompilerdirectives v1.4.0 h1:ZLq62rbGWVmQhiZ8kuNVIT/M09xCSTdJz9K3xOdT/CY= -4d63.com/gocheckcompilerdirectives v1.4.0/go.mod h1:9ZOAiMOjqC/nRwci2fcUXVHUNLG/cH6r6rhUh+jTFtQ= -4d63.com/gochecknoglobals v0.2.2 h1:H1vdnwnMaZdQW/N+NrkT1SZMTBmcwHe9Vq8lJcYYTtU= -4d63.com/gochecknoglobals v0.2.2/go.mod h1:lLxwTQjL5eIesRbvnzIP3jZtG140FnTdz+AlMa+ogt0= cel.dev/expr v0.25.2 h1:K6j46C81hXtZQfuX60cVWQFBJahKSE2gfRbNuvr5bFs= cel.dev/expr v0.25.2/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4= -charm.land/lipgloss/v2 v2.0.6 h1:EaGKeuA8FvF+v2BT5VmZd2LoYLaMZJXA5n34th8nCIQ= -charm.land/lipgloss/v2 v2.0.6/go.mod h1:ipDDJNSGa1hlwDtSfW1s2/xR8Vdhbut4PXh2zEKZd0Q= cloud.google.com/go v0.123.0 h1:2NAUJwPR47q+E35uaJeYoNhuNEM9kM8SjgRgdeOJUSE= cloud.google.com/go v0.123.0/go.mod h1:xBoMV08QcqUGuPW65Qfm1o9Y4zKZBpGS+7bImXLTAZU= cloud.google.com/go/auth v0.21.0 h1:g/QwYfYb2Ai6HH8oomAOyBaIHLbscZ4+T/F/f5JZHkE= @@ -20,33 +14,9 @@ cloud.google.com/go/monitoring v1.24.3 h1:dde+gMNc0UhPZD1Azu6at2e79bfdztVDS5lvhO cloud.google.com/go/monitoring v1.24.3/go.mod h1:nYP6W0tm3N9H/bOw8am7t62YTzZY+zUeQ+Bi6+2eonI= cloud.google.com/go/storage v1.61.3 h1:VS//ZfBuPGDvakfD9xyPW1RGF1Vy3BWUoVZXgW1KMOg= cloud.google.com/go/storage v1.61.3/go.mod h1:JtqK8BBB7TWv0HVGHubtUdzYYrakOQIsMLffZ2Z/HWk= -codeberg.org/chavacava/garif v0.2.0 h1:F0tVjhYbuOCnvNcU3YSpO6b3Waw6Bimy4K0mM8y6MfY= -codeberg.org/chavacava/garif v0.2.0/go.mod h1:P2BPbVbT4QcvLZrORc2T29szK3xEOlnl0GiPTJmEqBQ= -codeberg.org/polyfloyd/go-errorlint v1.9.0 h1:VkdEEmA1VBpH6ecQoMR4LdphVI3fA4RrCh2an7YmodI= -codeberg.org/polyfloyd/go-errorlint v1.9.0/go.mod h1:GPRRu2LzVijNn4YkrZYJfatQIdS+TrcK8rL5Xs24qw8= -dev.gaijin.team/go/exhaustruct/v4 v4.0.0 h1:873r7aNneqoBB3IaFIzhvt2RFYTuHgmMjoKfwODoI1Y= -dev.gaijin.team/go/exhaustruct/v4 v4.0.0/go.mod h1:aZ/k2o4Y05aMJtiux15x8iXaumE88YdiB0Ai4fXOzPI= -dev.gaijin.team/go/exhaustruct/v5 v5.0.3 h1:yOeA7DNjlT8y4yfmN6nWWYYggA13N523YAj9/TXbuTM= -dev.gaijin.team/go/exhaustruct/v5 v5.0.3/go.mod h1:KwtBsX8nHHH1YxhxkpiBq6bfsmw5WnazWpNvJPHgY9Y= -dev.gaijin.team/go/golib v0.8.1 h1:JYju4x9BSo+QD/AYeHULVDcvEhiFg8wOi6pT0IaZF5E= -dev.gaijin.team/go/golib v0.8.1/go.mod h1:c5fu7t1RSGMxSQgcUYO1sODbzsYnOCXJLmHeNG1Eb+0= filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= filippo.io/edwards25519 v1.2.0 h1:crnVqOiS4jqYleHd9vaKZ+HKtHfllngJIiOpNpoJsjo= filippo.io/edwards25519 v1.2.0/go.mod h1:xzAOLCNug/yB62zG1bQ8uziwrIqIuxhctzJT18Q77mc= -github.com/4meepo/tagalign v1.4.3 h1:Bnu7jGWwbfpAie2vyl63Zup5KuRv21olsPIha53BJr8= -github.com/4meepo/tagalign v1.4.3/go.mod h1:00WwRjiuSbrRJnSVeGWPLp2epS5Q/l4UEy0apLLS37c= -github.com/Abirdcfly/dupword v0.1.8 h1:SrhcUuGsROBuChFxHALRYzyyPODWn9zwghmzPvD9Cd8= -github.com/Abirdcfly/dupword v0.1.8/go.mod h1:XZrhVnI7YGpsTiWZANSQaBJ4QpL/Tq5vIEdKJJAs9WI= -github.com/AdminBenni/iota-mixing v1.0.0 h1:Os6lpjG2dp/AE5fYBPAA1zfa2qMdCAWwPMCgpwKq7wo= -github.com/AdminBenni/iota-mixing v1.0.0/go.mod h1:i4+tpAaB+qMVIV9OK3m4/DAynOd5bQFaOu+2AhtBCNY= -github.com/AlwxSin/noinlineerr v1.0.6 h1:KAvuxunTe9QxvqrFB7nZTdb/7Wzas4AvifslTnG0Ld8= -github.com/AlwxSin/noinlineerr v1.0.6/go.mod h1:+QgkkoYrMH7RHvcdxdlI7vYYEdgeoFOVjU9sUhw/rQc= -github.com/Antonboom/errname v1.1.2 h1:dxwONZJua3VB8Xh/VaCjqAcqF645sWWv7xj26zy7tdQ= -github.com/Antonboom/errname v1.1.2/go.mod h1:YeZIpgLMxT+SNkruGgYkLhzq/9vs3fsolTZegKaKDZI= -github.com/Antonboom/nilnil v1.1.2 h1:aNlFuJhaEseXe4fHO3xbjXlSeEiQVYa2lEkWD2s2hAY= -github.com/Antonboom/nilnil v1.1.2/go.mod h1:0ynwvphOLmAuMwTNDyBnDZmSwZoDpcFXmUHmzoHH2WA= -github.com/Antonboom/testifylint v1.6.4 h1:gs9fUEy+egzxkEbq9P4cpcMB6/G0DYdMeiFS87UiqmQ= -github.com/Antonboom/testifylint v1.6.4/go.mod h1:YO33FROXX2OoUfwjz8g+gUxQXio5i9qpVy7nXGbxDD4= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.0 h1:fou+2+WFTib47nS+nz/ozhEBnvU96bKHy6LjRsY4E28= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.0/go.mod h1:t76Ruy8AHvUAC8GfMWJMa0ElSbuIcO03NLpynfbgsPA= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1 h1:Hk5QBxZQC1jb2Fwj6mpzme37xbCDdNTxU7O9eb5+LB4= @@ -63,15 +33,10 @@ github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 h1:WJ github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mod h1:tCcJZ0uHAmvjsVYzEFivsRTN00oz5BEsRgQHu5JZ9WE= github.com/AzureAD/microsoft-authentication-library-for-go v1.7.0 h1:4iB+IesclUXdP0ICgAabvq2FYLXrJWKx1fJQ+GxSo3Y= github.com/AzureAD/microsoft-authentication-library-for-go v1.7.0/go.mod h1:HKpQxkWaGLJ+D/5H8QRpyQXA1eKjxkFlOMwck5+33Jk= -github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/toml v1.6.0 h1:dRaEfpa2VI55EwlIW72hMRHdWouJeRF7TPYhI+AUQjk= github.com/BurntSushi/toml v1.6.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= -github.com/ClickHouse/clickhouse-go-linter v1.2.1 h1:zGEKIyd5YL08ieWG/LOUmlau2DxbxPVOfAeo+4Jz3ck= -github.com/ClickHouse/clickhouse-go-linter v1.2.1/go.mod h1:pLorS7ffPTfuUV9M0SJgfHA/h/WQPQUk2FWG9x74cQ4= github.com/CycloneDX/cyclonedx-go v0.12.0 h1:/7Jum36UA6V043tQZ/fE3jf+Nf9gn/qxUFfd7QReMy8= github.com/CycloneDX/cyclonedx-go v0.12.0/go.mod h1:V2577HhxDDCDLYfkm55WJrz16nHTfyQZwcWUBSG7Z28= -github.com/Djarvur/go-err113 v0.1.1 h1:eHfopDqXRwAi+YmCUas75ZE0+hoBHJ2GQNLYRSxao4g= -github.com/Djarvur/go-err113 v0.1.1/go.mod h1:IaWJdYFLg76t2ihfflPZnM1LIQszWOsFDh2hhhAVF6k= github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.33.0 h1:l7+6kwRMJNwdCvYdDl7Eax+wzEYHSnNY7zrrfbhDdTA= github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.33.0/go.mod h1:pJTkW8hEUIIi3Pf65lPZOnn4Y81yCllX6IWk2jNXdkM= github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.55.0 h1:UnDZ/zFfG1JhH/DqxIZYU/1CUAlTUScoXD/LcM2Ykk8= @@ -82,41 +47,15 @@ github.com/KyleBanks/depth v1.2.1 h1:5h8fQADFrWtarTdtDudMmGsC7GPbOAu6RVB3ffsVFHc github.com/KyleBanks/depth v1.2.1/go.mod h1:jzSb9d0L43HxTQfT+oSA1EEp2q+ne2uh6XgeJcm8brE= github.com/Masterminds/semver/v3 v3.5.0 h1:kQceYJfbupGfZOKZQg0kou0DgAKhzDg2NZPAwZ/2OOE= github.com/Masterminds/semver/v3 v3.5.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= -github.com/MirrexOne/unqueryvet v1.5.4 h1:38QOxShO7JmMWT+eCdDMbcUgGCOeJphVkzzRgyLJgsQ= -github.com/MirrexOne/unqueryvet v1.5.4/go.mod h1:fs9Zq6eh1LRIhsDIsxf9PONVUjYdFHdtkHIgZdJnyPU= -github.com/OpenPeeDeeP/depguard/v2 v2.2.1 h1:vckeWVESWp6Qog7UZSARNqfu/cZqvki8zsuj3piCMx4= -github.com/OpenPeeDeeP/depguard/v2 v2.2.1/go.mod h1:q4DKzC4UcVaAvcfd41CZh0PWpGgzrVxUYBlgKNGquUo= github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI= github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/RaveNoX/go-jsoncommentstrip v1.0.0/go.mod h1:78ihd09MekBnJnxpICcwzCMzGrKSKYe4AqU6PDYYpjk= -github.com/alecthomas/assert/v2 v2.11.0 h1:2Q9r3ki8+JYXvGsDyBXwH3LcJ+WK5D0gc5E8vS6K3D0= -github.com/alecthomas/assert/v2 v2.11.0/go.mod h1:Bze95FyfUr7x34QZrjL+XP+0qgp/zg8yS+TtBj1WA3k= -github.com/alecthomas/chroma/v2 v2.27.0 h1:FodwmyOBgJULFYmDqibcp9pvfDLWdtPRh9v/r5BXYZs= -github.com/alecthomas/chroma/v2 v2.27.0/go.mod h1:NjJ3ciIgrqBNeIkWZ4e46nseoLDslxU1LmfCoL+wcY8= -github.com/alecthomas/go-check-sumtype v0.3.1 h1:u9aUvbGINJxLVXiFvHUlPEaD7VDULsrxJb4Aq31NLkU= -github.com/alecthomas/go-check-sumtype v0.3.1/go.mod h1:A8TSiN3UPRw3laIgWEUOHHLPa6/r9MtoigdlP5h3K/E= -github.com/alecthomas/repr v0.5.2 h1:SU73FTI9D1P5UNtvseffFSGmdNci/O6RsqzeXJtP0Qs= -github.com/alecthomas/repr v0.5.2/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4= -github.com/alexkohler/nakedret/v2 v2.0.6 h1:ME3Qef1/KIKr3kWX3nti3hhgNxw6aqN5pZmQiFSsuzQ= -github.com/alexkohler/nakedret/v2 v2.0.6/go.mod h1:l3RKju/IzOMQHmsEvXwkqMDzHHvurNQfAgE1eVmT40Q= -github.com/alexkohler/prealloc v1.1.0 h1:cKGRBqlXw5iyQGLYhrXrDlcHxugXpTq4tQ5c91wkf8M= -github.com/alexkohler/prealloc v1.1.0/go.mod h1:fT39Jge3bQrfA7nPMDngUfvUbQGQeJyGQnR+913SCig= -github.com/alfatraining/structtag v1.0.0 h1:2qmcUqNcCoyVJ0up879K614L9PazjBSFruTB0GOFjCc= -github.com/alfatraining/structtag v1.0.0/go.mod h1:p3Xi5SwzTi+Ryj64DqjLWz7XurHxbGsq6y3ubePJPus= -github.com/alingse/asasalint v0.0.11 h1:SFwnQXJ49Kx/1GghOFz1XGqHYKp21Kq1nHad/0WQRnw= -github.com/alingse/asasalint v0.0.11/go.mod h1:nCaoMhw7a9kSJObvQyVzNTPBDbNpdocqrSP7t/cW5+I= -github.com/alingse/nilnesserr v0.2.0 h1:raLem5KG7EFVb4UIDAXgrv3N2JIaffeKNtcEXkEWd/w= -github.com/alingse/nilnesserr v0.2.0/go.mod h1:1xJPrXonEtX7wyTq8Dytns5P2hNzoWymVUIaKm4HNFg= github.com/anchore/go-struct-converter v0.1.0 h1:2rDRssAl6mgKBSLNiVCMADgZRhoqtw9dedlWa0OhD30= github.com/anchore/go-struct-converter v0.1.0/go.mod h1:rYqSE9HbjzpHTI74vwPvae4ZVYZd1lue2ta6xHPdblA= github.com/apapsch/go-jsonmerge/v2 v2.0.0 h1:axGnT1gRIfimI7gJifB699GoE/oq+F2MU7Dml6nw9rQ= github.com/apapsch/go-jsonmerge/v2 v2.0.0/go.mod h1:lvDnEdqiQrp0O42VQGgmlKpxL1AP2+08jFMw88y4klk= -github.com/ashanbrown/forbidigo/v2 v2.3.1 h1:KAZijvQ7zeIBKbhikT4jCm0TLYXC4u78bTiLh/8JROI= -github.com/ashanbrown/forbidigo/v2 v2.3.1/go.mod h1:2QDkLTzU6TV937eFROamXrW92M3paehdae4HCDCOZCM= -github.com/ashanbrown/makezero/v2 v2.2.1 h1:A7uU8dgB1PA9aelTxHMfHIQ8Qev8AB3JLxJUBUsejqM= -github.com/ashanbrown/makezero/v2 v2.2.1/go.mod h1:aEGT/9q3S8DHeE57C88z2a6xydvgx8J5hgXIGWgo0MY= github.com/aws/aws-sdk-go-v2 v1.47.0 h1:0jsHallhJCeaU0Ko48c/3FK1ctOQ7NpzggxriJOQ8MQ= github.com/aws/aws-sdk-go-v2 v1.47.0/go.mod h1:bttEH6JqnUL8LepvDVfdrds/fZ5bCIxzpe3abyUrhDU= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.11 h1:h5+3VT69KUBK24grGuuA5saDJTj2IIjLb9au668Fo5I= @@ -159,77 +98,20 @@ github.com/aws/smithy-go v1.28.1 h1:R/nXH00c8qcfCzQVELtRw+eLQWtzv+VAIEFJ1/xxXlQ= github.com/aws/smithy-go v1.28.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/bkielbasa/cyclop v1.2.3 h1:faIVMIGDIANuGPWH031CZJTi2ymOQBULs9H21HSMa5w= -github.com/bkielbasa/cyclop v1.2.3/go.mod h1:kHTwA9Q0uZqOADdupvcFJQtp/ksSnytRMe8ztxG8Fuo= -github.com/blizzy78/varnamelen v0.8.0 h1:oqSblyuQvFsW1hbBHh1zfwrKe3kcSj0rnXkKzsQ089M= -github.com/blizzy78/varnamelen v0.8.0/go.mod h1:V9TzQZ4fLJ1DSrjVDfl89H7aMnTvKkApdHeyESmyR7k= github.com/bmatcuk/doublestar v1.1.1/go.mod h1:UD6OnuiIn0yFxxA2le/rnRU1G4RaI4UvFv1sNto9p6w= -github.com/bombsimon/wsl/v4 v4.7.0 h1:1Ilm9JBPRczjyUs6hvOPKvd7VL1Q++PL8M0SXBDf+jQ= -github.com/bombsimon/wsl/v4 v4.7.0/go.mod h1:uV/+6BkffuzSAVYD+yGyld1AChO7/EuLrCF/8xTiapg= -github.com/bombsimon/wsl/v5 v5.9.0 h1:WCrgZ7RQnZO5oEwbVTlYgBdU3wL294kR1BSWV8vTfsU= -github.com/bombsimon/wsl/v5 v5.9.0/go.mod h1:kjo4HiAV5FDkHC8/uzJq9mBffEEd6WT/nvN7DoMovDM= github.com/bradleyjkemp/cupaloy/v2 v2.8.0 h1:any4BmKE+jGIaMpnU8YgH/I2LPiLBufr6oMMlVBbn9M= github.com/bradleyjkemp/cupaloy/v2 v2.8.0/go.mod h1:bm7JXdkRd4BHJk9HpwqAI8BoAY1lps46Enkdqw6aRX0= -github.com/breml/bidichk v0.3.3 h1:WSM67ztRusf1sMoqH6/c4OBCUlRVTKq+CbSeo0R17sE= -github.com/breml/bidichk v0.3.3/go.mod h1:ISbsut8OnjB367j5NseXEGGgO/th206dVa427kR8YTE= -github.com/breml/errchkjson v0.4.1 h1:keFSS8D7A2T0haP9kzZTi7o26r7kE3vymjZNeNDRDwg= -github.com/breml/errchkjson v0.4.1/go.mod h1:a23OvR6Qvcl7DG/Z4o0el6BRAjKnaReoPQFciAl9U3s= -github.com/butuzov/ireturn v0.4.1 h1:vWb3NO4t77iku/sjCQ/2pHTQeOmxEhjIriJqRLg1Y+I= -github.com/butuzov/ireturn v0.4.1/go.mod h1:q+DXKzTDV5guNuXLnIab9fKXizTn2miZHLhxH7V/GB4= -github.com/butuzov/mirror v1.3.3 h1:v0RsWBhfFc1RQqE/f3sHpSttKDtodFn0gFmtYyD4/hA= -github.com/butuzov/mirror v1.3.3/go.mod h1:h9BzzwYnTiHO0GzgvaTqIg7VSsOUhdIv51cHFFBmX1w= -github.com/catenacyber/perfsprint v0.10.1 h1:u7Riei30bk46XsG8nknMhKLXG9BcXz3+3tl/WpKm0PQ= -github.com/catenacyber/perfsprint v0.10.1/go.mod h1:DJTGsi/Zufpuus6XPGJyKOTMELe347o6akPvWG9Zcsc= -github.com/ccojocar/zxcvbn-go v1.0.4 h1:FWnCIRMXPj43ukfX000kvBZvV6raSxakYr1nzyNrUcc= -github.com/ccojocar/zxcvbn-go v1.0.4/go.mod h1:3GxGX+rHmueTUMvm5ium7irpyjmm7ikxYFOSJB21Das= github.com/cenk/backoff v2.2.1+incompatible h1:djdFT7f4gF2ttuzRKPbMOWgZajgesItGLwG5FTQKmmE= github.com/cenk/backoff v2.2.1+incompatible/go.mod h1:7FtoeaSnHoZnmZzz47cM35Y9nSW7tNyaidugnHTaFDE= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/charithe/durationcheck v0.0.11 h1:g1/EX1eIiKS57NTWsYtHDZ/APfeXKhye1DidBcABctk= -github.com/charithe/durationcheck v0.0.11/go.mod h1:x5iZaixRNl8ctbM+3B2RrPG5t856TxRyVQEnbIEM2X4= -github.com/charmbracelet/colorprofile v0.4.3 h1:QPa1IWkYI+AOB+fE+mg/5/4HRMZcaXex9t5KX76i20Q= -github.com/charmbracelet/colorprofile v0.4.3/go.mod h1:/zT4BhpD5aGFpqQQqw7a+VtHCzu+zrQtt1zhMt9mR4Q= -github.com/charmbracelet/ultraviolet v0.0.0-20260811164956-006e29f97886 h1:rdnVWKgJpTVXKuKuJyxDJ+NFJdUaUqGvyGy61OcvlbA= -github.com/charmbracelet/ultraviolet v0.0.0-20260811164956-006e29f97886/go.mod h1:nAw0d9PhFp1qdzi2xhQU5YOu5sVpDIHWlaW2Uz/bCro= -github.com/charmbracelet/x/ansi v0.11.8 h1:JMFwp0CgDC2+jcOB162HH5k7I3FVbgFSMMYg7dSPBQQ= -github.com/charmbracelet/x/ansi v0.11.8/go.mod h1:ZNN+3mXny/516oTQPLMPIBeSINvNJJQ8uQXDgbeJxY0= -github.com/charmbracelet/x/term v0.2.2 h1:xVRT/S2ZcKdhhOuSP4t5cLi5o+JxklsoEObBSgfgZRk= -github.com/charmbracelet/x/term v0.2.2/go.mod h1:kF8CY5RddLWrsgVwpw4kAa6TESp6EB5y3uxGLeCqzAI= -github.com/charmbracelet/x/termios v0.1.1 h1:o3Q2bT8eqzGnGPOYheoYS8eEleT5ZVNYNy8JawjaNZY= -github.com/charmbracelet/x/termios v0.1.1/go.mod h1:rB7fnv1TgOPOyyKRJ9o+AsTU/vK5WHJ2ivHeut/Pcwo= -github.com/charmbracelet/x/windows v0.2.2 h1:IofanmuvaxnKHuV04sC0eBy/smG6kIKrWG2/jYn2GuM= -github.com/charmbracelet/x/windows v0.2.2/go.mod h1:/8XtdKZzedat74NQFn0NGlGL4soHB0YQZrETF96h75k= -github.com/ckaznocha/intrange v0.3.1 h1:j1onQyXvHUsPWujDH6WIjhyH26gkRt/txNlV7LspvJs= -github.com/ckaznocha/intrange v0.3.1/go.mod h1:QVepyz1AkUoFQkpEqksSYpNpUo3c5W7nWh/s6SHIJJk= -github.com/clipperhouse/displaywidth v0.11.0 h1:lBc6kY44VFw+TDx4I8opi/EtL9m20WSEFgwIwO+UVM8= -github.com/clipperhouse/displaywidth v0.11.0/go.mod h1:bkrFNkf81G8HyVqmKGxsPufD3JhNl3dSqnGhOoSD/o0= -github.com/clipperhouse/uax29/v2 v2.7.0 h1:+gs4oBZ2gPfVrKPthwbMzWZDaAFPGYK72F0NJv2v7Vk= -github.com/clipperhouse/uax29/v2 v2.7.0/go.mod h1:EFJ2TJMRUaplDxHKj1qAEhCtQPW2tJSwu5BF98AuoVM= github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2 h1:aBangftG7EVZoUb69Os8IaYg++6uMOdKK83QtkkvJik= github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2/go.mod h1:qwXFYgsP6T7XnJtbKlf1HP8AjxZZyzxMmc+Lq5GjlU4= -github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= -github.com/cpuguy83/go-md2man/v2 v2.0.6 h1:XJtiaUW6dEEqVuZiMTn1ldk455QWwEIsMIJlo5vtkx0= -github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/curioswitch/go-reassign v0.3.0 h1:dh3kpQHuADL3cobV/sSGETA8DOv457dwl+fbBAhrQPs= -github.com/curioswitch/go-reassign v0.3.0/go.mod h1:nApPCCTtqLJN/s8HfItCcKV0jIPwluBOvZP+dsJGA88= -github.com/daixiang0/gci v0.13.7 h1:+0bG5eK9vlI08J+J/NWGbWPTNiXPG4WhNLJOkSxWITQ= -github.com/daixiang0/gci v0.13.7/go.mod h1:812WVN6JLFY9S6Tv76twqmNqevN0pa3SX3nih0brVzQ= -github.com/dave/dst v0.27.3 h1:P1HPoMza3cMEquVf9kKy8yXsFirry4zEnWOdYPOoIzY= -github.com/dave/dst v0.27.3/go.mod h1:jHh6EOibnHgcUW3WjKHisiooEkYwqpHLBSX1iOBhEyc= -github.com/dave/jennifer v1.7.1 h1:B4jJJDHelWcDhlRQxWeo0Npa/pYKBLrirAQoTN45txo= -github.com/dave/jennifer v1.7.1/go.mod h1:nXbxhEmQfOZhWml3D1cDK5M1FLnMSozpbFN/m3RmGZc= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/denis-tingaikin/go-header v0.5.0 h1:SRdnP5ZKvcO9KKRP1KJrhFR3RrlGuD+42t4429eC9k8= -github.com/denis-tingaikin/go-header v0.5.0/go.mod h1:mMenU5bWrok6Wl2UsZjy+1okegmwQ3UgWl4V1D8gjlY= -github.com/dlclark/regexp2 v1.12.0 h1:0j4c5qQmnC6XOWNjP3PIXURXN2gWx76rd3KvgdPkCz8= -github.com/dlclark/regexp2 v1.12.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= -github.com/dlclark/regexp2/v2 v2.2.1 h1:mf4KkFUj0gJuarK8P+LgiS+Lit7m9N1yAwEfPbee7R0= -github.com/dlclark/regexp2/v2 v2.2.1/go.mod h1:avUrQvPaLz2DrFNHJF0taWAFFX2C1GMSSoeiqFjcBmU= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/ecosyste-ms/ecosystems-go v0.5.0 h1:qZqgar3Do9RrtW327OXlsrGKiRvNdeX8CmXb1aX4Njo= @@ -239,26 +121,10 @@ github.com/envoyproxy/go-control-plane/envoy v1.37.0 h1:u3riX6BoYRfF4Dr7dwSOroNf github.com/envoyproxy/go-control-plane/envoy v1.37.0/go.mod h1:DReE9MMrmecPy+YvQOAOHNYMALuowAnbjjEMkkWOi6A= github.com/envoyproxy/protoc-gen-validate v1.3.3 h1:MVQghNeW+LZcmXe7SY1V36Z+WFMDjpqGAGacLe2T0ds= github.com/envoyproxy/protoc-gen-validate v1.3.3/go.mod h1:TsndJ/ngyIdQRhMcVVGDDHINPLWB7C82oDArY51KfB0= -github.com/ettle/strcase v0.2.0 h1:fGNiVF21fHXpX1niBgk0aROov1LagYsOwV/xqKDKR/Q= -github.com/ettle/strcase v0.2.0/go.mod h1:DajmHElDSaX76ITe3/VHVyMin4LWSJN5Z909Wp+ED1A= github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a h1:yDWHCSQ40h88yih2JAcL6Ls/kVkSE8GFACTGVnMPruw= github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a/go.mod h1:7Ga40egUymuWXxAe151lTNnCv97MddSOVsjpPPkityA= -github.com/fatih/color v1.19.0 h1:Zp3PiM21/9Ld6FzSKyL5c/BULoe/ONr9KlbYVOfG8+w= -github.com/fatih/color v1.19.0/go.mod h1:zNk67I0ZUT1bEGsSGyCZYZNrHuTkJJB+r6Q9VuMi0LE= -github.com/fatih/structtag v1.2.0 h1:/OdNE99OxoI/PqaW/SuSK9uxxT3f/tcSZgon/ssNSx4= -github.com/fatih/structtag v1.2.0/go.mod h1:mBJUNpUnHmRKrKlQQlmCrh5PuhftFbNv8Ys4/aAZl94= github.com/felixge/httpsnoop v1.1.0 h1:3YtUj32ZZkqZtt3sZZsClsymw/QDuVfpNhoA31zeORc= github.com/felixge/httpsnoop v1.1.0/go.mod h1:Zqxgdd+1Rkcz8euOqdr7lqgCRJztwr5hp9vDSi5UZCE= -github.com/firefart/nonamedreturns v1.0.8 h1:iB32Dl17zJl1zlVEj/WlUWgx0HiRyQ85OUw1WHa4/II= -github.com/firefart/nonamedreturns v1.0.8/go.mod h1:vxFNvm5AfP/8rgAKFzYmnqx0yp1HjrYsErZ9pHPTznA= -github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE= -github.com/frankban/quicktest v1.14.3/go.mod h1:mgiwOwqx65TmIk1wJ6Q7wvnVMocbUorkibMOrVTHZps= -github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= -github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= -github.com/fzipp/gocyclo v0.6.0 h1:lsblElZG7d3ALtGMx9fmxeTKZaLLpU8mET09yN4BBLo= -github.com/fzipp/gocyclo v0.6.0/go.mod h1:rXPyn8fnlpa0R2csP/31uerbiVBugk5whMdlyaLkLoA= -github.com/ghostiam/protogetter v0.3.21 h1:EeWTGvL/Eyosp653hiWb6Byx4b69iJC4/E+za7vQHoI= -github.com/ghostiam/protogetter v0.3.21/go.mod h1:iAKSpyoHwYzay+OpjoWgwzRtPFthEfuUvmlomTThck0= github.com/git-pkgs/archives v0.8.1 h1:IcT1bVsAec8SqJ4m9vTnD644KiadQZ4ERrWnm/30t3o= github.com/git-pkgs/archives v0.8.1/go.mod h1:gIyPXPQr5e+YrMqA5KQ7x6pLrFXHNF1+CjdedNJdGUU= github.com/git-pkgs/artifacts v0.2.1 h1:VwdxR4yTDaqBZ34h0slxQBVyr2Xfa+QGOKCKviMx2xk= @@ -289,8 +155,6 @@ github.com/github/go-spdx/v2 v2.7.0 h1:GzfXx4wFdlilARxmFRXW/mgUy3A4vSqZocCMFV6XF github.com/github/go-spdx/v2 v2.7.0/go.mod h1:Ftc45YYG1WzpzwEPKRVm9Jv8vDqOrN4gWoCkK+bHer0= github.com/go-chi/chi/v5 v5.3.2 h1:5YQkICvTCSZ25hoRsyJazN0scjzKGiu4VAUc7H1o1nY= github.com/go-chi/chi/v5 v5.3.2/go.mod h1:R+tYY2hNuVUUjxoPtqUdgBqevM9s9njzkTLutVsOCto= -github.com/go-critic/go-critic v0.14.4 h1:dSX4C3pWSeuMVxvQh6yG8U0ReSf3YOmKi4nwX5q7n/8= -github.com/go-critic/go-critic v0.14.4/go.mod h1:xwntfW6SYAd7h1OqDzmN6hBX/JxsEKl5up/Y2bsxgVQ= github.com/go-jose/go-jose/v4 v4.1.4 h1:moDMcTHmvE6Groj34emNPLs/qtYXRVcd6S7NHbHz3kA= github.com/go-jose/go-jose/v4 v4.1.4/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= @@ -308,73 +172,13 @@ github.com/go-openapi/spec v0.20.4/go.mod h1:faYFR1CvsJZ0mNsmsphTMSoRrNV3TEDoAM7 github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-openapi/swag v0.19.15 h1:D2NRCBzS9/pEY3gP9Nl8aDqGUcPFrwG2p+CNFrLyrCM= github.com/go-openapi/swag v0.19.15/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= -github.com/go-quicktest/qt v1.102.0 h1:HSQxCeh5YZH3EL3W39ixjtyaEhcWSXQHtHnMBzSs474= -github.com/go-quicktest/qt v1.102.0/go.mod h1:p4lGIVX+8Wa6ZPNDvqcxq36XpUDLh42FLetFU7odllI= github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg= github.com/go-sql-driver/mysql v1.9.3 h1:U/N249h2WzJ3Ukj8SowVFjdtZKfu9vlLZxjPXV1aweo= github.com/go-sql-driver/mysql v1.9.3/go.mod h1:qn46aNg1333BRMNU69Lq93t8du/dwxI64Gl8i5p1WMU= -github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= -github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= -github.com/go-toolsmith/astcast v1.1.0 h1:+JN9xZV1A+Re+95pgnMgDboWNVnIMMQXwfBwLRPgSC8= -github.com/go-toolsmith/astcast v1.1.0/go.mod h1:qdcuFWeGGS2xX5bLM/c3U9lewg7+Zu4mr+xPwZIB4ZU= -github.com/go-toolsmith/astcopy v1.1.0 h1:YGwBN0WM+ekI/6SS6+52zLDEf8Yvp3n2seZITCUBt5s= -github.com/go-toolsmith/astcopy v1.1.0/go.mod h1:hXM6gan18VA1T/daUEHCFcYiW8Ai1tIwIzHY6srfEAw= -github.com/go-toolsmith/astequal v1.0.3/go.mod h1:9Ai4UglvtR+4up+bAD4+hCj7iTo4m/OXVTSLnCyTAx4= -github.com/go-toolsmith/astequal v1.1.0/go.mod h1:sedf7VIdCL22LD8qIvv7Nn9MuWJruQA/ysswh64lffQ= -github.com/go-toolsmith/astequal v1.2.0 h1:3Fs3CYZ1k9Vo4FzFhwwewC3CHISHDnVUPC4x0bI2+Cw= -github.com/go-toolsmith/astequal v1.2.0/go.mod h1:c8NZ3+kSFtFY/8lPso4v8LuJjdJiUFVnSuU3s0qrrDY= -github.com/go-toolsmith/astfmt v1.1.0 h1:iJVPDPp6/7AaeLJEruMsBUlOYCmvg0MoCfJprsOmcco= -github.com/go-toolsmith/astfmt v1.1.0/go.mod h1:OrcLlRwu0CuiIBp/8b5PYF9ktGVZUjlNMV634mhwuQ4= -github.com/go-toolsmith/astp v1.1.0 h1:dXPuCl6u2llURjdPLLDxJeZInAeZ0/eZwFJmqZMnpQA= -github.com/go-toolsmith/astp v1.1.0/go.mod h1:0T1xFGz9hicKs8Z5MfAqSUitoUYS30pDMsRVIDHs8CA= -github.com/go-toolsmith/pkgload v1.2.2 h1:0CtmHq/02QhxcF7E9N5LIFcYFsMR5rdovfqTtRKkgIk= -github.com/go-toolsmith/pkgload v1.2.2/go.mod h1:R2hxLNRKuAsiXCo2i5J6ZQPhnPMOVtU+f0arbFPWCus= -github.com/go-toolsmith/strparse v1.0.0/go.mod h1:YI2nUKP9YGZnL/L1/DLFBfixrcjslWct4wyljWhSRy8= -github.com/go-toolsmith/strparse v1.1.0 h1:GAioeZUK9TGxnLS+qfdqNbA4z0SSm5zVNtCQiyP2Bvw= -github.com/go-toolsmith/strparse v1.1.0/go.mod h1:7ksGy58fsaQkGQlY8WVoBFNyEPMGuJin1rfoPS4lBSQ= -github.com/go-toolsmith/typep v1.1.0 h1:fIRYDyF+JywLfqzyhdiHzRop/GQDxxNhLGQ6gFUNHus= -github.com/go-toolsmith/typep v1.1.0/go.mod h1:fVIw+7zjdsMxDA3ITWnH1yOiw1rnTQKCsF/sk2H/qig= -github.com/go-viper/mapstructure/v2 v2.5.0 h1:vM5IJoUAy3d7zRSVtIwQgBj7BiWtMPfmPEgAXnvj1Ro= -github.com/go-viper/mapstructure/v2 v2.5.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= -github.com/go-xmlfmt/xmlfmt v1.1.3 h1:t8Ey3Uy7jDSEisW2K3somuMKIpzktkWptA0iFCnRUWY= -github.com/go-xmlfmt/xmlfmt v1.1.3/go.mod h1:aUCEOzzezBEjDBbFBoSiya/gduyIiWYRP6CnSFIV8AM= -github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= -github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= -github.com/godoc-lint/godoc-lint v0.11.2 h1:Bp0FkJWoSdNsBikdNgIcgtaoo+xz6I/Y9s5WSBQUeeM= -github.com/godoc-lint/godoc-lint v0.11.2/go.mod h1:iVpGdL1JCikNH2gGeAn3Hh+AgN5Gx/I/cxV+91L41jo= -github.com/gofrs/flock v0.13.0 h1:95JolYOvGMqeH31+FC7D2+uULf6mG61mEZ/A8dRYMzw= -github.com/gofrs/flock v0.13.0/go.mod h1:jxeyy9R1auM5S6JYDBhDt+E2TCo7DkratH4Pgi8P+Z0= github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY= github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= -github.com/golangci/asciicheck v0.5.0 h1:jczN/BorERZwK8oiFBOGvlGPknhvq0bjnysTj4nUfo0= -github.com/golangci/asciicheck v0.5.0/go.mod h1:5RMNAInbNFw2krqN6ibBxN/zfRFa9S6tA1nPdM0l8qQ= -github.com/golangci/dupl v0.0.0-20260401084720-c99c5cf5c202 h1:CbTB8KpqnViI6lIXxp03Oclc4VFHi3K4BWC1TacsZ+A= -github.com/golangci/dupl v0.0.0-20260401084720-c99c5cf5c202/go.mod h1:NUw9Zr2Sy7+HxzdjIULge71wI6yEg1lWQr7Evcu8K0E= -github.com/golangci/go-printf-func-name v0.1.1 h1:hIYTFJqAGp1iwoIfsNTpoq1xZAarogrvjO9AfiW3B4U= -github.com/golangci/go-printf-func-name v0.1.1/go.mod h1:Es64MpWEZbh0UBtTAICOZiB+miW53w/K9Or/4QogJss= -github.com/golangci/gofmt v0.0.0-20260820135601-e84e05053792 h1:WL8YKrt3UbOBqSRU7GpP5BTtQTMWtVtj+mfPijgZeIg= -github.com/golangci/gofmt v0.0.0-20260820135601-e84e05053792/go.mod h1:te5hX0dW4C5r6YbXs+6ysNr8Q5UTmdIqGbb+mlFiYmA= -github.com/golangci/golangci-lint/v2 v2.13.1 h1:RuM4OcluM4xFQcGuRE6R7jA33pqxK/W1EsBxpugdZjg= -github.com/golangci/golangci-lint/v2 v2.13.1/go.mod h1:HwX7mDzqHbcSxlhrTygjX1GJbAfQ3sJAqOx41qQlhDE= -github.com/golangci/golines v0.15.0 h1:Qnph25g8Y1c5fdo1X7GaRDGgnMHgnxh4Gk4VfPTtRx0= -github.com/golangci/golines v0.15.0/go.mod h1:AZjXd23tbHMpowhtnGlj9KCNsysj72aeZVVHnVcZx10= -github.com/golangci/misspell v0.8.0 h1:qvxQhiE2/5z+BVRo1kwYA8yGz+lOlu5Jfvtx2b04Jbg= -github.com/golangci/misspell v0.8.0/go.mod h1:WZyyI2P3hxPY2UVHs3cS8YcllAeyfquQcKfdeE9AFVg= -github.com/golangci/plugin-module-register v0.1.2 h1:e5WM6PO6NIAEcij3B053CohVp3HIYbzSuP53UAYgOpg= -github.com/golangci/plugin-module-register v0.1.2/go.mod h1:1+QGTsKBvAIvPvoY/os+G5eoqxWn70HYDm2uvUyGuVw= -github.com/golangci/revgrep v0.8.0 h1:EZBctwbVd0aMeRnNUsFogoyayvKHyxlV3CdUA46FX2s= -github.com/golangci/revgrep v0.8.0/go.mod h1:U4R/s9dlXZsg8uJmaR1GrloUr14D7qDl8gi2iPXJH8k= -github.com/golangci/rowserrcheck v0.0.0-20260419091836-c5f79b8a11ba h1:lqtcnSMDuuJdu/LrKWi5RJzpSNLOJXYe/nzQutTI5kg= -github.com/golangci/rowserrcheck v0.0.0-20260419091836-c5f79b8a11ba/go.mod h1:sCBNcpRmhJCtbFGz49+IM3ETTFf7QdJ30AeYCd43NKk= -github.com/golangci/swaggoswag v0.0.0-20250504205917-77f2aca3143e h1:ai0EfmVYE2bRA5htgAG9r7s3tHsfjIhN98WshBTJ9jM= -github.com/golangci/swaggoswag v0.0.0-20250504205917-77f2aca3143e/go.mod h1:Vrn4B5oR9qRwM+f54koyeH3yzphlecwERs0el27Fr/s= -github.com/golangci/unconvert v0.0.0-20250410112200-a129a6e6413e h1:gD6P7NEo7Eqtt0ssnqSJNNndxe69DOQ24A5h7+i3KpM= -github.com/golangci/unconvert v0.0.0-20250410112200-a129a6e6413e/go.mod h1:h+wZwLjUTJnm/P2rwlbJdRPZXOzaT36/FwnPnY2inzc= -github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/go-replayers/grpcreplay v1.3.0 h1:1Keyy0m1sIpqstQmgz307zhiJ1pV4uIlFds5weTmxbo= @@ -395,54 +199,15 @@ github.com/googleapis/enterprise-certificate-proxy v0.3.18 h1:hvVi34VucdrV1IIsiW github.com/googleapis/enterprise-certificate-proxy v0.3.18/go.mod h1:rSEsBUemEBZEexP2y6jPp16LUmUbjmSbcPMQizR0o4k= github.com/googleapis/gax-go/v2 v2.23.0 h1:Tchl7qkvE7Ip3y+ztvNufYFvkfqTe7NfLTYGIdJRLuE= github.com/googleapis/gax-go/v2 v2.23.0/go.mod h1:rBQKOVJCdb8IFEzg+FCwlt1LP/xMDGuqUXhUG+XMXEg= -github.com/gordonklaus/ineffassign v0.2.0 h1:Uths4KnmwxNJNzq87fwQQDDnbNb7De00VOk9Nu0TySs= -github.com/gordonklaus/ineffassign v0.2.0/go.mod h1:TIpymnagPSexySzs7F9FnO1XFTy8IT3a59vmZp5Y9Lw= -github.com/gostaticanalysis/analysisutil v0.7.1 h1:ZMCjoue3DtDWQ5WyU16YbjbQEQ3VuzwxALrpYd+HeKk= -github.com/gostaticanalysis/analysisutil v0.7.1/go.mod h1:v21E3hY37WKMGSnbsw2S/ojApNWb6C1//mXO48CXbVc= -github.com/gostaticanalysis/comment v1.4.2/go.mod h1:KLUTGDv6HOCotCH8h2erHKmpci2ZoR8VPu34YA2uzdM= -github.com/gostaticanalysis/comment v1.5.0 h1:X82FLl+TswsUMpMh17srGRuKaaXprTaytmEpgnKIDu8= -github.com/gostaticanalysis/comment v1.5.0/go.mod h1:V6eb3gpCv9GNVqb6amXzEUX3jXLVK/AdA+IrAMSqvEc= -github.com/gostaticanalysis/forcetypeassert v0.2.0 h1:uSnWrrUEYDr86OCxWa4/Tp2jeYDlogZiZHzGkWFefTk= -github.com/gostaticanalysis/forcetypeassert v0.2.0/go.mod h1:M5iPavzE9pPqWyeiVXSFghQjljW1+l/Uke3PXHS6ILY= -github.com/gostaticanalysis/nilerr v0.1.2 h1:S6nk8a9N8g062nsx63kUkF6AzbHGw7zzyHMcpu52xQU= -github.com/gostaticanalysis/nilerr v0.1.2/go.mod h1:A19UHhoY3y8ahoL7YKz6sdjDtduwTSI4CsymaC2htPA= -github.com/gostaticanalysis/testutil v0.3.1-0.20210208050101-bfb5c8eec0e4/go.mod h1:D+FIZ+7OahH3ePw/izIEeH5I06eKs1IKI4Xr64/Am3M= -github.com/gostaticanalysis/testutil v0.5.0 h1:Dq4wT1DdTwTGCQQv3rl3IvD5Ld0E6HiY+3Zh0sUGqw8= -github.com/gostaticanalysis/testutil v0.5.0/go.mod h1:OLQSbuM6zw2EvCcXTz1lVq5unyoNft372msDY0nY5Hs= -github.com/hashicorp/go-immutable-radix/v2 v2.1.0 h1:CUW5RYIcysz+D3B+l1mDeXrQ7fUvGGCwJfdASSzbrfo= -github.com/hashicorp/go-immutable-radix/v2 v2.1.0/go.mod h1:hgdqLXA4f6NIjRVisM1TJ9aOJVNRqKZj+xDGF6m7PBw= -github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/Co8= -github.com/hashicorp/go-uuid v1.0.3/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/go-version v1.2.1/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= -github.com/hashicorp/go-version v1.9.0 h1:CeOIz6k+LoN3qX9Z0tyQrPtiB1DFYRPfCIBtaXPSCnA= -github.com/hashicorp/go-version v1.9.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= -github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= -github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= -github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= -github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg= -github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= -github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= -github.com/jgautheron/goconst v1.11.0 h1:KgN90z5qXt5f0Uzf3cWXev3hfMMFUyNeKdpkSBRvLDk= -github.com/jgautheron/goconst v1.11.0/go.mod h1:0p+wv1lFOiUr0IlNNT1nrm6+8DB8u2sU6KHGzFRXHDc= -github.com/jjti/go-spancheck v0.6.5 h1:lmi7pKxa37oKYIMScialXUK6hP3iY5F1gu+mLBPgYB8= -github.com/jjti/go-spancheck v0.6.5/go.mod h1:aEogkeatBrbYsyW6y5TgDfihCulDYciL1B7rG2vSsrU= github.com/jmoiron/sqlx v1.4.0 h1:1PLqN7S1UYp5t4SrVVnt4nUVNemrDAtxlulVe+Qgm3o= github.com/jmoiron/sqlx v1.4.0/go.mod h1:ZrZ7UsYB/weZdl2Bxg6jCRO9c3YHl8r3ahlKmRT4JLY= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/juju/gnuflag v0.0.0-20171113085948-2ce1bb71843d/go.mod h1:2PavIy+JPciBPrBUjwbNvtwB6RQlve+hkpll6QSNmOE= -github.com/julz/importas v0.2.0 h1:y+MJN/UdL63QbFJHws9BVC5RpA2iq0kpjrFajTGivjQ= -github.com/julz/importas v0.2.0/go.mod h1:pThlt589EnCYtMnmhmRYY/qn9lCf/frPOK+WMx3xiJY= -github.com/karamaru-alpha/copyloopvar v1.2.2 h1:yfNQvP9YaGQR7VaWLYcfZUlRP2eo2vhExWKxD/fP6q0= -github.com/karamaru-alpha/copyloopvar v1.2.2/go.mod h1:oY4rGZqZ879JkJMtX3RRkcXRkmUvH0x35ykgaKgsgJY= github.com/keybase/go-keychain v0.0.1 h1:way+bWYa6lDppZoZcgMbYsvC7GxljxrskdNInRtuthU= github.com/keybase/go-keychain v0.0.1/go.mod h1:PdEILRW3i9D8JcdM+FmY6RwkHGnhHxXwkPPMeUgOK1k= -github.com/kisielk/errcheck v1.20.0 h1:9rwHBNKzd4wkDWcROy3DvFGNqEPlkxBg305rvk7HabI= -github.com/kisielk/errcheck v1.20.0/go.mod h1:O+f80MKNwX8Oor2jwgpeQ9An7uJm+hRSgT+h22knRJU= -github.com/kkHAIKE/contextcheck v1.1.6 h1:7HIyRcnyzxL9Lz06NGhiKvenXq7Zw6Q0UQu/ttjfJCE= -github.com/kkHAIKE/contextcheck v1.1.6/go.mod h1:3dDbMRNBFaq8HFXWC1JyvDSPm43CmE6IuHam8Wr0rkg= github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA= github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= @@ -452,110 +217,35 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/kulti/thelper v0.7.1 h1:fI8QITAoFVLx+y+vSyuLBP+rcVIB8jKooNSCT2EiI98= -github.com/kulti/thelper v0.7.1/go.mod h1:NsMjfQEy6sd+9Kfw8kCP61W1I0nerGSYSFnGaxQkcbs= -github.com/kunwardeep/paralleltest v1.0.15 h1:ZMk4Qt306tHIgKISHWFJAO1IDQJLc6uDyJMLyncOb6w= -github.com/kunwardeep/paralleltest v1.0.15/go.mod h1:di4moFqtfz3ToSKxhNjhOZL+696QtJGCFe132CbBLGk= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= -github.com/lasiar/canonicalheader v1.1.2 h1:vZ5uqwvDbyJCnMhmFYimgMZnJMjwljN5VGY0VKbMXb4= -github.com/lasiar/canonicalheader v1.1.2/go.mod h1:qJCeLFS0G/QlLQ506T+Fk/fWMa2VmBUiEI2cuMK4djI= -github.com/ldez/exptostd v0.4.5 h1:kv2ZGUVI6VwRfp/+bcQ6Nbx0ghFWcGIKInkG/oFn1aQ= -github.com/ldez/exptostd v0.4.5/go.mod h1:QRjHRMXJrCTIm9WxVNH6VW7oN7KrGSht69bIRwvdFsM= -github.com/ldez/gomoddirectives v0.9.0 h1:2YV/EX7nVlWL4jySusYTzBKHuE3D2fgcRsQuMa3yIoo= -github.com/ldez/gomoddirectives v0.9.0/go.mod h1:DdZzfm9MdXCjn2/UGYXCFfo+tzrp2Ib4iD2Q0kIJkwE= -github.com/ldez/grignotin v0.10.1 h1:keYi9rYsgbvqAZGI1liek5c+jv9UUjbvdj3Tbn5fn4o= -github.com/ldez/grignotin v0.10.1/go.mod h1:UlDbXFCARrXbWGNGP3S5vsysNXAPhnSuBufpTEbwOas= -github.com/ldez/structtags v0.6.1 h1:bUooFLbXx41tW8SvkfwfFkkjPYvFFs59AAMgVg6DUBk= -github.com/ldez/structtags v0.6.1/go.mod h1:YDxVSgDy/MON6ariaxLF2X09bh19qL7MtGBN5MrvbdY= -github.com/ldez/tagliatelle v0.7.2 h1:KuOlL70/fu9paxuxbeqlicJnCspCRjH0x8FW+NfgYUk= -github.com/ldez/tagliatelle v0.7.2/go.mod h1:PtGgm163ZplJfZMZ2sf5nhUT170rSuPgBimoyYtdaSI= -github.com/ldez/usetesting v0.5.0 h1:3/QtzZObBKLy1F4F8jLuKJiKBjjVFi1IavpoWbmqLwc= -github.com/ldez/usetesting v0.5.0/go.mod h1:Spnb4Qppf8JTuRgblLrEWb7IE6rDmUpGvxY3iRrzvDQ= -github.com/leonklingele/grouper v1.1.2 h1:o1ARBDLOmmasUaNDesWqWCIFH3u7hoFlM84YrjT3mIY= -github.com/leonklingele/grouper v1.1.2/go.mod h1:6D0M/HVkhs2yRKRFZUoGjeDy7EZTfFBE9gl4kjmIGkA= github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/lib/pq v1.12.3 h1:tTWxr2YLKwIvK90ZXEw8GP7UFHtcbTtty8zsI+YjrfQ= github.com/lib/pq v1.12.3/go.mod h1:/p+8NSbOcwzAEI7wiMXFlgydTwcgTr3OSKMsD2BitpA= -github.com/lucasb-eyer/go-colorful v1.4.1 h1:1EO+WB73+EH8EVbzlrG3KLAfEypQWVHIBqlTf+2hNss= -github.com/lucasb-eyer/go-colorful v1.4.1/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= -github.com/macabu/inamedparam v0.2.0 h1:VyPYpOc10nkhI2qeNUdh3Zket4fcZjEWe35poddBCpE= -github.com/macabu/inamedparam v0.2.0/go.mod h1:+Pee9/YfGe5LJ62pYXqB89lJ+0k5bsR8Wgz/C0Zlq3U= -github.com/magiconair/properties v1.8.6 h1:5ibWZ6iY0NctNGWo87LalDlEZ6R41TqbbDamhfG/Qzo= -github.com/magiconair/properties v1.8.6/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= -github.com/manuelarte/embeddedstructfieldcheck v0.4.0 h1:3mAIyaGRtjK6EO9E73JlXLtiy7ha80b2ZVGyacxgfww= -github.com/manuelarte/embeddedstructfieldcheck v0.4.0/go.mod h1:z8dFSyXqp+fC6NLDSljRJeNQJJDWnY7RoWFzV3PC6UM= -github.com/manuelarte/funcorder v0.6.0 h1:0hBngc4fa1IgNiI65A7sFGkMvoMCc878RjqB5V7rWP0= -github.com/manuelarte/funcorder v0.6.0/go.mod h1:id3NDhXdQBmeqXH7eVC6Z89xS6JxvZ8kF9xUxpArU/g= -github.com/maratori/testableexamples v1.0.1 h1:HfOQXs+XgfeRBJ+Wz0XfH+FHnoY9TVqL6Fcevpzy4q8= -github.com/maratori/testableexamples v1.0.1/go.mod h1:XE2F/nQs7B9N08JgyRmdGjYVGqxWwClLPCGSQhXQSrQ= -github.com/maratori/testpackage v1.1.2 h1:ffDSh+AgqluCLMXhM19f/cpvQAKygKAJXFl9aUjmbqs= -github.com/maratori/testpackage v1.1.2/go.mod h1:8F24GdVDFW5Ew43Et02jamrVMNXLUNaOynhDssITGfc= -github.com/matoous/godox v1.1.0 h1:W5mqwbyWrwZv6OQ5Z1a/DHGMOvXYCBP3+Ht7KMoJhq4= -github.com/matoous/godox v1.1.0/go.mod h1:jgE/3fUXiTurkdHOLT5WEkThTSuE7yxHv5iWPa80afs= -github.com/matryer/is v1.4.0 h1:sosSmIWwkYITGrxZ25ULNDeKiMNzFSr4V/eqBQP0PeE= -github.com/matryer/is v1.4.0/go.mod h1:8I/i5uYgLzgsgEloJE1U6xx5HkBQpAZvepWuujKwMRU= -github.com/mattn/go-colorable v0.1.15 h1:+u9SLTRGnXv73cEsnsmoZBom+dMU88B2M0aDcWy0/jY= -github.com/mattn/go-colorable v0.1.15/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsReI= github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A= -github.com/mattn/go-runewidth v0.0.24 h1:cpokDiIn0MGnhdHwuWnJBITySJ20QyNGnY2kR/ay2DU= -github.com/mattn/go-runewidth v0.0.24/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs= github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU= github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= -github.com/mgechev/revive v1.15.0 h1:vJ0HzSBzfNyPbHKolgiFjHxLek9KUijhqh42yGoqZ8Q= -github.com/mgechev/revive v1.15.0/go.mod h1:LlAKO3QQe9OJ0pVZzI2GPa8CbXGZ/9lNpCGvK4T/a8A= -github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= -github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= -github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= -github.com/moricho/tparallel v0.3.2 h1:odr8aZVFA3NZrNybggMkYO3rgPRcqjeQUlBBFVxKHTI= -github.com/moricho/tparallel v0.3.2/go.mod h1:OQ+K3b4Ln3l2TZveGCywybl68glfLEwFGqvnjok8b+U= -github.com/muesli/cancelreader v0.2.2 h1:3I4Kt4BQjOR54NavqnDogx/MIoWBFa0StPA8ELUXHmA= -github.com/muesli/cancelreader v0.2.2/go.mod h1:3XuTXfFS2VjM+HTLZY9Ak0l6eUKfijIfMUZ4EgX0QYo= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/nakabonne/nestif v0.3.1 h1:wm28nZjhQY5HyYPx+weN3Q65k6ilSBxDb8v5S81B81U= -github.com/nakabonne/nestif v0.3.1/go.mod h1:9EtoZochLn5iUprVDmDjqGKPofoUEBL8U4Ngq6aY7OE= github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w= github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= -github.com/nishanths/exhaustive v0.12.0 h1:vIY9sALmw6T/yxiASewa4TQcFsVYZQQRUQJhKRf3Swg= -github.com/nishanths/exhaustive v0.12.0/go.mod h1:mEZ95wPIZW+x8kC4TgC+9YCUgiST7ecevsVDTgc2obs= -github.com/nishanths/predeclared v0.2.2 h1:V2EPdZPliZymNAn79T8RkNApBjMmVKh5XRpLm/w98Vk= -github.com/nishanths/predeclared v0.2.2/go.mod h1:RROzoN6TnGQupbC+lqggsOlcgysk3LMK/HI84Mp280c= -github.com/nunnatsa/ginkgolinter v0.24.0 h1:Mp0EagluLFP98JatP6nqp/gGEoljNG97uf9AcxcBVy8= -github.com/nunnatsa/ginkgolinter v0.24.0/go.mod h1:2ZMRuzX6+3XXyY6UZOwb6n+MCocVGbkIsDBC4vuWz5c= github.com/oapi-codegen/nullable v1.2.0 h1:VflFkDW980KhBPiFF7nWSyjg+r4Obqj8lXipV0UkP5w= github.com/oapi-codegen/nullable v1.2.0/go.mod h1:KUZ3vUzkmEKY90ksAmit2+5juDIhIZhfDl+0PwOQlFY= github.com/oapi-codegen/runtime v1.7.0 h1:t7358VYPvNbWJ9gdAkIK/smVeHpBf6yp8VTsaZsb/7k= github.com/oapi-codegen/runtime v1.7.0/go.mod h1:GwV7hC2hviaMzj+ITfHVRESK5J2W/GefVwIND/bMGvU= -github.com/onsi/ginkgo/v2 v2.32.0 h1:Hw7s2pVrQo/8Yz5N77qdnpHaoc+c6cC9WIV1Jce+J6E= -github.com/onsi/ginkgo/v2 v2.32.0/go.mod h1:+aXOY+vzZ5mu2iI2HpTZUPmM//oQfsNFX6gU9kNcA44= -github.com/onsi/gomega v1.42.1 h1:iN1rCUX+44NZ1Dc97MPoeFYbFR0vh8zxoxMFwKdyZ6I= -github.com/onsi/gomega v1.42.1/go.mod h1:REff/hsDsodHoKlWsP2mAPhu1+5/6hVYNf9rIEBpeSg= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= -github.com/otiai10/copy v1.2.0/go.mod h1:rrF5dJ5F0t/EWSYODDu4j9/vEeYHMkc8jt0zJChqQWw= -github.com/otiai10/copy v1.14.0 h1:dCI/t1iTdYGtkvCuBG2BgR6KZa83PTclw4U5n2wAllU= -github.com/otiai10/copy v1.14.0/go.mod h1:ECfuL02W+/FkTWZWgQqXPWZgW9oeKCSQ5qVfSc4qc4w= -github.com/otiai10/curr v0.0.0-20150429015615-9b4961190c95/go.mod h1:9qAhocn7zKJG+0mI8eUu6xqkFDYS2kb2saOteoSB3cE= -github.com/otiai10/curr v1.0.0/go.mod h1:LskTG5wDwr8Rs+nNQ+1LlxRjAtTZZjtJW4rMXl6j4vs= -github.com/otiai10/mint v1.3.0/go.mod h1:F5AjcsTsWUqX+Na9fpHb52P8pcRX2CI6A3ctIT91xUo= -github.com/otiai10/mint v1.3.1/go.mod h1:/yxELlJQ0ufhjUwhshSj+wFjZ78CnZ48/1wtmBH1OTc= github.com/package-url/packageurl-go v0.1.7 h1:iFWg6tzAjLA6F/qX3M5nZaiMHJgc+p2zxVyr/fY+sZY= github.com/package-url/packageurl-go v0.1.7/go.mod h1:nKAWB8E6uk1MHqiS/lQb9pYBGH2+mdJ2PJc2s50dQY0= github.com/pandatix/go-cvss v0.6.4 h1:9w2RCO/Q4UTiJyEgpCHRiVc6CfrsFEnkoX+OtATqKio= github.com/pandatix/go-cvss v0.6.4/go.mod h1:/ukvQnYlrKl3o/DVp7/GO2UZyZheuo/maOK0U1nBEhQ= -github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8= -github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= -github.com/pelletier/go-toml/v2 v2.4.3 h1:GTRvJQutkOSftxIFD5xw9aepkYNuPWmVJpffdDPYVpY= -github.com/pelletier/go-toml/v2 v2.4.3/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= github.com/peterbourgon/g2s v0.0.0-20170223122336-d4e7ad98afea h1:sKwxy1H95npauwu8vtF95vG/syrL0p8fSZo/XlDg5gk= github.com/peterbourgon/g2s v0.0.0-20170223122336-d4e7ad98afea/go.mod h1:1VcHEd3ro4QMoHfiNl/j7Jkln9+KQuorp0PItHMJYNg= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= @@ -573,155 +263,36 @@ github.com/prometheus/common v0.70.1 h1:1HvjP4D5oL3t8RsPlwxA9onvvStjtIHYE5XuuwOi github.com/prometheus/common v0.70.1/go.mod h1:VdFUQDMZK3VLkurFUVhia6uys/0suUp86TJz5qbJRhc= github.com/prometheus/procfs v0.21.1 h1:GljZCt+zSTS+NZq88cyQ1LjZ+RCHp3uVuabBWA5+OJI= github.com/prometheus/procfs v0.21.1/go.mod h1:aB55Cww9pdSJVHk0hUf0inxWyyjPogFIjmHKYgMKmtY= -github.com/quasilyte/go-ruleguard v0.4.5 h1:AGY0tiOT5hJX9BTdx/xBdoCubQUAE2grkqY2lSwvZcA= -github.com/quasilyte/go-ruleguard v0.4.5/go.mod h1:Vl05zJ538vcEEwu16V/Hdu7IYZWyKSwIy4c88Ro1kRE= -github.com/quasilyte/go-ruleguard/dsl v0.3.23 h1:lxjt5B6ZCiBeeNO8/oQsegE6fLeCzuMRoVWSkXC4uvY= -github.com/quasilyte/go-ruleguard/dsl v0.3.23/go.mod h1:KeCP03KrjuSO0H1kTuZQCWlQPulDV6YMIXmpQss17rU= -github.com/quasilyte/gogrep v0.5.0 h1:eTKODPXbI8ffJMN+W2aE0+oL0z/nh8/5eNdiO34SOAo= -github.com/quasilyte/gogrep v0.5.0/go.mod h1:Cm9lpz9NZjEoL1tgZ2OgeUKPIxL1meE7eo60Z6Sk+Ng= -github.com/quasilyte/regex/syntax v0.0.0-20210819130434-b3f0c404a727 h1:TCg2WBOl980XxGFEZSS6KlBGIV0diGdySzxATTWoqaU= -github.com/quasilyte/regex/syntax v0.0.0-20210819130434-b3f0c404a727/go.mod h1:rlzQ04UMyJXu/aOvhd8qT+hvDrFpiwqp8MRXDY9szc0= -github.com/quasilyte/stdinfo v0.0.0-20220114132959-f7386bf02567 h1:M8mH9eK4OUR4lu7Gd+PU1fV2/qnDNfzT635KRSObncs= -github.com/quasilyte/stdinfo v0.0.0-20220114132959-f7386bf02567/go.mod h1:DWNGW8A4Y+GyBgPuaQJuWiy0XYftx4Xm/y5Jqk9I6VQ= -github.com/raeperd/recvcheck v0.3.0 h1:PM+XYvyxIj3bo+kobJfFTdTuU3Lmfu96mKDbyHDbRt8= -github.com/raeperd/recvcheck v0.3.0/go.mod h1:PZNwG+HztFYMH2ZPq0Hu3QgkV2yiA6VrtNz9c1fXWJo= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= -github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= -github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/rogpeppe/go-internal v1.16.0 h1:O9DK+vNMDVGLr2BeZqmpLeMjiMNkuXfcqntWbZV6S5g= github.com/rogpeppe/go-internal v1.16.0/go.mod h1:DrUVZyrJU+txYW5/1kwtXQSMFio52ZOxX7yM1VHvnxs= github.com/rs/dnscache v0.0.0-20230804202142-fc85eb664529 h1:18kd+8ZUlt/ARXhljq+14TwAoKa61q6dX8jtwOf6DH8= github.com/rs/dnscache v0.0.0-20230804202142-fc85eb664529/go.mod h1:qe5TWALJ8/a1Lqznoc5BDHpYX/8HU60Hm2AwRmqzxqA= github.com/rubyist/circuitbreaker v2.2.1+incompatible h1:KUKd/pV8Geg77+8LNDwdow6rVCAYOp8+kHUyFvL6Mhk= github.com/rubyist/circuitbreaker v2.2.1+incompatible/go.mod h1:Ycs3JgJADPuzJDwffe12k6BZT8hxVi6lFK+gWYJLN4A= -github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= -github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/ryancurrah/gomodguard v1.4.1 h1:eWC8eUMNZ/wM/PWuZBv7JxxqT5fiIKSIyTvjb7Elr+g= -github.com/ryancurrah/gomodguard v1.4.1/go.mod h1:qnMJwV1hX9m+YJseXEBhd2s90+1Xn6x9dLz11ualI1I= -github.com/ryancurrah/gomodguard/v2 v2.1.3 h1:E7sz3PJwE9Ba1reVxSpF6XLCPJZ74Kfw/LabTNM4GIA= -github.com/ryancurrah/gomodguard/v2 v2.1.3/go.mod h1:CQicdLGatWMxLX53JzoBjYlsNZhHbmLv2AVa0s2aivU= -github.com/ryanrolds/sqlclosecheck v0.6.0 h1:pEyL9okISdg1F1SEpJNlrEotkTGerv5BMk7U4AG0eVg= -github.com/ryanrolds/sqlclosecheck v0.6.0/go.mod h1:xyX16hsDaCMXHrMJ3JMzGf5OpDfHTOTTQrT7HOFUmeU= -github.com/sanposhiho/wastedassign/v2 v2.1.0 h1:crurBF7fJKIORrV85u9UUpePDYGWnwvv3+A96WvwXT0= -github.com/sanposhiho/wastedassign/v2 v2.1.0/go.mod h1:+oSmSC+9bQ+VUAxA66nBb0Z7N8CK7mscKTDYC6aIek4= -github.com/santhosh-tekuri/jsonschema/v6 v6.0.3 h1:1EYB5IzjZawrrnELUi78f9fPu57HuXjmddZPjrls/28= -github.com/santhosh-tekuri/jsonschema/v6 v6.0.3/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU= -github.com/sashamelentyev/interfacebloat v1.1.0 h1:xdRdJp0irL086OyW1H/RTZTr1h/tMEOsumirXcOJqAw= -github.com/sashamelentyev/interfacebloat v1.1.0/go.mod h1:+Y9yU5YdTkrNvoX0xHc84dxiN1iBi9+G8zZIhPVoNjQ= -github.com/sashamelentyev/usestdlibvars v1.29.0 h1:8J0MoRrw4/NAXtjQqTHrbW9NN+3iMf7Knkq057v4XOQ= -github.com/sashamelentyev/usestdlibvars v1.29.0/go.mod h1:8PpnjHMk5VdeWlVb4wCdrB8PNbLqZ3wBZTZWkrpZZL8= -github.com/securego/gosec/v2 v2.28.0 h1:ZsSdiDb0AtTpLFVol5z91gbMei9ZiLEPG/pZjZujp7c= -github.com/securego/gosec/v2 v2.28.0/go.mod h1:lb4/9AHe+lJy/kjWmWRWWsEipvbwGKuxf+tY1Pmjdnk= -github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ= -github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= -github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= -github.com/sirupsen/logrus v1.10.1 h1:xi4336Zh11WpU14fXR6I67V3yaTPQYwRx2WEtHbRg4Q= -github.com/sirupsen/logrus v1.10.1/go.mod h1:vsQHnG7xzNsxk3NrwboUiWPnIC3dmbjcGPykD7+tiHk= -github.com/sivchari/containedctx v1.0.3 h1:x+etemjbsh2fB5ewm5FeLNi5bUjK0V8n0RB+Wwfd0XE= -github.com/sivchari/containedctx v1.0.3/go.mod h1:c1RDvCbnJLtH4lLcYD/GqwiBSSf4F5Qk0xld2rBqzJ4= -github.com/sonatard/noctx v0.5.1 h1:wklWg9c9ZYugOAk7qG4yP4PBrlQsmSLPTvW1K4PRQMs= -github.com/sonatard/noctx v0.5.1/go.mod h1:64XdbzFb18XL4LporKXp8poqZtPKbCrqQ402CV+kJas= -github.com/sourcegraph/go-diff v0.8.0 h1:ipIyu4cTsLbIrln4l0qtHA3r0a7gyK4ntKjtQytHhvY= -github.com/sourcegraph/go-diff v0.8.0/go.mod h1:hWlcO7Al+UZStZAP8rBumHpCK5ZHQ5BXsMls8p4+F5E= github.com/spdx/tools-golang v0.5.7 h1:+sWcKGnhwp3vLdMqPcLdA6QK679vd86cK9hQWH3AwCg= github.com/spdx/tools-golang v0.5.7/go.mod h1:jg7w0LOpoNAw6OxKEzCoqPC2GCTj45LyTlVmXubDsYw= -github.com/spf13/afero v1.15.0 h1:b/YBCLWAJdFWJTN9cLhiXXcD7mzKn9Dm86dNnfyQw1I= -github.com/spf13/afero v1.15.0/go.mod h1:NC2ByUVxtQs4b3sIUphxK0NioZnmxgyCrfzeuq8lxMg= -github.com/spf13/cast v1.5.0 h1:rj3WzYc11XZaIZMPKmwP96zkFEnnAmV8s6XbB2aY32w= -github.com/spf13/cast v1.5.0/go.mod h1:SpXXQ5YoyJw6s3/6cMTQuxvgRl3PCJiyaX9p6b155UU= -github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU= -github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4= -github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= -github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= -github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= -github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/viper v1.12.0 h1:CZ7eSOd3kZoaYDLbXnmzgQI5RlciuXBMA+18HwHRfZQ= -github.com/spf13/viper v1.12.0/go.mod h1:b6COn30jlNxbm/V2IqWiNWkJ+vZNiMNksliPCiuKtSI= github.com/spiffe/go-spiffe/v2 v2.7.0 h1:uXe1MflJoHw58wAUvxVlcM7WpKtijWG7I1UidcGh6g4= github.com/spiffe/go-spiffe/v2 v2.7.0/go.mod h1:47Q0Q9/AqGha8QLHp+kxpH4Wca7X7EnOtlIJy3mxZ3U= github.com/spkg/bom v0.0.0-20160624110644-59b7046e48ad/go.mod h1:qLr4V1qq6nMqFKkMo8ZTx3f+BZEkzsRUY10Xsm2mwU0= -github.com/ssgreg/nlreturn/v2 v2.2.1 h1:X4XDI7jstt3ySqGU86YGAURbxw3oTDPK9sPEi6YEwQ0= -github.com/ssgreg/nlreturn/v2 v2.2.1/go.mod h1:E/iiPB78hV7Szg2YfRgyIrk1AD6JVMTRkkxBiELzh2I= -github.com/stbenjam/no-sprintf-host-port v0.3.1 h1:AyX7+dxI4IdLBPtDbsGAyqiTSLpCP9hWRrXQDU4Cm/g= -github.com/stbenjam/no-sprintf-host-port v0.3.1/go.mod h1:ODbZesTCHMVKthBHskvUUexdcNHAQRXk9NpSsL8p/HQ= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.5.3 h1:jmXUvGomnU1o3W/V5h2VEradbpJDwGrzugQQvL0POH4= -github.com/stretchr/objx v0.5.3/go.mod h1:rDQraq+vQZU7Fde9LOZLr8Tax6zZvy4kuNKF+QYS+U0= -github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE= github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg= -github.com/subosito/gotenv v1.4.1 h1:jyEFiXpy21Wm81FBN71l9VoMMV8H8jG+qIK3GCpY6Qs= -github.com/subosito/gotenv v1.4.1/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0= github.com/swaggo/swag v1.16.6 h1:qBNcx53ZaX+M5dxVyTrgQ0PJ/ACK+NzhwcbieTt+9yI= github.com/swaggo/swag v1.16.6/go.mod h1:ngP2etMK5a0P3QBizic5MEwpRmluJZPHjXcMoj4Xesg= -github.com/tenntenn/modver v1.0.1 h1:2klLppGhDgzJrScMpkj9Ujy3rXPUspSjAcev9tSEBgA= -github.com/tenntenn/modver v1.0.1/go.mod h1:bePIyQPb7UeioSRkw3Q0XeMhYZSMx9B8ePqg6SAMGH0= -github.com/tenntenn/text/transform v0.0.0-20200319021203-7eef512accb3 h1:f+jULpRQGxTSkNYKJ51yaw6ChIqO+Je8UqsTKN/cDag= -github.com/tenntenn/text/transform v0.0.0-20200319021203-7eef512accb3/go.mod h1:ON8b8w4BN/kE1EOhwT0o+d62W65a6aPw1nouo9LMgyY= github.com/terminalstatic/go-xsd-validate v0.1.8 h1:UVrTCy1j3DhwaYTTUF+QYO/Nan13S0tf+Jwi+p45Bf0= github.com/terminalstatic/go-xsd-validate v0.1.8/go.mod h1:1kb47fi2c6onlf+B7UrrQ9VYraOhcYwFm3iG+J6F4Zo= -github.com/tetafro/godot v1.5.6 h1:IEkrFCwXaYHlOn4mGzGS3F3dkP6m9t0jpwqBFPIkKiA= -github.com/tetafro/godot v1.5.6/go.mod h1:eOkMrVQurDui411nBY2FA05EYH01r14LuWY/NrVDVcU= -github.com/timakin/bodyclose v0.0.0-20260129054331-73d1f95b84b4 h1:SiHe5XLTn9sFWJ5pBwJ5FN/4j34q9ZlOAD//kMoMYp0= -github.com/timakin/bodyclose v0.0.0-20260129054331-73d1f95b84b4/go.mod h1:sDHLK7rb/59v/ZxZ7KtymgcoxuUMxjXq8gtu9VMOK8M= -github.com/timonwong/loggercheck v0.11.0 h1:jdaMpYBl+Uq9mWPXv1r8jc5fC3gyXx4/WGwTnnNKn4M= -github.com/timonwong/loggercheck v0.11.0/go.mod h1:HEAWU8djynujaAVX7QI65Myb8qgfcZ1uKbdpg3ZzKl8= -github.com/tomarrell/wrapcheck/v2 v2.12.0 h1:H/qQ1aNWz/eeIhxKAFvkfIA+N7YDvq6TWVFL27Of9is= -github.com/tomarrell/wrapcheck/v2 v2.12.0/go.mod h1:AQhQuZd0p7b6rfW+vUwHm5OMCGgp63moQ9Qr/0BpIWo= -github.com/tommy-muehle/go-mnd/v2 v2.5.1 h1:NowYhSdyE/1zwK9QCLeRb6USWdoif80Ie+v+yU8u1Zw= -github.com/tommy-muehle/go-mnd/v2 v2.5.1/go.mod h1:WsUAkMJMYww6l/ufffCD3m+P7LEvr8TnZn9lwVDlgzw= github.com/ulikunitz/xz v0.5.16 h1:ld6NyySjx5lowVKwJvMRLnW5nxKX/xnpSiFYZ/Lxur0= github.com/ulikunitz/xz v0.5.16/go.mod h1:H9Rt/W6/Qj27PGauhQc6nfCDy7vHpzsOThBSaYDoEhw= -github.com/ultraware/funlen v0.2.0 h1:gCHmCn+d2/1SemTdYMiKLAHFYxTYz7z9VIDRaTGyLkI= -github.com/ultraware/funlen v0.2.0/go.mod h1:ZE0q4TsJ8T1SQcjmkhN/w+MceuatI6pBFSxxyteHIJA= -github.com/ultraware/whitespace v0.2.0 h1:TYowo2m9Nfj1baEQBjuHzvMRbp19i+RCcRYrSWoFa+g= -github.com/ultraware/whitespace v0.2.0/go.mod h1:XcP1RLD81eV4BW8UhQlpaR+SDc2givTvyI8a586WjW8= -github.com/urfave/cli/v2 v2.3.0 h1:qph92Y649prgesehzOrQjdWyxFOp/QVM+6imKHad91M= -github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI= -github.com/uudashr/gocognit v1.2.1 h1:CSJynt5txTnORn/DkhiB4mZjwPuifyASC8/6Q0I/QS4= -github.com/uudashr/gocognit v1.2.1/go.mod h1:acaubQc6xYlXFEMb9nWX2dYBzJ/bIjEkc1zzvyIZg5Q= -github.com/uudashr/iface v1.5.0 h1:PgdMt4uAettGG8K/Kbamc4B9FABgUgnS3TLbl6fnjEk= -github.com/uudashr/iface v1.5.0/go.mod h1:pbeBPlbuU2qkNDn0mmfrxP2X+wjPMIQAy+r1MBXSXtg= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f h1:J9EGpcZtP0E/raorCMxlFGSTBrsSlaDGf3jU/qvAE2c= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0= github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74= github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= -github.com/xen0n/gosmopolitan v1.3.0 h1:zAZI1zefvo7gcpbCOrPSHJZJYA9ZgLfJqtKzZ5pHqQM= -github.com/xen0n/gosmopolitan v1.3.0/go.mod h1:rckfr5T6o4lBtM1ga7mLGKZmLxswUoH1zxHgNXOsEt4= -github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no= -github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM= -github.com/yagipy/maintidx v1.0.0 h1:h5NvIsCz+nRDapQ0exNv4aJ0yXSI0420omVANTv3GJM= -github.com/yagipy/maintidx v1.0.0/go.mod h1:0qNf/I/CCZXSMhsRsrEPDZ+DkekpKLXAJfsTACwgXLk= -github.com/yeya24/promlinter v0.3.0 h1:JVDbMp08lVCP7Y6NP3qHroGAO6z2yGKQtS5JsjqtoFs= -github.com/yeya24/promlinter v0.3.0/go.mod h1:cDfJQQYv9uYciW60QT0eeHlFodotkYZlL+YcPQN+mW4= -github.com/ykadowak/zerologlint v0.1.5 h1:Gy/fMz1dFQN9JZTPjv1hxEk+sRWm05row04Yoolgdiw= -github.com/ykadowak/zerologlint v0.1.5/go.mod h1:KaUskqF3e/v59oPmdq1U1DnKcuHokl2/K1U4pmIELKg= -github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= -github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -gitlab.com/bosi/decorder v0.4.2 h1:qbQaV3zgwnBZ4zPMhGLW4KZe7A7NwxEhJx39R3shffo= -gitlab.com/bosi/decorder v0.4.2/go.mod h1:muuhHoaJkA9QLcYHq4Mj8FJUwDZ+EirSHRiaTcTf6T8= -go-simpler.org/assert v0.9.0 h1:PfpmcSvL7yAnWyChSjOz6Sp6m9j5lyK8Ok9pEL31YkQ= -go-simpler.org/assert v0.9.0/go.mod h1:74Eqh5eI6vCK6Y5l3PI8ZYFXG4Sa+tkr70OIPJAUr28= -go-simpler.org/musttag v0.14.0 h1:XGySZATqQYSEV3/YTy+iX+aofbZZllJaqwFWs+RTtSo= -go-simpler.org/musttag v0.14.0/go.mod h1:uP8EymctQjJ4Z1kUnjX0u2l60WfUdQxCwSNKzE1JEOE= -go-simpler.org/sloglint v0.12.0 h1:UzWDlLWNE5FLqsvyq3tWYHuQMbqrervOhT8qPl4Mmw4= -go-simpler.org/sloglint v0.12.0/go.mod h1:jBjjC2bm8rYrs88oTRlFX497kWjJsyZWYoNaXkGRI6I= -go.augendre.info/arangolint v0.4.0 h1:xSCZjRoS93nXazBSg5d0OGCi9APPLNMmmLrC995tR50= -go.augendre.info/arangolint v0.4.0/go.mod h1:l+f/b4plABuFISuKnTGD4RioXiCCgghv2xqst/xOvAA= -go.augendre.info/fatcontext v0.10.0 h1:HhFopmivh8U1+AU7f0kuwUeg2eiIns7YsGQOMHwSJ90= -go.augendre.info/fatcontext v0.10.0/go.mod h1:pqpGvA9GlrXy+aXkp8L2dKz12Zp4g2FhzcAtwToU+2w= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= go.opentelemetry.io/contrib/detectors/gcp v1.44.0 h1:NmLfL734pJhM0JKaYd2Y28+nY9dPRWYAAbxhRCrKXPw= @@ -744,109 +315,39 @@ go.opentelemetry.io/otel/trace v1.45.0 h1:l/mP6Uv7oNO7/TblbhpbgMidxhq1uO/rPsikOy go.opentelemetry.io/otel/trace v1.45.0/go.mod h1:qoJJA2xNMnxRrdISU/kLtfUH2wNeQbiv+jhs/CxI8bc= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= -go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= -go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= -go.uber.org/zap v1.27.1 h1:08RqriUEv8+ArZRYSTXy1LeBScaMpVSTBhCeaZYfMYc= -go.uber.org/zap v1.27.1/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ= go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ= -go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= gocloud.dev v0.46.0 h1:niIuZwSjMtBx8K+ITB2s5kZullB13PGOS2ZoQPZxQ4Q= gocloud.dev v0.46.0/go.mod h1:ACQe+2qO+hEO+pdcvvsM+RB63r8TyGD1W3ESCLFyzvM= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M= golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA= -golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa h1:Zt3DZoOFFYkKhDT3v7Lm9FDMEV06GpzjG2jrqW+QTE0= -golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa/go.mod h1:K79w1Vqn7PoiZn+TkNpx3BUWUQksGO3JcVX6qIjytmA= -golang.org/x/exp/typeparams v0.0.0-20220428152302-39d4317da171/go.mod h1:AbB0pIl9nAr9wVwH+Z2ZpaocVmF5I4GyWCDIsVjR0bk= -golang.org/x/exp/typeparams v0.0.0-20230203172020-98cc5a0785f9/go.mod h1:AbB0pIl9nAr9wVwH+Z2ZpaocVmF5I4GyWCDIsVjR0bk= -golang.org/x/exp/typeparams v0.0.0-20260811152304-ee035b5b010f h1:+lI8cDJ4uceLipg2f1ODay7fEuLkk0BIHXd6PB8icxo= -golang.org/x/exp/typeparams v0.0.0-20260811152304-ee035b5b010f/go.mod h1:PqrXSW65cXDZH0k4IeUbhmg/bcAZDbzNz3byBpKCsXo= -golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= -golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.41.0 h1:qJmnOUb4YB+FsEuM3HcWucdZASCPGhsX6uljO6pog0c= golang.org/x/mod v0.41.0/go.mod h1:Ek9pY8RKWXwsWvd3rQiHYtMqkjSUV+s1Rj7j4H5Ur6o= -golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1Kcs5dz7/ng1VjMUvfKvpfy+jM= -golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.59.0 h1:5zfYln+w5XCxwrnMMJPufRgNoXEaGxl0wo5GqPXyues= golang.org/x/net v0.59.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg= golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.23.0 h1:KameEIfc1IkluZyXWLn39Wd4tURc6GbCiISGiZm2bQk= golang.org/x/sync v0.23.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210420072515-93ed5bcd2bfe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211105183446-c75c47738b0c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= -golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI= golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E= golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20200329025819-fd4102a86c65/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= -golang.org/x/tools v0.0.0-20200724022722-7017fd6b1305/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.1.1-0.20210205202024-ef80cdb6ec6d/go.mod h1:9bzcO0MWcOuT0tm1iBGzDVPshzfwoVvREIui8C+MHqU= -golang.org/x/tools v0.1.1-0.20210302220138-2ac05c832e1a/go.mod h1:9bzcO0MWcOuT0tm1iBGzDVPshzfwoVvREIui8C+MHqU= -golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= -golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.49.0 h1:3NI7VXzL9+1WZD52Dx2ttoPwD5DWrFGpl9mFZDlmisI= golang.org/x/tools v0.49.0/go.mod h1:SJNXV9DBKT0UbdttsQjbfJlAE/q+y36++zo3uL3N0Oo= -golang.org/x/tools/go/expect v0.1.1-deprecated h1:jpBZDwmgPhXsKZC6WhL20P4b/wmnpsEAGHaNy0n/rJM= -golang.org/x/tools/go/expect v0.1.1-deprecated/go.mod h1:eihoPOH+FgIqa3FpoTwguz/bVUSGBlGQU67vpBeOrBY= -golang.org/x/tools/go/packages/packagestest v0.1.1-deprecated h1:1h2MnaIAIXISqTFKdENegdpAgUXz6NrPEsbIeWaBRvM= -golang.org/x/tools/go/packages/packagestest v0.1.1-deprecated/go.mod h1:RVAQXBGNv1ib0J382/DPCRS/BPnsGebyM1Gj5VSDpG8= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da h1:noIWHXmPHxILtqtCOPIhSt0ABwskkZKjD3bXGnZGpNY= golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= @@ -868,18 +369,13 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= -gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -honnef.co/go/tools v0.8.0 h1:UacpzPr7D6i5BAjTkA7sNVcx4kIbhAZcQ4zYtKiXx68= -honnef.co/go/tools v0.8.0/go.mod h1:XA+OnlRA9EDh/ukGvXMNSZNKGwFQJ+5dER0ioUkOxks= modernc.org/cc/v4 v4.29.2 h1:h6+9ciCnPKutf4I03CvheAvDLX7+IHlqR6Iy6J+cgd8= modernc.org/cc/v4 v4.29.2/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI= modernc.org/ccgo/v4 v4.35.0 h1:F+TUsmw09QxLzmi3aeYYGxjAXarmZaKgj3mKQHNaA8w= @@ -908,9 +404,3 @@ modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0= modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A= modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= -mvdan.cc/gofumpt v0.11.0 h1:0H01XB95PnN2QgCSR9ELdZyTlJqNZ7181B0BTMh5VZc= -mvdan.cc/gofumpt v0.11.0/go.mod h1:BeT5wCsOJt6J9zT2MZIOGszjUHzFkn1/l9g6xAzqsXo= -mvdan.cc/unparam v0.0.0-20260818115549-3f964bcb5673 h1:dEE6li4OPIE54oojY2qaayFS1fSp17G14si0gXRxl0U= -mvdan.cc/unparam v0.0.0-20260818115549-3f964bcb5673/go.mod h1:62roFV3D3nYOWIXv3PfGO4UYEKAotz2WgLywT87ONd8= -sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs= -sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4= diff --git a/tools.go b/tools.go deleted file mode 100644 index caf60b1..0000000 --- a/tools.go +++ /dev/null @@ -1,7 +0,0 @@ -//go:build tools - -package tools - -import ( - _ "github.com/swaggo/swag/cmd/swag" -) From b40a50aad838250d2feb76a0c264abc8414d5966 Mon Sep 17 00:00:00 2001 From: Daniil Skripchenko <90247855+DANIILSKRIPCHENKO@users.noreply.github.com> Date: Fri, 2 Oct 2026 15:28:20 +0700 Subject: [PATCH 168/633] Add storage.cache_artifacts to serve artifacts without storing them (#387) * Add storage.passthrough to serve artifacts without storing them With storage.passthrough enabled the proxy streams every artifact from upstream to the client and never writes it to storage or the cache database. Metadata filtering, cooldown and the denylist work as before, so the proxy can sit behind another cache (e.g. an Artifactory remote) purely as a policy layer without holding a second copy of every package. Artifacts whose digest is known up front (OCI, Swift, Helm) are verified while streaming. Their responses are sent chunked and the connection is aborted on a mismatch, so a client never receives a tampered artifact as a complete response. Passthrough is rejected together with scanning, direct_serve and mirror_api, and the mirror command refuses to run with it, since all of them depend on stored artifacts. Co-Authored-By: Claude Opus 5.5 * Address review: abort truncated streams, guard every cache read - serveArtifact aborts the response on any body read error, and when fewer bytes than the declared size were written, not only on a digest mismatch. A failed or short upstream body in passthrough mode no longer reaches the client as a complete 200. - Streamed upstream read failures are logged and counted as stream_failed upstream errors before the response is aborted. - checkCache reports a miss when artifact caching is off, so every cache read path, including Swift archive HEAD requests, ignores entries stored before the mode was enabled. Denylisted versions are still rejected. - Rename storage.passthrough to storage.cache_artifacts (default true, PROXY_STORAGE_CACHE_ARTIFACTS), which names what actually changes. Tests cover an OCI blob shorter than its Content-Length and an npm tarball whose chunked response ends without the final chunk, both served through the HTTP handlers, plus a Swift HEAD request against a cached archive. Co-Authored-By: Claude Opus 5.5 * Rename Proxy.Passthrough to StreamArtifacts Match the storage.cache_artifacts option: the field says what changes (artifacts are streamed instead of stored) and its zero value keeps the usual caching behaviour for every Proxy built without the server config, such as the mirror command and tests. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- cmd/proxy/main.go | 4 + config.example.yaml | 6 + docs/configuration.md | 14 + internal/config/config.go | 33 +- internal/config/config_test.go | 58 +++ internal/handler/handler.go | 118 ++++++- internal/handler/helm.go | 10 +- internal/handler/integrity.go | 28 +- internal/handler/stream_artifacts_test.go | 411 ++++++++++++++++++++++ internal/server/server.go | 1 + 10 files changed, 674 insertions(+), 9 deletions(-) create mode 100644 internal/handler/stream_artifacts_test.go diff --git a/cmd/proxy/main.go b/cmd/proxy/main.go index 9266de5..5b62563 100644 --- a/cmd/proxy/main.go +++ b/cmd/proxy/main.go @@ -479,6 +479,10 @@ func runMirror() { fmt.Fprintf(os.Stderr, "invalid configuration: %v\n", err) os.Exit(1) } + if !cfg.Storage.CacheArtifacts { + fmt.Fprintf(os.Stderr, "error: mirror is not available with storage.cache_artifacts: false: mirrored artifacts would never be served\n") + os.Exit(1) + } logger := setupLogger("info", "text") diff --git a/config.example.yaml b/config.example.yaml index 58a9de7..96a5ba2 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -52,6 +52,12 @@ storage: # Empty or "0" means unlimited max_size: "" + # Store fetched artifacts. Set to false to stream every download from + # upstream without storing it; metadata filtering, cooldown and the + # denylist still apply. Useful when another cache sits in front of the + # proxy. false is incompatible with direct_serve, scanning and mirror_api. + cache_artifacts: true + # Redirect cached artifact downloads to presigned storage URLs (HTTP 302) # instead of streaming through the proxy. Only effective for S3, GCS, and Azure. # Leave disabled if clients reach the proxy through an authenticating gateway, diff --git a/docs/configuration.md b/docs/configuration.md index f09e010..1510e70 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -43,9 +43,23 @@ storage: | `storage.url` | `PROXY_STORAGE_URL` | `-storage-url` | Storage URL (file:// or s3://) | | `storage.path` | `PROXY_STORAGE_PATH` | `-storage-path` | Local path (deprecated, use url) | | `storage.max_size` | `PROXY_STORAGE_MAX_SIZE` | - | Max cache size (e.g., "10GB") | +| `storage.cache_artifacts` | `PROXY_STORAGE_CACHE_ARTIFACTS` | - | Store fetched artifacts (default: true); `false` streams them from upstream | `storage.max_size` counts cached artifacts only. An artifact replaced by a refetch stays in storage for at least an hour, or `storage.direct_serve_ttl` if longer, so requests already reading it can finish, and storage use can exceed the limit by what was replaced in that time. +### Serving artifacts without storing them + +With `storage.cache_artifacts: false` the proxy streams every artifact download from upstream to the client and stores no artifacts. Metadata is still filtered and cached, so cooldown and the denylist apply as usual, and the database still holds the publish times cooldown needs. Use it when another caching layer, such as an Artifactory remote repository, sits in front of the proxy and caching artifacts twice only costs storage. + +```yaml +storage: + cache_artifacts: false +``` + +Every download is a fresh upstream fetch, and concurrent requests for the same artifact are not combined. Artifacts with a digest known up front (OCI blobs, Swift archives, Helm charts) are verified while streaming: the response is sent chunked, and on a mismatch the connection is aborted before the response completes so the client never receives a tampered artifact as a good one. The same happens when the upstream connection fails mid-download. + +`cache_artifacts: false` cannot be combined with `scanning.enabled`, `storage.direct_serve` or `mirror_api`, which all need stored artifacts, and the `mirror` command refuses to run with it. + ### Amazon S3 ```yaml diff --git a/internal/config/config.go b/internal/config/config.go index fcc41fa..6f8eda9 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -370,6 +370,14 @@ type StorageConfig struct { // storage at an internal address (e.g. 127.0.0.1 or a Docker hostname) // but clients must use a public one. DirectServeBaseURL string `json:"direct_serve_base_url" yaml:"direct_serve_base_url"` + + // CacheArtifacts stores fetched artifacts so later downloads are served + // from storage. When false, every download streams from upstream and + // nothing is stored; metadata is still cached, and cooldown and the + // denylist still apply. Useful when another caching layer sits in front + // of the proxy. False is incompatible with scanning, direct_serve and + // mirror_api, which all depend on stored artifacts. Default: true. + CacheArtifacts bool `json:"cache_artifacts" yaml:"cache_artifacts"` } // GradleConfig configures Gradle-specific features. @@ -757,8 +765,9 @@ func Default() *Config { Listen: ":8080", BaseURL: "http://localhost:8080", Storage: StorageConfig{ - Path: "./cache/artifacts", - MaxSize: "", + Path: "./cache/artifacts", + MaxSize: "", + CacheArtifacts: true, }, Database: DatabaseConfig{ Driver: "sqlite", @@ -893,6 +902,7 @@ func (c *Config) LoadFromEnv() { setEnvBool(&c.Storage.DirectServe, "PROXY_STORAGE_DIRECT_SERVE") setEnvString(&c.Storage.DirectServeTTL, "PROXY_STORAGE_DIRECT_SERVE_TTL") setEnvString(&c.Storage.DirectServeBaseURL, "PROXY_STORAGE_DIRECT_SERVE_BASE_URL") + setEnvBool(&c.Storage.CacheArtifacts, "PROXY_STORAGE_CACHE_ARTIFACTS") setEnvString(&c.Database.Driver, "PROXY_DATABASE_DRIVER") setEnvString(&c.Database.Path, "PROXY_DATABASE_PATH") setEnvString(&c.Database.URL, "PROXY_DATABASE_URL") @@ -1023,6 +1033,10 @@ func (c *Config) Validate() error { } } + if err := c.validateCacheArtifacts(); err != nil { + return err + } + // Validate metadata TTL if specified if c.MetadataTTL != "" && c.MetadataTTL != "0" { if _, err := time.ParseDuration(c.MetadataTTL); err != nil { @@ -1041,6 +1055,21 @@ func (c *Config) Validate() error { return c.validateComponents() } +func (c *Config) validateCacheArtifacts() error { + if c.Storage.CacheArtifacts { + return nil + } + switch { + case c.Scanning.Enabled: + return fmt.Errorf("storage.cache_artifacts: false cannot be combined with scanning.enabled: scanning needs stored artifacts") + case c.Storage.DirectServe: + return fmt.Errorf("storage.cache_artifacts: false cannot be combined with storage.direct_serve: no artifacts are stored to redirect to") + case c.MirrorAPI: + return fmt.Errorf("storage.cache_artifacts: false cannot be combined with mirror_api: mirrored artifacts would never be served") + } + return nil +} + func (c *Config) validateComponents() error { if _, err := denylist.New(c.Denylist.Packages); err != nil { return err diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 050b3ce..20bd103 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -1303,3 +1303,61 @@ func TestValidateNamedUpstreams(t *testing.T) { }) } } + +func TestValidateCacheArtifactsDisabled(t *testing.T) { + tests := []struct { + name string + modify func(*Config) + wantErr string + }{ + {"alone", func(*Config) {}, ""}, + {"with scanning", func(c *Config) { c.Scanning.Enabled = true }, "scanning.enabled"}, + {"with direct_serve", func(c *Config) { c.Storage.DirectServe = true }, "storage.direct_serve"}, + {"with mirror_api", func(c *Config) { c.MirrorAPI = true }, "mirror_api"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cfg := Default() + cfg.Storage.CacheArtifacts = false + tt.modify(cfg) + err := cfg.Validate() + if tt.wantErr == "" { + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + return + } + if err == nil || !strings.Contains(err.Error(), tt.wantErr) { + t.Fatalf("Validate() = %v, want an error mentioning %q", err, tt.wantErr) + } + }) + } +} + +func TestCacheArtifactsDefaultsToTrue(t *testing.T) { + if !Default().Storage.CacheArtifacts { + t.Fatal("Default().Storage.CacheArtifacts = false, want true") + } + + path := filepath.Join(t.TempDir(), "config.yaml") + if err := os.WriteFile(path, []byte("storage:\n url: \"file:///tmp/cache\"\n"), 0o600); err != nil { + t.Fatal(err) + } + cfg, err := Load(path) + if err != nil { + t.Fatalf("Load: %v", err) + } + if !cfg.Storage.CacheArtifacts { + t.Error("a config file without storage.cache_artifacts disabled artifact caching") + } +} + +func TestLoadCacheArtifactsFromEnv(t *testing.T) { + cfg := Default() + t.Setenv("PROXY_STORAGE_CACHE_ARTIFACTS", "false") + cfg.LoadFromEnv() + + if cfg.Storage.CacheArtifacts { + t.Error("Storage.CacheArtifacts should be false") + } +} diff --git a/internal/handler/handler.go b/internal/handler/handler.go index b9ecc62..37e353c 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -175,6 +175,11 @@ type Proxy struct { HTTPClient *http.Client AuthForURL func(string) (headerName, headerValue string) + // StreamArtifacts streams artifacts from upstream without storing them. + // Each request fetches its own copy: there is no cache to check and + // nothing for concurrent misses to share. + StreamArtifacts bool + // Scanners runs pre-cache artifact scanning (e.g. trivy, ClamAV, Wiz). // Nil or disabled means artifacts are cached without scanning. Scanners *scanner.Group @@ -242,6 +247,17 @@ func (p *Proxy) GetOrFetchArtifact(ctx context.Context, ecosystem, name, version return nil, errors.New("resolved artifact has no filename") } } + if p.StreamArtifacts { + if info == nil { + if info, err = p.resolveArtifact(ctx, ecosystem, name, version); err != nil { + return nil, err + } + } + return p.streamFromUpstream(ctx, ecosystem, name, version, filename, versionPURL, info.URL, "", + func(fetchCtx context.Context) (*fetch.Artifact, error) { + return p.Fetcher.Fetch(fetchCtx, info.URL) + }) + } if cached, err := p.checkCache(ctx, pkgPURL, versionPURL, filename); err != nil { return nil, err } else if cached != nil { @@ -289,10 +305,15 @@ func (p *Proxy) ClearCachedArtifact(ecosystem, name, version, filename string) e } // checkCache looks up an artifact in the cache. Returns nil if not cached. +// With StreamArtifacts set it always reports a miss, so entries stored before the +// mode was enabled are never served. func (p *Proxy) checkCache(ctx context.Context, pkgPURL, versionPURL, filename string) (*CacheResult, error) { if p.Denylist.Denied(versionPURL) { return nil, fmt.Errorf("%w: %s", ErrVersionDenied, versionPURL) } + if p.StreamArtifacts { + return nil, nil + } artifact, err := p.DB.GetCachedArtifact(pkgPURL, versionPURL, filename) if err != nil { return nil, fmt.Errorf("checking artifact cache: %w", err) @@ -764,7 +785,14 @@ func serveArtifact(w http.ResponseWriter, method string, result *CacheResult) { buffer := artifactCopyBufferPool.Get().(*[]byte) defer artifactCopyBufferPool.Put(buffer) // Hide optional ReaderFrom methods so io.CopyBuffer uses the pooled buffer. - _, _ = io.CopyBuffer(struct{ io.Writer }{w}, result.Reader, *buffer) + written, err := io.CopyBuffer(struct{ io.Writer }{w}, result.Reader, *buffer) + if err != nil || (result.Artifact.Size > 0 && written != result.Artifact.Size) { + // Headers are already committed, so an error status is no longer + // possible. Aborting leaves the response unterminated and the + // client discards it instead of keeping a truncated or unverified + // artifact. + panic(http.ErrAbortHandler) + } } } @@ -1320,6 +1348,12 @@ func (p *Proxy) getOrFetchArtifactFromURLWithCachePURLs(ctx context.Context, eco if p.versionDenied(ecosystem, name, version) { return nil, fmt.Errorf("%w: %s", ErrVersionDenied, canonicalVersionPURL(ecosystem, name, version)) } + if p.StreamArtifacts { + return p.streamFromUpstream(ctx, ecosystem, name, version, filename, versionPURL, downloadURL, upstreamHash, + func(fetchCtx context.Context) (*fetch.Artifact, error) { + return p.Fetcher.FetchWithHeaders(fetchCtx, downloadURL, headers) + }) + } if cached, err := p.getCachedArtifactWithUpstreamHash(ctx, pkgPURL, versionPURL, filename, upstreamHash); err != nil { return nil, err } else if cached != nil { @@ -1402,6 +1436,88 @@ func (p *Proxy) fetchAndCacheFromURL(ctx context.Context, ecosystem, name, versi return p.storeArtifact(ctx, ecosystem, name, version, filename, pkgPURL, versionPURL, downloadURL, upstreamHash, artifact) } +// streamFromUpstream fetches an artifact when StreamArtifacts is set and hands its +// body to the caller without storing it. +// +// With upstreamHash set the body is verified as it streams. The size is left +// unknown so the response goes out chunked: a mismatch only shows at EOF, +// after the bytes have been sent, and an unterminated chunked response is the +// only way left to make the client reject them (see serveArtifact). +func (p *Proxy) streamFromUpstream(ctx context.Context, ecosystem, name, version, filename, versionPURL, upstreamURL, upstreamHash string, fetchArtifact func(context.Context) (*fetch.Artifact, error)) (*CacheResult, error) { + p.Logger.Info("streaming from upstream", + "ecosystem", ecosystem, "name", name, "version", version, "url", upstreamURL) + + fetchStart := time.Now() + artifact, err := fetchArtifact(ctx) + metrics.RecordUpstreamFetch(ecosystem, time.Since(fetchStart)) + if err != nil { + metrics.RecordUpstreamError(ecosystem, "fetch_failed") + if errors.Is(err, fetch.ErrNotFound) { + return nil, ErrUpstreamNotFound + } + return nil, fmt.Errorf("fetching from upstream: %w", err) + } + + body := &streamErrorLogger{ + ReadCloser: artifact.Body, + onError: func(read int64, err error) { + p.Logger.Warn("streaming artifact from upstream failed", + "purl", versionPURL, "filename", filename, "url", upstreamURL, "bytes", read, "error", err) + metrics.RecordUpstreamError(ecosystem, "stream_failed") + }, + } + result := &CacheResult{ + Reader: body, + Artifact: artifacts.Artifact{ + PURL: versionPURL, + Size: artifact.Size, + Filename: filename, + MediaType: artifact.ContentType, + }, + } + if upstreamHash == "" { + return result, nil + } + + hash := strings.ToLower(upstreamHash) + checks, err := newIntegrityChecks(hash, "") + if err != nil { + _ = artifact.Body.Close() + return nil, fmt.Errorf("parsing upstream digest: %w", err) + } + result.Reader, err = checks.wrapFailOnMismatch(body, func(reason string) { + p.Logger.Error("streamed artifact failed integrity check", + "purl", versionPURL, "filename", filename, "url", upstreamURL, "reason", reason) + metrics.RecordIntegrityFailure(purl.NormalizeEcosystem(ecosystem)) + }) + if err != nil { + _ = artifact.Body.Close() + return nil, err + } + result.Artifact.Digest = digest.Digest("sha256:" + hash) + result.Artifact.Size = -1 + return result, nil +} + +// streamErrorLogger reports the first read error of a streamed upstream body. +// The error itself still reaches serveArtifact, which aborts the response. +type streamErrorLogger struct { + io.ReadCloser + onError func(read int64, err error) + read int64 + logged bool +} + +func (r *streamErrorLogger) Read(p []byte) (int, error) { + n, err := r.ReadCloser.Read(p) + r.read += int64(n) + if err != nil && err != io.EOF && !r.logged { + r.logged = true + r.onError(r.read, err) + } + return n, err +} + // ErrArtifactDigestMismatch indicates that fetched bytes did not match the // checksum the upstream declared and were not recorded in the cache database. var ErrArtifactDigestMismatch = errors.New("artifact digest mismatch") diff --git a/internal/handler/helm.go b/internal/handler/helm.go index 8cacdb6..c0f86fd 100644 --- a/internal/handler/helm.go +++ b/internal/handler/helm.go @@ -115,8 +115,14 @@ func (h *HelmHandler) handleChart(w http.ResponseWriter, r *http.Request) { return } - result, err := h.proxy.GetOrFetchArtifactFromURL( - r.Context(), helmMetadataEcosystem, repository, digest, filename, downloadURL) + // A streamed fetch is never stored, so serveChart's digest check has + // nothing to compare against: verify the stream itself instead. + expectedDigest := "" + if h.proxy.StreamArtifacts { + expectedDigest = "sha256:" + digest + } + result, err := h.proxy.GetOrFetchArtifactFromURLWithDigest( + r.Context(), helmMetadataEcosystem, repository, digest, filename, downloadURL, expectedDigest) if err != nil { h.proxy.serveArtifactError(w, err, "failed to fetch chart") return diff --git a/internal/handler/integrity.go b/internal/handler/integrity.go index 07963b9..97f41d9 100644 --- a/internal/handler/integrity.go +++ b/internal/handler/integrity.go @@ -40,6 +40,17 @@ func newIntegrityChecks(contentHash, native string) (integrityChecks, error) { } func (c integrityChecks) wrap(source io.ReadCloser, onMismatch func(string)) (io.ReadCloser, error) { + return c.newVerifyingReader(source, onMismatch, false) +} + +// wrapFailOnMismatch is wrap for bytes that have not been checked anywhere +// else: a mismatch is also returned from Read as ErrArtifactDigestMismatch in +// place of io.EOF, so the caller can abort rather than complete the response. +func (c integrityChecks) wrapFailOnMismatch(source io.ReadCloser, onMismatch func(string)) (io.ReadCloser, error) { + return c.newVerifyingReader(source, onMismatch, true) +} + +func (c integrityChecks) newVerifyingReader(source io.ReadCloser, onMismatch func(string), failOnMismatch bool) (io.ReadCloser, error) { if len(c.algorithms) == 0 { return source, nil } @@ -48,10 +59,11 @@ func (c integrityChecks) wrap(source io.ReadCloser, onMismatch func(string)) (io return nil, fmt.Errorf("create integrity reader: %w", err) } return &verifyingReader{ - source: source, - reader: reader, - checks: c, - onMismatch: onMismatch, + source: source, + reader: reader, + checks: c, + onMismatch: onMismatch, + failOnMismatch: failOnMismatch, }, nil } @@ -63,12 +75,18 @@ type verifyingReader struct { checks integrityChecks onMismatch func(reason string) verified bool + + failOnMismatch bool + mismatched bool } func (r *verifyingReader) Read(p []byte) (int, error) { n, err := r.reader.Read(p) if err == io.EOF { r.verify() + if r.failOnMismatch && r.mismatched { + return n, ErrArtifactDigestMismatch + } } return n, err } @@ -89,11 +107,13 @@ func (r *verifyingReader) verify() { if len(r.checks.contentHash) > 0 { if err := result.Verify(r.checks.contentHash); err != nil { + r.mismatched = true r.onMismatch("content_hash: " + err.Error()) } } if len(r.checks.native) > 0 { if err := result.Verify(r.checks.native); err != nil { + r.mismatched = true r.onMismatch("integrity: " + err.Error()) } } diff --git a/internal/handler/stream_artifacts_test.go b/internal/handler/stream_artifacts_test.go new file mode 100644 index 0000000..5edc5dd --- /dev/null +++ b/internal/handler/stream_artifacts_test.go @@ -0,0 +1,411 @@ +package handler + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/git-pkgs/artifacts" + "github.com/git-pkgs/cooldown" + "github.com/git-pkgs/proxy/internal/packageurl" + "github.com/git-pkgs/registries/fetch" +) + +func newStreamingProxy(t *testing.T, body string) (*Proxy, *mockStorage, *mockFetcher) { + t.Helper() + proxy, _, store, fetcher := setupTestProxy(t) + proxy.StreamArtifacts = true + fetcher.artifact = &fetch.Artifact{ + Body: io.NopCloser(strings.NewReader(body)), + Size: int64(len(body)), + ContentType: "application/gzip", + } + return proxy, store, fetcher +} + +func TestStreamArtifactsFromURLStreamsWithoutStoring(t *testing.T) { + proxy, store, fetcher := newStreamingProxy(t, "fetched content") + + result, err := proxy.GetOrFetchArtifactFromURL(context.Background(), "pypi", "newpkg", "1.0.0", + "newpkg-1.0.0.tar.gz", "https://pypi.org/files/newpkg-1.0.0.tar.gz") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + defer func() { _ = result.Reader.Close() }() + + body, err := io.ReadAll(result.Reader) + if err != nil { + t.Fatalf("reading body: %v", err) + } + if string(body) != "fetched content" { + t.Errorf("body = %q, want %q", body, "fetched content") + } + if fetcher.fetchedURL != "https://pypi.org/files/newpkg-1.0.0.tar.gz" { + t.Errorf("fetched URL = %q", fetcher.fetchedURL) + } + if result.Cached { + t.Error("streaming result reported as cached") + } + if result.Artifact.Size != int64(len("fetched content")) { + t.Errorf("Size = %d, want the upstream size", result.Artifact.Size) + } + if result.Artifact.MediaType != "application/gzip" { + t.Errorf("MediaType = %q", result.Artifact.MediaType) + } + if len(store.files) != 0 { + t.Errorf("streaming stored %d files, want none", len(store.files)) + } + cached, err := proxy.DB.GetCachedArtifact("pkg:pypi/newpkg", "pkg:pypi/newpkg@1.0.0", "newpkg-1.0.0.tar.gz") + if err != nil { + t.Fatalf("GetCachedArtifact: %v", err) + } + if cached != nil { + t.Error("streaming recorded the artifact in the cache database") + } +} + +func TestStreamArtifactsGetOrFetchArtifactStreamsWithoutStoring(t *testing.T) { + proxy, store, fetcher := newStreamingProxy(t, "tarball data") + + result, err := proxy.GetOrFetchArtifact(context.Background(), "npm", "leftpad", testVersion100, "leftpad-1.0.0.tgz") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + defer func() { _ = result.Reader.Close() }() + + body, _ := io.ReadAll(result.Reader) + if string(body) != "tarball data" { + t.Errorf("body = %q", body) + } + if !fetcher.fetchCalled { + t.Error("upstream was not fetched") + } + if len(store.files) != 0 { + t.Errorf("streaming stored %d files, want none", len(store.files)) + } +} + +func TestStreamArtifactsIgnoresCachedArtifacts(t *testing.T) { + proxy, _, store, fetcher := setupTestProxy(t) + fetcher.artifact = &fetch.Artifact{Body: io.NopCloser(strings.NewReader("old bytes"))} + url := "https://pypi.org/files/newpkg-1.0.0.tar.gz" + + // Populate the cache in normal mode first. + result, err := proxy.GetOrFetchArtifactFromURL(context.Background(), "pypi", "newpkg", "1.0.0", "newpkg-1.0.0.tar.gz", url) + if err != nil { + t.Fatalf("priming cache: %v", err) + } + _ = result.Reader.Close() + if len(store.files) != 1 { + t.Fatalf("expected the cache to hold the artifact, got %d files", len(store.files)) + } + + proxy.StreamArtifacts = true + cached, err := proxy.GetCachedArtifact(context.Background(), "pypi", "newpkg", "1.0.0", "newpkg-1.0.0.tar.gz") + if err != nil || cached != nil { + t.Fatalf("GetCachedArtifact = %v, %v; want nil, nil while streaming", cached, err) + } + + fetcher.fetchCalled = false + fetcher.artifact = &fetch.Artifact{Body: io.NopCloser(strings.NewReader("new bytes"))} + result, err = proxy.GetOrFetchArtifactFromURL(context.Background(), "pypi", "newpkg", "1.0.0", "newpkg-1.0.0.tar.gz", url) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + defer func() { _ = result.Reader.Close() }() + body, _ := io.ReadAll(result.Reader) + if !fetcher.fetchCalled || string(body) != "new bytes" { + t.Errorf("streaming served %q (fetched=%v), want a fresh upstream fetch", body, fetcher.fetchCalled) + } +} + +func TestStreamArtifactsVerifiesUpstreamDigest(t *testing.T) { + const content = "blob bytes" + + t.Run("matching digest streams the body", func(t *testing.T) { + proxy, _, _ := newStreamingProxy(t, content) + result, err := proxy.GetOrFetchArtifactFromURLWithDigest(context.Background(), "oci", "library/app", "v1", + "blob", "https://registry.test/blob", "sha256:"+sha256Hex(content)) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + defer func() { _ = result.Reader.Close() }() + + body, err := io.ReadAll(result.Reader) + if err != nil { + t.Fatalf("reading verified body: %v", err) + } + if string(body) != content { + t.Errorf("body = %q", body) + } + if got := result.Artifact.Digest.Encoded(); got != sha256Hex(content) { + t.Errorf("Digest = %q, want the upstream digest", got) + } + if result.Artifact.Size >= 0 { + t.Errorf("Size = %d, want unknown so the response is chunked", result.Artifact.Size) + } + }) + + t.Run("mismatched digest fails the read", func(t *testing.T) { + proxy, _, _ := newStreamingProxy(t, content) + result, err := proxy.GetOrFetchArtifactFromURLWithDigest(context.Background(), "oci", "library/app", "v1", + "blob", "https://registry.test/blob", "sha256:"+sha256Hex("other bytes")) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + defer func() { _ = result.Reader.Close() }() + + if _, err := io.ReadAll(result.Reader); !errors.Is(err, ErrArtifactDigestMismatch) { + t.Errorf("read error = %v, want ErrArtifactDigestMismatch", err) + } + }) +} + +func TestServeArtifactAbortsOnStreamedDigestMismatch(t *testing.T) { + proxy, _, _ := newStreamingProxy(t, "blob bytes") + result, err := proxy.GetOrFetchArtifactFromURLWithDigest(context.Background(), "oci", "library/app", "v1", + "blob", "https://registry.test/blob", "sha256:"+sha256Hex("other bytes")) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + rec := httptest.NewRecorder() + defer func() { + if r := recover(); r != http.ErrAbortHandler { + t.Fatalf("recovered %v, want http.ErrAbortHandler", r) + } + if rec.Header().Get(headerContentLength) != "" { + t.Errorf("Content-Length = %q, want none so the client sees an unterminated response", + rec.Header().Get(headerContentLength)) + } + }() + ServeArtifact(rec, result) +} + +func TestNPMDownloadCooldownWhileStreaming(t *testing.T) { + now := time.Now() + packument := `{ + "name": "leftpad", + "dist-tags": {"latest": "2.0.0"}, + "time": { + "1.0.0": "` + now.Add(-30*24*time.Hour).Format(time.RFC3339) + `", + "2.0.0": "` + now.Add(-1*time.Hour).Format(time.RFC3339) + `" + }, + "versions": {"1.0.0": {}, "2.0.0": {}} + }` + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", contentTypeJSON) + _, _ = io.WriteString(w, packument) + })) + defer upstream.Close() + + tests := []struct { + name string + version string + wantStatus int + }{ + {"published before the window streams the tarball", testVersion100, http.StatusOK}, + {"published inside the window is withheld", "2.0.0", http.StatusNotFound}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + proxy, store, fetcher := newStreamingProxy(t, "tarball data") + proxy.HTTPClient = upstream.Client() + proxy.Cooldown = &cooldown.Config{Default: "7d"} + + srv := httptest.NewServer(NewNPMHandler(proxy, "http://proxy.test", upstream.URL).Routes()) + defer srv.Close() + + resp, err := http.Get(srv.URL + "/leftpad/-/leftpad-" + tt.version + ".tgz") + if err != nil { + t.Fatalf("request failed: %v", err) + } + body, _ := io.ReadAll(resp.Body) + _ = resp.Body.Close() + + if resp.StatusCode != tt.wantStatus { + t.Fatalf("status = %d, want %d", resp.StatusCode, tt.wantStatus) + } + if tt.wantStatus == http.StatusOK && string(body) != "tarball data" { + t.Errorf("body = %q", body) + } + if tt.wantStatus == http.StatusNotFound && fetcher.fetchCalled { + t.Error("fetched a version that is still inside the cooldown window") + } + if len(store.files) != 0 { + t.Errorf("streaming stored %d files, want none", len(store.files)) + } + }) + } +} + +// truncatingUpstream answers every request matching match with raw, then +// closes the connection, so the response body ends early. +func truncatingUpstream(t *testing.T, match func(*http.Request) bool, raw string, fallback http.HandlerFunc) *httptest.Server { + t.Helper() + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if !match(r) { + fallback(w, r) + return + } + conn, buf, err := w.(http.Hijacker).Hijack() + if err != nil { + t.Errorf("hijack: %v", err) + return + } + _, _ = buf.WriteString(raw) + _ = buf.Flush() + _ = conn.Close() + })) + t.Cleanup(upstream.Close) + return upstream +} + +// requireIncompleteResponse fails unless reading the response surfaces an error, +// i.e. the client cannot mistake the body for a complete download. +func requireIncompleteResponse(t *testing.T, url string) { + t.Helper() + resp, err := http.Get(url) + if err != nil { + return + } + defer func() { _ = resp.Body.Close() }() + body, err := io.ReadAll(resp.Body) + if err == nil { + t.Fatalf("client read a complete %d response (Content-Length %q, body %q), want an incomplete one", + resp.StatusCode, resp.Header.Get(headerContentLength), body) + } +} + +func useRealFetcher(t *testing.T, proxy *Proxy, upstream *httptest.Server) { + t.Helper() + proxy.HTTPClient = upstream.Client() + fetcher := fetch.NewFetcher(fetch.WithHTTPClient(upstream.Client()), fetch.WithMaxRetries(0)) + proxy.Fetcher = fetcher + t.Cleanup(func() { _ = fetcher.Close() }) +} + +func TestStreamArtifactsOCIBlobShorterThanContentLengthIsIncomplete(t *testing.T) { + digest := "sha256:" + sha256Hex(strings.Repeat("x", 100)) + upstream := truncatingUpstream(t, + func(r *http.Request) bool { return strings.Contains(r.URL.Path, "/blobs/") }, + "HTTP/1.1 200 OK\r\nContent-Type: application/octet-stream\r\nContent-Length: 100\r\n\r\nshort", + http.NotFound) + + proxy, _, store, _ := setupTestProxy(t) + proxy.StreamArtifacts = true + useRealFetcher(t, proxy, upstream) + h := NewContainerHandler(proxy, "http://proxy.example", map[string]string{"ghcr": upstream.URL}) + srv := httptest.NewServer(h.Routes()) + defer srv.Close() + + requireIncompleteResponse(t, srv.URL+"/upstream/ghcr/owner/demo/blobs/"+digest) + if len(store.files) != 0 { + t.Errorf("streaming stored %d files, want none", len(store.files)) + } +} + +func TestStreamArtifactsNPMTarballWithoutFinalChunkIsIncomplete(t *testing.T) { + upstream := truncatingUpstream(t, + func(r *http.Request) bool { return strings.HasSuffix(r.URL.Path, ".tgz") }, + "HTTP/1.1 200 OK\r\nContent-Type: application/octet-stream\r\nTransfer-Encoding: chunked\r\n\r\n5\r\nshort\r\n", + http.NotFound) + + proxy, _, store, _ := setupTestProxy(t) + proxy.StreamArtifacts = true + useRealFetcher(t, proxy, upstream) + srv := httptest.NewServer(NewNPMHandler(proxy, "http://proxy.test", upstream.URL).Routes()) + defer srv.Close() + + requireIncompleteResponse(t, srv.URL+"/leftpad/-/leftpad-1.0.0.tgz") + if len(store.files) != 0 { + t.Errorf("streaming stored %d files, want none", len(store.files)) + } +} + +func TestServeArtifactAbortsOnShortRead(t *testing.T) { + tests := []struct { + name string + reader io.Reader + size int64 + }{ + {"read error", io.MultiReader(strings.NewReader("short"), iotestErrReader{io.ErrUnexpectedEOF}), -1}, + {"fewer bytes than the declared size", strings.NewReader("short"), 100}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := &CacheResult{Reader: io.NopCloser(tt.reader), Artifact: artifacts.Artifact{Size: tt.size}} + defer func() { + if r := recover(); r != http.ErrAbortHandler { + t.Fatalf("recovered %v, want http.ErrAbortHandler", r) + } + }() + ServeArtifact(httptest.NewRecorder(), result) + }) + } +} + +type iotestErrReader struct{ err error } + +func (r iotestErrReader) Read([]byte) (int, error) { return 0, r.err } + +func TestStreamArtifactsSwiftArchiveHeadIgnoresCachedEntry(t *testing.T) { + archive := []byte("cached archive") + checksum := sha256.Sum256(archive) + var archiveRequests int + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if strings.HasSuffix(r.URL.Path, ".zip") { + archiveRequests++ + http.NotFound(w, r) + return + } + w.Header().Set("Content-Type", "application/json") + _, _ = fmt.Fprintf(w, `{"id":"apple.example","version":"1.2.3","resources":[{"name":"source-archive","type":"application/zip","checksum":%q}]}`, hex.EncodeToString(checksum[:])) + })) + defer upstream.Close() + + proxy, _, _, fetcher := setupTestProxy(t) + proxy.HTTPClient = upstream.Client() + fetcher.artifact = &fetch.Artifact{ + Body: io.NopCloser(strings.NewReader(string(archive))), + Size: int64(len(archive)), + ContentType: "application/zip", + } + packagePURL, versionPURL := packageurl.MakeCacheStrings("swift", "apple/example", "1.2.3") + cached, err := proxy.getOrFetchArtifactFromURLWithCachePURLs( + context.Background(), "swift", "apple/example", "1.2.3", "example-1.2.3.zip", + packagePURL, versionPURL, upstream.URL+"/apple/example/1.2.3.zip", nil, hex.EncodeToString(checksum[:]), + ) + if err != nil { + t.Fatalf("seeding cache in normal mode: %v", err) + } + _ = cached.Reader.Close() + + proxy.StreamArtifacts = true + fetcher.artifact = nil + fetcher.fetchErr = fetch.ErrNotFound + handler := NewSwiftHandler(proxy, "https://proxy.example", upstream.URL).Routes() + + w := httptest.NewRecorder() + handler.ServeHTTP(w, httptest.NewRequest(http.MethodHead, "/apple/example/1.2.3.zip", nil)) + if w.Code == http.StatusOK { + t.Fatalf("HEAD served the cached archive (Content-Length %q) while streaming artifacts", w.Header().Get(headerContentLength)) + } + if archiveRequests == 0 { + t.Error("HEAD did not ask the upstream archive") + } + + w = httptest.NewRecorder() + handler.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/apple/example/1.2.3.zip", nil)) + if w.Code == http.StatusOK { + t.Fatalf("GET served the cached archive while streaming artifacts") + } +} diff --git a/internal/server/server.go b/internal/server/server.go index 3506648..da75aa0 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -257,6 +257,7 @@ func (s *Server) serve(listener net.Listener) error { proxy.DirectServe = s.cfg.Storage.DirectServe proxy.DirectServeTTL = s.cfg.ParseDirectServeTTL() proxy.DirectServeBaseURL = s.cfg.Storage.DirectServeBaseURL + proxy.StreamArtifacts = !s.cfg.Storage.CacheArtifacts // Create router with Chi r := chi.NewRouter() From 4261bcbc3fa36118b34156730c1a91ab01bf02ee Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Fri, 2 Oct 2026 10:34:41 +0100 Subject: [PATCH 169/633] Show the by-client table regardless of ui_request_sources The flag exists to withhold caller addresses. The by-client table carries none, and the same figures are already public at /metrics, so gating it left two metrics with no tile on the page in the default config. --- internal/server/sources_test.go | 10 ++++++++-- .../server/templates/components/runtime_metrics.html | 12 +++++++++--- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/internal/server/sources_test.go b/internal/server/sources_test.go index 1618d34..fa58dd7 100644 --- a/internal/server/sources_test.go +++ b/internal/server/sources_test.go @@ -423,8 +423,14 @@ func TestAnalyticsPageHidesSourcesByDefault(t *testing.T) { if strings.Contains(body, "10.1.2.3") { t.Error("a caller address rendered with ui_request_sources off") } - if strings.Contains(body, "Request sources") { - t.Error("the sources card rendered with ui_request_sources off") + // The by-client table carries no addresses and the same figures are already + // public at /metrics, so it renders either way; only the address table is + // gated. + if !strings.Contains(body, "By client") { + t.Error("the by-client table was gated along with the address table") + } + if !strings.Contains(body, "ui_request_sources") { + t.Error("the off-state note naming the config key did not render") } } diff --git a/internal/server/templates/components/runtime_metrics.html b/internal/server/templates/components/runtime_metrics.html index e464559..03fc9a0 100644 --- a/internal/server/templates/components/runtime_metrics.html +++ b/internal/server/templates/components/runtime_metrics.html @@ -93,19 +93,26 @@ - {{if .SourcesOn}}

Request sources

+ {{if .SourcesOn}}

{{if .TrustsForward}}Attributed by X-Forwarded-For{{else}}Attributed by peer address{{end}} {{- if .SourceCount}} · {{.SourceCount}} tracked{{end}}

+ {{end}}
- {{if .Sources}} + {{if not .SourcesOn}} +

+ Per-caller addresses are off. Set ui_request_sources: true + to show them here, once /ui is behind authentication; + the per-request detail is in the access log. +

+ {{else if .Sources}} @@ -153,7 +160,6 @@ - {{end}} {{if .ScanningOn}}
From 739bba316cb61b4ac1288574b775e1696b00fcdb Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Fri, 2 Oct 2026 10:42:59 +0100 Subject: [PATCH 170/633] Drop unused registry from TestUpdateEcosystemStatsSumsAliasedRows --- internal/metrics/analytics_test.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/internal/metrics/analytics_test.go b/internal/metrics/analytics_test.go index 6d391bc..427c38f 100644 --- a/internal/metrics/analytics_test.go +++ b/internal/metrics/analytics_test.go @@ -3,7 +3,6 @@ package metrics import ( "testing" - "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/testutil" ) @@ -68,9 +67,6 @@ func TestUpdateEcosystemStatsNormalizesLabels(t *testing.T) { // carrying both spellings of an aliased ecosystem yields two rows that // normalize to one label. They must sum rather than overwrite each other. func TestUpdateEcosystemStatsSumsAliasedRows(t *testing.T) { - reg := prometheus.NewRegistry() - reg.MustRegister(EcosystemDownloadedBytes, EcosystemCacheSize, EcosystemPackages) - UpdateEcosystemStats([]EcosystemStats{ {Ecosystem: "gem", DownloadedBytes: 100, CacheSize: 10, Packages: 1}, {Ecosystem: "rubygems", DownloadedBytes: 200, CacheSize: 20, Packages: 2}, From 8cc63c40eefa16ae09170a904e23d36179d8a441 Mon Sep 17 00:00:00 2001 From: montehurd Date: Sat, 3 Oct 2026 04:09:15 -0700 Subject: [PATCH 171/633] Cache rewritten npm and Composer metadata (#402) The npm and Composer handlers rewrite every metadata document they serve so that download URLs point at the proxy: decode the whole document into generic maps, change the URLs, encode it again, and for Composer expand the minified format first. That ran on every request, cached metadata included. With metadata caching on and upstream out of the picture, a cached request still cost 7 ms for @babel/core, 17 ms for symfony/console and 155 ms and 134 MB of allocations for typescript, and an 8 vCPU VM serving cached npm packuments to 20 clients ran the proxy at 450% CPU for about 1,100 requests a second. Rewritten documents are now kept in memory, keyed by the ecosystem, proxy URL, package and a SHA-256 of the raw document, so new bytes from upstream are rewritten again and nothing is served stale. Requests that arrive while a document is being rewritten wait for that rewrite rather than running their own; a waiter leaves when its client does, and the rewrite still completes and is cached. The denylist is fixed at startup, so it needs no place in the key. Cooldown filtering depends on the current time, so with cooldown on the cache is bypassed. metadata_rewrite_cache_size bounds the cache (default "256MB", least recently used out first, "0" to rewrite on every request). NewProxy callers keep the old behaviour unless they set it. Whole cached requests, measured locally: @babel/core 7.1 ms -> 0.38 ms, 54,489 -> 119 allocations typescript 155 ms -> 10.5 ms, 1.37M -> 133 allocations symfony/console 17.4 ms -> 0.48 ms, 165,909 -> 123 allocations What remains is reading the raw document from storage and hashing it. --- config.example.yaml | 5 + docs/configuration.md | 10 + internal/config/config.go | 42 ++++ internal/config/config_test.go | 48 +++++ internal/handler/composer.go | 5 +- internal/handler/handler.go | 4 + internal/handler/npm.go | 7 +- internal/handler/rewrite_cache.go | 160 +++++++++++++++ internal/handler/rewrite_cache_test.go | 258 +++++++++++++++++++++++++ internal/server/server.go | 1 + 10 files changed, 538 insertions(+), 2 deletions(-) create mode 100644 internal/handler/rewrite_cache.go create mode 100644 internal/handler/rewrite_cache_test.go diff --git a/config.example.yaml b/config.example.yaml index 15326de..44a3bfc 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -14,6 +14,11 @@ base_url: "http://localhost:8080" # Set to "0" to disable the timeout. Default: "30s". # http_timeout: "30s" +# Memory for rewritten npm and Composer metadata, so each upstream document is +# rewritten once rather than on every request. Set to "0" to rewrite on every +# request. Default: "256MB". +# metadata_rewrite_cache_size: "256MB" + # Public URL where the web UI is reached. Defaults to base_url when unset. # Set this separately when the UI is served on a different hostname than the # package endpoints — for example, the UI on a public domain behind auth while diff --git a/docs/configuration.md b/docs/configuration.md index d5003fb..ed5ad40 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -699,6 +699,16 @@ metadata_max_size: "100MB" # default Or via environment variable: `PROXY_METADATA_MAX_SIZE=250MB`. +### Rewritten metadata cache + +The npm and Composer handlers rewrite every metadata document they serve, so that download URLs point at the proxy. That means decoding the whole document and encoding it again, and Composer's minified documents are also expanded, which together cost milliseconds per typical package and far more for very large ones. The proxy keeps rewritten documents in memory, so each distinct upstream document is rewritten once and requests arriving while it is being rewritten wait for that rewrite. New bytes from upstream are rewritten again. With version cooldown enabled the cache is bypassed, because cooldown filtering depends on the current time. + +```yaml +metadata_rewrite_cache_size: "256MB" # default; "0" rewrites on every request +``` + +Or via environment variable: `PROXY_METADATA_REWRITE_CACHE_SIZE=1GB`. + ## Upstream HTTP timeout Protocol handlers use a shared HTTP client for upstream requests such as metadata fetches and pass-through file downloads. `http_timeout` sets that client's per-request timeout. Raise it if slow upstreams or large metadata responses cause `context deadline exceeded` errors. diff --git a/internal/config/config.go b/internal/config/config.go index f8dd1ab..05ed6d2 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -155,6 +155,12 @@ type Config struct { // size return ErrMetadataTooLarge. Default: "100MB". MetadataMaxSize string `json:"metadata_max_size" yaml:"metadata_max_size"` + // MetadataRewriteCacheSize is how much rewritten npm and Composer + // metadata to keep in memory, so a document is rewritten once rather than + // on every request (e.g. "256MB", "1GB"). Default: "256MB". Set to "0" to + // rewrite on every request. + MetadataRewriteCacheSize string `json:"metadata_rewrite_cache_size" yaml:"metadata_rewrite_cache_size"` + // HTTPTimeout is the timeout for individual upstream HTTP requests made // by protocol handlers (metadata fetches, pass-through file requests). // Uses Go duration syntax (e.g. "30s", "2m"). Default: "30s". @@ -976,6 +982,7 @@ func (c *Config) LoadFromEnv() { setEnvBool(&c.MirrorAPI, "PROXY_MIRROR_API") setEnvString(&c.MetadataTTL, "PROXY_METADATA_TTL") setEnvString(&c.MetadataMaxSize, "PROXY_METADATA_MAX_SIZE") + setEnvString(&c.MetadataRewriteCacheSize, "PROXY_METADATA_REWRITE_CACHE_SIZE") setEnvString(&c.HTTPTimeout, "PROXY_HTTP_TIMEOUT") setEnvBool(&c.Gradle.BuildCache.ReadOnly, "PROXY_GRADLE_BUILD_CACHE_READ_ONLY") setEnvString(&c.Gradle.BuildCache.MaxUploadSize, "PROXY_GRADLE_BUILD_CACHE_MAX_UPLOAD_SIZE") @@ -1103,6 +1110,10 @@ func (c *Config) validateCacheArtifacts() error { } func (c *Config) validateComponents() error { + if err := validateMetadataRewriteCacheSize(c.MetadataRewriteCacheSize); err != nil { + return err + } + if _, err := denylist.New(c.Denylist.Packages); err != nil { return err } @@ -1183,6 +1194,7 @@ const ( defaultHTTPTimeout = 30 * time.Second //nolint:mnd // sensible default defaultHitFlushInterval = time.Second defaultMetadataMaxSize = 100 << 20 + defaultMetadataRewriteCacheSize = 256 << 20 defaultGradleBuildCacheMaxUploadSize = 100 << 20 defaultGradleBuildCacheSweepInterval = 10 * time.Minute defaultGradleMaxUploadSizeStr = "100MB" @@ -1231,6 +1243,36 @@ func validateMetadataMaxSize(s string) error { return nil } +func validateMetadataRewriteCacheSize(s string) error { + if s == "" || s == "0" { + return nil + } + size, err := ParseSize(s) + if err != nil { + return fmt.Errorf("invalid metadata_rewrite_cache_size: %w", err) + } + if size < 0 { + return fmt.Errorf("invalid metadata_rewrite_cache_size %q: must not be negative", s) + } + return nil +} + +// ParseMetadataRewriteCacheSize returns how many bytes of rewritten metadata +// to keep in memory. Returns 256MB if unset or invalid, 0 if disabled. +func (c *Config) ParseMetadataRewriteCacheSize() int64 { + if c.MetadataRewriteCacheSize == "" { + return defaultMetadataRewriteCacheSize + } + if c.MetadataRewriteCacheSize == "0" { + return 0 + } + size, err := ParseSize(c.MetadataRewriteCacheSize) + if err != nil || size < 0 { + return defaultMetadataRewriteCacheSize + } + return size +} + // ParseMetadataMaxSize returns the maximum metadata response size in bytes. // Returns 100MB if unset or invalid. func (c *Config) ParseMetadataMaxSize() int64 { diff --git a/internal/config/config_test.go b/internal/config/config_test.go index c350898..2006970 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -940,6 +940,54 @@ func TestValidateHTTPTimeout(t *testing.T) { } } +func TestParseMetadataRewriteCacheSize(t *testing.T) { + tests := []struct { + name string + size string + want int64 + }{ + {"empty defaults to 256MB", "", 256 << 20}, + {"explicit zero disables", "0", 0}, + {"1GB", "1GB", 1 << 30}, + {"invalid defaults to 256MB", "lots", 256 << 20}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cfg := Default() + cfg.MetadataRewriteCacheSize = tt.size + if got := cfg.ParseMetadataRewriteCacheSize(); got != tt.want { + t.Errorf("ParseMetadataRewriteCacheSize() = %d, want %d", got, tt.want) + } + }) + } +} + +func TestValidateMetadataRewriteCacheSize(t *testing.T) { + cfg := Default() + cfg.MetadataRewriteCacheSize = "lots" + if err := cfg.Validate(); err == nil { + t.Error("expected validation error for invalid metadata_rewrite_cache_size") + } + + for _, ok := range []string{"512MB", "0", ""} { + cfg.MetadataRewriteCacheSize = ok + if err := cfg.Validate(); err != nil { + t.Errorf("unexpected error for metadata_rewrite_cache_size %q: %v", ok, err) + } + } +} + +func TestLoadMetadataRewriteCacheSizeFromEnv(t *testing.T) { + cfg := Default() + t.Setenv("PROXY_METADATA_REWRITE_CACHE_SIZE", "1GB") + cfg.LoadFromEnv() + + if cfg.MetadataRewriteCacheSize != "1GB" { + t.Errorf("MetadataRewriteCacheSize = %q, want %q", cfg.MetadataRewriteCacheSize, "1GB") + } +} + func TestLoadHTTPTimeoutFromEnv(t *testing.T) { cfg := Default() t.Setenv("PROXY_HTTP_TIMEOUT", "90s") diff --git a/internal/handler/composer.go b/internal/handler/composer.go index 47378f7..ce606e3 100644 --- a/internal/handler/composer.go +++ b/internal/handler/composer.go @@ -109,8 +109,11 @@ func (h *ComposerHandler) handlePackageMetadata(w http.ResponseWriter, r *http.R return } - rewritten, err := h.rewriteMetadata(body) + rewritten, err := h.proxy.cachedRewrite(r.Context(), "composer", h.proxyURL, packageName, body, h.rewriteMetadata) if err != nil { + if r.Context().Err() != nil { + return // the client left while waiting on a shared rewrite + } h.proxy.Logger.Warn("failed to rewrite metadata, proxying original", "error", err) w.Header().Set(headerContentType, "application/json") _, _ = w.Write(body) diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 37e353c..b04f941 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -197,6 +197,10 @@ type Proxy struct { // upstream fetch serves every waiting caller. Keyed by artifactCoalesceKey. fetchMu sync.Mutex inFlight map[string]*inflightFetch + + // rewrites caches metadata documents after their handler rewrites them. + // Nil leaves every request to rewrite its own copy. + rewrites *rewriteCache } // NewProxy creates a new Proxy with the given dependencies. diff --git a/internal/handler/npm.go b/internal/handler/npm.go index 27cf251..07dcc48 100644 --- a/internal/handler/npm.go +++ b/internal/handler/npm.go @@ -188,8 +188,13 @@ func (h *NPMHandler) handlePackageMetadata(w http.ResponseWriter, r *http.Reques return } - rewritten, err := h.rewriteMetadata(packageName, body) + rewritten, err := h.proxy.cachedRewrite(r.Context(), "npm", h.proxyURL, packageName, body, func(b []byte) ([]byte, error) { + return h.rewriteMetadata(packageName, b) + }) if err != nil { + if r.Context().Err() != nil { + return // the client left while waiting on a shared rewrite + } if len(h.proxy.Denylist.Versions(canonicalPackagePURL("npm", packageName))) != 0 { JSONError(w, http.StatusBadGateway, "failed to filter package metadata") return diff --git a/internal/handler/rewrite_cache.go b/internal/handler/rewrite_cache.go new file mode 100644 index 0000000..7626827 --- /dev/null +++ b/internal/handler/rewrite_cache.go @@ -0,0 +1,160 @@ +package handler + +import ( + "container/list" + "context" + "crypto/sha256" + "errors" + "strings" + "sync" +) + +// rewriteCache keeps metadata documents after a handler has rewritten them. +// Rewriting means decoding the whole document into generic maps, changing its +// download URLs and encoding it again, which for a large npm packument or an +// expanded Composer document costs milliseconds to hundreds of milliseconds +// and megabytes of allocations. It ran on every request, cached metadata +// included. The cache keeps one rewrite per distinct upstream document, and +// concurrent requests for a document that is not cached yet share a single +// rewrite. +// +// Entries are keyed by a hash of the raw document, so new bytes from upstream +// are rewritten again rather than served stale. Everything else a rewrite +// depends on is fixed for the life of the process: the proxy URL is part of +// the key, and the denylist is loaded at startup. +type rewriteCache struct { + maxBytes int64 + + mu sync.Mutex + size int64 + order *list.List // most recently used at the front + entries map[string]*list.Element + inFlight map[string]*inflightRewrite +} + +type rewriteEntry struct { + key string + out []byte +} + +// inflightRewrite is one rewrite that concurrent callers share. out and err +// are written before done closes and read only after, so the close is the +// handoff. +type inflightRewrite struct { + done chan struct{} + out []byte + err error +} + +// errSharedRewriteAbandoned is what waiters see if the caller running a +// shared rewrite panicked out of it. +var errSharedRewriteAbandoned = errors.New("shared metadata rewrite did not complete") + +// newRewriteCache returns a cache holding up to maxBytes of rewritten output. +// A size of zero or less disables it. +func newRewriteCache(maxBytes int64) *rewriteCache { + if maxBytes <= 0 { + return nil + } + return &rewriteCache{ + maxBytes: maxBytes, + order: list.New(), + entries: make(map[string]*list.Element), + inFlight: make(map[string]*inflightRewrite), + } +} + +// rewriteCacheKey identifies one rewrite: the ecosystem and proxy URL the +// handler rewrites for, the package, and the exact upstream bytes. +func rewriteCacheKey(ecosystem, proxyURL, name string, in []byte) string { + sum := sha256.Sum256(in) + return strings.Join([]string{ecosystem, proxyURL, name, string(sum[:])}, "\x00") +} + +// rewrite returns rewrite(in), from the cache when it can. Callers must treat +// the returned bytes as read-only: a cached result is shared. A caller waiting +// on another's rewrite leaves when ctx ends; the rewrite itself always runs +// to completion, so the result is cached for the next request. +func (c *rewriteCache) rewrite(ctx context.Context, key string, in []byte, rewrite func([]byte) ([]byte, error)) ([]byte, error) { + if c == nil { + return rewrite(in) + } + + c.mu.Lock() + if el, ok := c.entries[key]; ok { + c.order.MoveToFront(el) + out := el.Value.(*rewriteEntry).out + c.mu.Unlock() + return out, nil + } + if f, ok := c.inFlight[key]; ok { + c.mu.Unlock() + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-f.done: + return f.out, f.err + } + } + f := &inflightRewrite{done: make(chan struct{}), err: errSharedRewriteAbandoned} + c.inFlight[key] = f + c.mu.Unlock() + + defer func() { + c.mu.Lock() + delete(c.inFlight, key) + if f.err == nil { + c.add(key, f.out) + } + c.mu.Unlock() + close(f.done) + }() + f.out, f.err = rewrite(in) + return f.out, f.err +} + +// add stores out under key and evicts least recently used entries until the +// cache is back under its limit. Output larger than the whole cache is not +// stored. Called with mu held. +func (c *rewriteCache) add(key string, out []byte) { + n := int64(len(out)) + if n > c.maxBytes { + return + } + c.entries[key] = c.order.PushFront(&rewriteEntry{key: key, out: out}) + c.size += n + for c.size > c.maxBytes { + oldest := c.order.Back() + e := oldest.Value.(*rewriteEntry) + c.order.Remove(oldest) + delete(c.entries, e.key) + c.size -= int64(len(e.out)) + } +} + +// len reports how many rewrites are cached. +func (c *rewriteCache) len() int { + if c == nil { + return 0 + } + c.mu.Lock() + defer c.mu.Unlock() + return len(c.entries) +} + +// cachedRewrite rewrites a metadata document through the proxy's rewrite +// cache. Cooldown filtering depends on the current time, so with cooldown on +// a cached rewrite could keep hiding a version past its cooldown; those +// rewrites always run. +func (p *Proxy) cachedRewrite(ctx context.Context, ecosystem, proxyURL, name string, in []byte, rewrite func([]byte) ([]byte, error)) ([]byte, error) { + if p.rewrites == nil || (p.Cooldown != nil && p.Cooldown.Enabled()) { + return rewrite(in) + } + return p.rewrites.rewrite(ctx, rewriteCacheKey(ecosystem, proxyURL, name, in), in, rewrite) +} + +// SetMetadataRewriteCacheSize enables the cache of rewritten metadata with +// room for maxBytes of output. Zero or less disables it. +func (p *Proxy) SetMetadataRewriteCacheSize(maxBytes int64) { + p.rewrites = newRewriteCache(maxBytes) +} diff --git a/internal/handler/rewrite_cache_test.go b/internal/handler/rewrite_cache_test.go new file mode 100644 index 0000000..0edb94f --- /dev/null +++ b/internal/handler/rewrite_cache_test.go @@ -0,0 +1,258 @@ +package handler + +import ( + "context" + "errors" + "net/http" + "net/http/httptest" + "strings" + "sync" + "sync/atomic" + "testing" + + "github.com/git-pkgs/cooldown" +) + +// countingRewrite upper-cases its input and counts calls. If entered is set it +// is closed when the first call starts, and the call then waits on release. +type countingRewrite struct { + calls atomic.Int64 + entered chan struct{} + release chan struct{} + once sync.Once +} + +func (r *countingRewrite) fn(in []byte) ([]byte, error) { + r.calls.Add(1) + if r.entered != nil { + r.once.Do(func() { close(r.entered) }) + <-r.release + } + return []byte(strings.ToUpper(string(in))), nil +} + +func TestRewriteCache_RepeatServesCachedRewrite(t *testing.T) { + c := newRewriteCache(1 << 20) + var r countingRewrite + key := rewriteCacheKey("npm", "http://proxy", "left-pad", []byte("doc")) + + for range 3 { + out, err := c.rewrite(context.Background(), key, []byte("doc"), r.fn) + if err != nil || string(out) != "DOC" { + t.Fatalf("rewrite = %q, %v", out, err) + } + } + if got := r.calls.Load(); got != 1 { + t.Errorf("rewrites = %d, want 1", got) + } +} + +func TestRewriteCache_NewUpstreamBytesRewriteAgain(t *testing.T) { + c := newRewriteCache(1 << 20) + var r countingRewrite + + for _, doc := range []string{"v1", "v2"} { + key := rewriteCacheKey("npm", "http://proxy", "left-pad", []byte(doc)) + out, err := c.rewrite(context.Background(), key, []byte(doc), r.fn) + if err != nil || string(out) != strings.ToUpper(doc) { + t.Fatalf("rewrite(%s) = %q, %v", doc, out, err) + } + } + if got := r.calls.Load(); got != 2 { + t.Errorf("rewrites = %d, want 2", got) + } +} + +// TestRewriteCache_ConcurrentRequestsShareRewrite asserts that requests +// arriving while a document is being rewritten wait for that rewrite instead +// of running their own. Callers that arrive after it finishes hit the cache, +// so the count holds however the goroutines are scheduled. +func TestRewriteCache_ConcurrentRequestsShareRewrite(t *testing.T) { + c := newRewriteCache(1 << 20) + r := countingRewrite{entered: make(chan struct{}), release: make(chan struct{})} + key := rewriteCacheKey("npm", "http://proxy", "typescript", []byte("doc")) + const n = 10 + + outs := make(chan string, n) + run := func() { + out, err := c.rewrite(context.Background(), key, []byte("doc"), r.fn) + if err != nil { + t.Error(err) + } + outs <- string(out) + } + go run() + <-r.entered + for range n - 1 { + go run() + } + close(r.release) + + for range n { + if out := <-outs; out != "DOC" { + t.Errorf("out = %q", out) + } + } + if got := r.calls.Load(); got != 1 { + t.Errorf("rewrites = %d, want 1", got) + } +} + +// TestRewriteCache_WaiterLeavingKeepsRewrite asserts that a caller waiting on +// another's rewrite returns its own context error when its client leaves, +// while the rewrite completes and is cached for the next request. +func TestRewriteCache_WaiterLeavingKeepsRewrite(t *testing.T) { + c := newRewriteCache(1 << 20) + r := countingRewrite{entered: make(chan struct{}), release: make(chan struct{})} + key := rewriteCacheKey("npm", "http://proxy", "typescript", []byte("doc")) + + first := make(chan error, 1) + go func() { + _, err := c.rewrite(context.Background(), key, []byte("doc"), r.fn) + first <- err + }() + <-r.entered + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if _, err := c.rewrite(ctx, key, []byte("doc"), r.fn); !errors.Is(err, context.Canceled) { + t.Errorf("waiter err = %v, want context.Canceled", err) + } + close(r.release) + if err := <-first; err != nil { + t.Fatal(err) + } + if out, err := c.rewrite(context.Background(), key, []byte("doc"), r.fn); err != nil || string(out) != "DOC" { + t.Errorf("cached rewrite = %q, %v", out, err) + } + if got := r.calls.Load(); got != 1 { + t.Errorf("rewrites = %d, want 1", got) + } +} + +func TestRewriteCache_EvictsLeastRecentlyUsed(t *testing.T) { + c := newRewriteCache(6) // room for two three-byte outputs + var r countingRewrite + keyFor := func(doc string) string { return rewriteCacheKey("npm", "http://proxy", doc, []byte(doc)) } + + for _, doc := range []string{"aaa", "bbb"} { + _, _ = c.rewrite(context.Background(), keyFor(doc), []byte(doc), r.fn) + } + _, _ = c.rewrite(context.Background(), keyFor("aaa"), []byte("aaa"), r.fn) // aaa is now the most recent + _, _ = c.rewrite(context.Background(), keyFor("ccc"), []byte("ccc"), r.fn) // evicts bbb + + before := r.calls.Load() + _, _ = c.rewrite(context.Background(), keyFor("aaa"), []byte("aaa"), r.fn) + _, _ = c.rewrite(context.Background(), keyFor("ccc"), []byte("ccc"), r.fn) + if got := r.calls.Load() - before; got != 0 { + t.Errorf("aaa and ccc rewritten %d times, want both still cached", got) + } + _, _ = c.rewrite(context.Background(), keyFor("bbb"), []byte("bbb"), r.fn) + if got := r.calls.Load() - before; got != 1 { + t.Errorf("bbb should have been evicted and rewritten") + } +} + +func TestRewriteCache_OutputLargerThanCacheIsNotStored(t *testing.T) { + c := newRewriteCache(2) + var r countingRewrite + key := rewriteCacheKey("npm", "http://proxy", "big", []byte("big")) + + _, _ = c.rewrite(context.Background(), key, []byte("big"), r.fn) + if c.len() != 0 { + t.Errorf("cached %d entries, want 0", c.len()) + } +} + +func TestRewriteCache_ErrorsAreNotCached(t *testing.T) { + c := newRewriteCache(1 << 20) + var calls int + failing := func([]byte) ([]byte, error) { calls++; return nil, errors.New("bad document") } + key := rewriteCacheKey("npm", "http://proxy", "broken", []byte("doc")) + + for range 2 { + if _, err := c.rewrite(context.Background(), key, []byte("doc"), failing); err == nil { + t.Fatal("expected the rewrite error") + } + } + if calls != 2 { + t.Errorf("rewrites = %d, want 2", calls) + } +} + +func TestCachedRewrite_DisabledRewritesEveryTime(t *testing.T) { + proxy, _, _, _ := setupTestProxy(t) + proxy.SetMetadataRewriteCacheSize(0) + var r countingRewrite + + for range 2 { + _, _ = proxy.cachedRewrite(context.Background(), "npm", "http://proxy", "left-pad", []byte("doc"), r.fn) + } + if got := r.calls.Load(); got != 2 { + t.Errorf("rewrites = %d, want 2", got) + } +} + +// TestCachedRewrite_CooldownBypassesCache asserts that with cooldown on every +// request is rewritten: cooldown filtering depends on the current time, so a +// cached rewrite could keep hiding a version after its cooldown ends. +func TestCachedRewrite_CooldownBypassesCache(t *testing.T) { + proxy, _, _, _ := setupTestProxy(t) + proxy.SetMetadataRewriteCacheSize(1 << 20) + proxy.Cooldown = &cooldown.Config{Default: "3d"} + var r countingRewrite + + for range 2 { + _, _ = proxy.cachedRewrite(context.Background(), "npm", "http://proxy", "left-pad", []byte("doc"), r.fn) + } + if got := r.calls.Load(); got != 2 { + t.Errorf("rewrites = %d, want 2", got) + } +} + +// TestMetadataHandlers_ServeCachedRewrite asserts that repeated requests for +// the same npm or Composer metadata reuse one rewrite and return the same +// bytes, with the download URLs pointing at the proxy. +func TestMetadataHandlers_ServeCachedRewrite(t *testing.T) { + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + switch r.URL.Path { + case "/left-pad": + _, _ = w.Write([]byte(`{"name":"left-pad","versions":{"1.3.0":{"dist":{"tarball":"https://registry.npmjs.org/left-pad/-/left-pad-1.3.0.tgz"}}}}`)) + case "/p2/vendor/pkg.json": + _, _ = w.Write([]byte(`{"packages":{"vendor/pkg":[{"version":"1.0.0","dist":{"type":"zip","url":"https://example.com/pkg-1.0.0.zip"}}]}}`)) + default: + http.NotFound(w, r) + } + })) + defer upstream.Close() + + proxy, _, _, _ := setupTestProxy(t) + proxy.HTTPClient = upstream.Client() + proxy.SetMetadataRewriteCacheSize(1 << 20) + + cases := []struct { + name string + handler http.Handler + path string + }{ + {"npm", NewNPMHandler(proxy, "http://proxy.example", upstream.URL).Routes(), "/left-pad"}, + {"composer", NewComposerHandlerWithUpstreams(proxy, "http://proxy.example", upstream.URL, upstream.URL).Routes(), "/p2/vendor/pkg.json"}, + } + for i, c := range cases { + var bodies []string + for range 2 { + rec := httptest.NewRecorder() + c.handler.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, c.path, nil)) + if rec.Code != http.StatusOK { + t.Fatalf("%s: status %d", c.name, rec.Code) + } + bodies = append(bodies, rec.Body.String()) + } + if bodies[0] != bodies[1] || !strings.Contains(bodies[0], "http://proxy.example") { + t.Errorf("%s: bodies %q and %q", c.name, bodies[0], bodies[1]) + } + if got := proxy.rewrites.len(); got != i+1 { + t.Errorf("%s: cached rewrites = %d, want %d", c.name, got, i+1) + } + } +} diff --git a/internal/server/server.go b/internal/server/server.go index 1b82ce5..2483909 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -255,6 +255,7 @@ func (s *Server) serve(listener net.Listener) error { proxy.CacheMetadata = s.cfg.CacheMetadata proxy.MetadataTTL = s.cfg.ParseMetadataTTL() proxy.MetadataMaxSize = s.cfg.ParseMetadataMaxSize() + proxy.SetMetadataRewriteCacheSize(s.cfg.ParseMetadataRewriteCacheSize()) proxy.GradleReadOnly = s.cfg.Gradle.BuildCache.ReadOnly proxy.NPMFullMetadata = s.cfg.Upstream.NPMFullMetadata proxy.GradleMaxUploadSize = s.cfg.ParseGradleBuildCacheMaxUploadSize() From 4b1625b2eb7f1d262e1cfb4c1410cb13e2514d53 Mon Sep 17 00:00:00 2001 From: montehurd Date: Sat, 3 Oct 2026 04:10:44 -0700 Subject: [PATCH 172/633] Copy inherited Composer fields shallowly when expanding (#401) expandMinifiedVersions deep-copied every inherited field into every version (b68184c) so that rewriting one version's dist URL in place could not change the versions that inherited it. For a package with a long history that means recursively copying require, autoload and the rest hundreds of times per document, on every metadata request. dist is the only field the proxy changes after expansion, so copy just that: rewriteDistURL now gives the version its own dist map before setting the URL, and expansion shares the other inherited values. TestComposerExpandMinifiedSharedDistReferences still guards the original bug, and fails if the copy in rewriteDistURL is removed. On the symfony/console metadata from Packagist the rewritten output is byte-identical, and the rewrite drops from about 20.7 ms to 13.8 ms, with allocations down from 13.6 MB to 9 MB. --- internal/handler/composer.go | 35 ++++++++++++----------------------- 1 file changed, 12 insertions(+), 23 deletions(-) diff --git a/internal/handler/composer.go b/internal/handler/composer.go index ce606e3..75a6dc8 100644 --- a/internal/handler/composer.go +++ b/internal/handler/composer.go @@ -180,10 +180,12 @@ func expandMinifiedVersions(versionList []any) []any { } // Merge inherited fields into a new map, then overlay current fields. - // Deep copy values to avoid shared references between versions. + // Inherited values are shared between versions, not copied: the + // only one rewritten afterwards is dist, and rewriteDistURL copies + // it before changing it. merged := make(map[string]any, len(inherited)+len(vmap)) for k, val := range inherited { - merged[k] = deepCopyValue(val) + merged[k] = val } for k, val := range vmap { if val == composerUnset { @@ -202,26 +204,6 @@ func expandMinifiedVersions(versionList []any) []any { return expanded } -// deepCopyValue returns a deep copy of JSON-like values (maps, slices, scalars). -func deepCopyValue(v any) any { - switch val := v.(type) { - case map[string]any: - m := make(map[string]any, len(val)) - for k, v := range val { - m[k] = deepCopyValue(v) - } - return m - case []any: - s := make([]any, len(val)) - for i, v := range val { - s[i] = deepCopyValue(v) - } - return s - default: - return v - } -} - // filterAndRewriteVersions applies cooldown filtering and rewrites dist URLs // for a single package's version list. func (h *ComposerHandler) filterAndRewriteVersions(packageName string, versionList []any) []any { @@ -301,7 +283,14 @@ func (h *ComposerHandler) rewriteDistURL(vmap map[string]any, packageName, versi if len(parts) == vendorPackageParts { newURL := fmt.Sprintf("%s/composer/files/%s/%s/%s/%s", h.proxyURL, parts[0], parts[1], version, filename) - dist["url"] = newURL + // Expanded versions can share one inherited dist map, so give this + // version its own before changing its URL. + own := make(map[string]any, len(dist)) + for k, v := range dist { + own[k] = v + } + own["url"] = newURL + vmap["dist"] = own } } From f60772d2d87d7ba69ff43c19fa38030fcf3a4fc5 Mon Sep 17 00:00:00 2001 From: PhantomPhoton <120879831+PhantomPhoton@users.noreply.github.com> Date: Sun, 4 Oct 2026 09:48:41 -0700 Subject: [PATCH 173/633] Support HTTP byte-range requests for local file cached artifact downloads (#404) * feat(storage): expose efficient seek for local files Open file-backed artifacts with os.Open so local readers support efficient seeking. Leave cloud storage readers unchanged. Keep the legacy sidecar mapping test focused on fileblob's reader: Blob.Open now reads local files directly and no longer consults the sidecar. Continue checking sidecar cleanup and post-store reads. * feat(handler): preserve efficient seek through integrity checks Keep the cache result's single Reader field and preserve io.Seeker when the underlying storage reader supports efficient seeking. Sequential reads continue through whole-object integrity verification. A successful seek switches subsequent reads to the source for range responses, where full-object verification is not possible. Test both modes and verify the source is closed once. * feat(handler): add request-aware byte range responses Add a request-aware artifact serving helper for bounded, open-ended, and suffix byte ranges, including If-Range handling, 206 responses, and 416 responses for valid unsatisfiable ranges. Ignore malformed and multi-range requests by serving the full artifact. Advertise byte-range support only for seekable readers, preserving HEAD, redirect, and non-seekable behavior. Clarify that net/http recovers ErrAbortHandler per request and keeps the server running. * feat(handler): enable byte ranges across artifact downloads Forward each artifact request to the shared range-aware response helper, including cached generic release assets and OCI blobs. This lets the handler honor ranges only when its reader supports efficient seeking, without changing direct-storage redirect behavior. Add warm-cache endpoint tests that assert partial response headers and bytes without an upstream refetch. Document supported range behavior, unsupported reader fallback, and the integrity limitation of partial reads. --- docs/architecture.md | 8 ++ internal/handler/apk.go | 2 +- internal/handler/cargo.go | 2 +- internal/handler/composer.go | 2 +- internal/handler/conan.go | 4 +- internal/handler/conda.go | 2 +- internal/handler/container.go | 4 +- internal/handler/container_test.go | 13 +- internal/handler/cran.go | 4 +- internal/handler/debian.go | 2 +- internal/handler/filename_download.go | 2 +- internal/handler/generic.go | 2 +- internal/handler/generic_test.go | 20 ++- internal/handler/go.go | 2 +- internal/handler/handler.go | 176 ++++++++++++++++++++++++-- internal/handler/handler_test.go | 165 ++++++++++++++++++++++++ internal/handler/helm.go | 8 +- internal/handler/integrity.go | 45 ++++++- internal/handler/integrity_test.go | 67 ++++++++++ internal/handler/julia.go | 6 +- internal/handler/maven.go | 2 +- internal/handler/npm.go | 2 +- internal/handler/nuget.go | 2 +- internal/handler/pub.go | 2 +- internal/handler/pypi.go | 2 +- internal/handler/rpm.go | 2 +- internal/handler/swift.go | 4 +- internal/storage/blob.go | 11 ++ internal/storage/blob_test.go | 15 ++- internal/storage/storage.go | 3 + 30 files changed, 530 insertions(+), 51 deletions(-) diff --git a/docs/architecture.md b/docs/architecture.md index 42122c5..0d37a60 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -62,6 +62,14 @@ Metadata is not cached - always fetched fresh. This ensures clients see new vers - Return reader to handler - Handler streams file to client +Artifact handlers honor single byte ranges when the storage reader supports +efficient seeking (currently the local filesystem reader). They advertise +`Accept-Ranges: bytes`, return `206` or `416` as appropriate, and respect +`If-Range`. Malformed and multi-range requests fall back to the full response. +Non-seekable storage readers do not advertise range support, and direct-storage +redirects rely on the destination's capabilities. Range responses cannot verify +the full artifact digest because they read only part of the artifact. + ``` ┌────────┐ GET /npm/lodash/-/lodash-4.17.21.tgz ┌─────────────┐ │ Client │ ──────────────────────────────────────▶│ NPMHandler │ diff --git a/internal/handler/apk.go b/internal/handler/apk.go index 9acc3cc..e0b4b31 100644 --- a/internal/handler/apk.go +++ b/internal/handler/apk.go @@ -132,7 +132,7 @@ func (h *APKHandler) handlePackageDownload(w http.ResponseWriter, r *http.Reques if result.Artifact.MediaType == "" { result.Artifact.MediaType = "application/octet-stream" } - serveArtifact(w, r.Method, result) + ServeArtifactRequest(w, r, result) } // handleMetadata serves repository indexes and signatures through the diff --git a/internal/handler/cargo.go b/internal/handler/cargo.go index 01b62b9..f87819c 100644 --- a/internal/handler/cargo.go +++ b/internal/handler/cargo.go @@ -218,5 +218,5 @@ func (h *CargoHandler) handleDownload(w http.ResponseWriter, r *http.Request) { return } - ServeArtifact(w, result) + ServeArtifactRequest(w, r, result) } diff --git a/internal/handler/composer.go b/internal/handler/composer.go index 75a6dc8..2f475ac 100644 --- a/internal/handler/composer.go +++ b/internal/handler/composer.go @@ -350,7 +350,7 @@ func (h *ComposerHandler) handleDownload(w http.ResponseWriter, r *http.Request) return } - ServeArtifact(w, result) + ServeArtifactRequest(w, r, result) } // isDevVersion reports whether a Composer version string refers to a diff --git a/internal/handler/conan.go b/internal/handler/conan.go index 862f48c..aa41668 100644 --- a/internal/handler/conan.go +++ b/internal/handler/conan.go @@ -94,7 +94,7 @@ func (h *ConanHandler) handleRecipeFile(w http.ResponseWriter, r *http.Request) return } - ServeArtifact(w, result) + ServeArtifactRequest(w, r, result) } // handlePackageFile serves a package file, fetching and caching from upstream if needed. @@ -131,7 +131,7 @@ func (h *ConanHandler) handlePackageFile(w http.ResponseWriter, r *http.Request) return } - ServeArtifact(w, result) + ServeArtifactRequest(w, r, result) } // shouldCacheFile returns true if the file should be cached. diff --git a/internal/handler/conda.go b/internal/handler/conda.go index ad01f16..07c2aea 100644 --- a/internal/handler/conda.go +++ b/internal/handler/conda.go @@ -84,7 +84,7 @@ func (h *CondaHandler) handleDownload(w http.ResponseWriter, r *http.Request) { return } - ServeArtifact(w, result) + ServeArtifactRequest(w, r, result) } // isPackageFile returns true if the filename is a Conda package. diff --git a/internal/handler/container.go b/internal/handler/container.go index 5a47d79..f73f0f5 100644 --- a/internal/handler/container.go +++ b/internal/handler/container.go @@ -164,7 +164,7 @@ func (h *ContainerHandler) handleBlobDownload(w http.ResponseWriter, r *http.Req if cached.Artifact.MediaType == "" { cached.Artifact.MediaType = "application/octet-stream" } - serveArtifact(w, r.Method, cached) + ServeArtifactRequest(w, r, cached) return } @@ -208,7 +208,7 @@ func (h *ContainerHandler) handleBlobDownload(w http.ResponseWriter, r *http.Req if result.Artifact.MediaType == "" { result.Artifact.MediaType = "application/octet-stream" } - ServeArtifact(w, result) + ServeArtifactRequest(w, r, result) } // handleManifest serves immutable manifests from cache and revalidates mutable tags. diff --git a/internal/handler/container_test.go b/internal/handler/container_test.go index 5b7aa03..bd237cf 100644 --- a/internal/handler/container_test.go +++ b/internal/handler/container_test.go @@ -665,6 +665,7 @@ func TestContainerHandler_BlobDownload_CacheHitSkipsAuth(t *testing.T) { proxy, db, store, fetcher := setupTestProxy(t) digest := "sha256:abc123def456abc123def456abc123def456abc123def456abc123def456abcd" seedPackage(t, db, store, "oci", "library/nginx", digest, digest, "cached blob") + store.seekable = true upstreamRequests := 0 upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { @@ -680,14 +681,18 @@ func TestContainerHandler_BlobDownload_CacheHitSkipsAuth(t *testing.T) { } req := httptest.NewRequest(http.MethodGet, "/library/nginx/blobs/"+digest, nil) + req.Header.Set("Range", "bytes=0-5") w := httptest.NewRecorder() h.Routes().ServeHTTP(w, req) - if w.Code != http.StatusOK { - t.Fatalf("status = %d, want %d; body: %s", w.Code, http.StatusOK, w.Body.String()) + if w.Code != http.StatusPartialContent { + t.Fatalf("status = %d, want %d; body: %s", w.Code, http.StatusPartialContent, w.Body.String()) } - if got := w.Body.String(); got != "cached blob" { - t.Errorf("body = %q, want %q", got, "cached blob") + if got := w.Body.String(); got != "cached" { + t.Errorf("body = %q, want %q", got, "cached") + } + if got := w.Header().Get("Content-Range"); got != "bytes 0-5/11" { + t.Errorf("Content-Range = %q, want %q", got, "bytes 0-5/11") } if upstreamRequests != 0 { t.Errorf("upstream requests = %d, want 0", upstreamRequests) diff --git a/internal/handler/cran.go b/internal/handler/cran.go index 4a6ded8..6a363a4 100644 --- a/internal/handler/cran.go +++ b/internal/handler/cran.go @@ -83,7 +83,7 @@ func (h *CRANHandler) handleSourceDownload(w http.ResponseWriter, r *http.Reques return } - ServeArtifact(w, result) + ServeArtifactRequest(w, r, result) } // handleBinaryDownload serves a binary package, fetching and caching from upstream. @@ -117,7 +117,7 @@ func (h *CRANHandler) handleBinaryDownload(w http.ResponseWriter, r *http.Reques return } - ServeArtifact(w, result) + ServeArtifactRequest(w, r, result) } // parseSourceFilename extracts name and version from a CRAN source filename. diff --git a/internal/handler/debian.go b/internal/handler/debian.go index 513a6cc..7b6635e 100644 --- a/internal/handler/debian.go +++ b/internal/handler/debian.go @@ -156,7 +156,7 @@ func (h *DebianHandler) handlePackageDownload( } w.Header().Set(headerContentType, "application/vnd.debian.binary-package") - ServeArtifact(w, result) + ServeArtifactRequest(w, r, result) } // handleMetadata serves repository metadata files through the metadata cache, diff --git a/internal/handler/filename_download.go b/internal/handler/filename_download.go index e3c1162..fc96586 100644 --- a/internal/handler/filename_download.go +++ b/internal/handler/filename_download.go @@ -39,5 +39,5 @@ func (p *Proxy) handleFilenameDownload(w http.ResponseWriter, r *http.Request, d return } - ServeArtifact(w, result) + ServeArtifactRequest(w, r, result) } diff --git a/internal/handler/generic.go b/internal/handler/generic.go index 31f5dc3..a2458b2 100644 --- a/internal/handler/generic.go +++ b/internal/handler/generic.go @@ -133,7 +133,7 @@ func (h *GenericHandler) handleReleaseAsset(w http.ResponseWriter, r *http.Reque if result.Artifact.MediaType == "" { result.Artifact.MediaType = "application/octet-stream" } - serveArtifact(w, r.Method, result) + ServeArtifactRequest(w, r, result) } // handleMetadata serves any other path through the metadata cache. The query diff --git a/internal/handler/generic_test.go b/internal/handler/generic_test.go index 7d47cb9..e67a6b1 100644 --- a/internal/handler/generic_test.go +++ b/internal/handler/generic_test.go @@ -100,7 +100,8 @@ func TestGenericHandler_ReleaseAssetIsCachedAndServedWhenUpstreamDown(t *testing })) defer upstream.Close() - proxy, _, _, _ := setupTestProxy(t) + proxy, _, store, _ := setupTestProxy(t) + store.seekable = true fetcher := fetch.NewFetcher(fetch.WithHTTPClient(upstream.Client()), fetch.WithMaxRetries(0)) proxy.Fetcher = fetcher t.Cleanup(func() { _ = fetcher.Close() }) @@ -120,6 +121,23 @@ func TestGenericHandler_ReleaseAssetIsCachedAndServedWhenUpstreamDown(t *testing // Second request must be served from cache, even with the upstream down. available.Store(false) + rangeRequest := httptest.NewRequest(http.MethodGet, "/github"+testReleaseAssetPath, nil) + rangeRequest.Header.Set("Range", "bytes=0-2") + rangeResponse := httptest.NewRecorder() + h.Routes().ServeHTTP(rangeResponse, rangeRequest) + if rangeResponse.Code != http.StatusPartialContent { + t.Fatalf("range: status = %d, want 206: %s", rangeResponse.Code, rangeResponse.Body.String()) + } + if got := rangeResponse.Body.String(); got != string(asset[:3]) { + t.Errorf("range: body = %q, want %q", got, asset[:3]) + } + if got := rangeResponse.Header().Get("Content-Range"); got != "bytes 0-2/15" { + t.Errorf("range: Content-Range = %q, want %q", got, "bytes 0-2/15") + } + if got := upstreamRequests.Load(); got != 1 { + t.Errorf("upstream requests after range cache hit = %d, want 1", got) + } + w = serveGenericRequest(h, "/github"+testReleaseAssetPath) if w.Code != http.StatusOK { t.Fatalf("cached: status = %d, want 200: %s", w.Code, w.Body.String()) diff --git a/internal/handler/go.go b/internal/handler/go.go index 90b93b7..fb85587 100644 --- a/internal/handler/go.go +++ b/internal/handler/go.go @@ -126,7 +126,7 @@ func (h *GoHandler) handleDownload(w http.ResponseWriter, r *http.Request, modul return } - ServeArtifact(w, result) + ServeArtifactRequest(w, r, result) } // proxyUpstream forwards a request to proxy.golang.org without caching. diff --git a/internal/handler/handler.go b/internal/handler/handler.go index b04f941..df6a63c 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -756,7 +756,21 @@ func ServeArtifact(w http.ResponseWriter, result *CacheResult) { serveArtifact(w, http.MethodGet, result) } +// ServeArtifactRequest writes a CacheResult to an HTTP response using request +// headers such as Range and If-Range. +func ServeArtifactRequest(w http.ResponseWriter, r *http.Request, result *CacheResult) { + if r == nil { + ServeArtifact(w, result) + return + } + serveArtifactResponse(w, r.Method, r, result) +} + func serveArtifact(w http.ResponseWriter, method string, result *CacheResult) { + serveArtifactResponse(w, method, nil, result) +} + +func serveArtifactResponse(w http.ResponseWriter, method string, request *http.Request, result *CacheResult) { contentHash := "" if result.Artifact.Digest != "" { contentHash = result.Artifact.Digest.Encoded() @@ -777,26 +791,162 @@ func serveArtifact(w http.ResponseWriter, method string, result *CacheResult) { if result.Artifact.MediaType != "" { w.Header().Set(headerContentType, result.Artifact.MediaType) } - if result.Artifact.Size > 0 || (method == http.MethodHead && result.Artifact.Size == 0) { - w.Header().Set(headerContentLength, strconv.FormatInt(result.Artifact.Size, 10)) - } if contentHash != "" { w.Header().Set(headerETag, `"`+contentHash+`"`) } + var seeker io.Seeker + if result.Reader != nil { + seeker, _ = result.Reader.(io.Seeker) + } + if seeker != nil && result.Artifact.Size >= 0 { + w.Header().Set("Accept-Ranges", "bytes") + if request != nil && serveArtifactRange(w, request, result, seeker) { + return + } + } + + if result.Artifact.Size > 0 || (method == http.MethodHead && result.Artifact.Size == 0) { + w.Header().Set(headerContentLength, strconv.FormatInt(result.Artifact.Size, 10)) + } + w.WriteHeader(http.StatusOK) if method != http.MethodHead && result.Reader != nil { - buffer := artifactCopyBufferPool.Get().(*[]byte) - defer artifactCopyBufferPool.Put(buffer) - // Hide optional ReaderFrom methods so io.CopyBuffer uses the pooled buffer. - written, err := io.CopyBuffer(struct{ io.Writer }{w}, result.Reader, *buffer) - if err != nil || (result.Artifact.Size > 0 && written != result.Artifact.Size) { - // Headers are already committed, so an error status is no longer - // possible. Aborting leaves the response unterminated and the - // client discards it instead of keeping a truncated or unverified - // artifact. - panic(http.ErrAbortHandler) + copyArtifactBody(w, result.Reader, result.Artifact.Size, false) + } +} + +type parsedByteRange struct { + start int64 + end int64 +} + +func serveArtifactRange(w http.ResponseWriter, request *http.Request, result *CacheResult, seeker io.Seeker) bool { + if request.Method != http.MethodGet { + return false + } + ranges := request.Header.Values("Range") + if len(ranges) != 1 || !ifRangeMatches(request, w.Header().Get(headerETag)) { + return false + } + + byteRange, valid, satisfiable := parseByteRange(ranges[0], result.Artifact.Size) + if !valid { + return false + } + if !satisfiable { + w.Header().Set("Content-Range", fmt.Sprintf("bytes */%d", result.Artifact.Size)) + w.Header().Set(headerContentLength, "0") + w.WriteHeader(http.StatusRequestedRangeNotSatisfiable) + return true + } + + if _, err := seeker.Seek(byteRange.start, io.SeekStart); err != nil { + w.Header().Del(headerContentType) + w.Header().Del(headerContentLength) + w.Header().Del(headerETag) + w.Header().Del("Accept-Ranges") + http.Error(w, "failed to seek cached artifact", http.StatusInternalServerError) + return true + } + + length := byteRange.end - byteRange.start + 1 + w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", byteRange.start, byteRange.end, result.Artifact.Size)) + w.Header().Set(headerContentLength, strconv.FormatInt(length, 10)) + w.WriteHeader(http.StatusPartialContent) + copyArtifactBody(w, result.Reader, length, true) + return true +} + +func ifRangeMatches(request *http.Request, etag string) bool { + values := request.Header.Values("If-Range") + if len(values) == 0 { + return true + } + return len(values) == 1 && etag != "" && strings.TrimSpace(values[0]) == etag +} + +func parseByteRange(value string, size int64) (parsedByteRange, bool, bool) { + unit, spec, ok := strings.Cut(strings.TrimSpace(value), "=") + if !ok || !strings.EqualFold(strings.TrimSpace(unit), "bytes") { + return parsedByteRange{}, false, false + } + spec = strings.TrimSpace(spec) + if spec == "" || strings.Contains(spec, ",") { + return parsedByteRange{}, false, false + } + first, last, ok := strings.Cut(spec, "-") + if !ok { + return parsedByteRange{}, false, false + } + + if first == "" { + suffixLength, ok := parseRangeNumber(last) + if !ok { + return parsedByteRange{}, false, false } + if size == 0 || suffixLength == 0 { + return parsedByteRange{}, true, false + } + if suffixLength >= size { + return parsedByteRange{start: 0, end: size - 1}, true, true + } + return parsedByteRange{start: size - suffixLength, end: size - 1}, true, true + } + + start, ok := parseRangeNumber(first) + if !ok { + return parsedByteRange{}, false, false + } + if last == "" { + if size == 0 || start >= size { + return parsedByteRange{}, true, false + } + return parsedByteRange{start: start, end: size - 1}, true, true + } + end, ok := parseRangeNumber(last) + if !ok || start > end { + return parsedByteRange{}, false, false + } + if size == 0 || start >= size { + return parsedByteRange{}, true, false + } + if end >= size { + end = size - 1 + } + return parsedByteRange{start: start, end: end}, true, true +} + +func parseRangeNumber(value string) (int64, bool) { + if value == "" { + return 0, false + } + for _, digit := range value { + if digit < '0' || digit > '9' { + return 0, false + } + } + number, err := strconv.ParseInt(value, 10, 64) + return number, err == nil +} + +func copyArtifactBody(w http.ResponseWriter, reader io.Reader, expectedSize int64, bounded bool) { + buffer := artifactCopyBufferPool.Get().(*[]byte) + defer artifactCopyBufferPool.Put(buffer) + + source := reader + if bounded { + source = io.LimitReader(reader, expectedSize) + } + // Hide optional ReaderFrom methods so io.CopyBuffer uses the pooled buffer. + written, err := io.CopyBuffer(struct{ io.Writer }{w}, source, *buffer) + if err != nil || (bounded && written != expectedSize) || (!bounded && expectedSize > 0 && written != expectedSize) { + // Headers are already committed, so an error status is no longer + // possible. Aborting leaves the response unterminated so clients discard + // a truncated or unverified artifact. + // net/http recovers ErrAbortHandler for this request and keeps the server + // running; on HTTP/1.x it may close this connection as well. + panic(http.ErrAbortHandler) } } diff --git a/internal/handler/handler_test.go b/internal/handler/handler_test.go index c2714c3..4c05cbd 100644 --- a/internal/handler/handler_test.go +++ b/internal/handler/handler_test.go @@ -37,6 +37,7 @@ type mockStorage struct { openErr error signedURL string signErr error + seekable bool } func newMockStorage() *mockStorage { @@ -67,9 +68,18 @@ func (s *mockStorage) Open(_ context.Context, path string) (io.ReadCloser, error if !ok { return nil, storage.ErrNotFound } + if s.seekable { + return &mockSeekableReadCloser{Reader: bytes.NewReader(data)}, nil + } return io.NopCloser(bytes.NewReader(data)), nil } +type mockSeekableReadCloser struct { + *bytes.Reader +} + +func (r *mockSeekableReadCloser) Close() error { return nil } + func (s *mockStorage) Exists(_ context.Context, path string) (bool, error) { s.mu.Lock() defer s.mu.Unlock() @@ -737,6 +747,161 @@ func TestServeArtifact_Stream(t *testing.T) { } } +func TestServeArtifactRequestRanges(t *testing.T) { + const payload = "hello world" + tests := []struct { + name string + rangeHeader string + secondRange string + wantStatus int + wantRange string + wantLength string + wantBody string + }{ + {name: "bounded", rangeHeader: "bytes=1-3", wantStatus: http.StatusPartialContent, wantRange: "bytes 1-3/11", wantLength: "3", wantBody: "ell"}, + {name: "open ended", rangeHeader: "bytes=6-", wantStatus: http.StatusPartialContent, wantRange: "bytes 6-10/11", wantLength: "5", wantBody: "world"}, + {name: "suffix", rangeHeader: "bytes=-4", wantStatus: http.StatusPartialContent, wantRange: "bytes 7-10/11", wantLength: "4", wantBody: "orld"}, + {name: "suffix larger than artifact", rangeHeader: "bytes=-99", wantStatus: http.StatusPartialContent, wantRange: "bytes 0-10/11", wantLength: "11", wantBody: payload}, + {name: "unsatisfiable", rangeHeader: "bytes=11-", wantStatus: http.StatusRequestedRangeNotSatisfiable, wantRange: "bytes */11", wantLength: "0"}, + {name: "malformed", rangeHeader: "bytes=invalid", wantStatus: http.StatusOK, wantLength: "11", wantBody: payload}, + {name: "reversed", rangeHeader: "bytes=4-2", wantStatus: http.StatusOK, wantLength: "11", wantBody: payload}, + {name: "multiple ranges", rangeHeader: "bytes=0-1,4-5", wantStatus: http.StatusOK, wantLength: "11", wantBody: payload}, + {name: "multiple range fields", rangeHeader: "bytes=0-1", secondRange: "bytes=4-5", wantStatus: http.StatusOK, wantLength: "11", wantBody: payload}, + {name: "no range", wantStatus: http.StatusOK, wantLength: "11", wantBody: payload}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + request := httptest.NewRequest(http.MethodGet, "/artifact", nil) + if test.rangeHeader != "" { + request.Header.Set("Range", test.rangeHeader) + } + if test.secondRange != "" { + request.Header.Add("Range", test.secondRange) + } + w := httptest.NewRecorder() + ServeArtifactRequest(w, request, newRangeTestResult(payload)) + + if w.Code != test.wantStatus { + t.Errorf("status = %d, want %d", w.Code, test.wantStatus) + } + if got := w.Header().Get("Content-Range"); got != test.wantRange { + t.Errorf("Content-Range = %q, want %q", got, test.wantRange) + } + if got := w.Header().Get(headerContentLength); got != test.wantLength { + t.Errorf("Content-Length = %q, want %q", got, test.wantLength) + } + if got := w.Body.String(); got != test.wantBody { + t.Errorf("body = %q, want %q", got, test.wantBody) + } + if got := w.Header().Get("Accept-Ranges"); got != "bytes" { + t.Errorf("Accept-Ranges = %q, want bytes", got) + } + }) + } +} + +func TestServeArtifactRequestIfRange(t *testing.T) { + const payload = "hello world" + tests := []struct { + name string + ifRange string + wantStatus int + wantRange string + wantBody string + }{ + {name: "matching etag", ifRange: `"sha256-matching"`, wantStatus: http.StatusPartialContent, wantRange: "bytes 0-4/11", wantBody: "hello"}, + {name: "mismatching etag", ifRange: `"sha256-stale"`, wantStatus: http.StatusOK, wantBody: payload}, + {name: "weak etag", ifRange: `W/"sha256-matching"`, wantStatus: http.StatusOK, wantBody: payload}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + result := newRangeTestResult(payload) + request := httptest.NewRequest(http.MethodGet, "/artifact", nil) + request.Header.Set("Range", "bytes=0-4") + if test.name == "matching etag" { + test.ifRange = `"` + result.Artifact.Digest.Encoded() + `"` + } + request.Header.Set("If-Range", test.ifRange) + w := httptest.NewRecorder() + ServeArtifactRequest(w, request, result) + + if w.Code != test.wantStatus { + t.Errorf("status = %d, want %d", w.Code, test.wantStatus) + } + if got := w.Header().Get("Content-Range"); got != test.wantRange { + t.Errorf("Content-Range = %q, want %q", got, test.wantRange) + } + if got := w.Body.String(); got != test.wantBody { + t.Errorf("body = %q, want %q", got, test.wantBody) + } + }) + } +} + +func TestServeArtifactRequestHeadIgnoresRange(t *testing.T) { + request := httptest.NewRequest(http.MethodHead, "/artifact", nil) + request.Header.Set("Range", "bytes=0-1") + w := httptest.NewRecorder() + ServeArtifactRequest(w, request, newRangeTestResult("hello world")) + + if w.Code != http.StatusOK { + t.Errorf("status = %d, want %d", w.Code, http.StatusOK) + } + if got := w.Header().Get(headerContentLength); got != "11" { + t.Errorf("Content-Length = %q, want 11", got) + } + if got := w.Header().Get("Content-Range"); got != "" { + t.Errorf("Content-Range = %q, want empty", got) + } + if w.Body.Len() != 0 { + t.Errorf("HEAD body = %q, want empty", w.Body.String()) + } +} + +func TestServeArtifactRequestWithoutSeekCapability(t *testing.T) { + request := httptest.NewRequest(http.MethodGet, "/artifact", nil) + request.Header.Set("Range", "bytes=1-2") + w := httptest.NewRecorder() + ServeArtifactRequest(w, request, &CacheResult{ + Reader: io.NopCloser(strings.NewReader("payload")), + Artifact: testArtifact("payload", "pkg:npm/example@1.0.0", "example.tgz", "application/gzip"), + }) + + if w.Code != http.StatusOK || w.Body.String() != "payload" { + t.Errorf("response = %d %q, want 200 with full payload", w.Code, w.Body.String()) + } + if got := w.Header().Get("Accept-Ranges"); got != "" { + t.Errorf("Accept-Ranges = %q, want empty", got) + } +} + +func TestServeArtifactRequestRedirectDoesNotAdvertiseRanges(t *testing.T) { + request := httptest.NewRequest(http.MethodGet, "/artifact", nil) + request.Header.Set("Range", "bytes=1-2") + w := httptest.NewRecorder() + ServeArtifactRequest(w, request, &CacheResult{RedirectURL: "https://storage.example/artifact"}) + + if w.Code != http.StatusFound { + t.Errorf("status = %d, want %d", w.Code, http.StatusFound) + } + if got := w.Header().Get("Accept-Ranges"); got != "" { + t.Errorf("Accept-Ranges = %q, want empty", got) + } +} + +type rangeTestReadSeeker struct { + *bytes.Reader +} + +func (r *rangeTestReadSeeker) Close() error { return nil } + +func newRangeTestResult(payload string) *CacheResult { + return &CacheResult{ + Reader: &rangeTestReadSeeker{Reader: bytes.NewReader([]byte(payload))}, + Artifact: testArtifact(payload, "pkg:npm/example@1.0.0", "example.tgz", "application/gzip"), + } +} + func TestGetOrFetchArtifactFromURL_CacheHit(t *testing.T) { proxy, db, store, fetcher := setupTestProxy(t) seedPackage(t, db, store, "pypi", "requests", "2.28.0", "requests-2.28.0.tar.gz", "pypi content") diff --git a/internal/handler/helm.go b/internal/handler/helm.go index c0f86fd..31ef901 100644 --- a/internal/handler/helm.go +++ b/internal/handler/helm.go @@ -94,7 +94,7 @@ func (h *HelmHandler) handleChart(w http.ResponseWriter, r *http.Request) { return } if cached != nil { - h.serveChart(w, repository, digest, filename, cached) + h.serveChart(w, r, repository, digest, filename, cached) return } @@ -127,10 +127,10 @@ func (h *HelmHandler) handleChart(w http.ResponseWriter, r *http.Request) { h.proxy.serveArtifactError(w, err, "failed to fetch chart") return } - h.serveChart(w, repository, digest, filename, result) + h.serveChart(w, r, repository, digest, filename, result) } -func (h *HelmHandler) serveChart(w http.ResponseWriter, repository, digest, filename string, result *CacheResult) { +func (h *HelmHandler) serveChart(w http.ResponseWriter, r *http.Request, repository, digest, filename string, result *CacheResult) { if !strings.EqualFold(result.Artifact.Digest.Encoded(), digest) { if result.Reader != nil { _ = result.Reader.Close() @@ -145,7 +145,7 @@ func (h *HelmHandler) serveChart(w http.ResponseWriter, repository, digest, file if result.Artifact.MediaType == "" { w.Header().Set(headerContentType, "application/gzip") } - ServeArtifact(w, result) + ServeArtifactRequest(w, r, result) } func (h *HelmHandler) repositoryForRequest(r *http.Request) (name, upstreamURL string, ok bool) { diff --git a/internal/handler/integrity.go b/internal/handler/integrity.go index 97f41d9..437252f 100644 --- a/internal/handler/integrity.go +++ b/internal/handler/integrity.go @@ -40,7 +40,22 @@ func newIntegrityChecks(contentHash, native string) (integrityChecks, error) { } func (c integrityChecks) wrap(source io.ReadCloser, onMismatch func(string)) (io.ReadCloser, error) { - return c.newVerifyingReader(source, onMismatch, false) + if len(c.algorithms) == 0 { + return source, nil + } + verified, err := c.newVerifyingReader(source, onMismatch, false) + if err != nil { + return nil, err + } + seeker, ok := source.(io.Seeker) + if !ok { + return verified, nil + } + return &seekableVerifyingReader{ + source: source, + verified: verified, + seeker: seeker, + }, nil } // wrapFailOnMismatch is wrap for bytes that have not been checked anywhere @@ -80,6 +95,34 @@ type verifyingReader struct { mismatched bool } +// seekableVerifyingReader verifies ordinary reads until a successful seek, +// after which reads bypass whole-object verification for range responses. +type seekableVerifyingReader struct { + source io.ReadCloser + verified io.ReadCloser + seeker io.Seeker + bypass bool +} + +func (r *seekableVerifyingReader) Read(p []byte) (int, error) { + if r.bypass { + return r.source.Read(p) + } + return r.verified.Read(p) +} + +func (r *seekableVerifyingReader) Seek(offset int64, whence int) (int64, error) { + position, err := r.seeker.Seek(offset, whence) + if err == nil { + r.bypass = true + } + return position, err +} + +func (r *seekableVerifyingReader) Close() error { + return r.verified.Close() +} + func (r *verifyingReader) Read(p []byte) (int, error) { n, err := r.reader.Read(p) if err == io.EOF { diff --git a/internal/handler/integrity_test.go b/internal/handler/integrity_test.go index 95992c0..9aa9958 100644 --- a/internal/handler/integrity_test.go +++ b/internal/handler/integrity_test.go @@ -1,6 +1,7 @@ package handler import ( + "bytes" "crypto/sha256" "crypto/sha512" "encoding/base64" @@ -126,6 +127,72 @@ func TestVerifyingReader(t *testing.T) { } } +func TestVerifyingReaderPreservesSeekCapability(t *testing.T) { + const data = "hello world" + t.Run("ordinary reads stay verified", func(t *testing.T) { + source := &seekableCloseTrackingReader{Reader: bytes.NewReader([]byte(data))} + var calls int + reader := wrapIntegrityReader(t, source, sha256Hex("different"), "", func(string) { calls++ }) + + got, err := io.ReadAll(reader) + if err != nil { + t.Fatalf("ReadAll: %v", err) + } + if string(got) != data { + t.Errorf("data = %q, want %q", got, data) + } + if calls != 1 { + t.Errorf("onMismatch called %d times, want 1", calls) + } + if err := reader.Close(); err != nil { + t.Fatalf("Close: %v", err) + } + if source.closeCount != 1 { + t.Errorf("source closed %d times, want 1", source.closeCount) + } + }) + + t.Run("seek bypasses whole-object verification", func(t *testing.T) { + source := &seekableCloseTrackingReader{Reader: bytes.NewReader([]byte(data))} + var calls int + reader := wrapIntegrityReader(t, source, sha256Hex("different"), "", func(string) { calls++ }) + seeker, ok := reader.(io.Seeker) + if !ok { + t.Fatal("seekable source did not retain io.Seeker") + } + if _, err := seeker.Seek(6, io.SeekStart); err != nil { + t.Fatalf("Seek: %v", err) + } + + got, err := io.ReadAll(reader) + if err != nil { + t.Fatalf("ReadAll: %v", err) + } + if string(got) != "world" { + t.Errorf("data after seek = %q, want %q", got, "world") + } + if calls != 0 { + t.Errorf("onMismatch called %d times for a partial read, want 0", calls) + } + if err := reader.Close(); err != nil { + t.Fatalf("Close: %v", err) + } + if source.closeCount != 1 { + t.Errorf("source closed %d times, want 1", source.closeCount) + } + }) +} + +type seekableCloseTrackingReader struct { + *bytes.Reader + closeCount int +} + +func (r *seekableCloseTrackingReader) Close() error { + r.closeCount++ + return nil +} + func TestVerifyingReaderUsesStrongestNativeAlgorithm(t *testing.T) { const data = "artifact" tests := []struct { diff --git a/internal/handler/julia.go b/internal/handler/julia.go index 211d092..66a2245 100644 --- a/internal/handler/julia.go +++ b/internal/handler/julia.go @@ -103,7 +103,7 @@ func (h *JuliaHandler) handleRegistry(w http.ResponseWriter, r *http.Request) { go h.refreshNamesFromRegistry(uuid, hash) - ServeArtifact(w, result) + ServeArtifactRequest(w, r, result) } // handlePackage serves an immutable package source tarball. @@ -129,7 +129,7 @@ func (h *JuliaHandler) handlePackage(w http.ResponseWriter, r *http.Request) { return } - ServeArtifact(w, result) + ServeArtifactRequest(w, r, result) } // handleArtifact serves an immutable binary artifact tarball. Artifacts are @@ -150,7 +150,7 @@ func (h *JuliaHandler) handleArtifact(w http.ResponseWriter, r *http.Request) { return } - ServeArtifact(w, result) + ServeArtifactRequest(w, r, result) } // proxyUpstream forwards a request to the upstream Pkg server without caching. diff --git a/internal/handler/maven.go b/internal/handler/maven.go index 10e551e..3f1cd9d 100644 --- a/internal/handler/maven.go +++ b/internal/handler/maven.go @@ -134,7 +134,7 @@ func (h *MavenHandler) handleDownload(w http.ResponseWriter, r *http.Request, ur return } - ServeArtifact(w, result) + ServeArtifactRequest(w, r, result) } // parsePath extracts Maven coordinates from a URL path. diff --git a/internal/handler/npm.go b/internal/handler/npm.go index 07dcc48..663c8c6 100644 --- a/internal/handler/npm.go +++ b/internal/handler/npm.go @@ -406,7 +406,7 @@ func (h *NPMHandler) handleDownload(w http.ResponseWriter, r *http.Request) { return } - ServeArtifact(w, result) + ServeArtifactRequest(w, r, result) } // versionInCooldown reports whether a version is still inside the cooldown diff --git a/internal/handler/nuget.go b/internal/handler/nuget.go index d37e391..612f32e 100644 --- a/internal/handler/nuget.go +++ b/internal/handler/nuget.go @@ -192,7 +192,7 @@ func (h *NuGetHandler) handleDownload(w http.ResponseWriter, r *http.Request) { return } - ServeArtifact(w, result) + ServeArtifactRequest(w, r, result) } // proxyUpstream forwards a request to NuGet without caching. diff --git a/internal/handler/pub.go b/internal/handler/pub.go index 9d449ba..6e967e3 100644 --- a/internal/handler/pub.go +++ b/internal/handler/pub.go @@ -81,7 +81,7 @@ func (h *PubHandler) handleDownload(w http.ResponseWriter, r *http.Request) { return } - ServeArtifact(w, result) + ServeArtifactRequest(w, r, result) } // handlePackageMetadata proxies package metadata and rewrites archive URLs. diff --git a/internal/handler/pypi.go b/internal/handler/pypi.go index 21d9805..e37fddb 100644 --- a/internal/handler/pypi.go +++ b/internal/handler/pypi.go @@ -626,7 +626,7 @@ func (h *PyPIHandler) handleDownload(w http.ResponseWriter, r *http.Request) { return } - ServeArtifact(w, result) + ServeArtifactRequest(w, r, result) } // archiveExtensions are sdist formats of the form {name}-{version}{ext}. They diff --git a/internal/handler/rpm.go b/internal/handler/rpm.go index e06a3cc..6b138cc 100644 --- a/internal/handler/rpm.go +++ b/internal/handler/rpm.go @@ -95,7 +95,7 @@ func (h *RPMHandler) handlePackageDownload(w http.ResponseWriter, r *http.Reques } w.Header().Set(headerContentType, "application/x-rpm") - ServeArtifact(w, result) + ServeArtifactRequest(w, r, result) } // handleMetadata proxies repository metadata files (repomd.xml, primary.xml.gz, etc.). diff --git a/internal/handler/swift.go b/internal/handler/swift.go index 93bc9a7..6cc7d78 100644 --- a/internal/handler/swift.go +++ b/internal/handler/swift.go @@ -189,7 +189,7 @@ func (h *SwiftHandler) handleSourceArchive(w http.ResponseWriter, r *http.Reques result.Artifact.MediaType = "application/zip" setSwiftArchiveHeaders(w.Header(), name, version, result.Artifact.Digest.Encoded(), archiveInfo) - serveArtifact(w, r.Method, result) + ServeArtifactRequest(w, r, result) } func (h *SwiftHandler) handleSourceArchiveHead( @@ -208,7 +208,7 @@ func (h *SwiftHandler) handleSourceArchiveHead( if result != nil { result.Artifact.MediaType = "application/zip" setSwiftArchiveHeaders(w.Header(), name, version, result.Artifact.Digest.Encoded(), archiveInfo) - serveArtifact(w, r.Method, result) + ServeArtifactRequest(w, r, result) return } diff --git a/internal/storage/blob.go b/internal/storage/blob.go index 48574aa..4cadf8a 100644 --- a/internal/storage/blob.go +++ b/internal/storage/blob.go @@ -228,6 +228,17 @@ func (b *Blob) Store(ctx context.Context, path string, r io.Reader) (int64, stri } func (b *Blob) Open(ctx context.Context, path string) (io.ReadCloser, error) { + if localPath := b.localPath(path); localPath != "" { + r, err := os.Open(localPath) + if err != nil { + if os.IsNotExist(err) { + return nil, ErrNotFound + } + return nil, fmt.Errorf("opening local reader: %w", err) + } + return r, nil + } + r, err := b.bucket.NewReader(ctx, path, nil) if err != nil { if isNotExist(err) { diff --git a/internal/storage/blob_test.go b/internal/storage/blob_test.go index 57e85f5..efc1b76 100644 --- a/internal/storage/blob_test.go +++ b/internal/storage/blob_test.go @@ -67,13 +67,20 @@ func TestBlobOpen(t *testing.T) { t.Fatalf("Open failed: %v", err) } defer func() { _ = r.Close() }() + seeker, ok := r.(io.Seeker) + if !ok { + t.Fatal("local file reader does not implement io.Seeker") + } + if _, err := seeker.Seek(9, io.SeekStart); err != nil { + t.Fatalf("Seek failed: %v", err) + } data, err := io.ReadAll(r) if err != nil { t.Fatalf("ReadAll failed: %v", err) } - if string(data) != content { - t.Errorf("content = %q, want %q", string(data), content) + if string(data) != "content" { + t.Errorf("content after seek = %q, want %q", string(data), "content") } } @@ -529,7 +536,9 @@ func assertStoreClearsSidecar(t *testing.T, key string) { } b := openFileBlob(t, dir) - if _, err := b.Open(ctx, key); err == nil { + reader, err := b.bucket.NewReader(ctx, key, nil) + if err == nil { + _ = reader.Close() t.Fatal("corrupt sidecar did not fail the read, so it is not the file fileblob reads for this key") } diff --git a/internal/storage/storage.go b/internal/storage/storage.go index 0f64ed7..3b27ef4 100644 --- a/internal/storage/storage.go +++ b/internal/storage/storage.go @@ -46,6 +46,9 @@ type Storage interface { // Open returns a reader for the content at path. // The caller must close the reader when done. + // The reader may also implement io.Seeker when seeking is efficient for the + // storage backend. Backends must not expose io.Seeker when seeking requires + // reading and discarding the bytes before the requested position. // Returns ErrNotFound if the path does not exist. Open(ctx context.Context, path string) (io.ReadCloser, error) From dc1fc5baf38ca61844f66d9df10d46be4aa390f5 Mon Sep 17 00:00:00 2001 From: montehurd Date: Sun, 4 Oct 2026 09:50:45 -0700 Subject: [PATCH 174/633] Coalesce concurrent metadata fetches (#400) * Coalesce concurrent metadata fetches Concurrent cache misses for one artifact already share a single upstream fetch (#329), but metadata misses did not: every request for a package's metadata went to the registry, even when the same document was already being fetched. In CI that is the common case. Composer resolves versions on every job, since MediaWiki core and its extensions commit no composer.lock, and each package is one /p2/ document, so a burst of jobs fetched each document once per job. Metadata misses now follow the same pattern as artifacts: the first caller fetches, the rest wait for its result, and the caller that takes the key rechecks the cache first so a fetch that just finished is not repeated. The key includes the Accept and Accept-Encoding headers, which change the bytes upstream returns (npm's abbreviated and full documents), and whether the caller validates the response. One difference from artifacts: the shared fetch is detached from the first caller's cancellation. Artifact fetches keep it for scanning and mirroring, neither of which applies to metadata, and otherwise one job disconnecting would fail every job waiting on the same document. runScan detaches the same way, and the HTTP client's timeout still bounds the fetch. This covers every ecosystem that goes through fetchOrCacheMetadata (Composer, npm, PyPI, Maven, Cargo, Swift, Pub, Helm, NuGet), with metadata caching on or off. Streamed metadata with caching off is not coalesced. On an 8 vCPU test instance with caching off, 10 concurrent clients requesting the npm metadata of 1,263 packages went from 12,630 upstream requests to 1,898. With cache_metadata on, 20 clients requesting 1,538 packages made exactly 1,538. * Run each caller's validate on a shared metadata fetch A caller that joined another's metadata fetch got the shared bytes without running its own validate. NuGet's cooldown path uses validate to decode the document for the request, so a joined download found no publication date and was allowed, bypassing the cooldown. coalesceMetadata now reports whether the caller joined, and coalescedMetadataMiss runs that caller's validate on the shared bytes. Validation before caching is unchanged: the first caller still runs it inside the fetch. TestNuGetCooldownConcurrentDownloads reproduces the bypass through NuGetHandler.Routes() with two overlapping downloads of a version inside its cooldown. --- internal/handler/handler.go | 176 ++++++++++++--- internal/handler/metadata_coalesce_test.go | 249 +++++++++++++++++++++ internal/handler/nuget_cooldown_test.go | 54 +++++ 3 files changed, 451 insertions(+), 28 deletions(-) create mode 100644 internal/handler/metadata_coalesce_test.go diff --git a/internal/handler/handler.go b/internal/handler/handler.go index df6a63c..e5ccd61 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -198,6 +198,11 @@ type Proxy struct { fetchMu sync.Mutex inFlight map[string]*inflightFetch + // inFlightMeta does the same for metadata misses. Keyed by + // metadataCoalesceKey. + metaMu sync.Mutex + inFlightMeta map[string]*inflightMetadata + // rewrites caches metadata documents after their handler rewrites them. // Nil leaves every request to rewrite its own copy. rewrites *rewriteCache @@ -1059,34 +1064,20 @@ func (p *Proxy) FetchOrCacheMetadata(ctx context.Context, ecosystem, cacheKey, u // replayed as sent. The ProxyCached path uses "identity" for signed indexes and // "gzip" where both hops should stay compressed. // validate, when supplied, runs before caching or serving a document. Validation -// failures follow the same stale-cache fallback path as upstream failures. +// failures follow the same stale-cache fallback path as upstream failures. It +// runs for every caller, including one that shares another caller's fetch, so +// it can also decode the document into request-local state; a caller that +// joined a fetch gets its own validation error directly. It must not modify the +// body, which joined callers share. func (p *Proxy) fetchOrCacheMetadata(ctx context.Context, ecosystem, cacheKey, upstreamURL, acceptEncoding string, validate func([]byte) error, acceptHeaders ...string) ([]byte, string, string, error) { if containsPathTraversal(cacheKey) { return nil, "", "", fmt.Errorf("invalid cache key: %q", cacheKey) } - storagePath := metadataStoragePath(ecosystem, cacheKey) - - // Check for existing cache entry (for ETag revalidation and TTL) - var entry *database.MetadataCacheEntry - if p.CacheMetadata && p.DB != nil { - entry, _ = p.DB.GetMetadataCache(ecosystem, cacheKey) - } - // Serve from cache if within TTL (skip upstream entirely) - if entry != nil && p.MetadataTTL > 0 && entry.FetchedAt.Valid { - if time.Since(entry.FetchedAt.Time) < p.MetadataTTL { - data, ct, readErr := p.readCachedMetadata(ctx, entry, validate) - if readErr == nil { - metrics.RecordCacheHit(ecosystem) - return data, ct, entry.ContentEncoding.String, nil - } - if validate != nil { - // Do not revalidate an unusable cached body with its ETag. - entry = nil - } - // Cache file missing/unreadable, fall through to upstream - } + if _, hit := p.cachedMetadataState(ctx, ecosystem, cacheKey, validate); hit != nil { + metrics.RecordCacheHit(ecosystem) + return hit.body, hit.contentType, hit.contentEncoding, nil } p.recordMetadataCacheMiss(ecosystem) @@ -1095,7 +1086,62 @@ func (p *Proxy) fetchOrCacheMetadata(ctx context.Context, ecosystem, cacheKey, u accept = acceptHeaders[0] } - // Try upstream + res := p.coalescedMetadataMiss(ctx, ecosystem, cacheKey, upstreamURL, accept, acceptEncoding, validate) + return res.body, res.contentType, res.contentEncoding, res.err +} + +// coalescedMetadataMiss handles a metadata cache miss, sharing one upstream +// fetch among concurrent callers with the same key. +func (p *Proxy) coalescedMetadataMiss(ctx context.Context, ecosystem, cacheKey, upstreamURL, accept, acceptEncoding string, validate func([]byte) error) metadataResult { + key := metadataCoalesceKey(ecosystem, cacheKey, upstreamURL, accept, acceptEncoding, validate != nil) + res, shared := p.coalesceMetadata(ctx, key, func(fetchCtx context.Context) metadataResult { + // The caller's lookup ran before it took the key, so a fetch that + // finished in between has already refreshed the row. Recheck it + // rather than fetching again, and revalidate against the row as it + // is now. + entry, hit := p.cachedMetadataState(fetchCtx, ecosystem, cacheKey, validate) + if hit != nil { + return *hit + } + return p.fetchMetadataFromUpstream(fetchCtx, ecosystem, cacheKey, upstreamURL, accept, acceptEncoding, validate, entry) + }) + // The fetch ran the first caller's validate. A caller that joined it runs + // its own on the shared bytes, since validate may also decode them for it. + if shared && res.err == nil && validate != nil { + if err := validate(res.body); err != nil { + return metadataResult{err: err} + } + } + return res +} + +// cachedMetadataState reads the cache row for a metadata lookup. hit is set +// when the row is within TTL and its bytes are usable. Otherwise entry is the +// row to revalidate against with its ETag: nil when there is none, or when +// validate rejected the cached body, since revalidating an unusable body would +// keep it. +func (p *Proxy) cachedMetadataState(ctx context.Context, ecosystem, cacheKey string, validate func([]byte) error) (*database.MetadataCacheEntry, *metadataResult) { + if !p.CacheMetadata || p.DB == nil { + return nil, nil + } + entry, _ := p.DB.GetMetadataCache(ecosystem, cacheKey) + if entry != nil && p.MetadataTTL > 0 && entry.FetchedAt.Valid && time.Since(entry.FetchedAt.Time) < p.MetadataTTL { + data, ct, err := p.readCachedMetadata(ctx, entry, validate) + if err == nil { + return entry, &metadataResult{body: data, contentType: ct, contentEncoding: entry.ContentEncoding.String} + } + if validate != nil { + return nil, nil + } + // Cache file missing/unreadable, fall through to upstream + } + return entry, nil +} + +// fetchMetadataFromUpstream fetches metadata after a cache miss, caches it, +// and falls back to the cached copy if upstream fails. entry is the row to +// revalidate against, or nil. +func (p *Proxy) fetchMetadataFromUpstream(ctx context.Context, ecosystem, cacheKey, upstreamURL, accept, acceptEncoding string, validate func([]byte) error, entry *database.MetadataCacheEntry) metadataResult { meta, err := p.fetchUpstreamMetadata(ctx, upstreamURL, entry, accept, acceptEncoding) if errors.Is(err, errStale304) { // 304 but cached file is gone; retry without ETag @@ -1106,14 +1152,14 @@ func (p *Proxy) fetchOrCacheMetadata(ctx context.Context, ecosystem, cacheKey, u } if err == nil { if p.CacheMetadata { - p.cacheMetadataBlob(ctx, ecosystem, cacheKey, storagePath, meta) + p.cacheMetadataBlob(ctx, ecosystem, cacheKey, metadataStoragePath(ecosystem, cacheKey), meta) } - return meta.body, meta.contentType, meta.contentEncoding, nil + return metadataResult{body: meta.body, contentType: meta.contentType, contentEncoding: meta.contentEncoding} } // Upstream failed -- fall back to cache if available if !p.CacheMetadata || entry == nil { - return nil, "", "", fmt.Errorf("upstream failed and no cached metadata: %w", err) + return metadataResult{err: fmt.Errorf("upstream failed and no cached metadata: %w", err)} } p.Logger.Warn("upstream metadata fetch failed, checking cache", @@ -1126,12 +1172,86 @@ func (p *Proxy) fetchOrCacheMetadata(ctx context.Context, ecosystem, cacheKey, u data, ct, readErr := p.readCachedMetadata(ctx, entry, validate) if readErr != nil { - return nil, "", "", fmt.Errorf("upstream failed and cached metadata unusable (%v): %w", readErr, err) + return metadataResult{err: fmt.Errorf("upstream failed and cached metadata unusable (%v): %w", readErr, err)} } p.Logger.Info("serving metadata from cache", "ecosystem", ecosystem, "key", cacheKey) - return data, ct, entry.ContentEncoding.String, nil + return metadataResult{body: data, contentType: ct, contentEncoding: entry.ContentEncoding.String} +} + +// metadataResult is what a metadata lookup hands back. A shared fetch gives +// every waiter the same body, so callers must treat it as read-only. +type metadataResult struct { + body []byte + contentType string + contentEncoding string + err error +} + +// inflightMetadata is one metadata fetch that concurrent callers share. res is +// written before done closes and read only after, so the close is the handoff. +type inflightMetadata struct { + done chan struct{} + res metadataResult + + // waiters counts callers that joined the fetch, guarded by metaMu. Tests + // read it to know every caller has joined before the fetch finishes. + waiters int +} + +// metadataCoalesceKey identifies metadata requests that can share one fetch. +// Accept and Accept-Encoding are part of it because they change the bytes +// upstream returns: npm serves an abbreviated or a full document for the same +// package, and the cached-proxy paths ask for identity or gzip. Whether the +// caller validates is part of it so an unvalidated caller is never handed a +// result that skipped validation, or the reverse. +func metadataCoalesceKey(ecosystem, cacheKey, upstreamURL, accept, acceptEncoding string, validated bool) string { + return strings.Join([]string{ecosystem, cacheKey, upstreamURL, accept, acceptEncoding, strconv.FormatBool(validated)}, "\x00") +} + +// coalesceMetadata runs fetch at most once for concurrent callers sharing key +// and gives each the result, reporting whether this caller joined another's +// fetch rather than running it. It follows coalesceFetch with one difference: +// fetch runs on a context detached from the first caller's cancellation. A +// metadata fetch has no scan or mirror that depends on the caller aborting it, +// and CI jobs asking for the same Composer or npm metadata would otherwise all +// fail when the first of them disconnects. It stays bounded by the HTTP +// client's timeout. Waiters still leave when their own context ends. +func (p *Proxy) coalesceMetadata(ctx context.Context, key string, fetch func(context.Context) metadataResult) (metadataResult, bool) { + p.metaMu.Lock() + if p.inFlightMeta == nil { + p.inFlightMeta = make(map[string]*inflightMetadata) + } + f, joined := p.inFlightMeta[key] + if joined { + f.waiters++ + } else { + f = &inflightMetadata{done: make(chan struct{})} + p.inFlightMeta[key] = f + } + p.metaMu.Unlock() + + if joined { + select { + case <-ctx.Done(): + return metadataResult{err: ctx.Err()}, true + case <-f.done: + return f.res, true + } + } + + // Set before running so a panicking fetch leaves waiters with an error + // rather than an empty body. + f.res = metadataResult{err: errSharedFetchAbandoned} + defer func() { + p.metaMu.Lock() + delete(p.inFlightMeta, key) + p.metaMu.Unlock() + close(f.done) + }() + f.res = fetch(context.WithoutCancel(ctx)) + return f.res, false } func (p *Proxy) readCachedMetadata(ctx context.Context, entry *database.MetadataCacheEntry, validate func([]byte) error) ([]byte, string, error) { diff --git a/internal/handler/metadata_coalesce_test.go b/internal/handler/metadata_coalesce_test.go new file mode 100644 index 0000000..af8e2eb --- /dev/null +++ b/internal/handler/metadata_coalesce_test.go @@ -0,0 +1,249 @@ +package handler + +import ( + "context" + "errors" + "net/http" + "net/http/httptest" + "sync" + "sync/atomic" + "testing" + "time" +) + +// metadataUpstream is a stub registry that counts metadata requests. It +// signals entered when the first request arrives and holds every request until +// release is closed, so a test can keep the first fetch in flight while others +// arrive. +type metadataUpstream struct { + *httptest.Server + calls atomic.Int64 + entered chan struct{} + release chan struct{} + once sync.Once +} + +func newMetadataUpstream(t *testing.T, status int) *metadataUpstream { + t.Helper() + u := &metadataUpstream{entered: make(chan struct{}), release: make(chan struct{})} + u.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + u.calls.Add(1) + u.once.Do(func() { close(u.entered) }) + <-u.release + if status != http.StatusOK { + w.WriteHeader(status) + return + } + w.Header().Set("Content-Type", "application/json") + // Echo the Accept header so a test can tell which variant it got. + _, _ = w.Write([]byte(`{"accept":"` + r.Header.Get("Accept") + `"}`)) + })) + t.Cleanup(u.Close) + return u +} + +func metadataTestProxy(t *testing.T, u *metadataUpstream) *Proxy { + t.Helper() + proxy, _, _, _ := setupTestProxy(t) + proxy.HTTPClient = u.Client() + return proxy +} + +// waitForMetadataWaiters returns once n callers are waiting on the fetch for +// key, so a test can release upstream knowing every caller joined it. +func waitForMetadataWaiters(t *testing.T, p *Proxy, key string, n int) { + t.Helper() + deadline := time.Now().Add(5 * time.Second) + for time.Now().Before(deadline) { + p.metaMu.Lock() + f := p.inFlightMeta[key] + joined := 0 + if f != nil { + joined = f.waiters + } + p.metaMu.Unlock() + if joined >= n { + return + } + time.Sleep(time.Millisecond) + } + t.Fatalf("timed out waiting for %d callers to join the metadata fetch", n) +} + +type metadataCall struct { + body []byte + err error +} + +// startMetadataCalls starts n FetchOrCacheMetadata calls for one package. +func startMetadataCalls(p *Proxy, ctx context.Context, n int, url, accept string) []chan metadataCall { + out := make([]chan metadataCall, n) + for i := range out { + out[i] = make(chan metadataCall, 1) + go func(ch chan metadataCall) { + body, _, err := p.FetchOrCacheMetadata(ctx, "npm", "left-pad", url, accept) + ch <- metadataCall{body, err} + }(out[i]) + } + return out +} + +// TestFetchOrCacheMetadata_ConcurrentMissesCoalesce asserts that concurrent +// misses for one package make a single upstream request: the CI shape, where +// parallel jobs resolve the same Composer or npm dependencies at once. +func TestFetchOrCacheMetadata_ConcurrentMissesCoalesce(t *testing.T) { + u := newMetadataUpstream(t, http.StatusOK) + p := metadataTestProxy(t, u) + const n = 10 + + calls := startMetadataCalls(p, context.Background(), 1, u.URL, contentTypeJSON) + <-u.entered + calls = append(calls, startMetadataCalls(p, context.Background(), n-1, u.URL, contentTypeJSON)...) + key := metadataCoalesceKey("npm", "left-pad", u.URL, contentTypeJSON, "", false) + waitForMetadataWaiters(t, p, key, n-1) + close(u.release) + + for _, ch := range calls { + c := <-ch + if c.err != nil { + t.Fatalf("FetchOrCacheMetadata: %v", c.err) + } + if string(c.body) != `{"accept":"application/json"}` { + t.Errorf("body = %s", c.body) + } + } + if got := u.calls.Load(); got != 1 { + t.Errorf("upstream requests = %d, want 1", got) + } +} + +// TestFetchOrCacheMetadata_DifferentAcceptDoNotShare asserts that requests +// for different variants of one package, such as npm's abbreviated and full +// documents, each get their own fetch and their own bytes. +func TestFetchOrCacheMetadata_DifferentAcceptDoNotShare(t *testing.T) { + u := newMetadataUpstream(t, http.StatusOK) + p := metadataTestProxy(t, u) + const abbreviated = "application/vnd.npm.install-v1+json" + + full := startMetadataCalls(p, context.Background(), 1, u.URL, contentTypeJSON) + <-u.entered + abbr := startMetadataCalls(p, context.Background(), 1, u.URL, abbreviated) + deadline := time.Now().Add(5 * time.Second) + for u.calls.Load() < 2 && time.Now().Before(deadline) { + time.Sleep(time.Millisecond) + } + close(u.release) + + if c := <-full[0]; c.err != nil || string(c.body) != `{"accept":"application/json"}` { + t.Errorf("full document: body %s, err %v", c.body, c.err) + } + if c := <-abbr[0]; c.err != nil || string(c.body) != `{"accept":"`+abbreviated+`"}` { + t.Errorf("abbreviated document: body %s, err %v", c.body, c.err) + } + if got := u.calls.Load(); got != 2 { + t.Errorf("upstream requests = %d, want 2", got) + } +} + +// TestFetchOrCacheMetadata_FirstCallerLeavingDoesNotFailOthers asserts that +// when the caller running the shared fetch disconnects, the fetch continues +// and everyone waiting on it still gets the metadata. +func TestFetchOrCacheMetadata_FirstCallerLeavingDoesNotFailOthers(t *testing.T) { + u := newMetadataUpstream(t, http.StatusOK) + p := metadataTestProxy(t, u) + const n = 5 + + leaderCtx, cancelLeader := context.WithCancel(context.Background()) + leader := startMetadataCalls(p, leaderCtx, 1, u.URL, contentTypeJSON) + <-u.entered + waiters := startMetadataCalls(p, context.Background(), n, u.URL, contentTypeJSON) + key := metadataCoalesceKey("npm", "left-pad", u.URL, contentTypeJSON, "", false) + waitForMetadataWaiters(t, p, key, n) + cancelLeader() + close(u.release) + + <-leader[0] + for _, ch := range waiters { + if c := <-ch; c.err != nil { + t.Errorf("waiter failed after the first caller left: %v", c.err) + } + } + if got := u.calls.Load(); got != 1 { + t.Errorf("upstream requests = %d, want 1", got) + } +} + +// TestFetchOrCacheMetadata_WaiterLeavingKeepsFetch asserts that a waiter whose +// client disconnects returns its own context error without disturbing the +// fetch the others are waiting on. +func TestFetchOrCacheMetadata_WaiterLeavingKeepsFetch(t *testing.T) { + u := newMetadataUpstream(t, http.StatusOK) + p := metadataTestProxy(t, u) + + leader := startMetadataCalls(p, context.Background(), 1, u.URL, contentTypeJSON) + <-u.entered + waiterCtx, cancelWaiter := context.WithCancel(context.Background()) + leaving := startMetadataCalls(p, waiterCtx, 1, u.URL, contentTypeJSON) + key := metadataCoalesceKey("npm", "left-pad", u.URL, contentTypeJSON, "", false) + waitForMetadataWaiters(t, p, key, 1) + cancelWaiter() + + if c := <-leaving[0]; !errors.Is(c.err, context.Canceled) { + t.Errorf("leaving waiter err = %v, want context.Canceled", c.err) + } + close(u.release) + if c := <-leader[0]; c.err != nil { + t.Errorf("first caller failed: %v", c.err) + } +} + +// TestFetchOrCacheMetadata_SharedNotFound asserts that an upstream 404 reaches +// every caller sharing the fetch as ErrUpstreamNotFound, from one request. +func TestFetchOrCacheMetadata_SharedNotFound(t *testing.T) { + u := newMetadataUpstream(t, http.StatusNotFound) + p := metadataTestProxy(t, u) + const n = 5 + + calls := startMetadataCalls(p, context.Background(), 1, u.URL, contentTypeJSON) + <-u.entered + calls = append(calls, startMetadataCalls(p, context.Background(), n-1, u.URL, contentTypeJSON)...) + key := metadataCoalesceKey("npm", "left-pad", u.URL, contentTypeJSON, "", false) + waitForMetadataWaiters(t, p, key, n-1) + close(u.release) + + for _, ch := range calls { + if c := <-ch; !errors.Is(c.err, ErrUpstreamNotFound) { + t.Errorf("err = %v, want ErrUpstreamNotFound", c.err) + } + } + if got := u.calls.Load(); got != 1 { + t.Errorf("upstream requests = %d, want 1", got) + } +} + +// TestCoalescedMetadataMiss_ServesRowCommittedSinceLookup asserts that a +// caller which missed the cache, but finds the row fresh once it takes the +// key, serves that row instead of fetching again. That is a fetch finishing +// between a caller's lookup and its turn at the key. +func TestCoalescedMetadataMiss_ServesRowCommittedSinceLookup(t *testing.T) { + u := newMetadataUpstream(t, http.StatusOK) + close(u.release) + p := metadataTestProxy(t, u) + p.CacheMetadata = true + p.MetadataTTL = time.Hour + + if _, _, err := p.FetchOrCacheMetadata(context.Background(), "npm", "left-pad", u.URL); err != nil { + t.Fatalf("priming fetch: %v", err) + } + + res := p.coalescedMetadataMiss(context.Background(), "npm", "left-pad", u.URL, contentTypeJSON, "", nil) + if res.err != nil { + t.Fatalf("coalescedMetadataMiss: %v", res.err) + } + if string(res.body) != `{"accept":"application/json"}` { + t.Errorf("body = %s", res.body) + } + if got := u.calls.Load(); got != 1 { + t.Errorf("upstream requests = %d, want 1 (the priming fetch only)", got) + } +} diff --git a/internal/handler/nuget_cooldown_test.go b/internal/handler/nuget_cooldown_test.go index 74258da..3160bd5 100644 --- a/internal/handler/nuget_cooldown_test.go +++ b/internal/handler/nuget_cooldown_test.go @@ -3,12 +3,15 @@ package handler import ( "bytes" "compress/gzip" + "crypto/sha256" "encoding/json" "errors" + "fmt" "io" "net/http" "net/http/httptest" "strings" + "sync" "sync/atomic" "testing" "time" @@ -170,6 +173,57 @@ func TestNuGetCooldownColdDownload(t *testing.T) { } } +// TestNuGetCooldownConcurrentDownloads asserts that a download which joins +// another request's metadata fetch still applies the cooldown. Each caller +// decodes the leaf in its own validate callback, so a waiter on the shared +// fetch must run its own rather than finding no document and allowing it. +func TestNuGetCooldownConcurrentDownloads(t *testing.T) { + p, db, store, fetcher := setupTestProxy(t) + p.Cooldown = &cooldown.Config{Default: "14d"} + seedPackage(t, db, store, "nuget", "testpkg", "2.0.0", "testpkg.2.0.0.nupkg", "cached package") + entered, release := make(chan struct{}), make(chan struct{}) + var enteredOnce, releaseOnce sync.Once + releaseUpstream := func() { releaseOnce.Do(func() { close(release) }) } + var calls atomic.Int64 + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls.Add(1) + enteredOnce.Do(func() { close(entered) }) + <-release + _ = json.NewEncoder(w).Encode(map[string]string{"published": time.Now().Add(-time.Hour).Format(time.RFC3339)}) + })) + // Cleanups run last first: release the held request before closing the + // server, so a failed wait can't hang the test. + t.Cleanup(upstream.Close) + t.Cleanup(releaseUpstream) + routes := NewNuGetHandlerWithUpstreams(p, "http://proxy.test", upstream.URL, upstream.URL).Routes() + + statuses := make(chan int, 2) + download := func() { + w := httptest.NewRecorder() + routes.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/v3-flatcontainer/testpkg/2.0.0/testpkg.2.0.0.nupkg", nil)) + statuses <- w.Code + } + go download() + <-entered + go download() + target := upstream.URL + nugetRegistrationPath + "testpkg/2.0.0.json" + key := metadataCoalesceKey("nuget", fmt.Sprintf("_cooldown/%x", sha256.Sum256([]byte(target))), target, contentTypeJSON, "", true) + waitForMetadataWaiters(t, p, key, 1) + releaseUpstream() + + for range 2 { + if status := <-statuses; status != http.StatusNotFound { + t.Errorf("download status = %d, want %d", status, http.StatusNotFound) + } + } + if got := calls.Load(); got != 1 { + t.Errorf("upstream metadata requests = %d, want 1", got) + } + if fetcher.fetchCalled { + t.Error("blocked downloads must not fetch artifacts") + } +} + func TestNuGetRegistrationServiceAliases(t *testing.T) { h := NewNuGetHandler(nugetTestProxy(), "http://proxy.test") for _, tt := range []struct{ service, path string }{ From 34985b54d9567181129a2cc961f3c2dc37b842ab Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Sun, 4 Oct 2026 18:05:42 +0100 Subject: [PATCH 175/633] Support npm content-addressed tarball URLs --- internal/handler/npm.go | 95 ++++++++++++++--- internal/handler/npm_tarball_test.go | 154 +++++++++++++++++++++++++++ internal/handler/npm_test.go | 28 ++--- 3 files changed, 240 insertions(+), 37 deletions(-) create mode 100644 internal/handler/npm_tarball_test.go diff --git a/internal/handler/npm.go b/internal/handler/npm.go index 663c8c6..98ae10a 100644 --- a/internal/handler/npm.go +++ b/internal/handler/npm.go @@ -10,6 +10,7 @@ import ( "net/url" "sort" "strings" + "sync" "time" ) @@ -315,6 +316,13 @@ func (h *NPMHandler) rewriteTarballURLs(versions map[string]any, packageName str if idx := strings.LastIndex(tarball, "/"); idx >= 0 { filename = tarball[idx+1:] } + if h.extractVersionFromFilename(packageName, filename) != version { + _, shortName, scoped := strings.Cut(packageName, "/") + if !scoped { + shortName = packageName + } + filename = shortName + "-" + version + ".tgz" + } escapedName := url.PathEscape(packageName) newTarball := fmt.Sprintf("%s/npm/%s/-/%s", h.proxyURL, escapedName, filename) @@ -377,22 +385,23 @@ func (h *NPMHandler) handleDownload(w http.ResponseWriter, r *http.Request) { h.proxy.Logger.Info("npm download request", "package", packageName, "version", version, "filename", filename) - if h.versionInCooldown(r, packageName, version) { + if h.proxy.versionDenied("npm", packageName, version) { + JSONError(w, http.StatusForbidden, ErrVersionDenied.Error()+": "+canonicalVersionPURL("npm", packageName, version)) + return + } + metadata := sync.OnceValues(func() ([]byte, error) { + upstreamURL := h.upstreamURL + "/" + url.PathEscape(packageName) + body, _, err := h.proxy.FetchOrCacheMetadata(r.Context(), "npm", packageName, upstreamURL, contentTypeJSON) + return body, err + }) + if h.versionInCooldown(packageName, version, metadata) { h.proxy.Logger.Info("cooldown: withholding npm tarball", "package", packageName, "version", version) JSONError(w, http.StatusNotFound, "version not found") return } - downloadURL := fmt.Sprintf( - "%s/%s/-/%s", - h.upstreamURL, - escapeNPMDownloadPackage(packageName), - url.PathEscape(filename), - ) - result, err := h.proxy.GetOrFetchArtifactFromURL( - r.Context(), "npm", packageName, version, filename, downloadURL, - ) + result, err := h.getTarball(r, packageName, version, filename, metadata) if err != nil { switch { case errors.Is(err, ErrUpstreamNotFound): @@ -409,6 +418,60 @@ func (h *NPMHandler) handleDownload(w http.ResponseWriter, r *http.Request) { ServeArtifactRequest(w, r, result) } +func (h *NPMHandler) getTarball(r *http.Request, packageName, version, filename string, metadata func() ([]byte, error)) (*CacheResult, error) { + if cached, err := h.proxy.GetCachedArtifact(r.Context(), "npm", packageName, version, filename); err != nil || cached != nil { + return cached, err + } + downloadURL := fmt.Sprintf("%s/%s/-/%s", h.upstreamURL, escapeNPMDownloadPackage(packageName), url.PathEscape(filename)) + body, err := metadata() + if err == nil { + if tarball := npmVersionTarball(body, version); tarball != "" { + downloadURL, err = h.validateTarballURL(tarball) + if err != nil { + return nil, err + } + } + } + return h.proxy.GetOrFetchArtifactFromURL(r.Context(), "npm", packageName, version, filename, downloadURL) +} + +func npmVersionTarball(body []byte, version string) string { + var metadata struct { + Versions map[string]struct { + Dist struct { + Tarball string `json:"tarball"` + } `json:"dist"` + } `json:"versions"` + } + if err := json.Unmarshal(body, &metadata); err != nil { + return "" + } + return metadata.Versions[version].Dist.Tarball +} + +func (h *NPMHandler) validateTarballURL(raw string) (string, error) { + tarball, err := url.Parse(raw) + if err != nil { + return "", fmt.Errorf("parsing npm tarball URL: %w", err) + } + upstream, err := url.Parse(h.upstreamURL) + if err != nil { + return "", fmt.Errorf("parsing npm upstream URL: %w", err) + } + if tarball.User != nil || tarball.Fragment != "" || + tarball.Scheme != upstream.Scheme || !strings.EqualFold(tarball.Host, upstream.Host) { + return "", errors.New("npm tarball URL does not match upstream registry") + } + if containsPathTraversal(tarball.Path) || strings.Contains(tarball.Path, "\\") { + return "", errors.New("npm tarball URL contains path traversal") + } + basePath := strings.TrimRight(upstream.Path, "/") + if basePath != "" && !strings.HasPrefix(tarball.Path, basePath+"/") { + return "", errors.New("npm tarball URL is outside upstream base path") + } + return tarball.String(), nil +} + // versionInCooldown reports whether a version is still inside the cooldown // window. Filtering the packument is not enough on its own: tarball URLs are // predictable and lockfiles record them directly, so `npm ci` reaches the @@ -420,7 +483,7 @@ func (h *NPMHandler) handleDownload(w http.ResponseWriter, r *http.Request) { // fetched and parsed at most once per version. A version with no usable // publish time is allowed through, matching how applyCooldownFiltering // treats it. -func (h *NPMHandler) versionInCooldown(r *http.Request, packageName, version string) bool { +func (h *NPMHandler) versionInCooldown(packageName, version string, metadata func() ([]byte, error)) bool { if h.proxy.Cooldown == nil || !h.proxy.Cooldown.Enabled() { return false } @@ -430,25 +493,23 @@ func (h *NPMHandler) versionInCooldown(r *http.Request, packageName, version str return !h.proxy.Cooldown.IsAllowed("npm", canonicalPackagePURL("npm", packageName), ver.PublishedAt.Time) } - upstreamURL := fmt.Sprintf("%s/%s", h.upstreamURL, url.PathEscape(packageName)) - - body, _, err := h.proxy.FetchOrCacheMetadata(r.Context(), "npm", packageName, upstreamURL, contentTypeJSON) + body, err := metadata() if err != nil { h.proxy.Logger.Warn("cooldown: could not fetch npm metadata for download check", "package", packageName, "version", version, "error", err) return false } - var metadata struct { + var document struct { Time map[string]string `json:"time"` } - if err := json.Unmarshal(body, &metadata); err != nil { + if err := json.Unmarshal(body, &document); err != nil { h.proxy.Logger.Warn("cooldown: could not parse npm metadata for download check", "package", packageName, "version", version, "error", err) return false } - published, ok := metadata.Time[version] + published, ok := document.Time[version] if !ok { return false } diff --git a/internal/handler/npm_tarball_test.go b/internal/handler/npm_tarball_test.go new file mode 100644 index 0000000..303f218 --- /dev/null +++ b/internal/handler/npm_tarball_test.go @@ -0,0 +1,154 @@ +package handler + +import ( + "encoding/json" + "fmt" + "io" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/git-pkgs/cooldown" + "github.com/git-pkgs/registries/fetch" +) + +func TestNPMContentAddressedTarball(t *testing.T) { + for _, tc := range []struct { + cacheMetadata, stream bool + metadata, tarballs int64 + }{ + {false, false, 2, 1}, + {false, true, 3, 2}, + {true, false, 1, 1}, + {true, true, 1, 2}, + } { + t.Run(fmt.Sprintf("metadata=%t/stream=%t", tc.cacheMetadata, tc.stream), func(t *testing.T) { + var metadataCalls, tarballCalls atomic.Int64 + upstream := npmContentAddressedRegistry(t, &metadataCalls, &tarballCalls) + p, _, _, _ := setupTestProxy(t) + p.CacheMetadata = tc.cacheMetadata + p.MetadataTTL = time.Hour + p.StreamArtifacts = tc.stream + p.Cooldown = &cooldown.Config{Default: "7d"} + p.HTTPClient = upstream.Client() + fetcher := fetch.NewFetcher(fetch.WithHTTPClient(upstream.Client()), fetch.WithMaxRetries(0)) + p.Fetcher = fetcher + t.Cleanup(func() { _ = fetcher.Close() }) + h := NewNPMHandler(p, "http://proxy.test", upstream.URL+"/registry") + routes := http.StripPrefix("/npm", h.Routes()) + metadata := httptest.NewRecorder() + routes.ServeHTTP(metadata, httptest.NewRequest(http.MethodGet, "/npm/@example/widget", nil)) + if metadata.Code != http.StatusOK { + t.Fatalf("metadata status = %d: %s", metadata.Code, metadata.Body.String()) + } + tarball := npmVersionTarball(metadata.Body.Bytes(), testVersion100) + if want := "http://proxy.test/npm/@example%2Fwidget/-/widget-1.0.0.tgz"; tarball != want { + t.Fatalf("rewritten tarball = %q, want %q", tarball, want) + } + for i := range 2 { + response := httptest.NewRecorder() + routes.ServeHTTP(response, httptest.NewRequest(http.MethodGet, tarball, nil)) + if response.Code != http.StatusOK || response.Body.String() != "tarball bytes" { + t.Fatalf("download %d = %d %q", i, response.Code, response.Body.String()) + } + if !tc.stream { + upstream.Close() + } + } + if metadataCalls.Load() != tc.metadata || tarballCalls.Load() != tc.tarballs { + t.Errorf("upstream requests: metadata=%d tarballs=%d, want %d %d", metadataCalls.Load(), tarballCalls.Load(), tc.metadata, tc.tarballs) + } + }) + } +} + +func npmContentAddressedRegistry(t *testing.T, metadataCalls, tarballCalls *atomic.Int64) *httptest.Server { + t.Helper() + var upstream *httptest.Server + upstream = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/registry/@example/widget": + metadataCalls.Add(1) + w.Header().Set("Content-Type", contentTypeJSON) + _, _ = fmt.Fprintf(w, `{"name":"@example/widget","time":{"1.0.0":%q},"versions":{"1.0.0":{"dist":{"tarball":%q}}}}`, + time.Now().Add(-30*24*time.Hour).Format(time.RFC3339), upstream.URL+"/registry/download/@example/widget/1.0.0/abc123?token=example") + case "/registry/download/@example/widget/1.0.0/abc123": + if r.URL.RawQuery != "token=example" { + t.Errorf("tarball query = %q", r.URL.RawQuery) + } + tarballCalls.Add(1) + _, _ = io.WriteString(w, "tarball bytes") + default: + t.Errorf("unexpected upstream request: %s", r.URL) + http.NotFound(w, r) + } + })) + t.Cleanup(upstream.Close) + return upstream +} + +func TestNPMTarballMetadataFallback(t *testing.T) { + for _, body := range []string{`{"versions":{}}`, `{"versions":{"1.0.0":{"dist":{}}}}`, `not json`, "unavailable"} { + t.Run(body, func(t *testing.T) { + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + if body == "unavailable" { + w.WriteHeader(http.StatusServiceUnavailable) + } + _, _ = io.WriteString(w, body) + })) + t.Cleanup(upstream.Close) + p, _, _, fetcher := setupTestProxy(t) + p.CacheMetadata = true + p.MetadataTTL = time.Hour + fetcher.artifact = &fetch.Artifact{Body: io.NopCloser(strings.NewReader("package"))} + h := NewNPMHandler(p, "http://proxy.test", upstream.URL) + if body != "unavailable" { + if _, _, err := p.FetchOrCacheMetadata(t.Context(), "npm", "@example/widget", upstream.URL); err != nil { + t.Fatal(err) + } + upstream.Close() + } + response := httptest.NewRecorder() + h.Routes().ServeHTTP(response, httptest.NewRequest(http.MethodGet, "/@example/widget/-/widget-1.0.0.tgz", nil)) + if response.Code != http.StatusOK { + t.Fatalf("status = %d: %s", response.Code, response.Body.String()) + } + if want := upstream.URL + "/@example/widget/-/widget-1.0.0.tgz"; fetcher.fetchedURL != want { + t.Errorf("fetched URL = %q, want %q", fetcher.fetchedURL, want) + } + }) + } +} + +func TestNPMTarballRejectsUnsafeMetadataURL(t *testing.T) { + for _, target := range []string{ + "https://outside.invalid/package.tgz", "//outside.invalid/package.tgz", "/registry/package.tgz", + "UPSTREAM/outside/package.tgz", "UPSTREAM/registry-other/package.tgz", + "UPSTREAM/registry/../outside/package.tgz", "UPSTREAM/registry/%2e%2e/outside/package.tgz", + "UPSTREAM/registry/%252e%252e/outside/package.tgz", "UPSTREAM/registry/..%2foutside/package.tgz", + "UPSTREAM/registry/..%5coutside/package.tgz", "UPSTREAM/registry/package.tgz#fragment", + "USERINFO/registry/package.tgz", "SCHEME/registry/package.tgz", + } { + t.Run(target, func(t *testing.T) { + var upstream *httptest.Server + upstream = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + address, _ := url.Parse(upstream.URL) + address.User = url.UserPassword("user", "secret") + tarball := strings.NewReplacer("UPSTREAM", upstream.URL, "USERINFO", address.String(), "SCHEME", strings.Replace(upstream.URL, "http:", "https:", 1)).Replace(target) + _ = json.NewEncoder(w).Encode(map[string]any{"versions": map[string]any{testVersion100: map[string]any{"dist": map[string]string{"tarball": tarball}}}}) + })) + t.Cleanup(upstream.Close) + p, _, _, fetcher := setupTestProxy(t) + h := NewNPMHandler(p, "http://proxy.test", upstream.URL+"/registry") + response := httptest.NewRecorder() + h.Routes().ServeHTTP(response, httptest.NewRequest(http.MethodGet, "/widget/-/widget-1.0.0.tgz", nil)) + if response.Code != http.StatusBadGateway || fetcher.fetchCalled { + t.Errorf("status = %d, fetched = %t; body: %s", response.Code, fetcher.fetchCalled, response.Body.String()) + } + }) + } +} diff --git a/internal/handler/npm_test.go b/internal/handler/npm_test.go index d0c3889..5d8378f 100644 --- a/internal/handler/npm_test.go +++ b/internal/handler/npm_test.go @@ -538,22 +538,13 @@ func TestNPMDownloadCooldown(t *testing.T) { } func TestNPMDownloadCooldownDisabled(t *testing.T) { - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - t.Error("metadata must not be fetched when cooldown is disabled") - w.WriteHeader(http.StatusInternalServerError) - })) - defer upstream.Close() + proxy, _, _, _ := setupTestProxy(t) + h := NewNPMHandler(proxy, "http://proxy.test", "") - proxy, _, _, fetcher := setupTestProxy(t) - proxy.HTTPClient = upstream.Client() - fetcher.artifact = &fetch.Artifact{ - Body: io.NopCloser(strings.NewReader("tarball data")), - ContentType: "application/octet-stream", - } - - h := NewNPMHandler(proxy, "http://proxy.test", upstream.URL) - - if h.versionInCooldown(httptest.NewRequest(http.MethodGet, "/", nil), "leftpad", testVersion100) { + if h.versionInCooldown("leftpad", testVersion100, func() ([]byte, error) { + t.Fatal("cooldown must not request metadata when disabled") + return nil, nil + }) { t.Error("versionInCooldown = true, want false when cooldown is not configured") } } @@ -577,13 +568,10 @@ func TestNPMDownloadCooldownUsesStoredPublishTime(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - proxy, db, _, fetcher := setupTestProxy(t) + proxy, db, store, _ := setupTestProxy(t) proxy.HTTPClient = upstream.Client() proxy.Cooldown = &cooldown.Config{Default: "7d"} - fetcher.artifact = &fetch.Artifact{ - Body: io.NopCloser(strings.NewReader("tarball data")), - ContentType: "application/octet-stream", - } + seedPackage(t, db, store, "npm", "leftpad", tt.version, "leftpad-"+tt.version+".tgz", "tarball data") if err := db.SetVersionPublishedAt("pkg:npm/leftpad@"+tt.version, "pkg:npm/leftpad", tt.publishedAt); err != nil { t.Fatalf("seeding publish time failed: %v", err) From 866e55928551ae8ee1ddbbacd229af449f3f7d3a Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Sun, 4 Oct 2026 20:48:41 +0100 Subject: [PATCH 176/633] Add cooldown package pattern overrides (#407) * Add cooldown package pattern overrides * Expose full cooldown decisions through pattern policy * Reject unsafe cooldown pattern syntax --- config.example.yaml | 4 + docs/configuration.md | 9 +- internal/config/config.go | 4 + internal/config/config_test.go | 5 + internal/cooldownpolicy/policy.go | 114 +++++++++++ internal/cooldownpolicy/policy_test.go | 223 +++++++++++++++++++++ internal/handler/cooldown_patterns_test.go | 105 ++++++++++ internal/handler/denylist_test.go | 2 +- internal/handler/handler.go | 10 +- internal/server/cooldown_patterns_test.go | 30 +++ internal/server/server.go | 7 +- internal/server/server_test.go | 33 +++ 12 files changed, 542 insertions(+), 4 deletions(-) create mode 100644 internal/cooldownpolicy/policy.go create mode 100644 internal/cooldownpolicy/policy_test.go create mode 100644 internal/handler/cooldown_patterns_test.go create mode 100644 internal/server/cooldown_patterns_test.go diff --git a/config.example.yaml b/config.example.yaml index 44a3bfc..49b81ef 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -332,6 +332,10 @@ cooldown: # "pkg:npm/lodash": "0" # "pkg:npm/@babel/core": "14d" + # Per-package glob overrides, after exact packages and before ecosystems. + # package_patterns: + # "pkg:npm/@example/*": "0" + # Exact versions to deny, independently of cooldown and scanning. # Metadata filtering: npm, PyPI and Cargo. Shared artifact downloads, including # cache hits, are blocked with 403; signed APT metadata is left unchanged. diff --git a/docs/configuration.md b/docs/configuration.md index ed5ad40..8261eba 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -458,6 +458,8 @@ cooldown: packages: "pkg:npm/lodash": "0" "pkg:npm/@babel/core": "14d" + package_patterns: + "pkg:npm/@example/*": "0" ``` | Config | Environment | Description | @@ -465,12 +467,17 @@ cooldown: | `cooldown.default` | `PROXY_COOLDOWN_DEFAULT` | Global default cooldown | | `cooldown.ecosystems` | - | Per-ecosystem overrides | | `cooldown.packages` | - | Per-package overrides (keyed by PURL) | +| `cooldown.package_patterns` | - | Per-package glob overrides (keyed by PURL glob) | Durations support days (`7d`), hours (`48h`), and minutes (`30m`). Set to `0` to disable. Package PURL keys are normalized to canonical form before matching, so `pkg:npm/@babel/core` and `pkg:npm/%40babel/core` are equivalent, as are `pkg:pypi/Django` and `pkg:pypi/django`. If both forms configure the same package, the canonical entry wins. -Resolution order: package override, then ecosystem override, then global default. This lets you set a conservative default while exempting trusted packages. +`package_patterns` matches canonical, versionless PURLs using `*` for zero or more characters and `?` for one character. Neither wildcard crosses `/` separators. Character classes (`[...]`) and backslash escapes are rejected at startup. For example, `"pkg:npm/@example/*"` matches packages under the `@example` npm scope. Patterns accept `@` as an alias for `%40`; other characters must use their canonical PURL form. Equivalent patterns with different durations are rejected at startup. Equal durations, such as `1d` and `24h`, are accepted. + +Exact `packages` entries take precedence over patterns. When several patterns match, longer patterns win after excluding `*` and `?` from the length. Ties use lexical order of the normalized patterns. + +Resolution order: exact package override, then package pattern, then ecosystem override, then global default. This lets you set a conservative default while exempting trusted package families. Currently supported for npm, PyPI, pub.dev, Composer, Cargo, NuGet, Conda, RubyGems, and Hex. These ecosystems include publish timestamps in their metadata. diff --git a/internal/config/config.go b/internal/config/config.go index 05ed6d2..54f826f 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -192,6 +192,10 @@ type CooldownConfig struct { // Packages overrides the cooldown for specific packages (keyed by PURL). // Valid PURL keys are normalized to canonical form before use. Packages map[string]string `json:"packages" yaml:"packages"` + + // PackagePatterns overrides the cooldown for packages whose PURLs match a glob. + // Exact package overrides take precedence over matching patterns. + PackagePatterns map[string]string `json:"package_patterns" yaml:"package_patterns"` } // NormalizedPackages returns a copy of the package overrides with valid PURL diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 2006970..b1818ce 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -562,6 +562,8 @@ cooldown: packages: "pkg:npm/lodash": "0" "pkg:npm/@babel/core": "14d" + package_patterns: + "pkg:npm/@example/*": "0" ` if err := os.WriteFile(path, []byte(content), 0644); err != nil { t.Fatalf("writing config file: %v", err) @@ -590,6 +592,9 @@ cooldown: if got := cfg.Cooldown.NormalizedPackages()["pkg:npm/%40babel/core"]; got != "14d" { t.Errorf("normalized Cooldown.Packages[@babel/core] = %q, want %q", got, "14d") } + if cfg.Cooldown.PackagePatterns["pkg:npm/@example/*"] != "0" { + t.Errorf("Cooldown.PackagePatterns[example] = %q, want %q", cfg.Cooldown.PackagePatterns["pkg:npm/@example/*"], "0") + } } func TestCooldownConfigNormalizedPackages(t *testing.T) { diff --git a/internal/cooldownpolicy/policy.go b/internal/cooldownpolicy/policy.go new file mode 100644 index 0000000..096c4a4 --- /dev/null +++ b/internal/cooldownpolicy/policy.go @@ -0,0 +1,114 @@ +// Package cooldownpolicy applies package-pattern overrides to cooldown checks. +package cooldownpolicy + +import ( + "fmt" + "path" + "sort" + "strings" + "time" + + "github.com/git-pkgs/cooldown" +) + +// Policy applies exact PURL overrides before package-pattern overrides. +type Policy struct { + base *cooldown.Config + patterns []pattern + enabled bool +} + +type pattern struct { + glob string + duration time.Duration + config *cooldown.Config +} + +// New creates a Policy using the supplied exact and pattern overrides. +func New(base *cooldown.Config, packagePatterns map[string]string) (*Policy, error) { + if base == nil { + base = &cooldown.Config{} + } + + keys := make([]string, 0, len(packagePatterns)) + for glob := range packagePatterns { + keys = append(keys, glob) + } + sort.Strings(keys) + patterns := make([]pattern, 0, len(packagePatterns)) + seen := make(map[string]pattern) + enabled := base.Enabled() + for _, glob := range keys { + value := packagePatterns[glob] + if strings.ContainsAny(glob, "[\\") { + return nil, fmt.Errorf("invalid cooldown package pattern %q: character classes and escapes are not supported", glob) + } + canonicalGlob := strings.ReplaceAll(glob, "@", "%40") + if _, err := path.Match(canonicalGlob, ""); err != nil { + return nil, fmt.Errorf("invalid cooldown package pattern %q: %w", glob, err) + } + duration, err := cooldown.ParseDuration(value) + if err != nil { + return nil, fmt.Errorf("invalid cooldown duration for package pattern %q: %w", glob, err) + } + if previous, exists := seen[canonicalGlob]; exists { + if previous.duration != duration { + return nil, fmt.Errorf("conflicting cooldown package patterns %q and %q", previous.glob, glob) + } + continue + } + seen[canonicalGlob] = pattern{glob: glob, duration: duration} + config := &cooldown.Config{Default: value} + enabled = config.Enabled() || enabled + patterns = append(patterns, pattern{glob: canonicalGlob, duration: duration, config: config}) + } + sort.Slice(patterns, func(i, j int) bool { + left, right := literalLength(patterns[i].glob), literalLength(patterns[j].glob) + if left != right { + return left > right + } + return patterns[i].glob < patterns[j].glob + }) + + return &Policy{base: base, patterns: patterns, enabled: enabled}, nil +} + +func literalLength(glob string) int { + return len(glob) - strings.Count(glob, "*") - strings.Count(glob, "?") +} + +// For returns the duration, with exact overrides taking precedence over patterns. +func (p *Policy) For(ecosystem, packagePURL string) time.Duration { + return p.configFor(packagePURL).For(ecosystem, packagePURL) +} + +func (p *Policy) configFor(packagePURL string) *cooldown.Config { + if _, exact := p.base.Packages[packagePURL]; exact { + return p.base + } + + for _, candidate := range p.patterns { + matched, _ := path.Match(candidate.glob, packagePURL) + if !matched { + continue + } + return candidate.config + } + + return p.base +} + +// IsAllowed reports whether the package version has completed its cooldown. +func (p *Policy) IsAllowed(ecosystem, packagePURL string, publishedAt time.Time) bool { + return p.Evaluate(ecosystem, packagePURL, publishedAt, time.Now()).Allowed +} + +// Evaluate returns the cooldown decision at the supplied evaluation time. +func (p *Policy) Evaluate(ecosystem, packagePURL string, publishedAt, evaluatedAt time.Time) cooldown.Decision { + return p.configFor(packagePURL).Evaluate(ecosystem, packagePURL, publishedAt, evaluatedAt) +} + +// Enabled reports whether any configured cooldown can filter a package version. +func (p *Policy) Enabled() bool { + return p.enabled +} diff --git a/internal/cooldownpolicy/policy_test.go b/internal/cooldownpolicy/policy_test.go new file mode 100644 index 0000000..9e956c1 --- /dev/null +++ b/internal/cooldownpolicy/policy_test.go @@ -0,0 +1,223 @@ +package cooldownpolicy + +import ( + "strings" + "testing" + "time" + + "github.com/git-pkgs/cooldown" +) + +func TestPatternOverride(t *testing.T) { + policy, err := New(&cooldown.Config{ + Default: "7d", + Ecosystems: map[string]string{"npm": "7d"}, + }, map[string]string{ + "pkg:npm/@example/*": "0", + }) + if err != nil { + t.Fatalf("New returned error: %v", err) + } + + if !policy.IsAllowed("npm", "pkg:npm/%40example/widget", time.Now()) { + t.Fatal("matching package pattern should disable cooldown") + } + if policy.IsAllowed("npm", "pkg:npm/public-package", time.Now()) { + t.Fatal("non-matching package should use ecosystem cooldown") + } +} + +func TestExactOverrideTakesPrecedenceOverPattern(t *testing.T) { + purl := "pkg:npm/%40example/widget" + policy, err := New(&cooldown.Config{ + Default: "7d", + Packages: map[string]string{purl: "2d"}, + }, map[string]string{ + "pkg:npm/@example/*": "0", + }) + if err != nil { + t.Fatalf("New returned error: %v", err) + } + + if policy.IsAllowed("npm", purl, time.Now()) { + t.Fatal("exact package override should take precedence over pattern") + } +} + +func TestMoreSpecificPatternTakesPrecedence(t *testing.T) { + policy, err := New(&cooldown.Config{Default: "7d"}, map[string]string{ + "pkg:npm/@example/*": "0", + "pkg:npm/@example/critical": "2d", + }) + if err != nil { + t.Fatalf("New returned error: %v", err) + } + + if policy.IsAllowed("npm", "pkg:npm/%40example/critical", time.Now()) { + t.Fatal("more specific pattern should take precedence") + } +} + +func TestNewRejectsInvalidPattern(t *testing.T) { + for _, glob := range []string{"pkg:npm/[", "pkg:npm/[@a]*", "pkg:npm/[abcdef]*", `pkg:npm/\*`, `pkg:npm/\@example/*`} { + t.Run(glob, func(t *testing.T) { + if _, err := New(nil, map[string]string{glob: "0"}); err == nil || !strings.Contains(err.Error(), "character classes and escapes are not supported") { + t.Fatalf("unsupported pattern %q: error = %v", glob, err) + } + }) + } +} + +func TestQuestionMarkPattern(t *testing.T) { + policy, err := New(&cooldown.Config{Default: "7d"}, map[string]string{"pkg:npm/@example/widget-?": "0"}) + if err != nil { + t.Fatal(err) + } + for _, tc := range []struct { + purl string + want time.Duration + }{ + {"pkg:npm/%40example/widget-a", 0}, + {"pkg:npm/%40example/widget-ab", 7 * 24 * time.Hour}, + {"pkg:npm/%40example/widget-/", 7 * 24 * time.Hour}, + } { + if got := policy.For("npm", tc.purl); got != tc.want { + t.Errorf("For(%q) = %s, want %s", tc.purl, got, tc.want) + } + } +} + +func TestNormalizedPatternCollisions(t *testing.T) { + for _, duration := range []string{"0", "24h"} { + t.Run(duration, func(t *testing.T) { + for range 20 { + policy, err := New(nil, map[string]string{ + "pkg:npm/@example/*": "1d", + "pkg:npm/%40example/*": duration, + }) + if duration == "0" { + if err == nil || !strings.Contains(err.Error(), `conflicting cooldown package patterns "pkg:npm/%40example/*" and "pkg:npm/@example/*"`) { + t.Fatalf("conflict error = %v", err) + } + continue + } + if err != nil { + t.Fatal(err) + } + if got := policy.For("npm", "pkg:npm/%40example/widget"); got != 24*time.Hour { + t.Fatalf("duration = %s", got) + } + } + }) + } +} + +func TestForPrecedence(t *testing.T) { + policy, err := New(&cooldown.Config{ + Default: "1h", + Ecosystems: map[string]string{"npm": "2h"}, + Packages: map[string]string{"pkg:npm/%40example/exact": "0"}, + }, map[string]string{ + "pkg:npm/@example/*": "3h", + "pkg:npm/@example/specific*": "4h", + "pkg:npm/@example/ab*": "5h", + "pkg:npm/@example/a*c": "6h", + }) + if err != nil { + t.Fatal(err) + } + for _, tc := range []struct { + ecosystem, purl string + want time.Duration + }{ + {"npm", "pkg:npm/%40example/exact", 0}, + {"npm", "pkg:npm/%40example/widget", 3 * time.Hour}, + {"npm", "pkg:npm/%40example/specific-widget", 4 * time.Hour}, + {"npm", "pkg:npm/%40example/abc", 6 * time.Hour}, + {"npm", "pkg:npm/other", 2 * time.Hour}, + {"npm", "pkg:npm/%40example/nested/package", 2 * time.Hour}, + {"cargo", "pkg:cargo/serde", time.Hour}, + } { + t.Run(tc.purl, func(t *testing.T) { + if got := policy.For(tc.ecosystem, tc.purl); got != tc.want { + t.Errorf("For = %s, want %s", got, tc.want) + } + }) + } +} + +func TestPatternOnlyPolicy(t *testing.T) { + policy, err := New(nil, map[string]string{"pkg:nuget/example.*": "24h"}) + if err != nil { + t.Fatal(err) + } + if !policy.Enabled() { + t.Fatal("pattern-only policy should be enabled") + } + for _, tc := range []struct { + published time.Time + allowed bool + }{ + {time.Now().Add(-time.Hour), false}, + {time.Now().Add(-48 * time.Hour), true}, + {time.Time{}, true}, + } { + if got := policy.IsAllowed("nuget", "pkg:nuget/example.widget", tc.published); got != tc.allowed { + t.Errorf("published=%s: allowed=%t, want %t", tc.published, got, tc.allowed) + } + } + if !policy.IsAllowed("nuget", "pkg:nuget/other", time.Now()) { + t.Error("unmatched package should have no cooldown") + } +} + +func TestDisabledPolicy(t *testing.T) { + for _, patterns := range []map[string]string{nil, {"pkg:npm/@example/*": "0"}} { + policy, err := New(nil, patterns) + if err != nil { + t.Fatal(err) + } + if policy.Enabled() { + t.Error("zero cooldown should be disabled") + } + } +} + +func TestNewRejectsInvalidDuration(t *testing.T) { + if _, err := New(nil, map[string]string{"pkg:npm/*": "invalid"}); err == nil { + t.Fatal("invalid duration accepted") + } +} + +func TestEvaluate(t *testing.T) { + policy, err := New(&cooldown.Config{ + Default: "48h", + Ecosystems: map[string]string{"npm": "72h"}, + Packages: map[string]string{"pkg:npm/%40example/exact": "0"}, + }, map[string]string{"pkg:npm/@example/*": "7d"}) + if err != nil { + t.Fatal(err) + } + published := time.Date(2026, time.October, 1, 0, 0, 0, 0, time.UTC) + eligible := published.Add(7 * 24 * time.Hour) + for _, tc := range []struct { + name, ecosystem, purl string + published, at time.Time + want cooldown.Decision + }{ + {"waiting", "npm", "pkg:npm/%40example/widget", published, eligible.Add(-time.Nanosecond), cooldown.Decision{Cooldown: 7 * 24 * time.Hour, AvailableAt: eligible, Reason: cooldown.ReasonWaiting}}, + {"boundary", "npm", "pkg:npm/%40example/widget", published, eligible, cooldown.Decision{Allowed: true, Cooldown: 7 * 24 * time.Hour, AvailableAt: eligible, Reason: cooldown.ReasonElapsed}}, + {"elapsed", "npm", "pkg:npm/%40example/widget", published, eligible.Add(time.Hour), cooldown.Decision{Allowed: true, Cooldown: 7 * 24 * time.Hour, AvailableAt: eligible, Reason: cooldown.ReasonElapsed}}, + {"unknown publication", "npm", "pkg:npm/%40example/widget", time.Time{}, eligible, cooldown.Decision{Allowed: true, Cooldown: 7 * 24 * time.Hour, Reason: cooldown.ReasonUnknownPublicationTime}}, + {"exact exemption", "npm", "pkg:npm/%40example/exact", published, published, cooldown.Decision{Allowed: true, AvailableAt: published, Reason: cooldown.ReasonDisabled}}, + {"exemption without publication", "npm", "pkg:npm/%40example/exact", time.Time{}, eligible, cooldown.Decision{Allowed: true, Reason: cooldown.ReasonUnknownPublicationTime}}, + {"ecosystem fallback", "npm", "pkg:npm/other", published, published, cooldown.Decision{Cooldown: 72 * time.Hour, AvailableAt: published.Add(72 * time.Hour), Reason: cooldown.ReasonWaiting}}, + {"global fallback", "cargo", "pkg:cargo/serde", published, published, cooldown.Decision{Cooldown: 48 * time.Hour, AvailableAt: published.Add(48 * time.Hour), Reason: cooldown.ReasonWaiting}}, + } { + t.Run(tc.name, func(t *testing.T) { + if got := policy.Evaluate(tc.ecosystem, tc.purl, tc.published, tc.at); got != tc.want { + t.Errorf("Evaluate = %+v, want %+v", got, tc.want) + } + }) + } +} diff --git a/internal/handler/cooldown_patterns_test.go b/internal/handler/cooldown_patterns_test.go new file mode 100644 index 0000000..83034cf --- /dev/null +++ b/internal/handler/cooldown_patterns_test.go @@ -0,0 +1,105 @@ +package handler + +import ( + "fmt" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/git-pkgs/cooldown" + "github.com/git-pkgs/proxy/internal/cooldownpolicy" +) + +func TestNuGetCooldownPackagePatterns(t *testing.T) { + for _, tc := range []struct { + name, defaultDuration, pattern string + exact map[string]string + blocked bool + }{ + {"pattern enables cooldown", "", "14d", nil, true}, + {"pattern exempts package", "14d", "0", nil, false}, + {"exact overrides exemption", "", "0", map[string]string{"pkg:nuget/testpkg": "14d"}, true}, + {"exact exempts package", "", "14d", map[string]string{"pkg:nuget/testpkg": "0"}, false}, + } { + t.Run(tc.name, func(t *testing.T) { + p, db, store, fetcher := setupTestProxy(t) + policy, err := cooldownpolicy.New(&cooldown.Config{Default: tc.defaultDuration, Packages: tc.exact}, map[string]string{"pkg:nuget/test*": tc.pattern}) + if err != nil { + t.Fatal(err) + } + p.Cooldown = policy + seedPackage(t, db, store, "nuget", "testpkg", "2.0.0", "testpkg.2.0.0.nupkg", "cached package") + requests := 0 + upstream := newNuGetCooldownUpstream(t, &requests) + defer upstream.Close() + p.HTTPClient = upstream.Client() + h := NewNuGetHandlerWithUpstreams(p, "http://proxy.test", upstream.URL, upstream.URL) + list := nugetGet(t, h.Routes(), "/v3-flatcontainer/TestPkg/index.json", http.StatusOK) + if strings.Contains(list.Body.String(), "2.0.0") == tc.blocked { + t.Fatalf("incorrect version list: %s", list.Body.String()) + } + status := http.StatusOK + if tc.blocked { + status = http.StatusNotFound + } + nugetGet(t, h.Routes(), "/v3-flatcontainer/TestPkg/2.0.0/testpkg.2.0.0.nupkg", status) + if fetcher.fetchCalled { + t.Fatal("cached or withheld download fetched upstream artifact") + } + }) + } +} + +func TestNPMCooldownPatternExemptionRespectsDenylist(t *testing.T) { + p, db, store, fetcher := setupTestProxy(t) + var err error + p.Cooldown, err = cooldownpolicy.New(&cooldown.Config{Default: "7d"}, map[string]string{"pkg:npm/@example/*": "0"}) + if err != nil { + t.Fatal(err) + } + setTestDenylist(t, p, "pkg:npm/@example/widget@1.0.0") + published := time.Now().Add(-time.Hour) + for _, version := range []string{"1.0.0", "2.0.0"} { + seedPackage(t, db, store, "npm", "@example/widget", version, "widget-"+version+".tgz", "cached package") + if err := db.SetVersionPublishedAt("pkg:npm/%40example/widget@"+version, "pkg:npm/%40example/widget", published); err != nil { + t.Fatal(err) + } + } + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/@example/widget" { + t.Errorf("unexpected upstream request: %s", r.URL) + } + w.Header().Set("Content-Type", "application/json") + _, _ = fmt.Fprintf(w, `{"name":"@example/widget","dist-tags":{"latest":"1.0.0"},"versions":{"1.0.0":{},"2.0.0":{}},"time":{"1.0.0":%q,"2.0.0":%q}}`, published.Format(time.RFC3339), published.Format(time.RFC3339)) + })) + defer upstream.Close() + p.HTTPClient = upstream.Client() + h := NewNPMHandler(p, "http://proxy.test", upstream.URL) + server := httptest.NewServer(h.Routes()) + defer server.Close() + for _, tc := range []struct { + path string + status int + contains, absent string + }{ + {"/@example%2Fwidget", http.StatusOK, `"latest":"2.0.0"`, "1.0.0"}, + {"/@example%2Fwidget/-/widget-1.0.0.tgz", http.StatusForbidden, "denylist", "cached package"}, + {"/@example%2Fwidget/-/widget-2.0.0.tgz", http.StatusOK, "cached package", "denylist"}, + } { + response, err := server.Client().Get(server.URL + tc.path) + if err != nil { + t.Fatal(err) + } + body, err := io.ReadAll(response.Body) + _ = response.Body.Close() + if err != nil || response.StatusCode != tc.status || !strings.Contains(string(body), tc.contains) || strings.Contains(string(body), tc.absent) { + t.Errorf("GET %s: status=%d body=%s error=%v", tc.path, response.StatusCode, body, err) + } + } + if fetcher.fetchCalled { + t.Error("cached or denied downloads fetched an upstream artifact") + } +} diff --git a/internal/handler/denylist_test.go b/internal/handler/denylist_test.go index 5034a40..383f95c 100644 --- a/internal/handler/denylist_test.go +++ b/internal/handler/denylist_test.go @@ -98,7 +98,7 @@ func TestCargoDenylistWithoutTimestamps(t *testing.T) { setTestDenylist(t, p, "pkg:cargo/demo@1.0.0") h := &CargoHandler{proxy: p} input := "{\"name\":\"demo\",\"vers\":\"1.0.0\"}\n{\"name\":\"demo\",\"vers\":\"2.0.0\"}\n" - for _, cd := range []*cooldown.Config{nil, {Default: "3d"}} { + for _, cd := range []CooldownPolicy{nil, &cooldown.Config{Default: "3d"}} { p.Cooldown = cd w := httptest.NewRecorder() h.applyCooldownFiltering(w, []byte(input)) diff --git a/internal/handler/handler.go b/internal/handler/handler.go index e5ccd61..20dce9a 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -156,7 +156,7 @@ type Proxy struct { Fetcher fetch.FetcherInterface Resolver *fetch.Resolver Logger *slog.Logger - Cooldown *cooldown.Config + Cooldown CooldownPolicy Denylist *denylist.Policy CacheMetadata bool MetadataTTL time.Duration @@ -208,6 +208,14 @@ type Proxy struct { rewrites *rewriteCache } +// CooldownPolicy supplies version-age filtering and package-specific durations. +type CooldownPolicy interface { + IsAllowed(ecosystem, packagePURL string, publishedAt time.Time) bool + Evaluate(ecosystem, packagePURL string, publishedAt, evaluatedAt time.Time) cooldown.Decision + For(ecosystem, packagePURL string) time.Duration + Enabled() bool +} + // NewProxy creates a new Proxy with the given dependencies. func NewProxy(db *database.DB, store storage.Storage, fetcher fetch.FetcherInterface, resolver *fetch.Resolver, logger *slog.Logger) *Proxy { if logger == nil { diff --git a/internal/server/cooldown_patterns_test.go b/internal/server/cooldown_patterns_test.go new file mode 100644 index 0000000..0587ba0 --- /dev/null +++ b/internal/server/cooldown_patterns_test.go @@ -0,0 +1,30 @@ +package server + +import ( + "strings" + "testing" +) + +func TestStartRejectsInvalidCooldownPatterns(t *testing.T) { + for _, tc := range []struct { + name string + patterns map[string]string + message string + }{ + {"glob", map[string]string{"pkg:npm/[": "0"}, "invalid cooldown package pattern"}, + {"class containing scope alias", map[string]string{"pkg:npm/[@a]*": "0"}, "character classes and escapes are not supported"}, + {"character class", map[string]string{"pkg:npm/[abcdef]*": "0"}, "character classes and escapes are not supported"}, + {"escaped wildcard", map[string]string{`pkg:npm/\*`: "0"}, "character classes and escapes are not supported"}, + {"duration", map[string]string{"pkg:npm/*": "invalid"}, "invalid cooldown duration"}, + {"aliases", map[string]string{"pkg:npm/@example/*": "0", "pkg:npm/%40example/*": "7d"}, "conflicting cooldown package patterns"}, + } { + t.Run(tc.name, func(t *testing.T) { + s := newTestServer(t) + defer s.close() + s.server.cfg.Cooldown.PackagePatterns = tc.patterns + if err := s.server.Start(); err == nil || !strings.Contains(err.Error(), tc.message) { + t.Fatalf("Start error = %v, want %q", err, tc.message) + } + }) + } +} diff --git a/internal/server/server.go b/internal/server/server.go index 2483909..c5e08f1 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -68,6 +68,7 @@ import ( swaggerdoc "github.com/git-pkgs/proxy/docs/swagger" "github.com/git-pkgs/proxy/internal/accesslog" "github.com/git-pkgs/proxy/internal/config" + "github.com/git-pkgs/proxy/internal/cooldownpolicy" "github.com/git-pkgs/proxy/internal/database" "github.com/git-pkgs/proxy/internal/denylist" "github.com/git-pkgs/proxy/internal/enrichment" @@ -242,7 +243,11 @@ func (s *Server) serve(listener net.Listener) error { proxy := handler.NewProxy(s.db, s.storage, fetcher, resolver, s.logger) proxy.HTTPClient = &metadataClient proxy.AuthForURL = s.authForURL - proxy.Cooldown = cd + cooldownPolicy, err := cooldownpolicy.New(cd, s.cfg.Cooldown.PackagePatterns) + if err != nil { + return fmt.Errorf("configuring cooldown policy: %w", err) + } + proxy.Cooldown = cooldownPolicy policy, err := denylist.New(s.cfg.Denylist.Packages) if err != nil { return fmt.Errorf("configuring denylist: %w", err) diff --git a/internal/server/server_test.go b/internal/server/server_test.go index 57b8080..06b43fe 100644 --- a/internal/server/server_test.go +++ b/internal/server/server_test.go @@ -196,9 +196,15 @@ func testStartUsesConfiguredLoopbackUpstreams(t *testing.T) { case "/pypi/simple/ruff/": w.Header().Set("Content-Type", "application/vnd.pypi.simple.v1+json") _, _ = io.WriteString(w, `{"meta":{"api-version":"1.4"},"name":"ruff","files":[{"filename":"ruff-1.0.0.tar.gz","url":"ruff-1.0.0.tar.gz"},{"filename":"ruff-2.0.0.tar.gz","url":"ruff-2.0.0.tar.gz"}]}`) + case "/pypi/pypi/ruff/json": + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, `{"releases":{}}`) case "/v2/library/demo/manifests/latest": w.Header().Set("Content-Type", "application/vnd.oci.image.manifest.v1+json") _, _ = io.WriteString(w, `{"schemaVersion":2}`) + case "/npm/@example/widget", "/npm/@example/exact", "/npm/other": + w.Header().Set("Content-Type", "application/json") + _, _ = fmt.Fprintf(w, `{"time":{"1.0.0":"2020-01-01T00:00:00Z","2.0.0":%q},"versions":{"1.0.0":{},"2.0.0":{}}}`, time.Now().Add(-time.Hour).Format(time.RFC3339)) default: t.Errorf("unexpected upstream path: %q", r.URL.Path) http.NotFound(w, r) @@ -223,6 +229,12 @@ func testStartUsesConfiguredLoopbackUpstreams(t *testing.T) { cfg.Upstream.PyPIDownload = upstream.URL + "/pypi" cfg.Upstream.OCIDefault = upstream.URL cfg.Upstream.AllowLoopback = true + cfg.Upstream.NPM = upstream.URL + "/npm" + cfg.Cooldown = config.CooldownConfig{ + Default: "7d", + Packages: map[string]string{"pkg:npm/@example/exact": "7d"}, + PackagePatterns: map[string]string{"pkg:npm/@example/*": "0"}, + } cfg.Denylist.Packages = []string{"pkg:pypi/ruff@1.0.0"} if err := cfg.Validate(); err != nil { t.Fatalf("validating config: %v", err) @@ -294,6 +306,27 @@ func testStartUsesConfiguredLoopbackUpstreams(t *testing.T) { if !strings.Contains(string(body), `"schemaVersion":2`) { t.Fatalf("OCI response body = %s, want manifest", body) } + assertCooldownPatternMetadata(t, client, cfg.BaseURL) +} + +func assertCooldownPatternMetadata(t *testing.T, client *http.Client, baseURL string) { + t.Helper() + for _, name := range []string{"@example/widget", "@example/exact", "other"} { + resp, err := client.Get(baseURL + "/npm/" + url.PathEscape(name)) + if err != nil { + t.Fatal(err) + } + var metadata struct{ Versions map[string]json.RawMessage } + err = json.NewDecoder(resp.Body).Decode(&metadata) + _ = resp.Body.Close() + if err != nil || resp.StatusCode != http.StatusOK { + t.Fatalf("metadata for %s: status %d, error %v", name, resp.StatusCode, err) + } + _, recent := metadata.Versions["2.0.0"] + if recent != (name == "@example/widget") || metadata.Versions["1.0.0"] == nil { + t.Errorf("pattern policy for %s: versions %v", name, metadata.Versions) + } + } } // TestScanFetchRouteNotMountedWhenScanningDisabled verifies the internal From 129e6c6c1b48dbe9e0f856713ce4eaf1b1c8c1f6 Mon Sep 17 00:00:00 2001 From: Ondrej Kokes Date: Mon, 5 Oct 2026 09:20:36 +0200 Subject: [PATCH 177/633] Filter cached packages by ecosystem using pills (#319) * experimental filtering pills * align data in the dashboard table more nicely (#318) * Bump zizmorcore/zizmor-action from 0.6.2 to 0.6.3 (#333) Bumps [zizmorcore/zizmor-action](https://github.com/zizmorcore/zizmor-action) from 0.6.2 to 0.6.3. - [Release notes](https://github.com/zizmorcore/zizmor-action/releases) - [Commits](https://github.com/zizmorcore/zizmor-action/compare/3dc1ecc9bcb9e94e9b2c709687979e1298497054...70fb788f84895a7701f5643d103d587e460b5c99) --- updated-dependencies: - dependency-name: zizmorcore/zizmor-action dependency-version: 0.6.3 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump docker/setup-qemu-action from 4.2.0 to 4.3.0 (#331) Bumps [docker/setup-qemu-action](https://github.com/docker/setup-qemu-action) from 4.2.0 to 4.3.0. - [Release notes](https://github.com/docker/setup-qemu-action/releases) - [Commits](https://github.com/docker/setup-qemu-action/compare/96fe6ef7f33517b61c61be40b68a1882f3264fb8...1f40c72289eff860ee54a304f1438e3cff362e0a) --- updated-dependencies: - dependency-name: docker/setup-qemu-action dependency-version: 4.3.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump azure/setup-helm from 4.3.1 to 5.0.1 (#332) Bumps [azure/setup-helm](https://github.com/azure/setup-helm) from 4.3.1 to 5.0.1. - [Release notes](https://github.com/azure/setup-helm/releases) - [Changelog](https://github.com/Azure/setup-helm/blob/main/CHANGELOG.md) - [Commits](https://github.com/azure/setup-helm/compare/1a275c3b69536ee54be43f2070a358922e12c8d4...9bc31f4ebc9c6b171d7bfbaa5d006ae7abdb4310) --- updated-dependencies: - dependency-name: azure/setup-helm dependency-version: 5.0.1 dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump modernc.org/sqlite from 1.57.0 to 1.58.0 (#337) Bumps [modernc.org/sqlite](https://gitlab.com/cznic/sqlite) from 1.57.0 to 1.58.0. - [Changelog](https://gitlab.com/cznic/sqlite/blob/master/CHANGELOG.md) - [Commits](https://gitlab.com/cznic/sqlite/compare/v1.57.0...v1.58.0) --- updated-dependencies: - dependency-name: modernc.org/sqlite dependency-version: 1.58.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump github.com/aws/aws-sdk-go-v2/config from 1.32.40 to 1.33.2 (#336) Bumps [github.com/aws/aws-sdk-go-v2/config](https://github.com/aws/aws-sdk-go-v2) from 1.32.40 to 1.33.2. - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/config/v1.32.40...config/v1.33.2) --- updated-dependencies: - dependency-name: github.com/aws/aws-sdk-go-v2/config dependency-version: 1.33.2 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump github.com/prometheus/client_model from 0.6.2 to 0.6.3 (#330) Bumps [github.com/prometheus/client_model](https://github.com/prometheus/client_model) from 0.6.2 to 0.6.3. - [Release notes](https://github.com/prometheus/client_model/releases) - [Commits](https://github.com/prometheus/client_model/compare/v0.6.2...v0.6.3) --- updated-dependencies: - dependency-name: github.com/prometheus/client_model dependency-version: 0.6.3 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump golang.org/x/sync from 0.22.0 to 0.23.0 (#335) Bumps [golang.org/x/sync](https://github.com/golang/sync) from 0.22.0 to 0.23.0. - [Commits](https://github.com/golang/sync/compare/v0.22.0...v0.23.0) --- updated-dependencies: - dependency-name: golang.org/x/sync dependency-version: 0.23.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump github.com/aws/aws-sdk-go-v2/service/ecr from 1.61.0 to 1.64.0 (#334) Bumps [github.com/aws/aws-sdk-go-v2/service/ecr](https://github.com/aws/aws-sdk-go-v2) from 1.61.0 to 1.64.0. - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/service/s3/v1.61.0...service/s3/v1.64.0) --- updated-dependencies: - dependency-name: github.com/aws/aws-sdk-go-v2/service/ecr dependency-version: 1.63.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * fix(handler): let the ProxyCached path request a per-call Accept-Encoding (#324) * fix(handler): fetch conda repodata gzip-compressed on both hops #304 made the ProxyCached path request Accept-Encoding: identity so the metadata cache stores upstream bytes verbatim. That is required for the signed / hash-pinned index ecosystems, but conda's repodata.json is large plain JSON: linux-64 repodata.json is ~441 MB uncompressed (over the metadata_max_size cap, so it 502s today) versus ~34 MB gzip. Replace the ProxyCached path's verbatim bool with an explicit acceptEncoding string ('' = leave unset / transparent, 'identity', or 'gzip'), reusing #304's existing store-and-replay of Content-Encoding unchanged. ProxyCached keeps its exported signature and continues to send identity, so the nine other ecosystems and helm/maven are untouched; only conda's repodata.json / current_repodata.json now request gzip. Setting Accept-Encoding explicitly disables Go's transparent decompression, so the compressed bytes and the Content-Encoding: gzip header are cached and replayed exactly as identity bytes are. conda, mamba and pixi solicit and decode gzip on .json URLs; repodata.json.bz2 stays identity. Fixes #305 * fix(handler): pass metadata content-encoding with the body it describes The adversarial review of the conda gzip route found a reachable regression: writeMetadataCachedResponse took Content-Encoding from a fresh cache-row read while cacheMetadataBlob skips the row write when Storage.Store fails. Under identity that was benign (the body was plain anyway), but on the new gzip route a disk-full or object-store outage served raw gzip bytes as Content-Type: application/json with no Content-Encoding and HTTP 200 -- conda, mamba and pixi fail to parse them, with no HTTP signal and only a Warn log, on every request until a cache write succeeds. fetchOrCacheMetadata now returns the encoding of the body it hands back (the upstream value on a fetch, the stored row's value on a TTL hit or stale fallback) and proxyCachedWithEncoding passes it to writeMetadataCachedResponse, so the header always describes the bytes actually written. cachedMeta drops its now-unused content_encoding field. helm and maven pass "" -- both fetch transparently, so their stored encoding was always empty and behaviour is unchanged. Also fixes a vacuous assertion in the new conda test: the upstream request counter incremented behind the availability gate, so the cached-replay block could never observe a refetch. * fix(handler): pin the stale-fallback content-encoding and drop a dead guard Follow-ups from the adversarial review of the #305 branch, limited to code this branch introduced: - proxyMetadataStream is only ever reached with an explicit Accept-Encoding (ProxyCached passes identity, conda passes gzip or identity), so the guard around the header set was unreachable; replace it with the plain one-token substitution of the former literal, which is the smallest change from main. - The stale-fallback return of fetchOrCacheMetadata (encoding taken from the cache row) was the one #305 return site no test pinned: replacing it with an empty encoding survived the whole suite. Add a conda test that expires the entry, fails the upstream, and asserts the stored gzip blob is served with Content-Encoding: gzip. Not changed, by scope: cacheMetadataBlob still discards the UpsertMetadataCache error (pre-existing on main). If Storage.Store succeeds and the row write fails, a later stale fallback or TTL hit can serve the gzip blob with the row's stale encoding; that needs a DB write failure plus a second event and is tracked separately. * fix(handler): restore the pre-existing cachedMeta content-encoding field The third adversarial review classified deleting cachedMeta.contentEncoding and its lookupCachedMeta populate as elective: neither line was created by this branch nor forced by the fix (writeMetadataCachedResponse now reads the encoding from its parameter and ignores the row value). Under the rule that pre-existing code this branch did not have to touch stays untouched, restore both as they are on main. No behaviour change. Residuals the review documented, unchanged by scope (both share one root cause: the encoding lives in the cache row and the bytes in the blob, and neither is written or read atomically): - cacheMetadataBlob discards the UpsertMetadataCache error, so after a successful gzip Store and a failed row write a later stale fallback or TTL hit can serve the gzip blob with the row's stale encoding. - During the one-time identity->gzip rollout, a request that read a pre-branch identity row, lost the upstream race to a request that stored the gzip blob, and then failed upstream serves the gzip bytes with no Content-Encoding for that one response; later requests self-heal. - helm and maven now pass an empty encoding; on main a spec-violating upstream that answered a transparent gzip request with an encoding Go does not decode (e.g. br) would have had that header replayed from the row. Degenerate; documented rather than changed. * fix(handler): keep conda's proxyCached and .bz2 route as on main Threading acceptEncoding through CondaHandler.proxyCached changed the form of two pieces of original code the fix did not need to touch: the repodata.json.bz2 route (method value rewritten as a closure) and proxyCached itself (new parameter, new call). Restore both exactly as on main; ProxyCached still sends identity, so the .bz2 route is unchanged in behaviour. handleRepodata's non-cooldown branch now derives the cache key inline and calls proxyCachedWithEncoding with gzip directly, so the only original conda.go line that changes is that one call. * fix(handler): keep writeMetadataCachedResponse and its callers as on main Adding a contentEncoding parameter to writeMetadataCachedResponse changed a signature that predates #304 and dragged its two pre-#304 callers (helm.go, maven.go) into the diff, even though #304 only ever added the cm.contentEncoding block inside the function body. Restore writeMetadataCachedResponse's doc and signature exactly as on main and make it a delegate that passes an empty encoding to a new unexported writeMetadataCachedResponseWithEncoding, which carries the original body with #304's block reading the parameter instead of the cache row. proxyCachedWithEncoding calls the sibling with the encoding returned alongside the body. helm.go and maven.go drop out of the diff; their behaviour is unchanged (both fetch transparently, so their stored encoding was always empty). Same split pattern as ProxyCached -> proxyCachedWithEncoding. * fix(handler): move the conda gzip change to its own branch The conda call site in handleRepodata predates #304 and #304 never touched it, so under the rule that this PR only corrects code and behaviour #304 introduced it does not belong here. Restore conda.go and conda_test.go as on main; the conda change continues on a stacked branch against its own issue. Replace the conda-route tests with tests that exercise proxyCachedWithEncoding directly, so this PR still pins its own plumbing: gzip is requested and the compressed bytes plus Content-Encoding are cached and replayed (cached and streaming paths), the header survives a metadata cache write failure, and the stale fallback keeps the stored encoding. * fix(homebrew): fetch the JSON API gzip-compressed on both hops Homebrew (#254) routes every API path through ProxyCached and so, since #304, fetches formula.jws.json (~33 MB plain, ~5 MB gzip) uncompressed on every refresh -- the case that motivated #305. Request gzip for the JSON API via proxyCachedWithEncoding: brew fetches every API download with curl --compressed and decodes Content-Encoding itself, so the compressed bytes and header are cached and served as-is and both hops stay compressed. The analytics endpoints are the one brew consumer fetched without --compressed; they stay on identity. * fix(handler): leave Accept-Encoding unset in proxyMetadataStream for an empty value fetchUpstreamMetadata treats an empty acceptEncoding as 'do not set the header'; proxyMetadataStream set it unconditionally, which would send an empty Accept-Encoding line if a caller ever passed . Guard it the same way so both paths agree. No caller passes today. * fix(handler): keep the metadata row and blob from describing different bytes Two ways the cache row could stop describing the stored blob once a caller requests gzip, both raised by the review of #324: - cacheMetadataBlob stored the blob and then discarded the UpsertMetadataCache error. After a successful gzip store and a failed row write, a later TTL hit or stale fallback served the gzip blob with the previous row's encoding. On a row-write failure, log it and delete the blob just written, so the next request refetches instead. - fetchOrCacheMetadata read the row once up front and reused it for the stale fallback. A request that read an identity row, lost the upstream race to a request that stored the gzip blob, and then failed upstream labelled the new blob with the old row. Re-read the row before falling back so the encoding matches the blob as it is now. Both only become harmful with an encoding change, which this branch introduces; the pre-existing validator-from-row read is tracked separately. * Drop unused cachedMeta.contentEncoding and fix stale doc reference The field was added by #304 and its only reader is replaced in this branch by the encoding parameter passed alongside the body. The proxyCachedWithEncoding comment named conda repodata, which was moved out of this branch in 5991d95; Homebrew is the caller that ships here. --------- Co-authored-by: Andrew Nesbitt * Fix duplicate fetches and 502s on concurrent cache misses (#329) * Return the stored artifact from storeArtifact, not a reader storeArtifact returned a CacheResult holding an open file handle. A handle has one read position, so it can only ever serve a single caller, which is what blocks sharing one fetch between concurrent requests. Return the artifact and its storage path instead, and let each caller open its own reader through openStoredArtifact. Threading that type through fetchAndCache, fetchAndCacheFromURL and their error paths is mechanical; behaviour is unchanged. * Coalesce concurrent cache misses A cache miss went from checkCache straight to an upstream fetch with nothing tracking in-flight work, so N concurrent requests for one uncached artifact produced N upstream fetches and N stores to the same key. That is the CI shape: parallel jobs installing overlapping dependencies against a cold cache. The duplicate stores also fail requests, racing fileblob's per-key ".attrs" sidecar into a partial read served as a 502. Over 12 runs of 8 simultaneous requests for one uncached tarball, against bb2205a: before, 8 fetches per run and 12 of 96 responses were 502; after, 1 fetch per run and none failed. Route both miss paths through a shared in-flight map keyed on the artifact, including the download URL and upstream-declared hash so callers expecting different bytes never share a fetch. singleflight does not fit: Do gives waiters no way to leave, while DoChan lets the caller running the fetch abandon it, breaking storeArtifact's scan-on-disconnect contract. Deciding roles under a mutex gives both behaviours. The fetch runs on the first caller's context and is seen through; waiters leave when their own clients do. This removes the sidecar trigger on this path. The race is in fileblob and three writers bypass this path entirely, so it is fixed separately. Fewer failures now reach the circuit breaker, so it trips later. Sixteen concurrent callers against real file:// storage fail 10 of 10 runs on main and pass 10 of 10 here. Other tests pin key discrimination, failure propagation, resolver-path coalescing, per-caller readers, waiter cancellation, key release and panic safety. allocs/op is unchanged. mockStorage gains a mutex so concurrent tests can use it. * Normalize digest case in the coalescing key artifactHashMatches compares digests with strings.EqualFold, but the coalescing key used the hash verbatim. The same digest in two casings produced two keys, so two callers for one artifact each ran their own upstream fetch and store, which is what the coalescing is meant to prevent. * Make the panic coalescing test deterministic The test timed the second caller's arrival with a sleep, so which caller became the leader was left to the scheduler. When it lost that race the second caller ran the fetch itself, and its panic was not recovered, so the test binary died instead of the test failing. Whether a caller has reached the wait is not observable from outside: it runs a cache lookup against the database first, so releasing the leader on a timer races that query. Drive coalesceFetch directly and hold the shared entry instead, which removes the timing entirely. The panicking fetcher is no longer needed. * Recheck the cache before running a shared fetch A caller checks the cache before it reaches coalesceFetch, so a fetch that commits in that gap is invisible to it. Arriving after the sharing entry is gone, it became a new leader and fetched, stored and scanned an artifact the cache already held. The leader now rechecks the committed record first. It serves that record only if its bytes still open, because a record can outlive them, and refetching is the recovery the cache lookup already makes for that case. Waiters are unaffected: the record fills the same shared value a fetch would, and each caller opens its own reader from it. The recheck is the leader's alone. A waiter has a fetch in flight to wait on, and rechecking would race it for no gain. * Lock the mock fetcher's bookkeeping Coalescing tests call the handler from many goroutines. The key keeps the fetch itself serialized, but the mock should not lean on that: it now locks the fields it records, so any concurrency the handler applies is safe under the race detector. * Wait for the leader's fetch instead of sleeping The canceled-waiter test slept 200ms and assumed the leader had taken the key by then. On a slow scheduler the canceled call could become the leader and the test would no longer cover waiter cancellation. The fetcher now signals when its first fetch begins, which happens only once the key is held. * Bump google.golang.org/grpc from 1.83.1 to 1.83.2 (#346) Bumps [google.golang.org/grpc](https://github.com/grpc/grpc-go) from 1.83.1 to 1.83.2. - [Release notes](https://github.com/grpc/grpc-go/releases) - [Commits](https://github.com/grpc/grpc-go/compare/v1.83.1...v1.83.2) --- updated-dependencies: - dependency-name: google.golang.org/grpc dependency-version: 1.83.2 dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Stop writing fileblob's .attrs sidecar (#328) * Stop writing fileblob's .attrs sidecar fileblob stores blob metadata in an ".attrs" file per object and rewrites it with os.Create, truncating in place outside the atomic rename that protects the blob. A read overlapping a write decodes a partial file and fails with "opening reader: EOF", served as a 502. One writer against four readers on a single key failed 408 of 2000 reads. cacheMetadataBlob is most exposed to it, rewriting a key on every refresh while readers are served from it. Nothing in the proxy reads what the sidecar holds. gocloud.dev/blob is imported only by internal/storage, Store sets no ContentType, and Attributes is used only for Size, which comes from os.Stat. A missing sidecar already defaults cleanly, so "metadata=skip" removes the hazard rather than locking around it, and saves a write per store. * Clear .attrs sidecars left by earlier versions metadata=skip stops fileblob rewriting sidecars but does not delete ones already on disk, so a sidecar left partial by an interrupted write now fails every read of its key for good. Before, a later store repaired it by rewriting. Store therefore removes the sidecar for the key it writes. Removal is atomic where the rewrite was not, so a concurrent reader gets the whole old file or nothing. Delete already removes sidecars, so the two paths drain a cache between them. Deriving that path is necessary because fileblob's key escaping is unexported. It is the identity for a plain key and parts from one only for keys that are not valid local paths, which is what filepath.Localize rejects. That also keeps the removal inside the cache directory: without it a key holding ".." resolves outside. The clearing test runs one key per storage path the proxy builds, seeded through a bucket that still writes sidecars so the path under test is fileblob's own. * Explain why failed sidecar cleanup does not fail the store Move the removal into clearLegacySidecar and say why its error is dropped rather than returned. A failed removal leaves exactly the state this change inherited, while failing the write would turn a cleanup miss into a failed request. Windows makes that concrete: Go opens files with FILE_SHARE_READ and FILE_SHARE_WRITE but not FILE_SHARE_DELETE, so a reader holding the sidecar open blocks deletion, and that reader is the workload this change exists to protect. Propagating would fail stores during exactly the overlap being fixed. The next store of the key retries. A test pins it, using a non-empty directory at the sidecar path to make os.Remove fail with something other than not-exist on any platform. * Fail the concurrency test if its writer stops The writer returned silently when Store failed, so the test could pass with no concurrent writes at all. Its error is now reported, and the test also checks that at least one write completed. Reporting it showed the writer had been dying on Windows at its first collision: Go opens files without FILE_SHARE_DELETE, so a reader holding the file open makes the writer's rename fail with access denied. The test now skips there, since it cannot contend a writer with readers on that platform. * Clear legacy sidecars for keys fileblob escapes legacySidecarPath declined any key filepath.Localize rejects, which on Windows is every key with a colon: OCI digests and Debian epochs. Their sidecars were never cleared there, and a truncated one kept failing reads, since fileblob still reads a sidecar it finds under metadata=skip. fileblob hex-escapes such characters on the way to disk. The path is now derived the same way, so the sidecar is looked for where fileblob wrote it. Localize still validates the escaped form, which keeps the removal inside the cache directory. * Drop stale comment about declining colon keys on Windows 623ff3e made legacySidecarPath escape keys the way fileblob does, so colon-bearing keys are now cleared on Windows and the OCI and Debian rows in TestStoreClearsLegacyAttrsSidecar prove it. The comment described the behaviour before that commit. --------- Co-authored-by: Andrew Nesbitt * feat(debian): serve additional APT archives at /debian/{name}/ A Debian release is served by more than one archive. Security updates live on a different host than the main archive, so a single upstream.debian URL cannot serve a complete suite set and -security suites are unreachable. Add upstream.debian_repositories, a name-to-URL map served at /debian/{name}/, modelled on upstream.apk. The field is additive: upstream.debian keeps serving /debian/pool/... and /debian/dists/... with unchanged cache identities, so existing deployments and their warm caches are unaffected, and the scalar field and PROXY_UPSTREAM_DEBIAN are untouched. Named repositories scope both caches by name, since the same filename can hold different bytes in different archives. Metadata keys are hashed over name, upstream URL, and path, as APKHandler does. Names are validated through validateNamedUpstreams, and "pool" and "dists" are refused because they would shadow the main archive's own paths. An unconfigured first path segment stays a main-archive path rather than returning 404 as the APK handler does: the main archive is unnamed and serves paths of its own at the root. * fix(server): tune the shared upstream transport defaults (#351) * fix(server): tune the shared upstream transport defaults server.serve builds the shared client with safehttp.New, which clones Go's default transport: MaxIdleConnsPerHost stays 0 (an effective limit of two idle connections per host) and ResponseHeaderTimeout stays 0. Handing that client to fetch.NewFetcher via fetch.WithHTTPClient replaces the fetcher's own defaults of 10 idle connections per host and a 60-second response-header timeout. Set both on the shared transport, matching the fetcher defaults: a second burst of concurrent cache misses to one registry now reuses its connections instead of re-dialling most of them, and an upstream that accepts a request but stalls before sending headers is cut off after 60 seconds rather than only by the client's overall timeout. Tests measure connection reuse across two concurrent bursts against a TLS upstream that counts accepted connections (Go default: at most two reused; tuned: all eight) and assert that a stall before headers fails with the response-header timeout. Fixes #327 * fix(server): reconcile http_timeout docs and tighten the transport tests The http_timeout documentation and the config comment said "0" disables the upstream timeout entirely. With a fixed 60-second ResponseHeaderTimeout on the shared transport that is no longer the whole story, so both now say that waiting for response headers stays bounded independently of the setting. Test cleanup from review: the 50ms settle between bursts was dead time (the transport returns a connection to the idle pool before the body's final Read returns, so burst returning already means the pool is settled); the maxNewInBurst sentinel became explicit min/max bounds per case; the stall test dropped the client.Timeout override and the elapsed-time assertion, which was redundant with the error-text check in any realistic run and whose failure message misattributed the cause, and its comment now says plainly that the field assertions pin production while the behavioural half runs at a lowered timeout. * test(server): hold each burst at the upstream instead of sleeping The reuse test kept a burst in flight with a 100ms handler sleep, so a process stall longer than that between spawning the goroutines and their dials let a request finish early and hand its connection to a sibling. Review reproduced this with forced stalls: the default-transport case then dialled 5 instead of 6 connections. The handler now answers only once burstSize requests are waiting at the same time. With HTTP/1.1 pinned that forces every burst onto burstSize distinct connections regardless of scheduling, and the measured counts stay exactly 6 new for Go's default and 0 for the tuned transport, also under the same forced stalls. * fix(database): set connection pool limits for Postgres (#350) * fix(database): set connection pool limits for Postgres OpenPostgres returned sqlx.Open's handle with database/sql's defaults: no cap on open connections and two idle ones. Under load nearly every request opened a new Postgres session, ran its few statements and closed it again, paying a backend fork and SCRAM authentication each time. Set the pool limits the issue suggests: 32 open and 32 idle connections, idle connections closed after 5 minutes and every connection recycled after 30 minutes. The values live in named constants because the mnd linter rejects the literals inline. The test (skipped without PROXY_DATABASE_URL, like the other Postgres tests) takes 16 connections from the pool, releases them and checks that all 16 stay idle; with the default pool only two survive. Fixes #323 * test(database): pin the pool properties instead of the wiring Review pointed out that the pool test compared MaxOpenConnections to the constant it was set from, so the assertion followed the constant and would have accepted postgresMaxOpenConns = 0 (unlimited), and that its burst of 16 only proved MaxIdleConns >= 16. The burst now takes postgresMaxIdleConns connections, so every configured idle slot has to survive the release, and the open cap is checked to be finite and large enough for that burst before any connection is taken, so a cap below the idle count fails fast instead of blocking in db.Conn. The doc comment now says which settings the test covers; the idle-time and lifetime settings only show up in DBStats.MaxIdleTimeClosed and MaxLifetimeClosed after minutes of wall-clock time and stay unexercised. * test(database): guard the pool test against a vacuous burst Review showed that with the burst tied to postgresMaxIdleConns the test also passed for a constant of 2, database/sql's default, or of 0, where it took no connections at all. It now fails outright unless the configured idle count exceeds the default, and every connection it takes is released in a cleanup, so an assertion failure mid-burst no longer leaves sessions open for the rest of the test binary. * test(database): drop every schema table in the Postgres fixture (#349) createTestPostgresDB dropped artifacts, versions, packages and schema_info before calling CreateSchema, but not the migrations table. On a database that has seen one test the migration records survive, so the next CreateSchema fails while recording 001_add_packages_enrichment_ columns with a duplicate key on migrations_pkey. Running the package against one Postgres therefore failed from the second Postgres-backed test on. Drop vulnerabilities, metadata_cache and migrations as well, so the fixture clears every table CreateSchema creates. The package now passes repeatedly against the same database. * Discard a stale cache entry under the coalescing key (#348) The digest-aware cache check discarded a stale entry before its caller took the key. A slow caller could delete an entry another caller's fetch had just committed, and everyone sharing that fetch then failed to open it. The check now only reports the miss. The caller running the shared fetch discards the entry under the key, after the recheck, so one a previous fetch refreshed is served, not deleted. The mismatch warning fires once per refresh instead of once per request. Swift HEAD no longer discards either, having no fetch to do it under. It probes upstream as before, and the next GET replaces the entry. Different digests or URLs, or no digest, use different keys and can still collide on the storage path. That is the storage layout follow-up. * Separate OCI request timeout from readiness probes in loopback test (#347) The 250ms client is meant for the readiness poll, where a timeout is retried. Reusing it for the OCI manifest request makes the test flake under -race on Windows CI when fetch and cache I/O take longer, as seen on #328. The request checks upstream routing, not latency. * Fix NuGet cooldown enforcement for listings and downloads (#340) * fix(nuget): enforce cooldown across metadata and downloads * fix(nuget): support legacy registration and preserve valid metadata cache * refactor(nuget): address maintainer review cleanup * Fall back to embedded build info for Version (#352) go install does not apply goreleaser's -ldflags -X, so binaries installed that way reported 'dev'. Read debug.BuildInfo.Main.Version when the ldflag is unset. * docs(debian): trim comments on the named-repositories change The motivation for separate archives was repeated across six files and the reserved-name rationale across five. Keep each in one place -- the reference docs -- and leave the code comments to what an informed reader cannot get from the code: why an unknown first segment is not a 404 as it is for APK, and why the main archive keeps its legacy cache identities. Comments and prose only; no behaviour change. * docs(debian): drop a stray space in a test comment * Bump github.com/aws/aws-sdk-go-v2/service/ecr from 1.64.0 to 1.65.0 (#357) Bumps [github.com/aws/aws-sdk-go-v2/service/ecr](https://github.com/aws/aws-sdk-go-v2) from 1.64.0 to 1.65.0. - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/service/s3/v1.64.0...service/s3/v1.65.0) --- updated-dependencies: - dependency-name: github.com/aws/aws-sdk-go-v2/service/ecr dependency-version: 1.65.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump github.com/git-pkgs/spdx from 0.3.1 to 0.3.2 (#355) Bumps [github.com/git-pkgs/spdx](https://github.com/git-pkgs/spdx) from 0.3.1 to 0.3.2. - [Commits](https://github.com/git-pkgs/spdx/compare/v0.3.1...v0.3.2) --- updated-dependencies: - dependency-name: github.com/git-pkgs/spdx dependency-version: 0.3.2 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump zizmorcore/zizmor-action from 0.6.3 to 0.6.4 (#354) Bumps [zizmorcore/zizmor-action](https://github.com/zizmorcore/zizmor-action) from 0.6.3 to 0.6.4. - [Release notes](https://github.com/zizmorcore/zizmor-action/releases) - [Commits](https://github.com/zizmorcore/zizmor-action/compare/70fb788f84895a7701f5643d103d587e460b5c99...cc914d7f3750a2d13d75c7f184a1060aa0e9d482) --- updated-dependencies: - dependency-name: zizmorcore/zizmor-action dependency-version: 0.6.4 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump github.com/aws/aws-sdk-go-v2/config from 1.33.2 to 1.33.4 (#356) Bumps [github.com/aws/aws-sdk-go-v2/config](https://github.com/aws/aws-sdk-go-v2) from 1.33.2 to 1.33.4. - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/config/v1.33.2...config/v1.33.4) --- updated-dependencies: - dependency-name: github.com/aws/aws-sdk-go-v2/config dependency-version: 1.33.4 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * feat(debian): report a misspelled repository name A request to /debian/{name}/ for a name that is not configured fell through to the main archive, so a typo surfaced as the upstream's own HTML 404 with no mention of the repository. The reply now names the unknown repository and lists the configured ones. Scoped to {name}/dists/ and {name}/pool/. The main archive serves README, indices/ and project/ at its root, so an unknown first segment stays a main-archive path everywhere else. apt prints only the status line, so this is visible via curl or the proxy log rather than in apt output. * fix(helm): support OCI references in HTTP repository indexes * feat: add exact-version package denylist * Exit non-zero when mirror denylist is invalid * Omit malformed Helm chart releases instead of failing the index A single bad release in a third-party index made the whole repository unusable through the proxy. * End an eviction pass that cannot evict anything A failed delete leaves the record eligible, so the same batch came back on the next iteration while the recorded size never dropped. Against a backend refusing deletes the pass retried forever, burning CPU and flooding the log. A canceled context made it faster: every delete fails at once, so the loop spun as quickly as the database could answer. The pass now ends when a batch clears no record, and returns as soon as the context is canceled. Progress counts records rather than bytes, because a cached record can have no size recorded. A pass can therefore end with the cache still above its limit, leaving the records that failed for the next pass to retry. * Fix the handler test build The denylist tests call NewDebianHandler with three arguments, but the additional APT archives change gave it a fourth. Each passed CI on its own branch, and together they leave internal/handler without a test binary that compiles, so go vet and go test fail on main. * docs: explain APT denylist limitations and version pinning * Assert omitted Helm releases are not downloadable * build(deps): bump go.opentelemetry.io/otel/sdk from 1.44.0 to 1.45.0 Bumps [go.opentelemetry.io/otel/sdk](https://github.com/open-telemetry/opentelemetry-go) from 1.44.0 to 1.45.0. - [Release notes](https://github.com/open-telemetry/opentelemetry-go/releases) - [Changelog](https://github.com/open-telemetry/opentelemetry-go/blob/main/CHANGELOG.md) - [Commits](https://github.com/open-telemetry/opentelemetry-go/compare/v1.44.0...v1.45.0) --- updated-dependencies: - dependency-name: go.opentelemetry.io/otel/sdk dependency-version: 1.45.0 dependency-type: indirect ... Signed-off-by: dependabot[bot] * Widen wall-clock bound in flaky probe timeout test Loaded CI runners exceeded the 500ms bound; 5s still proves the probe timeout fired instead of blocking on Store. * Add and align CI security and dependency automation * fix(mirror): match cached artifacts by resolved filename * Standardize the README license footer * Test that GCS Delete of a missing object returns nil * Give each fetch its own storage path Every fetch of an artifact wrote one path, so fetches with different URLs or digests, which do not share a coalescing key, could overwrite each other's bytes, serve the wrong ones, or delete them on a digest mismatch while the other was about to open them. storeArtifact now writes {ecosystem}/{name}/{version}/{fetch id}/{filename}, with a random id per fetch. Most ecosystems learn the digest only after the fetch, and storage has no rename, so the id is the one rule that fits all of them. Existing records keep their paths and stay readable. On a file:// bucket, Delete now removes an emptied fetch directory, which fileblob leaves behind, and no other directory, since another fetch may be creating its own inside it. A path a record stops pointing at is queued in pending_deletes rather than deleted, since a request that read the record may still open it. A loop deletes queued paths after max(1h, direct_serve_ttl), whether or not max_size is set. UpsertArtifact and clears apply only if the record still holds the path the caller read, so racing commits each queue the path they replaced and a clear never orphans a newer commit. Eviction still deletes inline, and counts space as freed only when the record still pointed at what it deleted. * build(deps): bump docker/setup-buildx-action from 4.3.0 to 4.4.1 Bumps [docker/setup-buildx-action](https://github.com/docker/setup-buildx-action) from 4.3.0 to 4.4.1. - [Release notes](https://github.com/docker/setup-buildx-action/releases) - [Commits](https://github.com/docker/setup-buildx-action/compare/37fe631027851001ddb9b187196cc803df7f5f0e...f87e5991a6d7451dcb8d9637bfbc97413f497069) --- updated-dependencies: - dependency-name: docker/setup-buildx-action dependency-version: 4.4.1 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] * build(deps): bump docker/setup-qemu-action from 4.3.0 to 4.4.0 Bumps [docker/setup-qemu-action](https://github.com/docker/setup-qemu-action) from 4.3.0 to 4.4.0. - [Release notes](https://github.com/docker/setup-qemu-action/releases) - [Commits](https://github.com/docker/setup-qemu-action/compare/1f40c72289eff860ee54a304f1438e3cff362e0a...99012661954931238ded8c8b007157a8430204e1) --- updated-dependencies: - dependency-name: docker/setup-qemu-action dependency-version: 4.4.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] * build(deps): bump docker/build-push-action from 7.3.0 to 7.4.0 Bumps [docker/build-push-action](https://github.com/docker/build-push-action) from 7.3.0 to 7.4.0. - [Release notes](https://github.com/docker/build-push-action/releases) - [Commits](https://github.com/docker/build-push-action/compare/53b7df96c91f9c12dcc8a07bcb9ccacbed38856a...c3c9e263c25d99ce0380d002d59b67737d91b0dc) --- updated-dependencies: - dependency-name: docker/build-push-action dependency-version: 7.4.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] * build(deps): bump github.com/aws/aws-sdk-go-v2/service/ecr Bumps [github.com/aws/aws-sdk-go-v2/service/ecr](https://github.com/aws/aws-sdk-go-v2) from 1.65.0 to 1.66.0. - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/service/s3/v1.65.0...service/s3/v1.66.0) --- updated-dependencies: - dependency-name: github.com/aws/aws-sdk-go-v2/service/ecr dependency-version: 1.66.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] * build(deps): bump golang.org/x/net from 0.58.0 to 0.59.0 Bumps [golang.org/x/net](https://github.com/golang/net) from 0.58.0 to 0.59.0. - [Commits](https://github.com/golang/net/compare/v0.58.0...v0.59.0) --- updated-dependencies: - dependency-name: golang.org/x/net dependency-version: 0.59.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] * build(deps): bump github.com/git-pkgs/registries from 0.9.1 to 0.9.2 Bumps [github.com/git-pkgs/registries](https://github.com/git-pkgs/registries) from 0.9.1 to 0.9.2. - [Release notes](https://github.com/git-pkgs/registries/releases) - [Commits](https://github.com/git-pkgs/registries/compare/v0.9.1...v0.9.2) --- updated-dependencies: - dependency-name: github.com/git-pkgs/registries dependency-version: 0.9.2 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] * build(deps): bump github.com/git-pkgs/magic from 0.3.1 to 0.4.0 Bumps [github.com/git-pkgs/magic](https://github.com/git-pkgs/magic) from 0.3.1 to 0.4.0. - [Release notes](https://github.com/git-pkgs/magic/releases) - [Commits](https://github.com/git-pkgs/magic/compare/v0.3.1...v0.4.0) --- updated-dependencies: - dependency-name: github.com/git-pkgs/magic dependency-version: 0.4.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] * build(deps): bump modernc.org/sqlite from 1.58.0 to 1.59.0 Bumps [modernc.org/sqlite](https://gitlab.com/cznic/sqlite) from 1.58.0 to 1.59.0. - [Changelog](https://gitlab.com/cznic/sqlite/blob/master/CHANGELOG.md) - [Commits](https://gitlab.com/cznic/sqlite/compare/v1.58.0...v1.59.0) --- updated-dependencies: - dependency-name: modernc.org/sqlite dependency-version: 1.59.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] * build(deps): bump github.com/git-pkgs/vers from 0.7.0 to 0.7.1 Bumps [github.com/git-pkgs/vers](https://github.com/git-pkgs/vers) from 0.7.0 to 0.7.1. - [Commits](https://github.com/git-pkgs/vers/compare/v0.7.0...v0.7.1) --- updated-dependencies: - dependency-name: github.com/git-pkgs/vers dependency-version: 0.7.1 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] * build(deps): bump github.com/aws/aws-sdk-go-v2/config Bumps [github.com/aws/aws-sdk-go-v2/config](https://github.com/aws/aws-sdk-go-v2) from 1.33.4 to 1.33.5. - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/config/v1.33.4...config/v1.33.5) --- updated-dependencies: - dependency-name: github.com/aws/aws-sdk-go-v2/config dependency-version: 1.33.5 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] * build(deps): bump github.com/git-pkgs/archives from 0.7.0 to 0.7.1 Bumps [github.com/git-pkgs/archives](https://github.com/git-pkgs/archives) from 0.7.0 to 0.7.1. - [Commits](https://github.com/git-pkgs/archives/compare/v0.7.0...v0.7.1) --- updated-dependencies: - dependency-name: github.com/git-pkgs/archives dependency-version: 0.7.1 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] * fix(conda): preserve gzip for large repodata indexes * fix: centralize pass-through response relaying * Clear before deleting on eviction, and requeue failed reclaims Eviction deleted an object before checking its record still pointed at it. If a refetch had moved the record, that object was queued and a request that read the record earlier could still be opening it. Eviction now clears first, skips the delete if the record moved, and queues the object when the delete fails after a clear. Reclaim queues a failed delete again, which moves it behind the rest, so objects the backend keeps refusing cannot fill every batch. Also drop the clearArtifactCache passthrough and ClearCachedArtifact's unused context, note that only tests call ArtifactPath, and document that max_size does not count objects waiting to be reclaimed. * Log and count requests aborted mid-relay A truncated relay panics with http.ErrAbortHandler, which skipped the request log line, the request metrics and the access log entry. * Batch cache hit writes Every cached download wrote its hit in its own UPDATE. SQLite runs on one connection, so each download was a write transaction the rest queued behind, holding a warm npm cache near 1,000 requests a second with the CPU two thirds idle. Hits are now counted in memory and written in one transaction every database.hit_flush_interval, "1s" by default, "0" for a write per hit. A failed write keeps the hits for the next flush, and Close writes what is pending. The same test then served about 2,200 requests a second. Closes #322. * fix pnpm audit * apply review changes * Name every mounted route in requestEcosystem - Map /apk to alpine, matching what the handler writes to the database - Report /helm, /homebrew, /generic and /swift under their own names - Test every one of the 23 mounts, so the next route added cannot quietly fall through to "other" Part 1 of 6 splitting #381 up. Co-Authored-By: Claude Opus 5 (1M context) * test(handler): stop asserting on an upstream connection the proxy abandons TestRelayRoutes fails intermittently on CI with "write: connection reset by peer" at the upstream handler's final Flush, on /hex/packages/demo and /gem/info/demo. Those two routes are the only ones in the table that fan out a second upstream request: with cooldown enabled, the hex and gem handlers fetch version timestamps concurrently with the artifact. The test server answers that sidecar request with the same 64 KiB chunked 502, but the handler returns as soon as it sees a non-200 and closes the body unread, so the proxy drops the connection while the test server is still writing to it. Whether the reset lands before or after the Flush is a scheduling race, which is why it only surfaces on loaded runners. The flush error says nothing about the response under test, which is asserted on the downstream side, so stop reporting it. Co-Authored-By: Claude Opus 5 (1M context) * Per-ecosystem cache and download statistics - Add GetEcosystemStats, aggregating packages, versions, artifacts, cache size, downloads and downloaded bytes per ecosystem - Publish six proxy_ecosystem_* gauges from it on the existing one-minute cache-stats tick - Set and selectively delete rather than Reset, so no scrape lands on a half-populated vector - Report artifacts with no package row under "unattributed" rather than dropping them, so the figures still add up Part 2 of 6 splitting #381 up. Co-Authored-By: Claude Opus 5 (1M context) * Add the /ui/analytics page - Ring of download size by ecosystem, cache figures, per-ecosystem table, vulnerability overview and a Runtime card - Add metrics.Gather so the page can render counters that were never in the database - Add proxy_response_bytes_total and response-writer byte counting - Serve a retained snapshot behind a staleness banner when the aggregation fails, rather than rendering old figures as current - Extract the security overview into a shared component Part 3 of 6 splitting #381 up. Co-Authored-By: Claude Opus 5 (1M context) * Report the per-ecosystem breakdown from GET /stats - Add downloaded_bytes, downloads and an ecosystems array, served from the snapshot the gauges and the page already share - Add stats_unavailable so a failed aggregation is distinguishable from an idle proxy - Regenerate the OpenAPI spec Part 4 of 6 splitting #381 up. Co-Authored-By: Claude Opus 5 (1M context) * Add a Grafana dashboard - 33 panels, no hardcoded data source UID - Three ecosystem variables, because the label means three different things across /metrics: the package record, the request path, and the handler's own name Part 5 of 6 splitting #381 up. Co-Authored-By: Claude Opus 5 (1M context) * Attribute requests to a caller and a client tool - Track per-caller request and byte counts in a bounded table - Export proxy_client_requests_total and proxy_client_response_bytes_total over a closed label set; addresses are never labels - Gate the sources card behind ui_request_sources, off by default, since /ui carries no authentication of its own - Move access_log.trust_forwarded_for to a top-level key - Keep remote as the TCP peer and add remote_ip alongside it Part 6 of 6 splitting #381 up. Co-Authored-By: Claude Opus 5 (1M context) * Update enrichment to v0.7.2 (#394) * Update ecosystems-go to v0.5.0 * Update enrichment to v0.7.2 * Update git-pkgs dependencies (#396) * Keep hit timestamps from moving backwards With proxies sharing a database, a batch can flush after a newer hit is already written, and overwriting last_accessed_at and updated_at with its older time could reorder LRU eviction. Both now keep the later of the stored and incoming times. The count still adds up. * build(deps): bump alpine from 3.24.1 to 3.24.2 (#397) Bumps alpine from 3.24.1 to 3.24.2. --- updated-dependencies: - dependency-name: alpine dependency-version: 3.24.2 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Install development tools directly in CI (#398) * Add storage.cache_artifacts to serve artifacts without storing them (#387) * Add storage.passthrough to serve artifacts without storing them With storage.passthrough enabled the proxy streams every artifact from upstream to the client and never writes it to storage or the cache database. Metadata filtering, cooldown and the denylist work as before, so the proxy can sit behind another cache (e.g. an Artifactory remote) purely as a policy layer without holding a second copy of every package. Artifacts whose digest is known up front (OCI, Swift, Helm) are verified while streaming. Their responses are sent chunked and the connection is aborted on a mismatch, so a client never receives a tampered artifact as a complete response. Passthrough is rejected together with scanning, direct_serve and mirror_api, and the mirror command refuses to run with it, since all of them depend on stored artifacts. Co-Authored-By: Claude Opus 5.5 * Address review: abort truncated streams, guard every cache read - serveArtifact aborts the response on any body read error, and when fewer bytes than the declared size were written, not only on a digest mismatch. A failed or short upstream body in passthrough mode no longer reaches the client as a complete 200. - Streamed upstream read failures are logged and counted as stream_failed upstream errors before the response is aborted. - checkCache reports a miss when artifact caching is off, so every cache read path, including Swift archive HEAD requests, ignores entries stored before the mode was enabled. Denylisted versions are still rejected. - Rename storage.passthrough to storage.cache_artifacts (default true, PROXY_STORAGE_CACHE_ARTIFACTS), which names what actually changes. Tests cover an OCI blob shorter than its Content-Length and an npm tarball whose chunked response ends without the final chunk, both served through the HTTP handlers, plus a Swift HEAD request against a cached archive. Co-Authored-By: Claude Opus 5.5 * Rename Proxy.Passthrough to StreamArtifacts Match the storage.cache_artifacts option: the field says what changes (artifacts are streamed instead of stored) and its zero value keeps the usual caching behaviour for every Proxy built without the server config, such as the mirror command and tests. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 * Show the by-client table regardless of ui_request_sources The flag exists to withhold caller addresses. The by-client table carries none, and the same figures are already public at /metrics, so gating it left two metrics with no tile on the page in the default config. * Drop unused registry from TestUpdateEcosystemStatsSumsAliasedRows * Cache rewritten npm and Composer metadata (#402) The npm and Composer handlers rewrite every metadata document they serve so that download URLs point at the proxy: decode the whole document into generic maps, change the URLs, encode it again, and for Composer expand the minified format first. That ran on every request, cached metadata included. With metadata caching on and upstream out of the picture, a cached request still cost 7 ms for @babel/core, 17 ms for symfony/console and 155 ms and 134 MB of allocations for typescript, and an 8 vCPU VM serving cached npm packuments to 20 clients ran the proxy at 450% CPU for about 1,100 requests a second. Rewritten documents are now kept in memory, keyed by the ecosystem, proxy URL, package and a SHA-256 of the raw document, so new bytes from upstream are rewritten again and nothing is served stale. Requests that arrive while a document is being rewritten wait for that rewrite rather than running their own; a waiter leaves when its client does, and the rewrite still completes and is cached. The denylist is fixed at startup, so it needs no place in the key. Cooldown filtering depends on the current time, so with cooldown on the cache is bypassed. metadata_rewrite_cache_size bounds the cache (default "256MB", least recently used out first, "0" to rewrite on every request). NewProxy callers keep the old behaviour unless they set it. Whole cached requests, measured locally: @babel/core 7.1 ms -> 0.38 ms, 54,489 -> 119 allocations typescript 155 ms -> 10.5 ms, 1.37M -> 133 allocations symfony/console 17.4 ms -> 0.48 ms, 165,909 -> 123 allocations What remains is reading the raw document from storage and hashing it. * Copy inherited Composer fields shallowly when expanding (#401) expandMinifiedVersions deep-copied every inherited field into every version (b68184c) so that rewriting one version's dist URL in place could not change the versions that inherited it. For a package with a long history that means recursively copying require, autoload and the rest hundreds of times per document, on every metadata request. dist is the only field the proxy changes after expansion, so copy just that: rewriteDistURL now gives the version its own dist map before setting the URL, and expansion shares the other inherited values. TestComposerExpandMinifiedSharedDistReferences still guards the original bug, and fails if the copy in rewriteDistURL is removed. On the symfony/console metadata from Packagist the rewritten output is byte-identical, and the rewrite drops from about 20.7 ms to 13.8 ms, with allocations down from 13.6 MB to 9 MB. * Support HTTP byte-range requests for local file cached artifact downloads (#404) * feat(storage): expose efficient seek for local files Open file-backed artifacts with os.Open so local readers support efficient seeking. Leave cloud storage readers unchanged. Keep the legacy sidecar mapping test focused on fileblob's reader: Blob.Open now reads local files directly and no longer consults the sidecar. Continue checking sidecar cleanup and post-store reads. * feat(handler): preserve efficient seek through integrity checks Keep the cache result's single Reader field and preserve io.Seeker when the underlying storage reader supports efficient seeking. Sequential reads continue through whole-object integrity verification. A successful seek switches subsequent reads to the source for range responses, where full-object verification is not possible. Test both modes and verify the source is closed once. * feat(handler): add request-aware byte range responses Add a request-aware artifact serving helper for bounded, open-ended, and suffix byte ranges, including If-Range handling, 206 responses, and 416 responses for valid unsatisfiable ranges. Ignore malformed and multi-range requests by serving the full artifact. Advertise byte-range support only for seekable readers, preserving HEAD, redirect, and non-seekable behavior. Clarify that net/http recovers ErrAbortHandler per request and keeps the server running. * feat(handler): enable byte ranges across artifact downloads Forward each artifact request to the shared range-aware response helper, including cached generic release assets and OCI blobs. This lets the handler honor ranges only when its reader supports efficient seeking, without changing direct-storage redirect behavior. Add warm-cache endpoint tests that assert partial response headers and bytes without an upstream refetch. Document supported range behavior, unsupported reader fallback, and the integrity limitation of partial reads. * Coalesce concurrent metadata fetches (#400) * Coalesce concurrent metadata fetches Concurrent cache misses for one artifact already share a single upstream fetch (#329), but metadata misses did not: every request for a package's metadata went to the registry, even when the same document was already being fetched. In CI that is the common case. Composer resolves versions on every job, since MediaWiki core and its extensions commit no composer.lock, and each package is one /p2/ document, so a burst of jobs fetched each document once per job. Metadata misses now follow the same pattern as artifacts: the first caller fetches, the rest wait for its result, and the caller that takes the key rechecks the cache first so a fetch that just finished is not repeated. The key includes the Accept and Accept-Encoding headers, which change the bytes upstream returns (npm's abbreviated and full documents), and whether the caller validates the response. One difference from artifacts: the shared fetch is detached from the first caller's cancellation. Artifact fetches keep it for scanning and mirroring, neither of which applies to metadata, and otherwise one job disconnecting would fail every job waiting on the same document. runScan detaches the same way, and the HTTP client's timeout still bounds the fetch. This covers every ecosystem that goes through fetchOrCacheMetadata (Composer, npm, PyPI, Maven, Cargo, Swift, Pub, Helm, NuGet), with metadata caching on or off. Streamed metadata with caching off is not coalesced. On an 8 vCPU test instance with caching off, 10 concurrent clients requesting the npm metadata of 1,263 packages went from 12,630 upstream requests to 1,898. With cache_metadata on, 20 clients requesting 1,538 packages made exactly 1,538. * Run each caller's validate on a shared metadata fetch A caller that joined another's metadata fetch got the shared bytes without running its own validate. NuGet's cooldown path uses validate to decode the document for the request, so a joined download found no publication date and was allowed, bypassing the cooldown. coalesceMetadata now reports whether the caller joined, and coalescedMetadataMiss runs that caller's validate on the shared bytes. Validation before caching is unchanged: the first caller still runs it inside the fetch. TestNuGetCooldownConcurrentDownloads reproduces the bypass through NuGetHandler.Routes() with two overlapping downloads of a version inside its cooldown. * Support npm content-addressed tarball URLs * Add cooldown package pattern overrides (#407) * Add cooldown package pattern overrides * Expose full cooldown decisions through pattern policy * Reject unsafe cooldown pattern syntax * Fix cached package ecosystem filters * Update supportedEcosystems comment for filter pills --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: pinguinfuss Co-authored-by: Andrew Nesbitt Co-authored-by: montehurd Co-authored-by: Andrew Rich Co-authored-by: Abhinav Gautam Co-authored-by: montehurd Co-authored-by: wickedOne Co-authored-by: Wicliff Wolda Co-authored-by: Claude Opus 5 (1M context) Co-authored-by: Daniil Skripchenko <90247855+DANIILSKRIPCHENKO@users.noreply.github.com> Co-authored-by: PhantomPhoton <120879831+PhantomPhoton@users.noreply.github.com> --- internal/database/queries.go | 35 +++++ .../database/queries_packages_list_test.go | 45 ++++++ internal/server/dashboard.go | 106 +++++++++---- internal/server/packages_list_page_test.go | 148 ++++++++++++++++++ internal/server/server.go | 31 ++-- internal/server/static/packages-list.js | 6 + internal/server/templates.go | 1 + .../server/templates/pages/packages_list.html | 65 ++++---- internal/server/templates_test.go | 49 +++++- 9 files changed, 402 insertions(+), 84 deletions(-) create mode 100644 internal/server/packages_list_page_test.go create mode 100644 internal/server/static/packages-list.js diff --git a/internal/database/queries.go b/internal/database/queries.go index 4946133..9503022 100644 --- a/internal/database/queries.go +++ b/internal/database/queries.go @@ -1035,6 +1035,41 @@ func (db *DB) ListCachedPackages(ecosystem string, sortBy string, limit int, off return packages, nil } +// CountCachedPackagesByEcosystem counts packages with at least one stored artifact. +func (db *DB) CountCachedPackagesByEcosystem() (map[string]int64, error) { + hasArtifacts, err := db.HasTable("artifacts") + if err != nil { + return nil, err + } + counts := make(map[string]int64) + if !hasArtifacts { + return counts, nil + } + + rows, err := db.Query(` + SELECT p.ecosystem, COUNT(DISTINCT p.purl) + FROM packages p + JOIN versions v ON v.package_purl = p.purl + JOIN artifacts a ON a.version_purl = v.purl + WHERE a.storage_path IS NOT NULL + GROUP BY p.ecosystem + `) + if err != nil { + return nil, err + } + defer func() { _ = rows.Close() }() + + for rows.Next() { + var ecosystem string + var count int64 + if err := rows.Scan(&ecosystem, &count); err != nil { + return nil, err + } + counts[ecosystem] = count + } + return counts, rows.Err() +} + func (db *DB) CountCachedPackages(ecosystem string) (int64, error) { hasArtifacts, err := db.HasTable("artifacts") if err != nil { diff --git a/internal/database/queries_packages_list_test.go b/internal/database/queries_packages_list_test.go index fc9356c..e30d364 100644 --- a/internal/database/queries_packages_list_test.go +++ b/internal/database/queries_packages_list_test.go @@ -2,12 +2,57 @@ package database import ( "database/sql" + "maps" "testing" "time" ) const testEcosystemNPM = "npm" +func TestCountCachedPackagesByEcosystem(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + counts, err := db.CountCachedPackagesByEcosystem() + if err != nil || len(counts) != 0 { + t.Fatalf("empty cache counts = %v, error = %v", counts, err) + } + + seedArtifact(t, db, "npm", "express", "1.0.0", 100, 1) + seedArtifact(t, db, "npm", "express", "2.0.0", 200, 1) + seedArtifact(t, db, "npm", "lodash", "1.0.0", 100, 1) + seedArtifact(t, db, "cargo", "serde", "1.0.0", 100, 1) + seedArtifact(t, db, "pypi", "evicted", "1.0.0", 100, 1) + if _, err := db.Exec(`UPDATE artifacts SET storage_path = NULL WHERE version_purl = 'pkg:pypi/evicted@1.0.0'`); err != nil { + t.Fatal(err) + } + if err := db.UpsertPackage(&Package{PURL: "pkg:gem/metadata-only", Ecosystem: "gem", Name: "metadata-only"}); err != nil { + t.Fatal(err) + } + if err := db.UpsertArtifact(&Artifact{ + VersionPURL: "pkg:npm/express@1.0.0", + Filename: "extra.tgz", + UpstreamURL: "https://example.test/extra.tgz", + StoragePath: sql.NullString{String: "extra.tgz", Valid: true}, + }); err != nil { + t.Fatal(err) + } + + counts, err = db.CountCachedPackagesByEcosystem() + if err != nil { + t.Fatal(err) + } + want := map[string]int64{"npm": 2, "cargo": 1} + if !maps.Equal(counts, want) { + t.Fatalf("counts = %v, want %v", counts, want) + } + for ecosystem, count := range counts { + listed, err := db.CountCachedPackages(ecosystem) + if err != nil || count != listed { + t.Errorf("%s count = %d, list count = %d, error = %v", ecosystem, count, listed, err) + } + } + }) +} + func setupListCachedPackagesDB(t *testing.T) *DB { t.Helper() diff --git a/internal/server/dashboard.go b/internal/server/dashboard.go index a3ac9eb..05643f6 100644 --- a/internal/server/dashboard.go +++ b/internal/server/dashboard.go @@ -2,6 +2,7 @@ package server import ( "html/template" + "sort" "strings" "github.com/git-pkgs/proxy/internal/database" @@ -110,19 +111,27 @@ type SearchResultItem struct { // PackagesListPageData contains data for rendering the packages list page. type PackagesListPageData struct { Layout - Ecosystem string - SortBy string - Results []SearchResultItem - Count int - Page int - PerPage int - TotalPages int + Ecosystem string + SortBy string + Results []SearchResultItem + Count int + TotalPackages int64 + EcosystemFilters []EcosystemFilter + Page int + PerPage int + TotalPages int +} + +// EcosystemFilter represents an ecosystem filter pill with a package count. +type EcosystemFilter struct { + Ecosystem string + Count int64 } func supportedEcosystems() []string { // this list should be kept sorted in lexicographic order so - // that the 'select' list in the UI will be in the expected - // order + // that the ecosystem filter pills in the UI will be in the + // expected order return []string{ "alpine", "cargo", @@ -157,51 +166,84 @@ func ecosystemBadgeLabel(ecosystem string) string { } } -func ecosystemBadgeClasses(ecosystem string) string { - base := "inline-flex items-center px-2 py-0.5 rounded text-xs font-medium" - +func ecosystemColorClasses(ecosystem string) string { switch ecosystem { case "npm", "maven": - return base + " bg-red-100 text-red-700 dark:bg-red-900/50 dark:text-red-300" + return "bg-red-100 text-red-700 dark:bg-red-900/50 dark:text-red-300" case "cargo": - return base + " bg-orange-100 text-orange-700 dark:bg-orange-900/50 dark:text-orange-300" + return "bg-orange-100 text-orange-700 dark:bg-orange-900/50 dark:text-orange-300" case "gem": - return base + " bg-pink-100 text-pink-700 dark:bg-pink-900/50 dark:text-pink-300" - case "go": - return base + " bg-cyan-100 text-cyan-700 dark:bg-cyan-900/50 dark:text-cyan-300" + return "bg-pink-100 text-pink-700 dark:bg-pink-900/50 dark:text-pink-300" + case "go", "golang": + return "bg-cyan-100 text-cyan-700 dark:bg-cyan-900/50 dark:text-cyan-300" case "hex": - return base + " bg-purple-100 text-purple-700 dark:bg-purple-900/50 dark:text-purple-300" + return "bg-purple-100 text-purple-700 dark:bg-purple-900/50 dark:text-purple-300" case "pub": - return base + " bg-blue-100 text-blue-700 dark:bg-blue-900/50 dark:text-blue-300" + return "bg-blue-100 text-blue-700 dark:bg-blue-900/50 dark:text-blue-300" case "pypi": - return base + " bg-yellow-100 text-yellow-700 dark:bg-yellow-900/50 dark:text-yellow-300" + return "bg-yellow-100 text-yellow-700 dark:bg-yellow-900/50 dark:text-yellow-300" case "nuget": - return base + " bg-indigo-100 text-indigo-700 dark:bg-indigo-900/50 dark:text-indigo-300" + return "bg-indigo-100 text-indigo-700 dark:bg-indigo-900/50 dark:text-indigo-300" case "composer": - return base + " bg-violet-100 text-violet-700 dark:bg-violet-900/50 dark:text-violet-300" + return "bg-violet-100 text-violet-700 dark:bg-violet-900/50 dark:text-violet-300" case "conan": - return base + " bg-teal-100 text-teal-700 dark:bg-teal-900/50 dark:text-teal-300" + return "bg-teal-100 text-teal-700 dark:bg-teal-900/50 dark:text-teal-300" case "conda": - return base + " bg-green-100 text-green-700 dark:bg-green-900/50 dark:text-green-300" + return "bg-green-100 text-green-700 dark:bg-green-900/50 dark:text-green-300" case "cran": - return base + " bg-slate-100 text-slate-700 dark:bg-slate-800 dark:text-slate-300" + return "bg-slate-100 text-slate-700 dark:bg-slate-800 dark:text-slate-300" case "julia": - return base + " bg-emerald-100 text-emerald-700 dark:bg-emerald-900/50 dark:text-emerald-300" + return "bg-emerald-100 text-emerald-700 dark:bg-emerald-900/50 dark:text-emerald-300" case "swift": - return base + " bg-orange-100 text-orange-700 dark:bg-orange-900/50 dark:text-orange-300" + return "bg-orange-100 text-orange-700 dark:bg-orange-900/50 dark:text-orange-300" case "oci": - return base + " bg-sky-100 text-sky-700 dark:bg-sky-900/50 dark:text-sky-300" + return "bg-sky-100 text-sky-700 dark:bg-sky-900/50 dark:text-sky-300" case "deb": - return base + " bg-red-100 text-red-800 dark:bg-red-900/50 dark:text-red-300" + return "bg-red-100 text-red-800 dark:bg-red-900/50 dark:text-red-300" case "rpm": - return base + " bg-amber-100 text-amber-800 dark:bg-amber-900/50 dark:text-amber-300" + return "bg-amber-100 text-amber-800 dark:bg-amber-900/50 dark:text-amber-300" case "alpine": - return base + " bg-lime-100 text-lime-800 dark:bg-lime-900/50 dark:text-lime-300" + return "bg-lime-100 text-lime-800 dark:bg-lime-900/50 dark:text-lime-300" default: - return base + " bg-gray-100 text-gray-700 dark:bg-gray-800 dark:text-gray-300" + return "bg-gray-100 text-gray-700 dark:bg-gray-800 dark:text-gray-300" } } +func ecosystemBadgeClasses(ecosystem string) string { + return "inline-flex items-center px-2 py-0.5 rounded text-xs font-medium " + ecosystemColorClasses(ecosystem) +} + +func ecosystemPillClasses(ecosystem string) string { + return "inline-flex items-center gap-1.5 px-3 py-1.5 rounded-full text-xs font-medium hover:opacity-90 " + ecosystemColorClasses(ecosystem) +} + +func buildEcosystemFilters(counts map[string]int64) []EcosystemFilter { + var filters []EcosystemFilter + seen := make(map[string]bool, len(counts)) + + for _, eco := range supportedEcosystems() { + count := counts[eco] + if count > 0 { + filters = append(filters, EcosystemFilter{Ecosystem: eco, Count: count}) + seen[eco] = true + } + } + + var extra []string + for eco, count := range counts { + if count > 0 && !seen[eco] { + extra = append(extra, eco) + } + } + sort.Strings(extra) + + for _, eco := range extra { + filters = append(filters, EcosystemFilter{Ecosystem: eco, Count: counts[eco]}) + } + + return filters +} + func getRegistryConfigs(baseURL string) []RegistryConfig { swiftInsecureFlag := "" if strings.HasPrefix(strings.ToLower(baseURL), "http://") { diff --git a/internal/server/packages_list_page_test.go b/internal/server/packages_list_page_test.go new file mode 100644 index 0000000..8fdbc29 --- /dev/null +++ b/internal/server/packages_list_page_test.go @@ -0,0 +1,148 @@ +package server + +import ( + "database/sql" + "fmt" + "html" + "net/http" + "net/http/httptest" + "net/url" + "regexp" + "strings" + "testing" + + "github.com/git-pkgs/proxy/internal/database" +) + +func seedFilterPackage(t *testing.T, db *database.DB, ecosystem, name string, cached ...bool) { + t.Helper() + pkgPURL := "pkg:" + ecosystem + "/" + name + if err := db.UpsertPackage(&database.Package{PURL: pkgPURL, Ecosystem: ecosystem, Name: name}); err != nil { + t.Fatal(err) + } + versionPURL := pkgPURL + "@1.0.0" + if err := db.UpsertVersion(&database.Version{PURL: versionPURL, PackagePURL: pkgPURL}); err != nil { + t.Fatal(err) + } + for i, stored := range cached { + filename := fmt.Sprintf("%s-%d.tgz", name, i) + if err := db.UpsertArtifact(&database.Artifact{ + VersionPURL: versionPURL, + Filename: filename, + UpstreamURL: "https://example.test/" + filename, + StoragePath: sql.NullString{String: filename, Valid: stored}, + }); err != nil { + t.Fatal(err) + } + } +} + +func TestPackagesListFilters(t *testing.T) { + ts := newTestServer(t) + defer ts.close() + seedFilterPackage(t, ts.db, "npm", "express", true, true, false) + seedFilterPackage(t, ts.db, "npm", "lodash", true) + seedFilterPackage(t, ts.db, "cargo", "serde", true) + seedFilterPackage(t, ts.db, "gem", "metadata-only") + seedFilterPackage(t, ts.db, "pypi", "evicted", false) + seedFilterPackage(t, ts.db, "npm", "uncached", false) + + for _, tc := range []struct { + ecosystem string + count int + names []string + }{ + {"", 3, []string{"express", "lodash", "serde"}}, + {"npm", 2, []string{"express", "lodash"}}, + {"cargo", 1, []string{"serde"}}, + {"gem", 0, nil}, + } { + t.Run("ecosystem="+tc.ecosystem, func(t *testing.T) { + req := httptest.NewRequest(http.MethodGet, "/ui/packages?sort=name&ecosystem="+tc.ecosystem, nil) + w := httptest.NewRecorder() + ts.handler.ServeHTTP(w, req) + if w.Code != http.StatusOK { + t.Fatalf("status = %d: %s", w.Code, w.Body.String()) + } + body := w.Body.String() + assertFilterPills(t, body, tc.ecosystem, "name", map[string]string{"": "3", "npm": "2", "cargo": "1"}) + if !strings.Contains(body, fmt.Sprintf("%d packages", tc.count)) { + t.Errorf("missing heading count %d", tc.count) + } + links := regexp.MustCompile(`]*)>\s*[^<]+]*>(\d+)`).FindAllStringSubmatch(body, -1) + if len(pills) != len(want) { + t.Fatalf("got %d pills, want %d", len(pills), len(want)) + } + seen := make(map[string]bool) + for _, pill := range pills { + link, err := url.Parse(html.UnescapeString(pill[1])) + if err != nil { + t.Fatal(err) + } + params := link.Query() + ecosystem := params.Get("ecosystem") + if seen[ecosystem] || pill[3] != want[ecosystem] { + t.Errorf("%q pill count = %s, want %s (duplicate: %v)", ecosystem, pill[3], want[ecosystem], seen[ecosystem]) + } + seen[ecosystem] = true + if params.Get("sort") != sortBy || params.Has("page") { + t.Errorf("pill must preserve sorting and reset pagination: %s", link) + } + if active := strings.Contains(pill[2], `aria-current="page"`); active != (selected == ecosystem) { + t.Errorf("%q active = %v, selected = %q", ecosystem, active, selected) + } + } +} + +func TestPackagesListFiltersAcrossPages(t *testing.T) { + ts := newTestServer(t) + defer ts.close() + for i := range 51 { + seedFilterPackage(t, ts.db, "npm", fmt.Sprintf("package-%02d", i), true) + } + seedFilterPackage(t, ts.db, "cargo", "serde", true) + w := httptest.NewRecorder() + ts.handler.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/ui/packages?ecosystem=npm&sort=name&page=2", nil)) + if w.Code != http.StatusOK { + t.Fatalf("status = %d: %s", w.Code, w.Body.String()) + } + body := w.Body.String() + assertFilterPills(t, body, "npm", "name", map[string]string{"": "52", "npm": "51", "cargo": "1"}) + for _, text := range []string{"51 packages in npm", "Page 2 of 2", `href="/ui/package/npm/package-50"`, `href="?ecosystem=npm&sort=name&page=1"`} { + if !strings.Contains(html.UnescapeString(body), text) { + t.Errorf("missing %q", text) + } + } + if strings.Contains(body, `href="/ui/package/npm/package-00"`) { + t.Error("page 2 includes package from page 1") + } +} + +func TestPackagesListFiltersEmptyCache(t *testing.T) { + ts := newTestServer(t) + defer ts.close() + seedFilterPackage(t, ts.db, "npm", "metadata-only") + w := httptest.NewRecorder() + ts.handler.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/ui/packages", nil)) + if w.Code != http.StatusOK { + t.Fatalf("status = %d: %s", w.Code, w.Body.String()) + } + assertFilterPills(t, w.Body.String(), "", defaultSortBy, map[string]string{"": "0"}) + if !strings.Contains(w.Body.String(), "No cached packages found") { + t.Error("missing empty cache message") + } +} diff --git a/internal/server/server.go b/internal/server/server.go index c5e08f1..7494b94 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -821,10 +821,19 @@ func (s *Server) handlePackagesList(w http.ResponseWriter, r *http.Request) { return } - total, err := s.db.CountCachedPackages(ecosystem) + counts, err := s.db.CountCachedPackagesByEcosystem() if err != nil { s.logger.Error("failed to count packages", "error", err) - total = 0 + http.Error(w, "failed to count packages", http.StatusInternalServerError) + return + } + var totalPackages int64 + for _, count := range counts { + totalPackages += count + } + total := totalPackages + if ecosystem != "" { + total = counts[ecosystem] } items := make([]SearchResultItem, len(packages)) @@ -860,14 +869,16 @@ func (s *Server) handlePackagesList(w http.ResponseWriter, r *http.Request) { totalPages := int((total + int64(limit) - 1) / int64(limit)) data := PackagesListPageData{ - Layout: s.layoutFor(r), - Ecosystem: ecosystem, - SortBy: sortBy, - Results: items, - Count: int(total), - Page: page, - PerPage: limit, - TotalPages: totalPages, + Layout: s.layoutFor(r), + Ecosystem: ecosystem, + SortBy: sortBy, + Results: items, + Count: int(total), + TotalPackages: totalPackages, + EcosystemFilters: buildEcosystemFilters(counts), + Page: page, + PerPage: limit, + TotalPages: totalPages, } if err := s.templates.Render(w, "packages_list", data); err != nil { diff --git a/internal/server/static/packages-list.js b/internal/server/static/packages-list.js new file mode 100644 index 0000000..06f09ba --- /dev/null +++ b/internal/server/static/packages-list.js @@ -0,0 +1,6 @@ +document.getElementById('sort-by').addEventListener('change', function(e) { + const params = new URLSearchParams(window.location.search); + params.set('sort', e.target.value); + params.delete('page'); + window.location.href = '/ui/packages?' + params.toString(); +}); diff --git a/internal/server/templates.go b/internal/server/templates.go index 010abad..f37f4b6 100644 --- a/internal/server/templates.go +++ b/internal/server/templates.go @@ -29,6 +29,7 @@ func (t *Templates) load() error { "sub": func(a, b int) int { return a - b }, "supportedEcosystems": supportedEcosystems, "ecosystemBadgeClass": ecosystemBadgeClasses, + "ecosystemPillClass": ecosystemPillClasses, "ecosystemBadgeLabel": ecosystemBadgeLabel, "dict": templateDict, } diff --git a/internal/server/templates/pages/packages_list.html b/internal/server/templates/pages/packages_list.html index 95897db..61e73c0 100644 --- a/internal/server/templates/pages/packages_list.html +++ b/internal/server/templates/pages/packages_list.html @@ -4,23 +4,33 @@

Cached Packages

- {{.Count}} packages{{if .Ecosystem}} in {{.Ecosystem}}{{end}} + {{.Count}} packages{{if .Ecosystem}} in {{ecosystemBadgeLabel .Ecosystem}}{{end}}

-
-
- - +
-
+
- @@ -53,11 +63,14 @@ {{end}}
-
- {{.Hits}} hits - {{.SizeFormatted}} +
+ + {{.Hits}} + hits + + {{.SizeFormatted}} {{if .CachedAt}} - {{.CachedAt}} + {{.CachedAt}} {{end}}
@@ -66,7 +79,7 @@
{{else}}
-

No cached packages found{{if .Ecosystem}} in {{.Ecosystem}}{{end}}

+

No cached packages found{{if .Ecosystem}} in {{ecosystemBadgeLabel .Ecosystem}}{{end}}

Return to dashboard
{{end}} @@ -93,23 +106,5 @@ {{end}} - + {{end}} diff --git a/internal/server/templates_test.go b/internal/server/templates_test.go index c68a69a..fbd20c3 100644 --- a/internal/server/templates_test.go +++ b/internal/server/templates_test.go @@ -103,13 +103,15 @@ func TestTemplatesRenderAllPages(t *testing.T) { TotalPages: 0, }}, {"packages_list", PackagesListPageData{ - Ecosystem: "", - SortBy: defaultSortBy, - Results: []SearchResultItem{{Ecosystem: "npm", Name: "express", Hits: 200, SizeFormatted: "2 MB"}}, - Count: 1, - Page: 1, - PerPage: 50, - TotalPages: 1, + Ecosystem: "", + SortBy: defaultSortBy, + Results: []SearchResultItem{{Ecosystem: "npm", Name: "express", Hits: 200, SizeFormatted: "2 MB"}}, + Count: 1, + TotalPackages: 1, + EcosystemFilters: []EcosystemFilter{{Ecosystem: "npm", Count: 1}}, + Page: 1, + PerPage: 50, + TotalPages: 1, }}, {"package_show", PackageShowData{ Package: &database.Package{ @@ -608,6 +610,39 @@ func TestEcosystemBadgeClasses(t *testing.T) { } } +func TestBuildEcosystemFilters(t *testing.T) { + filters := buildEcosystemFilters(map[string]int64{ + "npm": 3, + "cargo": 2, + "empty": 0, + }) + + if len(filters) != 2 { + t.Fatalf("expected 2 filters, got %d", len(filters)) + } + if filters[0].Ecosystem != "cargo" || filters[0].Count != 2 { + t.Errorf("first filter = %#v, want cargo with count 2", filters[0]) + } + if filters[1].Ecosystem != "npm" || filters[1].Count != 3 { + t.Errorf("second filter = %#v, want npm with count 3", filters[1]) + } + + extra := buildEcosystemFilters(map[string]int64{"custom": 1}) + if len(extra) != 1 || extra[0].Ecosystem != "custom" { + t.Errorf("unexpected extra filters: %#v", extra) + } +} + +func TestEcosystemPillClasses(t *testing.T) { + classes := ecosystemPillClasses("npm") + if !strings.Contains(classes, "rounded-full") { + t.Error("pill classes should use rounded-full") + } + if !strings.Contains(classes, "bg-red-100") { + t.Error("npm pill should use npm colors") + } +} + func TestFormatSize(t *testing.T) { tests := []struct { bytes int64 From 1584499ac374b0f3de500881298ca5fc9ad5be3b Mon Sep 17 00:00:00 2001 From: simon chrzanowski Date: Thu, 8 Oct 2026 07:36:33 +0200 Subject: [PATCH 178/633] Cut memory use of npm metadata rewriting (#413) Rewriting a packument decoded the whole document into map[string]any and encoded it again. For a full packument of tens of megabytes that costs many times the document size per request, and with cooldown enabled (which skips the rewrite cache and forces full packuments) it happens on every request. - Rewrite npm metadata in place: walk the raw JSON for member offsets, decode only the version names, time and dist-tags for the existing filters, and assemble the response from slices of the original bytes with only tarball URLs (and filtered time/dist-tags) written fresh. Untouched values now reach clients byte for byte instead of round-tripping through encoding/json. - Look up versions[v].dist.tarball and time[v] directly on the tarball download path instead of decoding every version. - Record the stored metadata's SHA-256 in metadata_cache.content_digest and key the rewrite cache by it, so a repeat npm or Composer request inside the TTL is served from the rewrite cache without reading or hashing the stored document. On a synthetic 25 MB packument with 5000 versions, rewriteMetadata drops from ~150-174 MB / 706k allocations per call to ~31 MB / 90k, and runs in about two thirds of the time. npmVersionTarball drops from ~1 MB / 10k allocations to one allocation. --- internal/handler/composer.go | 6 + internal/handler/handler.go | 11 +- internal/handler/jsonscan.go | 213 ++++++++++++++ internal/handler/jsonscan_test.go | 98 +++++++ internal/handler/npm.go | 308 +++++++++++++++++---- internal/handler/npm_rewrite_bench_test.go | 61 ++++ internal/handler/npm_rewrite_test.go | 272 ++++++++++++++++++ internal/handler/rewrite_cache.go | 54 +++- 8 files changed, 957 insertions(+), 66 deletions(-) create mode 100644 internal/handler/jsonscan.go create mode 100644 internal/handler/jsonscan_test.go create mode 100644 internal/handler/npm_rewrite_bench_test.go create mode 100644 internal/handler/npm_rewrite_test.go diff --git a/internal/handler/composer.go b/internal/handler/composer.go index 2f475ac..3c10f6c 100644 --- a/internal/handler/composer.go +++ b/internal/handler/composer.go @@ -98,6 +98,12 @@ func (h *ComposerHandler) handlePackageMetadata(w http.ResponseWriter, r *http.R upstreamURL := fmt.Sprintf("%s/p2/%s/%s.json", h.repoURL, vendor, pkg) + if rewritten, ok := h.proxy.storedRewrite("composer", packageName, h.proxyURL, packageName); ok { + w.Header().Set(headerContentType, "application/json") + _, _ = w.Write(rewritten) + return + } + body, _, err := h.proxy.FetchOrCacheMetadata(r.Context(), "composer", packageName, upstreamURL) if err != nil { if errors.Is(err, ErrUpstreamNotFound) { diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 20dce9a..449fcef 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -1383,7 +1383,7 @@ func (p *Proxy) cacheMetadataBlob(ctx context.Context, ecosystem, cacheKey, stor return } - size, _, err := p.Storage.Store(ctx, storagePath, bytes.NewReader(meta.body)) + size, hash, err := p.Storage.Store(ctx, storagePath, bytes.NewReader(meta.body)) if err != nil { p.Logger.Warn("failed to cache metadata", "ecosystem", ecosystem, "key", cacheKey, "error", err) return @@ -1396,9 +1396,12 @@ func (p *Proxy) cacheMetadataBlob(ctx context.Context, ecosystem, cacheKey, stor ETag: sql.NullString{String: meta.etag, Valid: meta.etag != ""}, ContentType: sql.NullString{String: meta.contentType, Valid: meta.contentType != ""}, ContentEncoding: sql.NullString{String: meta.contentEncoding, Valid: meta.contentEncoding != ""}, - Size: sql.NullInt64{Int64: size, Valid: true}, - LastModified: sql.NullTime{Time: meta.lastModified, Valid: !meta.lastModified.IsZero()}, - FetchedAt: sql.NullTime{Time: time.Now(), Valid: true}, + // The digest identifies the stored bytes, so a rewrite cached for them + // can be found without reading them back (see storedRewrite). + ContentDigest: sql.NullString{String: "sha256:" + hash, Valid: hash != ""}, + Size: sql.NullInt64{Int64: size, Valid: true}, + LastModified: sql.NullTime{Time: meta.lastModified, Valid: !meta.lastModified.IsZero()}, + FetchedAt: sql.NullTime{Time: time.Now(), Valid: true}, }) if err != nil { // The blob is written but the row describing it is not, so a later diff --git a/internal/handler/jsonscan.go b/internal/handler/jsonscan.go new file mode 100644 index 0000000..b1b1050 --- /dev/null +++ b/internal/handler/jsonscan.go @@ -0,0 +1,213 @@ +package handler + +import ( + "bytes" + "encoding/json" + "errors" +) + +// The helpers in this file walk a JSON document in place, reporting where +// object members sit as offsets into the original bytes. They let a handler +// read or replace one value in a large metadata document without decoding the +// rest of it into Go values, which for an npm packument costs many times the +// document's size. +// +// They check structure only as far as they need to find their way through +// the document. Callers that copy unvisited bytes into a response validate the +// whole document first with json.Valid. + +var errMalformedJSON = errors.New("malformed JSON") + +// errNotJSONObject is returned when a value expected to be an object is not. +var errNotJSONObject = errors.New("JSON value is not an object") + +// jsonMember is one member of a JSON object as offsets into the bytes that +// were scanned. The key span includes its quotes. +type jsonMember struct { + keyStart, keyEnd int + valStart, valEnd int +} + +func (m jsonMember) key(data []byte) []byte { return data[m.keyStart:m.keyEnd] } +func (m jsonMember) value(data []byte) []byte { return data[m.valStart:m.valEnd] } + +func skipJSONSpace(data []byte, i int) int { + for i < len(data) { + switch data[i] { + case ' ', '\t', '\n', '\r': + i++ + default: + return i + } + } + return i +} + +// skipJSONString returns the index just past the string starting at data[i]. +func skipJSONString(data []byte, i int) (int, error) { + for j := i + 1; j < len(data); j++ { + switch data[j] { + case '\\': + j++ + case '"': + return j + 1, nil + } + } + return 0, errMalformedJSON +} + +// skipJSONValue returns the index just past the value starting at data[i]. +func skipJSONValue(data []byte, i int) (int, error) { + if i >= len(data) { + return 0, errMalformedJSON + } + switch data[i] { + case '"': + return skipJSONString(data, i) + case '{', '[': + depth := 0 + for j := i; j < len(data); j++ { + switch data[j] { + case '"': + end, err := skipJSONString(data, j) + if err != nil { + return 0, err + } + j = end - 1 + case '{', '[': + depth++ + case '}', ']': + depth-- + if depth == 0 { + return j + 1, nil + } + } + } + return 0, errMalformedJSON + case '}', ']', ',', ':': + return 0, errMalformedJSON + default: + // A number, true, false or null runs to the next delimiter. + j := i + for j < len(data) { + switch data[j] { + case ',', '}', ']', ' ', '\t', '\n', '\r': + return j, nil + } + j++ + } + return j, nil + } +} + +// forEachJSONMember calls fn for each member of the object obj holds, in +// document order. It returns errNotJSONObject if obj is not an object, and +// stops early with fn's error if fn returns one. +func forEachJSONMember(obj []byte, fn func(jsonMember) error) error { + i := skipJSONSpace(obj, 0) + if i >= len(obj) || obj[i] != '{' { + return errNotJSONObject + } + i = skipJSONSpace(obj, i+1) + if i < len(obj) && obj[i] == '}' { + return nil + } + for { + if i >= len(obj) || obj[i] != '"' { + return errMalformedJSON + } + keyEnd, err := skipJSONString(obj, i) + if err != nil { + return err + } + colon := skipJSONSpace(obj, keyEnd) + if colon >= len(obj) || obj[colon] != ':' { + return errMalformedJSON + } + valStart := skipJSONSpace(obj, colon+1) + valEnd, err := skipJSONValue(obj, valStart) + if err != nil { + return err + } + if err := fn(jsonMember{keyStart: i, keyEnd: keyEnd, valStart: valStart, valEnd: valEnd}); err != nil { + return err + } + next := skipJSONSpace(obj, valEnd) + if next >= len(obj) { + return errMalformedJSON + } + switch obj[next] { + case ',': + i = skipJSONSpace(obj, next+1) + case '}': + return nil + default: + return errMalformedJSON + } + } +} + +// jsonKey decodes a quoted key as forEachJSONMember reports it. +func jsonKey(raw []byte) (string, error) { + if bytes.IndexByte(raw, '\\') < 0 { + return string(raw[1 : len(raw)-1]), nil + } + var key string + err := json.Unmarshal(raw, &key) + return key, err +} + +// jsonKeyIs reports whether a quoted key decodes to name. +func jsonKeyIs(raw []byte, name string) bool { + if bytes.IndexByte(raw, '\\') < 0 { + return string(raw[1:len(raw)-1]) == name + } + key, err := jsonKey(raw) + return err == nil && key == name +} + +// findJSONMember returns the member of obj named name. When the key repeats, +// the last one wins, as it does for JSON.parse and encoding/json. +func findJSONMember(obj []byte, name string) (jsonMember, bool, error) { + var found jsonMember + ok := false + err := forEachJSONMember(obj, func(m jsonMember) error { + if jsonKeyIs(m.key(obj), name) { + found, ok = m, true + } + return nil + }) + return found, ok, err +} + +// lookupJSON follows path through nested objects in doc and returns the +// value at its end. It returns nil and no error when a key along the path is +// missing or names something other than an object. +func lookupJSON(doc []byte, path ...string) ([]byte, error) { + value := doc + for _, name := range path { + m, ok, err := findJSONMember(value, name) + if errors.Is(err, errNotJSONObject) { + return nil, nil + } + if err != nil || !ok { + return nil, err + } + value = m.value(value) + } + return value, nil +} + +// lookupJSONString is lookupJSON for a string value. ok is false when the +// value is missing or is not a string. +func lookupJSONString(doc []byte, path ...string) (string, bool, error) { + raw, err := lookupJSON(doc, path...) + if err != nil || len(raw) == 0 || raw[0] != '"' { + return "", false, err + } + var s string + if err := json.Unmarshal(raw, &s); err != nil { + return "", false, err + } + return s, true, nil +} diff --git a/internal/handler/jsonscan_test.go b/internal/handler/jsonscan_test.go new file mode 100644 index 0000000..5003809 --- /dev/null +++ b/internal/handler/jsonscan_test.go @@ -0,0 +1,98 @@ +package handler + +import ( + "bytes" + "errors" + "testing" +) + +func TestLookupJSONString(t *testing.T) { + doc := []byte(`{ + "name": "demo", + "versions": { + "1.0.0": {"dist": {"tarball": "https://example.com/a.tgz", "shasum": "x"}}, + "2.0.0": {"readme": "a } tricky \" string ] with {brackets}", "dist": {"tarball": "https://example.com/b.tgz"}}, + "3.0.0": "not an object", + "4.0.0": {"dist": {"tarball": 42}} + }, + "time": {"1.0.0": "2020-01-01T00:00:00Z", "1.0.0": "2021-01-01T00:00:00Z"}, + "n": -1.5e3, "t": true, "z": null, "list": [1, {"a": []}, "]"] + }`) + + cases := []struct { + path []string + want string + wantOK bool + }{ + {[]string{"versions", "1.0.0", "dist", "tarball"}, "https://example.com/a.tgz", true}, + {[]string{"versions", "2.0.0", "dist", "tarball"}, "https://example.com/b.tgz", true}, + {[]string{"versions", "3.0.0", "dist", "tarball"}, "", false}, + {[]string{"versions", "4.0.0", "dist", "tarball"}, "", false}, + {[]string{"versions", "9.9.9", "dist", "tarball"}, "", false}, + {[]string{"time", "1.0.0"}, "2021-01-01T00:00:00Z", true}, // last duplicate wins + {[]string{"name"}, "demo", true}, + {[]string{"name", "x"}, "", false}, + } + for _, c := range cases { + got, ok, err := lookupJSONString(doc, c.path...) + if err != nil || ok != c.wantOK || got != c.want { + t.Errorf("lookup %v = %q, %v, %v; want %q, %v", c.path, got, ok, err, c.want, c.wantOK) + } + } +} + +func TestLookupJSONEscapedKey(t *testing.T) { + doc := []byte(`{"versions": {"1.0.0": {"dist": {"tarball": "u"}}}}`) + got, ok, err := lookupJSONString(doc, "versions", "1.0.0", "dist", "tarball") + if err != nil || !ok || got != "u" { + t.Errorf("lookup = %q, %v, %v", got, ok, err) + } +} + +func TestLookupJSONMalformed(t *testing.T) { + for _, doc := range []string{ + `{"versions": {"1.0.0": {"dist": `, + `{"versions" {}}`, + `{"versions": {"a": 1 "b": 2}}`, + `{"a": "unterminated}`, + `{"a": [1, 2}`, + } { + _, _, err := lookupJSONString([]byte(doc), "versions", "1.0.0", "dist", "tarball") + if !errors.Is(err, errMalformedJSON) { + t.Errorf("%s: err = %v, want errMalformedJSON", doc, err) + } + } +} + +func TestForEachJSONMemberOffsets(t *testing.T) { + doc := []byte(` { "a" : 1 , "b":{"c":[true,null]} ,"d":"x\"y" } `) + var got [][2]string + err := forEachJSONMember(doc, func(m jsonMember) error { + got = append(got, [2]string{string(m.key(doc)), string(m.value(doc))}) + return nil + }) + want := [][2]string{{`"a"`, `1`}, {`"b"`, `{"c":[true,null]}`}, {`"d"`, `"x\"y"`}} + if err != nil || len(got) != len(want) { + t.Fatalf("members = %v, %v", got, err) + } + for i := range want { + if got[i] != want[i] { + t.Errorf("member %d = %v, want %v", i, got[i], want[i]) + } + } + + if err := forEachJSONMember([]byte(`[1]`), func(jsonMember) error { return nil }); !errors.Is(err, errNotJSONObject) { + t.Errorf("array: err = %v, want errNotJSONObject", err) + } + if err := forEachJSONMember([]byte(`{}`), func(jsonMember) error { t.Error("called for empty object"); return nil }); err != nil { + t.Errorf("empty object: err = %v", err) + } +} + +func TestWriteFilteredJSONObject(t *testing.T) { + var out bytes.Buffer + err := writeFilteredJSONObject(&out, []byte(`{"a": 1, "b": {"x": 2}, "c": 3}`), func(k string) bool { return k != "b" }) + if err != nil || out.String() != `{"a": 1,"c": 3}` { + t.Errorf("filtered = %s, %v", out.String(), err) + } +} diff --git a/internal/handler/npm.go b/internal/handler/npm.go index 98ae10a..31c5341 100644 --- a/internal/handler/npm.go +++ b/internal/handler/npm.go @@ -6,8 +6,10 @@ import ( "errors" "fmt" "io" + "maps" "net/http" "net/url" + "reflect" "sort" "strings" "sync" @@ -178,6 +180,13 @@ func (h *NPMHandler) handlePackageMetadata(w http.ResponseWriter, r *http.Reques accept = contentTypeJSON } + if rewritten, ok := h.proxy.storedRewrite("npm", packageName, h.proxyURL, packageName); ok { + w.Header().Set(headerContentType, contentTypeJSON) + w.WriteHeader(http.StatusOK) + _, _ = w.Write(rewritten) + return + } + body, _, err := h.proxy.FetchOrCacheMetadata(r.Context(), "npm", packageName, upstreamURL, accept) if err != nil { if errors.Is(err, ErrUpstreamNotFound) { @@ -215,26 +224,235 @@ func (h *NPMHandler) handlePackageMetadata(w http.ResponseWriter, r *http.Reques // rewriteMetadata rewrites tarball URLs in npm package metadata to point at this proxy. // If cooldown is enabled, versions published too recently are filtered out. +// +// The document is never decoded as a whole. A full packument can run to tens +// of megabytes, and decoding it into generic maps took many times that in +// memory for every rewrite. Instead the version names, time map and dist-tags +// are decoded to decide what to keep, and the response is assembled from +// slices of the original bytes: only each version's tarball URL, and the time +// map and dist-tags when filtering changed them, are written fresh. Everything +// else reaches the client byte for byte as upstream sent it. func (h *NPMHandler) rewriteMetadata(packageName string, body []byte) ([]byte, error) { - var metadata map[string]any - if err := json.Unmarshal(body, &metadata); err != nil { + doc, err := indexNPMPackument(body) + if err != nil { return nil, err } - // Rewrite tarball URLs in versions - versions, ok := metadata["versions"].(map[string]any) - if !ok { + versionsRaw := doc.value(doc.versionsAt) + if len(versionsRaw) == 0 || versionsRaw[0] != '{' { if len(h.proxy.Denylist.Versions(canonicalPackagePURL("npm", packageName))) != 0 { return nil, errors.New("npm metadata has no versions object") } return body, nil // No versions to rewrite } + // The filters only look at which versions exist, so they run on a map of + // version names alongside the decoded time map and dist-tags. + versions := map[string]any{} + if err := forEachJSONMember(versionsRaw, func(m jsonMember) error { + version, err := jsonKey(m.key(versionsRaw)) + versions[version] = nil + return err + }); err != nil { + return nil, err + } + metadata := map[string]any{} + timeMap := doc.decodeObject(doc.timeAt, metadata, "time") + distTags := doc.decodeObject(doc.tagsAt, metadata, "dist-tags") + timeEntries := len(timeMap) + originalTags := maps.Clone(distTags) + h.applyCooldownFiltering(metadata, versions, packageName) h.applyDenylistFiltering(metadata, versions, packageName) - h.rewriteTarballURLs(versions, packageName) - return json.Marshal(metadata) + // Untouched members are copied as they are; the three the filters + // handled are written from their filtered state, and only when it changed. + return doc.write(func(out *bytes.Buffer, i int) error { + switch { + case i == doc.versionsAt: + return h.writeNPMVersions(out, packageName, versionsRaw, versions) + case i == doc.timeAt && timeMap != nil && len(timeMap) != timeEntries: + return writeFilteredJSONObject(out, doc.value(i), func(key string) bool { + _, ok := timeMap[key] + return ok + }) + case i == doc.tagsAt && distTags != nil && !reflect.DeepEqual(distTags, originalTags): + encoded, err := json.Marshal(distTags) + out.Write(encoded) + return err + default: + out.Write(doc.value(i)) + return nil + } + }) +} + +// npmPackument is a packument's top-level members as offsets into its bytes, +// with the positions of the members rewriteMetadata filters, or -1 for those +// it lacks. A key that repeats is recorded at its last occurrence, the one +// JSON.parse keeps. +type npmPackument struct { + body []byte + members []jsonMember + versionsAt, timeAt, tagsAt int +} + +func indexNPMPackument(body []byte) (*npmPackument, error) { + if !json.Valid(body) { + return nil, errors.New("npm metadata is not valid JSON") + } + doc := &npmPackument{body: body, versionsAt: -1, timeAt: -1, tagsAt: -1} + err := forEachJSONMember(body, func(m jsonMember) error { + switch key := m.key(body); { + case jsonKeyIs(key, "versions"): + doc.versionsAt = len(doc.members) + case jsonKeyIs(key, "time"): + doc.timeAt = len(doc.members) + case jsonKeyIs(key, "dist-tags"): + doc.tagsAt = len(doc.members) + } + doc.members = append(doc.members, m) + return nil + }) + return doc, err +} + +// value returns the raw value of member i, or nil when i is -1. +func (d *npmPackument) value(i int) []byte { + if i < 0 { + return nil + } + return d.members[i].value(d.body) +} + +// decodeObject decodes member i into metadata[name] and returns it when it is +// an object. The filters edit the returned map in place. +func (d *npmPackument) decodeObject(i int, metadata map[string]any, name string) map[string]any { + raw := d.value(i) + if raw == nil { + return nil + } + var value any + if err := json.Unmarshal(raw, &value); err != nil { + return nil + } + metadata[name] = value + object, _ := value.(map[string]any) + return object +} + +// write assembles the document in member order, with writeValue writing each +// member's value. Earlier duplicates of the filtered keys are dropped: they +// would carry unfiltered data, and JSON.parse ignores them anyway. +func (d *npmPackument) write(writeValue func(out *bytes.Buffer, i int) error) ([]byte, error) { + var out bytes.Buffer + out.Grow(len(d.body) + len(d.body)/8) + out.WriteByte('{') + first := true + for i, m := range d.members { + if i != d.versionsAt && i != d.timeAt && i != d.tagsAt { + key := m.key(d.body) + if jsonKeyIs(key, "versions") || jsonKeyIs(key, "time") || jsonKeyIs(key, "dist-tags") { + continue + } + } + if !first { + out.WriteByte(',') + } + first = false + out.Write(m.key(d.body)) + out.WriteByte(':') + if err := writeValue(&out, i); err != nil { + return nil, err + } + } + out.WriteByte('}') + return out.Bytes(), nil +} + +// writeNPMVersions writes the versions object, keeping the versions still in +// keep and pointing each kept version's tarball at this proxy. +func (h *NPMHandler) writeNPMVersions(out *bytes.Buffer, packageName string, versionsRaw []byte, keep map[string]any) error { + out.WriteByte('{') + first := true + err := forEachJSONMember(versionsRaw, func(m jsonMember) error { + version, err := jsonKey(m.key(versionsRaw)) + if err != nil { + return err + } + if _, ok := keep[version]; !ok { + return nil + } + if !first { + out.WriteByte(',') + } + first = false + out.Write(m.key(versionsRaw)) + out.WriteByte(':') + h.writeNPMVersion(out, packageName, version, m.value(versionsRaw)) + return nil + }) + out.WriteByte('}') + return err +} + +// writeNPMVersion writes one version entry with its dist.tarball pointing at +// this proxy. An entry without a string tarball is written unchanged. +func (h *NPMHandler) writeNPMVersion(out *bytes.Buffer, packageName, version string, raw []byte) { + dist, ok, err := findJSONMember(raw, "dist") + if err != nil || !ok { + out.Write(raw) + return + } + distRaw := dist.value(raw) + tarball, ok, err := findJSONMember(distRaw, "tarball") + if err != nil || !ok || distRaw[tarball.valStart] != '"' { + out.Write(raw) + return + } + var oldTarball string + if err := json.Unmarshal(tarball.value(distRaw), &oldTarball); err != nil { + out.Write(raw) + return + } + + newTarball := h.proxyTarballURL(packageName, version, oldTarball) + encoded, err := json.Marshal(newTarball) + if err != nil { + out.Write(raw) + return + } + out.Write(raw[:dist.valStart+tarball.valStart]) + out.Write(encoded) + out.Write(raw[dist.valStart+tarball.valEnd:]) + + h.proxy.Logger.Debug("rewrote tarball URL", + "package", packageName, "version", version, + "old", oldTarball, "new", newTarball) +} + +// writeFilteredJSONObject writes the object obj holds, keeping the members +// whose decoded key keep accepts. +func writeFilteredJSONObject(out *bytes.Buffer, obj []byte, keep func(string) bool) error { + out.WriteByte('{') + first := true + err := forEachJSONMember(obj, func(m jsonMember) error { + key, err := jsonKey(m.key(obj)) + if err != nil { + return err + } + if !keep(key) { + return nil + } + if !first { + out.WriteByte(',') + } + first = false + out.Write(obj[m.keyStart:m.valEnd]) + return nil + }) + out.WriteByte('}') + return err } // applyCooldownFiltering removes versions that are too recently published, @@ -294,44 +512,22 @@ func (h *NPMHandler) updateDistTagsLatest(metadata, versions, timeMap map[string } } -// rewriteTarballURLs rewrites all tarball URLs in version entries to point at this proxy. -func (h *NPMHandler) rewriteTarballURLs(versions map[string]any, packageName string) { - for version, vdata := range versions { - vmap, ok := vdata.(map[string]any) - if !ok { - continue - } - - dist, ok := vmap["dist"].(map[string]any) - if !ok { - continue - } - - tarball, ok := dist["tarball"].(string) - if !ok { - continue - } - - filename := tarball - if idx := strings.LastIndex(tarball, "/"); idx >= 0 { - filename = tarball[idx+1:] - } - if h.extractVersionFromFilename(packageName, filename) != version { - _, shortName, scoped := strings.Cut(packageName, "/") - if !scoped { - shortName = packageName - } - filename = shortName + "-" + version + ".tgz" - } - - escapedName := url.PathEscape(packageName) - newTarball := fmt.Sprintf("%s/npm/%s/-/%s", h.proxyURL, escapedName, filename) - dist["tarball"] = newTarball - - h.proxy.Logger.Debug("rewrote tarball URL", - "package", packageName, "version", version, - "old", tarball, "new", newTarball) +// proxyTarballURL returns the proxy URL that serves the tarball upstream +// lists at tarball for this version. +func (h *NPMHandler) proxyTarballURL(packageName, version, tarball string) string { + filename := tarball + if idx := strings.LastIndex(tarball, "/"); idx >= 0 { + filename = tarball[idx+1:] } + if h.extractVersionFromFilename(packageName, filename) != version { + _, shortName, scoped := strings.Cut(packageName, "/") + if !scoped { + shortName = packageName + } + filename = shortName + "-" + version + ".tgz" + } + + return fmt.Sprintf("%s/npm/%s/-/%s", h.proxyURL, url.PathEscape(packageName), filename) } // findNewestVersion returns the version string with the most recent timestamp @@ -435,18 +631,12 @@ func (h *NPMHandler) getTarball(r *http.Request, packageName, version, filename return h.proxy.GetOrFetchArtifactFromURL(r.Context(), "npm", packageName, version, filename, downloadURL) } +// npmVersionTarball returns versions[version].dist.tarball from a packument, +// or "" if it has none. It reads that one value in place rather than decoding +// every version, since it runs on each tarball the proxy has not cached yet. func npmVersionTarball(body []byte, version string) string { - var metadata struct { - Versions map[string]struct { - Dist struct { - Tarball string `json:"tarball"` - } `json:"dist"` - } `json:"versions"` - } - if err := json.Unmarshal(body, &metadata); err != nil { - return "" - } - return metadata.Versions[version].Dist.Tarball + tarball, _, _ := lookupJSONString(body, "versions", version, "dist", "tarball") + return tarball } func (h *NPMHandler) validateTarballURL(raw string) (string, error) { @@ -500,16 +690,12 @@ func (h *NPMHandler) versionInCooldown(packageName, version string, metadata fun return false } - var document struct { - Time map[string]string `json:"time"` - } - if err := json.Unmarshal(body, &document); err != nil { + published, ok, err := lookupJSONString(body, "time", version) + if err != nil { h.proxy.Logger.Warn("cooldown: could not parse npm metadata for download check", "package", packageName, "version", version, "error", err) return false } - - published, ok := document.Time[version] if !ok { return false } diff --git a/internal/handler/npm_rewrite_bench_test.go b/internal/handler/npm_rewrite_bench_test.go new file mode 100644 index 0000000..b200c5f --- /dev/null +++ b/internal/handler/npm_rewrite_bench_test.go @@ -0,0 +1,61 @@ +package handler + +import ( + "fmt" + "strings" + "testing" +) + +// benchPackument builds a full packument shaped like a large real one: +// thousands of versions, each with dependencies, scripts and a readme. +func benchPackument(versions int) []byte { + var b strings.Builder + readme := strings.Repeat("Some documentation text. ", 160) + b.WriteString(`{"_id":"big","name":"big","dist-tags":{"latest":"1.0.` + fmt.Sprint(versions-1) + `"},"versions":{`) + for i := range versions { + if i > 0 { + b.WriteByte(',') + } + v := fmt.Sprintf("1.0.%d", i) + fmt.Fprintf(&b, `%q:{"name":"big","version":%q,"description":"A big package","main":"index.js",`+ + `"scripts":{"test":"node test.js","build":"tsc -p ."},"dependencies":{`, v, v) + for d := range 25 { + if d > 0 { + b.WriteByte(',') + } + fmt.Fprintf(&b, `"dep-%d":"^%d.0.0"`, d, d) + } + fmt.Fprintf(&b, `},"readme":%q,"dist":{"shasum":"0123456789abcdef0123456789abcdef01234567",`+ + `"integrity":"sha512-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==",`+ + `"tarball":"https://registry.npmjs.org/big/-/big-%s.tgz"}}`, readme, v) + } + b.WriteString(`},"time":{"created":"2015-01-01T00:00:00.000Z"`) + for i := range versions { + fmt.Fprintf(&b, `,"1.0.%d":"2016-01-01T00:00:00.000Z"`, i) + } + b.WriteString(`}}`) + return []byte(b.String()) +} + +func BenchmarkNPMRewriteMetadata(b *testing.B) { + body := benchPackument(5000) + h := &NPMHandler{proxy: testProxy(), proxyURL: "http://proxy.example"} + b.SetBytes(int64(len(body))) + b.ReportAllocs() + for b.Loop() { + if _, err := h.rewriteMetadata("big", body); err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkNPMVersionTarball(b *testing.B) { + body := benchPackument(5000) + b.SetBytes(int64(len(body))) + b.ReportAllocs() + for b.Loop() { + if npmVersionTarball(body, "1.0.2500") == "" { + b.Fatal("tarball not found") + } + } +} diff --git a/internal/handler/npm_rewrite_test.go b/internal/handler/npm_rewrite_test.go new file mode 100644 index 0000000..6503ae1 --- /dev/null +++ b/internal/handler/npm_rewrite_test.go @@ -0,0 +1,272 @@ +package handler + +import ( + "encoding/json" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/git-pkgs/cooldown" +) + +// TestNPMRewriteMetadataKeepsUpstreamBytes checks that only tarball URLs +// change: everything else, including numbers encoding/json would round +// through float64 and characters it would HTML-escape, is copied verbatim. +func TestNPMRewriteMetadataKeepsUpstreamBytes(t *testing.T) { + h := &NPMHandler{proxy: testProxy(), proxyURL: "http://proxy"} + input := `{"name":"demo","_big":12345678901234567890,"readme":"&",` + + `"versions":{"1.0.0":{"version":"1.0.0","dist":{"shasum":"s","tarball":"https://registry.npmjs.org/demo/-/demo-1.0.0.tgz","integrity":"i"}}}}` + + out, err := h.rewriteMetadata("demo", []byte(input)) + if err != nil { + t.Fatal(err) + } + want := strings.Replace(input, "https://registry.npmjs.org/demo/-/demo-1.0.0.tgz", "http://proxy/npm/demo/-/demo-1.0.0.tgz", 1) + if string(out) != want { + t.Errorf("rewrite =\n%s\nwant\n%s", out, want) + } +} + +func TestNPMRewriteMetadataUnusualEntries(t *testing.T) { + h := &NPMHandler{proxy: testProxy(), proxyURL: "http://proxy"} + input := `{ + "versions": { + "1.0.0": "not an object", + "2.0.0": {"dist": "no object either"}, + "3.0.0": {"dist": {"tarball": 7}}, + "4.0.0": {"dist": {"tarball": "https://registry.npmjs.org/demo/-/renamed.tgz"}}, + "5.0.0": {"dist": {}} + } + }` + out, err := h.rewriteMetadata("demo", []byte(input)) + if err != nil { + t.Fatal(err) + } + var doc struct { + Versions map[string]json.RawMessage `json:"versions"` + } + if err := json.Unmarshal(out, &doc); err != nil { + t.Fatalf("output is not JSON: %v\n%s", err, out) + } + for v, want := range map[string]string{ + "1.0.0": `"not an object"`, + "2.0.0": `{"dist": "no object either"}`, + "3.0.0": `{"dist": {"tarball": 7}}`, + "4.0.0": `{"dist": {"tarball": "http://proxy/npm/demo/-/demo-4.0.0.tgz"}}`, + "5.0.0": `{"dist": {}}`, + } { + if got := string(doc.Versions[v]); got != want { + t.Errorf("%s = %s, want %s", v, got, want) + } + } +} + +func TestNPMRewriteMetadataInvalidJSON(t *testing.T) { + h := &NPMHandler{proxy: testProxy(), proxyURL: "http://proxy"} + for _, input := range []string{`{"versions": {}`, `[1]`, `{"versions": {}} trailing`} { + if _, err := h.rewriteMetadata("demo", []byte(input)); err == nil { + t.Errorf("%s: expected an error", input) + } + } +} + +func TestNPMRewriteMetadataNoVersions(t *testing.T) { + h := &NPMHandler{proxy: testProxy(), proxyURL: "http://proxy"} + input := `{"name":"demo","versions":[]}` + out, err := h.rewriteMetadata("demo", []byte(input)) + if err != nil || string(out) != input { + t.Errorf("rewrite = %s, %v; want input unchanged", out, err) + } + + setTestDenylist(t, h.proxy, "pkg:npm/demo@1.0.0") + if _, err := h.rewriteMetadata("demo", []byte(input)); err == nil { + t.Error("expected an error when a denylisted package has no versions object") + } +} + +// TestNPMRewriteMetadataCooldownAndDenylist runs both filters together and +// checks that versions, their time entries and the tags pointing at them are +// all removed, with latest moved to the newest version left. +func TestNPMRewriteMetadataCooldownAndDenylist(t *testing.T) { + now := time.Now() + ts := func(age time.Duration) string { return now.Add(-age).UTC().Format(time.RFC3339) } + + proxy := testProxy() + proxy.Cooldown = &cooldown.Config{Default: "3d"} + setTestDenylist(t, proxy, "pkg:npm/demo@2.0.0") + h := &NPMHandler{proxy: proxy, proxyURL: "http://proxy"} + + version := func(v string) string { + return fmt.Sprintf(`%q:{"version":%q,"dist":{"tarball":"https://registry.npmjs.org/demo/-/demo-%s.tgz"}}`, v, v, v) + } + input := `{"name":"demo",` + + `"dist-tags":{"latest":"3.0.0","beta":"2.0.0","next":"1.0.0"},` + + `"versions":{` + version("1.0.0") + `,` + version("2.0.0") + `,` + version("3.0.0") + `},` + + `"time":{"created":"` + ts(1000*time.Hour) + `","1.0.0":"` + ts(900*time.Hour) + `","2.0.0":"` + ts(800*time.Hour) + `","3.0.0":"` + ts(time.Hour) + `"}}` + + out, err := h.rewriteMetadata("demo", []byte(input)) + if err != nil { + t.Fatal(err) + } + var doc struct { + DistTags map[string]string `json:"dist-tags"` + Versions map[string]json.RawMessage `json:"versions"` + Time map[string]string `json:"time"` + } + if err := json.Unmarshal(out, &doc); err != nil { + t.Fatalf("output is not JSON: %v\n%s", err, out) + } + if len(doc.Versions) != 1 || doc.Versions["1.0.0"] == nil { + t.Errorf("versions = %v, want only 1.0.0", keysOf(doc.Versions)) + } + if _, ok := doc.Time["2.0.0"]; ok { + t.Error("time still lists denylisted 2.0.0") + } + if _, ok := doc.Time["3.0.0"]; ok { + t.Error("time still lists 3.0.0, which is inside cooldown") + } + if _, ok := doc.Time["created"]; !ok { + t.Error("time lost its created entry") + } + want := map[string]string{"latest": "1.0.0", "next": "1.0.0"} + if fmt.Sprint(doc.DistTags) != fmt.Sprint(want) { + t.Errorf("dist-tags = %v, want %v", doc.DistTags, want) + } + if !strings.Contains(string(doc.Versions["1.0.0"]), "http://proxy/npm/demo/-/demo-1.0.0.tgz") { + t.Errorf("1.0.0 tarball not rewritten: %s", doc.Versions["1.0.0"]) + } +} + +// TestNPMRewriteMetadataDuplicateKeys checks that a repeated top-level key +// cannot smuggle an unfiltered copy of the versions past the filters. +func TestNPMRewriteMetadataDuplicateKeys(t *testing.T) { + proxy := testProxy() + setTestDenylist(t, proxy, "pkg:npm/demo@2.0.0") + h := &NPMHandler{proxy: proxy, proxyURL: "http://proxy"} + input := `{"versions":{"2.0.0":{"dist":{"tarball":"https://registry.npmjs.org/demo/-/demo-2.0.0.tgz"}}},` + + `"versions":{"1.0.0":{"dist":{"tarball":"https://registry.npmjs.org/demo/-/demo-1.0.0.tgz"}},"2.0.0":{"dist":{"tarball":"https://registry.npmjs.org/demo/-/demo-2.0.0.tgz"}}}}` + out, err := h.rewriteMetadata("demo", []byte(input)) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(out), "2.0.0") || strings.Count(string(out), `"versions"`) != 1 { + t.Errorf("rewrite = %s", out) + } +} + +func keysOf(m map[string]json.RawMessage) []string { + keys := make([]string, 0, len(m)) + for k := range m { + keys = append(keys, k) + } + return keys +} + +// TestMetadataServesRewriteWithoutReadingStorage checks that a repeat +// request inside the metadata TTL is answered from the rewrite cache by the +// stored digest alone, without reading the cached document back. +func TestMetadataServesRewriteWithoutReadingStorage(t *testing.T) { + var upstreamCalls atomic.Int64 + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + upstreamCalls.Add(1) + w.Header().Set("Content-Type", "application/json") + switch r.URL.Path { + case "/left-pad": + _, _ = w.Write([]byte(`{"name":"left-pad","versions":{"1.3.0":{"dist":{"tarball":"https://registry.npmjs.org/left-pad/-/left-pad-1.3.0.tgz"}}}}`)) + case "/p2/vendor/pkg.json": + _, _ = w.Write([]byte(`{"packages":{"vendor/pkg":[{"version":"1.0.0","dist":{"type":"zip","url":"https://example.com/pkg-1.0.0.zip"}}]}}`)) + default: + http.NotFound(w, r) + } + })) + defer upstream.Close() + + cases := []struct { + ecosystem, name, path string + handler func(*Proxy) http.Handler + }{ + {"npm", "left-pad", "/left-pad", func(p *Proxy) http.Handler { + return NewNPMHandler(p, "http://proxy.example", upstream.URL).Routes() + }}, + {"composer", "vendor/pkg", "/p2/vendor/pkg.json", func(p *Proxy) http.Handler { + return NewComposerHandlerWithUpstreams(p, "http://proxy.example", upstream.URL, upstream.URL).Routes() + }}, + } + for _, c := range cases { + t.Run(c.ecosystem, func(t *testing.T) { + upstreamCalls.Store(0) + proxy, db, store, _ := setupTestProxy(t) + proxy.HTTPClient = upstream.Client() + proxy.CacheMetadata = true + proxy.MetadataTTL = time.Hour + proxy.SetMetadataRewriteCacheSize(1 << 20) + handler := c.handler(proxy) + + get := func() string { + rec := httptest.NewRecorder() + handler.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, c.path, nil)) + if rec.Code != http.StatusOK { + t.Fatalf("status %d: %s", rec.Code, rec.Body.String()) + } + return rec.Body.String() + } + + first := get() + entry, err := db.GetMetadataCache(c.ecosystem, c.name) + if err != nil || entry == nil || !strings.HasPrefix(entry.ContentDigest.String, "sha256:") { + t.Fatalf("metadata row = %+v, %v; want a sha256 content digest", entry, err) + } + + store.mu.Lock() + store.openErr = errors.New("storage must not be read") + store.mu.Unlock() + + if second := get(); second != first { + t.Errorf("second response %q differs from first %q", second, first) + } + if got := upstreamCalls.Load(); got != 1 { + t.Errorf("upstream calls = %d, want 1", got) + } + }) + } +} + +// TestNPMStoredRewriteSkippedWhenStale checks that the digest fast path is +// only taken inside the metadata TTL. +func TestNPMStoredRewriteSkippedWhenStale(t *testing.T) { + proxy, db, _, _ := setupTestProxy(t) + proxy.CacheMetadata = true + proxy.MetadataTTL = time.Hour + proxy.SetMetadataRewriteCacheSize(1 << 20) + + body := []byte(`{}`) + proxy.cacheMetadataBlob(t.Context(), "npm", "demo", metadataStoragePath("npm", "demo"), &upstreamMetadata{body: body}) + key := rewriteCacheKey("npm", "http://proxy", "demo", body) + if _, err := proxy.rewrites.rewrite(t.Context(), key, body, func(b []byte) ([]byte, error) { return b, nil }); err != nil { + t.Fatal(err) + } + if _, ok := proxy.storedRewrite("npm", "demo", "http://proxy", "demo"); !ok { + t.Fatal("fresh row: expected a stored rewrite") + } + + entry, _ := db.GetMetadataCache("npm", "demo") + entry.FetchedAt.Time = time.Now().Add(-2 * time.Hour) + if err := db.UpsertMetadataCache(entry); err != nil { + t.Fatal(err) + } + if _, ok := proxy.storedRewrite("npm", "demo", "http://proxy", "demo"); ok { + t.Error("stale row: expected no stored rewrite") + } + + proxy.Cooldown = &cooldown.Config{Default: "3d"} + entry.FetchedAt.Time = time.Now() + _ = db.UpsertMetadataCache(entry) + if _, ok := proxy.storedRewrite("npm", "demo", "http://proxy", "demo"); ok { + t.Error("cooldown on: expected no stored rewrite") + } +} diff --git a/internal/handler/rewrite_cache.go b/internal/handler/rewrite_cache.go index 7626827..23096d6 100644 --- a/internal/handler/rewrite_cache.go +++ b/internal/handler/rewrite_cache.go @@ -4,9 +4,13 @@ import ( "container/list" "context" "crypto/sha256" + "encoding/hex" "errors" "strings" "sync" + "time" + + "github.com/git-pkgs/proxy/internal/metrics" ) // rewriteCache keeps metadata documents after a handler has rewritten them. @@ -68,7 +72,28 @@ func newRewriteCache(maxBytes int64) *rewriteCache { // handler rewrites for, the package, and the exact upstream bytes. func rewriteCacheKey(ecosystem, proxyURL, name string, in []byte) string { sum := sha256.Sum256(in) - return strings.Join([]string{ecosystem, proxyURL, name, string(sum[:])}, "\x00") + return rewriteCacheKeyForDigest(ecosystem, proxyURL, name, "sha256:"+hex.EncodeToString(sum[:])) +} + +// rewriteCacheKeyForDigest is rewriteCacheKey for upstream bytes known only by +// their digest, in the form the metadata cache records it. +func rewriteCacheKeyForDigest(ecosystem, proxyURL, name, digest string) string { + return strings.Join([]string{ecosystem, proxyURL, name, digest}, "\x00") +} + +// get returns the cached rewrite for key, if there is one. +func (c *rewriteCache) get(key string) ([]byte, bool) { + if c == nil { + return nil, false + } + c.mu.Lock() + defer c.mu.Unlock() + el, ok := c.entries[key] + if !ok { + return nil, false + } + c.order.MoveToFront(el) + return el.Value.(*rewriteEntry).out, true } // rewrite returns rewrite(in), from the cache when it can. Callers must treat @@ -153,6 +178,33 @@ func (p *Proxy) cachedRewrite(ctx context.Context, ecosystem, proxyURL, name str return p.rewrites.rewrite(ctx, rewriteCacheKey(ecosystem, proxyURL, name, in), in, rewrite) } +// storedRewrite returns the cached rewrite of the metadata stored for +// ecosystem and cacheKey, when that metadata is still within its TTL and a +// rewrite of it is cached. It reads only the cache row, never the stored +// document, so a repeated request for a large packument costs a database +// lookup instead of reading and hashing the whole document again. When it +// reports false the caller takes the usual fetch and rewrite path. +func (p *Proxy) storedRewrite(ecosystem, cacheKey, proxyURL, name string) ([]byte, bool) { + if p.rewrites == nil || (p.Cooldown != nil && p.Cooldown.Enabled()) { + return nil, false + } + if !p.CacheMetadata || p.DB == nil || p.MetadataTTL <= 0 { + return nil, false + } + entry, err := p.DB.GetMetadataCache(ecosystem, cacheKey) + if err != nil || entry == nil || !entry.ContentDigest.Valid || entry.ContentEncoding.String != "" { + return nil, false + } + if !entry.FetchedAt.Valid || time.Since(entry.FetchedAt.Time) >= p.MetadataTTL { + return nil, false + } + out, ok := p.rewrites.get(rewriteCacheKeyForDigest(ecosystem, proxyURL, name, entry.ContentDigest.String)) + if ok { + metrics.RecordCacheHit(ecosystem) + } + return out, ok +} + // SetMetadataRewriteCacheSize enables the cache of rewritten metadata with // room for maxBytes of output. Zero or less disables it. func (p *Proxy) SetMetadataRewriteCacheSize(maxBytes int64) { From fd4e9b74b8fbc437afa9ea0f5c301d88c563058f Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Thu, 8 Oct 2026 09:58:56 +0200 Subject: [PATCH 179/633] Mark Helm cooldown as supported in README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 70de9d7..56dfd31 100644 --- a/README.md +++ b/README.md @@ -68,7 +68,7 @@ The proxy never uploads artifact bytes to a scanner. Each scanner is notified wi | Arch | Arch Linux | | ✗ | | Chef | Chef | | ✗ | | Generic | Any | | ✓ | -| Helm | Kubernetes | | ✓ | +| Helm | Kubernetes | Yes | ✓ | | Vagrant | Vagrant | | ✗ | Cooldown requires publish timestamps in metadata. Registries without a "Yes" in the cooldown column either don't expose timestamps or haven't been wired up yet. From 52e8b5c2a4eec6eef64a6a9cd837e745f14bddd1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 9 Oct 2026 08:11:12 +0200 Subject: [PATCH 180/633] build(deps): bump modernc.org/sqlite from 1.59.0 to 1.60.1 (#421) Bumps [modernc.org/sqlite](https://gitlab.com/cznic/sqlite) from 1.59.0 to 1.60.1. - [Changelog](https://gitlab.com/cznic/sqlite/blob/master/CHANGELOG.md) - [Commits](https://gitlab.com/cznic/sqlite/compare/v1.59.0...v1.60.1) --- updated-dependencies: - dependency-name: modernc.org/sqlite dependency-version: 1.60.1 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 6 +++--- go.sum | 20 ++++++++++---------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/go.mod b/go.mod index 0fd1e1b..7b0d66b 100644 --- a/go.mod +++ b/go.mod @@ -33,7 +33,7 @@ require ( golang.org/x/sync v0.23.0 google.golang.org/protobuf v1.36.12 gopkg.in/yaml.v3 v3.0.1 - modernc.org/sqlite v1.59.0 + modernc.org/sqlite v1.60.1 ) require ( @@ -121,13 +121,13 @@ require ( golang.org/x/oauth2 v0.36.0 // indirect golang.org/x/sys v0.48.0 // indirect golang.org/x/text v0.42.0 // indirect - golang.org/x/tools v0.49.0 // indirect + golang.org/x/tools v0.50.0 // indirect golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect google.golang.org/api v0.288.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20260706201446-f0a921348800 // indirect google.golang.org/grpc v1.83.2 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect - modernc.org/libc v1.75.7 // indirect + modernc.org/libc v1.77.1 // indirect modernc.org/mathutil v1.7.1 // indirect modernc.org/memory v1.12.1 // indirect ) diff --git a/go.sum b/go.sum index 782b613..88ef0ee 100644 --- a/go.sum +++ b/go.sum @@ -346,8 +346,8 @@ golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E= golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.49.0 h1:3NI7VXzL9+1WZD52Dx2ttoPwD5DWrFGpl9mFZDlmisI= -golang.org/x/tools v0.49.0/go.mod h1:SJNXV9DBKT0UbdttsQjbfJlAE/q+y36++zo3uL3N0Oo= +golang.org/x/tools v0.50.0 h1:c2ifzfcuY7L90lZ2aKd8S4K2NpASF08SZx9ZuJkHmSU= +golang.org/x/tools v0.50.0/go.mod h1:7ulVMw3831Mwi5EZD6RomGyffr4VFjuNYXf2BbCEAV0= golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da h1:noIWHXmPHxILtqtCOPIhSt0ABwskkZKjD3bXGnZGpNY= golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= @@ -376,10 +376,10 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -modernc.org/cc/v4 v4.29.2 h1:h6+9ciCnPKutf4I03CvheAvDLX7+IHlqR6Iy6J+cgd8= -modernc.org/cc/v4 v4.29.2/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI= -modernc.org/ccgo/v4 v4.35.0 h1:F+TUsmw09QxLzmi3aeYYGxjAXarmZaKgj3mKQHNaA8w= -modernc.org/ccgo/v4 v4.35.0/go.mod h1:qrVGs9S3Sr2Ztcg9ve+kTAYMp5a3YvWjo+SoN06kJ5I= +modernc.org/cc/v4 v4.29.7 h1:q+NXGJ0bK3b4TXFYQQVr9pYETGnmwFWkrUzJnMya/Tg= +modernc.org/cc/v4 v4.29.7/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI= +modernc.org/ccgo/v4 v4.36.1 h1:ZNIUZAryN0UgnJwtyxrdEzcFc3yD4Cu4AzjfPXsLsIE= +modernc.org/ccgo/v4 v4.36.1/go.mod h1:rrtGc2QkS239nYb/mQNuBMyjq3/y3ZXWbBjPoV3wqzA= modernc.org/fileutil v1.4.0 h1:j6ZzNTftVS054gi281TyLjHPp6CPHr2KCxEXjEbD6SM= modernc.org/fileutil v1.4.0/go.mod h1:EqdKFDxiByqxLk8ozOxObDSfcVOv/54xDs/DUHdvCUU= modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI= @@ -388,8 +388,8 @@ modernc.org/gc/v3 v3.1.5 h1:21ldfPfRYE31Tb7B3mwAK8gy1AxP4+dKjrOQPfqakoc= modernc.org/gc/v3 v3.1.5/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY= modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks= modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI= -modernc.org/libc v1.75.7 h1:o3DTP9/0p9pKmY2WCKQaySW6wIiZhNM7wc2lUoyhfew= -modernc.org/libc v1.75.7/go.mod h1:bO5o2ztHxBb2rjz0PgdHN0sSMw57CgxGFLZ3Qd/QpVQ= +modernc.org/libc v1.77.1 h1:Ct8j47QtiZ1Enj2DtFXQtUqrPCAjdCmPjtCuvrYQ0Hs= +modernc.org/libc v1.77.1/go.mod h1:87/pZ4L6nD1zqW4nItuS12YO7hN1igAah34xjnQo/W0= modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= modernc.org/memory v1.12.1 h1:nFMiWrpStgZczNl6XI9GnIk/rWhYIyHGUaR04pGbp9g= @@ -398,8 +398,8 @@ modernc.org/opt v0.2.0 h1:tGyef5ApycA7FSEOMraay9SaTk5zmbx7Tu+cJs4QKZg= modernc.org/opt v0.2.0/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns= modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w= modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE= -modernc.org/sqlite v1.59.0 h1:X1es1GpqBlS/5T+vbM4HLUdaa8OtQx468DF2vrx+38A= -modernc.org/sqlite v1.59.0/go.mod h1:+paeT2A3iPRHkQDwG7oA6Tk0zQd5woMEI8q7orfry8k= +modernc.org/sqlite v1.60.1 h1:/blz53O951KWFOso4QQvEs/Fq6cDBKLtMVrYNSeJVKw= +modernc.org/sqlite v1.60.1/go.mod h1:1dIoEagfDE72QytD5scH1lxARtaUgKgHC/NuApA27r0= modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0= modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A= modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= From a691e992cc93334974e6574bd91ee92084801465 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 9 Oct 2026 08:11:55 +0200 Subject: [PATCH 181/633] build(deps): bump github.com/aws/aws-sdk-go-v2/service/ecr (#423) Bumps [github.com/aws/aws-sdk-go-v2/service/ecr](https://github.com/aws/aws-sdk-go-v2) from 1.66.0 to 1.66.1. - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/service/s3/v1.66.0...service/s3/v1.66.1) --- updated-dependencies: - dependency-name: github.com/aws/aws-sdk-go-v2/service/ecr dependency-version: 1.66.1 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 8 ++++---- go.sum | 16 ++++++++-------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/go.mod b/go.mod index 7b0d66b..d786791 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ require ( github.com/CycloneDX/cyclonedx-go v0.12.0 github.com/Masterminds/semver/v3 v3.5.0 github.com/aws/aws-sdk-go-v2/config v1.33.5 - github.com/aws/aws-sdk-go-v2/service/ecr v1.66.0 + github.com/aws/aws-sdk-go-v2/service/ecr v1.66.1 github.com/git-pkgs/archives v0.8.1 github.com/git-pkgs/artifacts v0.2.1 github.com/git-pkgs/cooldown v0.2.0 @@ -50,13 +50,13 @@ require ( github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect github.com/anchore/go-struct-converter v0.1.0 // indirect github.com/apapsch/go-jsonmerge/v2 v2.0.0 // indirect - github.com/aws/aws-sdk-go-v2 v1.47.0 // indirect + github.com/aws/aws-sdk-go-v2 v1.47.1 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.11 // indirect github.com/aws/aws-sdk-go-v2/credentials v1.20.5 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.0 // indirect github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.2.3 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.3 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.3 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.4 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.4 // indirect github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.3 // indirect github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.19 // indirect github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.18 // indirect diff --git a/go.sum b/go.sum index 88ef0ee..2b367a8 100644 --- a/go.sum +++ b/go.sum @@ -56,8 +56,8 @@ github.com/anchore/go-struct-converter v0.1.0 h1:2rDRssAl6mgKBSLNiVCMADgZRhoqtw9 github.com/anchore/go-struct-converter v0.1.0/go.mod h1:rYqSE9HbjzpHTI74vwPvae4ZVYZd1lue2ta6xHPdblA= github.com/apapsch/go-jsonmerge/v2 v2.0.0 h1:axGnT1gRIfimI7gJifB699GoE/oq+F2MU7Dml6nw9rQ= github.com/apapsch/go-jsonmerge/v2 v2.0.0/go.mod h1:lvDnEdqiQrp0O42VQGgmlKpxL1AP2+08jFMw88y4klk= -github.com/aws/aws-sdk-go-v2 v1.47.0 h1:0jsHallhJCeaU0Ko48c/3FK1ctOQ7NpzggxriJOQ8MQ= -github.com/aws/aws-sdk-go-v2 v1.47.0/go.mod h1:bttEH6JqnUL8LepvDVfdrds/fZ5bCIxzpe3abyUrhDU= +github.com/aws/aws-sdk-go-v2 v1.47.1 h1:uOIZnp4PK3ZhKI0dNrJrhTEsLxbpXHTAJlwoS1pvAtw= +github.com/aws/aws-sdk-go-v2 v1.47.1/go.mod h1:bttEH6JqnUL8LepvDVfdrds/fZ5bCIxzpe3abyUrhDU= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.11 h1:h5+3VT69KUBK24grGuuA5saDJTj2IIjLb9au668Fo5I= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.11/go.mod h1:dnakxebH6UwFvcvujL0LVggYQ8nEvBGjU4G/V79Nv94= github.com/aws/aws-sdk-go-v2/config v1.33.5 h1:UA1dmokBFOLFoOyVBhO6HjM6edy0MIk5AZSkJVcksQw= @@ -68,14 +68,14 @@ github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.0 h1:AM4hHjww+PSFtt6E+UrBrPl github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.0/go.mod h1:3x/yXezeQjpOvBb4jEMxrS8SXvpdvJ5abv6l5c1gWM8= github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.2.3 h1:w5OoDiMN6x53ROmiIImGzmVcxXv2q1GXY+aKV4WAJYM= github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.2.3/go.mod h1:dAhgYp776bX3LuWvnSCFwQEjNs6fuFg7YXIy5PXcP3Q= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.3 h1:Hp/VgjP0BysR3OgLlR057Vz2LcbbVnoWeJ+3qWiS/fY= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.3/go.mod h1:nwGV5qw7F1IZPgxCvA/ph8N2TAuz+BkRG/bXn808qMA= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.3 h1:MUaM4f+kj1ZIBPZfUS8cxP1GKXXZtHJjAthy93AN7SM= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.3/go.mod h1:6YmVmEVRI5ZZzRjCSsb9SryKH0hAlMRdgA7kG9aDvBU= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.4 h1:CLq4+8UHCI+ZZYl/EuJxXovaIVN2xeeT8JV+dsApQ5E= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.4/go.mod h1:Wv4q5sAM04xAMkoOedxLx2inVf6K5FdxYp+A61L+q/0= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.4 h1:dD4MR81I7YkpEBRk6UP9rocC2QnT3qVuXwzlYTtfGEs= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.4/go.mod h1:EcXV1kAFd5XwSkDHlj94gnF3q5CkJyYiIJfH8N0VmrE= github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.3 h1:fuSCw4Z2qfRCztMPO3GXJNSiEp6Wee+WOLwrHHUMy9c= github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.3/go.mod h1:6SxcHheD1pPR5+kWm1wGvjlL/YqUsh267sAfEmN4K7A= -github.com/aws/aws-sdk-go-v2/service/ecr v1.66.0 h1:9i19IigAYxnAxTUQcsxkiIXeeytHllTOetWBhr1DTQs= -github.com/aws/aws-sdk-go-v2/service/ecr v1.66.0/go.mod h1:iSlv4VibruxMhWfiKrNbn97Yjhe2855EFrgUitSESfM= +github.com/aws/aws-sdk-go-v2/service/ecr v1.66.1 h1:H63vyEXid/tHpv/UlvQUyM1c2QK5WgQRB3MK5gnAo8A= +github.com/aws/aws-sdk-go-v2/service/ecr v1.66.1/go.mod h1:WglfLchOYcHrYOwNV7jERuy0Xc+7jArLkEnQay93auY= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.19 h1:bAdDl/HkGCcGPoe25ToSHEw23VIxt6CT5fLcg111BKg= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.19/go.mod h1:KaUzbLxv4CeSxh6ZCl9B4m7CuFenS8kUEaDs+f/DQr4= github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.18 h1:W/EyPFl9A5rXrtoilfwHYEvzHER+K4SpBPtMXi24Mos= From 579ab79ef4349beeb8a9402c382de79edbad1120 Mon Sep 17 00:00:00 2001 From: Giles Westwood Date: Sat, 10 Oct 2026 07:17:53 +0100 Subject: [PATCH 182/633] Cache archive artifacts from Maven repositories (#425) The Maven handler cached only .jar, .war, .ear, .pom, .aar, .klib and .module files, so distributions published as archives, such as apache-maven-{version}-bin.tar.gz, were proxied uncached on every request. .zip, .tar.gz, .tgz, .tar.bz2 and .tar.xz files now go through the artifact cache like other artifacts. Co-authored-by: Giles Westwood <201443+gilesw@users.noreply.github.com> --- internal/handler/maven.go | 8 ++++++-- internal/handler/maven_test.go | 4 ++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/internal/handler/maven.go b/internal/handler/maven.go index 3f1cd9d..850cec3 100644 --- a/internal/handler/maven.go +++ b/internal/handler/maven.go @@ -157,8 +157,12 @@ func (h *MavenHandler) parsePath(urlPath string) (group, artifact, version, file // isArtifactFile returns true if the filename looks like a Maven artifact. func (h *MavenHandler) isArtifactFile(filename string) bool { - // Common artifact extensions - extensions := []string{".jar", ".war", ".ear", ".pom", ".aar", ".klib", ".module"} + // Common artifact extensions, and the archives distributions such as + // apache-maven-{version}-bin.tar.gz are published as. + extensions := []string{ + ".jar", ".war", ".ear", ".pom", ".aar", ".klib", ".module", + ".zip", ".tar.gz", ".tgz", ".tar.bz2", ".tar.xz", + } for _, ext := range extensions { if strings.HasSuffix(filename, ext) { return true diff --git a/internal/handler/maven_test.go b/internal/handler/maven_test.go index 9ca5eb6..9331a00 100644 --- a/internal/handler/maven_test.go +++ b/internal/handler/maven_test.go @@ -55,8 +55,12 @@ func TestMavenIsArtifactFile(t *testing.T) { {"guava-32.1.3-jre.module", true}, {"app-1.0.war", true}, {"lib-1.0.aar", true}, + {"apache-maven-3.9.9-bin.tar.gz", true}, + {"apache-maven-3.9.9-bin.zip", true}, + {"maven-mvnd-1.0.6-linux-amd64.tgz", true}, {"maven-metadata.xml", false}, {"guava-32.1.3-jre.jar.sha1", false}, + {"apache-maven-3.9.9-bin.tar.gz.sha512", false}, } for _, tt := range tests { From 73ed6bf5c60f83ced7b03742be85ee23293ee94d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 10 Oct 2026 07:20:45 +0100 Subject: [PATCH 183/633] build(deps): bump github.com/aws/aws-sdk-go-v2/config (#422) Bumps [github.com/aws/aws-sdk-go-v2/config](https://github.com/aws/aws-sdk-go-v2) from 1.33.5 to 1.33.6. - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/config/v1.33.5...config/v1.33.6) --- updated-dependencies: - dependency-name: github.com/aws/aws-sdk-go-v2/config dependency-version: 1.33.6 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 18 +++++++++--------- go.sum | 36 ++++++++++++++++++------------------ 2 files changed, 27 insertions(+), 27 deletions(-) diff --git a/go.mod b/go.mod index d786791..af9984d 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( github.com/BurntSushi/toml v1.6.0 github.com/CycloneDX/cyclonedx-go v0.12.0 github.com/Masterminds/semver/v3 v3.5.0 - github.com/aws/aws-sdk-go-v2/config v1.33.5 + github.com/aws/aws-sdk-go-v2/config v1.33.6 github.com/aws/aws-sdk-go-v2/service/ecr v1.66.1 github.com/git-pkgs/archives v0.8.1 github.com/git-pkgs/artifacts v0.2.1 @@ -52,21 +52,21 @@ require ( github.com/apapsch/go-jsonmerge/v2 v2.0.0 // indirect github.com/aws/aws-sdk-go-v2 v1.47.1 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.11 // indirect - github.com/aws/aws-sdk-go-v2/credentials v1.20.5 // indirect - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.0 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.20.6 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.1 // indirect github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.2.3 // indirect github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.4 // indirect github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.4 // indirect - github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.3 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.4 // indirect github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.19 // indirect github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.18 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.3 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.4 // indirect github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.25 // indirect github.com/aws/aws-sdk-go-v2/service/s3 v1.102.2 // indirect - github.com/aws/aws-sdk-go-v2/service/signin v1.10.0 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.38.0 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.0 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.51.0 // indirect + github.com/aws/aws-sdk-go-v2/service/signin v1.10.1 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.38.1 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.1 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.51.1 // indirect github.com/aws/smithy-go v1.28.1 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cenk/backoff v2.2.1+incompatible // indirect diff --git a/go.sum b/go.sum index 2b367a8..c6bed9e 100644 --- a/go.sum +++ b/go.sum @@ -60,40 +60,40 @@ github.com/aws/aws-sdk-go-v2 v1.47.1 h1:uOIZnp4PK3ZhKI0dNrJrhTEsLxbpXHTAJlwoS1pv github.com/aws/aws-sdk-go-v2 v1.47.1/go.mod h1:bttEH6JqnUL8LepvDVfdrds/fZ5bCIxzpe3abyUrhDU= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.11 h1:h5+3VT69KUBK24grGuuA5saDJTj2IIjLb9au668Fo5I= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.11/go.mod h1:dnakxebH6UwFvcvujL0LVggYQ8nEvBGjU4G/V79Nv94= -github.com/aws/aws-sdk-go-v2/config v1.33.5 h1:UA1dmokBFOLFoOyVBhO6HjM6edy0MIk5AZSkJVcksQw= -github.com/aws/aws-sdk-go-v2/config v1.33.5/go.mod h1:Dop8axzz0xx38GExIYWXdeyc8QQ7Cr+nPsxpD/LYy4U= -github.com/aws/aws-sdk-go-v2/credentials v1.20.5 h1:wklUVvHMc9xTQ3rcp49/ISpiMnhbCicJcA6n6S8m7J8= -github.com/aws/aws-sdk-go-v2/credentials v1.20.5/go.mod h1:fyEdrn6ccLFOkoK84j5bQyGTxp9zPt5l2XMhxf4DVZs= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.0 h1:AM4hHjww+PSFtt6E+UrBrPlZkWsePCLEt9AjkfQX+yM= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.0/go.mod h1:3x/yXezeQjpOvBb4jEMxrS8SXvpdvJ5abv6l5c1gWM8= +github.com/aws/aws-sdk-go-v2/config v1.33.6 h1:MBjkSTLczek/UgiK+EYPIoRTqE7gP8vtW3OFbFo7Nug= +github.com/aws/aws-sdk-go-v2/config v1.33.6/go.mod h1:grRAFzdAZJrwcbasJRg2MPvIrVjtlfXllHssN6+E1JE= +github.com/aws/aws-sdk-go-v2/credentials v1.20.6 h1:NpAFXCU7NzXNkdGK3zQTtsRJ+3v9tZQV0xcdRw8uBdw= +github.com/aws/aws-sdk-go-v2/credentials v1.20.6/go.mod h1:mcZCoiPnyMvP8VMNbygNX5lLqSlkYJIMPODylQMurOk= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.1 h1:8gALAAmacnIXh+z6VkdDanv4/IkG5APdg4DZLDTmLog= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.1/go.mod h1:Z7IJhJU+poOdJjUR2wpyY21ossQ1XS/R3Lk9Msq5kM4= github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.2.3 h1:w5OoDiMN6x53ROmiIImGzmVcxXv2q1GXY+aKV4WAJYM= github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.2.3/go.mod h1:dAhgYp776bX3LuWvnSCFwQEjNs6fuFg7YXIy5PXcP3Q= github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.4 h1:CLq4+8UHCI+ZZYl/EuJxXovaIVN2xeeT8JV+dsApQ5E= github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.4/go.mod h1:Wv4q5sAM04xAMkoOedxLx2inVf6K5FdxYp+A61L+q/0= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.4 h1:dD4MR81I7YkpEBRk6UP9rocC2QnT3qVuXwzlYTtfGEs= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.4/go.mod h1:EcXV1kAFd5XwSkDHlj94gnF3q5CkJyYiIJfH8N0VmrE= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.3 h1:fuSCw4Z2qfRCztMPO3GXJNSiEp6Wee+WOLwrHHUMy9c= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.3/go.mod h1:6SxcHheD1pPR5+kWm1wGvjlL/YqUsh267sAfEmN4K7A= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.4 h1:7Wo47d/xn/7KttCSBd8EGYeZ7ULRFRkUHr6vkZPBzVQ= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.4/go.mod h1:tDB2IVC1xC3vX8o+6uRlzhTxP3g1b77CZXFX/oD2FnQ= github.com/aws/aws-sdk-go-v2/service/ecr v1.66.1 h1:H63vyEXid/tHpv/UlvQUyM1c2QK5WgQRB3MK5gnAo8A= github.com/aws/aws-sdk-go-v2/service/ecr v1.66.1/go.mod h1:WglfLchOYcHrYOwNV7jERuy0Xc+7jArLkEnQay93auY= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.19 h1:bAdDl/HkGCcGPoe25ToSHEw23VIxt6CT5fLcg111BKg= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.19/go.mod h1:KaUzbLxv4CeSxh6ZCl9B4m7CuFenS8kUEaDs+f/DQr4= github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.18 h1:W/EyPFl9A5rXrtoilfwHYEvzHER+K4SpBPtMXi24Mos= github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.18/go.mod h1:UG50K+pvd/uy6xExbobg0rjqFBFZe6I3l75EPDZw4tg= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.3 h1:bON1rJf67TSTDCKg816AAIE4xSTtoo9tl0XRkO72R+I= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.3/go.mod h1:c5BBpjJcQXpfeq9iASyVKA3T6vX6B6LEXY4mL/gklDY= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.4 h1:29SvnfGhXjTl8ONxFwbj2rs6lbhiFXD2CgFQmbT/bXY= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.4/go.mod h1:wm04I5DMuNVvZHFe/dHnUxincvNbbK7AiNBbYsQivek= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.25 h1:2pQEbwf+/6EDbiit/GcBE2K4IUpMZymaA0kOz3xK978= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.25/go.mod h1:KvT6NCcQ0EZ+ZkVRrlBMt04Po3ok23YELEp7WimhLhM= github.com/aws/aws-sdk-go-v2/service/s3 v1.102.2 h1:ie4ElCmUKS26pzrZcIk/lmt4yWjAqLLcawstyQCh298= github.com/aws/aws-sdk-go-v2/service/s3 v1.102.2/go.mod h1:zjsomFeX5duj+4PlMB+o4JoWTIx+G0XMyzjYrUbQkN0= -github.com/aws/aws-sdk-go-v2/service/signin v1.10.0 h1:ZD5qFpWcaOKdTuhBi431pIDkCgrMkMlMT6jlpSPoIRI= -github.com/aws/aws-sdk-go-v2/service/signin v1.10.0/go.mod h1:8Nuuf+tR346PjJ3MvZPh9pekbLiLQFWJhzMXfwy7alA= -github.com/aws/aws-sdk-go-v2/service/sso v1.38.0 h1:JGeeBcMlhg1xtOXYpeCaTQBZObtXMPQCUqBcmr65NRA= -github.com/aws/aws-sdk-go-v2/service/sso v1.38.0/go.mod h1:XwteswG9EOMRFm73UT0t+MbTwyLxMrEXkU6e+v92Lzo= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.0 h1:obhahQXDEdVEv8y5bTKXR30LVaxYe1kyYM0L7l2Iq+k= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.0/go.mod h1:6twZZ/aXHNy1vXUO8koUbp++MYzMASkOgEBdkbJYmO0= -github.com/aws/aws-sdk-go-v2/service/sts v1.51.0 h1:Zpnqa6XtrNzXZnwbdCqHOXpXhMsa01ql/pcRQ1sb4hk= -github.com/aws/aws-sdk-go-v2/service/sts v1.51.0/go.mod h1:/8JRcdTt//hG0Q4BTmGbuOplT7ABe+5rdtqUHqXvYIM= +github.com/aws/aws-sdk-go-v2/service/signin v1.10.1 h1:DzCCWLzcIRQ77F3DEUljud7bEjTgFOIKXP52NmVRyhU= +github.com/aws/aws-sdk-go-v2/service/signin v1.10.1/go.mod h1:xpo/geVldu8payT375WekctUzopG/hBU7miiqItMUlw= +github.com/aws/aws-sdk-go-v2/service/sso v1.38.1 h1:Umtl/0YZhng4xndfW3lKJrYYP7NLEjI6bGXVomwLcs0= +github.com/aws/aws-sdk-go-v2/service/sso v1.38.1/go.mod h1:rRD/dnm7q0HYE/I5TMaPgkWyyUGLcwuxHLABsLnQ3e0= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.1 h1:orIWdNiLgzrhu/11RcPPKO/SBzUUymbUQuZbSPImghg= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.1/go.mod h1:skwM/xsbR/1ReUTesv9BhpJp1VjajR7DWQnuVLwiXsQ= +github.com/aws/aws-sdk-go-v2/service/sts v1.51.1 h1:0HOqZXRvMytH6bFHVIc0oJX07sZjfhz0zXtjs6gdE8s= +github.com/aws/aws-sdk-go-v2/service/sts v1.51.1/go.mod h1:26zA0GhDrLo+yiLI2yXWxqB1PdsShfLikoI7GOEgugM= github.com/aws/smithy-go v1.28.1 h1:R/nXH00c8qcfCzQVELtRw+eLQWtzv+VAIEFJ1/xxXlQ= github.com/aws/smithy-go v1.28.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= From 449683a67626b52d4e4b343aaaa4a41c1a52a799 Mon Sep 17 00:00:00 2001 From: simon chrzanowski Date: Sat, 10 Oct 2026 08:31:37 +0200 Subject: [PATCH 184/633] Rewrite Composer metadata in place and keep it minified (#418) - Walk version lists in place instead of decoding the document into map[string]any and expanding the minified format - Keep minified metadata minified: each written version carries what differs from the client's expanded state, so changes made by versions dropped by cooldown fold into the next kept one; every version gets its own dist - Copy everything else byte for byte; only dist values are written fresh - Resolve download URLs by walking the list in place, tracking inherited fields, instead of decoding and expanding the whole document - Add forEachJSONElement, rewriteJSONMembers and jsonStringIs to the scanner --- internal/handler/composer.go | 435 ++++++++++++------ .../handler/composer_rewrite_bench_test.go | 81 ++++ internal/handler/composer_rewrite_test.go | 282 ++++++++++++ internal/handler/composer_test.go | 93 ++-- internal/handler/jsonscan.go | 61 +++ internal/handler/jsonscan_test.go | 57 +++ 6 files changed, 789 insertions(+), 220 deletions(-) create mode 100644 internal/handler/composer_rewrite_bench_test.go create mode 100644 internal/handler/composer_rewrite_test.go diff --git a/internal/handler/composer.go b/internal/handler/composer.go index 3c10f6c..da7a957 100644 --- a/internal/handler/composer.go +++ b/internal/handler/composer.go @@ -1,10 +1,12 @@ package handler import ( + "bytes" "context" "encoding/json" "errors" "fmt" + "io" "net/http" "path" "strings" @@ -14,10 +16,14 @@ import ( const ( composerUpstream = "https://packagist.org" composerRepo = "https://repo.packagist.org" + composerMinified = "composer/2.0" composerUnset = "__unset" + composerDevReset = "~dev" vendorPackageParts = 2 ) +var composerUnsetJSON = []byte(`"` + composerUnset + `"`) + // ComposerHandler handles Composer/Packagist registry protocol requests. type ComposerHandler struct { proxy *Proxy @@ -131,118 +137,148 @@ func (h *ComposerHandler) handlePackageMetadata(w http.ResponseWriter, r *http.R } // rewriteMetadata rewrites dist URLs in Composer metadata to point at this proxy. -// If the metadata uses the minified Composer v2 format, it is expanded first so -// that every version entry contains all fields. If cooldown is enabled, versions -// published too recently are filtered out. +// If cooldown is enabled, versions published too recently are filtered out. +// +// The document is never decoded as a whole. Packagist serves large packages +// as minified lists of thousands of versions, and expanding them into generic +// maps took many times the document's size for every rewrite. Instead each +// version list is walked in place and the response is assembled from slices +// of the original bytes: only dist values are written fresh, and minified +// metadata stays minified. func (h *ComposerHandler) rewriteMetadata(body []byte) ([]byte, error) { - var metadata map[string]any - if err := json.Unmarshal(body, &metadata); err != nil { + if !json.Valid(body) { + return nil, errors.New("composer metadata is not valid JSON") + } + format, _, err := lookupJSONString(body, "minified") + if err != nil { return nil, err } + minified := format == composerMinified - packages, ok := metadata["packages"].(map[string]any) - if !ok { - return body, nil - } - - minified := metadata["minified"] == "composer/2.0" - - for packageName, versions := range packages { - versionList, ok := versions.([]any) - if !ok { - continue + var out bytes.Buffer + out.Grow(len(body) + len(body)/8) + err = rewriteJSONMembers(&out, body, func(out *bytes.Buffer, m jsonMember) (bool, error) { + packages := m.value(body) + if !jsonKeyIs(m.key(body), "packages") || packages[0] != '{' { + return false, nil } - - if minified { - versionList = expandMinifiedVersions(versionList) - } - - packages[packageName] = h.filterAndRewriteVersions(packageName, versionList) - } - - delete(metadata, "minified") - - return json.Marshal(metadata) -} - -// expandMinifiedVersions expands the Composer v2 minified format where each -// version entry only contains fields that differ from the previous entry. -// The "~dev" sentinel string resets the inheritance chain, and the "__unset" -// value removes a field from the inherited state. -func expandMinifiedVersions(versionList []any) []any { - expanded := make([]any, 0, len(versionList)) - inherited := map[string]any{} - - for _, v := range versionList { - // The "~dev" sentinel resets the inheritance chain for dev versions. - if s, ok := v.(string); ok && s == "~dev" { - inherited = map[string]any{} - continue - } - - vmap, ok := v.(map[string]any) - if !ok { - continue - } - - // Merge inherited fields into a new map, then overlay current fields. - // Inherited values are shared between versions, not copied: the - // only one rewritten afterwards is dist, and rewriteDistURL copies - // it before changing it. - merged := make(map[string]any, len(inherited)+len(vmap)) - for k, val := range inherited { - merged[k] = val - } - for k, val := range vmap { - if val == composerUnset { - delete(merged, k) - continue + return true, rewriteJSONMembers(out, packages, func(out *bytes.Buffer, p jsonMember) (bool, error) { + versions := p.value(packages) + if versions[0] != '[' { + return false, nil } - merged[k] = val - } - - // Update inherited state for next iteration. - inherited = merged - - expanded = append(expanded, merged) + packageName, err := jsonKey(p.key(packages)) + if err != nil { + return false, err + } + return true, h.writeVersions(out, packageName, versions, minified) + }) + }) + if err != nil { + return nil, err } - - return expanded + return out.Bytes(), nil } -// filterAndRewriteVersions applies cooldown filtering and rewrites dist URLs -// for a single package's version list. -func (h *ComposerHandler) filterAndRewriteVersions(packageName string, versionList []any) []any { +// writeVersions writes one package's version list without the versions in +// cooldown and with each dist pointing at this proxy. +// +// Minified lists stay minified. Each version written carries the fields that +// differ from what the client has expanded so far, which is the upstream +// entry itself unless filtered versions were dropped before it; then it also +// carries the changes those versions made. Every version gets its own dist, +// since the proxied URL names the version. +func (h *ComposerHandler) writeVersions(out *bytes.Buffer, packageName string, versions []byte, minified bool) error { packagePURL := canonicalPackagePURL("composer", packageName) + upstream, written := newComposerFields(), newComposerFields() + devReset, first := false, true - filtered := versionList[:0] - for _, v := range versionList { - vmap, ok := v.(map[string]any) - if !ok { - continue + out.WriteByte('[') + err := forEachJSONElement(versions, func(entry []byte) error { + if minified && jsonStringIs(entry, composerDevReset) { + upstream.reset() + written.reset() + devReset = true + return nil + } + if entry[0] != '{' { + return nil + } + if !minified { + upstream.reset() + written.reset() + } + if err := upstream.apply(entry, minified); err != nil { + return err } - version, _ := vmap["version"].(string) - - if h.shouldFilterVersion(packagePURL, packageName, version, vmap) { - continue + version := upstream.string("version") + if h.shouldFilterVersion(packagePURL, packageName, version, upstream.get("time")) { + return nil } - h.rewriteDistURL(vmap, packageName, version) - filtered = append(filtered, v) + if !first { + out.WriteByte(',') + } + first = false + if devReset { + out.WriteString(`"` + composerDevReset + `",`) + devReset = false + } + h.writeVersion(out, packageName, version, upstream, written) + return nil + }) + out.WriteByte(']') + return err +} + +// writeVersion writes the fields of upstream that differ from written, with +// dist pointing at this proxy, and "__unset" for those written has and +// upstream lacks. written is then what the client expands this version to. +func (h *ComposerHandler) writeVersion(out *bytes.Buffer, packageName, version string, upstream, written *composerFields) { + out.WriteByte('{') + first := true + writeMember := func(key, value []byte) { + if !first { + out.WriteByte(',') + } + first = false + out.Write(key) + out.WriteByte(':') + out.Write(value) } - return filtered + for _, f := range upstream.fields { + value := f.value + if value == nil { + continue + } + if f.name == "dist" { + value = h.rewriteDist(packageName, version, value) + } + if bytes.Equal(value, written.get(f.name)) { + continue + } + writeMember(f.key, value) + written.set(f.name, f.key, value) + } + for _, f := range written.fields { + if f.value != nil && upstream.get(f.name) == nil { + writeMember(f.key, composerUnsetJSON) + written.set(f.name, f.key, nil) + } + } + out.WriteByte('}') } // shouldFilterVersion returns true if the version should be excluded due to cooldown. -func (h *ComposerHandler) shouldFilterVersion(packagePURL, packageName, version string, vmap map[string]any) bool { +func (h *ComposerHandler) shouldFilterVersion(packagePURL, packageName, version string, published []byte) bool { if h.proxy.Cooldown == nil || !h.proxy.Cooldown.Enabled() { return false } - timeStr, ok := vmap["time"].(string) - if !ok { + var timeStr string + if err := json.Unmarshal(published, &timeStr); err != nil { return false } @@ -260,44 +296,133 @@ func (h *ComposerHandler) shouldFilterVersion(packagePURL, packageName, version return false } -// rewriteDistURL rewrites the dist URL in a version entry to point at this proxy. -func (h *ComposerHandler) rewriteDistURL(vmap map[string]any, packageName, version string) { - dist, ok := vmap["dist"].(map[string]any) - if !ok { - return +// rewriteDist returns a version's dist object with its url pointing at this +// proxy. A dist without a string url is returned unchanged. +func (h *ComposerHandler) rewriteDist(packageName, version string, dist []byte) []byte { + urlMember, ok, err := findJSONMember(dist, "url") + if err != nil || !ok || dist[urlMember.valStart] != '"' { + return dist } + var upstreamURL string + if err := json.Unmarshal(urlMember.value(dist), &upstreamURL); err != nil { + return dist + } + distType, _, _ := lookupJSONString(dist, "type") - url, ok := dist["url"].(string) - if !ok || url == "" { - return + proxyURL, ok := h.proxyDistURL(packageName, version, upstreamURL, distType) + if !ok { + return dist + } + encoded, err := json.Marshal(proxyURL) + if err != nil { + return dist + } + rewritten := make([]byte, 0, len(dist)-len(urlMember.value(dist))+len(encoded)) + rewritten = append(rewritten, dist[:urlMember.valStart]...) + rewritten = append(rewritten, encoded...) + return append(rewritten, dist[urlMember.valEnd:]...) +} + +// proxyDistURL returns the URL this proxy serves a version's archive at, or +// false when the upstream URL or package name gives nothing to build it from. +func (h *ComposerHandler) proxyDistURL(packageName, version, upstreamURL, distType string) (string, bool) { + if upstreamURL == "" { + return "", false } filename := "package.zip" - if idx := strings.LastIndex(url, "/"); idx >= 0 { - filename = url[idx+1:] + if idx := strings.LastIndex(upstreamURL, "/"); idx >= 0 { + filename = upstreamURL[idx+1:] } // GitHub zipball URLs end with a bare commit hash (no extension). // Append .zip so the archives library can detect the format. - if path.Ext(filename) == "" { - if distType, _ := dist["type"].(string); distType == "zip" { - filename += ".zip" - } + if path.Ext(filename) == "" && distType == "zip" { + filename += ".zip" } parts := strings.SplitN(packageName, "/", vendorPackageParts) - if len(parts) == vendorPackageParts { - newURL := fmt.Sprintf("%s/composer/files/%s/%s/%s/%s", - h.proxyURL, parts[0], parts[1], version, filename) - // Expanded versions can share one inherited dist map, so give this - // version its own before changing its URL. - own := make(map[string]any, len(dist)) - for k, v := range dist { - own[k] = v - } - own["url"] = newURL - vmap["dist"] = own + if len(parts) != vendorPackageParts { + return "", false } + return fmt.Sprintf("%s/composer/files/%s/%s/%s/%s", + h.proxyURL, parts[0], parts[1], version, filename), true +} + +// composerFields holds one version's fields as Composer sees them after +// expanding the minified format: raw values under their decoded names, in the +// order the names first appeared. A removed field keeps its slot with a nil +// value, so the order stays stable when it comes back. +type composerFields struct { + fields []composerField + index map[string]int +} + +type composerField struct { + name string + key, value []byte +} + +func newComposerFields() *composerFields { + return &composerFields{index: map[string]int{}} +} + +func (f *composerFields) reset() { + f.fields = f.fields[:0] + clear(f.index) +} + +// get returns the raw value of the named field, or nil when it is not set. +func (f *composerFields) get(name string) []byte { + if i, ok := f.index[name]; ok { + return f.fields[i].value + } + return nil +} + +// string returns the named field when it is a string, or "". +func (f *composerFields) string(name string) string { + raw := f.get(name) + if len(raw) == 0 || raw[0] != '"' { + return "" + } + var s string + _ = json.Unmarshal(raw, &s) + return s +} + +// set stores value under name, written with the quoted key. A nil value +// removes the field. +func (f *composerFields) set(name string, key, value []byte) { + if i, ok := f.index[name]; ok { + f.fields[i].key, f.fields[i].value = key, value + return + } + f.index[name] = len(f.fields) + f.fields = append(f.fields, composerField{name: name, key: key, value: value}) +} + +// apply overlays the members of a version entry. In minified metadata the +// value "__unset" removes the field instead. +func (f *composerFields) apply(entry []byte, minified bool) error { + return forEachJSONMember(entry, func(m jsonMember) error { + key, value := m.key(entry), m.value(entry) + if minified && jsonStringIs(value, composerUnset) { + value = nil + } + if bytes.IndexByte(key, '\\') < 0 { + if i, ok := f.index[string(key[1:len(key)-1])]; ok { + f.fields[i].key, f.fields[i].value = key, value + return nil + } + } + name, err := jsonKey(key) + if err != nil { + return err + } + f.set(name, key, value) + return nil + }) } // handleDownload serves a package file, fetching and caching from upstream if needed. @@ -407,62 +532,66 @@ func (h *ComposerHandler) findDownloadURLFromMetadata(ctx context.Context, metaU return "", nil } - var metadata map[string]any - if err := json.NewDecoder(resp.Body).Decode(&metadata); err != nil { + body, err := io.ReadAll(resp.Body) + if err != nil { return "", err } - // Expand minified Composer v2 format so that inherited fields (including - // dist) are present on every version entry. Without this, versions that - // inherit dist from a previous entry will appear to have no download URL. - if metadata["minified"] == "composer/2.0" { - h.proxy.Logger.Debug("expanding minified metadata", "url", metaURL) - if packages, ok := metadata["packages"].(map[string]any); ok { - for pkgName, versions := range packages { - versionList, ok := versions.([]any) - if !ok { - continue - } - packages[pkgName] = expandMinifiedVersions(versionList) - } - } + url, err := composerDistURL(body, packageName, version) + if err != nil { + return "", err } - - url := h.findDownloadURL(metadata, packageName, version) h.proxy.Logger.Debug("download URL lookup result", "url", metaURL, "package", packageName, "version", version, "download_url", url) return url, nil } -// findDownloadURL finds the dist URL for a specific version in metadata. -func (h *ComposerHandler) findDownloadURL(metadata map[string]any, packageName, version string) string { - packages, ok := metadata["packages"].(map[string]any) - if !ok { - return "" +// composerDistURL returns the upstream dist URL of version from Composer +// metadata, or "" when the package has no such version or it has no dist URL. +// It walks the version list in place, expanding only as much of the minified +// format as it needs, since it runs on each archive the proxy has not cached +// yet. +func composerDistURL(body []byte, packageName, version string) (string, error) { + format, _, err := lookupJSONString(body, "minified") + if err != nil { + return "", err } - - versions, ok := packages[packageName].([]any) - if !ok { - return "" + versions, err := lookupJSON(body, "packages", packageName) + if err != nil || len(versions) == 0 || versions[0] != '[' { + return "", err } + minified := format == composerMinified - for _, v := range versions { - vmap, ok := v.(map[string]any) - if !ok { - continue + fields := newComposerFields() + var url string + err = forEachJSONElement(versions, func(entry []byte) error { + if minified && jsonStringIs(entry, composerDevReset) { + fields.reset() + return nil } - - if vmap["version"] == version { - if dist, ok := vmap["dist"].(map[string]any); ok { - if url, ok := dist["url"].(string); ok { - return url - } - } + if entry[0] != '{' { + return nil } + if !minified { + fields.reset() + } + if err := fields.apply(entry, minified); err != nil { + return err + } + if !jsonStringIs(fields.get("version"), version) { + return nil + } + url, _, _ = lookupJSONString(fields.get("dist"), "url") + if url != "" { + return errStopScan + } + return nil + }) + if errors.Is(err, errStopScan) { + err = nil } - - return "" + return url, err } // proxyUpstream forwards a request to packagist.org without caching. diff --git a/internal/handler/composer_rewrite_bench_test.go b/internal/handler/composer_rewrite_bench_test.go new file mode 100644 index 0000000..dd1ba57 --- /dev/null +++ b/internal/handler/composer_rewrite_bench_test.go @@ -0,0 +1,81 @@ +package handler + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +// benchComposerMetadata builds minified Composer v2 metadata shaped like a +// large real package: thousands of versions, where the first carries every +// field and the rest mostly change version, time, dist and source, with the +// requirements changing every so often. +func benchComposerMetadata(versions int) []byte { + var b strings.Builder + b.WriteString(`{"minified":"composer/2.0","packages":{"big/sdk":[`) + for i := range versions { + if i > 0 { + b.WriteByte(',') + } + v := fmt.Sprintf("3.%d.0", versions-i) + ref := fmt.Sprintf("%040x", i) + b.WriteByte('{') + if i == 0 { + b.WriteString(`"name":"big/sdk","description":"A big SDK for a big cloud",` + + `"keywords":["cloud","sdk","api","storage","queue"],"homepage":"https://example.com/sdk",` + + `"license":["Apache-2.0"],"authors":[{"name":"Big Cloud","homepage":"https://example.com"}],` + + `"type":"library","autoload":{"psr-4":{"Big\\Sdk\\":"src/"},"files":["src/functions.php"]},` + + `"support":{"issues":"https://example.com/sdk/issues","source":"https://example.com/sdk/tree/main"},` + + `"extra":{"branch-alias":{"dev-master":"3.0-dev"}},`) + } + if i%50 == 0 { + b.WriteString(`"require":{"php":">=8.1",`) + for d := range 12 { + if d > 0 { + b.WriteByte(',') + } + fmt.Fprintf(&b, `"vendor/dep-%d":"^%d.%d"`, d, d+1, i/50) + } + b.WriteString(`},`) + } + fmt.Fprintf(&b, `"version":%q,"version_normalized":"%s.0","time":"2024-01-01T00:00:00+00:00",`+ + `"source":{"url":"https://github.com/big/sdk.git","type":"git","reference":%q},`+ + `"dist":{"url":"https://api.github.com/repos/big/sdk/zipball/%s","type":"zip","shasum":"","reference":%q}}`, + v, v, ref, ref, ref) + } + b.WriteString(`]}}`) + return []byte(b.String()) +} + +func BenchmarkComposerRewriteMetadata(b *testing.B) { + body := benchComposerMetadata(5000) + h := &ComposerHandler{proxy: testProxy(), proxyURL: "http://proxy.example"} + b.SetBytes(int64(len(body))) + b.ReportAllocs() + for b.Loop() { + if _, err := h.rewriteMetadata(body); err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkComposerFindDownloadURL(b *testing.B) { + body := benchComposerMetadata(5000) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write(body) + })) + defer srv.Close() + h := &ComposerHandler{proxy: testProxy(), proxyURL: "http://proxy.example"} + h.proxy.HTTPClient = srv.Client() + b.SetBytes(int64(len(body))) + b.ReportAllocs() + for b.Loop() { + url, err := h.findDownloadURLFromMetadata(context.Background(), srv.URL, "big/sdk", "3.2500.0") + if err != nil || url == "" { + b.Fatalf("download URL not found: %q, %v", url, err) + } + } +} diff --git a/internal/handler/composer_rewrite_test.go b/internal/handler/composer_rewrite_test.go new file mode 100644 index 0000000..9d2bb51 --- /dev/null +++ b/internal/handler/composer_rewrite_test.go @@ -0,0 +1,282 @@ +package handler + +import ( + "encoding/json" + "log/slog" + "maps" + "strings" + "testing" + "time" + + "github.com/git-pkgs/cooldown" +) + +// composerVersions decodes rewritten metadata and returns packageName's +// versions as a Composer client sees them: minified lists are expanded the +// way Composer's MetadataMinifier::expand does it. +func composerVersions(t *testing.T, output []byte, packageName string) []map[string]any { + t.Helper() + var doc struct { + Minified string `json:"minified"` + Packages map[string]any `json:"packages"` + } + if err := json.Unmarshal(output, &doc); err != nil { + t.Fatalf("output is not JSON: %v\n%s", err, output) + } + var versions []map[string]any + var expanded map[string]any + entries, _ := doc.Packages[packageName].([]any) + for _, entry := range entries { + if doc.Minified == "composer/2.0" && entry == "~dev" { + expanded = nil + continue + } + fields, ok := entry.(map[string]any) + if !ok { + t.Fatalf("version entry is not an object: %v", entry) + } + if doc.Minified != "composer/2.0" || expanded == nil { + expanded = fields + versions = append(versions, fields) + continue + } + expanded = maps.Clone(expanded) + for k, v := range fields { + if v == composerUnset { + delete(expanded, k) + continue + } + expanded[k] = v + } + versions = append(versions, expanded) + } + return versions +} + +func versionNames(versions []map[string]any) []string { + names := make([]string, len(versions)) + for i, v := range versions { + names[i], _ = v["version"].(string) + } + return names +} + +func distURL(t *testing.T, version map[string]any) string { + t.Helper() + dist, ok := version["dist"].(map[string]any) + if !ok { + t.Fatalf("version %v has no dist object", version["version"]) + } + url, _ := dist["url"].(string) + return url +} + +func cooldownComposerHandler() *ComposerHandler { + proxy := &Proxy{Logger: slog.Default()} + proxy.Cooldown = &cooldown.Config{Default: "3d"} + return &ComposerHandler{proxy: proxy, proxyURL: "http://proxy"} +} + +// TestComposerRewriteMetadataKeepsUpstreamBytes checks that only dist URLs +// change: numbers encoding/json would round through float64, characters it +// would HTML-escape and the key order all reach the client as upstream sent +// them. +func TestComposerRewriteMetadataKeepsUpstreamBytes(t *testing.T) { + h := &ComposerHandler{proxy: testProxy(), proxyURL: "http://proxy"} + input := `{"packages":{"v/p":[{"version":"1.0.0","extra":{"big":12345678901234567890,"html":"&"},` + + `"dist":{"type":"zip","url":"https://example.com/abc.zip","reference":"abc"}}]},"security-advisories":[]}` + + out, err := h.rewriteMetadata([]byte(input)) + if err != nil { + t.Fatal(err) + } + want := strings.Replace(input, "https://example.com/abc.zip", "http://proxy/composer/files/v/p/1.0.0/abc.zip", 1) + if string(out) != want { + t.Errorf("rewrite =\n%s\nwant\n%s", out, want) + } +} + +// TestComposerRewriteMetadataStaysMinified checks that minified metadata is +// rewritten without expanding it: each entry still carries only what changed +// from the one before, plus its own dist now that dist URLs name the version. +func TestComposerRewriteMetadataStaysMinified(t *testing.T) { + h := &ComposerHandler{proxy: testProxy(), proxyURL: "http://proxy"} + input := `{"minified":"composer/2.0","packages":{"v/p":[` + + `{"name":"v/p","version":"2.0.0","require":{"php": ">=8.1"},"dist":{"url":"https://example.com/a.zip","type":"zip"}},` + + `{"version":"1.1.0","dist":{"url":"https://example.com/b.zip","type":"zip"}},` + + `{"version":"1.0.0","require":"__unset"}` + + `]}}` + + out, err := h.rewriteMetadata([]byte(input)) + if err != nil { + t.Fatal(err) + } + want := `{"minified":"composer/2.0","packages":{"v/p":[` + + `{"name":"v/p","version":"2.0.0","require":{"php": ">=8.1"},"dist":{"url":"http://proxy/composer/files/v/p/2.0.0/a.zip","type":"zip"}},` + + `{"version":"1.1.0","dist":{"url":"http://proxy/composer/files/v/p/1.1.0/b.zip","type":"zip"}},` + + `{"version":"1.0.0","dist":{"url":"http://proxy/composer/files/v/p/1.0.0/b.zip","type":"zip"},"require":"__unset"}` + + `]}}` + if string(out) != want { + t.Errorf("rewrite =\n%s\nwant\n%s", out, want) + } +} + +// TestComposerRewriteMetadataCooldownCarriesChanges checks that the changes a +// filtered entry made to the inherited fields still reach the next entry +// that is kept. +func TestComposerRewriteMetadataCooldownCarriesChanges(t *testing.T) { + old := time.Now().Add(-10 * 24 * time.Hour).Format(time.RFC3339) + recent := time.Now().Add(-1 * time.Hour).Format(time.RFC3339) + input := `{"minified":"composer/2.0","packages":{"v/p":[ + {"name":"v/p","description":"Seven","version":"7.0.0","time":"` + recent + `", + "require-dev":{"phpunit/phpunit":"^11"},"dist":{"url":"https://example.com/7.zip","type":"zip"}}, + {"version":"6.4.0","time":"` + old + `","dist":{"url":"https://example.com/640.zip","type":"zip"}}, + {"description":"Backport","version":"5.4.9","time":"` + recent + `","require-dev":"__unset", + "dist":{"url":"https://example.com/549.zip","type":"zip"}}, + {"version":"5.4.8","time":"` + old + `"} + ]}}` + + out, err := cooldownComposerHandler().rewriteMetadata([]byte(input)) + if err != nil { + t.Fatal(err) + } + versions := composerVersions(t, out, "v/p") + if got := versionNames(versions); strings.Join(got, ",") != "6.4.0,5.4.8" { + t.Fatalf("versions = %v, want [6.4.0 5.4.8]", got) + } + + v640, v548 := versions[0], versions[1] + if v640["name"] != "v/p" || v640["description"] != "Seven" || v640["require-dev"] == nil { + t.Errorf("6.4.0 lost fields inherited from the filtered 7.0.0: %v", v640) + } + if v548["description"] != "Backport" { + t.Errorf("5.4.8 description = %v, want the one set by the filtered 5.4.9", v548["description"]) + } + if _, ok := v548["require-dev"]; ok { + t.Errorf("5.4.8 require-dev = %v, want it unset by the filtered 5.4.9", v548["require-dev"]) + } + if got := distURL(t, v548); got != "http://proxy/composer/files/v/p/5.4.8/549.zip" { + t.Errorf("5.4.8 dist url = %q", got) + } +} + +// TestComposerRewriteMetadataCooldownAfterDevReset checks that a "~dev" +// reset survives when the first dev entry after it is filtered. +func TestComposerRewriteMetadataCooldownAfterDevReset(t *testing.T) { + old := time.Now().Add(-10 * 24 * time.Hour).Format(time.RFC3339) + recent := time.Now().Add(-1 * time.Hour).Format(time.RFC3339) + input := `{"minified":"composer/2.0","packages":{"v/p":[ + {"name":"v/p","license":["MIT"],"version":"1.0.0","time":"` + old + `", + "dist":{"url":"https://example.com/1.zip","type":"zip"}}, + "~dev", + {"name":"v/p","description":"Dev","version":"dev-main","time":"` + recent + `", + "dist":{"url":"https://example.com/main.zip","type":"zip"}}, + {"version":"1.x-dev","time":"` + old + `"} + ]}}` + + out, err := cooldownComposerHandler().rewriteMetadata([]byte(input)) + if err != nil { + t.Fatal(err) + } + versions := composerVersions(t, out, "v/p") + if got := versionNames(versions); strings.Join(got, ",") != "1.0.0,1.x-dev" { + t.Fatalf("versions = %v, want [1.0.0 1.x-dev]", got) + } + dev := versions[1] + if _, ok := dev["license"]; ok { + t.Error("1.x-dev inherited license across the ~dev reset") + } + if dev["name"] != "v/p" || dev["description"] != "Dev" { + t.Errorf("1.x-dev lost fields inherited from the filtered dev-main: %v", dev) + } + if got := distURL(t, dev); got != "http://proxy/composer/files/v/p/1.x-dev/main.zip" { + t.Errorf("1.x-dev dist url = %q", got) + } +} + +func TestComposerRewriteMetadataInvalidJSON(t *testing.T) { + h := &ComposerHandler{proxy: testProxy(), proxyURL: "http://proxy"} + if _, err := h.rewriteMetadata([]byte(`{"packages":{"v/p":[{"version":"1.0.0"}]}`)); err == nil { + t.Error("expected an error for truncated JSON") + } +} + +func TestComposerRewriteMetadataWithoutPackages(t *testing.T) { + h := &ComposerHandler{proxy: testProxy(), proxyURL: "http://proxy"} + for _, input := range []string{`{"other":1}`, `{"packages":[]}`} { + out, err := h.rewriteMetadata([]byte(input)) + if err != nil { + t.Fatalf("%s: %v", input, err) + } + if string(out) != input { + t.Errorf("rewrite of %s = %s, want it unchanged", input, out) + } + } +} + +func TestComposerDistURL(t *testing.T) { + minified := `{"minified":"composer/2.0","packages":{"v/p":[ + {"name":"v/p","version":"3.0.0","dist":{"url":"https://example.com/3.zip","type":"zip"}}, + {"version":"2.0.0"}, + {"version":"1.0.0","dist":"__unset"}, + "~dev", + {"version":"dev-main","dist":{"url":"https://example.com/main.zip","type":"zip"}} + ]}}` + expanded := `{"packages":{"v/p":[ + {"version":"2.0.0","dist":{"url":"https://example.com/2.zip","type":"zip"}}, + {"version":"1.0.0"} + ]}}` + + tests := []struct { + name, body, version, want string + }{ + {"own dist", minified, "3.0.0", "https://example.com/3.zip"}, + {"inherited dist", minified, "2.0.0", "https://example.com/3.zip"}, + {"unset dist", minified, "1.0.0", ""}, + {"after dev reset", minified, "dev-main", "https://example.com/main.zip"}, + {"missing version", minified, "9.9.9", ""}, + {"expanded", expanded, "2.0.0", "https://example.com/2.zip"}, + {"expanded inherits nothing", expanded, "1.0.0", ""}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := composerDistURL([]byte(tt.body), "v/p", tt.version) + if err != nil { + t.Fatal(err) + } + if got != tt.want { + t.Errorf("composerDistURL(%s) = %q, want %q", tt.version, got, tt.want) + } + }) + } +} + +func TestComposerDistURLMalformed(t *testing.T) { + if _, err := composerDistURL([]byte(`{"packages":{"v/p":[{"version":`), "v/p", "1.0.0"); err == nil { + t.Error("expected an error for truncated JSON") + } +} + +// TestComposerRewriteMetadataRewritesEveryPackage checks that each package +// list in the document is rewritten under its own name, as before. +func TestComposerRewriteMetadataRewritesEveryPackage(t *testing.T) { + h := &ComposerHandler{proxy: testProxy(), proxyURL: "http://proxy"} + input := `{"packages":{ + "v/a":[{"version":"1.0.0","dist":{"url":"https://example.com/a.zip","type":"zip"}}], + "v/b":[{"version":"2.0.0","dist":{"url":"https://example.com/b.zip","type":"zip"}}], + "v/c":"not a list" + }}` + out, err := h.rewriteMetadata([]byte(input)) + if err != nil { + t.Fatal(err) + } + if got := distURL(t, composerVersions(t, out, "v/a")[0]); got != "http://proxy/composer/files/v/a/1.0.0/a.zip" { + t.Errorf("v/a dist url = %q", got) + } + if got := distURL(t, composerVersions(t, out, "v/b")[0]); got != "http://proxy/composer/files/v/b/2.0.0/b.zip" { + t.Errorf("v/b dist url = %q", got) + } + if !strings.Contains(string(out), `"v/c":"not a list"`) { + t.Errorf("non-list package value was not copied unchanged:\n%s", out) + } +} diff --git a/internal/handler/composer_test.go b/internal/handler/composer_test.go index 9898664..f6f077c 100644 --- a/internal/handler/composer_test.go +++ b/internal/handler/composer_test.go @@ -54,15 +54,16 @@ func TestComposerRewriteMetadata(t *testing.T) { } } -func TestComposerRewriteMetadataExpandsMinified(t *testing.T) { +func TestComposerRewriteMetadataMinifiedInheritance(t *testing.T) { h := &ComposerHandler{ proxy: testProxy(), proxyURL: "http://localhost:8080", } // Minified format: first version has all fields, subsequent versions - // only include fields that changed. The proxy must expand this so every - // version has all fields (including "name"). + // only include fields that changed. The proxy keeps the format, so once + // Composer expands it every version still has all fields (including + // "name"). input := `{ "minified": "composer/2.0", "packages": { @@ -97,16 +98,14 @@ func TestComposerRewriteMetadataExpandsMinified(t *testing.T) { t.Fatalf("failed to parse output: %v", err) } - // The minified key should be removed from output - if _, ok := result["minified"]; ok { - t.Error("expected minified key to be removed from output") + if result["minified"] != "composer/2.0" { + t.Errorf("minified = %v, want the format kept", result["minified"]) } - packages := result["packages"].(map[string]any) - versions := packages["symfony/console"].([]any) + versions := composerVersions(t, output, "symfony/console") // Second version should have inherited the "name" and "description" fields - v1 := versions[1].(map[string]any) + v1 := versions[1] if v1["name"] != "symfony/console" { t.Errorf("second version name = %v, want %q", v1["name"], "symfony/console") } @@ -154,13 +153,7 @@ func TestComposerRewriteMetadataMinifiedDevReset(t *testing.T) { t.Fatalf("rewriteMetadata failed: %v", err) } - var result map[string]any - if err := json.Unmarshal(output, &result); err != nil { - t.Fatalf("failed to parse output: %v", err) - } - - packages := result["packages"].(map[string]any) - versions := packages["symfony/console"].([]any) + versions := composerVersions(t, output, "symfony/console") if len(versions) != 2 { t.Fatalf("expected 2 versions, got %d", len(versions)) @@ -168,7 +161,7 @@ func TestComposerRewriteMetadataMinifiedDevReset(t *testing.T) { // Dev version should NOT have inherited "license" or "description" // from the tagged version (the ~dev sentinel resets inheritance). - devVersion := versions[1].(map[string]any) + devVersion := versions[1] if devVersion["version"] != "dev-main" { t.Errorf("dev version = %v, want %q", devVersion["version"], "dev-main") } @@ -221,19 +214,13 @@ func TestComposerRewriteMetadataUnset(t *testing.T) { t.Fatalf("rewriteMetadata failed: %v", err) } - var result map[string]any - if err := json.Unmarshal(output, &result); err != nil { - t.Fatalf("failed to parse output: %v", err) - } - - versions := result["packages"].(map[string]any)["venturecraft/revisionable"].([]any) + versions := composerVersions(t, output, "venturecraft/revisionable") if len(versions) != 4 { t.Fatalf("expected 4 versions, got %d", len(versions)) } byVersion := map[string]map[string]any{} - for _, v := range versions { - vmap := v.(map[string]any) + for _, vmap := range versions { byVersion[vmap["version"].(string)] = vmap } @@ -270,7 +257,7 @@ func TestComposerRewriteMetadataCooldownPreservesNames(t *testing.T) { // Minified format where "name" only appears in first version. // When cooldown filters the first version, remaining versions must - // still have the "name" field after expansion. + // still have the "name" field once Composer expands them. input := `{ "minified": "composer/2.0", "packages": { @@ -301,13 +288,7 @@ func TestComposerRewriteMetadataCooldownPreservesNames(t *testing.T) { t.Fatalf("rewriteMetadata failed: %v", err) } - var result map[string]any - if err := json.Unmarshal(output, &result); err != nil { - t.Fatalf("failed to parse output: %v", err) - } - - packages := result["packages"].(map[string]any) - versions := packages["symfony/console"].([]any) + versions := composerVersions(t, output, "symfony/console") // v7.0.0 should be filtered by cooldown, leaving v6.0.0 and v5.0.0 if len(versions) != 2 { @@ -315,8 +296,7 @@ func TestComposerRewriteMetadataCooldownPreservesNames(t *testing.T) { } // Both remaining versions must have the "name" field - for _, v := range versions { - vmap := v.(map[string]any) + for _, vmap := range versions { if vmap["name"] != "symfony/console" { t.Errorf("version %v missing name field, got %v", vmap["version"], vmap["name"]) } @@ -332,21 +312,12 @@ func TestComposerRewriteDistURLGitHubZipball(t *testing.T) { proxyURL: "http://localhost:8080", } - vmap := map[string]any{ - "version": "v7.4.8", - "dist": map[string]any{ - "url": "https://api.github.com/repos/symfony/asset/zipball/d2e2f014ccd6ec9fae8dbe6336a4164346a2a856", - "type": "zip", - "shasum": "", - "reference": "d2e2f014ccd6ec9fae8dbe6336a4164346a2a856", - }, + url, ok := h.proxyDistURL("symfony/asset", "v7.4.8", + "https://api.github.com/repos/symfony/asset/zipball/d2e2f014ccd6ec9fae8dbe6336a4164346a2a856", "zip") + if !ok { + t.Fatal("dist URL was not rewritten") } - h.rewriteDistURL(vmap, "symfony/asset", "v7.4.8") - - dist := vmap["dist"].(map[string]any) - url := dist["url"].(string) - // The rewritten URL's filename must have a .zip extension if !strings.HasSuffix(url, ".zip") { t.Errorf("rewritten dist URL filename has no .zip extension: %s", url) @@ -399,16 +370,16 @@ func TestComposerRewriteMetadataGitHubZipballFilenames(t *testing.T) { func TestComposerExpandMinifiedSharedDistReferences(t *testing.T) { // When a minified version inherits the dist field from a previous version - // (i.e. it doesn't include its own dist), expanding + rewriting must not - // corrupt the dist URLs via shared map references. + // (i.e. it doesn't include its own dist), rewriting must still give each + // version a dist URL naming its own version. h := &ComposerHandler{ proxy: testProxy(), proxyURL: "http://localhost:8080", } // In this minified payload, v5.3.0 does NOT include a dist field, - // so it inherits v5.4.0's dist. After expansion and URL rewriting, - // each version must have its own correct dist URL. + // so it inherits v5.4.0's dist. After URL rewriting, each version must + // have its own correct dist URL. input := `{ "minified": "composer/2.0", "packages": { @@ -434,25 +405,13 @@ func TestComposerExpandMinifiedSharedDistReferences(t *testing.T) { t.Fatalf("rewriteMetadata failed: %v", err) } - var result map[string]any - if err := json.Unmarshal(output, &result); err != nil { - t.Fatalf("failed to parse output: %v", err) - } - - packages := result["packages"].(map[string]any) - versions := packages["vendor/pkg"].([]any) + versions := composerVersions(t, output, "vendor/pkg") if len(versions) != 2 { t.Fatalf("expected 2 versions, got %d", len(versions)) } - v1 := versions[0].(map[string]any) - v2 := versions[1].(map[string]any) - - dist1 := v1["dist"].(map[string]any) - dist2 := v2["dist"].(map[string]any) - - url1 := dist1["url"].(string) - url2 := dist2["url"].(string) + url1 := distURL(t, versions[0]) + url2 := distURL(t, versions[1]) // Each version must have its own URL with its own version in the path if !strings.Contains(url1, "/5.4.0/") { diff --git a/internal/handler/jsonscan.go b/internal/handler/jsonscan.go index b1b1050..2c95aa4 100644 --- a/internal/handler/jsonscan.go +++ b/internal/handler/jsonscan.go @@ -21,6 +21,10 @@ var errMalformedJSON = errors.New("malformed JSON") // errNotJSONObject is returned when a value expected to be an object is not. var errNotJSONObject = errors.New("JSON value is not an object") +// errStopScan is returned by a callback to end a walk early once it has +// found what it was looking for. +var errStopScan = errors.New("stop scanning") + // jsonMember is one member of a JSON object as offsets into the bytes that // were scanned. The key span includes its quotes. type jsonMember struct { @@ -147,6 +151,58 @@ func forEachJSONMember(obj []byte, fn func(jsonMember) error) error { } } +// forEachJSONElement calls fn with each element of the array arr holds, in +// order, and stops early with fn's error if fn returns one. +func forEachJSONElement(arr []byte, fn func(element []byte) error) error { + i := skipJSONSpace(arr, 0) + if i >= len(arr) || arr[i] != '[' { + return errMalformedJSON + } + i = skipJSONSpace(arr, i+1) + if i < len(arr) && arr[i] == ']' { + return nil + } + for { + end, err := skipJSONValue(arr, i) + if err != nil { + return err + } + if err := fn(arr[i:end]); err != nil { + return err + } + next := skipJSONSpace(arr, end) + if next >= len(arr) { + return errMalformedJSON + } + switch arr[next] { + case ',': + i = skipJSONSpace(arr, next+1) + case ']': + return nil + default: + return errMalformedJSON + } + } +} + +// rewriteJSONMembers copies the object obj holds to out byte for byte, +// except that rewrite may write a member's value itself. It returns true +// when it did; otherwise the original value is copied. +func rewriteJSONMembers(out *bytes.Buffer, obj []byte, rewrite func(out *bytes.Buffer, m jsonMember) (bool, error)) error { + copied := 0 + err := forEachJSONMember(obj, func(m jsonMember) error { + out.Write(obj[copied:m.valStart]) + copied = m.valStart + rewritten, err := rewrite(out, m) + if rewritten { + copied = m.valEnd + } + return err + }) + out.Write(obj[copied:]) + return err +} + // jsonKey decodes a quoted key as forEachJSONMember reports it. func jsonKey(raw []byte) (string, error) { if bytes.IndexByte(raw, '\\') < 0 { @@ -166,6 +222,11 @@ func jsonKeyIs(raw []byte, name string) bool { return err == nil && key == name } +// jsonStringIs reports whether raw is a JSON string that decodes to s. +func jsonStringIs(raw []byte, s string) bool { + return len(raw) >= 2 && raw[0] == '"' && jsonKeyIs(raw, s) +} + // findJSONMember returns the member of obj named name. When the key repeats, // the last one wins, as it does for JSON.parse and encoding/json. func findJSONMember(obj []byte, name string) (jsonMember, bool, error) { diff --git a/internal/handler/jsonscan_test.go b/internal/handler/jsonscan_test.go index 5003809..9066a5f 100644 --- a/internal/handler/jsonscan_test.go +++ b/internal/handler/jsonscan_test.go @@ -96,3 +96,60 @@ func TestWriteFilteredJSONObject(t *testing.T) { t.Errorf("filtered = %s, %v", out.String(), err) } } + +func TestForEachJSONElement(t *testing.T) { + var got []string + err := forEachJSONElement([]byte(` [ 1, "a,]", {"b": [2]} ,[] ] `), func(e []byte) error { + got = append(got, string(e)) + return nil + }) + want := []string{`1`, `"a,]"`, `{"b": [2]}`, `[]`} + if err != nil || len(got) != len(want) { + t.Fatalf("elements = %q, %v; want %q", got, err, want) + } + for i := range want { + if got[i] != want[i] { + t.Errorf("element %d = %s, want %s", i, got[i], want[i]) + } + } + + if err := forEachJSONElement([]byte(`[]`), func([]byte) error { t.Error("called for empty array"); return nil }); err != nil { + t.Errorf("empty array: err = %v", err) + } + for _, bad := range []string{`{}`, `[1,`, `[1 2]`, `[1`} { + if err := forEachJSONElement([]byte(bad), func([]byte) error { return nil }); err == nil { + t.Errorf("%s: expected an error", bad) + } + } +} + +func TestRewriteJSONMembers(t *testing.T) { + obj := []byte(`{ "a": 1, "b": [2], "c" : 3 }`) + var out bytes.Buffer + err := rewriteJSONMembers(&out, obj, func(out *bytes.Buffer, m jsonMember) (bool, error) { + if !jsonKeyIs(m.key(obj), "b") { + return false, nil + } + out.WriteString(`"B"`) + return true, nil + }) + if want := `{ "a": 1, "b": "B", "c" : 3 }`; err != nil || out.String() != want { + t.Errorf("rewritten = %s, %v; want %s", out.String(), err, want) + } +} + +func TestJSONStringIs(t *testing.T) { + for raw, want := range map[string]bool{ + `"~dev"`: true, + `"~\u0064ev"`: true, + `"~dev2"`: false, + `{"~dev":1}`: false, + `"`: false, + ``: false, + `["~dev"]`: false, + } { + if got := jsonStringIs([]byte(raw), "~dev"); got != want { + t.Errorf("jsonStringIs(%s) = %v, want %v", raw, got, want) + } + } +} From 8b3b933fd4b8d28a5e533e37f65d2e2926941c38 Mon Sep 17 00:00:00 2001 From: Abhinav Gautam Date: Sat, 10 Oct 2026 12:04:31 +0530 Subject: [PATCH 185/633] Stream archives when listing and reading browsed files (#408) * Stream archives when listing and reading browsed files * Fail browse file responses on archive read errors --- internal/server/browse.go | 176 +++++-- internal/server/browse_archive.go | 405 +++++++++++++++ internal/server/browse_archive_test.go | 680 +++++++++++++++++++++++++ internal/server/server.go | 4 + 4 files changed, 1209 insertions(+), 56 deletions(-) create mode 100644 internal/server/browse_archive.go create mode 100644 internal/server/browse_archive_test.go diff --git a/internal/server/browse.go b/internal/server/browse.go index fc5e658..26ee63e 100644 --- a/internal/server/browse.go +++ b/internal/server/browse.go @@ -3,6 +3,7 @@ package server import ( "bufio" "encoding/json" + "errors" "fmt" "io" "net/http" @@ -20,10 +21,13 @@ import ( const ( contentTypePlainText = "text/plain; charset=utf-8" browseSniffSize = 512 + // browsePrefetchSize is how much of a browsed file is read before the + // response starts. Read errors within it still produce an error status. + browsePrefetchSize = 64 << 10 ) -// maxBrowseArchiveSize caps how much data openArchive will buffer for -// prefix detection. Artifacts larger than this are rejected to prevent +// maxBrowseArchiveSize caps the compressed artifact size the browse and diff +// endpoints will read. Artifacts larger than this are rejected to prevent // memory exhaustion from a single request. const maxBrowseArchiveSize = 512 << 20 // 512 MB @@ -57,45 +61,32 @@ func isMetadataSidecar(filename string) bool { // "repo-hash/"). Returns "" if there's no single root or the archive is flat. func detectSingleRootDir(reader archives.Reader) string { files, err := reader.List() - if err != nil || len(files) == 0 { + if err != nil { return "" } - var root string + var root rootDetector for _, f := range files { - parts := strings.SplitN(f.Path, "/", 2) //nolint:mnd // split into dir + rest - if len(parts) == 0 { - continue - } - dir := parts[0] - if root == "" { - root = dir - } else if dir != root { - return "" + if !root.add(f.Path) { + break } } - - if root == "" { - return "" - } - return root + "/" + return root.prefix() } -// openArchive opens a cached artifact as an archive reader, auto-detecting -// and stripping a single top-level directory prefix (like GitHub zipballs). -// For npm, the hardcoded "package/" prefix takes precedence. +// openArchive opens a cached artifact as a random-access archive reader, +// auto-detecting and stripping a single top-level directory prefix (like +// GitHub zipballs). For npm, the hardcoded "package/" prefix takes precedence. +// The whole artifact is buffered, so only the version diff uses it; listing +// and file reads stream through browseArchive instead. func openArchive(filename string, content io.Reader, ecosystem string) (archives.Reader, error) { //nolint:ireturn // wraps multiple archive implementations - limited := io.LimitReader(content, maxBrowseArchiveSize+1) - data, err := io.ReadAll(limited) + data, err := readBrowseInput(content, maxBrowseArchiveSize) if err != nil { - return nil, fmt.Errorf("reading artifact: %w", err) - } - if int64(len(data)) > maxBrowseArchiveSize { - return nil, fmt.Errorf("artifact too large for browsing (%d bytes)", len(data)) + return nil, err } if ecosystem == "npm" { - return archives.OpenBytesWithPrefix(filename, data, "package/") + return archives.OpenBytesWithPrefix(filename, data, npmPackagePrefix) } probe, err := archives.OpenBytes(filename, data) @@ -108,6 +99,35 @@ func openArchive(filename string, content io.Reader, ecosystem string) (archives return archives.OpenBytesWithPrefix(filename, data, prefix) } +// browseLimitsOrDefault returns the stream limits for listing and file reads. +func (s *Server) browseLimitsOrDefault() archives.StreamOptions { + if s.browseLimits != (archives.StreamOptions{}) { + return s.browseLimits + } + return defaultBrowseLimits +} + +// openBrowseArchive prepares a cached artifact for streaming. content is the +// already open storage reader; later passes reopen the artifact. +func (s *Server) openBrowseArchive(r *http.Request, artifact *database.Artifact, content io.Reader, ecosystem string) (*browseArchive, error) { + storagePath := artifact.StoragePath.String + reopen := func() (io.ReadCloser, error) { + return s.storage.Open(r.Context(), storagePath) + } + return newBrowseArchive(artifact.Filename, ecosystem, content, reopen, s.browseLimitsOrDefault()) +} + +// browseArchiveError reports a failure to read an archive for browsing. +func (s *Server) browseArchiveError(w http.ResponseWriter, err error, filename string) { + if isBrowseLimitError(err) { + s.logger.Warn("archive exceeds browse limits", "error", err, "filename", filename) + internalError(w, "archive exceeds browse limits") + return + } + s.logger.Error("failed to open archive", "error", err, "filename", filename) + internalError(w, "failed to open archive") +} + // BrowseListResponse contains the file listing for a directory in an archives. type BrowseListResponse struct { Path string `json:"path"` @@ -253,20 +273,16 @@ func (s *Server) browseList(w http.ResponseWriter, r *http.Request, ecosystem, n } defer func() { _ = artifactReader.Close() }() - // Open archive with auto-detected prefix stripping - archiveReader, err := openArchive(cachedArtifact.Filename, artifactReader, ecosystem) + archive, err := s.openBrowseArchive(r, cachedArtifact, artifactReader, ecosystem) if err != nil { - s.logger.Error("failed to open archive", "error", err, "filename", cachedArtifact.Filename) - internalError(w, "failed to open archive") + s.browseArchiveError(w, err, cachedArtifact.Filename) return } - defer func() { _ = archiveReader.Close() }() - // List files in the directory - files, err := archiveReader.ListDir(dirPath) + // List files in the directory, with the root prefix stripped + files, err := archive.ListDir(dirPath) if err != nil { - s.logger.Error("failed to list directory", "error", err, "path", dirPath) - internalError(w, "failed to list directory") + s.browseArchiveError(w, err, cachedArtifact.Filename) return } @@ -340,35 +356,48 @@ func (s *Server) browseFile(w http.ResponseWriter, r *http.Request, ecosystem, n } defer func() { _ = artifactReader.Close() }() - // Open archive with auto-detected prefix stripping - archiveReader, err := openArchive(cachedArtifact.Filename, artifactReader, ecosystem) + archive, err := s.openBrowseArchive(r, cachedArtifact, artifactReader, ecosystem) if err != nil { - s.logger.Error("failed to open archive", "error", err, "filename", cachedArtifact.Filename) - internalError(w, "failed to open archive") + s.browseArchiveError(w, err, cachedArtifact.Filename) return } - defer func() { _ = archiveReader.Close() }() - // Extract the file - fileReader, err := archiveReader.Extract(filePath) - if err != nil { - if strings.Contains(err.Error(), "not found") { - notFound(w, "file not found") - return - } + // Find the file and stream it straight from the archive + fileReader, err := archive.Extract(filePath) + switch { + case errors.Is(err, errBrowseNotFound): + notFound(w, "file not found") + return + case errors.Is(err, errBrowseIsDir): s.logger.Error("failed to extract file", "error", err, "path", filePath) internalError(w, "failed to extract file") return + case err != nil: + s.browseArchiveError(w, err, cachedArtifact.Filename) + return } defer func() { _ = fileReader.Close() }() + s.writeBrowseFile(w, fileReader, filePath, cachedArtifact.Filename) +} + +// writeBrowseFile sends a file streamed from an archive. The start of the file +// is read before any headers, so a truncated or over-limit entry that fails +// there still gets an error status. A failure after the response has started +// aborts it, so the client sees an incomplete download rather than a +// successful one. +func (s *Server) writeBrowseFile(w http.ResponseWriter, file io.Reader, filePath, artifactName string) { + source := &trackedReader{reader: file} + content := bufio.NewReaderSize(source, browsePrefetchSize) + head, err := content.Peek(browsePrefetchSize) + if err != nil && !errors.Is(err, io.EOF) { + s.browseReadError(w, err, filePath, artifactName) + return + } + contentType, knownPath := detectContentTypeFromPath(filePath) - var content io.Reader = fileReader if !knownPath { - bufferedFile := bufio.NewReaderSize(fileReader, browseSniffSize) - prefix, _ := bufferedFile.Peek(browseSniffSize) - contentType = detectContentTypeFromPrefix(prefix) - content = bufferedFile + contentType = detectContentTypeFromPrefix(head[:min(len(head), browseSniffSize)]) } w.Header().Set("Content-Type", contentType) w.Header().Set("Content-Security-Policy", "sandbox") @@ -377,8 +406,43 @@ func (s *Server) browseFile(w http.ResponseWriter, r *http.Request, ecosystem, n _, filename := path.Split(filePath) w.Header().Set("Content-Disposition", fmt.Sprintf("inline; filename=%q", filename)) - // Stream the file - _, _ = io.Copy(w, content) + written, err := io.Copy(w, content) + if err == nil || source.err == nil { + // A write error without a read error means the client went away. + return + } + s.logger.Error("failed to stream file from archive", "error", source.err, + "path", filePath, "filename", artifactName, "bytes", written) + // Headers are already committed: finishing normally would turn a truncated + // file into a seemingly successful download. Let net/http close the HTTP/1 + // connection or reset the HTTP/2 stream instead. + panic(http.ErrAbortHandler) +} + +// browseReadError reports a failure reading a file before its response starts. +func (s *Server) browseReadError(w http.ResponseWriter, err error, filePath, artifactName string) { + if isBrowseLimitError(err) { + s.logger.Warn("archive exceeds browse limits", "error", err, "path", filePath, "filename", artifactName) + internalError(w, "archive exceeds browse limits") + return + } + s.logger.Error("failed to read file from archive", "error", err, "path", filePath, "filename", artifactName) + internalError(w, "failed to read file") +} + +// trackedReader records the first read error other than io.EOF, so a failed +// copy can be told apart from a failed write to the client. +type trackedReader struct { + reader io.Reader + err error +} + +func (r *trackedReader) Read(p []byte) (int, error) { + n, err := r.reader.Read(p) + if err != nil && !errors.Is(err, io.EOF) && r.err == nil { + r.err = err + } + return n, err } func detectContentTypeFromPath(filename string) (string, bool) { diff --git a/internal/server/browse_archive.go b/internal/server/browse_archive.go new file mode 100644 index 0000000..4af6afe --- /dev/null +++ b/internal/server/browse_archive.go @@ -0,0 +1,405 @@ +package server + +import ( + "bufio" + "errors" + "fmt" + "io" + "path" + "strings" + + "github.com/git-pkgs/archives" + "github.com/git-pkgs/magic" +) + +// Limits applied when listing or reading a single file. Bodies are streamed +// rather than retained, so these bound decompression work per request; they +// match the limits the buffered archive reader enforces. +const ( + maxBrowseExpandedSize = 512 << 20 // 512 MB across all entries + maxBrowseEntrySize = 512 << 20 // 512 MB for any one entry + maxBrowseEntries = 100_000 +) + +// npmPackagePrefix is the directory npm wraps every tarball's contents in. +const npmPackagePrefix = "package/" + +var defaultBrowseLimits = archives.StreamOptions{ + MaxInputBytes: maxBrowseArchiveSize, + MaxEntryBytes: maxBrowseEntrySize, + MaxExpandedBytes: maxBrowseExpandedSize, + MaxEntries: maxBrowseEntries, +} + +var ( + errArtifactTooLarge = errors.New("artifact too large for browsing") + errBrowseNotFound = errors.New("file not found") + errBrowseIsDir = errors.New("path is a directory") + + // errStopWalk ends a walk early without reporting an error. + errStopWalk = errors.New("stop walk") +) + +// isBrowseLimitError reports whether err comes from one of the browse size or +// entry-count limits rather than a malformed archive. +func isBrowseLimitError(err error) bool { + return errors.Is(err, errArtifactTooLarge) || + errors.Is(err, archives.ErrInputLimit) || + errors.Is(err, archives.ErrEntrySizeLimit) || + errors.Is(err, archives.ErrDecompressLimit) || + errors.Is(err, archives.ErrEntryLimit) +} + +// browseArchive reads a cached artifact sequentially for the browse endpoints +// without holding expanded file bodies in memory. TAR based formats stream from +// storage, reopening it when a second pass is needed. ZIP and conda need random +// access, so their input is read once and shared by every pass. +type browseArchive struct { + filename string + ecosystem string + limits archives.StreamOptions + reopen func() (io.ReadCloser, error) + + // first is the caller's reader, used for the first pass only. + first io.Reader + // data holds buffered ZIP or conda input. + data []byte +} + +// newBrowseArchive prepares content for streaming. The caller keeps ownership +// of content; reopen supplies a fresh reader for any later pass. +func newBrowseArchive(filename, ecosystem string, content io.Reader, reopen func() (io.ReadCloser, error), limits archives.StreamOptions) (*browseArchive, error) { + buffered := bufio.NewReaderSize(content, browseSniffSize) + sniff, err := buffered.Peek(browseSniffSize) + if err != nil && err != io.EOF { + return nil, fmt.Errorf("reading artifact: %w", err) + } + + a := &browseArchive{filename: filename, ecosystem: ecosystem, limits: limits, reopen: reopen} + if !needsRandomAccess(filename, sniff) { + a.first = buffered + return a, nil + } + + a.data, err = readBrowseInput(buffered, limits.MaxInputBytes) + if err != nil { + return nil, err + } + return a, nil +} + +// readBrowseInput reads all of r, failing once it exceeds limit bytes. +func readBrowseInput(r io.Reader, limit int64) ([]byte, error) { + data, err := io.ReadAll(io.LimitReader(r, limit+1)) + if err != nil { + return nil, fmt.Errorf("reading artifact: %w", err) + } + if int64(len(data)) > limit { + return nil, fmt.Errorf("%w (%d bytes)", errArtifactTooLarge, len(data)) + } + return data, nil +} + +// needsRandomAccess reports whether the archives library buffers this format +// when streaming it, following the library's own detection order: a known +// extension first, then the content's leading bytes. +func needsRandomAccess(filename string, sniff []byte) bool { + name := strings.ToLower(filename) + for _, suffix := range []string{".tar.gz", ".tar.bz2", ".tar.xz", ".tar.zst"} { + if strings.HasSuffix(name, suffix) { + return false + } + } + switch path.Ext(name) { + case ".zip", ".jar", ".whl", ".nupkg", ".egg", ".vsix", ".conda": + return true + case ".tar", ".tgz", ".crate", ".gem": + return false + } + return magic.DetectPrefix(sniff).Format == "zip" +} + +// openStream starts a pass over the archive. The returned function releases +// the stream and any reader opened for it. +func (a *browseArchive) openStream() (*archives.Stream, func(), error) { + if a.data != nil { + stream, err := archives.OpenStreamBytes(a.filename, a.data, a.limits) + if err != nil { + return nil, nil, err + } + return stream, func() { _ = stream.Close() }, nil + } + + if a.first != nil { + content := a.first + a.first = nil + stream, err := archives.OpenStream(a.filename, content, a.limits) + if err != nil { + return nil, nil, err + } + return stream, func() { _ = stream.Close() }, nil + } + + content, err := a.reopen() + if err != nil { + return nil, nil, fmt.Errorf("reopening artifact: %w", err) + } + stream, err := archives.OpenStream(a.filename, content, a.limits) + if err != nil { + _ = content.Close() + return nil, nil, err + } + return stream, func() { + _ = stream.Close() + _ = content.Close() + }, nil +} + +// walk calls fn for each entry in archive order until fn returns an error or +// the archive ends. fn may read the entry body from the stream. +func (a *browseArchive) walk(fn func(entry *archives.StreamEntry) error) error { + stream, release, err := a.openStream() + if err != nil { + return err + } + defer release() + + for { + entry, err := stream.Next() + if err == io.EOF { + return nil + } + if err != nil { + return err + } + if err := fn(entry); err != nil { + if errors.Is(err, errStopWalk) { + return nil + } + return err + } + } +} + +// prefix returns the directory stripped from every path: npm's "package/", or +// a single top-level directory shared by all entries (like GitHub zipballs). +// For other ecosystems this costs one pass, which stops early once two +// top-level names differ. +func (a *browseArchive) prefix() (string, error) { + if a.ecosystem == "npm" { + return npmPackagePrefix, nil + } + + var root rootDetector + err := a.walk(func(entry *archives.StreamEntry) error { + if !root.add(entry.Path) { + return errStopWalk + } + return nil + }) + if err != nil { + return "", err + } + return root.prefix(), nil +} + +// ListDir returns the entries directly inside dirPath, after prefix stripping, +// in a single pass that keeps only entry metadata. +func (a *browseArchive) ListDir(dirPath string) ([]archives.FileInfo, error) { + var entries []archives.FileInfo + var root rootDetector + err := a.walk(func(entry *archives.StreamEntry) error { + entries = append(entries, entry.FileInfo) + root.add(entry.Path) + return nil + }) + if err != nil { + return nil, err + } + + prefix := npmPackagePrefix + if a.ecosystem != "npm" { + prefix = root.prefix() + } + return listBrowseDir(entries, prefix, dirPath), nil +} + +// Extract returns the body of the first entry at filePath, streamed from the +// archive. The caller must close the returned reader. +func (a *browseArchive) Extract(filePath string) (io.ReadCloser, error) { + prefix, err := a.prefix() + if err != nil { + return nil, err + } + target := prefix + filePath + + stream, release, err := a.openStream() + if err != nil { + return nil, err + } + for { + entry, err := stream.Next() + if err == io.EOF { + release() + return nil, fmt.Errorf("%w: %s", errBrowseNotFound, filePath) + } + if err != nil { + release() + return nil, err + } + if entry.Path != target { + continue + } + if entry.IsDir { + release() + return nil, fmt.Errorf("%w: %s", errBrowseIsDir, filePath) + } + return &streamedFile{Reader: stream, release: release}, nil + } +} + +// streamedFile is an entry body that releases its archive stream on Close. +type streamedFile struct { + io.Reader + release func() +} + +func (f *streamedFile) Close() error { + if f.release != nil { + f.release() + f.release = nil + } + return nil +} + +// rootDetector tracks whether every path seen so far lives under one +// top-level directory. +type rootDetector struct { + root string + mixed bool + seen bool +} + +// add records p and reports whether a single root is still possible. +func (d *rootDetector) add(p string) bool { + if d.mixed { + return false + } + d.seen = true + dir, _, _ := strings.Cut(p, "/") + if d.root == "" { + d.root = dir + } else if dir != d.root { + d.mixed = true + } + return !d.mixed +} + +// prefix returns the shared root with a trailing slash, or "" if paths have +// more than one root or none were seen. +func (d *rootDetector) prefix() string { + if d.mixed || !d.seen || d.root == "" { + return "" + } + return d.root + "/" +} + +// listBrowseDir lists dirPath from entry metadata the same way a reader from +// archives.OpenWithPrefix does: the prefix is joined to dirPath before +// listing, then stripped from the results. +func listBrowseDir(entries []archives.FileInfo, prefix, dirPath string) []archives.FileInfo { + files := listDir(entries, prefix+dirPath) + if prefix == "" { + return files + } + return stripBrowsePrefix(files, prefix) +} + +// listDir mirrors archives.Reader.ListDir: entries directly in dirPath are +// returned in archive order. Deeper paths add one synthesized entry for each +// immediate subdirectory. +func listDir(entries []archives.FileInfo, dirPath string) []archives.FileInfo { + dirPath = normalizeBrowseDir(dirPath) + var files []archives.FileInfo + seenDirs := make(map[string]bool) + + for _, f := range entries { + p := f.Path + + if isInBrowseDir(p, dirPath) { + if f.IsDir { + name := strings.TrimSuffix(strings.TrimPrefix(p, dirPath), "/") + if seenDirs[name] { + continue + } + seenDirs[name] = true + } + files = append(files, f) + continue + } + + if dirPath != "" && !strings.HasPrefix(p, dirPath) { + continue + } + rel := strings.TrimSuffix(strings.TrimPrefix(p, dirPath), "/") + name, _, nested := strings.Cut(rel, "/") + if nested && !seenDirs[name] { + seenDirs[name] = true + files = append(files, archives.FileInfo{ + Path: dirPath + name + "/", + Name: name, + IsDir: true, + }) + } + } + + return files +} + +func normalizeBrowseDir(dirPath string) string { + dirPath = strings.Trim(strings.TrimSpace(dirPath), "/") + if dirPath == "" { + return "" + } + return dirPath + "/" +} + +// isInBrowseDir reports whether filePath is directly in the normalized +// dirPath. An entry naming the directory itself counts as inside it. +func isInBrowseDir(filePath, dirPath string) bool { + filePath = strings.TrimSuffix(filePath, "/") + if dirPath == "" { + return !strings.Contains(filePath, "/") + } + if filePath == dirPath[:len(dirPath)-1] { + return true + } + rest, ok := strings.CutPrefix(filePath, dirPath) + return ok && !strings.Contains(rest, "/") +} + +// stripBrowsePrefix removes prefix from each path, dropping entries outside +// it and the prefix directory itself. +func stripBrowsePrefix(files []archives.FileInfo, prefix string) []archives.FileInfo { + result := make([]archives.FileInfo, 0, len(files)) + for _, f := range files { + stripped, ok := strings.CutPrefix(f.Path, prefix) + if !ok || stripped == "" || stripped == "/" { + continue + } + f.Path = stripped + f.Name = entryName(stripped) + result = append(result, f) + } + return result +} + +// entryName returns the last element of an archive path, ignoring a trailing +// slash. +func entryName(p string) string { + p = strings.TrimSuffix(p, "/") + if i := strings.LastIndexByte(p, '/'); i >= 0 { + return p[i+1:] + } + return p +} diff --git a/internal/server/browse_archive_test.go b/internal/server/browse_archive_test.go new file mode 100644 index 0000000..71491f2 --- /dev/null +++ b/internal/server/browse_archive_test.go @@ -0,0 +1,680 @@ +package server + +import ( + "archive/tar" + "archive/zip" + "bytes" + "compress/gzip" + "database/sql" + "encoding/json" + "fmt" + "io" + "net/http" + "net/http/httptest" + "net/url" + "os" + "path/filepath" + "runtime" + "strings" + "testing" + + "github.com/git-pkgs/archives" + "github.com/git-pkgs/proxy/internal/database" +) + +type archiveEntry struct { + name string + body string + dir bool +} + +func buildTar(t testing.TB, entries []archiveEntry) []byte { + t.Helper() + buf := new(bytes.Buffer) + tw := tar.NewWriter(buf) + for _, e := range entries { + header := &tar.Header{Name: e.name, Mode: 0o644, Size: int64(len(e.body)), Typeflag: tar.TypeReg} + if e.dir { + header.Typeflag = tar.TypeDir + header.Mode = 0o755 + header.Size = 0 + } + if err := tw.WriteHeader(header); err != nil { + t.Fatalf("writing tar header: %v", err) + } + if _, err := io.WriteString(tw, e.body); err != nil { + t.Fatalf("writing tar body: %v", err) + } + } + if err := tw.Close(); err != nil { + t.Fatalf("closing tar: %v", err) + } + return buf.Bytes() +} + +func gzipBytes(t testing.TB, data []byte) []byte { + t.Helper() + buf := new(bytes.Buffer) + gw := gzip.NewWriter(buf) + if _, err := gw.Write(data); err != nil { + t.Fatalf("writing gzip: %v", err) + } + if err := gw.Close(); err != nil { + t.Fatalf("closing gzip: %v", err) + } + return buf.Bytes() +} + +func buildTarGz(t testing.TB, entries []archiveEntry) []byte { + t.Helper() + return gzipBytes(t, buildTar(t, entries)) +} + +func buildZip(t testing.TB, entries []archiveEntry) []byte { + t.Helper() + buf := new(bytes.Buffer) + zw := zip.NewWriter(buf) + for _, e := range entries { + f, err := zw.Create(e.name) + if err != nil { + t.Fatalf("creating zip entry: %v", err) + } + if _, err := io.WriteString(f, e.body); err != nil { + t.Fatalf("writing zip entry: %v", err) + } + } + if err := zw.Close(); err != nil { + t.Fatalf("closing zip: %v", err) + } + return buf.Bytes() +} + +// buildGem wraps entries in a gem: an outer tar holding data.tar.gz. +func buildGem(t testing.TB, entries []archiveEntry) []byte { + t.Helper() + return buildTar(t, []archiveEntry{ + {name: "metadata.gz", body: string(gzipBytes(t, []byte("--- {}\n")))}, + {name: "data.tar.gz", body: string(buildTarGz(t, entries))}, + }) +} + +// testBrowseVersion is the version every browse fixture is cached under. +const testBrowseVersion = "1.0.0" + +// cacheBrowseArtifact stores data as the cached artifact for ecosystem/name. +func cacheBrowseArtifact(t *testing.T, ts *testServer, ecosystem, name, filename string, data []byte) { + t.Helper() + + storagePath := fmt.Sprintf("%s-%s-%s", ecosystem, name, filename) + artifactsDir := filepath.Join(ts.tempDir, "artifacts") + if err := os.MkdirAll(artifactsDir, 0o755); err != nil { + t.Fatalf("creating artifacts dir: %v", err) + } + if err := os.WriteFile(filepath.Join(artifactsDir, storagePath), data, 0o644); err != nil { + t.Fatalf("writing artifact: %v", err) + } + + pkg := &database.Package{ + PURL: fmt.Sprintf("pkg:%s/%s", ecosystem, name), + Ecosystem: ecosystem, + Name: name, + } + if err := ts.db.UpsertPackage(pkg); err != nil { + t.Fatalf("upserting package: %v", err) + } + ver := &database.Version{PURL: pkg.PURL + "@" + testBrowseVersion, PackagePURL: pkg.PURL} + if err := ts.db.UpsertVersion(ver); err != nil { + t.Fatalf("upserting version: %v", err) + } + artifact := &database.Artifact{ + VersionPURL: ver.PURL, + Filename: filename, + UpstreamURL: "https://example.com/" + filename, + StoragePath: sql.NullString{String: storagePath, Valid: true}, + } + if err := ts.db.UpsertArtifact(artifact); err != nil { + t.Fatalf("upserting artifact: %v", err) + } +} + +func browseGet(ts *testServer, target string) *httptest.ResponseRecorder { + req := httptest.NewRequest(http.MethodGet, target, nil) + w := httptest.NewRecorder() + ts.handler.ServeHTTP(w, req) + return w +} + +func browseListURL(ecosystem, name, dir string) string { + return fmt.Sprintf("/ui/api/browse/%s/%s/%s?path=%s", ecosystem, name, testBrowseVersion, url.QueryEscape(dir)) +} + +func browseFileURL(ecosystem, name, file string) string { + return fmt.Sprintf("/ui/api/browse/%s/%s/%s/file/%s", ecosystem, name, testBrowseVersion, file) +} + +func decodeListing(t *testing.T, w *httptest.ResponseRecorder) BrowseListResponse { + t.Helper() + if w.Code != http.StatusOK { + t.Fatalf("listing status = %d, body %s", w.Code, w.Body.String()) + } + var resp BrowseListResponse + if err := json.NewDecoder(w.Body).Decode(&resp); err != nil { + t.Fatalf("decoding listing: %v", err) + } + return resp +} + +func listingPaths(resp BrowseListResponse) []string { + paths := make([]string, len(resp.Files)) + for i, f := range resp.Files { + paths[i] = f.Path + } + return paths +} + +type browseFixture struct { + name string + ecosystem string + filename string + data []byte +} + +func browseFixtures(t *testing.T) []browseFixture { + t.Helper() + + rooted := []archiveEntry{ + {name: "repo-abc123/", dir: true}, + {name: "repo-abc123/README.md", body: "# rooted\n"}, + {name: "repo-abc123/src/", dir: true}, + {name: "repo-abc123/src/main.go", body: "package main\n"}, + {name: "repo-abc123/src/util/strings.go", body: "package util\n"}, + {name: "repo-abc123/docs/guide.md", body: "guide\n"}, + } + npm := []archiveEntry{ + {name: "package/package.json", body: `{"name":"x"}`}, + {name: "package/README.md", body: "# npm\n"}, + {name: "package/src/main.go", body: "package main\n"}, + {name: "package/src/util/strings.go", body: "package util\n"}, + {name: "other/ignored.txt", body: "outside the package prefix\n"}, + } + flat := []archiveEntry{ + {name: "README.md", body: "# flat\n"}, + {name: "src/main.go", body: "package main\n"}, + {name: "src/util/strings.go", body: "package util\n"}, + {name: "docs/guide.md", body: "guide\n"}, + } + + return []browseFixture{ + {"tar.gz with root dir", "cargo", "demo-1.0.0.crate", buildTarGz(t, rooted)}, + {"extensionless tar.gz", "cargo", "artifact", buildTarGz(t, rooted)}, + {"npm package prefix", "npm", "demo-1.0.0.tgz", buildTarGz(t, npm)}, + {"flat tar.gz", "cargo", "demo-1.0.0.tar.gz", buildTarGz(t, flat)}, + {"zip with root dir", "composer", "demo.zip", buildZip(t, rooted)}, + {"extensionless zip", "composer", "d2e2f014ccd6ec9fae8dbe6336a4164346a2a856", buildZip(t, rooted)}, + {"flat zip", "composer", "demo.zip", buildZip(t, flat)}, + {"gem", "gem", "demo-1.0.0.gem", buildGem(t, flat)}, + } +} + +func TestBrowseListingStripsPrefix(t *testing.T) { + for _, fx := range browseFixtures(t) { + t.Run(fx.name, func(t *testing.T) { + ts := newTestServer(t) + defer ts.close() + cacheBrowseArtifact(t, ts, fx.ecosystem, "demo", fx.filename, fx.data) + + root := listingPaths(decodeListing(t, browseGet(ts, browseListURL(fx.ecosystem, "demo", "")))) + for _, p := range root { + if strings.HasPrefix(p, "repo-abc123/") || strings.HasPrefix(p, "package/") { + t.Errorf("root listing kept archive prefix: %q", p) + } + } + if !containsPath(root, "README.md") || !containsPath(root, "src/") { + t.Errorf("root listing = %v, want README.md and src/", root) + } + + src := listingPaths(decodeListing(t, browseGet(ts, browseListURL(fx.ecosystem, "demo", "src")))) + if !containsPath(src, "src/main.go") || !containsPath(src, "src/util/") { + t.Errorf("src listing = %v, want src/main.go and src/util/", src) + } + }) + } +} + +func containsPath(paths []string, want string) bool { + for _, p := range paths { + if p == want { + return true + } + } + return false +} + +// TestBrowseListingMatchesBufferedReader checks the streamed listing returns +// exactly what the buffered archive reader returned for the same request. +func TestBrowseListingMatchesBufferedReader(t *testing.T) { + dirs := []string{"", "/", "src", "src/", "/src", "src/util", "docs", "missing", "README.md"} + + for _, fx := range browseFixtures(t) { + t.Run(fx.name, func(t *testing.T) { + ts := newTestServer(t) + defer ts.close() + cacheBrowseArtifact(t, ts, fx.ecosystem, "demo", fx.filename, fx.data) + + buffered, err := openArchive(fx.filename, bytes.NewReader(fx.data), fx.ecosystem) + if err != nil { + t.Fatalf("openArchive: %v", err) + } + defer func() { _ = buffered.Close() }() + + for _, dir := range dirs { + want, err := buffered.ListDir(dir) + if err != nil { + t.Fatalf("buffered ListDir(%q): %v", dir, err) + } + got := decodeListing(t, browseGet(ts, browseListURL(fx.ecosystem, "demo", dir))) + if len(got.Files) != len(want) { + t.Fatalf("ListDir(%q) = %v, want %d entries", dir, listingPaths(got), len(want)) + } + for i, f := range got.Files { + w := want[i] + if f.Path != w.Path || f.Name != w.Name || f.IsDir != w.IsDir || f.Size != w.Size { + t.Errorf("ListDir(%q)[%d] = %+v, want %+v", dir, i, f, w) + } + } + } + }) + } +} + +func TestBrowseFileContents(t *testing.T) { + for _, fx := range browseFixtures(t) { + t.Run(fx.name, func(t *testing.T) { + ts := newTestServer(t) + defer ts.close() + cacheBrowseArtifact(t, ts, fx.ecosystem, "demo", fx.filename, fx.data) + + buffered, err := openArchive(fx.filename, bytes.NewReader(fx.data), fx.ecosystem) + if err != nil { + t.Fatalf("openArchive: %v", err) + } + defer func() { _ = buffered.Close() }() + + for _, file := range []string{"README.md", "src/main.go", "src/util/strings.go"} { + rc, err := buffered.Extract(file) + if err != nil { + t.Fatalf("buffered Extract(%q): %v", file, err) + } + want, _ := io.ReadAll(rc) + _ = rc.Close() + + w := browseGet(ts, browseFileURL(fx.ecosystem, "demo", file)) + if w.Code != http.StatusOK { + t.Fatalf("file %s status = %d, body %s", file, w.Code, w.Body.String()) + } + if w.Body.String() != string(want) { + t.Errorf("file %s = %q, want %q", file, w.Body.String(), want) + } + wantType, _ := detectContentTypeFromPath(file) + if got := w.Header().Get("Content-Type"); got != wantType { + t.Errorf("file %s Content-Type = %q, want %q", file, got, wantType) + } + if got := w.Header().Get("Content-Disposition"); !strings.Contains(got, filepath.Base(file)) { + t.Errorf("file %s Content-Disposition = %q", file, got) + } + } + + w := browseGet(ts, browseFileURL(fx.ecosystem, "demo", "missing.txt")) + if w.Code != http.StatusNotFound { + t.Errorf("missing file status = %d, want 404", w.Code) + } + }) + } +} + +func TestBrowseFileFirstDuplicateWins(t *testing.T) { + ts := newTestServer(t) + defer ts.close() + data := buildTarGz(t, []archiveEntry{ + {name: "package/index.js", body: "first"}, + {name: "package/index.js", body: "second"}, + }) + cacheBrowseArtifact(t, ts, "npm", "dup", "dup-1.0.0.tgz", data) + + w := browseGet(ts, browseFileURL("npm", "dup", "index.js")) + if w.Code != http.StatusOK || w.Body.String() != "first" { + t.Fatalf("got %d %q, want 200 \"first\"", w.Code, w.Body.String()) + } +} + +func TestBrowseLimits(t *testing.T) { + entries := []archiveEntry{ + {name: "package/a.txt", body: strings.Repeat("a", 100)}, + {name: "package/b.txt", body: strings.Repeat("b", 100)}, + {name: "package/c.txt", body: strings.Repeat("c", 100)}, + } + tarGz := buildTarGz(t, entries) + zipped := buildZip(t, entries) + + tests := []struct { + name string + filename string + data []byte + limits archives.StreamOptions + file string + }{ + {"entry count", "limit.tgz", tarGz, archives.StreamOptions{MaxEntries: 2}, "c.txt"}, + {"entry size", "limit.tgz", tarGz, archives.StreamOptions{MaxEntryBytes: 50}, "a.txt"}, + {"expanded size", "limit.tgz", tarGz, archives.StreamOptions{MaxExpandedBytes: 250}, "c.txt"}, + {"input size", "limit.tgz", tarGz, archives.StreamOptions{MaxInputBytes: int64(len(tarGz) / 2)}, "c.txt"}, + {"zip entry count", "limit.zip", zipped, archives.StreamOptions{MaxEntries: 2}, "c.txt"}, + {"zip input size", "limit.zip", zipped, archives.StreamOptions{MaxInputBytes: int64(len(zipped) / 2)}, "a.txt"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ts := newTestServer(t) + defer ts.close() + ts.server.browseLimits = tt.limits + cacheBrowseArtifact(t, ts, "npm", "limited", tt.filename, tt.data) + + for _, target := range []string{ + browseListURL("npm", "limited", ""), + browseFileURL("npm", "limited", tt.file), + } { + w := browseGet(ts, target) + if w.Code != http.StatusInternalServerError { + t.Fatalf("%s status = %d, want 500 (body %s)", target, w.Code, w.Body.String()) + } + if !strings.Contains(w.Body.String(), "archive exceeds browse limits") { + t.Errorf("%s body = %s, want limit error", target, w.Body.String()) + } + } + }) + } +} + +// largeBrowseArchive is a gzip tarball with many unrelated entries and one +// small file. The padding compresses well, so the fixture is small on disk +// while its expanded size dwarfs what a streamed request should allocate. +func largeBrowseArchive(t testing.TB, entries, entrySize int) []byte { + t.Helper() + buf := new(bytes.Buffer) + gw := gzip.NewWriter(buf) + tw := tar.NewWriter(gw) + padding := make([]byte, entrySize) + write := func(name string, body []byte) { + if err := tw.WriteHeader(&tar.Header{Name: name, Mode: 0o644, Size: int64(len(body))}); err != nil { + t.Fatalf("writing tar header: %v", err) + } + if _, err := tw.Write(body); err != nil { + t.Fatalf("writing tar body: %v", err) + } + } + write("repo-large/README.md", []byte("small file\n")) + for i := range entries { + write(fmt.Sprintf("repo-large/vendor/blob%05d.bin", i), padding) + } + if err := tw.Close(); err != nil { + t.Fatalf("closing tar: %v", err) + } + if err := gw.Close(); err != nil { + t.Fatalf("closing gzip: %v", err) + } + return buf.Bytes() +} + +// allocatedBytes returns the bytes allocated while fn runs. Total allocation +// bounds the peak heap growth fn can cause. +func allocatedBytes(fn func()) uint64 { + runtime.GC() + var before, after runtime.MemStats + runtime.ReadMemStats(&before) + fn() + runtime.ReadMemStats(&after) + return after.TotalAlloc - before.TotalAlloc +} + +func TestBrowseLargeArchiveMemory(t *testing.T) { + const ( + entries = 1024 + entrySize = 32 << 10 + expanded = entries * entrySize // 32 MiB + ) + data := largeBrowseArchive(t, entries, entrySize) + + ts := newTestServer(t) + defer ts.close() + cacheBrowseArtifact(t, ts, "cargo", "large", "large-1.0.0.crate", data) + + buffered := allocatedBytes(func() { + r, err := openArchive("large-1.0.0.crate", bytes.NewReader(data), "cargo") + if err != nil { + t.Fatalf("openArchive: %v", err) + } + _ = r.Close() + }) + + var listing BrowseListResponse + list := allocatedBytes(func() { + listing = decodeListing(t, browseGet(ts, browseListURL("cargo", "large", ""))) + }) + if !containsPath(listingPaths(listing), "README.md") || !containsPath(listingPaths(listing), "vendor/") { + t.Fatalf("listing = %v, want README.md and vendor/", listingPaths(listing)) + } + + var w *httptest.ResponseRecorder + file := allocatedBytes(func() { + w = browseGet(ts, browseFileURL("cargo", "large", "README.md")) + }) + if w.Code != http.StatusOK || w.Body.String() != "small file\n" { + t.Fatalf("file = %d %q, want 200 \"small file\\n\"", w.Code, w.Body.String()) + } + + t.Logf("compressed %d bytes, expanded %d bytes", len(data), expanded) + t.Logf("allocated: buffered open %d, streamed listing %d, streamed file %d", buffered, list, file) + + if buffered < expanded { + t.Fatalf("buffered open allocated %d bytes, expected at least the expanded %d", buffered, expanded) + } + limit := uint64(expanded / 4) + if list > limit { + t.Errorf("streamed listing allocated %d bytes, want under %d", list, limit) + } + if file > limit { + t.Errorf("streamed file read allocated %d bytes, want under %d", file, limit) + } +} + +func BenchmarkBrowseLargeArchive(b *testing.B) { + data := largeBrowseArchive(b, 1024, 32<<10) + artifactPath := filepath.Join(b.TempDir(), "large.crate") + if err := os.WriteFile(artifactPath, data, 0o644); err != nil { + b.Fatal(err) + } + + b.Run("buffered-list", func(b *testing.B) { benchBufferedBrowse(b, data, false) }) + b.Run("streamed-list", func(b *testing.B) { benchStreamedBrowse(b, artifactPath, false) }) + b.Run("buffered-file", func(b *testing.B) { benchBufferedBrowse(b, data, true) }) + b.Run("streamed-file", func(b *testing.B) { benchStreamedBrowse(b, artifactPath, true) }) +} + +// benchBufferedBrowse lists the root, or reads README.md, through openArchive. +func benchBufferedBrowse(b *testing.B, data []byte, readFile bool) { + b.ReportAllocs() + for b.Loop() { + r, err := openArchive("large.crate", bytes.NewReader(data), "cargo") + if err != nil { + b.Fatal(err) + } + if readFile { + drainBrowseFile(b, r.Extract) + } else if _, err := r.ListDir(""); err != nil { + b.Fatal(err) + } + _ = r.Close() + } +} + +// benchStreamedBrowse lists the root, or reads README.md, through browseArchive. +func benchStreamedBrowse(b *testing.B, artifactPath string, readFile bool) { + reopen := func() (io.ReadCloser, error) { return os.Open(artifactPath) } + b.ReportAllocs() + for b.Loop() { + content, err := reopen() + if err != nil { + b.Fatal(err) + } + a, err := newBrowseArchive("large.crate", "cargo", content, reopen, defaultBrowseLimits) + if err != nil { + b.Fatal(err) + } + if readFile { + drainBrowseFile(b, a.Extract) + } else if _, err := a.ListDir(""); err != nil { + b.Fatal(err) + } + _ = content.Close() + } +} + +func drainBrowseFile(b *testing.B, extract func(string) (io.ReadCloser, error)) { + b.Helper() + rc, err := extract("README.md") + if err != nil { + b.Fatal(err) + } + _, _ = io.Copy(io.Discard, rc) + _ = rc.Close() +} + +// truncatedTar is a TAR whose only entry declares size bytes but carries just +// body, ending without the rest of the entry or the end-of-archive marker. +func truncatedTar(t testing.TB, name string, size int64, body []byte) []byte { + t.Helper() + buf := new(bytes.Buffer) + tw := tar.NewWriter(buf) + if err := tw.WriteHeader(&tar.Header{Name: name, Mode: 0o644, Size: size}); err != nil { + t.Fatalf("writing tar header: %v", err) + } + if _, err := tw.Write(body); err != nil { + t.Fatalf("writing tar body: %v", err) + } + // Deliberately not closed: tw.Close would fail on the short entry. + return buf.Bytes() +} + +func TestBrowseFileReadErrorBeforeResponse(t *testing.T) { + complete := buildTar(t, []archiveEntry{{name: "package/data.txt", body: strings.Repeat("x", 4096)}}) + + tests := []struct { + name string + filename string + data []byte + limits archives.StreamOptions + wantBody string + }{ + { + name: "truncated entry body", + filename: "short-1.0.0.tgz", + data: gzipBytes(t, truncatedTar(t, "package/data.txt", 4096, []byte("abc"))), + wantBody: "failed to read file", + }, + { + name: "input limit inside entry", + filename: "short-1.0.0.tar", + data: complete, + limits: archives.StreamOptions{MaxInputBytes: 1024}, + wantBody: "archive exceeds browse limits", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ts := newTestServer(t) + defer ts.close() + ts.server.browseLimits = tt.limits + cacheBrowseArtifact(t, ts, "npm", "short", tt.filename, tt.data) + + w := browseGet(ts, browseFileURL("npm", "short", "data.txt")) + if w.Code != http.StatusInternalServerError { + t.Fatalf("status = %d, want 500 (body %q)", w.Code, w.Body.String()) + } + if strings.Contains(w.Body.String(), "abc") || strings.Contains(w.Body.String(), "xxx") { + t.Errorf("error response leaked partial file content: %q", w.Body.String()) + } + if !strings.Contains(w.Body.String(), tt.wantBody) { + t.Errorf("body = %q, want %q", w.Body.String(), tt.wantBody) + } + }) + } +} + +func TestBrowseFileReadErrorAfterResponseStarts(t *testing.T) { + const size = 4 * browsePrefetchSize + body := bytes.Repeat([]byte("0123456789abcdef"), size/16) + complete := buildTar(t, []archiveEntry{{name: "package/big.txt", body: string(body)}}) + + tests := []struct { + name string + filename string + data []byte + limits archives.StreamOptions + wantErr bool + }{ + { + name: "complete file", + filename: "big-1.0.0.tar", + data: complete, + }, + { + name: "truncated entry body", + filename: "big-1.0.0.tgz", + data: gzipBytes(t, truncatedTar(t, "package/big.txt", size, body[:size/2])), + wantErr: true, + }, + { + name: "input limit inside entry", + filename: "big-1.0.0.tar", + data: complete, + limits: archives.StreamOptions{MaxInputBytes: size / 2}, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ts := newTestServer(t) + defer ts.close() + ts.server.browseLimits = tt.limits + cacheBrowseArtifact(t, ts, "npm", "big", tt.filename, tt.data) + + srv := httptest.NewServer(ts.handler) + defer srv.Close() + + resp, err := http.Get(srv.URL + browseFileURL("npm", "big", "big.txt")) + if err != nil { + if tt.wantErr { + return + } + t.Fatalf("GET: %v", err) + } + defer func() { _ = resp.Body.Close() }() + got, readErr := io.ReadAll(resp.Body) + + if !tt.wantErr { + if resp.StatusCode != http.StatusOK || readErr != nil || !bytes.Equal(got, body) { + t.Fatalf("status %d, read error %v, %d of %d bytes", resp.StatusCode, readErr, len(got), len(body)) + } + return + } + if readErr == nil { + t.Fatalf("status %d with %d of %d bytes and no read error; want an aborted response", + resp.StatusCode, len(got), len(body)) + } + if len(got) >= len(body) { + t.Errorf("read %d bytes, want fewer than %d", len(got), len(body)) + } + }) + } +} diff --git a/internal/server/server.go b/internal/server/server.go index 7494b94..69707f7 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -64,6 +64,7 @@ import ( "strings" "time" + "github.com/git-pkgs/archives" "github.com/git-pkgs/cooldown" swaggerdoc "github.com/git-pkgs/proxy/docs/swagger" "github.com/git-pkgs/proxy/internal/accesslog" @@ -117,6 +118,9 @@ type Server struct { breakers *breakerMonitor ecoStats ecosystemStatsCache sources sourceTracker + + // browseLimits overrides defaultBrowseLimits when set; tests use it. + browseLimits archives.StreamOptions } // New creates a new Server with the given configuration. From 1559be63e27114b68b15513a1abb22a32dcde1a0 Mon Sep 17 00:00:00 2001 From: pinguinfuss Date: Sat, 10 Oct 2026 08:35:50 +0200 Subject: [PATCH 186/633] Route containerd ns mirror requests to configured registries (#405) * Route containerd ns mirror requests to configured registries containerd's hosts.toml mirrors append ?ns= to every request. The container handler ignored it, so a single _default mirror entry could not serve more than one registry. ns is a closed-world lookup key and is never dialed. Docker Hub aliases and the host of upstream.oci_default select the default route, hosts of upstream.oci entries select their named upstream. Unknown hosts return NAME_UNKNOWN so containerd falls back to its next host. Registry URLs with a path are not indexed, because ns names the registry at the root of a host. Host collisions log a warning and resolve deterministically. With ns the path is the verbatim upstream repository, the reserved upstream/ prefix is rejected, and repository prefix routes such as Homebrew's do not apply. Cache names match the unprefixed and upstream/{name}/ routes, so all routes share cached blobs, manifests and tag lists. ns is no longer forwarded on tag-list requests, and the pagination Link is rewritten per request instead of at store time so clients on different routes get links for their own route. Refs #303 * Document containerd ns mirroring Refs #303 * Version the tag-list cache identity for the raw-link format Tag-list rows now store the upstream Link verbatim and rewrite it per request. Rows written by earlier versions hold a Link already rewritten for one route. Under the unchanged key an older binary running next to this one (rolling update on shared Postgres, or a rollback) would serve the raw Link unrewritten, sending clients to the wrong route for the next page. A format marker in the key keeps the two apart at the cost of one cache miss per tag list after the upgrade. Adversarial review finding F1: tag-list cache rows hold raw upstream Links under unchanged keys, so an older binary serves them unrewritten. * Qualify the README claim about ns mirror coverage Registries whose URL has a path and hosts shared by two entries are not reachable through ns; point to the configuration guide for both rules. Adversarial review finding F3: README says ns covers every upstream.oci registry. * Redact registry URLs in ns index warnings Configured registry URLs may carry userinfo credentials, which the new startup warnings wrote to the log verbatim. Log them with the password masked, and say precisely what a path-prefixed default registry means: its host is not indexed, while the Docker Hub aliases still select it. Adversarial review finding F2: startup warnings print registry URLs unredacted and the default-registry message is misleading. * Keep non-default ports apart in the ns index Dropping 80 and 443 regardless of scheme turned https://host:80 and https://host into the same key, so a request for one of them could land on the other registry. Only the default port of the URL's scheme is optional now. A configured host is indexed with and without that port, because image references spell it either way; any other port has to match exactly, and the ns value is used as containerd sends it, apart from case and IPv6 brackets. Since a URL now yields two keys that can belong to different routes, the collision warning lists the owner per key. The handler test runs two registries on one host that differ only in the port. Tests can't bind 80 or 443, so a dialer maps those addresses to the fake servers. * Allow upstream/{name}/ together with ns when they agree containerd sends ns on every request to a mirror host, override_path or not. Per-registry mirrors pointing at /v2/upstream/{name} therefore arrive as upstream/{name}/...?ns=, and refusing every prefixed name under ns broke them, including the only way to mirror an upstream whose URL has a path. Such requests are now handled like the prefix route without ns, as long as ns names that upstream's own host under the same port rules as the index. Any other ns on a prefixed name is still NAME_UNKNOWN, so ns can't be used to create cache entries under another registry's name. * README: containerd config_path and how to check it A _default/hosts.toml does nothing while CRI has no hosts directory configured, so add the config.toml snippet for containerd 2.x and 1.x. To check the setup, look at containerd config dump and pull with crictl, then find the request in the proxy log; ctr --hosts-dir reads the directory on its own and would succeed even without config_path. Also mention that existing per-registry override_path entries keep working and that k3s generates the directory itself. * Let the prefix route take ns for aliases and mirrored registries With upstream.oci.hub pointing at registry-1.docker.io and a docker.io hosts.toml mirror on /v2/upstream/hub, containerd sends ns=docker.io, and the prefix check only accepted the upstream's own host. The same happens whenever the upstream is a mirror of the registry the nodes pull from, say an Artifactory remote for ghcr.io: ns carries ghcr.io, the upstream host is something else, and the pull got a 404. The prefix already decides where the content comes from and which cache entries are used, so ns can't redirect anything there. The check now only refuses an ns that contradicts the prefix, meaning a host the proxy knows that belongs to another route. The Docker Hub aliases count as one host, and a host the proxy doesn't know at all is accepted, since a client can only arrive at the prefix through its own mirror entry. * Take docker.io on every prefix route A Docker Hub mirror configured as a named upstream (mirror.gcr.io, an Artifactory remote) with a docker.io hosts.toml pointing at /v2/upstream/hub got a 404 again: docker.io is always in the index for the default route, so the check saw a known host of another route. That setup worked before this branch. Docker Hub repository names have exactly two path components, so docker.io/upstream/... can never be a real image, and accepting the aliases on every prefix route shadows nothing. While here: the collision warning claimed ns requests go to the other route, which is only true without the prefix, and the README said pulls for unknown registries always get a 404, which isn't the case for paths under upstream/{name}/. * Only take a foreign ns on a prefix route when it is configured A containerd _default mirror sends the same request for a pull of unconfigured.example/upstream/mirror/owner/app as a ghcr.io hosts.toml with override_path does for a mirror upstream. Accepting any host the proxy didn't know therefore let such a pull be answered with owner/app from the mirror upstream, a different image, instead of a 404 that lets containerd go to the registry the image actually names. The prefix route now accepts ns only for the upstream's own host, for Docker Hub, and for hosts listed in the new upstream.oci_mirrors setting. That setting is how an Artifactory remote of ghcr.io keeps working: upstream: oci_mirrors: ghcr: ["ghcr.io"] The listed hosts also select the upstream for unprefixed ns requests, which makes a mirror with a path in its URL reachable through the _default entry as well. Hosts from configured URLs still win over mirror entries; the proxy warns about such overlaps at startup. * Fix the docs sentence on which ns a prefix route takes configuration.md still said a prefix request with ns is only refused when ns names another route's host. Since the last commit it is the other way round: only the upstream's own host, its oci_mirrors entries and Docker Hub get through. * Note what a host accepted on a prefix route gives up Listing ghcr.io for an upstream (or the upstream's own host) still has the ambiguity from the wildcard case: ghcr.io/upstream/ghcr/owner/app lands on that upstream. That is the price of an explicit entry and fine in practice, but it should be written down. * Test that oci_mirrors entries stay with their own upstream Two gaps in the coverage of the last change: nothing failed if a host listed for one upstream was also accepted on another upstream's prefix, and nothing checked that the server actually hands upstream.oci_mirrors to the container handler. The handler test now has a second upstream that must refuse ghcr.io, and a small server test pulls through a mirror entry on both the prefix and the unprefixed route. * Log when a prefix route refuses an ns Setups with a ghcr.io hosts.toml pointing at a mirror upstream worked before this branch and now need an oci_mirrors entry. Without one, containerd gets a 404, quietly pulls from ghcr.io directly, and nobody notices the proxy is being skipped. The refusal is now logged as a warning that names the upstream and the ns and points at upstream.oci_mirrors. * Drop a check in validateOCIMirrors that could never fire user@ghcr.io parses with ghcr.io as its host, so the parsed.Host != host comparison already rejects it and the extra parsed.User test was dead. A test case for a host with credentials keeps that path covered. * Reject an oci_mirrors host that is listed for two upstreams With ghcr.io under both art and nexus, both prefix routes took ns=ghcr.io while unprefixed requests went to whichever name sorted first, with nothing but a startup warning. Config validation now refuses the second entry, comparing hosts without case and without the default :443. * Log each refused ns once per upstream and refuse overlong values The refusal warning fired on every request, and a node with a missing oci_mirrors entry asks for the manifest and each layer, so the same line piled up. Each upstream/ns pair is now logged once, for up to 1024 pairs; after that every refusal is logged again. ns comes straight from the client and can be as long as the request line allows, so remembering it as is would let anyone pin a lot of memory with a handful of requests. No registry host is longer than 259 characters (a 253-character name plus a port), so anything longer is refused right away, before it is logged or remembered. * Compare oci_mirrors hosts the way the handler does The duplicate check lowercased the host and trimmed :443, while the handler splits host and port, brackets IPv6 literals and drops an empty port. So ghcr.io and ghcr.io:, or [::1] and [::1]:, passed as different hosts in two upstreams and then collided in the handler. The check now builds the same key the handler uses. * Rewrap the oci_mirrors paragraphs in the configuration docs Two sentences had been dropped into the middle of wrapped lines, and the prefix-route paragraph named Docker Hub twice in a row. Same content, shorter lines, one mention. * Stop logging ns refusals once the set is full Past 1024 different upstream/ns pairs every refusal was logged again, so anyone could fill the set with made-up hosts and then flood the log at warning level. When the set is full the proxy now says so once and keeps quiet after that. A real misconfiguration shows up long before that, with the first pull. * Test the limit on remembered ns refusals Nothing covered what happens once 1024 pairs are remembered. The new test fills the set, then sends a known pair and two new ones: the proxy must log the limit notice exactly once, log none of the refusals and keep the set at its size. --- README.md | 59 ++ config.example.yaml | 6 + docs/configuration.md | 51 ++ internal/config/config.go | 60 ++ internal/config/config_test.go | 78 +++ internal/handler/container.go | 357 +++++++++++- internal/handler/container_ns_test.go | 801 ++++++++++++++++++++++++++ internal/handler/container_tags.go | 56 +- internal/server/oci_mirrors_test.go | 45 ++ internal/server/server.go | 1 + 10 files changed, 1479 insertions(+), 35 deletions(-) create mode 100644 internal/handler/container_ns_test.go create mode 100644 internal/server/oci_mirrors_test.go diff --git a/README.md b/README.md index 56dfd31..083f18b 100644 --- a/README.md +++ b/README.md @@ -447,6 +447,65 @@ Or pull images directly: docker pull localhost:8080/library/nginx:latest ``` +#### containerd (Kubernetes, k3s, nerdctl) + +containerd mirrors send the original registry host in an `ns` query +parameter, so one mirror entry can serve Docker Hub and the registries +configured in `upstream.oci` whose URL has no path. Point containerd's CRI +plugin at a hosts directory in `/etc/containerd/config.toml` and restart +containerd. For containerd 2.x: + +```toml +version = 3 + +[plugins."io.containerd.cri.v1.images".registry] + config_path = "/etc/containerd/certs.d" +``` + +For containerd 1.x: + +```toml +version = 2 + +[plugins."io.containerd.grpc.v1.cri".registry] + config_path = "/etc/containerd/certs.d" +``` + +Then create `/etc/containerd/certs.d/_default/hosts.toml`: + +```toml +[host."http://proxy.example.com:8080"] + capabilities = ["pull", "resolve"] +``` + +To check that CRI picked up the directory, confirm the setting containerd +runs with and pull through CRI (on k3s use `k3s crictl`): + +```bash +containerd config dump | grep config_path +crictl pull docker.io/library/nginx:latest +``` + +The proxy logs a `container manifest request` line for the pull and, when +`access_log.path` is set, an access-log entry. `ctr images pull --hosts-dir` +is no substitute for this check: it reads the directory itself and succeeds +even while CRI still has no `config_path`. + +`docker.io` and the host of `upstream.oci_default` use the default registry; +the host of each `upstream.oci` URL uses that named registry, and so do the +hosts listed for it in `upstream.oci_mirrors`. Pulls for any other registry get +`404 NAME_UNKNOWN`, and containerd falls back to the registry itself. Existing +per-registry `hosts.toml` files that point at `/v2/upstream/{name}` with +`override_path = true` keep working for that upstream's own host and for +Docker Hub. If the upstream is a mirror of the registry the nodes pull from, +say an Artifactory remote of `ghcr.io`, list that registry in +`upstream.oci_mirrors` (`ghcr: ["ghcr.io"]`); otherwise those pulls get the +404 as well. See [docs/configuration.md](docs/configuration.md) for how +hosts are matched and which entry wins when two share a host. k3s generates +the hosts directory itself from `/etc/rancher/k3s/registries.yaml`; `mirrors: +{"*": {endpoint: ["http://proxy.example.com:8080"]}}` produces the same +`_default` entry. + ### Helm Configure each HTTP chart repository with a name, then add the matching proxy diff --git a/config.example.yaml b/config.example.yaml index 49b81ef..a34462a 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -229,6 +229,12 @@ upstream: # oci: # ghcr: "https://ghcr.io" + # Registries a named OCI upstream mirrors, e.g. when ghcr above points at + # an Artifactory remote of ghcr.io. containerd mirror requests whose ns + # parameter names one of these hosts are served by that upstream. + # oci_mirrors: + # ghcr: ["ghcr.io"] + # Named Alpine APK repositories (used by /apk/{name}/). # Defaults to {"alpine": "https://dl-cdn.alpinelinux.org/alpine"} when empty; # configuring any entry replaces that default. diff --git a/docs/configuration.md b/docs/configuration.md index 8261eba..331db95 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -290,6 +290,57 @@ mise section in the README for the client-side `url_replacements`. while `upstream.oci` selects named registries through the `upstream/{name}/` repository prefix. For example, `oci://proxy.example.com/upstream/ghcr/owner/chart` uses the `ghcr` registry with `owner/chart` as its repository. + +containerd mirror requests carry the original registry host in an `ns` query +parameter (see the containerd section in the README). The proxy only looks the +host up and never connects to it: `docker.io`, `index.docker.io`, +`registry-1.docker.io` and the host of `upstream.oci_default` select the default +registry, and the host of each `upstream.oci` URL selects that named registry. +Hosts are compared case-insensitively. The scheme's default port (443 for +`https`, 80 for `http`) may be spelled out or left out in the image reference; +any other port must match exactly, so `https://registry.example:80` and +`https://registry.example` are two different registries. An +unknown host returns `404 NAME_UNKNOWN`, so containerd falls back to its next +host. Registry URLs with a path (for example an Artifactory repository path) +are not reachable through their own host in `ns`, only through +`upstream/{name}/` or through `upstream.oci_mirrors`. When two +entries share a host, the proxy logs a warning at startup; the default registry +wins, otherwise the alphabetically first name. Pulls through `ns`, +`upstream/{name}/` and unprefixed requests share the same cache entries. +Requests that combine the `upstream/{name}/` prefix with `ns`, as per-registry +containerd mirrors with `override_path = true` send them, are routed by the +prefix. They are accepted only when `ns` is the upstream's own host, a host +listed for it in `upstream.oci_mirrors`, or Docker Hub (`docker.io` and its +aliases). Docker Hub repository names have two path components and can never +start with `upstream/`, so a Docker Hub mirror behind any prefix stays +reachable. Any other host gets `404 NAME_UNKNOWN`: a containerd `_default` +mirror sends the same request for an image such as +`unconfigured.example/upstream/ghcr/owner/app`, and that pull must not be +answered from the `ghcr` upstream. + +When a named upstream mirrors another registry, for example an Artifactory +remote of `ghcr.io`, say so in `upstream.oci_mirrors`: + +```yaml +upstream: + oci: + ghcr: "https://artifactory.example.com/artifactory/api/docker/ghcr-remote" + oci_mirrors: + ghcr: ["ghcr.io"] +``` + +Each entry lists bare registry hosts as containerd sends them, optionally with +a port. A host can be listed for one upstream only. Those hosts select the +upstream for unprefixed `ns` requests and are accepted as `ns` on its +`upstream/{name}/` prefix. A host that is also the host of a configured +registry URL stays with that registry for unprefixed requests; the proxy logs a +warning at startup. + +A host that is accepted on a prefix route gives up image paths of the form +`/upstream/{name}/...` under a `_default` mirror: with `ghcr: ["ghcr.io"]`, +a pull of `ghcr.io/upstream/ghcr/owner/app` is answered with `owner/app` from +the `ghcr` upstream, because containerd sends the same request for it as a +per-registry mirror does. The same applies to the upstream's own host. When the proxy uses plain HTTP (for example `localhost:8080`), pass `--plain-http` to Helm OCI commands. diff --git a/internal/config/config.go b/internal/config/config.go index 54f826f..2df21d8 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -63,6 +63,7 @@ import ( "encoding/base64" "encoding/json" "fmt" + "net" "net/url" "os" "path/filepath" @@ -634,6 +635,12 @@ type UpstreamConfig struct { // oci://proxy.example.com/upstream/ghcr/owner/chart. OCI map[string]string `json:"oci" yaml:"oci"` + // OCIMirrors lists, per upstream.oci name, the registry hosts that + // upstream mirrors, for example {"ghcr": ["ghcr.io"]} for an Artifactory + // remote of ghcr.io. containerd requests whose ns query parameter names + // one of these hosts are served by that upstream. + OCIMirrors map[string][]string `json:"oci_mirrors" yaml:"oci_mirrors"` + // Generic maps names to plain HTTP upstream base URLs, served at // /generic/{name}/. The remaining request path and query string are // appended to the upstream URL. GitHub release asset paths @@ -691,6 +698,9 @@ func (u *UpstreamConfig) Validate() error { if err := validateNamedUpstreams("upstream.oci", u.OCI); err != nil { return err } + if err := validateOCIMirrors(u.OCIMirrors, u.OCI); err != nil { + return err + } if err := validateNamedUpstreams("upstream.generic", u.Generic); err != nil { return err } @@ -712,6 +722,56 @@ func (u *UpstreamConfig) Validate() error { // paths, which a repository of the same name would shadow. var debianReservedRepositoryNames = []string{"pool", "dists"} +// validateOCIMirrors checks that every upstream.oci_mirrors entry belongs to +// an upstream.oci name and lists bare registry hosts, the form containerd +// sends in the ns query parameter. A host may belong to one upstream only; +// otherwise both prefixes would take it while unprefixed requests silently +// went to one of them. +func validateOCIMirrors(mirrors map[string][]string, upstreams map[string]string) error { + names := make([]string, 0, len(mirrors)) + for name := range mirrors { + names = append(names, name) + } + sort.Strings(names) + owners := make(map[string]string) + for _, name := range names { + if _, ok := upstreams[name]; !ok { + return fmt.Errorf("invalid upstream.oci_mirrors name %q: no upstream.oci entry of that name", name) + } + for _, host := range mirrors[name] { + parsed, err := url.Parse("https://" + host) + if host == "" || err != nil || parsed.Host != host { + return fmt.Errorf("invalid upstream.oci_mirrors.%s host %q: must be a registry host such as ghcr.io or registry.example:5000", name, host) + } + key := ociMirrorHostKey(host) + if owner, taken := owners[key]; taken && owner != name { + return fmt.Errorf("invalid upstream.oci_mirrors.%s host %q: already listed for %s", name, host, owner) + } + owners[key] = name + } + } + return nil +} + +// ociMirrorHostKey normalizes an upstream.oci_mirrors host the way the +// container handler builds its ns lookup keys: lowercase, IPv6 literals in +// brackets, an empty or default https port dropped. Two entries with the same +// key would select the same route. +func ociMirrorHostKey(host string) string { + name, port, err := net.SplitHostPort(host) + if err != nil { + name, port = strings.TrimSuffix(strings.TrimPrefix(host, "["), "]"), "" + } + name = strings.ToLower(name) + if strings.Contains(name, ":") { + name = "[" + name + "]" + } + if port == "" || port == "443" { + return name + } + return name + ":" + port +} + func validateNamedUpstreams(field string, upstreams map[string]string) error { for name, upstreamURL := range upstreams { if name == "" || name == "." || name == ".." || strings.ContainsAny(name, `/\\`) { diff --git a/internal/config/config_test.go b/internal/config/config_test.go index b1818ce..c1cc2ff 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -1341,6 +1341,84 @@ func TestValidateNamedUpstreams(t *testing.T) { }, wantErr: true, }, + { + name: "OCI mirrors for a configured upstream", + modify: func(cfg *Config) { + cfg.Upstream.OCI = map[string]string{"ghcr": "https://artifactory.example/api/docker/ghcr-remote"} + cfg.Upstream.OCIMirrors = map[string][]string{"ghcr": {"ghcr.io", "registry.example:5000"}} + }, + }, + { + name: "OCI mirrors for an unknown upstream", + modify: func(cfg *Config) { + cfg.Upstream.OCI = map[string]string{"ghcr": "https://ghcr.io"} + cfg.Upstream.OCIMirrors = map[string][]string{"quay": {"quay.io"}} + }, + wantErr: true, + }, + { + name: "OCI mirror given as URL", + modify: func(cfg *Config) { + cfg.Upstream.OCI = map[string]string{"ghcr": "https://mirror.example"} + cfg.Upstream.OCIMirrors = map[string][]string{"ghcr": {"https://ghcr.io"}} + }, + wantErr: true, + }, + { + name: "OCI mirror with a path", + modify: func(cfg *Config) { + cfg.Upstream.OCI = map[string]string{"ghcr": "https://mirror.example"} + cfg.Upstream.OCIMirrors = map[string][]string{"ghcr": {"ghcr.io/owner"}} + }, + wantErr: true, + }, + { + name: "OCI mirror host with credentials", + modify: func(cfg *Config) { + cfg.Upstream.OCI = map[string]string{"ghcr": "https://mirror.example"} + cfg.Upstream.OCIMirrors = map[string][]string{"ghcr": {"user@ghcr.io"}} + }, + wantErr: true, + }, + { + name: "OCI mirror host listed for two upstreams", + modify: func(cfg *Config) { + cfg.Upstream.OCI = map[string]string{"art": "https://art.example", "nexus": "https://nexus.example"} + cfg.Upstream.OCIMirrors = map[string][]string{"art": {"ghcr.io"}, "nexus": {"GHCR.io:443"}} + }, + wantErr: true, + }, + { + name: "OCI mirror host listed for two upstreams with an empty port", + modify: func(cfg *Config) { + cfg.Upstream.OCI = map[string]string{"art": "https://art.example", "nexus": "https://nexus.example"} + cfg.Upstream.OCIMirrors = map[string][]string{"art": {"ghcr.io"}, "nexus": {"ghcr.io:"}} + }, + wantErr: true, + }, + { + name: "OCI mirror IPv6 host listed for two upstreams", + modify: func(cfg *Config) { + cfg.Upstream.OCI = map[string]string{"art": "https://art.example", "nexus": "https://nexus.example"} + cfg.Upstream.OCIMirrors = map[string][]string{"art": {"[::1]"}, "nexus": {"[::1]:"}} + }, + wantErr: true, + }, + { + name: "OCI mirror hosts that differ only in a non-default port", + modify: func(cfg *Config) { + cfg.Upstream.OCI = map[string]string{"art": "https://art.example", "nexus": "https://nexus.example"} + cfg.Upstream.OCIMirrors = map[string][]string{"art": {"registry.example"}, "nexus": {"registry.example:5000"}} + }, + }, + { + name: "OCI mirror host is empty", + modify: func(cfg *Config) { + cfg.Upstream.OCI = map[string]string{"ghcr": "https://mirror.example"} + cfg.Upstream.OCIMirrors = map[string][]string{"ghcr": {""}} + }, + wantErr: true, + }, { name: "OCI upstream URL is not absolute", modify: func(cfg *Config) { diff --git a/internal/handler/container.go b/internal/handler/container.go index f73f0f5..63afd68 100644 --- a/internal/handler/container.go +++ b/internal/handler/container.go @@ -4,9 +4,14 @@ import ( "encoding/json" "errors" "fmt" + "maps" + "net" "net/http" + "net/url" "regexp" + "slices" "strings" + "sync" ) const ( @@ -15,8 +20,43 @@ const ( manifestMatchCount = 3 // full match + name + reference tagsListMatchCount = 2 // full match + name registrySelectorParts = 3 // upstream + name + repository + + // namespaceQueryParam is the query parameter containerd appends to mirror + // requests to name the registry the image reference points at. + namespaceQueryParam = "ns" + // defaultNamespaceRoute marks namespace hosts served by the default registry. + defaultNamespaceRoute = "" + // maxReportedNamespaceRefusals bounds the upstream/ns pairs remembered + // for logging refusals once; past it further refusals are not logged. + maxReportedNamespaceRefusals = 1024 + // maxNamespaceLength is the longest ns a registry host can produce: a + // 253-character DNS name plus ":65535". Longer values are refused before + // they are logged or remembered. + maxNamespaceLength = 259 ) +// dockerHubNamespaces are the registry URLs clients use for Docker Hub. +var dockerHubNamespaces = []string{"https://docker.io", "https://index.docker.io", "https://registry-1.docker.io"} //nolint:gochecknoglobals // fixed alias list + +// schemeDefaultPorts are the ports an image reference may leave out. +var schemeDefaultPorts = map[string]string{"https": "443", "http": "80"} //nolint:gochecknoglobals // fixed table + +// dockerHubNamespaceKeys are the ns lookup keys of the Docker Hub aliases. +var dockerHubNamespaceKeys = dockerHubKeys() //nolint:gochecknoglobals // fixed table + +func dockerHubKeys() []string { + var keys []string + for _, registryURL := range dockerHubNamespaces { + hostKeys, _ := namespaceKeysForURL(registryURL) + keys = append(keys, hostKeys...) + } + return keys +} + +func isDockerHubKey(key string) bool { + return slices.Contains(dockerHubNamespaceKeys, key) +} + // ContainerHandler handles OCI/Docker container registry protocol requests. // It implements the OCI Distribution Spec for pulling images. // Reference: https://github.com/opencontainers/distribution-spec/blob/main/spec.md @@ -26,6 +66,19 @@ type ContainerHandler struct { proxyURL string namedRegistries map[string]string registries []containerRegistry + // namespaces maps normalized registry hosts from containerd's ns query + // parameter to a named upstream, or to defaultNamespaceRoute. + namespaces map[string]string + // mirrorKeys holds, per named upstream, the ns lookup keys of the + // registries it is configured to mirror. + mirrorKeys map[string][]string + + refusalsMu sync.Mutex + // refusals remembers upstream/ns pairs whose refusal was already logged. + refusals map[string]struct{} + // refusalsFull is set once the refusals set reached its limit and that + // was logged. + refusalsFull bool } type containerRegistry struct { @@ -38,18 +91,7 @@ type containerRegistry struct { // upstream/{name}/, leaving unprefixed requests compatible with the Docker Hub // mirror behavior. func NewContainerHandler(proxy *Proxy, proxyURL string, namedRegistries ...map[string]string) *ContainerHandler { - h := &ContainerHandler{ - proxy: proxy, - registryURL: dockerHubRegistry, - proxyURL: strings.TrimSuffix(proxyURL, "/"), - } - if len(namedRegistries) > 0 { - h.namedRegistries = make(map[string]string, len(namedRegistries[0])) - for name, registryURL := range namedRegistries[0] { - h.namedRegistries[name] = strings.TrimSuffix(registryURL, "/") - } - } - return h + return newContainerHandler(proxy, proxyURL, dockerHubRegistry, namedRegistries...) } // NewContainerHandlerWithRegistry creates a container handler with a custom @@ -59,11 +101,182 @@ func NewContainerHandlerWithRegistry( proxyURL, registryURL string, namedRegistries ...map[string]string, ) *ContainerHandler { - h := NewContainerHandler(proxy, proxyURL, namedRegistries...) - h.registryURL = configuredUpstreamURL(registryURL, dockerHubRegistry) + return newContainerHandler(proxy, proxyURL, configuredUpstreamURL(registryURL, dockerHubRegistry), namedRegistries...) +} + +func newContainerHandler( + proxy *Proxy, + proxyURL, registryURL string, + namedRegistries ...map[string]string, +) *ContainerHandler { + h := &ContainerHandler{ + proxy: proxy, + registryURL: registryURL, + proxyURL: strings.TrimSuffix(proxyURL, "/"), + } + if len(namedRegistries) > 0 { + h.namedRegistries = make(map[string]string, len(namedRegistries[0])) + for name, registryURL := range namedRegistries[0] { + h.namedRegistries[name] = strings.TrimSuffix(registryURL, "/") + } + } + // The index needs the final default registry URL, so build it last. + h.buildNamespaceIndex() return h } +// buildNamespaceIndex maps the registry hosts containerd may send in the ns +// query parameter to the configured routes. The index is closed-world: a host +// is only ever looked up, never dialed. Docker Hub aliases and the default +// registry's host select the default route; hosts of upstream.oci entries +// select their named upstream. Registry URLs with a path are skipped because +// ns names the registry at the root of that host, not a repository mounted +// below it. On collisions the default route wins, then the alphabetically +// first upstream name. +func (h *ContainerHandler) buildNamespaceIndex() { + h.namespaces = make(map[string]string) + for _, registryURL := range dockerHubNamespaces { + h.indexNamespace(defaultNamespaceRoute, registryURL) + } + if !h.indexNamespace(defaultNamespaceRoute, h.registryURL) { + h.warn("host of the default OCI registry is not indexed for ns lookups: URL has a path; Docker Hub aliases still select it", + "url", redactedURL(h.registryURL)) + } + for _, name := range slices.Sorted(maps.Keys(h.namedRegistries)) { + if !h.indexNamespace(name, h.namedRegistries[name]) { + h.warn("OCI upstream is not reachable through the ns query parameter: URL has a path", + "upstream", name, "url", redactedURL(h.namedRegistries[name])) + } + } +} + +// SetMirroredRegistries records which registry hosts each named upstream +// mirrors, for example "ghcr" -> ["ghcr.io"] for an Artifactory remote of +// ghcr.io. Those hosts select the upstream in ns lookups and are accepted as +// ns on its upstream/{name}/ prefix. They rank below every host taken from a +// configured URL, so they are indexed after the constructor's index. +func (h *ContainerHandler) SetMirroredRegistries(mirrors map[string][]string) { + h.mirrorKeys = make(map[string][]string, len(mirrors)) + for _, name := range slices.Sorted(maps.Keys(mirrors)) { + if _, ok := h.namedRegistries[name]; !ok { + h.warn("OCI mirror entry has no upstream of that name", "upstream", name) + continue + } + for _, host := range mirrors[name] { + registryURL := "https://" + host + keys, ok := namespaceKeysForURL(registryURL) + if !ok { + h.warn("OCI mirror host is not a registry host", "upstream", name, "host", host) + continue + } + h.mirrorKeys[name] = append(h.mirrorKeys[name], keys...) + h.indexNamespace(name, registryURL) + } + } +} + +// indexNamespace maps the ns lookup keys of registryURL to route. Keys that +// another route already owns stay with that route and are reported once. It +// reports false when the URL is not a bare registry root. +func (h *ContainerHandler) indexNamespace(route, registryURL string) bool { + keys, ok := namespaceKeysForURL(registryURL) + if !ok { + return false + } + var taken []string + for _, key := range keys { + owner, exists := h.namespaces[key] + switch { + case !exists: + h.namespaces[key] = route + case owner != route: + if owner == defaultNamespaceRoute { + owner = "default registry" + } + taken = append(taken, key+"="+owner) + } + } + if len(taken) > 0 { + h.warn("OCI upstream shares a registry host with another route; unprefixed ns requests for it go to the other route", + "upstream", route, "hosts", strings.Join(taken, ", ")) + } + return true +} + +// namespaceKeysForURL returns the ns lookup keys of a registry URL. The +// scheme-default port is optional in image references, so such a URL is +// indexed both without and with the port. Any other port is kept as is, which +// keeps https://host:80 and https://host apart. It reports false for URLs +// that are not a bare registry root; namespaceKeysForHost serves those. +func namespaceKeysForURL(registryURL string) ([]string, bool) { + parsed, err := url.Parse(registryURL) + if err != nil || parsed.Host == "" || (parsed.Path != "" && parsed.Path != "/") { + return nil, false + } + return namespaceKeysForHost(parsed), true +} + +// namespaceKeysForHost returns the lookup keys of a URL's host, ignoring its +// path. +func namespaceKeysForHost(parsed *url.URL) []string { + host, port := splitNamespaceHost(parsed.Host) + defaultPort := schemeDefaultPorts[strings.ToLower(parsed.Scheme)] + switch { + case port != "" && port != defaultPort: + return []string{namespaceKey(host, port)} + case defaultPort == "": + return []string{namespaceKey(host, "")} + default: + return []string{namespaceKey(host, ""), namespaceKey(host, defaultPort)} + } +} + +// namespaceKeyForRequest normalizes the ns value of a request. containerd +// sends the registry host of the image reference, with or without a port, so +// the value is matched as sent apart from case and IPv6 bracket form. +func namespaceKeyForRequest(namespace string) string { + host, port := splitNamespaceHost(namespace) + return namespaceKey(host, port) +} + +// splitNamespaceHost splits host[:port], accepting bracketed and bare IPv6 +// hosts without a port. +func splitNamespaceHost(hostport string) (host, port string) { + host, port, err := net.SplitHostPort(hostport) + if err != nil { + return strings.TrimSuffix(strings.TrimPrefix(hostport, "["), "]"), "" + } + return host, port +} + +// namespaceKey builds a lookup key from a host and an optional port: the host +// lowercased, IPv6 literals in brackets. +func namespaceKey(host, port string) string { + host = strings.ToLower(host) + if strings.Contains(host, ":") { + host = "[" + host + "]" + } + if port == "" { + return host + } + return host + ":" + port +} + +// redactedURL returns a registry URL for logging with any password masked. +func redactedURL(raw string) string { + parsed, err := url.Parse(raw) + if err != nil { + return "" + } + return parsed.Redacted() +} + +func (h *ContainerHandler) warn(msg string, args ...any) { + if h.proxy != nil && h.proxy.Logger != nil { + h.proxy.Logger.Warn(msg, args...) + } +} + // RegisterRegistry routes a repository and its descendants to a specific OCI // registry. The longest matching repository prefix wins. func (h *ContainerHandler) RegisterRegistry(repositoryPrefix, registryURL string) { @@ -144,7 +357,7 @@ func (h *ContainerHandler) handleBlobDownload(w http.ResponseWriter, r *http.Req return } - registryURL, upstreamName, cacheName, ok := h.registryForName(name) + registryURL, upstreamName, cacheName, ok := h.registryForRequest(r, name) if !ok { h.containerError(w, http.StatusNotFound, "NAME_UNKNOWN", "unknown upstream registry") return @@ -225,7 +438,7 @@ func (h *ContainerHandler) handleManifest(w http.ResponseWriter, r *http.Request return } - registryURL, upstreamName, _, ok := h.registryForName(name) + registryURL, upstreamName, _, ok := h.registryForRequest(r, name) if !ok { h.containerError(w, http.StatusNotFound, "NAME_UNKNOWN", "unknown upstream registry") return @@ -248,7 +461,7 @@ func (h *ContainerHandler) handleTagsList(w http.ResponseWriter, r *http.Request return } - registryURL, upstreamName, _, ok := h.registryForName(name) + registryURL, upstreamName, _, ok := h.registryForRequest(r, name) if !ok { h.containerError(w, http.StatusNotFound, "NAME_UNKNOWN", "unknown upstream registry") return @@ -286,6 +499,116 @@ func (h *ContainerHandler) proxyBlobHead(w http.ResponseWriter, r *http.Request, }) } +// registryForRequest resolves the repository name of a request. containerd +// mirror requests name the target registry in the ns query parameter; without +// it the name is routed by registryForName. +func (h *ContainerHandler) registryForRequest(r *http.Request, name string) (registryURL, upstreamName, cacheName string, ok bool) { + namespaces := r.URL.Query()[namespaceQueryParam] + switch { + case len(namespaces) == 0 || (len(namespaces) == 1 && namespaces[0] == ""): + return h.registryForName(name) + case len(namespaces) > 1: + return "", "", "", false + } + return h.registryForNamespace(namespaces[0], name) +} + +// registryForNamespace resolves a repository name verbatim against the registry +// named by ns. Cache names match the unprefixed and upstream/{name}/ routes, +// so all routes to one registry share blobs. +// +// Per-registry containerd mirrors with override_path address the reserved +// upstream/{name}/ prefix and still send ns. Such requests are served like +// the prefix route without ns unless ns contradicts the prefix, see +// namespaceNamesPrefixUpstream. +func (h *ContainerHandler) registryForNamespace(namespace, name string) (registryURL, upstreamName, cacheName string, ok bool) { + if len(namespace) > maxNamespaceLength { + return "", "", "", false + } + if strings.HasPrefix(name, "upstream/") { + if !h.namespaceNamesPrefixUpstream(namespace, name) { + return "", "", "", false + } + return h.registryForName(name) + } + route, ok := h.namespaces[namespaceKeyForRequest(namespace)] + if !ok { + return "", "", "", false + } + if route == defaultNamespaceRoute { + // ns explicitly names the default registry, so repository prefix + // routes such as Homebrew's do not apply. + if h.registryURL == "" { + return "", "", "", false + } + return h.registryURL, name, name, true + } + registryURL = h.namedRegistries[route] + if registryURL == "" { + return "", "", "", false + } + return registryURL, name, "upstream/" + route + "/" + name, true +} + +// namespaceNamesPrefixUpstream decides whether an upstream/{name}/ request +// may carry the given ns. A per-registry mirror with override_path and a +// _default wildcard pull of /upstream/{name}/... send the same request, +// so ns has to name a registry the upstream is known to serve: its own host +// (the URL's path is irrelevant), a host listed for it in upstream.oci_mirrors, +// or Docker Hub. Docker Hub repository names have two path components, so +// docker.io/upstream/... is never a real image and a Docker Hub mirror behind +// any prefix stays reachable. Any other ns gets a 404 and containerd falls +// back to the registry the image actually names. +func (h *ContainerHandler) namespaceNamesPrefixUpstream(namespace, name string) bool { + rest, _ := strings.CutPrefix(name, "upstream/") + upstream, _, _ := strings.Cut(rest, "/") + parsed, err := url.Parse(h.namedRegistries[upstream]) + if err != nil || parsed.Host == "" { + return false + } + key := namespaceKeyForRequest(namespace) + if isDockerHubKey(key) || + slices.Contains(namespaceKeysForHost(parsed), key) || + slices.Contains(h.mirrorKeys[upstream], key) { + return true + } + h.reportNamespaceRefusal(upstream, key, namespace) + return false +} + +// reportNamespaceRefusal logs a refused ns on a prefix route. containerd +// falls back to the registry itself on that 404 without reporting anything, +// so this line is all an operator gets to see. Each upstream/ns pair is +// logged once, since a misconfigured node repeats the same request for every +// layer. ns comes from the client, so the set is bounded: once it is full, +// that is logged one time and later refusals stay quiet. Overlong values +// never get here (see maxNamespaceLength). +func (h *ContainerHandler) reportNamespaceRefusal(upstream, key, namespace string) { + pair := upstream + "\x00" + key + h.refusalsMu.Lock() + if _, seen := h.refusals[pair]; seen { + h.refusalsMu.Unlock() + return + } + if len(h.refusals) >= maxReportedNamespaceRefusals { + announce := !h.refusalsFull + h.refusalsFull = true + h.refusalsMu.Unlock() + if announce { + h.warn("too many different ns values refused on OCI upstream prefixes; further refusals are not logged", + "limit", maxReportedNamespaceRefusals) + } + return + } + if h.refusals == nil { + h.refusals = make(map[string]struct{}) + } + h.refusals[pair] = struct{}{} + h.refusalsMu.Unlock() + h.warn("refusing ns on OCI upstream prefix; if the upstream mirrors this registry, list it in upstream.oci_mirrors", + "upstream", upstream, "ns", namespace) +} + // registryForName resolves a client-visible OCI repository name to an upstream // registry and its repository name. Named upstreams use upstream/{name}/ as a // reserved prefix. Other names are matched against registered repository diff --git a/internal/handler/container_ns_test.go b/internal/handler/container_ns_test.go new file mode 100644 index 0000000..e4a1e12 --- /dev/null +++ b/internal/handler/container_ns_test.go @@ -0,0 +1,801 @@ +package handler + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "log/slog" + "net" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "sync" + "testing" + "time" + + "github.com/git-pkgs/registries/fetch" +) + +const ( + nsTestBlob = "layer bytes" + nsTestProxyURL = "http://proxy.example.test" +) + +// nsTestRegistry is a fake OCI registry serving one repository below an +// optional path prefix. It records every request URI it receives. +type nsTestRegistry struct { + *httptest.Server + repository string + + mu sync.Mutex + requests []string +} + +func newNSTestRegistry(t *testing.T, repository, pathPrefix string) *nsTestRegistry { + t.Helper() + registry := &nsTestRegistry{repository: repository} + manifest := nsTestManifest() + blobDigest := "sha256:" + sha256Hex(nsTestBlob) + manifestDigest := "sha256:" + sha256Hex(manifest) + base := pathPrefix + "/v2/" + repository + registry.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + registry.mu.Lock() + registry.requests = append(registry.requests, r.URL.RequestURI()) + registry.mu.Unlock() + + switch r.URL.Path { + case base + "/manifests/latest", base + "/manifests/" + manifestDigest: + w.Header().Set("Content-Type", "application/vnd.oci.image.manifest.v1+json") + w.Header().Set("Docker-Content-Digest", manifestDigest) + _, _ = io.WriteString(w, manifest) + case base + "/blobs/" + blobDigest: + w.Header().Set("Content-Type", "application/octet-stream") + _, _ = io.WriteString(w, nsTestBlob) + case base + "/tags/list": + w.Header().Set("Content-Type", "application/json") + if r.URL.Query().Get("last") == "" { + w.Header().Set("Link", `<`+base+`/tags/list?last=1.0&n=1>; rel="next"`) + _, _ = io.WriteString(w, `{"name":"`+repository+`","tags":["1.0"]}`) + return + } + _, _ = io.WriteString(w, `{"name":"`+repository+`","tags":["2.0"]}`) + default: + http.NotFound(w, r) + } + })) + t.Cleanup(registry.Close) + return registry +} + +func nsTestManifest() string { + return `{"schemaVersion":2,"layers":[{"digest":"sha256:` + sha256Hex(nsTestBlob) + `"}]}` +} + +func (r *nsTestRegistry) host() string { + return strings.TrimPrefix(r.URL, "http://") +} + +func (r *nsTestRegistry) requestCount() int { + r.mu.Lock() + defer r.mu.Unlock() + return len(r.requests) +} + +func (r *nsTestRegistry) lastRequest() string { + r.mu.Lock() + defer r.mu.Unlock() + if len(r.requests) == 0 { + return "" + } + return r.requests[len(r.requests)-1] +} + +// newNSTestHandler builds a container handler the way the server does, with a +// real fetcher so blob downloads reach the fake registries. Warnings logged +// while building the handler are written to the returned buffer. +func newNSTestHandler(t *testing.T, defaultURL string, named map[string]string) (http.Handler, *ContainerHandler, *bytes.Buffer) { + t.Helper() + proxy, _, _, _ := setupTestProxy(t) + logs := &bytes.Buffer{} + proxy.Logger = slog.New(slog.NewTextHandler(logs, nil)) + client := &http.Client{} + proxy.HTTPClient = client + proxy.MetadataTTL = time.Hour + fetcher := fetch.NewFetcher(fetch.WithHTTPClient(client), fetch.WithMaxRetries(0)) + proxy.Fetcher = fetcher + t.Cleanup(func() { _ = fetcher.Close() }) + h := NewContainerHandlerWithRegistry(proxy, nsTestProxyURL, defaultURL, named) + return http.StripPrefix("/v2", h.Routes()), h, logs +} + +func serveNS(routes http.Handler, target string) *httptest.ResponseRecorder { + response := httptest.NewRecorder() + routes.ServeHTTP(response, httptest.NewRequest(http.MethodGet, target, nil)) + return response +} + +func assertNameUnknown(t *testing.T, response *httptest.ResponseRecorder) { + t.Helper() + if response.Code != http.StatusNotFound { + t.Fatalf("status = %d, want 404: %s", response.Code, response.Body.String()) + } + if !strings.Contains(response.Body.String(), `"NAME_UNKNOWN"`) { + t.Errorf("body = %s, want NAME_UNKNOWN error", response.Body.String()) + } +} + +func TestContainerHandler_NamespaceSelectsDefaultRegistry(t *testing.T) { + registry := newNSTestRegistry(t, "library/nginx", "") + // The default registry is a custom oci_default, so its host is only + // resolvable when the index is built from the final registry URL. + routes, _, _ := newNSTestHandler(t, registry.URL, nil) + + for _, namespace := range []string{"docker.io", "index.docker.io", "registry-1.docker.io", registry.host()} { + t.Run(namespace, func(t *testing.T) { + response := serveNS(routes, "/v2/library/nginx/manifests/latest?ns="+namespace) + if response.Code != http.StatusOK { + t.Fatalf("status = %d, want 200: %s", response.Code, response.Body.String()) + } + if got, want := registry.lastRequest(), "/v2/library/nginx/manifests/latest"; got != want { + t.Errorf("upstream request = %q, want %q", got, want) + } + }) + } +} + +func TestContainerHandler_NamespaceSelectsNamedRegistry(t *testing.T) { + fallback := newNSTestRegistry(t, "owner/app", "") + named := newNSTestRegistry(t, "owner/app", "") + routes, _, _ := newNSTestHandler(t, fallback.URL, map[string]string{"ghcr": named.URL}) + digest := "sha256:" + sha256Hex(nsTestBlob) + + manifest := serveNS(routes, "/v2/owner/app/manifests/latest?ns="+named.host()) + if manifest.Code != http.StatusOK { + t.Fatalf("manifest status = %d, want 200: %s", manifest.Code, manifest.Body.String()) + } + blob := serveNS(routes, "/v2/owner/app/blobs/"+digest+"?ns="+named.host()) + if blob.Code != http.StatusOK { + t.Fatalf("blob status = %d, want 200: %s", blob.Code, blob.Body.String()) + } + if got := blob.Body.String(); got != nsTestBlob { + t.Errorf("blob body = %q, want %q", got, nsTestBlob) + } + if got := named.requestCount(); got != 2 { + t.Errorf("named registry requests = %d, want 2", got) + } + if got := fallback.requestCount(); got != 0 { + t.Errorf("default registry requests = %d, want 0", got) + } +} + +func TestContainerHandler_NamespaceRejectsUnresolvableRequests(t *testing.T) { + registry := newNSTestRegistry(t, "owner/app", "") + other := newNSTestRegistry(t, "owner/app", "") + routes, _, _ := newNSTestHandler(t, registry.URL, map[string]string{"ghcr": registry.URL, "quay": other.URL}) + digest := "sha256:" + sha256Hex(nsTestBlob) + + tests := map[string]string{ + "unknown host manifest": "/v2/owner/app/manifests/latest?ns=quay.io", + "unknown host blob": "/v2/owner/app/blobs/" + digest + "?ns=quay.io", + "unknown host tags": "/v2/owner/app/tags/list?ns=quay.io", + "multiple ns values": "/v2/owner/app/manifests/latest?ns=docker.io&ns=" + registry.host(), + "prefix route with default host": "/v2/upstream/quay/owner/app/manifests/latest?ns=" + registry.host(), + "prefix route with other upstream": "/v2/upstream/ghcr/owner/app/blobs/" + digest + "?ns=" + other.host(), + "prefix route with unknown upstream": "/v2/upstream/nope/owner/app/manifests/latest?ns=" + registry.host(), + "prefix route without repository": "/v2/upstream/ghcr/manifests/latest?ns=" + registry.host(), + } + for name, target := range tests { + t.Run(name, func(t *testing.T) { + assertNameUnknown(t, serveNS(routes, target)) + }) + } + if got := registry.requestCount() + other.requestCount(); got != 0 { + t.Errorf("upstream requests = %d, want 0", got) + } +} + +func TestContainerHandler_NamespaceAcceptsPrefixRouteForOwnHost(t *testing.T) { + // Per-registry containerd mirrors with override_path address the + // upstream/{name}/ prefix and still send ns. They must keep working and + // share the prefix route's cache entries. + registry := newNSTestRegistry(t, "owner/app", "") + routes, _, _ := newNSTestHandler(t, "", map[string]string{"ghcr": registry.URL}) + digest := "sha256:" + sha256Hex(nsTestBlob) + + for _, target := range []string{ + "/v2/upstream/ghcr/owner/app/manifests/latest?ns=" + registry.host(), + "/v2/upstream/ghcr/owner/app/blobs/" + digest + "?ns=" + registry.host(), + } { + if response := serveNS(routes, target); response.Code != http.StatusOK { + t.Fatalf("%s status = %d: %s", target, response.Code, response.Body.String()) + } + } + if got, want := registry.lastRequest(), "/v2/owner/app/blobs/"+digest; got != want { + t.Errorf("upstream request = %q, want %q", got, want) + } + warmed := registry.requestCount() + + for _, target := range []string{ + "/v2/upstream/ghcr/owner/app/manifests/latest", + "/v2/upstream/ghcr/owner/app/blobs/" + digest, + } { + if response := serveNS(routes, target); response.Code != http.StatusOK { + t.Fatalf("%s status = %d: %s", target, response.Code, response.Body.String()) + } + } + if got := registry.requestCount(); got != warmed { + t.Errorf("upstream requests after prefix pulls without ns = %d, want %d (cache hits)", got, warmed) + } + + tags := serveNS(routes, "/v2/upstream/ghcr/owner/app/tags/list?n=1&ns="+registry.host()) + if tags.Code != http.StatusOK { + t.Fatalf("tags status = %d: %s", tags.Code, tags.Body.String()) + } + wantLink := `<` + nsTestProxyURL + `/v2/upstream/ghcr/owner/app/tags/list?last=1.0&n=1&ns=` + url.QueryEscape(registry.host()) + `>; rel="next"` + if got := tags.Header().Get("Link"); got != wantLink { + t.Errorf("Link = %q, want %q", got, wantLink) + } + if got, want := registry.lastRequest(), "/v2/owner/app/tags/list?n=1"; got != want { + t.Errorf("upstream tags request = %q, want %q (ns must not be forwarded)", got, want) + } +} + +func TestContainerHandler_NamespacePrefixRouteAcceptsDockerHubAliases(t *testing.T) { + // upstream.oci.hub points at Docker Hub and a docker.io hosts.toml mirror + // with override_path addresses /v2/upstream/hub, so containerd sends + // ns=docker.io. The dialer stands in for registry-1.docker.io. + hub := newNSTestRegistry(t, "library/nginx", "") + other := newNSTestRegistry(t, "library/nginx", "") + routes, h, _ := newNSTestHandler(t, "", map[string]string{ + "hub": "http://registry-1.docker.io", + "quay": other.URL, + }) + transport := &http.Transport{DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) { + if addr != "registry-1.docker.io:80" { + return nil, fmt.Errorf("unexpected dial to %s", addr) + } + return (&net.Dialer{}).DialContext(ctx, network, hub.Listener.Addr().String()) + }} + t.Cleanup(transport.CloseIdleConnections) + h.proxy.HTTPClient = &http.Client{Transport: transport} + + for _, query := range []string{"?ns=docker.io", "?ns=index.docker.io", "?ns=registry-1.docker.io", "?ns=docker.io:443", ""} { + response := serveNS(routes, "/v2/upstream/hub/library/nginx/manifests/latest"+query) + if response.Code != http.StatusOK { + t.Fatalf("%q status = %d: %s", query, response.Code, response.Body.String()) + } + } + if got := hub.requestCount(); got != 1 { + t.Errorf("hub requests = %d, want 1 (the rest are cache hits)", got) + } + + // A known host of another route still contradicts the prefix. + assertNameUnknown(t, serveNS(routes, "/v2/upstream/hub/library/nginx/manifests/latest?ns="+other.host())) + if got := other.requestCount(); got != 0 { + t.Errorf("other registry requests = %d, want 0", got) + } + + t.Run("Docker Hub mirror as named upstream", func(t *testing.T) { + // The upstream is a Docker Hub mirror on some other host (mirror.gcr.io, + // an Artifactory remote); nodes still pull docker.io/... through + // /v2/upstream/hub, so ns says docker.io. + mirror := newNSTestRegistry(t, "library/nginx", "") + routes, _, _ := newNSTestHandler(t, "", map[string]string{"hub": mirror.URL}) + + response := serveNS(routes, "/v2/upstream/hub/library/nginx/manifests/latest?ns=docker.io") + if response.Code != http.StatusOK { + t.Fatalf("status = %d: %s", response.Code, response.Body.String()) + } + if got, want := mirror.lastRequest(), "/v2/library/nginx/manifests/latest"; got != want { + t.Errorf("upstream request = %q, want %q", got, want) + } + }) +} + +func TestContainerHandler_NamespacePrefixRouteRejectsUnknownHosts(t *testing.T) { + // A _default wildcard mirror turns a pull of + // unconfigured.example/upstream/mirror/owner/app into exactly this + // request. Serving it would hand out owner/app from the mirror upstream + // instead of letting containerd fall back to unconfigured.example. + mirror := newNSTestRegistry(t, "owner/app", "") + routes, h, logs := newNSTestHandler(t, "", map[string]string{"mirror": mirror.URL}) + + for _, namespace := range []string{"unconfigured.example", "ghcr.io"} { + assertNameUnknown(t, serveNS(routes, "/v2/upstream/mirror/owner/app/manifests/latest?ns="+namespace)) + } + if got := mirror.requestCount(); got != 0 { + t.Errorf("upstream requests = %d, want 0", got) + } + // containerd swallows the 404, so the refusal has to show up in the log. + for _, want := range []string{"upstream.oci_mirrors", "upstream=mirror", "ns=ghcr.io"} { + if !strings.Contains(logs.String(), want) { + t.Errorf("logs missing %q:\n%s", want, logs.String()) + } + } + + // A node repeats the refused request for every pull; one line per + // upstream/ns pair is enough. An ns longer than any registry host is + // refused before it is logged or remembered. + assertNameUnknown(t, serveNS(routes, "/v2/upstream/mirror/owner/app/manifests/latest?ns=GHCR.io")) + long := strings.Repeat("a", 300) + ".example" + assertNameUnknown(t, serveNS(routes, "/v2/upstream/mirror/owner/app/manifests/latest?ns="+long)) + if got := strings.Count(logs.String(), "upstream.oci_mirrors"); got != 2 { + t.Errorf("refusal warnings = %d, want 2 (one per upstream/ns pair):\n%s", got, logs.String()) + } + if strings.Contains(logs.String(), "aaaa") { + t.Errorf("logs mention the overlong ns, want it refused silently:\n%s", logs.String()) + } + if got := len(h.refusals); got != 2 { + t.Errorf("remembered refusals = %d, want 2", got) + } +} + +func TestContainerHandler_NamespaceRefusalLogIsBounded(t *testing.T) { + mirror := newNSTestRegistry(t, "owner/app", "") + routes, h, logs := newNSTestHandler(t, "", map[string]string{"mirror": mirror.URL}) + h.refusals = make(map[string]struct{}, maxReportedNamespaceRefusals) + for i := range maxReportedNamespaceRefusals { + h.refusals[fmt.Sprintf("mirror\x00filler-%d.example", i)] = struct{}{} + } + + for _, namespace := range []string{"filler-0.example", "first.example", "second.example"} { + assertNameUnknown(t, serveNS(routes, "/v2/upstream/mirror/owner/app/manifests/latest?ns="+namespace)) + } + if got := strings.Count(logs.String(), "further refusals are not logged"); got != 1 { + t.Errorf("limit notices = %d, want 1:\n%s", got, logs.String()) + } + if strings.Contains(logs.String(), "refusing ns") { + t.Errorf("refusal logged past the limit:\n%s", logs.String()) + } + if got := len(h.refusals); got != maxReportedNamespaceRefusals { + t.Errorf("remembered refusals = %d, want %d", got, maxReportedNamespaceRefusals) + } +} + +func TestContainerHandler_NamespaceMirroredRegistries(t *testing.T) { + // The upstream mirrors ghcr.io, and upstream.oci_mirrors says so. Nodes + // keep pulling ghcr.io/..., either through a ghcr.io hosts.toml pointing + // at /v2/upstream/ghcr or through the _default wildcard mirror. + tests := map[string]string{ + "plain mirror host": "", + "artifactory remote": "/artifactory/api/docker/ghcr-remote", + } + for name, pathPrefix := range tests { + t.Run(name, func(t *testing.T) { + mirror := newNSTestRegistry(t, "owner/app", pathPrefix) + routes, h, _ := newNSTestHandler(t, "", map[string]string{"ghcr": mirror.URL + pathPrefix}) + h.SetMirroredRegistries(map[string][]string{"ghcr": {"ghcr.io"}}) + + for _, target := range []string{ + "/v2/upstream/ghcr/owner/app/manifests/latest?ns=ghcr.io", + "/v2/upstream/ghcr/owner/app/manifests/latest?ns=GHCR.io:443", + "/v2/owner/app/manifests/latest?ns=ghcr.io", + } { + before := mirror.requestCount() + response := serveNS(routes, target) + if response.Code != http.StatusOK { + t.Fatalf("GET %s status = %d: %s", target, response.Code, response.Body.String()) + } + // The manifest is cached after the first request, so only + // the first one may reach the upstream. + if before == 0 { + if got, want := mirror.lastRequest(), pathPrefix+"/v2/owner/app/manifests/latest"; got != want { + t.Errorf("upstream request = %q, want %q", got, want) + } + } + } + if got := mirror.requestCount(); got != 1 { + t.Errorf("upstream requests = %d, want 1 (all routes share the cache)", got) + } + + assertNameUnknown(t, serveNS(routes, "/v2/upstream/ghcr/owner/app/manifests/latest?ns=quay.io")) + }) + } +} + +func TestContainerHandler_NamespaceMirrorHostsRankBelowConfiguredURLs(t *testing.T) { + proxy, _, _, _ := setupTestProxy(t) + logs := &bytes.Buffer{} + proxy.Logger = slog.New(slog.NewTextHandler(logs, nil)) + h := NewContainerHandlerWithRegistry(proxy, nsTestProxyURL, "", map[string]string{ + "ghcr": "https://ghcr.io", + "art": "https://artifactory.example/api/docker/ghcr-remote", + "other": "https://other.example", + }) + h.SetMirroredRegistries(map[string][]string{"art": {"ghcr.io"}, "missing": {"quay.io"}}) + + if got := h.namespaces["ghcr.io"]; got != "ghcr" { + t.Errorf("index[ghcr.io] = %q, want ghcr (configured URL wins over a mirror entry)", got) + } + if !h.namespaceNamesPrefixUpstream("ghcr.io", "upstream/art/owner/app") { + t.Error("ns ghcr.io refused on upstream/art/, want accepted as its mirrored registry") + } + if h.namespaceNamesPrefixUpstream("ghcr.io", "upstream/other/owner/app") { + t.Error("ns ghcr.io accepted on upstream/other/, want refused: only art lists it") + } + if _, ok := h.namespaces["quay.io"]; ok { + t.Error("index has quay.io from a mirror entry without upstream") + } + for _, want := range []string{"ghcr.io=ghcr", "upstream=missing"} { + if !strings.Contains(logs.String(), want) { + t.Errorf("logs missing %q:\n%s", want, logs.String()) + } + } +} + +func TestContainerHandler_NamespaceSharesCacheWithOtherRoutes(t *testing.T) { + digest := "sha256:" + sha256Hex(nsTestBlob) + manifestDigest := "sha256:" + sha256Hex(nsTestManifest()) + + t.Run("named registry", func(t *testing.T) { + registry := newNSTestRegistry(t, "owner/app", "") + routes, _, _ := newNSTestHandler(t, "", map[string]string{"ghcr": registry.URL}) + + for _, target := range []string{ + "/v2/upstream/ghcr/owner/app/blobs/" + digest, + "/v2/upstream/ghcr/owner/app/manifests/" + manifestDigest, + } { + if response := serveNS(routes, target); response.Code != http.StatusOK { + t.Fatalf("warm %s status = %d: %s", target, response.Code, response.Body.String()) + } + } + warmed := registry.requestCount() + for _, target := range []string{ + "/v2/owner/app/blobs/" + digest + "?ns=" + registry.host(), + "/v2/owner/app/manifests/" + manifestDigest + "?ns=" + registry.host(), + } { + if response := serveNS(routes, target); response.Code != http.StatusOK { + t.Fatalf("ns %s status = %d: %s", target, response.Code, response.Body.String()) + } + } + if got := registry.requestCount(); got != warmed { + t.Errorf("upstream requests after ns pulls = %d, want %d (cache hits)", got, warmed) + } + }) + + t.Run("default registry", func(t *testing.T) { + registry := newNSTestRegistry(t, "library/nginx", "") + routes, _, _ := newNSTestHandler(t, registry.URL, nil) + + for _, target := range []string{ + "/v2/library/nginx/blobs/" + digest + "?ns=docker.io", + "/v2/library/nginx/manifests/" + manifestDigest + "?ns=docker.io", + } { + if response := serveNS(routes, target); response.Code != http.StatusOK { + t.Fatalf("warm %s status = %d: %s", target, response.Code, response.Body.String()) + } + } + warmed := registry.requestCount() + for _, target := range []string{ + "/v2/library/nginx/blobs/" + digest, + "/v2/library/nginx/manifests/" + manifestDigest, + } { + if response := serveNS(routes, target); response.Code != http.StatusOK { + t.Fatalf("unprefixed %s status = %d: %s", target, response.Code, response.Body.String()) + } + } + if got := registry.requestCount(); got != warmed { + t.Errorf("upstream requests after unprefixed pulls = %d, want %d (cache hits)", got, warmed) + } + }) +} + +func TestContainerHandler_NamespaceTagsList(t *testing.T) { + registry := newNSTestRegistry(t, "owner/app", "") + routes, _, _ := newNSTestHandler(t, "", map[string]string{"ghcr": registry.URL}) + + // The prefix route fills the shared cache entry first. + prefixed := serveNS(routes, "/v2/upstream/ghcr/owner/app/tags/list?n=1") + if prefixed.Code != http.StatusOK { + t.Fatalf("prefixed status = %d: %s", prefixed.Code, prefixed.Body.String()) + } + const wantPrefixedLink = `<` + nsTestProxyURL + `/v2/upstream/ghcr/owner/app/tags/list?last=1.0&n=1>; rel="next"` + if got := prefixed.Header().Get("Link"); got != wantPrefixedLink { + t.Errorf("prefixed Link = %q, want %q", got, wantPrefixedLink) + } + + namespaced := serveNS(routes, "/v2/owner/app/tags/list?n=1&ns="+registry.host()) + if namespaced.Code != http.StatusOK { + t.Fatalf("ns status = %d: %s", namespaced.Code, namespaced.Body.String()) + } + if got := registry.requestCount(); got != 1 { + t.Errorf("upstream requests = %d, want 1 (ns request served from the shared cache)", got) + } + wantNSLink := `<` + nsTestProxyURL + `/v2/owner/app/tags/list?last=1.0&n=1&ns=` + url.QueryEscape(registry.host()) + `>; rel="next"` + link := namespaced.Header().Get("Link") + if link != wantNSLink { + t.Fatalf("ns Link = %q, want %q", link, wantNSLink) + } + + next := serveNS(routes, strings.TrimPrefix(strings.SplitN(link, ">", 2)[0], "<")) + if next.Code != http.StatusOK { + t.Fatalf("next page status = %d: %s", next.Code, next.Body.String()) + } + if got, want := next.Body.String(), `{"name":"owner/app","tags":["2.0"]}`; got != want { + t.Errorf("next page body = %q, want %q", got, want) + } + if got, want := registry.lastRequest(), "/v2/owner/app/tags/list?last=1.0&n=1"; got != want { + t.Errorf("upstream request = %q, want %q (ns must not be forwarded)", got, want) + } +} + +func TestContainerHandler_NamespaceDefaultBypassesRepositoryPrefixRoutes(t *testing.T) { + hub := newNSTestRegistry(t, "homebrew/core/jq", "") + brew := newNSTestRegistry(t, "homebrew/core/jq", "") + routes, h, _ := newNSTestHandler(t, hub.URL, nil) + RegisterHomebrewArtifacts(h, brew.URL) + + if response := serveNS(routes, "/v2/homebrew/core/jq/manifests/latest?ns=docker.io"); response.Code != http.StatusOK { + t.Fatalf("ns status = %d: %s", response.Code, response.Body.String()) + } + if hub.requestCount() != 1 || brew.requestCount() != 0 { + t.Errorf("ns request reached hub=%d brew=%d, want hub=1 brew=0", hub.requestCount(), brew.requestCount()) + } + + if response := serveNS(routes, "/v2/homebrew/core/jq/manifests/latest"); response.Code != http.StatusOK { + t.Fatalf("unprefixed status = %d: %s", response.Code, response.Body.String()) + } + if hub.requestCount() != 1 || brew.requestCount() != 1 { + t.Errorf("unprefixed request reached hub=%d brew=%d, want hub=1 brew=1", hub.requestCount(), brew.requestCount()) + } +} + +func TestContainerHandler_NamespaceMatchesConfiguredHosts(t *testing.T) { + // Each pair is a configured registry URL and an ns value containerd sends + // for an image reference on that registry. The scheme-default port may be + // spelled out or left out on either side; any other port must match. + matches := []struct { + configured string + namespace string + }{ + {"https://GHCR.io", "ghcr.io"}, + {"https://ghcr.io/", "GHCR.IO"}, + {"http://[fd00::1]:5000", "[fd00::1]:5000"}, + {"https://[fd00::1]", "[fd00::1]"}, + {"https://[fd00::1]", "[fd00::1]:443"}, + {"https://reg.example:80", "reg.example:80"}, + {"https://reg.example:443", "reg.example"}, + {"https://reg.example", "reg.example:443"}, + {"http://reg.example:80", "reg.example"}, + {"http://reg.example", "reg.example:80"}, + {"http://reg.example:5000", "reg.example:5000"}, + } + for _, tt := range matches { + t.Run(tt.configured+" "+tt.namespace, func(t *testing.T) { + h := NewContainerHandlerWithRegistry(nil, nsTestProxyURL, "", map[string]string{"lab": tt.configured}) + registryURL, upstreamName, cacheName, ok := h.registryForNamespace(tt.namespace, "owner/app") + if !ok { + t.Fatalf("ns %q did not resolve for upstream %q", tt.namespace, tt.configured) + } + if registryURL != strings.TrimSuffix(tt.configured, "/") || upstreamName != "owner/app" || cacheName != "upstream/lab/owner/app" { + t.Errorf("route = (%q, %q, %q), want (%q, owner/app, upstream/lab/owner/app)", + registryURL, upstreamName, cacheName, tt.configured) + } + }) + } + + mismatches := []struct { + configured string + namespace string + }{ + {"http://reg.example:5000", "reg.example:5001"}, + {"https://reg.example:80", "reg.example"}, + {"https://reg.example", "reg.example:80"}, + {"http://reg.example:443", "reg.example"}, + } + for _, tt := range mismatches { + t.Run("mismatch "+tt.configured+" "+tt.namespace, func(t *testing.T) { + h := NewContainerHandlerWithRegistry(nil, nsTestProxyURL, "", map[string]string{"lab": tt.configured}) + if _, _, _, ok := h.registryForNamespace(tt.namespace, "owner/app"); ok { + t.Errorf("ns %q resolved for upstream %q, want no match", tt.namespace, tt.configured) + } + }) + } +} + +func TestContainerHandler_NamespaceKeepsNonDefaultPorts(t *testing.T) { + // Two registries on one host that differ only in the port, one of them on + // the scheme-default port. Tests cannot listen on ports 80 or 443, so the + // dialer maps those addresses to the fake registries. + onDefaultPort := newNSTestRegistry(t, "owner/app", "") + onOtherPort := newNSTestRegistry(t, "owner/app", "") + routes, h, _ := newNSTestHandler(t, "", map[string]string{ + "std": "http://registry.example", + "alt": "http://registry.example:443", + }) + endpoints := map[string]string{ + "registry.example:80": onDefaultPort.Listener.Addr().String(), + "registry.example:443": onOtherPort.Listener.Addr().String(), + } + transport := &http.Transport{DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) { + target, ok := endpoints[addr] + if !ok { + return nil, fmt.Errorf("unexpected dial to %s", addr) + } + return (&net.Dialer{}).DialContext(ctx, network, target) + }} + t.Cleanup(transport.CloseIdleConnections) + h.proxy.HTTPClient = &http.Client{Transport: transport} + + // Every request below misses the cache: a different registry or a + // different endpoint than the request before it. + tests := []struct { + target string + wantDefault int + wantOther int + }{ + {"/v2/owner/app/manifests/latest?ns=registry.example:80", 1, 0}, + {"/v2/owner/app/manifests/latest?ns=registry.example:443", 1, 1}, + {"/v2/owner/app/tags/list?ns=registry.example", 2, 1}, + } + for _, tt := range tests { + response := serveNS(routes, tt.target) + if response.Code != http.StatusOK { + t.Fatalf("%s status = %d: %s", tt.target, response.Code, response.Body.String()) + } + if onDefaultPort.requestCount() != tt.wantDefault || onOtherPort.requestCount() != tt.wantOther { + t.Errorf("after %s: requests default-port=%d other-port=%d, want %d/%d", + tt.target, onDefaultPort.requestCount(), onOtherPort.requestCount(), tt.wantDefault, tt.wantOther) + } + } +} + +func TestContainerHandler_NamespaceHostCollisions(t *testing.T) { + proxy, _, _, _ := setupTestProxy(t) + logs := &bytes.Buffer{} + proxy.Logger = slog.New(slog.NewTextHandler(logs, nil)) + h := NewContainerHandlerWithRegistry(proxy, nsTestProxyURL, "https://mirror.example", map[string]string{ + "zeta": "https://shared.example", + "alpha": "https://Shared.example:443", + "mirror": "https://mirror.example", + "hub": "https://registry-1.docker.io", + }) + + tests := map[string]string{ + "shared.example": "https://Shared.example:443", + "mirror.example": "https://mirror.example", + "registry-1.docker.io": "https://mirror.example", + } + for namespace, want := range tests { + registryURL, _, cacheName, ok := h.registryForNamespace(namespace, "owner/app") + if !ok || registryURL != want { + t.Errorf("ns %q resolved to (%q, %v), want %q", namespace, registryURL, ok, want) + } + if namespace == "shared.example" && cacheName != "upstream/alpha/owner/app" { + t.Errorf("ns %q cache name = %q, want upstream/alpha/owner/app", namespace, cacheName) + } + } + for _, upstream := range []string{"upstream=zeta", "upstream=mirror", "upstream=hub"} { + if !strings.Contains(logs.String(), upstream) { + t.Errorf("missing collision warning for %s in logs:\n%s", upstream, logs.String()) + } + } +} + +func TestContainerHandler_NamespaceCollisionWarningNamesEachOwner(t *testing.T) { + // r's two keys end up with two different owners; the warning has to + // name both, or an admin fixing one entry misses the other. + proxy, _, _, _ := setupTestProxy(t) + logs := &bytes.Buffer{} + proxy.Logger = slog.New(slog.NewTextHandler(logs, nil)) + h := NewContainerHandlerWithRegistry(proxy, nsTestProxyURL, "", map[string]string{ + "p": "http://h.example", + "q": "http://h.example:443", + "r": "https://h.example", + }) + + for namespace, want := range map[string]string{"h.example": "p", "h.example:80": "p", "h.example:443": "q"} { + if got := h.namespaces[namespace]; got != want { + t.Errorf("index[%q] = %q, want %q", namespace, got, want) + } + } + for _, want := range []string{"upstream=r", "h.example=p", "h.example:443=q"} { + if !strings.Contains(logs.String(), want) { + t.Errorf("logs missing %q:\n%s", want, logs.String()) + } + } +} + +func TestContainerHandler_NamespaceSkipsRegistryURLsWithPath(t *testing.T) { + registry := newNSTestRegistry(t, "owner/app", "/artifactory/api/docker/remote") + routes, _, logs := newNSTestHandler(t, "", map[string]string{ + "art": registry.URL + "/artifactory/api/docker/remote", + }) + + assertNameUnknown(t, serveNS(routes, "/v2/owner/app/manifests/latest?ns="+registry.host())) + if got := registry.requestCount(); got != 0 { + t.Errorf("upstream requests via ns = %d, want 0", got) + } + if !strings.Contains(logs.String(), "upstream=art") { + t.Errorf("missing warning for path-prefixed upstream in logs:\n%s", logs.String()) + } + + // Per-registry mirrors with override_path reach it with ns attached. + if response := serveNS(routes, "/v2/upstream/art/owner/app/manifests/latest?ns="+registry.host()); response.Code != http.StatusOK { + t.Fatalf("prefix route with ns status = %d, want 200: %s", response.Code, response.Body.String()) + } + if got := registry.requestCount(); got != 1 { + t.Errorf("upstream requests via prefix route with ns = %d, want 1", got) + } + if response := serveNS(routes, "/v2/upstream/art/owner/app/manifests/latest"); response.Code != http.StatusOK { + t.Fatalf("prefix route status = %d, want 200: %s", response.Code, response.Body.String()) + } + + t.Run("default registry", func(t *testing.T) { + mirror := newNSTestRegistry(t, "library/nginx", "/hub") + routes, _, logs := newNSTestHandler(t, mirror.URL+"/hub", nil) + + assertNameUnknown(t, serveNS(routes, "/v2/library/nginx/manifests/latest?ns="+mirror.host())) + if got := mirror.requestCount(); got != 0 { + t.Errorf("upstream requests via ns host = %d, want 0", got) + } + if response := serveNS(routes, "/v2/library/nginx/manifests/latest?ns=docker.io"); response.Code != http.StatusOK { + t.Fatalf("ns=docker.io status = %d, want 200: %s", response.Code, response.Body.String()) + } + if !strings.Contains(logs.String(), "default OCI registry") { + t.Errorf("missing warning for path-prefixed default registry in logs:\n%s", logs.String()) + } + }) +} + +func TestContainerHandler_TagsListIgnoresLegacyCacheRows(t *testing.T) { + registry := newNSTestRegistry(t, "owner/app", "") + routes, h, _ := newNSTestHandler(t, "", map[string]string{"ghcr": registry.URL}) + + // Rows written before the raw-link format hold a Link already rewritten + // for the route that filled them. Seed one under the legacy identity. + query := url.Values{"n": {"1"}} + legacySum := sha256.Sum256([]byte(registry.URL + "\x00owner/app\x00" + query.Encode())) + legacyKey := hex.EncodeToString(legacySum[:]) + if legacyKey == h.containerTagsCacheKey(registry.URL, "owner/app", query) { + t.Fatal("legacy and current tag-list cache keys are equal") + } + legacy := &cachedContainerTags{ + body: []byte(`{"name":"owner/app","tags":["legacy"]}`), + contentType: contentTypeJSON, + link: `<` + nsTestProxyURL + `/v2/upstream/ghcr/owner/app/tags/list?last=legacy&n=1>; rel="next"`, + fetchedAt: time.Now(), + } + if err := h.storeContainerTags(context.Background(), legacyKey, legacy); err != nil { + t.Fatalf("store legacy tag list: %v", err) + } + + response := serveNS(routes, "/v2/owner/app/tags/list?n=1&ns="+registry.host()) + if response.Code != http.StatusOK { + t.Fatalf("status = %d: %s", response.Code, response.Body.String()) + } + if got, want := response.Body.String(), `{"name":"owner/app","tags":["1.0"]}`; got != want { + t.Errorf("body = %q, want %q (legacy row must not be served)", got, want) + } + if got := registry.requestCount(); got != 1 { + t.Errorf("upstream requests = %d, want 1 (legacy row must not be served)", got) + } +} + +func TestContainerHandler_NamespaceWarningsRedactCredentials(t *testing.T) { + proxy, _, _, _ := setupTestProxy(t) + logs := &bytes.Buffer{} + proxy.Logger = slog.New(slog.NewTextHandler(logs, nil)) + NewContainerHandlerWithRegistry(proxy, nsTestProxyURL, "https://svc:s3cret@mirror.example/hub", map[string]string{ + "art": "https://bot:hunter2@art.example/artifactory/api/docker/remote", + }) + + for _, secret := range []string{"s3cret", "hunter2"} { + if strings.Contains(logs.String(), secret) { + t.Errorf("logs contain credential %q:\n%s", secret, logs.String()) + } + } + for _, want := range []string{ + "svc:xxxxx@mirror.example/hub", + "bot:xxxxx@art.example/artifactory", + "Docker Hub aliases still select it", + } { + if !strings.Contains(logs.String(), want) { + t.Errorf("logs missing %q:\n%s", want, logs.String()) + } + } +} diff --git a/internal/handler/container_tags.go b/internal/handler/container_tags.go index e9803ed..a0ac9c1 100644 --- a/internal/handler/container_tags.go +++ b/internal/handler/container_tags.go @@ -13,7 +13,14 @@ import ( "time" ) -const containerTagsCacheEcosystem = "oci-tags" +const ( + containerTagsCacheEcosystem = "oci-tags" + // containerTagsCacheFormat versions the tag-list cache identity. Rows + // written before it hold a Link already rewritten for the route that + // filled them, while this format stores the upstream Link verbatim. The + // two must not share rows: an older binary would serve a raw Link as is. + containerTagsCacheFormat = "raw-link" +) var containerLinkTargetPattern = regexp.MustCompile(`<([^>]*)>`) @@ -27,20 +34,24 @@ type cachedContainerTags struct { } func (h *ContainerHandler) serveTagsList(w http.ResponseWriter, r *http.Request, registryURL, name string) { - cacheKey := h.containerTagsCacheKey(registryURL, name, r.URL.Query()) + // ns only selects the registry; it is neither forwarded upstream nor part + // of the cache identity, so all routes to one registry share tag lists. + query := r.URL.Query() + query.Del(namespaceQueryParam) + cacheKey := h.containerTagsCacheKey(registryURL, name, query) cached, err := h.loadContainerTags(r.Context(), cacheKey) if err != nil { h.proxy.Logger.Warn("failed to read cached container tag list", "error", err) cached = nil } if cached != nil && h.containerTagsFresh(cached) { - writeContainerTags(w, cached, false) + h.writeContainerTags(w, r, registryURL, cached, false) return } upstreamURL := fmt.Sprintf("%s/v2/%s/tags/list", registryURL, name) - if query := r.URL.Query().Encode(); query != "" { - upstreamURL += "?" + query + if encoded := query.Encode(); encoded != "" { + upstreamURL += "?" + encoded } req, err := http.NewRequestWithContext(r.Context(), http.MethodGet, upstreamURL, nil) if err != nil { @@ -54,7 +65,7 @@ func (h *ContainerHandler) serveTagsList(w http.ResponseWriter, r *http.Request, resp, err := h.proxy.HTTPClient.Do(req) if err != nil { - h.serveStaleTagsOrError(w, cached, err) + h.serveStaleTagsOrError(w, r, registryURL, cached, err) return } defer func() { _ = resp.Body.Close() }() @@ -64,12 +75,12 @@ func (h *ContainerHandler) serveTagsList(w http.ResponseWriter, r *http.Request, if err := h.storeContainerTags(r.Context(), cacheKey, cached); err != nil { h.proxy.Logger.Warn("failed to refresh cached container tag list", "error", err) } - writeContainerTags(w, cached, false) + h.writeContainerTags(w, r, registryURL, cached, false) return } if resp.StatusCode != http.StatusOK { if cached != nil && shouldServeStaleManifest(resp.StatusCode) { - writeContainerTags(w, cached, true) + h.writeContainerTags(w, r, registryURL, cached, true) return } h.proxy.relayResponse(w, r, resp, copyContainerTagsHeaders) @@ -78,14 +89,16 @@ func (h *ContainerHandler) serveTagsList(w http.ResponseWriter, r *http.Request, body, err := h.proxy.ReadMetadata(resp.Body) if err != nil { - h.serveStaleTagsOrError(w, cached, fmt.Errorf("reading tag list: %w", err)) + h.serveStaleTagsOrError(w, r, registryURL, cached, fmt.Errorf("reading tag list: %w", err)) return } + // The upstream Link is stored verbatim and rewritten per request because + // clients on different routes share this cache entry. tags := &cachedContainerTags{ body: body, contentType: resp.Header.Get(headerContentType), etag: resp.Header.Get(headerETag), - link: h.rewriteContainerTagsLink(strings.Join(resp.Header.Values("Link"), ", "), registryURL, r.URL.Path), + link: strings.Join(resp.Header.Values("Link"), ", "), size: int64(len(body)), fetchedAt: time.Now(), } @@ -95,13 +108,13 @@ func (h *ContainerHandler) serveTagsList(w http.ResponseWriter, r *http.Request, if err := h.storeContainerTags(r.Context(), cacheKey, tags); err != nil { h.proxy.Logger.Warn("failed to cache container tag list", "error", err) } - writeContainerTags(w, tags, false) + h.writeContainerTags(w, r, registryURL, tags, false) } -func (h *ContainerHandler) serveStaleTagsOrError(w http.ResponseWriter, cached *cachedContainerTags, err error) { +func (h *ContainerHandler) serveStaleTagsOrError(w http.ResponseWriter, r *http.Request, registryURL string, cached *cachedContainerTags, err error) { if cached != nil { h.proxy.Logger.Warn("upstream tag list fetch failed, serving stale cache", "error", err) - writeContainerTags(w, cached, true) + h.writeContainerTags(w, r, registryURL, cached, true) return } h.proxy.Logger.Error("failed to fetch container tag list", "error", err) @@ -109,7 +122,7 @@ func (h *ContainerHandler) serveStaleTagsOrError(w http.ResponseWriter, cached * } func (h *ContainerHandler) containerTagsCacheKey(registryURL, name string, query url.Values) string { - identity := registryURL + "\x00" + name + "\x00" + query.Encode() + identity := containerTagsCacheFormat + "\x00" + registryURL + "\x00" + name + "\x00" + query.Encode() sum := sha256.Sum256([]byte(identity)) return hex.EncodeToString(sum[:]) } @@ -165,14 +178,15 @@ func (h *ContainerHandler) storeContainerTags(ctx context.Context, cacheKey stri return nil } -func writeContainerTags(w http.ResponseWriter, tags *cachedContainerTags, stale bool) { +func (h *ContainerHandler) writeContainerTags(w http.ResponseWriter, r *http.Request, registryURL string, tags *cachedContainerTags, stale bool) { w.Header().Set(headerContentType, tags.contentType) w.Header().Set(headerContentLength, strconv.FormatInt(tags.size, 10)) if tags.etag != "" { w.Header().Set(headerETag, tags.etag) } - if tags.link != "" { - w.Header().Set("Link", tags.link) + link := h.rewriteContainerTagsLink(tags.link, registryURL, r.URL.Path, r.URL.Query().Get(namespaceQueryParam)) + if link != "" { + w.Header().Set("Link", link) } if stale { w.Header().Set("Warning", containerStaleWarning) @@ -189,7 +203,7 @@ func copyContainerTagsHeaders(destination, source http.Header) { } } -func (h *ContainerHandler) rewriteContainerTagsLink(link, registryURL, requestPath string) string { +func (h *ContainerHandler) rewriteContainerTagsLink(link, registryURL, requestPath, namespace string) string { if link == "" { return "" } @@ -221,6 +235,12 @@ func (h *ContainerHandler) rewriteContainerTagsLink(link, registryURL, requestPa linkURL.User = proxyURL.User linkURL.Path = strings.TrimSuffix(proxyURL.Path, "/") + "/v2" + requestPath linkURL.RawPath = "" + if namespace != "" { + // Keep follow-up pages on the ns route the client is using. + query := linkURL.Query() + query.Set(namespaceQueryParam, namespace) + linkURL.RawQuery = query.Encode() + } return "<" + linkURL.String() + ">" }) } diff --git a/internal/server/oci_mirrors_test.go b/internal/server/oci_mirrors_test.go new file mode 100644 index 0000000..5668dd4 --- /dev/null +++ b/internal/server/oci_mirrors_test.go @@ -0,0 +1,45 @@ +package server + +import ( + "io" + "log/slog" + "net/http" + "net/http/httptest" + "testing" + + "github.com/git-pkgs/proxy/internal/config" + "github.com/git-pkgs/proxy/internal/handler" + "github.com/go-chi/chi/v5" +) + +func TestOCIMirrorsReachContainerHandler(t *testing.T) { + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/art/v2/owner/app/manifests/latest" { + http.NotFound(w, r) + return + } + w.Header().Set("Content-Type", "application/vnd.oci.image.manifest.v1+json") + _, _ = io.WriteString(w, `{"schemaVersion":2}`) + })) + defer upstream.Close() + + cfg := config.Default() + cfg.BaseURL = "https://proxy.example" + cfg.Upstream.OCI = map[string]string{"art": upstream.URL + "/art"} + cfg.Upstream.OCIMirrors = map[string][]string{"art": {"ghcr.io"}} + p := &handler.Proxy{HTTPClient: upstream.Client(), Logger: slog.New(slog.NewTextHandler(io.Discard, nil))} + s := &Server{cfg: cfg} + r := chi.NewRouter() + s.mountProtocolHandlers(r, p) + + for _, target := range []string{ + "/v2/upstream/art/owner/app/manifests/latest?ns=ghcr.io", + "/v2/owner/app/manifests/latest?ns=ghcr.io", + } { + w := httptest.NewRecorder() + r.ServeHTTP(w, httptest.NewRequest(http.MethodGet, target, nil)) + if w.Code != http.StatusOK { + t.Errorf("GET %s status = %d, want 200 through upstream.oci_mirrors: %s", target, w.Code, w.Body.String()) + } + } +} diff --git a/internal/server/server.go b/internal/server/server.go index 69707f7..66ed14b 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -438,6 +438,7 @@ func (s *Server) mountProtocolHandlers(r chi.Router, proxy *handler.Proxy) { s.cfg.Upstream.OCIDefault, s.cfg.Upstream.OCI, ) + containerHandler.SetMirroredRegistries(s.cfg.Upstream.OCIMirrors) handler.RegisterHomebrewArtifacts(containerHandler, s.cfg.Upstream.HomebrewArtifact) helmHandler := handler.NewHelmHandlerWithOCIRegistries( proxy, s.cfg.BaseURL, s.cfg.Upstream.Helm, s.cfg.Upstream.OCIDefault, s.cfg.Upstream.OCI) From 56833497547587a9f90c94eca85ab4102144e220 Mon Sep 17 00:00:00 2001 From: Christian Heim Date: Sun, 11 Oct 2026 17:56:15 +0200 Subject: [PATCH 187/633] Cache Conan 2 recipe and package archives (#431) * Conan 2 downloads recipe and package files from /v2/conans/.../revisions/{rrev}/files/{file} and .../packages/{id}/revisions/{prev}/files/{file}, but the handler only cached the invalid /v1/files and /v2/files paths, so archives were never stored. This adds the real routes and drops the old ones; manifests and conaninfo.txt stay in the metadata cache. * Conan 2.25 can also compress archives with xz or zstd (conan_package.txz, conan_package.tzst and so on), so cache those too. * Store Conan archives under plain name and version, like pkg:conan/zlib@1.3.1, with the revisions and any user/channel in the filename. The old name/version@user/channel with one version per revision never matched a denylist entry or what scanners look up. --- internal/handler/conan.go | 60 +++---- internal/handler/conan_test.go | 180 ++++++++++++++++++- internal/handler/download_test.go | 8 +- internal/handler/notfound_ecosystems_test.go | 2 +- internal/handler/relay_test.go | 2 +- 5 files changed, 213 insertions(+), 39 deletions(-) diff --git a/internal/handler/conan.go b/internal/handler/conan.go index aa41668..d4040e2 100644 --- a/internal/handler/conan.go +++ b/internal/handler/conan.go @@ -1,7 +1,6 @@ package handler import ( - "fmt" "net/http" "strings" ) @@ -41,13 +40,12 @@ func (h *ConanHandler) Routes() http.Handler { mux.HandleFunc("GET /v1/ping", h.handlePing) mux.HandleFunc("GET /v2/ping", h.handlePing) - // Recipe file downloads (cache these) - mux.HandleFunc("GET /v1/files/{name}/{version}/{user}/{channel}/{revision}/recipe/{filename}", h.handleRecipeFile) - mux.HandleFunc("GET /v2/files/{name}/{version}/{user}/{channel}/{revision}/recipe/{filename}", h.handleRecipeFile) - - // Package file downloads (cache these) - mux.HandleFunc("GET /v1/files/{name}/{version}/{user}/{channel}/{revision}/package/{pkgref}/{pkgrev}/{filename}", h.handlePackageFile) - mux.HandleFunc("GET /v2/files/{name}/{version}/{user}/{channel}/{revision}/package/{pkgref}/{pkgrev}/{filename}", h.handlePackageFile) + // Recipe and package file downloads, as built by the Conan 2 client + // (ClientV2Router.recipe_file and package_file). The v1 protocol hands out + // signed absolute URLs on the upstream host, so its downloads never pass + // through here. + mux.HandleFunc("GET /v2/conans/{name}/{version}/{user}/{channel}/revisions/{revision}/files/{filename}", h.handleRecipeFile) + mux.HandleFunc("GET /v2/conans/{name}/{version}/{user}/{channel}/revisions/{revision}/packages/{pkgref}/revisions/{pkgrev}/files/{filename}", h.handlePackageFile) // Proxy all other endpoints (metadata, search, etc.) with caching mux.HandleFunc("GET /", h.proxyCached) @@ -70,25 +68,19 @@ func (h *ConanHandler) handleRecipeFile(w http.ResponseWriter, r *http.Request) revision := r.PathValue("revision") filename := r.PathValue("filename") - // Only cache specific files + // Only the archives are artifacts; the rest goes through the metadata cache. if !h.shouldCacheFile(filename) { - h.proxyUpstream(w, r) + h.proxyCached(w, r) return } - // Conan package name format: name/version@user/channel - packageName := fmt.Sprintf("%s/%s@%s/%s", name, version, user, channel) - h.proxy.Logger.Info("conan recipe download", "name", name, "version", version, "user", user, "channel", channel, "filename", filename) upstreamURL := h.upstreamURL + r.URL.Path + storageFilename := conanStorageFilename(user, channel, revision+"_"+filename) - // Use revision as part of version for storage - storageVersion := fmt.Sprintf("%s_%s", version, revision) - storageFilename := fmt.Sprintf("recipe_%s", filename) - - result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "conan", packageName, storageVersion, storageFilename, upstreamURL) + result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "conan", name, version, storageFilename, upstreamURL) if err != nil { h.proxy.serveArtifactError(w, err, "failed to fetch file") return @@ -108,24 +100,19 @@ func (h *ConanHandler) handlePackageFile(w http.ResponseWriter, r *http.Request) pkgrev := r.PathValue("pkgrev") filename := r.PathValue("filename") - // Only cache specific files + // Only the archives are artifacts; the rest goes through the metadata cache. if !h.shouldCacheFile(filename) { - h.proxyUpstream(w, r) + h.proxyCached(w, r) return } - packageName := fmt.Sprintf("%s/%s@%s/%s", name, version, user, channel) - h.proxy.Logger.Info("conan package download", "name", name, "version", version, "pkgref", pkgref, "filename", filename) upstreamURL := h.upstreamURL + r.URL.Path + storageFilename := conanStorageFilename(user, channel, revision+"_"+pkgref+"_"+pkgrev+"_"+filename) - // Use revision and package ref as part of version for storage - storageVersion := fmt.Sprintf("%s_%s_%s_%s", version, revision, pkgref, pkgrev) - storageFilename := fmt.Sprintf("package_%s", filename) - - result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "conan", packageName, storageVersion, storageFilename, upstreamURL) + result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "conan", name, version, storageFilename, upstreamURL) if err != nil { h.proxy.serveArtifactError(w, err, "failed to fetch file") return @@ -134,13 +121,24 @@ func (h *ConanHandler) handlePackageFile(w http.ResponseWriter, r *http.Request) ServeArtifactRequest(w, r, result) } +// conanStorageFilename names a cached archive. Archives are stored under +// pkg:conan/{name}@{version}, so the file name carries the revisions and, when +// set, user and channel, to keep every recipe and binary apart. +func conanStorageFilename(user, channel, file string) string { + if user == "_" && channel == "_" { + return file + } + return user + "@" + channel + "_" + file +} + // shouldCacheFile returns true if the file should be cached. func (h *ConanHandler) shouldCacheFile(filename string) bool { - // Cache the large archive files + // Cache the large archive files. Since Conan 2.25 they can also be xz or + // zstd (core.upload:compression_format). cacheFiles := []string{ - "conan_sources.tgz", - "conan_export.tgz", - "conan_package.tgz", + "conan_sources.tgz", "conan_sources.txz", "conan_sources.tzst", + "conan_export.tgz", "conan_export.txz", "conan_export.tzst", + "conan_package.tgz", "conan_package.txz", "conan_package.tzst", } for _, f := range cacheFiles { diff --git a/internal/handler/conan_test.go b/internal/handler/conan_test.go index a7bd362..f2fb909 100644 --- a/internal/handler/conan_test.go +++ b/internal/handler/conan_test.go @@ -6,7 +6,9 @@ import ( "net/http" "net/http/httptest" "strings" + "sync/atomic" "testing" + "time" ) const testProxyURL = "http://localhost:8080" @@ -28,6 +30,13 @@ func TestConanShouldCacheFile(t *testing.T) { {"conan_sources.tgz", true}, {"conan_export.tgz", true}, {"conan_package.tgz", true}, + {"conan_sources.txz", true}, + {"conan_export.txz", true}, + {"conan_package.txz", true}, + {"conan_sources.tzst", true}, + {"conan_export.tzst", true}, + {"conan_package.tzst", true}, + {"conan_package.tar.gz", false}, {"conanfile.py", false}, {"conanmanifest.txt", false}, {"conaninfo.txt", false}, @@ -225,7 +234,7 @@ func TestConanRecipeFileNonCacheable(t *testing.T) { proxyURL: "http://proxy.local", } - req := httptest.NewRequest(http.MethodGet, "/v2/files/zlib/1.2.13/_/_/abc123/recipe/conanfile.py", nil) + req := httptest.NewRequest(http.MethodGet, "/v2/conans/zlib/1.2.13/_/_/revisions/abc123/files/conanfile.py", nil) req.SetPathValue("name", "zlib") req.SetPathValue("version", "1.2.13") req.SetPathValue("user", "_") @@ -260,7 +269,7 @@ func TestConanPackageFileNonCacheable(t *testing.T) { proxyURL: "http://proxy.local", } - req := httptest.NewRequest(http.MethodGet, "/v2/files/zlib/1.2.13/_/_/abc123/package/pkgref1/pkgrev1/conaninfo.txt", nil) + req := httptest.NewRequest(http.MethodGet, "/v2/conans/zlib/1.2.13/_/_/revisions/abc123/packages/pkgref1/revisions/pkgrev1/files/conaninfo.txt", nil) req.SetPathValue("name", "zlib") req.SetPathValue("version", "1.2.13") req.SetPathValue("user", "_") @@ -283,6 +292,173 @@ func TestConanPackageFileNonCacheable(t *testing.T) { } } +// The Conan 2 client downloads recipe and package files over these routes; +// the archives must be stored as artifacts and served from the cache after. +func TestConanV2FileRoutesCacheArchives(t *testing.T) { + tests := []struct { + name string + path string + filename string + }{ + {"recipe", "/v2/conans/zlib/1.3.1/_/_/revisions/rrev1/files/conan_sources.tgz", + "rrev1_conan_sources.tgz"}, + {"package", "/v2/conans/zlib/1.3.1/_/_/revisions/rrev1/packages/pkgid1/revisions/prev1/files/conan_package.tgz", + "rrev1_pkgid1_prev1_conan_package.tgz"}, + {"recipe xz", "/v2/conans/zlib/1.3.1/_/_/revisions/rrev1/files/conan_export.txz", + "rrev1_conan_export.txz"}, + {"package zstd", "/v2/conans/zlib/1.3.1/_/_/revisions/rrev1/packages/pkgid1/revisions/prev1/files/conan_package.tzst", + "rrev1_pkgid1_prev1_conan_package.tzst"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + proxy, db, _, fetcher := setupTestProxy(t) + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + t.Errorf("archive %s was proxied instead of fetched as an artifact", r.URL.Path) + http.Error(w, "unexpected", http.StatusInternalServerError) + })) + defer upstream.Close() + proxy.HTTPClient = upstream.Client() + + h := NewConanHandlerWithUpstream(proxy, testProxyURL, upstream.URL) + srv := httptest.NewServer(h.Routes()) + defer srv.Close() + + for i := range 2 { + fetcher.artifact = artifactBody("archive bytes") + fetcher.fetchCalled = false + + resp, err := http.Get(srv.URL + tt.path) + if err != nil { + t.Fatalf("request %d failed: %v", i+1, err) + } + body, _ := io.ReadAll(resp.Body) + _ = resp.Body.Close() + + if resp.StatusCode != http.StatusOK || string(body) != "archive bytes" { + t.Fatalf("request %d: status = %d, body = %q, want 200 %q", i+1, resp.StatusCode, body, "archive bytes") + } + if fetcher.fetchCalled != (i == 0) { + t.Errorf("request %d: fetched = %v, want %v", i+1, fetcher.fetchCalled, i == 0) + } + if i == 0 && fetcher.fetchedURL != upstream.URL+tt.path { + t.Errorf("upstream URL = %q, want %q", fetcher.fetchedURL, upstream.URL+tt.path) + } + } + + recordedStoragePath(t, db, "pkg:conan/zlib@1.3.1", tt.filename) + }) + } +} + +// References that differ only in user and channel share pkg:conan/zlib@1.3.1 +// but must not share files, even with the same recipe revision. +func TestConanUserChannelKeptApart(t *testing.T) { + proxy, db, _, fetcher := setupTestProxy(t) + h := NewConanHandlerWithUpstream(proxy, testProxyURL, "http://upstream.invalid") + srv := httptest.NewServer(h.Routes()) + defer srv.Close() + + for _, ref := range []struct{ path, body, filename string }{ + {"/v2/conans/zlib/1.3.1/_/_/revisions/rrev1/files/conan_sources.tgz", + "center sources", "rrev1_conan_sources.tgz"}, + {"/v2/conans/zlib/1.3.1/acme/stable/revisions/rrev1/files/conan_sources.tgz", + "acme sources", "acme@stable_rrev1_conan_sources.tgz"}, + } { + fetcher.artifact = artifactBody(ref.body) + fetcher.fetchCalled = false + + resp, err := http.Get(srv.URL + ref.path) + if err != nil { + t.Fatalf("GET %s failed: %v", ref.path, err) + } + body, _ := io.ReadAll(resp.Body) + _ = resp.Body.Close() + + if string(body) != ref.body || !fetcher.fetchCalled { + t.Errorf("GET %s: body = %q, fetched = %v, want %q from upstream", ref.path, body, fetcher.fetchCalled, ref.body) + } + recordedStoragePath(t, db, "pkg:conan/zlib@1.3.1", ref.filename) + } +} + +// A denylist entry for a Conan version blocks its recipe and binary archives. +func TestConanDenylistMatchesVersion(t *testing.T) { + proxy, _, _, fetcher := setupTestProxy(t) + setTestDenylist(t, proxy, "pkg:conan/zlib@1.3.1") + h := NewConanHandlerWithUpstream(proxy, testProxyURL, "http://upstream.invalid") + srv := httptest.NewServer(h.Routes()) + defer srv.Close() + + for _, path := range []string{ + "/v2/conans/zlib/1.3.1/_/_/revisions/rrev1/files/conan_sources.tgz", + "/v2/conans/zlib/1.3.1/acme/stable/revisions/rrev1/packages/pkgid1/revisions/prev1/files/conan_package.tgz", + } { + fetcher.artifact = artifactBody("archive bytes") + resp, err := http.Get(srv.URL + path) + if err != nil { + t.Fatalf("GET %s failed: %v", path, err) + } + _ = resp.Body.Close() + if resp.StatusCode != http.StatusForbidden { + t.Errorf("GET %s: status = %d, want %d", path, resp.StatusCode, http.StatusForbidden) + } + } + if fetcher.fetchCalled { + t.Error("denied archive was fetched from upstream") + } +} + +// The small files next to the archives keep going through the metadata cache. +func TestConanV2FileRoutesKeepSmallFilesAsMetadata(t *testing.T) { + for name, path := range map[string]string{ + "recipe": "/v2/conans/zlib/1.3.1/_/_/revisions/rrev1/files/conanmanifest.txt", + "package": "/v2/conans/zlib/1.3.1/_/_/revisions/rrev1/packages/pkgid1/revisions/prev1/files/conaninfo.txt", + } { + t.Run(name, func(t *testing.T) { + var requests atomic.Int32 + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests.Add(1) + if r.URL.Path != path { + http.NotFound(w, r) + return + } + w.Header().Set("Content-Type", "text/plain") + _, _ = w.Write([]byte("small file")) + })) + defer upstream.Close() + + proxy, _, _, fetcher := setupTestProxy(t) + proxy.CacheMetadata = true + proxy.MetadataTTL = time.Hour + proxy.HTTPClient = upstream.Client() + + h := NewConanHandlerWithUpstream(proxy, testProxyURL, upstream.URL) + srv := httptest.NewServer(h.Routes()) + defer srv.Close() + + for i := range 2 { + resp, err := http.Get(srv.URL + path) + if err != nil { + t.Fatalf("request %d failed: %v", i+1, err) + } + body, _ := io.ReadAll(resp.Body) + _ = resp.Body.Close() + if resp.StatusCode != http.StatusOK || string(body) != "small file" { + t.Fatalf("request %d: status = %d, body = %q, want 200 %q", i+1, resp.StatusCode, body, "small file") + } + } + + if got := requests.Load(); got != 1 { + t.Errorf("upstream requests = %d, want 1 (second served from the metadata cache)", got) + } + if fetcher.fetchCalled { + t.Error("small file was fetched as an artifact") + } + }) + } +} + func TestConanRoutes(t *testing.T) { h := &ConanHandler{ proxy: conanTestProxy(), diff --git a/internal/handler/download_test.go b/internal/handler/download_test.go index d91b68a..0df443d 100644 --- a/internal/handler/download_test.go +++ b/internal/handler/download_test.go @@ -1163,7 +1163,7 @@ func TestConanHandler_RecipeFileCacheMiss(t *testing.T) { srv := httptest.NewServer(h.Routes()) defer srv.Close() - resp, err := http.Get(srv.URL + "/v2/files/zlib/1.3/_/_/abc123/recipe/conan_export.tgz") + resp, err := http.Get(srv.URL + "/v2/conans/zlib/1.3/_/_/revisions/abc123/files/conan_export.tgz") if err != nil { t.Fatalf("request failed: %v", err) } @@ -1173,7 +1173,7 @@ func TestConanHandler_RecipeFileCacheMiss(t *testing.T) { t.Error("expected fetcher to be called on cache miss") } - want := "https://center.conan.io/v2/files/zlib/1.3/_/_/abc123/recipe/conan_export.tgz" + want := "https://center.conan.io/v2/conans/zlib/1.3/_/_/revisions/abc123/files/conan_export.tgz" if fetcher.fetchedURL != want { t.Errorf("upstream URL = %q, want %q", fetcher.fetchedURL, want) } @@ -1190,7 +1190,7 @@ func TestConanHandler_PackageFileCacheMiss(t *testing.T) { srv := httptest.NewServer(h.Routes()) defer srv.Close() - resp, err := http.Get(srv.URL + "/v2/files/zlib/1.3/_/_/abc123/package/def456/ghi789/conan_package.tgz") + resp, err := http.Get(srv.URL + "/v2/conans/zlib/1.3/_/_/revisions/abc123/packages/def456/revisions/ghi789/files/conan_package.tgz") if err != nil { t.Fatalf("request failed: %v", err) } @@ -1200,7 +1200,7 @@ func TestConanHandler_PackageFileCacheMiss(t *testing.T) { t.Error("expected fetcher to be called on cache miss") } - want := "https://center.conan.io/v2/files/zlib/1.3/_/_/abc123/package/def456/ghi789/conan_package.tgz" + want := "https://center.conan.io/v2/conans/zlib/1.3/_/_/revisions/abc123/packages/def456/revisions/ghi789/files/conan_package.tgz" if fetcher.fetchedURL != want { t.Errorf("upstream URL = %q, want %q", fetcher.fetchedURL, want) } diff --git a/internal/handler/notfound_ecosystems_test.go b/internal/handler/notfound_ecosystems_test.go index 22cc3d2..9e229fc 100644 --- a/internal/handler/notfound_ecosystems_test.go +++ b/internal/handler/notfound_ecosystems_test.go @@ -31,7 +31,7 @@ func TestArtifactDownloadUpstreamNotFoundReturns404(t *testing.T) { func(p *Proxy) http.Handler { return NewCRANHandler(p, "http://localhost").Routes() }}, {"conda", "/conda-forge/linux-64/numpy-1.26.0-py311_0.tar.bz2", func(p *Proxy) http.Handler { return NewCondaHandler(p, "http://localhost").Routes() }}, - {"conan", "/v1/files/zlib/1.3.1/_/_/0/recipe/conan_sources.tgz", + {"conan", "/v2/conans/zlib/1.3.1/_/_/revisions/0/files/conan_sources.tgz", func(p *Proxy) http.Handler { return NewConanHandler(p, "http://localhost").Routes() }}, {"gem", "/gems/rails-7.1.0.gem", func(p *Proxy) http.Handler { return NewGemHandler(p, "http://localhost").Routes() }}, diff --git a/internal/handler/relay_test.go b/internal/handler/relay_test.go index 67af121..a7a8fb7 100644 --- a/internal/handler/relay_test.go +++ b/internal/handler/relay_test.go @@ -50,7 +50,7 @@ func relayTestRoutes(proxy *Proxy, upstream string) http.Handler { func TestRelayRoutes(t *testing.T) { for _, route := range []string{ "/upstream", "/file", "/metadata", "/nuget/query", - "/conan/v2/files/demo/1.0/user/stable/rev/recipe/other.txt", + "/conan/v2/conans/demo/1.0/user/stable/revisions/rev/files/other.txt", "/composer/search.json", "/pypi/simple/", "/gem/api/v1/dependencies", "/gem/info/demo", "/conda/conda-forge/noarch/repodata.json", "/hex/packages/demo", "/swift/scope/demo/1.0.0/Package.swift", From 3c095235f317cb3926905496dc1610f6c6db560a Mon Sep 17 00:00:00 2001 From: Christian Heim Date: Sun, 11 Oct 2026 17:57:37 +0200 Subject: [PATCH 188/633] Fetch the pinned commit for Composer branch downloads (#430) * Fetch the pinned commit for Composer branch downloads The download handler resolved a version's dist URL from the current metadata and cached the archive under whatever file name the client asked for. A lock file pinning an older commit of a branch like dev-master got the branch head, stored and served as that older commit. Downloads now fetch the commit the file name stands for when the dist URL ends in a commit hash (GitHub zipballs, Bitbucket archives), and return 404 for other names the metadata no longer lists. * Accept bare commit names from older Composer lock files Proxy versions before the .zip suffix handed out GitHub zipball URLs ending in the bare commit hash. Lock files written back then still ask for those names, and the new file name check turned them into 404s. * Rebuild Composer commit URLs only for branch versions A tagged release lists one commit and lock files ask for that one. Rebuilding the URL for any requested commit let a client store another commit, even one from a fork, as the release, and the browse view could then show it. * Note that the Composer cache key leaves out the query --- internal/handler/composer.go | 113 ++++++++--- .../handler/composer_rewrite_bench_test.go | 2 +- internal/handler/composer_rewrite_test.go | 4 +- internal/handler/composer_test.go | 186 +++++++++++++++++- 4 files changed, 271 insertions(+), 34 deletions(-) diff --git a/internal/handler/composer.go b/internal/handler/composer.go index da7a957..5135915 100644 --- a/internal/handler/composer.go +++ b/internal/handler/composer.go @@ -9,6 +9,7 @@ import ( "io" "net/http" "path" + "regexp" "strings" "time" ) @@ -329,7 +330,17 @@ func (h *ComposerHandler) proxyDistURL(packageName, version, upstreamURL, distTy if upstreamURL == "" { return "", false } + parts := strings.SplitN(packageName, "/", vendorPackageParts) + if len(parts) != vendorPackageParts { + return "", false + } + return fmt.Sprintf("%s/composer/files/%s/%s/%s/%s", + h.proxyURL, parts[0], parts[1], version, distFilename(upstreamURL, distType)), true +} +// distFilename returns the file name this proxy serves an archive under: the +// last segment of its upstream URL. +func distFilename(upstreamURL, distType string) string { filename := "package.zip" if idx := strings.LastIndex(upstreamURL, "/"); idx >= 0 { filename = upstreamURL[idx+1:] @@ -340,13 +351,36 @@ func (h *ComposerHandler) proxyDistURL(packageName, version, upstreamURL, distTy if path.Ext(filename) == "" && distType == "zip" { filename += ".zip" } + return filename +} - parts := strings.SplitN(packageName, "/", vendorPackageParts) - if len(parts) != vendorPackageParts { +// composerCommitPattern matches a full git commit hash, SHA-1 or SHA-256. +var composerCommitPattern = regexp.MustCompile(`^([0-9a-f]{40}|[0-9a-f]{64})$`) + +// upstreamDistURL returns the upstream URL of the archive served as filename, +// given the dist URL the version's metadata lists now. A branch version only +// lists its current head while lock files pin older commits, so for a branch +// a dist URL ending in a commit hash is rebuilt for the requested commit. A +// tagged release has to ask for the listed name. Any other name gives false: +// fetching the listed archive would cache it under a name it does not have. +func upstreamDistURL(listedURL, distType, filename string, branch bool) (string, bool) { + listed := distFilename(listedURL, distType) + // Proxy versions before the .zip suffix handed out bare commit names, + // and lock files written back then still ask for them. + if bare := distFilename(listedURL, ""); listed != bare && path.Ext(filename) == "" { + filename += listed[len(bare):] + } + if filename == listed { + return listedURL, true + } + ext := path.Ext(listed) + commit := strings.TrimSuffix(listed, ext) + requested, ok := strings.CutSuffix(filename, ext) + if !branch || !ok || !composerCommitPattern.MatchString(commit) || !composerCommitPattern.MatchString(requested) { return "", false } - return fmt.Sprintf("%s/composer/files/%s/%s/%s/%s", - h.proxyURL, parts[0], parts[1], version, filename), true + idx := strings.LastIndex(listedURL, commit) + return listedURL[:idx] + requested + listedURL[idx+len(commit):], true } // composerFields holds one version's fields as Composer sees them after @@ -449,16 +483,16 @@ func (h *ComposerHandler) handleDownload(w http.ResponseWriter, r *http.Request) "package", packageName, "version", version, "metadata_urls", metaURLs) - var downloadURL string + var downloadURL, distType string for _, metaURL := range metaURLs { - url, err := h.findDownloadURLFromMetadata(r.Context(), metaURL, packageName, version) + url, typ, err := h.findDownloadURLFromMetadata(r.Context(), metaURL, packageName, version) if err != nil { h.proxy.Logger.Error("failed to fetch metadata", "error", err, "url", metaURL) http.Error(w, "failed to fetch metadata", http.StatusBadGateway) return } if url != "" { - downloadURL = url + downloadURL, distType = url, typ break } } @@ -471,6 +505,24 @@ func (h *ComposerHandler) handleDownload(w http.ResponseWriter, r *http.Request) return } + // The archive is cached under the file name from the path, so fetch the + // one that name stands for. Compare it as sent, the way rewriteDist handed + // it out. A query such as GitLab's ?sha= is compared but is not part of + // the cache key. + requested := path.Base(r.URL.EscapedPath()) + if r.URL.RawQuery != "" { + requested += "?" + r.URL.RawQuery + } + listedURL := downloadURL + downloadURL, ok := upstreamDistURL(listedURL, distType, requested, isDevVersion(version)) + if !ok { + h.proxy.Logger.Info("composer file not in upstream metadata", + "package", packageName, "version", version, + "filename", requested, "listed_url", listedURL) + http.Error(w, "file not found", http.StatusNotFound) + return + } + h.proxy.Logger.Debug("resolved download URL", "package", packageName, "version", version, "download_url", downloadURL) @@ -507,21 +559,22 @@ func (h *ComposerHandler) metadataURLsForVersion(vendor, pkg, version string) [] } // findDownloadURLFromMetadata fetches a metadata document and returns the dist -// URL for the given version, or an empty string if the version is not present. -// An error is returned only on transport failure; a missing document (non-200) -// or a missing version both yield an empty string so the caller can fall back. -func (h *ComposerHandler) findDownloadURLFromMetadata(ctx context.Context, metaURL, packageName, version string) (string, error) { +// URL and type for the given version, or an empty URL if the version is not +// present. An error is returned only on transport failure; a missing document +// (non-200) or a missing version both yield an empty URL so the caller can +// fall back. +func (h *ComposerHandler) findDownloadURLFromMetadata(ctx context.Context, metaURL, packageName, version string) (string, string, error) { h.proxy.Logger.Debug("fetching upstream metadata for download lookup", "url", metaURL, "package", packageName, "version", version) req, err := http.NewRequestWithContext(ctx, http.MethodGet, metaURL, nil) if err != nil { - return "", err + return "", "", err } resp, err := h.proxy.HTTPClient.Do(req) if err != nil { - return "", err + return "", "", err } defer func() { _ = resp.Body.Close() }() @@ -529,42 +582,42 @@ func (h *ComposerHandler) findDownloadURLFromMetadata(ctx context.Context, metaU "url", metaURL, "status", resp.StatusCode) if resp.StatusCode != http.StatusOK { - return "", nil + return "", "", nil } body, err := io.ReadAll(resp.Body) if err != nil { - return "", err + return "", "", err } - url, err := composerDistURL(body, packageName, version) + url, distType, err := composerDistURL(body, packageName, version) if err != nil { - return "", err + return "", "", err } h.proxy.Logger.Debug("download URL lookup result", "url", metaURL, "package", packageName, "version", version, "download_url", url) - return url, nil + return url, distType, nil } -// composerDistURL returns the upstream dist URL of version from Composer -// metadata, or "" when the package has no such version or it has no dist URL. -// It walks the version list in place, expanding only as much of the minified -// format as it needs, since it runs on each archive the proxy has not cached -// yet. -func composerDistURL(body []byte, packageName, version string) (string, error) { +// composerDistURL returns the upstream dist URL and type of version from +// Composer metadata, or "" when the package has no such version or it has no +// dist URL. It walks the version list in place, expanding only as much of the +// minified format as it needs, since it runs on each archive the proxy has not +// cached yet. +func composerDistURL(body []byte, packageName, version string) (string, string, error) { format, _, err := lookupJSONString(body, "minified") if err != nil { - return "", err + return "", "", err } versions, err := lookupJSON(body, "packages", packageName) if err != nil || len(versions) == 0 || versions[0] != '[' { - return "", err + return "", "", err } minified := format == composerMinified fields := newComposerFields() - var url string + var url, distType string err = forEachJSONElement(versions, func(entry []byte) error { if minified && jsonStringIs(entry, composerDevReset) { fields.reset() @@ -582,8 +635,10 @@ func composerDistURL(body []byte, packageName, version string) (string, error) { if !jsonStringIs(fields.get("version"), version) { return nil } - url, _, _ = lookupJSONString(fields.get("dist"), "url") + dist := fields.get("dist") + url, _, _ = lookupJSONString(dist, "url") if url != "" { + distType, _, _ = lookupJSONString(dist, "type") return errStopScan } return nil @@ -591,7 +646,7 @@ func composerDistURL(body []byte, packageName, version string) (string, error) { if errors.Is(err, errStopScan) { err = nil } - return url, err + return url, distType, err } // proxyUpstream forwards a request to packagist.org without caching. diff --git a/internal/handler/composer_rewrite_bench_test.go b/internal/handler/composer_rewrite_bench_test.go index dd1ba57..9f0ab7b 100644 --- a/internal/handler/composer_rewrite_bench_test.go +++ b/internal/handler/composer_rewrite_bench_test.go @@ -73,7 +73,7 @@ func BenchmarkComposerFindDownloadURL(b *testing.B) { b.SetBytes(int64(len(body))) b.ReportAllocs() for b.Loop() { - url, err := h.findDownloadURLFromMetadata(context.Background(), srv.URL, "big/sdk", "3.2500.0") + url, _, err := h.findDownloadURLFromMetadata(context.Background(), srv.URL, "big/sdk", "3.2500.0") if err != nil || url == "" { b.Fatalf("download URL not found: %q, %v", url, err) } diff --git a/internal/handler/composer_rewrite_test.go b/internal/handler/composer_rewrite_test.go index 9d2bb51..c484e1a 100644 --- a/internal/handler/composer_rewrite_test.go +++ b/internal/handler/composer_rewrite_test.go @@ -240,7 +240,7 @@ func TestComposerDistURL(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, err := composerDistURL([]byte(tt.body), "v/p", tt.version) + got, _, err := composerDistURL([]byte(tt.body), "v/p", tt.version) if err != nil { t.Fatal(err) } @@ -252,7 +252,7 @@ func TestComposerDistURL(t *testing.T) { } func TestComposerDistURLMalformed(t *testing.T) { - if _, err := composerDistURL([]byte(`{"packages":{"v/p":[{"version":`), "v/p", "1.0.0"); err == nil { + if _, _, err := composerDistURL([]byte(`{"packages":{"v/p":[{"version":`), "v/p", "1.0.0"); err == nil { t.Error("expected an error for truncated JSON") } } diff --git a/internal/handler/composer_test.go b/internal/handler/composer_test.go index f6f077c..f47550e 100644 --- a/internal/handler/composer_test.go +++ b/internal/handler/composer_test.go @@ -3,6 +3,8 @@ package handler import ( "context" "encoding/json" + "fmt" + "io" "log/slog" "net/http" "net/http/httptest" @@ -11,6 +13,7 @@ import ( "time" "github.com/git-pkgs/cooldown" + "github.com/git-pkgs/registries/fetch" ) func TestComposerRewriteMetadata(t *testing.T) { @@ -489,7 +492,7 @@ func TestComposerDownloadDevVersionUsesDevMetadata(t *testing.T) { // OLD behavior: fetching only the regular file fails to resolve the dev // version, which is what produced the 404 before the fix. stableURL := srv.URL + "/p2/phpmd/phpmd.json" - got, err := h.findDownloadURLFromMetadata(ctx, stableURL, pkg, version) + got, _, err := h.findDownloadURLFromMetadata(ctx, stableURL, pkg, version) if err != nil { t.Fatalf("unexpected error fetching regular metadata: %v", err) } @@ -507,7 +510,7 @@ func TestComposerDownloadDevVersionUsesDevMetadata(t *testing.T) { var resolved string for _, u := range urls { - resolved, err = h.findDownloadURLFromMetadata(ctx, u, pkg, version) + resolved, _, err = h.findDownloadURLFromMetadata(ctx, u, pkg, version) if err != nil { t.Fatalf("unexpected error fetching metadata %q: %v", u, err) } @@ -521,6 +524,185 @@ func TestComposerDownloadDevVersionUsesDevMetadata(t *testing.T) { } } +const ( + composerCommitA = "1111111111111111111111111111111111111111" + composerCommitB = "2222222222222222222222222222222222222222" +) + +// setupComposerBranch returns a caching Composer handler whose upstream lists +// distPath as the dist of v/p's dev-master and 1.0.0 and serves archives by +// request URI. +func setupComposerBranch(t *testing.T, distPath string, archives map[string]string) (http.Handler, *Proxy) { + t.Helper() + var srv *httptest.Server + srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/p2/v/p.json" || r.URL.Path == "/p2/v/p~dev.json" { + dist := fmt.Sprintf(`"dist":{"url":%q,"type":"zip"}`, srv.URL+distPath) + _, _ = fmt.Fprintf(w, `{"packages":{"v/p":[{"version":"dev-master",%s},{"version":"1.0.0",%s}]}}`, dist, dist) + return + } + body, ok := archives[r.URL.RequestURI()] + if !ok { + http.NotFound(w, r) + return + } + _, _ = io.WriteString(w, body) + })) + t.Cleanup(srv.Close) + + proxy, _, _, _ := setupTestProxy(t) + fetcher := fetch.NewFetcher(fetch.WithHTTPClient(srv.Client()), fetch.WithMaxRetries(0)) + proxy.Fetcher = fetcher + t.Cleanup(func() { _ = fetcher.Close() }) + return NewComposerHandlerWithUpstreams(proxy, "http://proxy", srv.URL, srv.URL).Routes(), proxy +} + +// cachedComposerFile returns the cached bytes of v/p dev-master's filename, +// or "" when nothing is cached under it. +func cachedComposerFile(t *testing.T, proxy *Proxy, filename string) string { + t.Helper() + cached, err := proxy.GetCachedArtifact(context.Background(), "composer", "v/p", "dev-master", filename) + if err != nil { + t.Fatalf("reading cache for %s: %v", filename, err) + } + if cached == nil { + return "" + } + defer func() { _ = cached.Reader.Close() }() + data, err := io.ReadAll(cached.Reader) + if err != nil { + t.Fatalf("reading cached %s: %v", filename, err) + } + return string(data) +} + +// TestComposerDownloadPinnedBranchCommit checks that a lock file pinning an +// older commit of a branch gets that commit rather than the head the branch's +// metadata lists now, and that the head is not cached under the older name. +func TestComposerDownloadPinnedBranchCommit(t *testing.T) { + tests := []struct { + name, distPath, filename string + }{ + {"github zipball", "/repos/v/p/zipball/%s", "%s.zip"}, + {"github zipball without .zip", "/repos/v/p/zipball/%s", "%s"}, + {"bitbucket archive", "/v/p/get/%s.zip", "%s.zip"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + h, proxy := setupComposerBranch(t, fmt.Sprintf(tt.distPath, composerCommitB), map[string]string{ + fmt.Sprintf(tt.distPath, composerCommitA): "commit A", + fmt.Sprintf(tt.distPath, composerCommitB): "commit B", + }) + for _, c := range []struct{ commit, want string }{ + {composerCommitA, "commit A"}, + {composerCommitB, "commit B"}, + } { + filename, want := fmt.Sprintf(tt.filename, c.commit), c.want + w := httptest.NewRecorder() + h.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/files/v/p/dev-master/"+filename, nil)) + if w.Code != http.StatusOK || w.Body.String() != want { + t.Errorf("%s: status %d, body %q, want 200 %q", filename, w.Code, w.Body.String(), want) + } + if got := cachedComposerFile(t, proxy, filename); got != want { + t.Errorf("%s cached as %q, want %q", filename, got, want) + } + } + }) + } +} + +// TestComposerDownloadUnlistedFile checks that a file the metadata no longer +// lists, and that names no commit to fetch instead, is a 404 rather than the +// listed archive cached under the requested name. +func TestComposerDownloadUnlistedFile(t *testing.T) { + tests := []struct { + name, distPath, listed, stale string + }{ + {"plain name", "/dist/main.zip", "main.zip", "old.zip"}, + {"escaped name", "/dist/p%2Bdev-main.zip", "p%2Bdev-main.zip", "old.zip"}, + {"gitlab archive", "/api/v4/projects/v%2Fp/repository/archive.zip?sha=" + composerCommitB, + "archive.zip?sha=" + composerCommitB, "archive.zip?sha=" + composerCommitA}, + {"branch name for a commit", "/repos/v/p/zipball/" + composerCommitB, composerCommitB + ".zip", "master.zip"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + h, proxy := setupComposerBranch(t, tt.distPath, map[string]string{tt.distPath: "head"}) + + w := httptest.NewRecorder() + h.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/files/v/p/dev-master/"+tt.stale, nil)) + if w.Code != http.StatusNotFound { + t.Errorf("%s: status %d, body %q, want 404", tt.stale, w.Code, w.Body.String()) + } + staleName, _, _ := strings.Cut(tt.stale, "?") + if got := cachedComposerFile(t, proxy, staleName); got != "" { + t.Errorf("%s cached as %q, want nothing", staleName, got) + } + + w = httptest.NewRecorder() + h.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/files/v/p/dev-master/"+tt.listed, nil)) + if w.Code != http.StatusOK || w.Body.String() != "head" { + t.Errorf("%s: status %d, body %q, want 200 %q", tt.listed, w.Code, w.Body.String(), "head") + } + }) + } +} + +// TestComposerDownloadTaggedReleaseCommit checks that a tagged release is only +// served as the commit its metadata lists, so a client cannot have another +// commit stored as that release. +func TestComposerDownloadTaggedReleaseCommit(t *testing.T) { + zipball := "/repos/v/p/zipball/" + h, proxy := setupComposerBranch(t, zipball+composerCommitB, map[string]string{ + zipball + composerCommitA: "commit A", + zipball + composerCommitB: "commit B", + }) + + w := httptest.NewRecorder() + h.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/files/v/p/1.0.0/"+composerCommitA+".zip", nil)) + if w.Code != http.StatusNotFound { + t.Errorf("other commit: status %d, body %q, want 404", w.Code, w.Body.String()) + } + cached, err := proxy.GetCachedArtifact(context.Background(), "composer", "v/p", "1.0.0", composerCommitA+".zip") + if err != nil || cached != nil { + t.Errorf("other commit cached as 1.0.0: %v, %v", cached, err) + } + + w = httptest.NewRecorder() + h.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/files/v/p/1.0.0/"+composerCommitB+".zip", nil)) + if w.Code != http.StatusOK || w.Body.String() != "commit B" { + t.Errorf("listed commit: status %d, body %q, want 200 %q", w.Code, w.Body.String(), "commit B") + } +} + +func TestComposerUpstreamDistURL(t *testing.T) { + const zipball = "https://api.github.com/repos/v/p/zipball/" + sha256A := strings.Repeat("a", 64) + tests := []struct { + name, listed, distType, filename, want string + }{ + {"listed file", zipball + composerCommitB, "zip", composerCommitB + ".zip", zipball + composerCommitB}, + {"older commit", zipball + composerCommitB, "zip", composerCommitA + ".zip", zipball + composerCommitA}, + {"sha256 commit", zipball + composerCommitB, "zip", sha256A + ".zip", zipball + sha256A}, + {"bare listed commit", zipball + composerCommitB, "zip", composerCommitB, zipball + composerCommitB}, + {"bare older commit", zipball + composerCommitB, "zip", composerCommitA, zipball + composerCommitA}, + {"tar dist keeps bare name", zipball + composerCommitB, "tar", composerCommitA, zipball + composerCommitA}, + {"short hash", zipball + composerCommitB, "zip", "1111111.zip", ""}, + {"upper-case hash", zipball + composerCommitB, "zip", strings.ToUpper(sha256A) + ".zip", ""}, + {"other name", "https://example.com/dist/main.zip", "zip", "old.zip", ""}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, ok := upstreamDistURL(tt.listed, tt.distType, tt.filename, true) + if got != tt.want || ok != (tt.want != "") { + t.Errorf("upstreamDistURL(%q, %q) = %q, %v; want %q", tt.listed, tt.filename, got, ok, tt.want) + } + }) + } + if got, ok := upstreamDistURL(zipball+composerCommitB, "zip", composerCommitA+".zip", false); ok { + t.Errorf("tagged release rebuilt to %q, want only the listed archive", got) + } +} + func TestComposerRewriteMetadataCooldown(t *testing.T) { now := time.Now() old := now.Add(-10 * 24 * time.Hour).Format(time.RFC3339) From 3adae30f7eabb87893feb36febdbce3ac4bab13d Mon Sep 17 00:00:00 2001 From: Christian Heim Date: Sun, 11 Oct 2026 17:58:45 +0200 Subject: [PATCH 189/633] Explain APT signing keys and add a deb822 example (#429) Signed indexes pass through unchanged, so apt keeps using the distro's archive keyring. No proxy key needed. --- README.md | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 083f18b..c9cb887 100644 --- a/README.md +++ b/README.md @@ -547,12 +547,31 @@ Configure APT to use the proxy in `/etc/apt/sources.list.d/proxy.list`: deb http://localhost:8080/debian stable main contrib ``` -Replace your existing sources.list entries, then: +Or in deb822 format, in `/etc/apt/sources.list.d/proxy.sources`: + +``` +Types: deb +URIs: http://localhost:8080/debian +Suites: stable +Components: main contrib +Signed-By: /usr/share/keyrings/debian-archive-keyring.gpg +``` + +Replace your existing sources.list entries (on newer releases also +`debian.sources` or `ubuntu.sources` in `/etc/apt/sources.list.d/`), then: ```bash sudo apt update ``` +There is no proxy key to install. The proxy serves the upstream's signed +`InRelease` and `Release` files unchanged, so APT verifies them with the +upstream archive's own key from the distribution's keyring package: +`debian-archive-keyring` on Debian, `ubuntu-keyring` on Ubuntu +(`/usr/share/keyrings/ubuntu-archive-keyring.gpg`). `Signed-By:` in a +`.sources` file, or `[signed-by=...]` in a `.list` line, restricts the source +to that key. For a third-party upstream, use the key its vendor publishes. + The upstream defaults to `http://deb.debian.org/debian`. To proxy a different APT repository (e.g. Ubuntu), set `upstream.debian` in the config file or `PROXY_UPSTREAM_DEBIAN` in the environment: ```yaml From 832a44178110a5db60eb274c1d1cf367ea935147 Mon Sep 17 00:00:00 2001 From: simon chrzanowski Date: Sun, 11 Oct 2026 19:06:18 +0200 Subject: [PATCH 190/633] Add package routes for private Composer and npm registries (#426) * Add Composer package routes for private repositories - Add upstream.composer_routes mapping package name patterns to repositories - Serve matching packages only from their route, with no fallback to the default repository - Qualify metadata and artifact cache keys with the route's upstream - Give routed versions an empty notification-url so installs are not reported to notify-batch - Document routes in configuration docs, example config and README * Add npm package routes for private registries - Add upstream.npm_routes mapping package name patterns to registries - Serve matching packages only from their route, with no fallback to upstream.npm - Resolve and validate tarball URLs against the route's registry - Qualify metadata and artifact cache keys with the route's upstream - Document npm routes and the audit endpoint limitation * Share routed artifact filename qualification between handlers * Match Composer routes without the ~dev suffix Packagist serves a package's dev versions from {name}~dev.json, and the handler matched routes against the name with ~dev still attached. An exact route for example/library therefore missed /p2/example/library~dev.json, and the private package's dev metadata came from the default repository. - Match routes against the package name without ~dev, keeping it in the upstream URL and the cache key so the two files stay apart - Add a regression test with an exact route * Keep routed npm publish times out of the shared versions row The download cooldown check stores publish times under the plain version PURL, which does not say which registry a version came from. A time read from the default registry was trusted for a private version of the same name, so a private release published an hour ago passed a seven-day cooldown when the public one was a month old. - Skip the stored row for routed packages, both reading and writing, and take the publish time from the route's metadata, which is cached under the route - This also stops a private time from holding back the public version, and a route's previous upstream from lending its times to the new one - Cover adding a route after a public download, the reverse order, and moving a route to another upstream --- README.md | 4 + config.example.yaml | 10 + docs/configuration.md | 30 +++ internal/config/config.go | 34 +++ internal/config/config_test.go | 55 +++++ internal/handler/composer.go | 53 +++- internal/handler/composer_routes_test.go | 302 +++++++++++++++++++++++ internal/handler/npm.go | 71 ++++-- internal/handler/npm_routes_test.go | 296 ++++++++++++++++++++++ internal/handler/package_routes.go | 74 ++++++ internal/handler/package_routes_test.go | 60 +++++ internal/server/package_routes_test.go | 53 ++++ internal/server/server.go | 5 +- 13 files changed, 1019 insertions(+), 28 deletions(-) create mode 100644 internal/handler/composer_routes_test.go create mode 100644 internal/handler/npm_routes_test.go create mode 100644 internal/handler/package_routes.go create mode 100644 internal/handler/package_routes_test.go create mode 100644 internal/server/package_routes_test.go diff --git a/README.md b/README.md index c9cb887..06cd0ed 100644 --- a/README.md +++ b/README.md @@ -161,6 +161,8 @@ therefore come from upstream's database, not from the proxy's own vulnerability data, and versions withheld by [cooldown](#version-cooldown) are not excluded from the report. +To serve private packages through the same registry URL, see [Private package routes](docs/configuration.md#private-package-routes). + ### Cargo Create or edit `~/.cargo/config.toml`: @@ -344,6 +346,8 @@ Or set globally: composer config -g repositories.proxy composer http://localhost:8080/composer ``` +To serve private packages through the same URL, see [Private package routes](docs/configuration.md#private-package-routes). + ### Conan (C/C++) Add the proxy as a remote: diff --git a/config.example.yaml b/config.example.yaml index a34462a..d29b4ba 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -140,6 +140,11 @@ upstream: # gate on publish age, e.g. Yarn's npmMinimalAgeGate. Default: false. # npm_full_metadata: true + # npm packages served only from a private registry, by name pattern. + # Matching packages are never looked up on the npm registry above. + # npm_routes: + # "@example/*": "https://npm.example.com/registry" + # Cargo sparse index URL cargo: "https://index.crates.io" @@ -185,6 +190,11 @@ upstream: # Packagist repository URL composer_repository: "https://repo.packagist.org" + # Composer packages served only from a private repository, by name pattern. + # Matching packages are never looked up on composer_repository. + # composer_routes: + # "example/*": "https://composer.example.com/packages" + # Conan registry URL conan: "https://center.conan.io" diff --git a/docs/configuration.md b/docs/configuration.md index 331db95..675d45a 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -229,6 +229,36 @@ upstream: `upstream.hex_api` is used for cooldown timestamps and must expose Hex's `/api/packages/{name}` JSON endpoint. +### Private package routes + +Clients that use the proxy as their only registry can still install private packages. A package route sends every package whose name matches a pattern to a private registry, and never to the default upstream: + +```yaml +upstream: + composer_routes: + "example/*": "https://packages.example.com/composer" + npm_routes: + "@example/*": "https://packages.example.com/npm" + allow_private_hosts: + - "packages.example.com" + auth: + "https://packages.example.com": + type: bearer + token: "${PRIVATE_REGISTRY_TOKEN}" +``` + +- Patterns use Go's [`path.Match`](https://pkg.go.dev/path#Match) syntax and are matched case-insensitively. `*` does not cross `/`, so `example/*` matches `example/library` but not `example-fork/library`. The longest matching pattern wins. +- A Composer route URL is the repository base that serves `/p2/{vendor}/{package}.json`. A route also covers the package's dev versions in `/p2/{vendor}/{package}~dev.json`; patterns are matched against the package name without `~dev`. +- An npm route URL is the registry base that serves `/{package}`. Tarball URLs in its packuments must be on the same host and below the same path. +- A routed package is not looked up anywhere else. If the private registry does not have it, the client gets a 404. If the registry fails or rejects the credentials, the client gets a 502. A package with the same name on the public registry is never served in its place (dependency confusion). +- Metadata and artifacts of routed packages are cached under keys that include the route's URL. Entries cached from the public registry before a route was added are never served for a routed package. +- With cooldown enabled, the publish time of a routed npm version always comes from the route's metadata. Times recorded for the public package of the same name, or for a route's previous URL, are not used, and routed downloads do not record times of their own. +- Composer metadata of routed packages carries an empty `notification-url`, so Composer does not report their installs to the default upstream's `notify-batch` endpoint. +- Use `upstream.auth` for the registry's credentials. The proxy has no authentication of its own, so every client that can reach it can download routed packages. +- Search (`/composer/search.json`) and the package list (`/composer/packages/list.json`) still come from the default upstream only. +- npm audit and signing-key requests still go to `upstream.npm`. An audit request lists every package in the dependency tree, routed ones included. +- Routes can only be set in the configuration file, not with environment variables. + Helm HTTP repositories and additional OCI registries are configured as named maps: ```yaml diff --git a/internal/config/config.go b/internal/config/config.go index 2df21d8..fc314bd 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -66,6 +66,7 @@ import ( "net" "net/url" "os" + "path" "path/filepath" "sort" "strconv" @@ -512,6 +513,13 @@ type UpstreamConfig struct { // Default: false (the abbreviated format is preferred). NPMFullMetadata bool `json:"npm_full_metadata" yaml:"npm_full_metadata"` + // NPMRoutes maps package name patterns to npm registry URLs. A package + // whose name matches a pattern is fetched only from that registry, never + // from upstream.npm, so a public package cannot stand in for it. Patterns + // use path.Match syntax, e.g. "@example/*"; the longest matching pattern + // wins. + NPMRoutes map[string]string `json:"npm_routes" yaml:"npm_routes"` + // Cargo is the upstream cargo index URL. // Default: https://index.crates.io Cargo string `json:"cargo" yaml:"cargo"` @@ -573,6 +581,14 @@ type UpstreamConfig struct { // Default: https://repo.packagist.org ComposerRepository string `json:"composer_repository" yaml:"composer_repository"` + // ComposerRoutes maps package name patterns to Composer repository URLs + // (the base that serves /p2/{vendor}/{package}.json). A package whose + // name matches a pattern is fetched only from that repository, never + // from upstream.composer_repository, so a public package cannot stand in + // for it. Patterns use path.Match syntax, e.g. "example/*"; the longest + // matching pattern wins. + ComposerRoutes map[string]string `json:"composer_routes" yaml:"composer_routes"` + // Conan is the upstream Conan registry URL. // Default: https://center.conan.io Conan string `json:"conan" yaml:"conan"` @@ -692,6 +708,12 @@ func (u *UpstreamConfig) Validate() error { if err := validateNamedUpstreams("upstream.helm", u.Helm); err != nil { return err } + if err := validatePackageRoutes("upstream.composer_routes", u.ComposerRoutes); err != nil { + return err + } + if err := validatePackageRoutes("upstream.npm_routes", u.NPMRoutes); err != nil { + return err + } if err := validateNamedUpstreams("upstream.apk", u.APK); err != nil { return err } @@ -784,6 +806,18 @@ func validateNamedUpstreams(field string, upstreams map[string]string) error { return nil } +func validatePackageRoutes(field string, routes map[string]string) error { + for pattern, upstreamURL := range routes { + if _, err := path.Match(pattern, ""); pattern == "" || err != nil { + return fmt.Errorf("invalid %s pattern %q", field, pattern) + } + if err := validateAbsoluteURL(field+"."+pattern, upstreamURL); err != nil { + return err + } + } + return nil +} + func parseAuthURL(value string) (*url.URL, error) { parsed, err := url.Parse(value) if err != nil || !parsed.IsAbs() || parsed.Hostname() == "" || parsed.Opaque != "" { diff --git a/internal/config/config_test.go b/internal/config/config_test.go index c1cc2ff..5566afd 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -1302,6 +1302,61 @@ func TestValidateUpstreamAuthURLs(t *testing.T) { }) } +func TestValidatePackageRoutes(t *testing.T) { + tests := []struct { + name string + modify func(*Config) + wantErr bool + }{ + { + name: "valid Composer and npm routes", + modify: func(cfg *Config) { + cfg.Upstream.ComposerRoutes = map[string]string{"acme/*": "https://composer.example.com/packages"} + cfg.Upstream.NPMRoutes = map[string]string{"@acme/*": "https://npm.example.com/registry"} + }, + }, + { + name: "Composer route URL is not absolute", + modify: func(cfg *Config) { + cfg.Upstream.ComposerRoutes = map[string]string{"acme/*": "composer.example.com"} + }, + wantErr: true, + }, + { + name: "npm route URL is not absolute", + modify: func(cfg *Config) { + cfg.Upstream.NPMRoutes = map[string]string{"@acme/*": "npm.example.com"} + }, + wantErr: true, + }, + { + name: "Composer route pattern is malformed", + modify: func(cfg *Config) { + cfg.Upstream.ComposerRoutes = map[string]string{"acme/[": "https://composer.example.com"} + }, + wantErr: true, + }, + { + name: "npm route pattern is empty", + modify: func(cfg *Config) { + cfg.Upstream.NPMRoutes = map[string]string{"": "https://npm.example.com"} + }, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cfg := Default() + tt.modify(cfg) + err := cfg.Validate() + if (err != nil) != tt.wantErr { + t.Errorf("Validate() error = %v, wantErr %t", err, tt.wantErr) + } + }) + } +} + func TestValidateNamedUpstreams(t *testing.T) { tests := []struct { name string diff --git a/internal/handler/composer.go b/internal/handler/composer.go index 5135915..182fe7a 100644 --- a/internal/handler/composer.go +++ b/internal/handler/composer.go @@ -20,6 +20,7 @@ const ( composerMinified = "composer/2.0" composerUnset = "__unset" composerDevReset = "~dev" + composerDevFile = "~dev" // suffix of the metadata file holding dev versions vendorPackageParts = 2 ) @@ -31,6 +32,7 @@ type ComposerHandler struct { upstreamURL string repoURL string proxyURL string + routes packageRoutes } // NewComposerHandler creates a new Composer protocol handler. @@ -52,6 +54,13 @@ func NewComposerHandlerWithUpstreams(proxy *Proxy, proxyURL, upstreamURL, repoUR return h } +// WithPackageRoutes sends packages matching a pattern to a dedicated Composer +// repository instead of the default one. See packageRoutes. +func (h *ComposerHandler) WithPackageRoutes(routes map[string]string) *ComposerHandler { + h.routes = newPackageRoutes(routes) + return h +} + // Routes returns the HTTP handler for Composer requests. func (h *ComposerHandler) Routes() http.Handler { mux := http.NewServeMux() @@ -87,6 +96,18 @@ func (h *ComposerHandler) handleServiceIndex(w http.ResponseWriter, r *http.Requ _ = json.NewEncoder(w).Encode(index) } +// sourceFor returns the repository serving a package and the key its metadata +// is cached under. packageName may carry the ~dev suffix of the file holding +// the package's dev versions: routes match the package without it, so its dev +// versions come from the same repository, while the cache key keeps it so the +// two files stay apart. +func (h *ComposerHandler) sourceFor(packageName string) (repoURL, cacheKey string) { + if route, routed := h.routes.match(strings.TrimSuffix(packageName, composerDevFile)); routed { + return route.url, route.cacheKey(packageName) + } + return h.repoURL, packageName +} + // handlePackageMetadata proxies and rewrites package metadata. func (h *ComposerHandler) handlePackageMetadata(w http.ResponseWriter, r *http.Request) { // Parse path: /p2/{vendor}/{package}.json @@ -103,15 +124,16 @@ func (h *ComposerHandler) handlePackageMetadata(w http.ResponseWriter, r *http.R h.proxy.Logger.Info("composer metadata request", "package", packageName) - upstreamURL := fmt.Sprintf("%s/p2/%s/%s.json", h.repoURL, vendor, pkg) + repoURL, cacheKey := h.sourceFor(packageName) + upstreamURL := fmt.Sprintf("%s/p2/%s/%s.json", repoURL, vendor, pkg) - if rewritten, ok := h.proxy.storedRewrite("composer", packageName, h.proxyURL, packageName); ok { + if rewritten, ok := h.proxy.storedRewrite("composer", cacheKey, h.proxyURL, cacheKey); ok { w.Header().Set(headerContentType, "application/json") _, _ = w.Write(rewritten) return } - body, _, err := h.proxy.FetchOrCacheMetadata(r.Context(), "composer", packageName, upstreamURL) + body, _, err := h.proxy.FetchOrCacheMetadata(r.Context(), "composer", cacheKey, upstreamURL) if err != nil { if errors.Is(err, ErrUpstreamNotFound) { http.Error(w, "not found", http.StatusNotFound) @@ -122,7 +144,7 @@ func (h *ComposerHandler) handlePackageMetadata(w http.ResponseWriter, r *http.R return } - rewritten, err := h.proxy.cachedRewrite(r.Context(), "composer", h.proxyURL, packageName, body, h.rewriteMetadata) + rewritten, err := h.proxy.cachedRewrite(r.Context(), "composer", h.proxyURL, cacheKey, body, h.rewriteMetadata) if err != nil { if r.Context().Err() != nil { return // the client left while waiting on a shared rewrite @@ -193,6 +215,7 @@ func (h *ComposerHandler) writeVersions(out *bytes.Buffer, packageName string, v packagePURL := canonicalPackagePURL("composer", packageName) upstream, written := newComposerFields(), newComposerFields() devReset, first := false, true + _, routed := h.routes.match(packageName) out.WriteByte('[') err := forEachJSONElement(versions, func(entry []byte) error { @@ -226,6 +249,9 @@ func (h *ComposerHandler) writeVersions(out *bytes.Buffer, packageName string, v out.WriteString(`"` + composerDevReset + `",`) devReset = false } + if routed { + disableDefaultNotification(upstream) + } h.writeVersion(out, packageName, version, upstream, written) return nil }) @@ -297,6 +323,17 @@ func (h *ComposerHandler) shouldFilterVersion(packagePURL, packageName, version return false } +// disableDefaultNotification stops Composer from reporting installs of a routed +// package to the default repository's notify-batch URL, which would disclose +// the package name to it. Composer only falls back to notify-batch when a +// version has no notification-url of its own, and skips an empty one. Setting +// it on the expanded fields keeps it set for the versions that inherit them. +func disableDefaultNotification(fields *composerFields) { + if fields.get("notification-url") == nil { + fields.set("notification-url", []byte(`"notification-url"`), []byte(`""`)) + } +} + // rewriteDist returns a version's dist object with its url pointing at this // proxy. A dist without a string url is returned unchanged. func (h *ComposerHandler) rewriteDist(packageName, version string, dist []byte) []byte { @@ -527,7 +564,8 @@ func (h *ComposerHandler) handleDownload(w http.ResponseWriter, r *http.Request) "package", packageName, "version", version, "download_url", downloadURL) - result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "composer", packageName, version, filename, downloadURL) + cacheFilename := h.routes.cacheFilename(packageName, filename) + result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "composer", packageName, version, cacheFilename, downloadURL) if err != nil { h.proxy.serveArtifactError(w, err, "failed to fetch package") return @@ -549,8 +587,9 @@ func isDevVersion(version string) bool { // file, tagged releases from the regular file; the other file is included as a // fallback so an unexpected classification still resolves. func (h *ComposerHandler) metadataURLsForVersion(vendor, pkg, version string) []string { - stable := fmt.Sprintf("%s/p2/%s/%s.json", h.repoURL, vendor, pkg) - dev := fmt.Sprintf("%s/p2/%s/%s~dev.json", h.repoURL, vendor, pkg) + repoURL, _ := h.sourceFor(vendor + "/" + pkg) + stable := fmt.Sprintf("%s/p2/%s/%s.json", repoURL, vendor, pkg) + dev := fmt.Sprintf("%s/p2/%s/%s~dev.json", repoURL, vendor, pkg) if isDevVersion(version) { return []string{dev, stable} diff --git a/internal/handler/composer_routes_test.go b/internal/handler/composer_routes_test.go new file mode 100644 index 0000000..5a65440 --- /dev/null +++ b/internal/handler/composer_routes_test.go @@ -0,0 +1,302 @@ +package handler + +import ( + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + "time" + + "github.com/git-pkgs/registries/fetch" +) + +// composerUpstreamRecorder serves canned Composer metadata and records which +// paths were requested. +type composerUpstreamRecorder struct { + *httptest.Server + mu sync.Mutex + requests []string +} + +func newComposerUpstream(t *testing.T, documents map[string]string, status int) *composerUpstreamRecorder { + t.Helper() + u := &composerUpstreamRecorder{} + u.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + u.mu.Lock() + u.requests = append(u.requests, r.URL.Path) + u.mu.Unlock() + if status != http.StatusOK { + w.WriteHeader(status) + return + } + body, ok := documents[r.URL.Path] + if !ok { + http.NotFound(w, r) + return + } + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, body) + })) + t.Cleanup(u.Close) + return u +} + +func (u *composerUpstreamRecorder) requested() []string { + u.mu.Lock() + defer u.mu.Unlock() + return append([]string(nil), u.requests...) +} + +const routedComposerMetadata = `{ + "packages": { + "example/library": [ + {"name": "example/library", "version": "1.0.0", "dist": {"url": "https://private.example.com/archives/library-1.0.0.zip", "type": "zip"}} + ] + } +}` + +const publicComposerMetadata = `{ + "packages": { + "example/library": [ + {"name": "example/library", "version": "9.9.9", "dist": {"url": "https://public.example.com/library-9.9.9.zip", "type": "zip"}} + ] + } +}` + +func newRoutedComposerHandler(t *testing.T, proxy *Proxy, public, private *composerUpstreamRecorder) *ComposerHandler { + t.Helper() + return NewComposerHandlerWithUpstreams(proxy, "http://proxy.test", public.URL, public.URL). + WithPackageRoutes(map[string]string{"example/*": private.URL + "/group/packages/composer"}) +} + +func serveComposer(h *ComposerHandler, target string) *httptest.ResponseRecorder { + w := httptest.NewRecorder() + h.Routes().ServeHTTP(w, httptest.NewRequest(http.MethodGet, target, nil)) + return w +} + +func TestComposerRoutedMetadataComesFromRouteOnly(t *testing.T) { + public := newComposerUpstream(t, map[string]string{"/p2/example/library.json": publicComposerMetadata}, http.StatusOK) + private := newComposerUpstream(t, map[string]string{"/group/packages/composer/p2/example/library.json": routedComposerMetadata}, http.StatusOK) + proxy, _, _, _ := setupTestProxy(t) + h := newRoutedComposerHandler(t, proxy, public, private) + + w := serveComposer(h, "/p2/example/library.json") + + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want %d; body: %s", w.Code, http.StatusOK, w.Body.String()) + } + if got := public.requested(); len(got) != 0 { + t.Errorf("default repository was queried for a routed package: %v", got) + } + + var metadata map[string]any + if err := json.Unmarshal(w.Body.Bytes(), &metadata); err != nil { + t.Fatalf("parsing response: %v", err) + } + versions := metadata["packages"].(map[string]any)["example/library"].([]any) + version := versions[0].(map[string]any) + if version["version"] != "1.0.0" { + t.Errorf("version = %v, want the routed upstream's 1.0.0", version["version"]) + } + wantDist := "http://proxy.test/composer/files/example/library/1.0.0/library-1.0.0.zip" + if dist := version["dist"].(map[string]any)["url"]; dist != wantDist { + t.Errorf("dist url = %v, want %q", dist, wantDist) + } + if notification, ok := version["notification-url"]; !ok || notification != "" { + t.Errorf("notification-url = %v (present %t), want empty so Composer does not report routed installs to notify-batch", notification, ok) + } +} + +func TestComposerRoutedMetadataDoesNotFallBack(t *testing.T) { + tests := []struct { + name string + privateStatus int + wantStatus int + }{ + {"package missing on route", http.StatusOK, http.StatusNotFound}, + {"route rejects credentials", http.StatusUnauthorized, http.StatusBadGateway}, + {"route unavailable", http.StatusInternalServerError, http.StatusBadGateway}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + public := newComposerUpstream(t, map[string]string{"/p2/example/library.json": publicComposerMetadata}, http.StatusOK) + private := newComposerUpstream(t, map[string]string{}, tt.privateStatus) + proxy, _, _, _ := setupTestProxy(t) + h := newRoutedComposerHandler(t, proxy, public, private) + + w := serveComposer(h, "/p2/example/library.json") + + if w.Code != tt.wantStatus { + t.Errorf("status = %d, want %d", w.Code, tt.wantStatus) + } + if got := public.requested(); len(got) != 0 { + t.Errorf("default repository was queried for a routed package: %v", got) + } + }) + } +} + +func TestComposerRoutedMetadataIgnoresDefaultRepositoryCache(t *testing.T) { + public := newComposerUpstream(t, map[string]string{"/p2/example/library.json": publicComposerMetadata}, http.StatusOK) + private := newComposerUpstream(t, map[string]string{"/group/packages/composer/p2/example/library.json": routedComposerMetadata}, http.StatusOK) + proxy, _, _, _ := setupTestProxy(t) + proxy.CacheMetadata = true + proxy.MetadataTTL = time.Hour + + unrouted := NewComposerHandlerWithUpstreams(proxy, "http://proxy.test", public.URL, public.URL) + if w := serveComposer(unrouted, "/p2/example/library.json"); w.Code != http.StatusOK { + t.Fatalf("priming the default repository cache: status %d", w.Code) + } + + w := serveComposer(newRoutedComposerHandler(t, proxy, public, private), "/p2/example/library.json") + + if strings.Contains(w.Body.String(), "9.9.9") { + t.Errorf("routed package served metadata cached from the default repository: %s", w.Body.String()) + } + if len(private.requested()) == 0 { + t.Error("routed upstream was not queried") + } +} + +func TestComposerUnroutedMetadataUsesDefaultRepository(t *testing.T) { + public := newComposerUpstream(t, map[string]string{"/p2/other/library.json": strings.ReplaceAll(publicComposerMetadata, "example/", "other/")}, http.StatusOK) + private := newComposerUpstream(t, map[string]string{}, http.StatusOK) + proxy, _, _, _ := setupTestProxy(t) + h := newRoutedComposerHandler(t, proxy, public, private) + + w := serveComposer(h, "/p2/other/library.json") + + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want %d", w.Code, http.StatusOK) + } + if got := private.requested(); len(got) != 0 { + t.Errorf("route was queried for an unrouted package: %v", got) + } + if strings.Contains(w.Body.String(), "notification-url") { + t.Errorf("unrouted package metadata gained a notification-url: %s", w.Body.String()) + } +} + +func TestComposerRoutedDownloadResolvesFromRoute(t *testing.T) { + public := newComposerUpstream(t, map[string]string{"/p2/example/library.json": publicComposerMetadata}, http.StatusOK) + private := newComposerUpstream(t, map[string]string{"/group/packages/composer/p2/example/library.json": routedComposerMetadata}, http.StatusOK) + proxy, _, _, artifactFetcher := setupTestProxy(t) + proxy.HTTPClient = http.DefaultClient + artifactFetcher.artifact = &fetch.Artifact{ + Body: io.NopCloser(strings.NewReader("archive")), + ContentType: "application/zip", + } + h := newRoutedComposerHandler(t, proxy, public, private) + + w := serveComposer(h, "/files/example/library/1.0.0/library-1.0.0.zip") + + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want %d; body: %s", w.Code, http.StatusOK, w.Body.String()) + } + if want := "https://private.example.com/archives/library-1.0.0.zip"; artifactFetcher.fetchedURL != want { + t.Errorf("fetched URL = %q, want %q", artifactFetcher.fetchedURL, want) + } + if got := public.requested(); len(got) != 0 { + t.Errorf("default repository was queried for a routed download: %v", got) + } +} + +func TestComposerRoutedDownloadIgnoresDefaultRepositoryArtifact(t *testing.T) { + public := newComposerUpstream(t, map[string]string{"/p2/example/library.json": strings.ReplaceAll(publicComposerMetadata, "9.9.9", "1.0.0")}, http.StatusOK) + private := newComposerUpstream(t, map[string]string{"/group/packages/composer/p2/example/library.json": routedComposerMetadata}, http.StatusOK) + proxy, _, _, artifactFetcher := setupTestProxy(t) + proxy.HTTPClient = http.DefaultClient + artifactFetcher.artifact = &fetch.Artifact{ + Body: io.NopCloser(strings.NewReader("public archive")), + ContentType: "application/zip", + } + + unrouted := NewComposerHandlerWithUpstreams(proxy, "http://proxy.test", public.URL, public.URL) + if w := serveComposer(unrouted, "/files/example/library/1.0.0/library-1.0.0.zip"); w.Code != http.StatusOK { + t.Fatalf("priming the artifact cache: status %d; body: %s", w.Code, w.Body.String()) + } + + artifactFetcher.artifact = &fetch.Artifact{ + Body: io.NopCloser(strings.NewReader("private archive")), + ContentType: "application/zip", + } + w := serveComposer(newRoutedComposerHandler(t, proxy, public, private), "/files/example/library/1.0.0/library-1.0.0.zip") + + if w.Body.String() != "private archive" { + t.Errorf("routed download served %q, want the routed upstream's archive", w.Body.String()) + } +} + +// TestComposerRoutedMinifiedMetadataDisablesNotification checks that every +// version of a routed package, as Composer expands minified metadata, has an +// empty notification-url: versions that inherit their fields, versions after a +// ~dev reset, and versions after one that set its own URL and then unset it. +func TestComposerRoutedMinifiedMetadataDisablesNotification(t *testing.T) { + minified := `{"minified":"composer/2.0","packages":{"example/library":[` + + `{"name":"example/library","version":"1.0.0","dist":{"url":"https://private.example.com/a/1.0.0.zip","type":"zip"}},` + + `{"version":"1.1.0","dist":{"url":"https://private.example.com/a/1.1.0.zip","type":"zip"}},` + + `{"version":"1.2.0","notification-url":"https://private.example.com/notify","dist":{"url":"https://private.example.com/a/1.2.0.zip","type":"zip"}},` + + `{"version":"1.3.0","notification-url":"__unset","dist":{"url":"https://private.example.com/a/1.3.0.zip","type":"zip"}},` + + `"~dev",` + + `{"name":"example/library","version":"dev-main","dist":{"url":"https://private.example.com/a/main.zip","type":"zip"}}]}}` + public := newComposerUpstream(t, map[string]string{}, http.StatusOK) + private := newComposerUpstream(t, map[string]string{"/group/packages/composer/p2/example/library.json": minified}, http.StatusOK) + proxy, _, _, _ := setupTestProxy(t) + h := newRoutedComposerHandler(t, proxy, public, private) + + w := serveComposer(h, "/p2/example/library.json") + if w.Code != http.StatusOK { + t.Fatalf("status = %d; body: %s", w.Code, w.Body.String()) + } + want := map[string]string{ + "1.0.0": "", + "1.1.0": "", + "1.2.0": "https://private.example.com/notify", // the route's own URL is kept + "1.3.0": "", + "dev-main": "", + } + versions := composerVersions(t, w.Body.Bytes(), "example/library") + if len(versions) != len(want) { + t.Fatalf("versions = %v, want %d", versionNames(versions), len(want)) + } + for _, v := range versions { + name, _ := v["version"].(string) + if got, ok := v["notification-url"]; !ok || got != want[name] { + t.Errorf("%s: notification-url = %v (present %t), want %q", name, got, ok, want[name]) + } + } +} + +// TestComposerExactRouteCoversDevMetadata is a regression test: Packagist +// serves a package's dev versions from a separate {name}~dev.json file, and an +// exact route for the package must cover that file too. Matching the route +// against the name with ~dev still attached sent the private package's dev +// metadata to the default repository. +func TestComposerExactRouteCoversDevMetadata(t *testing.T) { + const devMetadata = `{"packages":{"example/library":[{"name":"example/library","version":"dev-main","dist":{"url":"https://private.example.com/archives/main.zip","type":"zip"}}]}}` + public := newComposerUpstream(t, map[string]string{"/p2/example/library~dev.json": publicComposerMetadata}, http.StatusOK) + private := newComposerUpstream(t, map[string]string{"/group/packages/composer/p2/example/library~dev.json": devMetadata}, http.StatusOK) + proxy, _, _, _ := setupTestProxy(t) + h := NewComposerHandlerWithUpstreams(proxy, "http://proxy.test", public.URL, public.URL). + WithPackageRoutes(map[string]string{"example/library": private.URL + "/group/packages/composer"}) + + w := serveComposer(h, "/p2/example/library~dev.json") + + if w.Code != http.StatusOK { + t.Fatalf("status = %d; body: %s", w.Code, w.Body.String()) + } + if got := public.requested(); len(got) != 0 { + t.Errorf("default repository was queried for a routed package's dev metadata: %v", got) + } + if got, want := private.requested(), []string{"/group/packages/composer/p2/example/library~dev.json"}; strings.Join(got, ",") != strings.Join(want, ",") { + t.Errorf("route requests = %v, want %v", got, want) + } + if !strings.Contains(w.Body.String(), "/composer/files/example/library/dev-main/") { + t.Errorf("dev metadata was not rewritten through the proxy: %s", w.Body.String()) + } +} diff --git a/internal/handler/npm.go b/internal/handler/npm.go index 31c5341..5c8fbd7 100644 --- a/internal/handler/npm.go +++ b/internal/handler/npm.go @@ -38,6 +38,7 @@ type NPMHandler struct { proxy *Proxy upstreamURL string proxyURL string // URL where this proxy is hosted + routes packageRoutes } // NewNPMHandler creates a new npm protocol handler. @@ -53,6 +54,13 @@ func NewNPMHandler(proxy *Proxy, proxyURL, upstreamURL string) *NPMHandler { } } +// WithPackageRoutes sends packages matching a pattern to a dedicated npm +// registry instead of the default one. See packageRoutes. +func (h *NPMHandler) WithPackageRoutes(routes map[string]string) *NPMHandler { + h.routes = newPackageRoutes(routes) + return h +} + // Routes returns the HTTP handler for npm requests. // Mount this at /npm on your router. func (h *NPMHandler) Routes() http.Handler { @@ -156,6 +164,15 @@ func (h *NPMHandler) handleSecurity(w http.ResponseWriter, r *http.Request) { h.proxy.relayResponse(w, r, resp, nil) } +// sourceFor returns the registry serving a package and the key its metadata is +// cached under. +func (h *NPMHandler) sourceFor(packageName string) (registryURL, cacheKey string) { + if route, routed := h.routes.match(packageName); routed { + return route.url, route.cacheKey(packageName) + } + return h.upstreamURL, packageName +} + // handlePackageMetadata proxies package metadata from upstream and rewrites tarball URLs. func (h *NPMHandler) handlePackageMetadata(w http.ResponseWriter, r *http.Request) { packageName := h.extractPackageName(r) @@ -166,7 +183,8 @@ func (h *NPMHandler) handlePackageMetadata(w http.ResponseWriter, r *http.Reques h.proxy.Logger.Info("npm metadata request", "package", packageName) - upstreamURL := fmt.Sprintf("%s/%s", h.upstreamURL, url.PathEscape(packageName)) + registryURL, cacheKey := h.sourceFor(packageName) + upstreamURL := fmt.Sprintf("%s/%s", registryURL, url.PathEscape(packageName)) // Prefer the smaller abbreviated packument format but include application/json // as a fallback so upstreams that reject the abbreviated type (e.g. JFrog @@ -180,14 +198,14 @@ func (h *NPMHandler) handlePackageMetadata(w http.ResponseWriter, r *http.Reques accept = contentTypeJSON } - if rewritten, ok := h.proxy.storedRewrite("npm", packageName, h.proxyURL, packageName); ok { + if rewritten, ok := h.proxy.storedRewrite("npm", cacheKey, h.proxyURL, cacheKey); ok { w.Header().Set(headerContentType, contentTypeJSON) w.WriteHeader(http.StatusOK) _, _ = w.Write(rewritten) return } - body, _, err := h.proxy.FetchOrCacheMetadata(r.Context(), "npm", packageName, upstreamURL, accept) + body, _, err := h.proxy.FetchOrCacheMetadata(r.Context(), "npm", cacheKey, upstreamURL, accept) if err != nil { if errors.Is(err, ErrUpstreamNotFound) { JSONError(w, http.StatusNotFound, "package not found") @@ -198,7 +216,7 @@ func (h *NPMHandler) handlePackageMetadata(w http.ResponseWriter, r *http.Reques return } - rewritten, err := h.proxy.cachedRewrite(r.Context(), "npm", h.proxyURL, packageName, body, func(b []byte) ([]byte, error) { + rewritten, err := h.proxy.cachedRewrite(r.Context(), "npm", h.proxyURL, cacheKey, body, func(b []byte) ([]byte, error) { return h.rewriteMetadata(packageName, b) }) if err != nil { @@ -585,9 +603,10 @@ func (h *NPMHandler) handleDownload(w http.ResponseWriter, r *http.Request) { JSONError(w, http.StatusForbidden, ErrVersionDenied.Error()+": "+canonicalVersionPURL("npm", packageName, version)) return } + registryURL, cacheKey := h.sourceFor(packageName) metadata := sync.OnceValues(func() ([]byte, error) { - upstreamURL := h.upstreamURL + "/" + url.PathEscape(packageName) - body, _, err := h.proxy.FetchOrCacheMetadata(r.Context(), "npm", packageName, upstreamURL, contentTypeJSON) + upstreamURL := registryURL + "/" + url.PathEscape(packageName) + body, _, err := h.proxy.FetchOrCacheMetadata(r.Context(), "npm", cacheKey, upstreamURL, contentTypeJSON) return body, err }) if h.versionInCooldown(packageName, version, metadata) { @@ -597,7 +616,7 @@ func (h *NPMHandler) handleDownload(w http.ResponseWriter, r *http.Request) { return } - result, err := h.getTarball(r, packageName, version, filename, metadata) + result, err := h.getTarball(r, registryURL, packageName, version, filename, metadata) if err != nil { switch { case errors.Is(err, ErrUpstreamNotFound): @@ -614,21 +633,22 @@ func (h *NPMHandler) handleDownload(w http.ResponseWriter, r *http.Request) { ServeArtifactRequest(w, r, result) } -func (h *NPMHandler) getTarball(r *http.Request, packageName, version, filename string, metadata func() ([]byte, error)) (*CacheResult, error) { - if cached, err := h.proxy.GetCachedArtifact(r.Context(), "npm", packageName, version, filename); err != nil || cached != nil { +func (h *NPMHandler) getTarball(r *http.Request, registryURL, packageName, version, filename string, metadata func() ([]byte, error)) (*CacheResult, error) { + cacheFilename := h.routes.cacheFilename(packageName, filename) + if cached, err := h.proxy.GetCachedArtifact(r.Context(), "npm", packageName, version, cacheFilename); err != nil || cached != nil { return cached, err } - downloadURL := fmt.Sprintf("%s/%s/-/%s", h.upstreamURL, escapeNPMDownloadPackage(packageName), url.PathEscape(filename)) + downloadURL := fmt.Sprintf("%s/%s/-/%s", registryURL, escapeNPMDownloadPackage(packageName), url.PathEscape(filename)) body, err := metadata() if err == nil { if tarball := npmVersionTarball(body, version); tarball != "" { - downloadURL, err = h.validateTarballURL(tarball) + downloadURL, err = validateTarballURL(tarball, registryURL) if err != nil { return nil, err } } } - return h.proxy.GetOrFetchArtifactFromURL(r.Context(), "npm", packageName, version, filename, downloadURL) + return h.proxy.GetOrFetchArtifactFromURL(r.Context(), "npm", packageName, version, cacheFilename, downloadURL) } // npmVersionTarball returns versions[version].dist.tarball from a packument, @@ -639,12 +659,12 @@ func npmVersionTarball(body []byte, version string) string { return tarball } -func (h *NPMHandler) validateTarballURL(raw string) (string, error) { +func validateTarballURL(raw, registryURL string) (string, error) { tarball, err := url.Parse(raw) if err != nil { return "", fmt.Errorf("parsing npm tarball URL: %w", err) } - upstream, err := url.Parse(h.upstreamURL) + upstream, err := url.Parse(registryURL) if err != nil { return "", fmt.Errorf("parsing npm upstream URL: %w", err) } @@ -673,14 +693,25 @@ func (h *NPMHandler) validateTarballURL(raw string) (string, error) { // fetched and parsed at most once per version. A version with no usable // publish time is allowed through, matching how applyCooldownFiltering // treats it. +// +// The versions row is keyed by the plain version PURL, which does not say +// which registry the version came from, so routed packages leave it alone: +// their publish time always comes from the route's own metadata, cached under +// the route. Otherwise a time read from the default registry would be trusted +// for a private version of the same name, a private time would hold back the +// public version, and moving a route to another upstream would keep the old +// one's times. func (h *NPMHandler) versionInCooldown(packageName, version string, metadata func() ([]byte, error)) bool { if h.proxy.Cooldown == nil || !h.proxy.Cooldown.Enabled() { return false } + _, routed := h.routes.match(packageName) versionPURL := canonicalVersionPURL("npm", packageName, version) - if ver, err := h.proxy.DB.GetVersionByPURL(versionPURL); err == nil && ver != nil && ver.PublishedAt.Valid { - return !h.proxy.Cooldown.IsAllowed("npm", canonicalPackagePURL("npm", packageName), ver.PublishedAt.Time) + if !routed { + if ver, err := h.proxy.DB.GetVersionByPURL(versionPURL); err == nil && ver != nil && ver.PublishedAt.Valid { + return !h.proxy.Cooldown.IsAllowed("npm", canonicalPackagePURL("npm", packageName), ver.PublishedAt.Time) + } } body, err := metadata() @@ -705,9 +736,11 @@ func (h *NPMHandler) versionInCooldown(packageName, version string, metadata fun return false } - if err := h.proxy.DB.SetVersionPublishedAt(versionPURL, canonicalPackagePURL("npm", packageName), publishedAt); err != nil { - h.proxy.Logger.Warn("cooldown: could not store npm publish time", - "package", packageName, "version", version, "error", err) + if !routed { + if err := h.proxy.DB.SetVersionPublishedAt(versionPURL, canonicalPackagePURL("npm", packageName), publishedAt); err != nil { + h.proxy.Logger.Warn("cooldown: could not store npm publish time", + "package", packageName, "version", version, "error", err) + } } return !h.proxy.Cooldown.IsAllowed("npm", canonicalPackagePURL("npm", packageName), publishedAt) diff --git a/internal/handler/npm_routes_test.go b/internal/handler/npm_routes_test.go new file mode 100644 index 0000000..a2ffb38 --- /dev/null +++ b/internal/handler/npm_routes_test.go @@ -0,0 +1,296 @@ +package handler + +import ( + "io" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + "time" + + "github.com/git-pkgs/cooldown" + "github.com/git-pkgs/registries/fetch" +) + +// npmUpstreamRecorder serves canned packuments and records which paths were +// requested. +type npmUpstreamRecorder struct { + *httptest.Server + mu sync.Mutex + requests []string +} + +func newNPMUpstream(t *testing.T, packuments map[string]func(baseURL string) string, status int) *npmUpstreamRecorder { + t.Helper() + u := &npmUpstreamRecorder{} + u.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + u.mu.Lock() + u.requests = append(u.requests, r.URL.Path) + u.mu.Unlock() + if status != http.StatusOK { + w.WriteHeader(status) + return + } + packument, ok := packuments[r.URL.Path] + if !ok { + http.NotFound(w, r) + return + } + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, packument(u.URL)) + })) + t.Cleanup(u.Close) + return u +} + +func (u *npmUpstreamRecorder) requested() []string { + u.mu.Lock() + defer u.mu.Unlock() + return append([]string(nil), u.requests...) +} + +func routedTestPackument(version, tarballBase string) func(string) string { + return func(baseURL string) string { + return `{"name":"@example/widgets","dist-tags":{"latest":"` + version + `"},"versions":{"` + version + + `":{"version":"` + version + `","dist":{"tarball":"` + baseURL + tarballBase + `/@example/widgets/-/widgets-` + version + `.tgz"}}}}` + } +} + +const routedNPMPath = "/registry/@example/widgets" + +func newRoutedNPMHandler(proxy *Proxy, public, private *npmUpstreamRecorder) *NPMHandler { + return NewNPMHandler(proxy, "http://proxy.test", public.URL). + WithPackageRoutes(map[string]string{"@example/*": private.URL + "/registry"}) +} + +func TestNPMRoutedMetadataComesFromRouteOnly(t *testing.T) { + public := newNPMUpstream(t, map[string]func(string) string{"/@example/widgets": routedTestPackument("9.9.9", "")}, http.StatusOK) + private := newNPMUpstream(t, map[string]func(string) string{routedNPMPath: routedTestPackument("1.0.0", "/registry")}, http.StatusOK) + proxy, _, _, _ := setupTestProxy(t) + proxy.HTTPClient = http.DefaultClient + h := newRoutedNPMHandler(proxy, public, private) + + w := serveNPM(t, h, http.MethodGet, "/@example%2fwidgets", nil, nil) + + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want %d; body: %s", w.Code, http.StatusOK, w.Body.String()) + } + if got := public.requested(); len(got) != 0 { + t.Errorf("default registry was queried for a routed package: %v", got) + } + if strings.Contains(w.Body.String(), "9.9.9") { + t.Errorf("routed package served the default registry's metadata: %s", w.Body.String()) + } + wantTarball := "http://proxy.test/npm/@example%2Fwidgets/-/widgets-1.0.0.tgz" + if !strings.Contains(w.Body.String(), wantTarball) { + t.Errorf("tarball not rewritten to %q: %s", wantTarball, w.Body.String()) + } +} + +func TestNPMRoutedMetadataDoesNotFallBack(t *testing.T) { + tests := []struct { + name string + privateStatus int + wantStatus int + }{ + {"package missing on route", http.StatusOK, http.StatusNotFound}, + {"route rejects credentials", http.StatusUnauthorized, http.StatusBadGateway}, + {"route unavailable", http.StatusInternalServerError, http.StatusBadGateway}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + public := newNPMUpstream(t, map[string]func(string) string{"/@example/widgets": routedTestPackument("9.9.9", "")}, http.StatusOK) + private := newNPMUpstream(t, map[string]func(string) string{}, tt.privateStatus) + proxy, _, _, _ := setupTestProxy(t) + proxy.HTTPClient = http.DefaultClient + h := newRoutedNPMHandler(proxy, public, private) + + w := serveNPM(t, h, http.MethodGet, "/@example%2fwidgets", nil, nil) + + if w.Code != tt.wantStatus { + t.Errorf("status = %d, want %d", w.Code, tt.wantStatus) + } + if got := public.requested(); len(got) != 0 { + t.Errorf("default registry was queried for a routed package: %v", got) + } + }) + } +} + +func TestNPMRoutedMetadataIgnoresDefaultRegistryCache(t *testing.T) { + public := newNPMUpstream(t, map[string]func(string) string{"/@example/widgets": routedTestPackument("9.9.9", "")}, http.StatusOK) + private := newNPMUpstream(t, map[string]func(string) string{routedNPMPath: routedTestPackument("1.0.0", "/registry")}, http.StatusOK) + proxy, _, _, _ := setupTestProxy(t) + proxy.HTTPClient = http.DefaultClient + proxy.CacheMetadata = true + proxy.MetadataTTL = time.Hour + + if w := serveNPM(t, NewNPMHandler(proxy, "http://proxy.test", public.URL), http.MethodGet, "/@example%2fwidgets", nil, nil); w.Code != http.StatusOK { + t.Fatalf("priming the default registry cache: status %d", w.Code) + } + + w := serveNPM(t, newRoutedNPMHandler(proxy, public, private), http.MethodGet, "/@example%2fwidgets", nil, nil) + + if strings.Contains(w.Body.String(), "9.9.9") { + t.Errorf("routed package served metadata cached from the default registry: %s", w.Body.String()) + } +} + +func TestNPMUnroutedMetadataUsesDefaultRegistry(t *testing.T) { + public := newNPMUpstream(t, map[string]func(string) string{"/lodash": routedTestPackument("4.17.21", "")}, http.StatusOK) + private := newNPMUpstream(t, map[string]func(string) string{}, http.StatusOK) + proxy, _, _, _ := setupTestProxy(t) + proxy.HTTPClient = http.DefaultClient + h := newRoutedNPMHandler(proxy, public, private) + + w := serveNPM(t, h, http.MethodGet, "/lodash", nil, nil) + + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want %d", w.Code, http.StatusOK) + } + if got := private.requested(); len(got) != 0 { + t.Errorf("route was queried for an unrouted package: %v", got) + } +} + +func TestNPMRoutedDownloadResolvesFromRoute(t *testing.T) { + public := newNPMUpstream(t, map[string]func(string) string{"/@example/widgets": routedTestPackument("1.0.0", "")}, http.StatusOK) + private := newNPMUpstream(t, map[string]func(string) string{routedNPMPath: routedTestPackument("1.0.0", "/registry")}, http.StatusOK) + proxy, _, _, artifactFetcher := setupTestProxy(t) + proxy.HTTPClient = http.DefaultClient + artifactFetcher.artifact = &fetch.Artifact{ + Body: io.NopCloser(strings.NewReader("package")), + ContentType: "application/gzip", + } + h := newRoutedNPMHandler(proxy, public, private) + + w := serveNPM(t, h, http.MethodGet, "/@example/widgets/-/widgets-1.0.0.tgz", nil, nil) + + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want %d; body: %s", w.Code, http.StatusOK, w.Body.String()) + } + if want := private.URL + "/registry/@example/widgets/-/widgets-1.0.0.tgz"; artifactFetcher.fetchedURL != want { + t.Errorf("fetched URL = %q, want %q", artifactFetcher.fetchedURL, want) + } + if got := public.requested(); len(got) != 0 { + t.Errorf("default registry was queried for a routed download: %v", got) + } +} + +func TestNPMRoutedDownloadWithoutMetadataStaysOnRoute(t *testing.T) { + public := newNPMUpstream(t, map[string]func(string) string{}, http.StatusOK) + private := newNPMUpstream(t, map[string]func(string) string{}, http.StatusInternalServerError) + proxy, _, _, artifactFetcher := setupTestProxy(t) + proxy.HTTPClient = http.DefaultClient + artifactFetcher.artifact = &fetch.Artifact{ + Body: io.NopCloser(strings.NewReader("package")), + ContentType: "application/gzip", + } + h := newRoutedNPMHandler(proxy, public, private) + + serveNPM(t, h, http.MethodGet, "/@example/widgets/-/widgets-1.0.0.tgz", nil, nil) + + if !strings.HasPrefix(artifactFetcher.fetchedURL, private.URL+"/registry/") { + t.Errorf("fetched URL = %q, want one on the route %q", artifactFetcher.fetchedURL, private.URL+"/registry/") + } +} + +func TestNPMRoutedDownloadIgnoresDefaultRegistryArtifact(t *testing.T) { + public := newNPMUpstream(t, map[string]func(string) string{"/@example/widgets": routedTestPackument("1.0.0", "")}, http.StatusOK) + private := newNPMUpstream(t, map[string]func(string) string{routedNPMPath: routedTestPackument("1.0.0", "/registry")}, http.StatusOK) + proxy, _, _, artifactFetcher := setupTestProxy(t) + proxy.HTTPClient = http.DefaultClient + artifactFetcher.artifact = &fetch.Artifact{ + Body: io.NopCloser(strings.NewReader("public package")), + ContentType: "application/gzip", + } + + if w := serveNPM(t, NewNPMHandler(proxy, "http://proxy.test", public.URL), http.MethodGet, "/@example/widgets/-/widgets-1.0.0.tgz", nil, nil); w.Code != http.StatusOK { + t.Fatalf("priming the artifact cache: status %d; body: %s", w.Code, w.Body.String()) + } + + artifactFetcher.artifact = &fetch.Artifact{ + Body: io.NopCloser(strings.NewReader("private package")), + ContentType: "application/gzip", + } + w := serveNPM(t, newRoutedNPMHandler(proxy, public, private), http.MethodGet, "/@example/widgets/-/widgets-1.0.0.tgz", nil, nil) + + if w.Body.String() != "private package" { + t.Errorf("routed download served %q, want the routed registry's package", w.Body.String()) + } +} + +// timedTestPackument is routedTestPackument with a publish time for the +// version, published age ago. +func timedTestPackument(version, tarballBase string, age time.Duration) func(string) string { + published := time.Now().Add(-age).UTC().Format(time.RFC3339) + return func(baseURL string) string { + return `{"name":"@example/widgets","dist-tags":{"latest":"` + version + `"},"versions":{"` + version + + `":{"version":"` + version + `","dist":{"tarball":"` + baseURL + tarballBase + `/@example/widgets/-/widgets-` + version + `.tgz"}}},` + + `"time":{"` + version + `":"` + published + `"}}` + } +} + +// TestNPMRoutedCooldownUsesRoutePublishTimes is a regression test: the +// download cooldown check stored publish times under the package's plain +// version PURL, so a time read from one registry was trusted for the same +// version from another. A route's version published an hour ago must stay in +// a seven-day cooldown even when the default registry's version of the same +// name is a month old, whichever was downloaded first, and changing a route's +// upstream must not carry over the old one's times. +func TestNPMRoutedCooldownUsesRoutePublishTimes(t *testing.T) { + const tarball = "/@example/widgets/-/widgets-1.0.0.tgz" + month, hour := 30*24*time.Hour, time.Hour + + type step struct { + handler string // "public", "route" or "other route" + wantStatus int + } + tests := []struct { + name string + steps []step + }{ + {"route added after a public download", []step{ + {"public", http.StatusOK}, + {"route", http.StatusNotFound}, + }}, + {"public download after a routed one", []step{ + {"route", http.StatusNotFound}, + {"public", http.StatusOK}, + }}, + {"route moved to another upstream", []step{ + {"other route", http.StatusOK}, + {"route", http.StatusNotFound}, + }}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + public := newNPMUpstream(t, map[string]func(string) string{"/@example/widgets": timedTestPackument("1.0.0", "", month)}, http.StatusOK) + private := newNPMUpstream(t, map[string]func(string) string{routedNPMPath: timedTestPackument("1.0.0", "/registry", hour)}, http.StatusOK) + old := newNPMUpstream(t, map[string]func(string) string{routedNPMPath: timedTestPackument("1.0.0", "/registry", month)}, http.StatusOK) + proxy, _, _, artifactFetcher := setupTestProxy(t) + proxy.HTTPClient = http.DefaultClient + proxy.CacheMetadata = true + proxy.MetadataTTL = time.Hour + proxy.Cooldown = &cooldown.Config{Default: "7d"} + handlers := map[string]*NPMHandler{ + "public": NewNPMHandler(proxy, "http://proxy.test", public.URL), + "route": newRoutedNPMHandler(proxy, public, private), + "other route": newRoutedNPMHandler(proxy, public, old), + } + + for _, s := range tt.steps { + artifactFetcher.artifact = &fetch.Artifact{ + Body: io.NopCloser(strings.NewReader("package")), + ContentType: "application/gzip", + } + w := serveNPM(t, handlers[s.handler], http.MethodGet, tarball, nil, nil) + if w.Code != s.wantStatus { + t.Errorf("%s download: status %d, want %d; body: %s", s.handler, w.Code, s.wantStatus, w.Body.String()) + } + } + }) + } +} diff --git a/internal/handler/package_routes.go b/internal/handler/package_routes.go new file mode 100644 index 0000000..8af7099 --- /dev/null +++ b/internal/handler/package_routes.go @@ -0,0 +1,74 @@ +package handler + +import ( + "crypto/sha256" + "encoding/hex" + "path" + "sort" + "strings" +) + +// routeIDLength is the number of hex characters of the upstream URL digest +// that qualify cache entries of a routed package. +const routeIDLength = 16 + +// packageRoutes sends packages whose names match a pattern to a dedicated +// upstream instead of the ecosystem's default registry. A matched package is +// never looked up anywhere else, so a public package with the same name cannot +// replace it. +type packageRoutes struct { + routes []packageRoute // longest pattern first +} + +type packageRoute struct { + pattern string + url string + id string +} + +func newPackageRoutes(patterns map[string]string) packageRoutes { + routes := make([]packageRoute, 0, len(patterns)) + for pattern, upstreamURL := range patterns { + upstreamURL = strings.TrimSuffix(upstreamURL, "/") + digest := sha256.Sum256([]byte(upstreamURL)) + routes = append(routes, packageRoute{ + pattern: strings.ToLower(pattern), + url: upstreamURL, + id: hex.EncodeToString(digest[:])[:routeIDLength], + }) + } + sort.Slice(routes, func(i, j int) bool { + if len(routes[i].pattern) != len(routes[j].pattern) { + return len(routes[i].pattern) > len(routes[j].pattern) + } + return routes[i].pattern < routes[j].pattern + }) + return packageRoutes{routes: routes} +} + +// match returns the route for a package name, if any pattern matches it. +func (r packageRoutes) match(packageName string) (packageRoute, bool) { + name := strings.ToLower(packageName) + for _, route := range r.routes { + if ok, _ := path.Match(route.pattern, name); ok { + return route, true + } + } + return packageRoute{}, false +} + +// cacheKey qualifies a package's metadata cache key with the route's upstream, +// so entries cached from the default registry, or from a route's previous +// upstream, are never served for a routed package. +func (r packageRoute) cacheKey(packageName string) string { + return "routes/" + r.id + "/" + packageName +} + +// cacheFilename qualifies a routed package's artifact filename the same way as +// cacheKey, and leaves other packages' filenames unchanged. +func (r packageRoutes) cacheFilename(packageName, filename string) string { + if route, routed := r.match(packageName); routed { + return route.id + "/" + filename + } + return filename +} diff --git a/internal/handler/package_routes_test.go b/internal/handler/package_routes_test.go new file mode 100644 index 0000000..dcb6f5f --- /dev/null +++ b/internal/handler/package_routes_test.go @@ -0,0 +1,60 @@ +package handler + +import "testing" + +func TestPackageRoutesMatch(t *testing.T) { + routes := newPackageRoutes(map[string]string{ + "@example/*": "https://npm.example.com/scoped/", + "@example/tool-*": "https://npm.example.com/tools", + "example/*": "https://composer.example.com/packages", + "single-package": "https://registry.example.com/single", + }) + + tests := []struct { + name string + pkg string + wantURL string + wantOK bool + }{ + {"scoped npm package", "@example/widgets", "https://npm.example.com/scoped", true}, + {"longest pattern wins", "@example/tool-kit", "https://npm.example.com/tools", true}, + {"composer vendor", "example/library", "https://composer.example.com/packages", true}, + {"exact name", "single-package", "https://registry.example.com/single", true}, + {"names are case-insensitive", "Example/Library", "https://composer.example.com/packages", true}, + {"wildcard does not cross slash", "example/library/extra", "", false}, + {"similar vendor does not match", "example-fork/library", "", false}, + {"unrouted package", "lodash", "", false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + route, ok := routes.match(tt.pkg) + if ok != tt.wantOK { + t.Fatalf("match(%q) ok = %t, want %t", tt.pkg, ok, tt.wantOK) + } + if ok && route.url != tt.wantURL { + t.Errorf("match(%q) url = %q, want %q", tt.pkg, route.url, tt.wantURL) + } + }) + } +} + +func TestPackageRoutesCacheKeysAreDistinct(t *testing.T) { + routes := newPackageRoutes(map[string]string{ + "example/*": "https://one.example.com", + "other-example/*": "https://two.example.com", + }) + + first, _ := routes.match("example/library") + second, _ := routes.match("other-example/library") + + if first.cacheKey("example/library") == "example/library" { + t.Error("routed cache key must differ from the unrouted package name") + } + if first.cacheKey("example/library") == second.cacheKey("example/library") { + t.Error("routes with different upstreams must not share cache keys") + } + if containsPathTraversal(first.cacheKey("example/library")) { + t.Error("routed cache key must be usable as a storage path") + } +} diff --git a/internal/server/package_routes_test.go b/internal/server/package_routes_test.go new file mode 100644 index 0000000..57bf946 --- /dev/null +++ b/internal/server/package_routes_test.go @@ -0,0 +1,53 @@ +package server + +import ( + "io" + "log/slog" + "net/http" + "net/http/httptest" + "testing" + + "github.com/git-pkgs/proxy/internal/config" + "github.com/git-pkgs/proxy/internal/handler" + "github.com/git-pkgs/registries/fetch" + "github.com/go-chi/chi/v5" +) + +func TestMountProtocolHandlersAppliesPackageRoutes(t *testing.T) { + routed := make(chan string, 2) + private := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + routed <- r.URL.Path + http.NotFound(w, r) + })) + defer private.Close() + + cfg := config.Default() + cfg.BaseURL = "http://proxy.test" + cfg.Upstream.Composer = "http://default.invalid" + cfg.Upstream.ComposerRepository = "http://default.invalid" + cfg.Upstream.NPM = "http://default.invalid" + cfg.Upstream.ComposerRoutes = map[string]string{"example/*": private.URL + "/composer"} + cfg.Upstream.NPMRoutes = map[string]string{"@example/*": private.URL + "/npm"} + + logger := slog.New(slog.NewTextHandler(io.Discard, nil)) + proxy := handler.NewProxy(nil, nil, fetch.NewFetcher(), fetch.NewResolver(), logger) + proxy.HTTPClient = private.Client() + s := &Server{cfg: cfg, logger: logger} + r := chi.NewRouter() + s.mountProtocolHandlers(r, proxy) + + for path, wantUpstreamPath := range map[string]string{ + "/composer/p2/example/library.json": "/composer/p2/example/library.json", + "/npm/@example%2fwidgets": "/npm/@example/widgets", + } { + r.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest(http.MethodGet, path, nil)) + select { + case got := <-routed: + if got != wantUpstreamPath { + t.Errorf("%s reached route upstream at %q, want %q", path, got, wantUpstreamPath) + } + default: + t.Errorf("%s did not reach the configured route", path) + } + } +} diff --git a/internal/server/server.go b/internal/server/server.go index 66ed14b..380892a 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -385,7 +385,8 @@ func (s *Server) serve(listener net.Listener) error { // mountProtocolHandlers constructs every ecosystem handler and mounts it on // r under its protocol prefix. func (s *Server) mountProtocolHandlers(r chi.Router, proxy *handler.Proxy) { - npmHandler := handler.NewNPMHandler(proxy, s.cfg.BaseURL, s.cfg.Upstream.NPM) + npmHandler := handler.NewNPMHandler(proxy, s.cfg.BaseURL, s.cfg.Upstream.NPM). + WithPackageRoutes(s.cfg.Upstream.NPMRoutes) cargoHandler := handler.NewCargoHandler( proxy, s.cfg.BaseURL, @@ -425,7 +426,7 @@ func (s *Server) mountProtocolHandlers(r chi.Router, proxy *handler.Proxy) { s.cfg.BaseURL, s.cfg.Upstream.Composer, s.cfg.Upstream.ComposerRepository, - ) + ).WithPackageRoutes(s.cfg.Upstream.ComposerRoutes) conanHandler := handler.NewConanHandlerWithUpstream(proxy, s.cfg.BaseURL, s.cfg.Upstream.Conan) condaHandler := handler.NewCondaHandlerWithUpstream(proxy, s.cfg.BaseURL, s.cfg.Upstream.Conda) cranHandler := handler.NewCRANHandlerWithUpstream(proxy, s.cfg.BaseURL, s.cfg.Upstream.CRAN) From 2ed0327946e10546688d7474bbe981da46ba736a Mon Sep 17 00:00:00 2001 From: Christian Heim Date: Sun, 11 Oct 2026 23:29:52 +0200 Subject: [PATCH 191/633] Keep fetching an artifact after the client that started the fetch disconnects. pacman gives up after 10 seconds without data, and since the proxy only starts sending once the whole file is stored, a large package on a slow upstream was cancelled on every try and never got cached. Client requests now mark their context so the shared fetch runs on context.WithoutCancel, still bounded by the artifact client timeout. Mirror jobs don't set the marker, so cancelling a job still aborts its fetches. --- internal/handler/detached_fetch_test.go | 139 ++++++++++++++++++++++++ internal/handler/handler.go | 28 ++++- internal/server/server.go | 5 + 3 files changed, 170 insertions(+), 2 deletions(-) create mode 100644 internal/handler/detached_fetch_test.go diff --git a/internal/handler/detached_fetch_test.go b/internal/handler/detached_fetch_test.go new file mode 100644 index 0000000..2e3196a --- /dev/null +++ b/internal/handler/detached_fetch_test.go @@ -0,0 +1,139 @@ +package handler + +import ( + "context" + "errors" + "io" + "net/http" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/git-pkgs/registries/fetch" +) + +// gatedFetcher holds every fetch until release is closed, and gives up early +// when the fetch context is cancelled, the way a real HTTP fetch does. +type gatedFetcher struct { + calls atomic.Int64 + content string + entered chan struct{} + release chan struct{} +} + +func newGatedFetcher(content string) *gatedFetcher { + return &gatedFetcher{content: content, entered: make(chan struct{}, 1), release: make(chan struct{})} +} + +func (f *gatedFetcher) Fetch(ctx context.Context, url string) (*fetch.Artifact, error) { + return f.FetchWithHeaders(ctx, url, nil) +} + +func (f *gatedFetcher) FetchWithHeaders(ctx context.Context, _ string, _ http.Header) (*fetch.Artifact, error) { + f.calls.Add(1) + select { + case f.entered <- struct{}{}: + default: + } + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-f.release: + } + return &fetch.Artifact{ + Body: io.NopCloser(strings.NewReader(f.content)), + ContentType: "application/octet-stream", + }, nil +} + +func (f *gatedFetcher) Head(context.Context, string) (int64, string, error) { + return 0, "", nil +} + +const detachedFetchURL = "https://geo.mirror.pkgbuild.com/extra/os/x86_64/big-1.0-1-x86_64.pkg.tar.zst" + +// startCancelledFetch starts a fetch under ctx, waits until it reached the +// upstream, cancels ctx, lets the upstream answer, and returns the caller's +// error. +func startCancelledFetch(t *testing.T, proxy *Proxy, fetcher *gatedFetcher, ctx context.Context) error { + t.Helper() + ctx, cancel := context.WithCancel(ctx) + done := make(chan error, 1) + go func() { + res, err := proxy.GetOrFetchArtifactFromURL(ctx, "arch", "big", "1.0-1", "big-1.0-1-x86_64.pkg.tar.zst", detachedFetchURL) + drain(res) + done <- err + }() + + select { + case <-fetcher.entered: + case <-time.After(5 * time.Second): + t.Fatal("fetch never reached the upstream") + } + cancel() + // Give a non-detached fetch the chance to observe the cancellation before + // the upstream answers. + time.Sleep(50 * time.Millisecond) + close(fetcher.release) + + select { + case err := <-done: + return err + case <-time.After(5 * time.Second): + t.Fatal("fetch did not return") + return nil + } +} + +// TestDetachedFetch_ClientCancelStillCaches covers a client that gives up on a +// slow cache miss. pacman aborts after 10 s without data; if that cancelled +// the upstream fetch, every retry would start from zero and a large package +// would never reach the cache. +func TestDetachedFetch_ClientCancelStillCaches(t *testing.T) { + proxy, _, _, _ := setupTestProxy(t) + fetcher := newGatedFetcher("package bytes") + proxy.Fetcher = fetcher + + _ = startCancelledFetch(t, proxy, fetcher, DetachUpstreamFetch(context.Background())) + + res, err := proxy.GetOrFetchArtifactFromURL(context.Background(), + "arch", "big", "1.0-1", "big-1.0-1-x86_64.pkg.tar.zst", detachedFetchURL) + if err != nil { + t.Fatalf("follow-up request: %v", err) + } + body, _ := io.ReadAll(res.Reader) + _ = res.Reader.Close() + if !res.Cached { + t.Error("follow-up was not a cache hit: the cancelled client aborted the fetch") + } + if string(body) != "package bytes" { + t.Errorf("body = %q, want %q", body, "package bytes") + } + if got := fetcher.calls.Load(); got != 1 { + t.Errorf("upstream fetches = %d, want 1", got) + } +} + +// TestDetachedFetch_UnmarkedContextStillCancels keeps mirror jobs cancellable: +// without the marker, cancelling the context aborts the fetch and nothing is +// cached. +func TestDetachedFetch_UnmarkedContextStillCancels(t *testing.T) { + proxy, _, _, _ := setupTestProxy(t) + fetcher := newGatedFetcher("package bytes") + proxy.Fetcher = fetcher + + err := startCancelledFetch(t, proxy, fetcher, context.Background()) + if !errors.Is(err, context.Canceled) { + t.Fatalf("error = %v, want context.Canceled", err) + } + + cached, err := proxy.GetCachedArtifact(context.Background(), "arch", "big", "1.0-1", "big-1.0-1-x86_64.pkg.tar.zst") + if err != nil { + t.Fatalf("cache lookup: %v", err) + } + if cached != nil { + _ = cached.Reader.Close() + t.Error("cancelled undetached fetch was still committed to the cache") + } +} diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 449fcef..53445f7 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -612,7 +612,9 @@ type inflightFetch struct { // commit runs on that caller's context, so cancellation behaves as it did // uncoalesced and mirroring still relies on it aborting the fetch. If that // caller goes away, everyone sharing the fetch gets its error and the key is -// released for a later retry. +// released for a later retry. A context marked with DetachUpstreamFetch is the +// exception: there commit keeps running after the caller goes away, so the +// artifact still lands in the cache. // // Every sharing caller still records a cache miss, so the gap between // proxy_cache_misses_total and upstream fetch observations is what coalescing @@ -669,7 +671,11 @@ func (p *Proxy) runSharedFetch(ctx context.Context, key string, f *inflightFetch } } - stored, path, err := commit(ctx) + commitCtx := ctx + if upstreamFetchDetached(ctx) { + commitCtx = context.WithoutCancel(ctx) + } + stored, path, err := commit(commitCtx) f.val, f.err = fetchedArtifact{artifact: stored, storagePath: path}, err if err != nil { return nil, err @@ -677,6 +683,24 @@ func (p *Proxy) runSharedFetch(ctx context.Context, key string, f *inflightFetch return p.openStoredArtifact(ctx, stored, path) } +type detachUpstreamFetchKey struct{} + +// DetachUpstreamFetch marks ctx so that an artifact fetch started under it +// runs to completion even when ctx is cancelled. The server sets it on client +// requests: a client that gives up on a slow cache miss (pacman aborts after +// 10 s without data) would otherwise cancel the fetch, and every retry would +// start again from zero, so a large artifact never reaches the cache. The +// fetch stays bounded by the artifact HTTP client's timeout. Mirror jobs leave +// it unset so cancelling a job still aborts its fetches. +func DetachUpstreamFetch(ctx context.Context) context.Context { + return context.WithValue(ctx, detachUpstreamFetchKey{}, true) +} + +func upstreamFetchDetached(ctx context.Context) bool { + detached, _ := ctx.Value(detachUpstreamFetchKey{}).(bool) + return detached +} + // fetchedArtifact is what a shared fetch hands its callers: metadata and a // storage path, neither holding reader state. type fetchedArtifact struct { diff --git a/internal/server/server.go b/internal/server/server.go index 380892a..82c47de 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -290,6 +290,11 @@ func (s *Server) serve(listener net.Listener) error { next.ServeHTTP(w, r) }) }) + r.Use(func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + next.ServeHTTP(w, r.WithContext(handler.DetachUpstreamFetch(r.Context()))) + }) + }) // Mount protocol handlers s.mountProtocolHandlers(r, proxy) From 055540a540614aa7d01dc550ca5cab451211b77b Mon Sep 17 00:00:00 2001 From: Christian Heim Date: Sun, 11 Oct 2026 23:59:29 +0200 Subject: [PATCH 192/633] Manifests are now cached under a key that ignores the client's Accept header: a digest names one document, and a tag is shared by all clients that accept an index type, so Docker, containerd and Podman read the same entry and a pre-fetched image is found offline. Clients without index support keep their per-Accept entry, and the old keys are still written and read as fallback. --- internal/handler/container_manifest.go | 95 +++++++--- .../handler/container_manifest_key_test.go | 165 ++++++++++++++++++ internal/handler/container_test.go | 10 +- 3 files changed, 240 insertions(+), 30 deletions(-) create mode 100644 internal/handler/container_manifest_key_test.go diff --git a/internal/handler/container_manifest.go b/internal/handler/container_manifest.go index 01df3af..4ff750e 100644 --- a/internal/handler/container_manifest.go +++ b/internal/handler/container_manifest.go @@ -8,6 +8,7 @@ import ( "mime" "net/http" "regexp" + "slices" "sort" "strconv" "strings" @@ -23,6 +24,14 @@ const ( containerAcceptExactSpecificity ) +const ( + // containerManifestIndexCacheClass replaces the Accept header in tag cache + // keys for clients that accept an index type. + containerManifestIndexCacheClass = "index" + ociImageIndexMediaType = "application/vnd.oci.image.index.v1+json" + dockerManifestListMediaType = "application/vnd.docker.distribution.manifest.list.v2+json" +) + var manifestDigestReferencePattern = regexp.MustCompile(`^[a-z0-9]+:[a-f0-9]+$`) type cachedContainerManifest struct { @@ -38,8 +47,7 @@ type cachedContainerManifest struct { func (h *ContainerHandler) serveManifest(w http.ResponseWriter, r *http.Request, registryURL, name, reference string) { accept := containerManifestAccept(r) cacheAccept := normalizeContainerManifestAccept(accept) - cacheKey := h.containerManifestCacheKey(registryURL, name, reference, cacheAccept) - cached := h.loadContainerManifestForAccept(r.Context(), registryURL, name, reference, accept, cacheKey) + cached := h.loadContainerManifestForAccept(r.Context(), registryURL, name, reference, accept, cacheAccept) immutable := manifestDigestReferencePattern.MatchString(reference) if cached != nil && (immutable || h.containerManifestFresh(cached)) { @@ -140,7 +148,44 @@ func (h *ContainerHandler) containerManifestCacheKey(registryURL, name, referenc return hex.EncodeToString(sum[:]) } -func (h *ContainerHandler) loadContainerManifestForAccept(ctx context.Context, registryURL, name, reference, accept, cacheKey string) *cachedContainerManifest { +// containerManifestSharedCacheKey is the key clients with different Accept +// headers share. A digest names exactly one document. A tag is stored once for +// all clients that accept an index type, whatever upstream returned for it, so +// Docker, containerd and Podman read the same entry. Clients without index +// support keep a key per normalized Accept header, because registries may +// convert an index into a single platform manifest for them. +func (h *ContainerHandler) containerManifestSharedCacheKey(registryURL, name, reference, cacheAccept string) string { + if manifestDigestReferencePattern.MatchString(reference) { + identity := strings.Join([]string{registryURL, name, reference}, "\x00") + sum := sha256.Sum256([]byte(identity)) + return hex.EncodeToString(sum[:]) + } + if containerManifestAcceptsIndex(cacheAccept) { + return h.containerManifestCacheKey(registryURL, name, reference, containerManifestIndexCacheClass) + } + return h.containerManifestCacheKey(registryURL, name, reference, cacheAccept) +} + +func containerManifestAcceptsIndex(accept string) bool { + return containerManifestAccepts(accept, ociImageIndexMediaType) || containerManifestAccepts(accept, dockerManifestListMediaType) +} + +// containerManifestLegacyCacheKeys are the per-Accept keys older versions +// used. They are still written so a rollback finds the cache, and read when +// the shared key has no entry yet. +func (h *ContainerHandler) containerManifestLegacyCacheKeys(registryURL, name, reference, accept, cacheAccept, sharedKey string) []string { + var keys []string + for _, variant := range []string{cacheAccept, accept} { + key := h.containerManifestCacheKey(registryURL, name, reference, variant) + if key != sharedKey && !slices.Contains(keys, key) { + keys = append(keys, key) + } + } + return keys +} + +func (h *ContainerHandler) loadContainerManifestForAccept(ctx context.Context, registryURL, name, reference, accept, cacheAccept string) *cachedContainerManifest { + cacheKey := h.containerManifestSharedCacheKey(registryURL, name, reference, cacheAccept) cached, err := h.loadContainerManifest(ctx, cacheKey) if err != nil { h.proxy.Logger.Warn("failed to read cached container manifest", "error", err) @@ -153,36 +198,36 @@ func (h *ContainerHandler) loadContainerManifestForAccept(ctx context.Context, r return nil } - legacyCacheKey := h.containerManifestCacheKey(registryURL, name, reference, accept) - if legacyCacheKey == cacheKey { - return nil + for _, legacyCacheKey := range h.containerManifestLegacyCacheKeys(registryURL, name, reference, accept, cacheAccept, cacheKey) { + cached, err = h.loadContainerManifest(ctx, legacyCacheKey) + if err != nil { + h.proxy.Logger.Warn("failed to read legacy cached container manifest", "error", err) + return nil + } + if cached == nil { + continue + } + if !containerManifestCacheCompatible(accept, cached) { + return nil + } + if err := h.storeContainerManifest(ctx, cacheKey, cached); err != nil { + h.proxy.Logger.Warn("failed to migrate cached container manifest", "error", err) + } + return cached } - cached, err = h.loadContainerManifest(ctx, legacyCacheKey) - if err != nil { - h.proxy.Logger.Warn("failed to read legacy cached container manifest", "error", err) - return nil - } - if cached == nil || !containerManifestCacheCompatible(accept, cached) { - return nil - } - if err := h.storeContainerManifest(ctx, cacheKey, cached); err != nil { - h.proxy.Logger.Warn("failed to migrate cached container manifest", "error", err) - } - return cached + return nil } func (h *ContainerHandler) storeContainerManifestForAccept(ctx context.Context, registryURL, name, reference, accept, cacheAccept string, manifest *cachedContainerManifest) { - cacheKey := h.containerManifestCacheKey(registryURL, name, reference, cacheAccept) + cacheKey := h.containerManifestSharedCacheKey(registryURL, name, reference, cacheAccept) if err := h.storeContainerManifest(ctx, cacheKey, manifest); err != nil { h.proxy.Logger.Warn("failed to cache container manifest", "error", err) } - legacyCacheKey := h.containerManifestCacheKey(registryURL, name, reference, accept) - if legacyCacheKey == cacheKey { - return - } - if err := h.storeContainerManifest(ctx, legacyCacheKey, manifest); err != nil { - h.proxy.Logger.Warn("failed to cache legacy container manifest", "error", err) + for _, legacyCacheKey := range h.containerManifestLegacyCacheKeys(registryURL, name, reference, accept, cacheAccept, cacheKey) { + if err := h.storeContainerManifest(ctx, legacyCacheKey, manifest); err != nil { + h.proxy.Logger.Warn("failed to cache legacy container manifest", "error", err) + } } } diff --git a/internal/handler/container_manifest_key_test.go b/internal/handler/container_manifest_key_test.go new file mode 100644 index 0000000..e339ced --- /dev/null +++ b/internal/handler/container_manifest_key_test.go @@ -0,0 +1,165 @@ +package handler + +import ( + "io" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" +) + +const ( + dockerManifestAccept = "application/vnd.docker.distribution.manifest.v2+json, " + + "application/vnd.docker.distribution.manifest.list.v2+json, " + + "application/vnd.oci.image.index.v1+json, application/vnd.oci.image.manifest.v1+json" + containerdManifestAccept = "application/vnd.docker.distribution.manifest.v2+json, " + + "application/vnd.docker.distribution.manifest.list.v2+json, " + + "application/vnd.oci.image.manifest.v1+json, application/vnd.oci.image.index.v1+json, */*" + ociManifestMediaType = "application/vnd.oci.image.manifest.v1+json" +) + +// fakeManifestRegistry serves one document per manifest path and fails with +// 503 once it is switched off. +type fakeManifestRegistry struct { + server *httptest.Server + offline atomic.Bool + docs map[string]fakeManifest +} + +type fakeManifest struct { + contentType string + body string +} + +func newFakeManifestRegistry(t *testing.T) *fakeManifestRegistry { + t.Helper() + registry := &fakeManifestRegistry{docs: make(map[string]fakeManifest)} + registry.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if registry.offline.Load() { + http.Error(w, "offline", http.StatusServiceUnavailable) + return + } + doc, ok := registry.docs[r.URL.Path] + if !ok { + http.NotFound(w, r) + return + } + w.Header().Set("Content-Type", doc.contentType) + w.Header().Set("Docker-Content-Digest", sha256Digest([]byte(doc.body))) + _, _ = io.WriteString(w, doc.body) + })) + t.Cleanup(registry.server.Close) + return registry +} + +func (r *fakeManifestRegistry) set(path, contentType, body string) { + r.docs[path] = fakeManifest{contentType: contentType, body: body} +} + +func newManifestKeyTestHandler(t *testing.T, registry *fakeManifestRegistry) *ContainerHandler { + t.Helper() + proxy, _, _, _ := setupTestProxy(t) + proxy.HTTPClient = registry.server.Client() + // A zero TTL makes every tag request revalidate, so an offline upstream + // exercises the stale path a real offline pull takes. + proxy.MetadataTTL = 0 + return &ContainerHandler{proxy: proxy, registryURL: registry.server.URL, proxyURL: "http://localhost:8080"} +} + +func getManifest(t *testing.T, h *ContainerHandler, path, accept string) *httptest.ResponseRecorder { + t.Helper() + request := httptest.NewRequest(http.MethodGet, path, nil) + request.Header.Set("Accept", accept) + response := httptest.NewRecorder() + h.Routes().ServeHTTP(response, request) + return response +} + +func TestContainerHandler_SingleArchTagSharedAcrossClients(t *testing.T) { + manifest := `{"schemaVersion":2,"mediaType":"application/vnd.oci.image.manifest.v1+json"}` + registry := newFakeManifestRegistry(t) + registry.set("/v2/library/app/manifests/1.0", ociManifestMediaType, manifest) + h := newManifestKeyTestHandler(t, registry) + + if response := getManifest(t, h, "/library/app/manifests/1.0", dockerManifestAccept); response.Code != http.StatusOK { + t.Fatalf("docker pull status = %d: %s", response.Code, response.Body.String()) + } + registry.offline.Store(true) + + response := getManifest(t, h, "/library/app/manifests/1.0", containerdManifestAccept) + if response.Code != http.StatusOK { + t.Fatalf("offline containerd pull status = %d, want 200: %s", response.Code, response.Body.String()) + } + if response.Body.String() != manifest { + t.Errorf("body = %q, want %q", response.Body.String(), manifest) + } +} + +func TestContainerHandler_DigestManifestSharedAcrossClients(t *testing.T) { + manifest := `{"schemaVersion":2,"mediaType":"application/vnd.oci.image.manifest.v1+json","config":{}}` + digest := sha256Digest([]byte(manifest)) + registry := newFakeManifestRegistry(t) + registry.set("/v2/library/app/manifests/"+digest, ociManifestMediaType, manifest) + h := newManifestKeyTestHandler(t, registry) + + if response := getManifest(t, h, "/library/app/manifests/"+digest, dockerManifestAccept); response.Code != http.StatusOK { + t.Fatalf("docker pull status = %d: %s", response.Code, response.Body.String()) + } + registry.offline.Store(true) + + response := getManifest(t, h, "/library/app/manifests/"+digest, ociManifestMediaType) + if response.Code != http.StatusOK { + t.Fatalf("offline pull by digest status = %d, want 200: %s", response.Code, response.Body.String()) + } +} + +func TestContainerHandler_RepushedTagServesLatestDocument(t *testing.T) { + index := `{"schemaVersion":2,"mediaType":"application/vnd.oci.image.index.v1+json","manifests":[]}` + manifest := `{"schemaVersion":2,"mediaType":"application/vnd.oci.image.manifest.v1+json"}` + registry := newFakeManifestRegistry(t) + registry.set("/v2/library/app/manifests/latest", ociImageIndexMediaType, index) + h := newManifestKeyTestHandler(t, registry) + + if response := getManifest(t, h, "/library/app/manifests/latest", dockerManifestAccept); response.Body.String() != index { + t.Fatalf("first pull body = %q, want index", response.Body.String()) + } + registry.set("/v2/library/app/manifests/latest", ociManifestMediaType, manifest) + if response := getManifest(t, h, "/library/app/manifests/latest", containerdManifestAccept); response.Body.String() != manifest { + t.Fatalf("pull after re-push body = %q, want manifest", response.Body.String()) + } + registry.offline.Store(true) + + response := getManifest(t, h, "/library/app/manifests/latest", dockerManifestAccept) + if response.Code != http.StatusOK { + t.Fatalf("offline pull status = %d, want 200: %s", response.Code, response.Body.String()) + } + if response.Body.String() != manifest { + t.Errorf("offline body = %q, want the re-pushed manifest", response.Body.String()) + } +} + +func TestContainerHandler_IndexlessClientKeepsOwnTagEntry(t *testing.T) { + index := `{"schemaVersion":2,"mediaType":"application/vnd.oci.image.index.v1+json","manifests":[]}` + converted := `{"schemaVersion":2,"mediaType":"application/vnd.docker.distribution.manifest.v2+json"}` + const schema2 = "application/vnd.docker.distribution.manifest.v2+json" + registry := newFakeManifestRegistry(t) + registry.set("/v2/library/app/manifests/latest", schema2, converted) + h := newManifestKeyTestHandler(t, registry) + + // A client without index support gets the converted platform manifest. + if response := getManifest(t, h, "/library/app/manifests/latest", schema2); response.Body.String() != converted { + t.Fatalf("index-less pull body = %q", response.Body.String()) + } + registry.set("/v2/library/app/manifests/latest", ociImageIndexMediaType, index) + if response := getManifest(t, h, "/library/app/manifests/latest", dockerManifestAccept); response.Body.String() != index { + t.Fatalf("index-capable pull body = %q, want the index", response.Body.String()) + } + registry.offline.Store(true) + + if response := getManifest(t, h, "/library/app/manifests/latest", containerdManifestAccept); response.Body.String() != index { + t.Errorf("offline index-capable body = %q, want the index", response.Body.String()) + } + if response := getManifest(t, h, "/library/app/manifests/latest", schema2); response.Body.String() != converted { + t.Errorf("offline index-less body = %q, want the converted manifest", response.Body.String()) + } +} diff --git a/internal/handler/container_test.go b/internal/handler/container_test.go index bd237cf..bd165c2 100644 --- a/internal/handler/container_test.go +++ b/internal/handler/container_test.go @@ -889,7 +889,7 @@ func TestContainerHandler_ManifestDigestMismatchIsNotCached(t *testing.T) { if !strings.Contains(w.Body.String(), "DIGEST_INVALID") { t.Errorf("body = %q, want DIGEST_INVALID", w.Body.String()) } - cacheKey := h.containerManifestCacheKey(upstream.URL, "homebrew/core/jq", digest, containerManifestAccept(req)) + cacheKey := h.containerManifestSharedCacheKey(upstream.URL, "homebrew/core/jq", digest, normalizeContainerManifestAccept(containerManifestAccept(req))) entry, err := db.GetMetadataCache(containerManifestCacheEcosystem, cacheKey) if err != nil { t.Fatalf("checking manifest cache: %v", err) @@ -951,7 +951,7 @@ func TestContainerHandler_ManifestTagWithInvalidDigestIsNotAliased(t *testing.T) if w.Code != http.StatusBadGateway { t.Fatalf("status = %d, want %d; body: %s", w.Code, http.StatusBadGateway, w.Body.String()) } - cacheKey := h.containerManifestCacheKey(upstream.URL, "homebrew/core/jq", invalidDigest, containerManifestAccept(req)) + cacheKey := h.containerManifestSharedCacheKey(upstream.URL, "homebrew/core/jq", invalidDigest, normalizeContainerManifestAccept(containerManifestAccept(req))) entry, err := db.GetMetadataCache(containerManifestCacheEcosystem, cacheKey) if err != nil { t.Fatalf("checking manifest cache: %v", err) @@ -1075,9 +1075,9 @@ func TestContainerHandler_ManifestMigratesLegacyAcceptCacheKey(t *testing.T) { accept := "application/vnd.oci.image.index.v1+json;q=1, application/vnd.oci.image.manifest.v1+json;q=1" cacheAccept := normalizeContainerManifestAccept(accept) legacyCacheKey := h.containerManifestCacheKey(upstream.URL, "library/nginx", "latest", accept) - cacheKey := h.containerManifestCacheKey(upstream.URL, "library/nginx", "latest", cacheAccept) + cacheKey := h.containerManifestSharedCacheKey(upstream.URL, "library/nginx", "latest", cacheAccept) if legacyCacheKey == cacheKey { - t.Fatal("legacy and normalized cache keys are equal") + t.Fatal("legacy and shared cache keys are equal") } request := httptest.NewRequest(http.MethodGet, "/library/nginx/manifests/latest", nil) @@ -1108,7 +1108,7 @@ func TestContainerHandler_ManifestMigratesLegacyAcceptCacheKey(t *testing.T) { t.Fatalf("load migrated manifest: %v", err) } if migrated == nil { - t.Error("normalized cache entry was not created") + t.Error("shared cache entry was not created") } } From 111b271273ed8579ee402974c55e1604afe07a9f Mon Sep 17 00:00:00 2001 From: Christian Heim Date: Sun, 11 Oct 2026 23:41:16 +0200 Subject: [PATCH 193/633] Arch Linux pacman repositories are now proxied at /arch/{name}/, with geo.mirror.pkgbuild.com as the built-in archlinux repository and more mirrors configurable under upstream.arch. Databases and file lists go through the metadata cache byte for byte, and a database signature is refreshed together with its database so the two always match. Packages and package signatures go to the artifact cache and stay available offline. The path is passed through unchanged, so Arch Linux ARM's $arch/$repo layout works too. README, config example, configuration docs and the dashboard have pacman instructions. --- README.md | 66 ++- config.example.yaml | 9 + docs/configuration.md | 20 + internal/config/config.go | 9 + internal/config/config_test.go | 20 +- internal/handler/arch.go | 218 +++++++++ internal/handler/arch_test.go | 463 +++++++++++++++++++ internal/handler/notfound_ecosystems_test.go | 2 + internal/server/dashboard.go | 15 + internal/server/middleware.go | 2 +- internal/server/middleware_test.go | 2 +- internal/server/server.go | 3 + internal/server/sources.go | 1 + internal/server/sources_test.go | 1 + 14 files changed, 827 insertions(+), 4 deletions(-) create mode 100644 internal/handler/arch.go create mode 100644 internal/handler/arch_test.go diff --git a/README.md b/README.md index 06cd0ed..c3b8571 100644 --- a/README.md +++ b/README.md @@ -65,7 +65,7 @@ The proxy never uploads artifact bytes to a scanner. Each scanner is notified wi | Debian | Debian/Ubuntu | | ✓ | | RPM | RHEL/Fedora | | ✓ | | Alpine | Alpine Linux | | ✓ | -| Arch | Arch Linux | | ✗ | +| Arch | Arch Linux | | ✓ | | Chef | Chef | | ✗ | | Generic | Any | | ✓ | | Helm | Kubernetes | Yes | ✓ | @@ -665,6 +665,69 @@ http://localhost:8080/apk/private apk appends the architecture and index filename to each repository line itself. +### Arch Linux / pacman + +Put the proxy at the top of `/etc/pacman.d/mirrorlist`. The default repository +name `archlinux` proxies the official GeoIP mirror +(`https://geo.mirror.pkgbuild.com`): + +``` +Server = http://localhost:8080/arch/archlinux/$repo/os/$arch +``` + +Then: + +```bash +pacman -Syu +``` + +Repository databases (`core.db`, `core.files` and their `.tar.*` forms), +database signatures, packages and package signatures are served byte-for-byte +unchanged, so pacman's signature checks keep working. Databases use the +metadata cache (`metadata_ttl`, stale fallback); a database signature is +refreshed together with its database rather than on its own TTL, so the two +always match. Packages and their `.sig` files are stored in the shared artifact +cache and remain available when the upstream is unreachable. + +The proxy only starts sending a package once it has the whole file, and pacman +gives up on a download that sends nothing for 10 seconds. The fetch keeps +running and the next try is served from the cache. On a slow upstream, set +`DisableDownloadTimeout` in `/etc/pacman.conf` to avoid the failed first try. + +`extra.files` is about 50 MB and is read into memory on each `pacman -F` +request. Raise `metadata_max_size` (100 MB by default) for larger custom +repositories. + +A custom repository that rebuilds a package under the same version leaves the +old package and signature in the cache. Bump `pkgrel` instead, or remove the +cached version. + +#### Arch Linux ARM + +Arch Linux ARM has no built-in repository. Add one under `upstream.arch`, using a +regional mirror from that serves HTTPS: + +```yaml +upstream: + arch: + archlinux: "https://geo.mirror.pkgbuild.com" + archlinuxarm: "https://.mirror.archlinuxarm.org" +``` + +Setting `upstream.arch` replaces the built-in default, so keep `archlinux` in the map +if you also proxy x86_64 Arch. + +Arch Linux ARM mirrors use `$arch/$repo` instead of `$repo/os/$arch`. In +`/etc/pacman.d/mirrorlist`: + +``` +Server = http://localhost:8080/arch/archlinuxarm/$arch/$repo +``` + +Avoid `mirror.archlinuxarm.org` as the upstream. It redirects each request to a +changing regional mirror over plain HTTP, so the database and the packages can come +from mirrors with a different sync state. + ### GitHub Releases / mise (aqua backend) Configure named generic upstreams: @@ -993,6 +1056,7 @@ Recently cached: | `GET /v2/*` | OCI/Docker registry protocol | | `GET /v2/homebrew/core/*` | Homebrew core bottle manifests and blobs from GHCR | | `GET /apk/{repository}/*` | Alpine APK repository protocol | +| `GET /arch/{repository}/*` | Arch Linux pacman repository protocol | | `GET /generic/{name}/*` | Generic HTTP download proxy (GitHub release assets, mise/aqua) | | `GET /debian/*` | Debian/APT repository protocol (main archive) | | `GET /debian/{repository}/*` | Debian/APT repository protocol (named archive, e.g. security) | diff --git a/config.example.yaml b/config.example.yaml index d29b4ba..6704b33 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -252,6 +252,15 @@ upstream: # alpine: "https://dl-cdn.alpinelinux.org/alpine" # private: "https://apk.example.com" + # Named Arch Linux pacman mirrors (used by /arch/{name}/). + # Defaults to {"archlinux": "https://geo.mirror.pkgbuild.com"} when empty; + # configuring any entry replaces that default. Arch Linux ARM needs a + # regional HTTPS mirror of your choice, not the mirror.archlinuxarm.org + # redirector. + # arch: + # archlinux: "https://geo.mirror.pkgbuild.com" + # archlinuxarm: "https://.mirror.archlinuxarm.org" + # Named generic HTTP upstreams (used by /generic/{name}/). The remaining # request path and query are appended to the upstream URL. GitHub release # assets ({owner}/{repo}/releases/download/{tag}/{asset}) are cached diff --git a/docs/configuration.md b/docs/configuration.md index 675d45a..442ec0d 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -390,6 +390,26 @@ verification keeps working; `.apk` packages use the shared artifact cache. When `upstream.apk` is empty, a single repository named `alpine` pointing at the official mirror is available; configuring any entry replaces that default. +```yaml +upstream: + + # Named Arch Linux pacman mirrors, served at /arch/{name}/. + arch: + archlinux: "https://geo.mirror.pkgbuild.com" +``` + +Arch Linux repositories are read-only. Requests to `/arch/{name}/…` are passed +to the mirror unchanged, so both the Arch Linux layout +(`/arch/archlinux/core/os/x86_64/core.db`) and the Arch Linux ARM layout +(`/arch/archlinuxarm/aarch64/core/core.db`) work. Databases (`{repo}.db`, +`{repo}.files` and their `.tar.*` forms) are cached with the metadata TTL and +served byte-for-byte unchanged; a database's `.sig` is refreshed together with +the database. Packages and package signatures use the shared artifact cache. +When `upstream.arch` is empty, a single repository named `archlinux` pointing at +`https://geo.mirror.pkgbuild.com` is available; configuring any entry replaces +that default. Arch Linux ARM has no default: pick a regional HTTPS mirror, not +the `mirror.archlinuxarm.org` redirector. + ### Debian archives ```yaml diff --git a/internal/config/config.go b/internal/config/config.go index fc314bd..2e4c850 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -646,6 +646,12 @@ type UpstreamConfig struct { // Default when empty: {"alpine": "https://dl-cdn.alpinelinux.org/alpine"}. APK map[string]string `json:"apk" yaml:"apk"` + // Arch maps repository names to Arch Linux pacman mirror base URLs, + // served at /arch/{name}/. The remaining request path mirrors the + // upstream layout, e.g. /arch/archlinux/core/os/x86_64/core.db. + // Default when empty: {"archlinux": "https://geo.mirror.pkgbuild.com"}. + Arch map[string]string `json:"arch" yaml:"arch"` + // OCI maps names to OCI registry URLs. Requests to a named registry use // the repository prefix upstream/{name}/, for example // oci://proxy.example.com/upstream/ghcr/owner/chart. @@ -717,6 +723,9 @@ func (u *UpstreamConfig) Validate() error { if err := validateNamedUpstreams("upstream.apk", u.APK); err != nil { return err } + if err := validateNamedUpstreams("upstream.arch", u.Arch); err != nil { + return err + } if err := validateNamedUpstreams("upstream.oci", u.OCI); err != nil { return err } diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 5566afd..492d4a7 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -1364,11 +1364,15 @@ func TestValidateNamedUpstreams(t *testing.T) { wantErr bool }{ { - name: "valid Helm, OCI, APK, and generic upstreams", + name: "valid Helm, OCI, APK, Arch, and generic upstreams", modify: func(cfg *Config) { cfg.Upstream.Helm = map[string]string{"bitnami": "https://charts.bitnami.com/bitnami"} cfg.Upstream.OCI = map[string]string{"ghcr": "https://ghcr.io"} cfg.Upstream.APK = map[string]string{"alpine": "https://dl-cdn.alpinelinux.org/alpine"} + cfg.Upstream.Arch = map[string]string{ + "archlinux": "https://geo.mirror.pkgbuild.com", + "archlinuxarm": "https://de3.mirror.archlinuxarm.org", + } cfg.Upstream.Generic = map[string]string{ "github": "https://github.com", "github-api": "https://api.github.com", @@ -1495,6 +1499,20 @@ func TestValidateNamedUpstreams(t *testing.T) { }, wantErr: true, }, + { + name: "Arch upstream name contains path separator", + modify: func(cfg *Config) { + cfg.Upstream.Arch = map[string]string{"arch/linux": "https://geo.mirror.pkgbuild.com"} + }, + wantErr: true, + }, + { + name: "Arch upstream URL is not absolute", + modify: func(cfg *Config) { + cfg.Upstream.Arch = map[string]string{"archlinux": "geo.mirror.pkgbuild.com"} + }, + wantErr: true, + }, { name: "valid Debian repository", modify: func(cfg *Config) { diff --git a/internal/handler/arch.go b/internal/handler/arch.go new file mode 100644 index 0000000..fd745e6 --- /dev/null +++ b/internal/handler/arch.go @@ -0,0 +1,218 @@ +package handler + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "net/http" + "regexp" + "strings" +) + +const ( + archEcosystem = "arch" + defaultArchRepositoryName = "archlinux" + // defaultArchUpstream is the official Arch Linux GeoIP mirror. + defaultArchUpstream = "https://geo.mirror.pkgbuild.com" + archMatchCount = 4 // full match + name + version + arch + archSignatureSuffix = ".sig" + archAccept = "*/*" + archPackageMediaType = "application/octet-stream" +) + +// ArchHandler serves pacman repositories at /arch/{repository}/. The rest of +// the path goes to the mirror unchanged, so the Arch Linux ARM layout works +// too. Databases are served byte for byte through the metadata cache, packages +// and their signatures through the artifact cache. +type ArchHandler struct { + proxy *Proxy + proxyURL string + repositories map[string]string +} + +// NewArchHandler creates an Arch Linux handler. Without repositories it serves +// the official GeoIP mirror as "archlinux". +func NewArchHandler(proxy *Proxy, proxyURL string, repositories map[string]string) *ArchHandler { + h := &ArchHandler{ + proxy: proxy, + proxyURL: strings.TrimSuffix(proxyURL, "/"), + repositories: make(map[string]string, len(repositories)), + } + for name, repositoryURL := range repositories { + h.repositories[name] = strings.TrimSuffix(repositoryURL, "/") + } + if len(h.repositories) == 0 { + h.repositories[defaultArchRepositoryName] = defaultArchUpstream + } + return h +} + +// Routes returns the HTTP handler for pacman requests. +// Mount this at /arch on your router. +func (h *ArchHandler) Routes() http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet && r.Method != http.MethodHead { + http.Error(w, "method not allowed", http.StatusMethodNotAllowed) + return + } + + path := strings.TrimPrefix(r.URL.Path, "/") + + if containsPathTraversal(path) { + http.Error(w, "invalid path", http.StatusBadRequest) + return + } + + repository, rest, ok := strings.Cut(path, "/") + upstreamURL, found := h.repositories[repository] + if !ok || rest == "" || !found { + http.NotFound(w, r) + return + } + + switch { + case isArchDatabase(rest): + h.handleMetadata(w, r, repository, upstreamURL, rest) + case isArchDatabaseSignature(rest): + h.handleDatabaseSignature(w, r, repository, upstreamURL, rest) + case isArchPackage(rest) || isArchPackageSignature(rest): + h.handlePackageDownload(w, r, repository, upstreamURL, rest) + default: + h.proxyFile(w, r, upstreamURL, rest) + } + }) +} + +// archDatabasePattern matches {repo}.db and {repo}.files, bare or as .tar.*. +var archDatabasePattern = regexp.MustCompile(`^[^/]+\.(?:db|files)(?:\.tar(?:\.(?:gz|xz|zst|bz2|lz4|lzo|lrz|lz|Z))?)?$`) + +func archBase(path string) string { + return path[strings.LastIndex(path, "/")+1:] +} + +func isArchDatabase(path string) bool { + return archDatabasePattern.MatchString(archBase(path)) +} + +func isArchDatabaseSignature(path string) bool { + base, ok := strings.CutSuffix(archBase(path), archSignatureSuffix) + return ok && archDatabasePattern.MatchString(base) +} + +// archPackagePattern matches {pkgname}-{pkgver}-{pkgrel}-{arch}.pkg.tar[.ext], +// e.g. lz4-1:1.10.0-2-x86_64.pkg.tar.zst. pkgname may contain '-', and pkgver +// may carry an epoch. +var archPackagePattern = regexp.MustCompile(`^(.+)-((?:\d+:)?[^-/:]+-[^-/:]+)-([^-/:]+)\.pkg\.tar(?:\.(?:zst|xz|gz|bz2|lz4|lzo|lrz|lz|Z))?$`) + +func isArchPackage(path string) bool { + return archPackagePattern.MatchString(archBase(path)) +} + +func isArchPackageSignature(path string) bool { + base, ok := strings.CutSuffix(archBase(path), archSignatureSuffix) + return ok && archPackagePattern.MatchString(base) +} + +// parseArchPackage returns name, pkgver-pkgrel and arch of a package or .sig. +func parseArchPackage(filename string) (name, version, arch string) { + matches := archPackagePattern.FindStringSubmatch(strings.TrimSuffix(filename, archSignatureSuffix)) + if len(matches) != archMatchCount { + return "", "", "" + } + return matches[1], matches[2], matches[3] +} + +// handlePackageDownload caches packages and their signatures. The cache key +// holds repository and full path, since two repositories may ship different +// bytes under one filename. +func (h *ArchHandler) handlePackageDownload(w http.ResponseWriter, r *http.Request, repository, upstreamURL, path string) { + name, version, arch := parseArchPackage(archBase(path)) + if name == "" { + h.proxyFile(w, r, upstreamURL, path) + return + } + + downloadURL := upstreamURL + "/" + path + cacheFilename := repository + "/" + path + + h.proxy.Logger.Info("arch package download", + "repository", repository, "name", name, "version", version, "arch", arch, + "signature", strings.HasSuffix(path, archSignatureSuffix)) + + result, err := h.proxy.GetOrFetchArtifactFromURL( + r.Context(), archEcosystem, name, version, cacheFilename, downloadURL) + if err != nil { + h.proxy.serveArtifactError(w, err, "failed to fetch package") + return + } + + if result.Artifact.MediaType == "" { + result.Artifact.MediaType = archPackageMediaType + } + ServeArtifactRequest(w, r, result) +} + +// handleMetadata serves databases unchanged through the metadata cache. +func (h *ArchHandler) handleMetadata(w http.ResponseWriter, r *http.Request, repository, upstreamURL, path string) { + h.proxy.ProxyCached(w, r, upstreamURL+"/"+path, archEcosystem, + h.metadataCacheKey(repository, upstreamURL, path), archAccept) +} + +// handleDatabaseSignature lets a database's .sig follow the database instead +// of its own TTL, since pacman rejects a mismatched pair. A signature older +// than the cached database is fetched again. +func (h *ArchHandler) handleDatabaseSignature(w http.ResponseWriter, r *http.Request, repository, upstreamURL, path string) { + p := h.proxy + sigKey := h.metadataCacheKey(repository, upstreamURL, path) + if !p.CacheMetadata || p.DB == nil { + h.handleMetadata(w, r, repository, upstreamURL, path) + return + } + + dbKey := h.metadataCacheKey(repository, upstreamURL, strings.TrimSuffix(path, archSignatureSuffix)) + dbEntry, _ := p.DB.GetMetadataCache(archEcosystem, dbKey) + sigEntry, _ := p.DB.GetMetadataCache(archEcosystem, sigKey) + if dbEntry == nil || sigEntry == nil || !dbEntry.FetchedAt.Valid || !sigEntry.FetchedAt.Valid { + h.handleMetadata(w, r, repository, upstreamURL, path) + return + } + + if !sigEntry.FetchedAt.Time.Before(dbEntry.FetchedAt.Time) { + body, contentType, err := p.readCachedMetadata(r.Context(), sigEntry, nil) + if err == nil { + p.writeMetadataCachedResponseWithEncoding(w, r, archEcosystem, sigKey, body, contentType, sigEntry.ContentEncoding.String) + return + } + // The cached bytes are gone; refetch below. + } + + signatureURL := upstreamURL + "/" + path + // Own key: a regular fetch would return the cached copy within its TTL. + key := metadataCoalesceKey(archEcosystem, sigKey, signatureURL, archAccept, "identity", false) + "\x00follow-database" + res, _ := p.coalesceMetadata(r.Context(), key, func(fetchCtx context.Context) metadataResult { + return p.fetchMetadataFromUpstream(fetchCtx, archEcosystem, sigKey, signatureURL, archAccept, "identity", nil, sigEntry) + }) + if res.err != nil { + if errors.Is(res.err, ErrUpstreamNotFound) { + http.Error(w, "not found", http.StatusNotFound) + return + } + p.Logger.Error("metadata fetch failed", "error", res.err) + http.Error(w, "failed to fetch from upstream", http.StatusBadGateway) + return + } + p.writeMetadataCachedResponseWithEncoding(w, r, archEcosystem, sigKey, res.body, res.contentType, res.contentEncoding) +} + +// metadataCacheKey hashes repository, upstream URL and path, as APKHandler does. +func (h *ArchHandler) metadataCacheKey(repository, upstreamURL, path string) string { + identity := repository + "\x00" + upstreamURL + "\x00" + path + digest := sha256.Sum256([]byte(identity)) + return hex.EncodeToString(digest[:]) +} + +// proxyFile passes other files (lastupdate, ISOs) through uncached. +func (h *ArchHandler) proxyFile(w http.ResponseWriter, r *http.Request, upstreamURL, path string) { + h.proxy.ProxyFile(w, r, upstreamURL+"/"+path) +} diff --git a/internal/handler/arch_test.go b/internal/handler/arch_test.go new file mode 100644 index 0000000..778c2dd --- /dev/null +++ b/internal/handler/arch_test.go @@ -0,0 +1,463 @@ +package handler + +import ( + "fmt" + "net/http" + "net/http/httptest" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/git-pkgs/proxy/internal/packageurl" + "github.com/git-pkgs/registries/fetch" +) + +func TestParseArchPackage(t *testing.T) { + assertPathParser(t, "parseArchPackage", parseArchPackage, []pathParseCase{ + {"bash-5.2.037-1-x86_64.pkg.tar.zst", "bash", "5.2.037-1", "x86_64"}, + {"python-pip-24.3.1-1-any.pkg.tar.zst", "python-pip", "24.3.1-1", "any"}, + {"lz4-1:1.10.0-2-x86_64.pkg.tar.zst", "lz4", "1:1.10.0-2", "x86_64"}, + {"wpa_supplicant-2:2.12-1-x86_64.pkg.tar.zst", "wpa_supplicant", "2:2.12-1", "x86_64"}, + {"linux-aarch64-6.12.1-1-aarch64.pkg.tar.xz", "linux-aarch64", "6.12.1-1", "aarch64"}, + {"libsigc++-3.6.0-1.1-x86_64.pkg.tar.lz4", "libsigc++", "3.6.0-1.1", "x86_64"}, + {"plain-1.0-1-x86_64.pkg.tar", "plain", "1.0-1", "x86_64"}, + {"bash-5.2.037-1-x86_64.pkg.tar.zst.sig", "bash", "5.2.037-1", "x86_64"}, + {"bash-x86_64.pkg.tar.zst", "", "", ""}, + {"bash-5.2.037-1-x86_64.tar.zst", "", "", ""}, + {"core.db", "", "", ""}, + }) +} + +func TestArchFileClassification(t *testing.T) { + tests := []struct { + path string + database, databaseSig, pkg, pkgSig bool + }{ + {path: "core/os/x86_64/core.db", database: true}, + {path: "core/os/x86_64/core.db.tar.gz", database: true}, + {path: "core/os/x86_64/core.files", database: true}, + {path: "core/os/x86_64/core.files.tar.zst", database: true}, + {path: "aarch64/core/core.db", database: true}, + {path: "core/os/x86_64/core.db.sig", databaseSig: true}, + {path: "custom/os/x86_64/custom.db.tar.xz.sig", databaseSig: true}, + {path: "core/os/x86_64/bash-5.2.037-1-x86_64.pkg.tar.zst", pkg: true}, + {path: "core/os/x86_64/bash-5.2.037-1-x86_64.pkg.tar.zst.sig", pkgSig: true}, + {path: "core/os/x86_64/core.db.tar.gz.old"}, + {path: "lastupdate"}, + {path: "iso/latest/archlinux-x86_64.iso"}, + } + for _, tt := range tests { + if got := isArchDatabase(tt.path); got != tt.database { + t.Errorf("isArchDatabase(%q) = %v, want %v", tt.path, got, tt.database) + } + if got := isArchDatabaseSignature(tt.path); got != tt.databaseSig { + t.Errorf("isArchDatabaseSignature(%q) = %v, want %v", tt.path, got, tt.databaseSig) + } + if got := isArchPackage(tt.path); got != tt.pkg { + t.Errorf("isArchPackage(%q) = %v, want %v", tt.path, got, tt.pkg) + } + if got := isArchPackageSignature(tt.path); got != tt.pkgSig { + t.Errorf("isArchPackageSignature(%q) = %v, want %v", tt.path, got, tt.pkgSig) + } + } +} + +// TestArchPURL pins the package URL type: git-pkgs/purl maps the "arch" +// ecosystem to pkg:alpm with the "arch" namespace. +func TestArchPURL(t *testing.T) { + pkg, version := packageurl.MakeCacheStrings(archEcosystem, "lz4", "1:1.10.0-2") + if pkg != "pkg:alpm/arch/lz4" { + t.Errorf("package PURL = %q, want %q", pkg, "pkg:alpm/arch/lz4") + } + if version != "pkg:alpm/arch/lz4@1:1.10.0-2" { + t.Errorf("version PURL = %q, want %q", version, "pkg:alpm/arch/lz4@1:1.10.0-2") + } +} + +func TestArchHandler_Routes(t *testing.T) { + h := NewArchHandler(nil, "http://localhost:8080", nil) + assertRoutesBasics(t, h.Routes(), "/archlinux/core/os/x86_64/core.db", "/archlinux/core/../../../etc/passwd") +} + +func TestArchHandler_DefaultsToGeoMirror(t *testing.T) { + h := NewArchHandler(nil, "http://localhost:8080", nil) + if got := h.repositories[defaultArchRepositoryName]; got != defaultArchUpstream { + t.Errorf("default repository = %q, want %q", got, defaultArchUpstream) + } + if len(h.repositories) != 1 { + t.Errorf("repositories = %v, want only the default", h.repositories) + } +} + +func TestArchHandler_UnknownRepositoryReturns404(t *testing.T) { + proxy, _, _, _ := setupTestProxy(t) + h := NewArchHandler(proxy, "http://localhost:8080", map[string]string{"archlinux": "https://example.test"}) + + for _, target := range []string{"/unknown/core/os/x86_64/core.db", "/archlinux", "/"} { + if w := serveArchRequest(h, http.MethodGet, target, nil); w.Code != http.StatusNotFound { + t.Errorf("%s: status = %d, want 404", target, w.Code) + } + } +} + +// archUpstream is a fake pacman mirror. Files can be swapped and the mirror +// taken offline; requests are counted per path. +type archUpstream struct { + *httptest.Server + mu sync.Mutex + files map[string][]byte + modified map[string]time.Time + hits map[string]int + available atomic.Bool + auth atomic.Value +} + +func newArchUpstream(t *testing.T, files map[string][]byte) *archUpstream { + t.Helper() + u := &archUpstream{files: files, modified: map[string]time.Time{}, hits: map[string]int{}} + u.available.Store(true) + u.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if !u.available.Load() { + http.Error(w, "unavailable", http.StatusServiceUnavailable) + return + } + u.auth.Store(r.Header.Get("Authorization")) + u.mu.Lock() + data, ok := u.files[r.URL.Path] + modified := u.modified[r.URL.Path] + u.hits[r.URL.Path]++ + u.mu.Unlock() + if !ok { + http.NotFound(w, r) + return + } + if !modified.IsZero() { + w.Header().Set("Last-Modified", modified.UTC().Format(http.TimeFormat)) + } + w.Header().Set("Content-Type", "application/octet-stream") + _, _ = w.Write(data) + })) + t.Cleanup(u.Close) + return u +} + +func (u *archUpstream) set(path string, data []byte) { + u.mu.Lock() + defer u.mu.Unlock() + u.files[path] = data +} + +func (u *archUpstream) hitCount(path string) int { + u.mu.Lock() + defer u.mu.Unlock() + return u.hits[path] +} + +func newArchTestProxy(t *testing.T, upstream *archUpstream) *Proxy { + t.Helper() + proxy, _, _, _ := setupTestProxy(t) + proxy.CacheMetadata = true + proxy.MetadataTTL = time.Hour + proxy.HTTPClient = upstream.Client() + fetcher := fetch.NewFetcher(fetch.WithHTTPClient(upstream.Client()), fetch.WithMaxRetries(0)) + proxy.Fetcher = fetcher + t.Cleanup(func() { _ = fetcher.Close() }) + return proxy +} + +func assertArchBody(t *testing.T, w *httptest.ResponseRecorder, label string, want []byte) { + t.Helper() + if w.Code != http.StatusOK { + t.Fatalf("%s: status = %d, want 200: %s", label, w.Code, w.Body.String()) + } + if got := w.Body.String(); got != string(want) { + t.Errorf("%s: body altered:\ngot %q\nwant %q", label, got, want) + } +} + +// TestArchHandler_DatabasesServedUnchanged covers database and file-list +// downloads under the names pacman uses (core.db, core.files) and their +// .tar.* forms: bytes are served unchanged so signatures stay valid, cached +// copies are served within the TTL without contacting the upstream, and the +// cache keeps serving them when the upstream is down. +func TestArchHandler_DatabasesServedUnchanged(t *testing.T) { + files := map[string][]byte{ + "/core/os/x86_64/core.db": []byte("\x1f\x8b\x08core-db-bytes"), + "/core/os/x86_64/core.db.tar.gz": []byte("\x1f\x8b\x08core-db-bytes"), + "/core/os/x86_64/core.db.sig": []byte("core-db-signature"), + "/core/os/x86_64/core.files": []byte("\x1f\x8b\x08core-files-bytes"), + "/core/os/x86_64/core.files.tar.zst": []byte("\x28\xb5\x2f\xfdcore-files-zst"), + } + upstream := newArchUpstream(t, files) + proxy := newArchTestProxy(t, upstream) + proxy.AuthForURL = func(string) (string, string) { return "Authorization", "Bearer arch-token" } + h := NewArchHandler(proxy, "http://proxy.example", map[string]string{"archlinux": upstream.URL}) + + // The database goes first, as pacman requests it. + paths := []string{ + "/core/os/x86_64/core.db", "/core/os/x86_64/core.db.sig", "/core/os/x86_64/core.db.tar.gz", + "/core/os/x86_64/core.files", "/core/os/x86_64/core.files.tar.zst", + } + for _, path := range paths { + assertArchBody(t, serveArchRequest(h, http.MethodGet, "/archlinux"+path, nil), path, files[path]) + } + if got, _ := upstream.auth.Load().(string); got != "Bearer arch-token" { + t.Errorf("Authorization = %q, want %q", got, "Bearer arch-token") + } + + for _, path := range paths { + assertArchBody(t, serveArchRequest(h, http.MethodGet, "/archlinux"+path, nil), path+" cached", files[path]) + if got := upstream.hitCount(path); got != 1 { + t.Errorf("%s: upstream hits = %d, want 1 (second read from cache)", path, got) + } + } + + upstream.available.Store(false) + for _, path := range paths { + assertArchBody(t, serveArchRequest(h, http.MethodGet, "/archlinux"+path, nil), path+" offline", files[path]) + } +} + +// TestArchHandler_StaleDatabaseServedWhenUpstreamDown covers the stale +// fallback after the TTL. +func TestArchHandler_StaleDatabaseServedWhenUpstreamDown(t *testing.T) { + db := []byte("core-db-bytes") + upstream := newArchUpstream(t, map[string][]byte{"/core/os/x86_64/core.db": db}) + proxy := newArchTestProxy(t, upstream) + h := NewArchHandler(proxy, "http://proxy.example", map[string]string{"archlinux": upstream.URL}) + + assertArchBody(t, serveArchRequest(h, http.MethodGet, "/archlinux/core/os/x86_64/core.db", nil), "seed", db) + ageMetadata(t, proxy, h, "archlinux", upstream.URL, "core/os/x86_64/core.db", 2*time.Hour) + upstream.available.Store(false) + + w := serveArchRequest(h, http.MethodGet, "/archlinux/core/os/x86_64/core.db", nil) + assertArchBody(t, w, "stale", db) + if got := w.Header().Get("Warning"); got == "" { + t.Error("stale database served without a Warning header") + } +} + +// TestArchHandler_DatabaseIfModifiedSince covers pacman's database refresh, +// which sends If-Modified-Since with the local copy's time. +func TestArchHandler_DatabaseIfModifiedSince(t *testing.T) { + modified := time.Date(2026, 10, 1, 12, 0, 0, 0, time.UTC) + upstream := newArchUpstream(t, map[string][]byte{"/core/os/x86_64/core.db": []byte("core-db-bytes")}) + upstream.modified["/core/os/x86_64/core.db"] = modified + proxy := newArchTestProxy(t, upstream) + h := NewArchHandler(proxy, "http://proxy.example", map[string]string{"archlinux": upstream.URL}) + + target := "/archlinux/core/os/x86_64/core.db" + assertArchBody(t, serveArchRequest(h, http.MethodGet, target, nil), "seed", []byte("core-db-bytes")) + + upToDate := http.Header{"If-Modified-Since": {modified.Format(http.TimeFormat)}} + if w := serveArchRequest(h, http.MethodGet, target, upToDate); w.Code != http.StatusNotModified { + t.Errorf("unchanged database: status = %d, want 304", w.Code) + } + older := http.Header{"If-Modified-Since": {modified.Add(-time.Hour).Format(http.TimeFormat)}} + assertArchBody(t, serveArchRequest(h, http.MethodGet, target, older), "older local copy", []byte("core-db-bytes")) +} + +// TestArchHandler_MissingDatabaseSignature covers the official repositories, +// which sign no databases: pacman with DatabaseOptional asks for core.db.sig +// and must get a 404 while the upstream is up. With the upstream down the +// answer is a 502, which pacman also treats as "no signature". +func TestArchHandler_MissingDatabaseSignature(t *testing.T) { + upstream := newArchUpstream(t, map[string][]byte{"/core/os/x86_64/core.db": []byte("core-db-bytes")}) + proxy := newArchTestProxy(t, upstream) + h := NewArchHandler(proxy, "http://proxy.example", map[string]string{"archlinux": upstream.URL}) + + assertArchBody(t, serveArchRequest(h, http.MethodGet, "/archlinux/core/os/x86_64/core.db", nil), "db", []byte("core-db-bytes")) + if w := serveArchRequest(h, http.MethodGet, "/archlinux/core/os/x86_64/core.db.sig", nil); w.Code != http.StatusNotFound { + t.Errorf("missing signature: status = %d, want 404", w.Code) + } + + upstream.available.Store(false) + assertArchBody(t, serveArchRequest(h, http.MethodGet, "/archlinux/core/os/x86_64/core.db", nil), "db offline", []byte("core-db-bytes")) + if w := serveArchRequest(h, http.MethodGet, "/archlinux/core/os/x86_64/core.db.sig", nil); w.Code != http.StatusBadGateway { + t.Errorf("missing signature offline: status = %d, want 502", w.Code) + } +} + +// TestArchHandler_DatabaseSignatureFollowsDatabase covers custom repositories +// with signed databases. The database and its signature are separate cache +// entries; when the database is refreshed, the signature must be refreshed +// with it even inside its own TTL, or pacman would get a new database with +// the old signature. While the database is not refreshed, the cached +// signature keeps matching it and is served without asking the upstream. +func TestArchHandler_DatabaseSignatureFollowsDatabase(t *testing.T) { + const dbPath, sigPath = "/custom/os/x86_64/custom.db", "/custom/os/x86_64/custom.db.sig" + upstream := newArchUpstream(t, map[string][]byte{dbPath: []byte("db v1"), sigPath: []byte("sig v1")}) + proxy := newArchTestProxy(t, upstream) + h := NewArchHandler(proxy, "http://proxy.example", map[string]string{"custom": upstream.URL}) + + assertArchBody(t, serveArchRequest(h, http.MethodGet, "/custom"+dbPath, nil), "db v1", []byte("db v1")) + assertArchBody(t, serveArchRequest(h, http.MethodGet, "/custom"+sigPath, nil), "sig v1", []byte("sig v1")) + assertArchBody(t, serveArchRequest(h, http.MethodGet, "/custom"+sigPath, nil), "sig v1 again", []byte("sig v1")) + if got := upstream.hitCount(sigPath); got != 1 { + t.Errorf("signature upstream hits = %d, want 1 while the database is unchanged", got) + } + + // Upstream publishes a new database. The cached database expires before + // the signature does. + upstream.set(dbPath, []byte("db v2")) + upstream.set(sigPath, []byte("sig v2")) + ageMetadata(t, proxy, h, "custom", upstream.URL, dbPath[1:], 2*time.Hour) + + assertArchBody(t, serveArchRequest(h, http.MethodGet, "/custom"+dbPath, nil), "db v2", []byte("db v2")) + assertArchBody(t, serveArchRequest(h, http.MethodGet, "/custom"+sigPath, nil), "sig after db refresh", []byte("sig v2")) + assertArchBody(t, serveArchRequest(h, http.MethodGet, "/custom"+sigPath, nil), "sig v2 again", []byte("sig v2")) + if got := upstream.hitCount(sigPath); got != 2 { + t.Errorf("signature upstream hits = %d, want 2", got) + } +} + +// ageMetadata moves a cached metadata entry's fetch time into the past, as if +// it had been fetched age ago. +func ageMetadata(t *testing.T, proxy *Proxy, h *ArchHandler, repository, upstreamURL, path string, age time.Duration) { + t.Helper() + entry, err := proxy.DB.GetMetadataCache(archEcosystem, h.metadataCacheKey(repository, upstreamURL, path)) + if err != nil || entry == nil { + t.Fatalf("metadata entry for %s: %v", path, err) + } + entry.FetchedAt.Time = time.Now().Add(-age) + if err := proxy.DB.UpsertMetadataCache(entry); err != nil { + t.Fatalf("aging metadata entry: %v", err) + } +} + +// TestArchHandler_PackagesAndSignatures covers package and signature +// downloads, cache hits, offline reads and epoch filenames (both as pacman +// sends them and percent-encoded). +func TestArchHandler_PackagesAndSignatures(t *testing.T) { + files := map[string][]byte{ + "/core/os/x86_64/bash-5.2.037-1-x86_64.pkg.tar.zst": []byte("bash package"), + "/core/os/x86_64/bash-5.2.037-1-x86_64.pkg.tar.zst.sig": []byte("bash signature"), + "/core/os/x86_64/lz4-1:1.10.0-2-x86_64.pkg.tar.zst": []byte("lz4 package"), + "/core/os/x86_64/lz4-1:1.10.0-2-x86_64.pkg.tar.zst.sig": []byte("lz4 signature"), + } + upstream := newArchUpstream(t, files) + proxy := newArchTestProxy(t, upstream) + h := NewArchHandler(proxy, "http://proxy.example", map[string]string{"archlinux": upstream.URL}) + + for path, want := range files { + assertArchBody(t, serveArchRequest(h, http.MethodGet, "/archlinux"+path, nil), path, want) + } + + upstream.available.Store(false) + for path, want := range files { + assertArchBody(t, serveArchRequest(h, http.MethodGet, "/archlinux"+path, nil), path+" offline", want) + if got := upstream.hitCount(path); got != 1 { + t.Errorf("%s: upstream hits = %d, want 1", path, got) + } + } + assertArchBody(t, serveArchRequest(h, http.MethodGet, + "/archlinux/core/os/x86_64/lz4-1%3A1.10.0-2-x86_64.pkg.tar.zst", nil), "percent-encoded epoch", files["/core/os/x86_64/lz4-1:1.10.0-2-x86_64.pkg.tar.zst"]) +} + +// TestArchHandler_ArchLinuxARMLayout covers the Arch Linux ARM mirror layout +// ($arch/$repo), served through the same route. +func TestArchHandler_ArchLinuxARMLayout(t *testing.T) { + files := map[string][]byte{ + "/aarch64/core/core.db": []byte("alarm core db"), + "/aarch64/core/bash-5.2.037-1-aarch64.pkg.tar.xz": []byte("alarm bash"), + "/aarch64/core/bash-5.2.037-1-aarch64.pkg.tar.xz.sig": []byte("alarm bash signature"), + } + upstream := newArchUpstream(t, files) + proxy := newArchTestProxy(t, upstream) + h := NewArchHandler(proxy, "http://proxy.example", map[string]string{"archlinuxarm": upstream.URL}) + + for path, want := range files { + assertArchBody(t, serveArchRequest(h, http.MethodGet, "/archlinuxarm"+path, nil), path, want) + } + upstream.available.Store(false) + for path, want := range files { + assertArchBody(t, serveArchRequest(h, http.MethodGet, "/archlinuxarm"+path, nil), path+" offline", want) + } +} + +// TestArchHandler_RepositoriesDoNotShareCache checks that two repositories +// with the same path keep their own bytes, for databases and packages. +func TestArchHandler_RepositoriesDoNotShareCache(t *testing.T) { + const dbPath, pkgPath = "/core/os/x86_64/core.db", "/core/os/x86_64/bash-5.2.037-1-x86_64.pkg.tar.zst" + a := newArchUpstream(t, map[string][]byte{dbPath: []byte("db A"), pkgPath: []byte("pkg A")}) + b := newArchUpstream(t, map[string][]byte{dbPath: []byte("db B"), pkgPath: []byte("pkg B")}) + proxy := newArchTestProxy(t, a) + proxy.HTTPClient = http.DefaultClient + fetcher := fetch.NewFetcher(fetch.WithHTTPClient(http.DefaultClient), fetch.WithMaxRetries(0)) + proxy.Fetcher = fetcher + t.Cleanup(func() { _ = fetcher.Close() }) + h := NewArchHandler(proxy, "http://proxy.example", map[string]string{"a": a.URL, "b": b.URL}) + + for _, repo := range []string{"a", "b"} { + suffix := map[string]string{"a": "A", "b": "B"}[repo] + assertArchBody(t, serveArchRequest(h, http.MethodGet, "/"+repo+dbPath, nil), repo+" db", []byte("db "+suffix)) + assertArchBody(t, serveArchRequest(h, http.MethodGet, "/"+repo+pkgPath, nil), repo+" pkg", []byte("pkg "+suffix)) + } +} + +// TestArchHandler_PackageHead covers HEAD: headers without a body. Range +// requests (pacman resuming a .part download) go through ServeArtifactRequest +// and are covered by its tests on seekable storage. +func TestArchHandler_PackageHead(t *testing.T) { + const path = "/core/os/x86_64/bash-5.2.037-1-x86_64.pkg.tar.zst" + pkg := []byte("0123456789 package bytes") + upstream := newArchUpstream(t, map[string][]byte{path: pkg}) + proxy := newArchTestProxy(t, upstream) + h := NewArchHandler(proxy, "http://proxy.example", map[string]string{"archlinux": upstream.URL}) + + assertArchBody(t, serveArchRequest(h, http.MethodGet, "/archlinux"+path, nil), "seed", pkg) + + w := serveArchRequest(h, http.MethodHead, "/archlinux"+path, nil) + if w.Code != http.StatusOK || w.Body.Len() != 0 { + t.Errorf("HEAD: status = %d, body length = %d, want 200 and 0", w.Code, w.Body.Len()) + } + if got := w.Header().Get("Content-Length"); got != fmt.Sprint(len(pkg)) { + t.Errorf("HEAD Content-Length = %q, want %d", got, len(pkg)) + } +} + +// TestArchHandler_PackageDownloadSendsUpstreamAuth covers upstream.auth for +// package downloads; databases are covered in +// TestArchHandler_DatabasesServedUnchanged. +func TestArchHandler_PackageDownloadSendsUpstreamAuth(t *testing.T) { + const path = "/private/os/x86_64/tool-1.0-1-x86_64.pkg.tar.zst" + upstream := newArchUpstream(t, map[string][]byte{path: []byte("private package")}) + proxy := newArchTestProxy(t, upstream) + client := upstream.Client() + client.Transport = &authRoundTripper{base: client.Transport, header: "Authorization", value: "Bearer arch-token"} + fetcher := fetch.NewFetcher(fetch.WithHTTPClient(client), fetch.WithMaxRetries(0)) + proxy.Fetcher = fetcher + t.Cleanup(func() { _ = fetcher.Close() }) + h := NewArchHandler(proxy, "http://proxy.example", map[string]string{"private": upstream.URL}) + + assertArchBody(t, serveArchRequest(h, http.MethodGet, "/private"+path, nil), path, []byte("private package")) + if got, _ := upstream.auth.Load().(string); got != "Bearer arch-token" { + t.Errorf("Authorization = %q, want %q", got, "Bearer arch-token") + } +} + +// TestArchHandler_OtherFilesProxiedUncached covers files that are neither +// databases nor packages, such as lastupdate. +func TestArchHandler_OtherFilesProxiedUncached(t *testing.T) { + upstream := newArchUpstream(t, map[string][]byte{"/lastupdate": []byte("1760000000")}) + proxy := newArchTestProxy(t, upstream) + h := NewArchHandler(proxy, "http://proxy.example", map[string]string{"archlinux": upstream.URL}) + + for i := range 2 { + assertArchBody(t, serveArchRequest(h, http.MethodGet, "/archlinux/lastupdate", nil), fmt.Sprint("lastupdate ", i), []byte("1760000000")) + } + if got := upstream.hitCount("/lastupdate"); got != 2 { + t.Errorf("upstream hits = %d, want 2 (not cached)", got) + } +} + +func serveArchRequest(h *ArchHandler, method, target string, header http.Header) *httptest.ResponseRecorder { + req := httptest.NewRequest(method, target, nil) + for k, v := range header { + req.Header[k] = v + } + w := httptest.NewRecorder() + h.Routes().ServeHTTP(w, req) + return w +} diff --git a/internal/handler/notfound_ecosystems_test.go b/internal/handler/notfound_ecosystems_test.go index 9e229fc..e214124 100644 --- a/internal/handler/notfound_ecosystems_test.go +++ b/internal/handler/notfound_ecosystems_test.go @@ -21,6 +21,8 @@ func TestArtifactDownloadUpstreamNotFoundReturns404(t *testing.T) { func(p *Proxy) http.Handler { return NewRPMHandler(p, "http://localhost").Routes() }}, {"apk", "/alpine/v3.22/main/x86_64/busybox-1.37.0-r12.apk", func(p *Proxy) http.Handler { return NewAPKHandler(p, "http://localhost", nil).Routes() }}, + {"arch", "/archlinux/core/os/x86_64/bash-5.2.037-1-x86_64.pkg.tar.zst", + func(p *Proxy) http.Handler { return NewArchHandler(p, "http://localhost", nil).Routes() }}, {"nuget", "/v3-flatcontainer/newtonsoft.json/13.0.3/newtonsoft.json.13.0.3.nupkg", func(p *Proxy) http.Handler { return NewNuGetHandler(p, "http://localhost").Routes() }}, {"pypi", "/packages/packages/ab/cd/ef0123456789/requests-2.31.0-py3-none-any.whl", diff --git a/internal/server/dashboard.go b/internal/server/dashboard.go index 05643f6..dbc3a69 100644 --- a/internal/server/dashboard.go +++ b/internal/server/dashboard.go @@ -134,6 +134,7 @@ func supportedEcosystems() []string { // expected order return []string{ "alpine", + "arch", "cargo", "composer", "conan", @@ -204,6 +205,8 @@ func ecosystemColorClasses(ecosystem string) string { return "bg-amber-100 text-amber-800 dark:bg-amber-900/50 dark:text-amber-300" case "alpine": return "bg-lime-100 text-lime-800 dark:bg-lime-900/50 dark:text-lime-300" + case "arch": + return "bg-blue-100 text-blue-800 dark:bg-blue-900/50 dark:text-blue-300" default: return "bg-gray-100 text-gray-700 dark:bg-gray-800 dark:text-gray-300" } @@ -521,5 +524,17 @@ sudo dnf update`), # Then run: apk update`), }, + { + ID: "arch", + Name: "Arch Linux pacman", + Language: "Arch Linux", + Endpoint: "/arch/", + Instructions: template.HTML(`

Configure pacman to use the proxy:

+
# In /etc/pacman.d/mirrorlist, above any other Server line
+Server = ` + baseURL + `/arch/archlinux/$repo/os/$arch
+
+# Then run:
+pacman -Syu
`), + }, } } diff --git a/internal/server/middleware.go b/internal/server/middleware.go index a5433ce..eabe9f4 100644 --- a/internal/server/middleware.go +++ b/internal/server/middleware.go @@ -106,7 +106,7 @@ func requestEcosystem(path string) string { segment, _, _ := strings.Cut(strings.TrimPrefix(path, "/"), "/") switch segment { case "npm", "cargo", "hex", "pub", "pypi", "maven", "gradle", "nuget", - "conan", "conda", "cran", "julia", "debian", "rpm", + "conan", "conda", "cran", "julia", "debian", "rpm", "arch", "helm", "homebrew", "generic", "swift": return segment case "apk": diff --git a/internal/server/middleware_test.go b/internal/server/middleware_test.go index 595d7ca..875b152 100644 --- a/internal/server/middleware_test.go +++ b/internal/server/middleware_test.go @@ -313,7 +313,7 @@ func TestRequestEcosystemCoversEveryMountedRoute(t *testing.T) { "/gradle/x": "gradle", "/nuget/x": "nuget", "/composer/x": "packagist", "/conan/x": "conan", "/conda/x": "conda", "/cran/x": "cran", "/julia/x": "julia", "/swift/x": "swift", "/v2/x": "oci", - "/apk/x": "alpine", "/debian/x": "debian", "/rpm/x": "rpm", + "/apk/x": "alpine", "/arch/x": "arch", "/debian/x": "debian", "/rpm/x": "rpm", "/helm/x": "helm", "/homebrew/x": "homebrew", "/generic/x": "generic", } for path, want := range mounts { diff --git a/internal/server/server.go b/internal/server/server.go index 82c47de..e5df624 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -19,6 +19,7 @@ // - /swift/* - Swift Package Registry protocol // - /v2/* - OCI/Docker container registry protocol // - /apk/* - Alpine APK repository protocol +// - /arch/* - Arch Linux pacman repository protocol // - /debian/* - Debian/APT repository protocol // - /rpm/* - RPM/Yum repository protocol // @@ -449,6 +450,7 @@ func (s *Server) mountProtocolHandlers(r chi.Router, proxy *handler.Proxy) { helmHandler := handler.NewHelmHandlerWithOCIRegistries( proxy, s.cfg.BaseURL, s.cfg.Upstream.Helm, s.cfg.Upstream.OCIDefault, s.cfg.Upstream.OCI) apkHandler := handler.NewAPKHandler(proxy, s.cfg.BaseURL, s.cfg.Upstream.APK) + archHandler := handler.NewArchHandler(proxy, s.cfg.BaseURL, s.cfg.Upstream.Arch) debianHandler := handler.NewDebianHandler( proxy, s.cfg.BaseURL, @@ -478,6 +480,7 @@ func (s *Server) mountProtocolHandlers(r chi.Router, proxy *handler.Proxy) { r.Mount("/v2", http.StripPrefix("/v2", containerHandler.Routes())) r.Mount("/helm", http.StripPrefix("/helm", helmHandler.Routes())) r.Mount("/apk", http.StripPrefix("/apk", apkHandler.Routes())) + r.Mount("/arch", http.StripPrefix("/arch", archHandler.Routes())) r.Mount("/debian", http.StripPrefix("/debian", debianHandler.Routes())) r.Mount("/rpm", http.StripPrefix("/rpm", rpmHandler.Routes())) r.Mount("/generic", http.StripPrefix("/generic", genericHandler.Routes())) diff --git a/internal/server/sources.go b/internal/server/sources.go index 923cb9d..2a0e47c 100644 --- a/internal/server/sources.go +++ b/internal/server/sources.go @@ -64,6 +64,7 @@ var knownClients = map[string]string{ "dnf": "dnf", "urlgrabber": "dnf", "apk": "apk", + "pacman": "pacman", "conda": "conda", "mamba": "conda", "conan": "conan", diff --git a/internal/server/sources_test.go b/internal/server/sources_test.go index fa58dd7..560ca93 100644 --- a/internal/server/sources_test.go +++ b/internal/server/sources_test.go @@ -21,6 +21,7 @@ func TestClientName(t *testing.T) { {"docker/24.0.7 go/go1.20.10 kernel/6.5.0 os/linux", "docker"}, {"Debian APT-HTTP/1.3 (2.6.1)", "apt"}, {"bundler/2.4.22 rubygems/3.4.22", "bundler"}, + {"pacman/7.0.0 (Linux x86_64) libalpm/15.0.0", "pacman"}, // Unknown and hostile input collapses, so a User-Agent cannot mint a // new Prometheus time series. {"", unknownClient}, From 094a962697f80ba757070d8e7638ab1aa6495de6 Mon Sep 17 00:00:00 2001 From: Christian Heim Date: Sun, 11 Oct 2026 23:51:43 +0200 Subject: [PATCH 194/633] At most 64 fetches keep running after their client hangs up; beyond that a fetch is tied to its client again, so nobody can start an unbounded number of large downloads and leave. A server test now checks that client requests actually get the detach marker. --- internal/handler/detached_fetch_test.go | 26 +++++ internal/handler/handler.go | 40 +++++++- internal/server/detached_fetch_test.go | 122 ++++++++++++++++++++++++ 3 files changed, 185 insertions(+), 3 deletions(-) create mode 100644 internal/server/detached_fetch_test.go diff --git a/internal/handler/detached_fetch_test.go b/internal/handler/detached_fetch_test.go index 2e3196a..554761d 100644 --- a/internal/handler/detached_fetch_test.go +++ b/internal/handler/detached_fetch_test.go @@ -137,3 +137,29 @@ func TestDetachedFetch_UnmarkedContextStillCancels(t *testing.T) { t.Error("cancelled undetached fetch was still committed to the cache") } } + +// TestDetachedFetch_CapFallsBackToClientContext checks the cap on detached +// fetches: with every slot taken, a cancelled client cancels its fetch again, +// as it did before fetches were detached. +func TestDetachedFetch_CapFallsBackToClientContext(t *testing.T) { + proxy, _, _, _ := setupTestProxy(t) + fetcher := newGatedFetcher("package bytes") + proxy.Fetcher = fetcher + + for range maxDetachedFetches { + if !proxy.acquireDetachSlot() { + t.Fatal("could not take a free detach slot") + } + } + err := startCancelledFetch(t, proxy, fetcher, DetachUpstreamFetch(context.Background())) + if !errors.Is(err, context.Canceled) { + t.Fatalf("error = %v, want context.Canceled once the detach slots are used up", err) + } + + for range maxDetachedFetches { + proxy.releaseDetachSlot() + } + if !proxy.acquireDetachSlot() { + t.Error("slots not available again after release") + } +} diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 53445f7..2552692 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -198,6 +198,11 @@ type Proxy struct { fetchMu sync.Mutex inFlight map[string]*inflightFetch + // detachSlots caps the artifact fetches that keep running after their + // client went away (see DetachUpstreamFetch). Guarded by fetchMu while it + // is created. + detachSlots chan struct{} + // inFlightMeta does the same for metadata misses. Keyed by // metadataCoalesceKey. metaMu sync.Mutex @@ -672,7 +677,8 @@ func (p *Proxy) runSharedFetch(ctx context.Context, key string, f *inflightFetch } commitCtx := ctx - if upstreamFetchDetached(ctx) { + if upstreamFetchDetached(ctx) && p.acquireDetachSlot() { + defer p.releaseDetachSlot() commitCtx = context.WithoutCancel(ctx) } stored, path, err := commit(commitCtx) @@ -690,8 +696,9 @@ type detachUpstreamFetchKey struct{} // requests: a client that gives up on a slow cache miss (pacman aborts after // 10 s without data) would otherwise cancel the fetch, and every retry would // start again from zero, so a large artifact never reaches the cache. The -// fetch stays bounded by the artifact HTTP client's timeout. Mirror jobs leave -// it unset so cancelling a job still aborts its fetches. +// fetch stays bounded by the artifact HTTP client's timeout, and at most +// maxDetachedFetches run detached at once. Mirror jobs leave it unset so +// cancelling a job still aborts its fetches. func DetachUpstreamFetch(ctx context.Context) context.Context { return context.WithValue(ctx, detachUpstreamFetchKey{}, true) } @@ -701,6 +708,33 @@ func upstreamFetchDetached(ctx context.Context) bool { return detached } +// maxDetachedFetches caps the artifact fetches that may outlive their client. +// Without a cap a client could start any number of large uncached downloads +// and hang up straight away, leaving each to run to the artifact client's +// timeout. Fetches beyond the cap stay tied to their client, as before +// detaching existed. +const maxDetachedFetches = 64 + +func (p *Proxy) acquireDetachSlot() bool { + p.fetchMu.Lock() + if p.detachSlots == nil { + p.detachSlots = make(chan struct{}, maxDetachedFetches) + } + slots := p.detachSlots + p.fetchMu.Unlock() + + select { + case slots <- struct{}{}: + return true + default: + return false + } +} + +func (p *Proxy) releaseDetachSlot() { + <-p.detachSlots +} + // fetchedArtifact is what a shared fetch hands its callers: metadata and a // storage path, neither holding reader state. type fetchedArtifact struct { diff --git a/internal/server/detached_fetch_test.go b/internal/server/detached_fetch_test.go new file mode 100644 index 0000000..557de51 --- /dev/null +++ b/internal/server/detached_fetch_test.go @@ -0,0 +1,122 @@ +package server + +import ( + "context" + "errors" + "io" + "log/slog" + "net" + "net/http" + "net/http/httptest" + "os" + "os/exec" + "path/filepath" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/git-pkgs/proxy/internal/config" +) + +// TestServerKeepsFetchAfterClientHangsUp checks that the server marks client +// requests with handler.DetachUpstreamFetch: a pacman client that gives up on +// a slow cache miss must not cancel the upstream fetch, so the next request +// is served from the cache without fetching again. +func TestServerKeepsFetchAfterClientHangsUp(t *testing.T) { + if os.Getenv("PROXY_TEST_DETACHED_FETCH") == "1" { + testServerKeepsFetchAfterClientHangsUp(t) + return + } + + cmd := exec.Command(os.Args[0], "-test.run=^TestServerKeepsFetchAfterClientHangsUp$") + cmd.Env = append(os.Environ(), "PROXY_TEST_DETACHED_FETCH=1") + if output, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("detached fetch test failed: %v\n%s", err, output) + } +} + +func testServerKeepsFetchAfterClientHangsUp(t *testing.T) { + const pkgPath = "/core/os/x86_64/big-1.0-1-x86_64.pkg.tar.zst" + release := make(chan struct{}) + var releaseOnce sync.Once + t.Cleanup(func() { releaseOnce.Do(func() { close(release) }) }) + var hits atomic.Int32 + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != pkgPath { + http.NotFound(w, r) + return + } + hits.Add(1) + <-release + _, _ = io.WriteString(w, "big package") + })) + defer upstream.Close() + + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("reserving proxy address: %v", err) + } + t.Cleanup(func() { _ = listener.Close() }) + + tempDir := t.TempDir() + cfg := config.Default() + cfg.Listen = listener.Addr().String() + cfg.BaseURL = "http://" + cfg.Listen + cfg.Database.Path = filepath.Join(tempDir, "proxy.db") + cfg.Storage.URL = "file://" + filepath.Join(tempDir, "artifacts") + cfg.Upstream.Arch = map[string]string{"archlinux": upstream.URL} + cfg.Upstream.AllowLoopback = true + if err := cfg.Validate(); err != nil { + t.Fatalf("validating config: %v", err) + } + + proxyServer, err := New(cfg, slog.New(slog.NewTextHandler(io.Discard, nil)), BuildInfo{Version: "test", Commit: "test"}) + if err != nil { + t.Fatalf("creating server: %v", err) + } + startErr := make(chan error, 1) + go func() { startErr <- proxyServer.Start(listener) }() + defer func() { + releaseOnce.Do(func() { close(release) }) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if err := proxyServer.Shutdown(ctx); err != nil { + t.Errorf("shutting down server: %v", err) + } + if err := <-startErr; !errors.Is(err, http.ErrServerClosed) { + t.Errorf("Start() error = %v, want %v", err, http.ErrServerClosed) + } + }() + + target := cfg.BaseURL + "/arch/archlinux" + pkgPath + + // The client gives up while the upstream is still sending nothing. + impatient := &http.Client{Timeout: 300 * time.Millisecond} + deadline := time.Now().Add(5 * time.Second) + for hits.Load() == 0 { + if _, err := impatient.Get(target); err == nil { + t.Fatal("request succeeded while the upstream was held") + } + if time.Now().After(deadline) { + t.Fatal("upstream fetch never started") + } + } + // Let the server notice the hang-up before the upstream answers. + time.Sleep(100 * time.Millisecond) + releaseOnce.Do(func() { close(release) }) + + client := &http.Client{Timeout: 5 * time.Second} + resp, err := client.Get(target) + if err != nil { + t.Fatalf("follow-up request: %v", err) + } + body, _ := io.ReadAll(resp.Body) + _ = resp.Body.Close() + if resp.StatusCode != http.StatusOK || string(body) != "big package" { + t.Fatalf("follow-up: status %d, body %q", resp.StatusCode, body) + } + if got := hits.Load(); got != 1 { + t.Errorf("upstream fetches = %d, want 1: the client hanging up cancelled the first fetch", got) + } +} From 5ce445f5d3f64ba2bf49a785c745abd223bc2e91 Mon Sep 17 00:00:00 2001 From: Christian Heim Date: Sun, 11 Oct 2026 23:51:43 +0200 Subject: [PATCH 195/633] A database signature that belongs to an older database is no longer served. When the database was refreshed and the signature refetch fails, the proxy answers 404 or 502 instead of falling back to the cached signature, which pacman would reject. Paths with a decoded ? or # are refused, since they would fetch one file and cache it under another name. The README now says that pacman -Sy only works offline with cache_metadata enabled. --- README.md | 6 ++-- internal/handler/arch.go | 9 +++-- internal/handler/arch_test.go | 66 +++++++++++++++++++++++++++++++++++ 3 files changed, 76 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index c3b8571..ad47805 100644 --- a/README.md +++ b/README.md @@ -685,8 +685,10 @@ Repository databases (`core.db`, `core.files` and their `.tar.*` forms), database signatures, packages and package signatures are served byte-for-byte unchanged, so pacman's signature checks keep working. Databases use the metadata cache (`metadata_ttl`, stale fallback); a database signature is -refreshed together with its database rather than on its own TTL, so the two -always match. Packages and their `.sig` files are stored in the shared artifact +refreshed together with its database rather than on its own TTL, and a +signature that belongs to an older database is never served in its place. Enable `cache_metadata` (or set `PROXY_CACHE_METADATA=true`) so +`pacman -Sy` keeps working while the upstream is down; without it databases are +fetched fresh on every request. Packages and their `.sig` files are stored in the shared artifact cache and remain available when the upstream is unreachable. The proxy only starts sending a package once it has the whole file, and pacman diff --git a/internal/handler/arch.go b/internal/handler/arch.go index fd745e6..a38fbc1 100644 --- a/internal/handler/arch.go +++ b/internal/handler/arch.go @@ -59,7 +59,9 @@ func (h *ArchHandler) Routes() http.Handler { path := strings.TrimPrefix(r.URL.Path, "/") - if containsPathTraversal(path) { + // A decoded '?' or '#' would cut the upstream path short and cache one + // file under another file's name. + if containsPathTraversal(path) || strings.ContainsAny(path, "?#") { http.Error(w, "invalid path", http.StatusBadRequest) return } @@ -161,7 +163,7 @@ func (h *ArchHandler) handleMetadata(w http.ResponseWriter, r *http.Request, rep // handleDatabaseSignature lets a database's .sig follow the database instead // of its own TTL, since pacman rejects a mismatched pair. A signature older -// than the cached database is fetched again. +// than the cached database is fetched again and never served as a fallback. func (h *ArchHandler) handleDatabaseSignature(w http.ResponseWriter, r *http.Request, repository, upstreamURL, path string) { p := h.proxy sigKey := h.metadataCacheKey(repository, upstreamURL, path) @@ -191,7 +193,8 @@ func (h *ArchHandler) handleDatabaseSignature(w http.ResponseWriter, r *http.Req // Own key: a regular fetch would return the cached copy within its TTL. key := metadataCoalesceKey(archEcosystem, sigKey, signatureURL, archAccept, "identity", false) + "\x00follow-database" res, _ := p.coalesceMetadata(r.Context(), key, func(fetchCtx context.Context) metadataResult { - return p.fetchMetadataFromUpstream(fetchCtx, archEcosystem, sigKey, signatureURL, archAccept, "identity", nil, sigEntry) + // No entry: a 304 or stale fallback would return the old signature. + return p.fetchMetadataFromUpstream(fetchCtx, archEcosystem, sigKey, signatureURL, archAccept, "identity", nil, nil) }) if res.err != nil { if errors.Is(res.err, ErrUpstreamNotFound) { diff --git a/internal/handler/arch_test.go b/internal/handler/arch_test.go index 778c2dd..587ab34 100644 --- a/internal/handler/arch_test.go +++ b/internal/handler/arch_test.go @@ -312,6 +312,72 @@ func TestArchHandler_DatabaseSignatureFollowsDatabase(t *testing.T) { } } +// TestArchHandler_OutdatedSignatureNeverServed covers a signature refetch that +// fails after the database was refreshed. The cached signature belongs to +// the old database, and pacman rejects a present but invalid signature even +// with DatabaseOptional, so the proxy must answer 404 or 502 rather than fall +// back to it. +func TestArchHandler_OutdatedSignatureNeverServed(t *testing.T) { + const dbPath, sigPath = "/custom/os/x86_64/custom.db", "/custom/os/x86_64/custom.db.sig" + + for _, tt := range []struct { + name string + breakSig func(u *archUpstream) + want int + }{ + {"signature removed upstream", func(u *archUpstream) { + u.mu.Lock() + delete(u.files, sigPath) + u.mu.Unlock() + }, http.StatusNotFound}, + {"upstream down after database refresh", func(u *archUpstream) { u.available.Store(false) }, http.StatusBadGateway}, + } { + t.Run(tt.name, func(t *testing.T) { + upstream := newArchUpstream(t, map[string][]byte{dbPath: []byte("db v1"), sigPath: []byte("sig v1")}) + proxy := newArchTestProxy(t, upstream) + h := NewArchHandler(proxy, "http://proxy.example", map[string]string{"custom": upstream.URL}) + + assertArchBody(t, serveArchRequest(h, http.MethodGet, "/custom"+dbPath, nil), "db v1", []byte("db v1")) + assertArchBody(t, serveArchRequest(h, http.MethodGet, "/custom"+sigPath, nil), "sig v1", []byte("sig v1")) + + upstream.set(dbPath, []byte("db v2")) + ageMetadata(t, proxy, h, "custom", upstream.URL, dbPath[1:], 2*time.Hour) + assertArchBody(t, serveArchRequest(h, http.MethodGet, "/custom"+dbPath, nil), "db v2", []byte("db v2")) + + tt.breakSig(upstream) + for i := range 2 { + w := serveArchRequest(h, http.MethodGet, "/custom"+sigPath, nil) + if w.Code != tt.want { + t.Errorf("request %d: status = %d, body = %q, want %d", i, w.Code, w.Body.String(), tt.want) + } + } + }) + } +} + +// TestArchHandler_RejectsQueryAndFragmentInPath covers a decoded '?' or '#' +// in the path, which would otherwise fetch one file and cache it under the +// name that follows. +func TestArchHandler_RejectsQueryAndFragmentInPath(t *testing.T) { + upstream := newArchUpstream(t, map[string][]byte{}) + proxy := newArchTestProxy(t, upstream) + h := NewArchHandler(proxy, "http://proxy.example", map[string]string{"archlinux": upstream.URL}) + + for _, target := range []string{ + "/archlinux/core/os/x86_64/bash-5.2-1-x86_64.pkg.tar.zst%3F/foo-9-9-any.pkg.tar.zst", + "/archlinux/core/os/x86_64/bash-5.2-1-x86_64.pkg.tar.zst%23/foo-9-9-any.pkg.tar.zst", + } { + if w := serveArchRequest(h, http.MethodGet, target, nil); w.Code != http.StatusBadRequest { + t.Errorf("%s: status = %d, want 400", target, w.Code) + } + } + upstream.mu.Lock() + defer upstream.mu.Unlock() + if len(upstream.hits) != 0 { + t.Errorf("upstream was asked for %v", upstream.hits) + } +} + // ageMetadata moves a cached metadata entry's fetch time into the past, as if // it had been fetched age ago. func ageMetadata(t *testing.T, proxy *Proxy, h *ArchHandler, repository, upstreamURL, path string, age time.Duration) { From cc18c889ad1b1e25e3bec4fc71b446668ffc2ed0 Mon Sep 17 00:00:00 2001 From: Christian Heim Date: Mon, 12 Oct 2026 00:02:31 +0200 Subject: [PATCH 196/633] proxy mirror now builds its upstream clients the same way as serve, so it follows OCI Bearer challenges, uses configured upstream credentials and the SSRF guard, and artifact downloads have no overall time limit. Before this the CLI used a plain client and Docker Hub answered every request with 401. --- cmd/proxy/main.go | 8 +--- internal/server/mirror_proxy_test.go | 64 ++++++++++++++++++++++++++++ internal/server/server.go | 48 ++++++++++++++------- 3 files changed, 99 insertions(+), 21 deletions(-) create mode 100644 internal/server/mirror_proxy_test.go diff --git a/cmd/proxy/main.go b/cmd/proxy/main.go index 5b62563..acd32cc 100644 --- a/cmd/proxy/main.go +++ b/cmd/proxy/main.go @@ -113,11 +113,9 @@ import ( "github.com/git-pkgs/proxy/internal/config" "github.com/git-pkgs/proxy/internal/database" "github.com/git-pkgs/proxy/internal/denylist" - "github.com/git-pkgs/proxy/internal/handler" "github.com/git-pkgs/proxy/internal/mirror" "github.com/git-pkgs/proxy/internal/server" "github.com/git-pkgs/proxy/internal/storage" - "github.com/git-pkgs/registries/fetch" ) const defaultTopN = 10 @@ -518,10 +516,8 @@ func runMirror() { os.Exit(1) //nolint:gocritic // db closed above } - // Build proxy (reuses same pipeline as serve) - fetcher := fetch.NewFetcher() - resolver := fetch.NewResolver() - proxy := handler.NewProxy(db, store, fetcher, resolver, logger) + // Build proxy with the same upstream clients and authentication as serve + proxy := server.NewMirrorProxy(cfg, db, store, logger) proxy.Denylist, err = denylist.New(cfg.Denylist.Packages) if err != nil { _ = db.Close() diff --git a/internal/server/mirror_proxy_test.go b/internal/server/mirror_proxy_test.go new file mode 100644 index 0000000..2df395a --- /dev/null +++ b/internal/server/mirror_proxy_test.go @@ -0,0 +1,64 @@ +package server + +import ( + "io" + "log/slog" + "net/http" + "net/http/httptest" + "testing" + + "github.com/git-pkgs/proxy/internal/config" +) + +// newTokenRegistry answers 401 with a Bearer challenge until the request +// carries the token its realm hands out. +func newTokenRegistry(t *testing.T) *httptest.Server { + t.Helper() + var registry *httptest.Server + registry = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/token": + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, `{"token":"pull-token"}`) + case "/v2/library/app/manifests/1.0": + if r.Header.Get("Authorization") != "Bearer pull-token" { + w.Header().Set("WWW-Authenticate", `Bearer realm="`+registry.URL+`/token",service="registry",scope="repository:library/app:pull"`) + http.Error(w, "unauthorized", http.StatusUnauthorized) + return + } + w.Header().Set("Content-Type", "application/vnd.oci.image.manifest.v1+json") + _, _ = io.WriteString(w, `{"schemaVersion":2}`) + default: + http.NotFound(w, r) + } + })) + t.Cleanup(registry.Close) + return registry +} + +func TestNewMirrorProxyFollowsBearerChallenge(t *testing.T) { + registry := newTokenRegistry(t) + cfg := config.Default() + cfg.Upstream.AllowLoopback = true + proxy := NewMirrorProxy(cfg, nil, nil, slog.New(slog.NewTextHandler(io.Discard, nil))) + + resp, err := proxy.HTTPClient.Get(registry.URL + "/v2/library/app/manifests/1.0") + if err != nil { + t.Fatalf("GET manifest: %v", err) + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode != http.StatusOK { + t.Errorf("status = %d, want 200 after the Bearer challenge", resp.StatusCode) + } +} + +func TestNewMirrorProxyRefusesLoopbackByDefault(t *testing.T) { + registry := newTokenRegistry(t) + proxy := NewMirrorProxy(config.Default(), nil, nil, slog.New(slog.NewTextHandler(io.Discard, nil))) + + resp, err := proxy.HTTPClient.Get(registry.URL + "/v2/library/app/manifests/1.0") + if err == nil { + _ = resp.Body.Close() + t.Fatal("GET to a loopback registry succeeded without upstream.allow_loopback") + } +} diff --git a/internal/server/server.go b/internal/server/server.go index 380892a..de6204c 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -220,22 +220,10 @@ func (s *Server) Start(listeners ...net.Listener) error { } func (s *Server) serve(listener net.Listener) error { - // Use one authentication-aware transport for metadata and artifacts so - // configured credentials and cached OCI challenges apply consistently. - safeClient := newUpstreamClient(s.cfg.Upstream) - baseTransport := safeClient.Transport - if s.accessLog != nil { - baseTransport = upstreamhttp.NewAccessLogTransport(baseTransport, s.accessLog, s.logger) - } - authTransport := upstreamhttp.NewTransport(baseTransport, upstreamhttp.AuthFunc(s.authForURL)) - metadataClient := *safeClient - metadataClient.Timeout = s.cfg.ParseHTTPTimeout() - metadataClient.Transport = authTransport - artifactClient := metadataClient - artifactClient.Timeout = serverWriteTimeout + metadataClient, artifactClient := s.newUpstreamClients(serverWriteTimeout) // Create shared components with circuit breaker. - baseFetcher := fetch.NewFetcher(fetch.WithHTTPClient(&artifactClient)) + baseFetcher := fetch.NewFetcher(fetch.WithHTTPClient(artifactClient)) fetcher := fetch.NewCircuitBreakerFetcher(baseFetcher) s.breakers = newBreakerMonitor(fetcher, s.logger) resolver := fetch.NewResolver() @@ -245,7 +233,7 @@ func (s *Server) serve(listener net.Listener) error { Packages: s.cfg.Cooldown.NormalizedPackages(), } proxy := handler.NewProxy(s.db, s.storage, fetcher, resolver, s.logger) - proxy.HTTPClient = &metadataClient + proxy.HTTPClient = metadataClient proxy.AuthForURL = s.authForURL cooldownPolicy, err := cooldownpolicy.New(cd, s.cfg.Cooldown.PackagePatterns) if err != nil { @@ -481,6 +469,36 @@ func (s *Server) mountProtocolHandlers(r chi.Router, proxy *handler.Proxy) { // configureScanning builds the scanner group from cfg and wires it into // proxy, returning the group so the caller can decide whether to mount the // internal scan-fetch route. +// newUpstreamClients builds the metadata and artifact clients. Both share one +// authentication-aware transport so configured credentials and cached OCI +// challenges apply consistently. An artifactTimeout of zero means no limit. +func (s *Server) newUpstreamClients(artifactTimeout time.Duration) (metadataClient, artifactClient *http.Client) { + safeClient := newUpstreamClient(s.cfg.Upstream) + baseTransport := safeClient.Transport + if s.accessLog != nil { + baseTransport = upstreamhttp.NewAccessLogTransport(baseTransport, s.accessLog, s.logger) + } + authTransport := upstreamhttp.NewTransport(baseTransport, upstreamhttp.AuthFunc(s.authForURL)) + metadata := *safeClient + metadata.Timeout = s.cfg.ParseHTTPTimeout() + metadata.Transport = authTransport + artifact := metadata + artifact.Timeout = artifactTimeout + return &metadata, &artifact +} + +// NewMirrorProxy builds the proxy for the mirror command with the same +// upstream clients, SSRF guard and upstream authentication as serve. Artifact +// downloads have no overall time limit; the caller cancels through ctx. +func NewMirrorProxy(cfg *config.Config, db *database.DB, store storage.Storage, logger *slog.Logger) *handler.Proxy { + s := &Server{cfg: cfg, db: db, storage: store, logger: logger, ecr: newECRTokens(logger)} + metadataClient, artifactClient := s.newUpstreamClients(0) + proxy := handler.NewProxy(db, store, fetch.NewFetcher(fetch.WithHTTPClient(artifactClient)), fetch.NewResolver(), logger) + proxy.HTTPClient = metadataClient + proxy.AuthForURL = s.authForURL + return proxy +} + func configureScanning(proxy *handler.Proxy, cfg config.ScanningConfig, baseURL string, logger *slog.Logger) (*scanner.Group, error) { scanGroup, err := scanner.NewGroup(cfg, logger) if err != nil { From 7b6c64af0a07ef062a0472e03d215cbc1bd2e619 Mon Sep 17 00:00:00 2001 From: Christian Heim Date: Mon, 12 Oct 2026 00:20:10 +0200 Subject: [PATCH 197/633] proxy mirror and the mirror API now pre-fetch container images from pkg:docker and pkg:oci PURLs: the mirror walks index, platform manifests (with their attestations) and blobs through the container handler, so a later pull is served from cache even when the upstream is down. A --platform filter (platforms in the API) limits multi-arch images, repository_url only resolves to configured registries, and names, tags and digests are validated before anything is fetched. --- README.md | 3 + cmd/proxy/main.go | 24 +- docs/architecture.md | 2 +- docs/configuration.md | 27 ++ internal/handler/container_mirror.go | 373 ++++++++++++++++++++++ internal/handler/container_mirror_test.go | 295 +++++++++++++++++ internal/mirror/image.go | 126 ++++++++ internal/mirror/image_test.go | 174 ++++++++++ internal/mirror/job.go | 10 +- internal/mirror/mirror.go | 8 + internal/mirror/source.go | 22 +- internal/server/mirror_proxy_test.go | 4 +- internal/server/server.go | 31 +- 13 files changed, 1076 insertions(+), 23 deletions(-) create mode 100644 internal/handler/container_mirror.go create mode 100644 internal/handler/container_mirror_test.go create mode 100644 internal/mirror/image.go create mode 100644 internal/mirror/image_test.go diff --git a/README.md b/README.md index 06cd0ed..05cc362 100644 --- a/README.md +++ b/README.md @@ -890,6 +890,9 @@ proxy mirror --dry-run pkg:npm/lodash # Control parallelism proxy mirror --concurrency 8 pkg:npm/lodash@4.17.21 + +# Mirror a container image for one platform +proxy mirror --platform linux/amd64 pkg:docker/library/alpine@3.20 ``` The mirror command accepts the same storage and database flags as `serve`. Already-cached artifacts are skipped. diff --git a/cmd/proxy/main.go b/cmd/proxy/main.go index acd32cc..b4208d1 100644 --- a/cmd/proxy/main.go +++ b/cmd/proxy/main.go @@ -419,6 +419,7 @@ func runMirror() { sbomPath := fs.String("sbom", "", "Path to CycloneDX or SPDX SBOM file") concurrency := fs.Int("concurrency", 4, "Number of parallel downloads") //nolint:mnd // default concurrency dryRun := fs.Bool("dry-run", false, "Show what would be mirrored without downloading") + platforms := fs.String("platform", "", "Comma-separated os/arch[/variant] list limiting which platforms of container images are mirrored") fs.Usage = func() { fmt.Fprintf(os.Stderr, "git-pkgs proxy - Pre-populate cache\n\n") @@ -426,13 +427,22 @@ func runMirror() { fmt.Fprintf(os.Stderr, "Examples:\n") fmt.Fprintf(os.Stderr, " proxy mirror pkg:npm/lodash@4.17.21\n") fmt.Fprintf(os.Stderr, " proxy mirror --sbom sbom.cdx.json\n") - fmt.Fprintf(os.Stderr, " proxy mirror pkg:npm/lodash # all versions\n\n") + fmt.Fprintf(os.Stderr, " proxy mirror pkg:npm/lodash # all versions\n") + fmt.Fprintf(os.Stderr, " proxy mirror --platform linux/amd64 pkg:docker/library/alpine@3.20\n\n") fmt.Fprintf(os.Stderr, "Flags:\n") fs.PrintDefaults() } _ = fs.Parse(os.Args[1:]) purls := fs.Args() + var platformList []string + if *platforms != "" { + platformList = []string{*platforms} + } + if _, err := mirror.ParsePlatforms(platformList); err != nil { + fmt.Fprintf(os.Stderr, "error: %v\n", err) + os.Exit(1) + } // Determine source var source mirror.Source @@ -443,9 +453,9 @@ func runMirror() { fmt.Fprintf(os.Stderr, "error reading SBOM %s: %v\n", *sbomPath, err) os.Exit(1) } - source = &mirror.SBOMSource{Data: data, Name: *sbomPath} + source = &mirror.SBOMSource{Data: data, Name: *sbomPath, Platforms: platformList} case len(purls) > 0: - source = &mirror.PURLSource{PURLs: purls} + source = &mirror.PURLSource{PURLs: purls, Platforms: platformList} default: fmt.Fprintf(os.Stderr, "error: provide PURLs or --sbom\n") fs.Usage() @@ -517,7 +527,7 @@ func runMirror() { } // Build proxy with the same upstream clients and authentication as serve - proxy := server.NewMirrorProxy(cfg, db, store, logger) + proxy, images := server.NewMirrorProxy(cfg, db, store, logger) proxy.Denylist, err = denylist.New(cfg.Denylist.Packages) if err != nil { _ = db.Close() @@ -529,6 +539,12 @@ func runMirror() { proxy.MetadataMaxSize = cfg.ParseMetadataMaxSize() m := mirror.New(proxy, db, store, logger, *concurrency) + m.Images = images + if cfg.Scanning.Enabled { + // Scanners fetch staged layers from the running server, which this + // command does not start. + m.Images = mirror.UnavailableImages{Reason: "container images cannot be mirrored by this command while scanning is enabled; use the mirror API of the running server"} + } ctx, cancel := context.WithCancel(context.Background()) go func() { diff --git a/docs/architecture.md b/docs/architecture.md index 0d37a60..7529d5e 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -316,7 +316,7 @@ Package metadata enrichment. Fetches license, description, homepage, repository ### `internal/mirror` -Selective package mirroring for pre-populating the proxy cache. Supports multiple input sources: individual PURLs (versioned or unversioned), CycloneDX/SPDX SBOM files, and full registry enumeration. Uses a bounded worker pool backed by `errgroup` to download artifacts in parallel, reusing `handler.Proxy.GetOrFetchArtifact()` for the actual fetch-and-cache work. +Selective package mirroring for pre-populating the proxy cache. Supports multiple input sources: individual PURLs (versioned or unversioned), CycloneDX/SPDX SBOM files, and full registry enumeration. Uses a bounded worker pool backed by `errgroup` to download artifacts in parallel, reusing `handler.Proxy.GetOrFetchArtifact()` for the actual fetch-and-cache work. Container images (`pkg:docker`, `pkg:oci`) go through `handler.ContainerHandler.MirrorImage()` instead, which walks index, platform manifests and blobs and stores them under the keys a pull reads. The package also provides a `MetadataCache` for storing raw upstream metadata blobs so the proxy can serve metadata responses offline. The `JobStore` manages async mirror jobs exposed via the `/api/mirror` endpoints. diff --git a/docs/configuration.md b/docs/configuration.md index 675d45a..4986a7d 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -829,6 +829,14 @@ curl -X POST http://localhost:8080/api/mirror \ -d '{"sbom":{"bomFormat":"CycloneDX","components":[{"purl":"pkg:npm/lodash@4.17.21"}]}}' ``` +`platforms` limits which platforms of container images are mirrored, like the command's `--platform` flag: + +```bash +curl -X POST http://localhost:8080/api/mirror \ + -H "Content-Type: application/json" \ + -d '{"purls":["pkg:docker/library/alpine@3.20"],"platforms":["linux/amd64"]}' +``` + ## Mirror Command The `proxy mirror` command pre-populates the cache from various sources. It accepts the same storage and database flags as `serve`. @@ -838,6 +846,7 @@ The `proxy mirror` command pre-populates the cache from various sources. It acce | `--sbom` | | Path to CycloneDX or SPDX SBOM file | | `--concurrency` | `4` | Number of parallel downloads | | `--dry-run` | `false` | Show what would be mirrored without downloading | +| `--platform` | | Comma-separated `os/arch[/variant]` list limiting which platforms of container images are mirrored | | `--config` | | Path to configuration file | | `--storage-url` | | Storage URL | | `--database-driver` | | Database driver | @@ -850,6 +859,24 @@ Positional arguments are treated as PURLs: proxy mirror pkg:npm/lodash@4.17.21 pkg:cargo/serde@1.0.0 ``` +### Container images + +`pkg:docker` and `pkg:oci` PURLs mirror a container image the way a pull does: the manifest or index, the platform manifests, then config and layer blobs. Everything is stored under the keys a pull through `/v2` reads, so a later `docker pull` or containerd pull is served from cache, also when the upstream is unreachable. + +```bash +proxy mirror --platform linux/amd64,linux/arm64 pkg:docker/library/alpine@3.20 +proxy mirror 'pkg:docker/containerd/alpine@3.14.0?repository_url=ghcr.io' +proxy mirror 'pkg:oci/debian@sha256:...?repository_url=docker.io/library/debian&tag=bookworm' +``` + +- The version is a tag or a digest. Unversioned image PURLs fail; mirroring all tags is not supported. +- Without `repository_url` the name is routed like a pull without containerd's `ns` parameter: the default OCI registry, repository prefix routes and `upstream/{name}/`. Single-segment Docker Hub names get the `library/` prefix. +- `repository_url` is resolved like containerd's `ns` parameter, so it has to name the default registry, Docker Hub, a configured `upstream.oci` host or a host listed in `upstream.oci_mirrors`. Other hosts are refused. +- `pkg:oci` needs `repository_url`. With `tag=` the tag is mirrored as well; if it now points to another digest, both are mirrored and a warning is logged. `arch=` limits the platforms when `--platform` is not given. +- Without `--platform` every platform of an index is mirrored, including Windows images, which can be large. Attestation manifests of the selected platforms are mirrored too, because Docker fetches them on every pull. +- Non-distributable (foreign) layers are skipped. +- While `scanning.enabled` is set, the command refuses images, because scanners fetch staged layers from the running server. Use the mirror API instead. + ## Docker ### SQLite with Local Storage diff --git a/internal/handler/container_mirror.go b/internal/handler/container_mirror.go new file mode 100644 index 0000000..f9e3d49 --- /dev/null +++ b/internal/handler/container_mirror.go @@ -0,0 +1,373 @@ +package handler + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "net/url" + "regexp" + "slices" + "strings" +) + +const ( + // maxMirrorIndexDepth bounds how deep nested indexes are followed. + maxMirrorIndexDepth = 2 + // mirrorErrorBodyLimit bounds how much of an error response is quoted. + mirrorErrorBodyLimit = 200 + + dockerReferenceTypeAnnotation = "vnd.docker.reference.type" + dockerReferenceDigestAnnotation = "vnd.docker.reference.digest" + attestationManifestReference = "attestation-manifest" +) + +var ( + // Grammar from the OCI distribution spec. + mirrorNamePattern = regexp.MustCompile(`^[a-z0-9]+((\.|_|__|-+)[a-z0-9]+)*(/[a-z0-9]+((\.|_|__|-+)[a-z0-9]+)*)*$`) + mirrorTagPattern = regexp.MustCompile(`^[a-zA-Z0-9_][a-zA-Z0-9._-]{0,127}$`) + mirrorDigestPattern = regexp.MustCompile(`^sha256:[a-f0-9]{64}$`) +) + +// dockerHubMirrorAliases are registry hosts that name Docker Hub in image +// references. hub.docker.com is the default registry of the docker PURL type. +var dockerHubMirrorAliases = []string{"docker.io", "index.docker.io", "registry-1.docker.io", "hub.docker.com"} //nolint:gochecknoglobals // fixed alias list + +// ImageRef names an image to mirror. +type ImageRef struct { + // Registry is the registry host the image reference names, as containerd + // sends it in the ns query parameter. Empty routes Name like a pull + // without ns. + Registry string + // Name is the repository name. + Name string + // References are the tags and digests to mirror. + References []string +} + +// Platform selects the entries of an image index to mirror. Empty fields +// match any value. +type Platform struct { + OS string + Architecture string + Variant string +} + +// ParsePlatform parses os/architecture[/variant]. +func ParsePlatform(value string) (Platform, error) { + os, rest, _ := strings.Cut(value, "/") + architecture, variant, _ := strings.Cut(rest, "/") + if os == "" || architecture == "" || strings.Contains(variant, "/") { + return Platform{}, fmt.Errorf("invalid platform %q, want os/architecture[/variant]", value) + } + return Platform{OS: os, Architecture: architecture, Variant: variant}, nil +} + +func (p Platform) matches(entry *ociPlatform) bool { + if entry == nil { + return false + } + if p.OS != "" && p.OS != entry.OS { + return false + } + if p.Architecture != "" && p.Architecture != entry.Architecture { + return false + } + return p.Variant == "" || p.Variant == platformVariant(entry) +} + +// platformVariant fills in the variant arm64 images usually leave out. +func platformVariant(entry *ociPlatform) string { + if entry.Variant == "" && entry.Architecture == "arm64" { + return "v8" + } + return entry.Variant +} + +// ImageResult reports what mirroring one image did. +type ImageResult struct { + // Fetched counts manifests and blobs that were not cached before. + Fetched int + // Bytes is the size of the blobs fetched for this image. + Bytes int64 + Warnings []string +} + +type ociDescriptor struct { + MediaType string `json:"mediaType"` + Digest string `json:"digest"` + URLs []string `json:"urls"` + Platform *ociPlatform `json:"platform"` + Annotations map[string]string `json:"annotations"` +} + +type ociPlatform struct { + OS string `json:"os"` + Architecture string `json:"architecture"` + Variant string `json:"variant"` +} + +type ociManifest struct { + SchemaVersion int `json:"schemaVersion"` + MediaType string `json:"mediaType"` + Manifests []ociDescriptor `json:"manifests"` + Config *ociDescriptor `json:"config"` + Layers []ociDescriptor `json:"layers"` +} + +type imageMirror struct { + h *ContainerHandler + ref ImageRef + registryURL string + upstreamName string + cacheName string + platforms []Platform + seen map[string]bool + result ImageResult +} + +// MirrorImage fetches an image the way a client pull does: the manifest or +// index for each reference, the platform manifests, then config and layer +// blobs. Everything is stored under the keys a later pull through this handler +// reads, so the pull is served from cache. Platforms limit which index entries +// are followed; without them every platform is mirrored. +func (h *ContainerHandler) MirrorImage(ctx context.Context, ref ImageRef, platforms []Platform) (ImageResult, error) { + ref, err := normalizeImageRef(ref, h.registryURL) + if err != nil { + return ImageResult{}, err + } + route := &http.Request{URL: &url.URL{RawQuery: url.Values{namespaceQueryParam: {ref.Registry}}.Encode()}} + if ref.Registry == "" { + route.URL.RawQuery = "" + } + registryURL, upstreamName, cacheName, ok := h.registryForRequest(route, ref.Name) + if !ok { + if ref.Registry != "" { + return ImageResult{}, fmt.Errorf("registry %q is not configured as an OCI upstream", ref.Registry) + } + return ImageResult{}, fmt.Errorf("no OCI upstream for %q", ref.Name) + } + m := &imageMirror{ + h: h, + ref: ref, + registryURL: registryURL, + upstreamName: upstreamName, + cacheName: cacheName, + platforms: platforms, + seen: make(map[string]bool), + } + var digests []string + for _, reference := range ref.References { + digest, err := m.mirrorManifest(ctx, reference, 0) + if err != nil { + return m.result, fmt.Errorf("%s:%s: %w", ref.Name, reference, err) + } + digests = append(digests, digest) + } + if len(digests) == 2 && digests[0] != digests[1] && mirrorDigestPattern.MatchString(ref.References[0]) { + m.result.Warnings = append(m.result.Warnings, fmt.Sprintf("tag %s now points to %s, not %s; both were mirrored", + ref.References[1], digests[1], ref.References[0])) + } + return m.result, nil +} + +// normalizeImageRef validates the reference before any network access and +// adds the library/ prefix Docker Hub clients use for official images. +func normalizeImageRef(ref ImageRef, defaultRegistryURL string) (ImageRef, error) { + if slices.Contains(dockerHubMirrorAliases, strings.ToLower(ref.Registry)) { + ref.Registry = "docker.io" + } + dockerHub := ref.Registry == "docker.io" || (ref.Registry == "" && defaultRegistryURL == dockerHubRegistry) + if dockerHub && !strings.Contains(ref.Name, "/") { + ref.Name = "library/" + ref.Name + } + if !mirrorNamePattern.MatchString(ref.Name) { + return ref, fmt.Errorf("invalid repository name %q", ref.Name) + } + if len(ref.References) == 0 { + return ref, errors.New("a tag or digest is required; mirroring all tags is not supported") + } + for _, reference := range ref.References { + if !mirrorTagPattern.MatchString(reference) && !mirrorDigestPattern.MatchString(reference) { + return ref, fmt.Errorf("invalid tag or digest %q", reference) + } + } + return ref, nil +} + +// mirrorManifest stores the manifest for reference through the handler's own +// manifest route, then follows what it references. It returns the manifest +// digest. +func (m *imageMirror) mirrorManifest(ctx context.Context, reference string, depth int) (string, error) { + cacheAccept := normalizeContainerManifestAccept(containerManifestAccept(&http.Request{Header: http.Header{}})) + before := m.h.loadContainerManifestForAccept(ctx, m.registryURL, m.upstreamName, reference, cacheAccept, cacheAccept) + + target := "/" + m.ref.Name + "/manifests/" + reference + if m.ref.Registry != "" { + target += "?" + url.Values{namespaceQueryParam: {m.ref.Registry}}.Encode() + } + request, err := http.NewRequestWithContext(ctx, http.MethodGet, target, nil) + if err != nil { + return "", err + } + response := newMirrorResponse() + m.h.Routes().ServeHTTP(response, request) + if response.status != http.StatusOK { + return "", fmt.Errorf("manifest %s: status %d: %s", reference, response.status, response.errorText()) + } + if response.header.Get("Warning") != "" { + m.result.Warnings = append(m.result.Warnings, fmt.Sprintf("manifest %s served from stale cache, upstream unreachable", reference)) + } + digest := response.header.Get("Docker-Content-Digest") + if !mirrorDigestPattern.MatchString(digest) { + digest = sha256Digest(response.body.Bytes()) + } + if before == nil || before.contentDigest != digest { + m.result.Fetched++ + } + if m.seen[digest] { + return digest, nil + } + m.seen[digest] = true + + var manifest ociManifest + if err := json.Unmarshal(response.body.Bytes(), &manifest); err != nil { + return "", fmt.Errorf("manifest %s: %w", reference, err) + } + if manifest.SchemaVersion == 1 { + return "", fmt.Errorf("manifest %s: schema 1 manifests are not supported", reference) + } + if manifest.Manifests != nil { + return digest, m.mirrorIndex(ctx, reference, &manifest, depth) + } + return digest, m.mirrorBlobs(ctx, &manifest) +} + +// mirrorIndex follows the index entries for the wanted platforms. Docker +// fetches the attestation manifests of an index on every pull, so those of the +// selected platforms are mirrored too. +func (m *imageMirror) mirrorIndex(ctx context.Context, reference string, index *ociManifest, depth int) error { + if depth >= maxMirrorIndexDepth { + return fmt.Errorf("index %s: nested deeper than %d levels", reference, maxMirrorIndexDepth) + } + var selected, attestations []ociDescriptor + selectedDigests := make(map[string]bool) + for _, entry := range index.Manifests { + if !mirrorDigestPattern.MatchString(entry.Digest) { + return fmt.Errorf("index %s: invalid digest %q", reference, entry.Digest) + } + if entry.Annotations[dockerReferenceTypeAnnotation] == attestationManifestReference { + attestations = append(attestations, entry) + continue + } + if m.wantsPlatform(entry.Platform) { + selected = append(selected, entry) + selectedDigests[entry.Digest] = true + } + } + if len(selected) == 0 && len(m.platforms) > 0 { + return fmt.Errorf("index %s: no manifest for the requested platforms", reference) + } + for _, entry := range attestations { + if len(m.platforms) == 0 || selectedDigests[entry.Annotations[dockerReferenceDigestAnnotation]] { + selected = append(selected, entry) + } + } + for _, entry := range selected { + if _, err := m.mirrorManifest(ctx, entry.Digest, depth+1); err != nil { + return err + } + } + return nil +} + +func (m *imageMirror) wantsPlatform(platform *ociPlatform) bool { + if len(m.platforms) == 0 || platform == nil { + // Entries without a platform are nested indexes or artifacts; a + // filter cannot judge them, so they are followed. + return true + } + for _, want := range m.platforms { + if want.matches(platform) { + return true + } + } + return false +} + +func (m *imageMirror) mirrorBlobs(ctx context.Context, manifest *ociManifest) error { + blobs := manifest.Layers + if manifest.Config != nil { + blobs = append([]ociDescriptor{*manifest.Config}, blobs...) + } + for _, blob := range blobs { + if len(blob.URLs) > 0 || strings.Contains(blob.MediaType, "nondistributable") || strings.Contains(blob.MediaType, "foreign") { + m.result.Warnings = append(m.result.Warnings, fmt.Sprintf("skipped non-distributable layer %s", blob.Digest)) + continue + } + if !mirrorDigestPattern.MatchString(blob.Digest) { + return fmt.Errorf("invalid blob digest %q", blob.Digest) + } + if m.seen[blob.Digest] { + continue + } + m.seen[blob.Digest] = true + if err := m.mirrorBlob(ctx, blob.Digest); err != nil { + return err + } + } + return nil +} + +// mirrorBlob stores a blob under the same identity handleBlobDownload uses. +func (m *imageMirror) mirrorBlob(ctx context.Context, digest string) error { + proxy := m.h.proxy + cached, err := proxy.GetCachedArtifact(ctx, "oci", m.cacheName, digest, digest) + if err != nil { + return fmt.Errorf("blob %s: %w", digest, err) + } + if cached != nil { + if cached.Reader != nil { + _ = cached.Reader.Close() + } + return nil + } + result, err := proxy.GetOrFetchArtifactFromURLWithDigest(ctx, "oci", m.cacheName, digest, digest, + fmt.Sprintf("%s/v2/%s/blobs/%s", m.registryURL, m.upstreamName, digest), digest) + if err != nil { + return fmt.Errorf("blob %s: %w", digest, err) + } + if result.Reader != nil { + _ = result.Reader.Close() + } + m.result.Fetched++ + m.result.Bytes += result.Artifact.Size + return nil +} + +// mirrorResponse collects the handler's manifest response in memory. +type mirrorResponse struct { + header http.Header + status int + body bytes.Buffer +} + +func newMirrorResponse() *mirrorResponse { + return &mirrorResponse{header: make(http.Header), status: http.StatusOK} +} + +func (r *mirrorResponse) Header() http.Header { return r.header } + +func (r *mirrorResponse) WriteHeader(status int) { r.status = status } + +func (r *mirrorResponse) Write(p []byte) (int, error) { return r.body.Write(p) } + +func (r *mirrorResponse) errorText() string { + text := strings.TrimSpace(r.body.String()) + if len(text) > mirrorErrorBodyLimit { + text = text[:mirrorErrorBodyLimit] + "..." + } + return text +} diff --git a/internal/handler/container_mirror_test.go b/internal/handler/container_mirror_test.go new file mode 100644 index 0000000..fc4e8a6 --- /dev/null +++ b/internal/handler/container_mirror_test.go @@ -0,0 +1,295 @@ +package handler + +import ( + "context" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "sync/atomic" + "testing" + + "github.com/git-pkgs/registries/fetch" +) + +// fakeImageRegistry serves manifests and blobs of a small multi-arch image, +// a single-arch image and a moved tag. It fails with 503 once offline. +type fakeImageRegistry struct { + server *httptest.Server + offline atomic.Bool + manifests map[string]fakeManifest + blobs map[string]string + blobGets atomic.Int32 +} + +type fakeImage struct { + indexDigest string + amd64Digest string + arm64Digest string + amd64Layer string + arm64Layer string + singleDigest string + singleLayer string + // attestationLayers holds the amd64 and arm64 provenance statements. + attestationLayers []string +} + +func newFakeImageRegistry(t *testing.T) (*fakeImageRegistry, fakeImage) { + t.Helper() + const repository = "team/app" + r := &fakeImageRegistry{manifests: make(map[string]fakeManifest), blobs: make(map[string]string)} + var image fakeImage + + addBlob := func(content string) string { + digest := sha256Digest([]byte(content)) + r.blobs["/v2/"+repository+"/blobs/"+digest] = content + return digest + } + addManifest := func(contentType string, doc any, tags ...string) string { + body, err := json.Marshal(doc) + if err != nil { + t.Fatal(err) + } + digest := sha256Digest(body) + for _, reference := range append(tags, digest) { + r.manifests["/v2/"+repository+"/manifests/"+reference] = fakeManifest{contentType: contentType, body: string(body)} + } + return digest + } + imageManifest := func(platform string) (string, string) { + config := addBlob(`{"architecture":"` + platform + `"}`) + layer := addBlob("layer for " + platform) + digest := addManifest(ociManifestMediaType, map[string]any{ + "schemaVersion": 2, "mediaType": ociManifestMediaType, + "config": map[string]any{"mediaType": "application/vnd.oci.image.config.v1+json", "digest": config}, + "layers": []map[string]any{ + {"mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": layer}, + {"mediaType": "application/vnd.oci.image.layer.nondistributable.v1.tar+gzip", "digest": sha256Digest([]byte("foreign")), "urls": []string{"https://example.invalid/layer"}}, + }, + }) + return digest, layer + } + + image.amd64Digest, image.amd64Layer = imageManifest("amd64") + image.arm64Digest, image.arm64Layer = imageManifest("arm64") + attestation := func(platformDigest, platform string) map[string]any { + statement := addBlob("provenance for " + platform) + digest := addManifest(ociManifestMediaType, map[string]any{ + "schemaVersion": 2, "mediaType": ociManifestMediaType, + "config": map[string]any{"mediaType": "application/vnd.oci.image.config.v1+json", "digest": addBlob("{}")}, + "layers": []map[string]any{{"mediaType": "application/vnd.in-toto+json", "digest": statement}}, + }) + image.attestationLayers = append(image.attestationLayers, statement) + return map[string]any{ + "mediaType": ociManifestMediaType, "digest": digest, + "platform": map[string]string{"os": "unknown", "architecture": "unknown"}, + "annotations": map[string]string{ + dockerReferenceTypeAnnotation: attestationManifestReference, + dockerReferenceDigestAnnotation: platformDigest, + }, + } + } + image.indexDigest = addManifest(ociImageIndexMediaType, map[string]any{ + "schemaVersion": 2, "mediaType": ociImageIndexMediaType, + "manifests": []map[string]any{ + {"mediaType": ociManifestMediaType, "digest": image.amd64Digest, "platform": map[string]string{"os": "linux", "architecture": "amd64"}}, + {"mediaType": ociManifestMediaType, "digest": image.arm64Digest, "platform": map[string]string{"os": "linux", "architecture": "arm64", "variant": "v8"}}, + attestation(image.amd64Digest, "amd64"), + attestation(image.arm64Digest, "arm64"), + }, + }, "multi") + + image.singleDigest, image.singleLayer = imageManifest("single") + r.manifests["/v2/"+repository+"/manifests/single"] = r.manifests["/v2/"+repository+"/manifests/"+image.singleDigest] + + r.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + if r.offline.Load() { + http.Error(w, "offline", http.StatusServiceUnavailable) + return + } + if doc, ok := r.manifests[req.URL.Path]; ok { + w.Header().Set("Content-Type", doc.contentType) + w.Header().Set("Docker-Content-Digest", sha256Digest([]byte(doc.body))) + _, _ = io.WriteString(w, doc.body) + return + } + if blob, ok := r.blobs[req.URL.Path]; ok { + r.blobGets.Add(1) + _, _ = io.WriteString(w, blob) + return + } + http.NotFound(w, req) + })) + t.Cleanup(r.server.Close) + return r, image +} + +func newMirrorTestProxy(t *testing.T, registry *fakeImageRegistry) *Proxy { + t.Helper() + proxy, _, _, _ := setupTestProxy(t) + proxy.HTTPClient = registry.server.Client() + proxy.Fetcher = fetch.NewFetcher(fetch.WithHTTPClient(registry.server.Client()), fetch.WithMaxRetries(0)) + proxy.MetadataTTL = 0 + return proxy +} + +func pull(t *testing.T, h *ContainerHandler, target, accept string) *httptest.ResponseRecorder { + t.Helper() + request := httptest.NewRequest(http.MethodGet, target, nil) + if accept != "" { + request.Header.Set("Accept", accept) + } + response := httptest.NewRecorder() + h.Routes().ServeHTTP(response, request) + return response +} + +func TestMirrorImageThenPullOffline(t *testing.T) { + registry, image := newFakeImageRegistry(t) + proxy := newMirrorTestProxy(t, registry) + mirrorHandler := NewContainerHandlerWithRegistry(proxy, "http://proxy.example", registry.server.URL) + + result, err := mirrorHandler.MirrorImage(context.Background(), ImageRef{Name: "team/app", References: []string{"multi", "single"}}, nil) + if err != nil { + t.Fatalf("MirrorImage: %v", err) + } + if len(result.Warnings) == 0 || !strings.Contains(strings.Join(result.Warnings, "\n"), "non-distributable") { + t.Errorf("warnings = %v, want the skipped foreign layer", result.Warnings) + } + registry.offline.Store(true) + + // A fresh handler instance, as after a restart, serves the pull. + h := NewContainerHandlerWithRegistry(proxy, "http://proxy.example", registry.server.URL) + for _, accept := range []string{dockerManifestAccept, containerdManifestAccept} { + for _, reference := range []string{"multi", image.indexDigest, image.amd64Digest, image.arm64Digest, "single", image.singleDigest} { + if response := pull(t, h, "/team/app/manifests/"+reference, accept); response.Code != http.StatusOK { + t.Errorf("offline manifest %s with %.30q: status %d: %s", reference, accept, response.Code, response.Body.String()) + } + } + } + for _, layer := range append([]string{image.amd64Layer, image.arm64Layer, image.singleLayer}, image.attestationLayers...) { + if response := pull(t, h, "/team/app/blobs/"+layer, ""); response.Code != http.StatusOK { + t.Errorf("offline blob %s: status %d", layer, response.Code) + } + } + + registry.offline.Store(false) + gets := registry.blobGets.Load() + again, err := mirrorHandler.MirrorImage(context.Background(), ImageRef{Name: "team/app", References: []string{"multi"}}, nil) + if err != nil { + t.Fatalf("second MirrorImage: %v", err) + } + if again.Fetched != 0 || registry.blobGets.Load() != gets { + t.Errorf("second run fetched %d items and %d blobs, want nothing new", again.Fetched, registry.blobGets.Load()-gets) + } +} + +func TestMirrorImagePlatformFilter(t *testing.T) { + registry, image := newFakeImageRegistry(t) + proxy := newMirrorTestProxy(t, registry) + h := NewContainerHandlerWithRegistry(proxy, "http://proxy.example", registry.server.URL) + + platform, err := ParsePlatform("linux/arm64") + if err != nil { + t.Fatal(err) + } + if _, err := h.MirrorImage(context.Background(), ImageRef{Name: "team/app", References: []string{"multi"}}, []Platform{platform}); err != nil { + t.Fatalf("MirrorImage: %v", err) + } + registry.offline.Store(true) + if response := pull(t, h, "/team/app/blobs/"+image.arm64Layer, ""); response.Code != http.StatusOK { + t.Errorf("arm64 layer status = %d, want 200", response.Code) + } + if response := pull(t, h, "/team/app/blobs/"+image.attestationLayers[1], ""); response.Code != http.StatusOK { + t.Errorf("arm64 attestation status = %d, want 200", response.Code) + } + for _, layer := range []string{image.amd64Layer, image.attestationLayers[0]} { + if response := pull(t, h, "/team/app/blobs/"+layer, ""); response.Code == http.StatusOK { + t.Errorf("amd64 blob %s was mirrored despite the linux/arm64 filter", layer) + } + } + + registry.offline.Store(false) + riscv, _ := ParsePlatform("linux/riscv64") + if _, err := h.MirrorImage(context.Background(), ImageRef{Name: "team/app", References: []string{"multi"}}, []Platform{riscv}); err == nil { + t.Error("a filter matching no index entry did not fail") + } +} + +func TestMirrorImageMovedTagMirrorsBothTargets(t *testing.T) { + registry, image := newFakeImageRegistry(t) + proxy := newMirrorTestProxy(t, registry) + h := NewContainerHandlerWithRegistry(proxy, "http://proxy.example", registry.server.URL) + + result, err := h.MirrorImage(context.Background(), ImageRef{Name: "team/app", References: []string{image.singleDigest, "multi"}}, nil) + if err != nil { + t.Fatalf("MirrorImage: %v", err) + } + if !strings.Contains(strings.Join(result.Warnings, "\n"), "now points to") { + t.Errorf("warnings = %v, want the moved-tag warning", result.Warnings) + } + registry.offline.Store(true) + for _, layer := range []string{image.singleLayer, image.amd64Layer} { + if response := pull(t, h, "/team/app/blobs/"+layer, ""); response.Code != http.StatusOK { + t.Errorf("blob %s status = %d, want 200", layer, response.Code) + } + } +} + +func TestMirrorImageNamespaceRouteMatchesContainerdPull(t *testing.T) { + registry, image := newFakeImageRegistry(t) + proxy := newMirrorTestProxy(t, registry) + named := map[string]string{"internal": registry.server.URL} + h := NewContainerHandlerWithRegistry(proxy, "http://proxy.example", dockerHubRegistry, named) + host := strings.TrimPrefix(registry.server.URL, "http://") + + if _, err := h.MirrorImage(context.Background(), ImageRef{Registry: host, Name: "team/app", References: []string{"single"}}, nil); err != nil { + t.Fatalf("MirrorImage: %v", err) + } + registry.offline.Store(true) + ns := "?" + url.Values{namespaceQueryParam: {host}}.Encode() + if response := pull(t, h, "/team/app/manifests/single"+ns, containerdManifestAccept); response.Code != http.StatusOK { + t.Errorf("containerd ns manifest status = %d: %s", response.Code, response.Body.String()) + } + if response := pull(t, h, "/team/app/blobs/"+image.singleLayer+ns, ""); response.Code != http.StatusOK { + t.Errorf("containerd ns blob status = %d", response.Code) + } + if response := pull(t, h, "/upstream/internal/team/app/blobs/"+image.singleLayer, ""); response.Code != http.StatusOK { + t.Errorf("prefix-route blob status = %d", response.Code) + } + + if _, err := h.MirrorImage(context.Background(), ImageRef{Registry: "unknown.example", Name: "team/app", References: []string{"single"}}, nil); err == nil { + t.Error("an unconfigured registry host was accepted") + } +} + +func TestNormalizeImageRef(t *testing.T) { + tests := []struct { + name string + ref ImageRef + defaultRT string + wantName string + wantErr bool + }{ + {name: "docker hub default adds library", ref: ImageRef{Name: "alpine", References: []string{"3.20"}}, defaultRT: dockerHubRegistry, wantName: "library/alpine"}, + {name: "docker hub ns adds library", ref: ImageRef{Registry: "hub.docker.com", Name: "alpine", References: []string{"3.20"}}, defaultRT: "https://registry.example", wantName: "library/alpine"}, + {name: "other default keeps name", ref: ImageRef{Name: "alpine", References: []string{"3.20"}}, defaultRT: "https://registry.example", wantName: "alpine"}, + {name: "path traversal", ref: ImageRef{Name: "../../x", References: []string{"1"}}, defaultRT: dockerHubRegistry, wantErr: true}, + {name: "query in tag", ref: ImageRef{Name: "library/alpine", References: []string{"3.20?x#y"}}, defaultRT: dockerHubRegistry, wantErr: true}, + {name: "short digest", ref: ImageRef{Name: "library/alpine", References: []string{"sha256:abcd"}}, defaultRT: dockerHubRegistry, wantErr: true}, + {name: "no reference", ref: ImageRef{Name: "library/alpine"}, defaultRT: dockerHubRegistry, wantErr: true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := normalizeImageRef(tt.ref, tt.defaultRT) + if (err != nil) != tt.wantErr { + t.Fatalf("err = %v, wantErr %v", err, tt.wantErr) + } + if !tt.wantErr && got.Name != tt.wantName { + t.Errorf("name = %q, want %q", got.Name, tt.wantName) + } + }) + } +} diff --git a/internal/mirror/image.go b/internal/mirror/image.go new file mode 100644 index 0000000..bc21937 --- /dev/null +++ b/internal/mirror/image.go @@ -0,0 +1,126 @@ +package mirror + +import ( + "context" + "errors" + "fmt" + "strings" + + "github.com/git-pkgs/proxy/internal/handler" +) + +// ImageMirrorer fetches container images into the cache. +type ImageMirrorer interface { + MirrorImage(ctx context.Context, ref handler.ImageRef, platforms []handler.Platform) (handler.ImageResult, error) +} + +// UnavailableImages refuses every image with the given reason. +type UnavailableImages struct { + Reason string +} + +func (u UnavailableImages) MirrorImage(context.Context, handler.ImageRef, []handler.Platform) (handler.ImageResult, error) { + return handler.ImageResult{}, errors.New(u.Reason) +} + +func isImageEcosystem(ecosystem string) bool { + return ecosystem == "docker" || ecosystem == "oci" +} + +// ParsePlatforms parses os/architecture[/variant] values. +func ParsePlatforms(values []string) ([]handler.Platform, error) { + var platforms []handler.Platform + for _, value := range values { + for _, item := range strings.Split(value, ",") { + item = strings.TrimSpace(item) + if item == "" { + continue + } + platform, err := handler.ParsePlatform(item) + if err != nil { + return nil, err + } + platforms = append(platforms, platform) + } + } + return platforms, nil +} + +// imageRef maps a docker or oci PURL to the image reference and platforms to +// mirror. +// +// - pkg:docker//@, optionally with +// repository_url=[/]. +// - pkg:oci/@?repository_url=/, optionally +// with tag= and arch=. OCI PURLs have no default +// registry, so repository_url is required. +func imageRef(pv PackageVersion) (handler.ImageRef, []handler.Platform, error) { + platforms, err := ParsePlatforms(pv.Platforms) + if err != nil { + return handler.ImageRef{}, nil, err + } + ref := handler.ImageRef{Name: pv.Name} + if pv.Version != "" { + ref.References = []string{pv.Version} + } + repositoryURL := pv.Qualifiers["repository_url"] + if pv.Ecosystem == "oci" { + if repositoryURL == "" { + return ref, nil, errors.New("pkg:oci needs a repository_url qualifier") + } + if tag := pv.Qualifiers["tag"]; tag != "" && tag != pv.Version { + ref.References = append(ref.References, tag) + } + if arch := pv.Qualifiers["arch"]; arch != "" && len(platforms) == 0 { + platforms = []handler.Platform{{Architecture: arch}} + } + } + if repositoryURL != "" { + repositoryURL = strings.TrimPrefix(strings.TrimPrefix(repositoryURL, "https://"), "http://") + host, repository, _ := strings.Cut(strings.Trim(repositoryURL, "/"), "/") + ref.Registry = host + switch { + case pv.Ecosystem == "oci" && repository != "": + // The repository URL names the repository itself. + ref.Name = repository + case repository != "": + ref.Name = repository + "/" + pv.Name + } + } + return ref, platforms, nil +} + +func (m *Mirror) mirrorImage(ctx context.Context, pv PackageVersion, tracker *progressTracker) { + fail := func(err error) { + tracker.failed.Add(1) + tracker.addError(pv.Ecosystem, pv.Name, pv.Version, err.Error()) + m.logger.Warn("mirror failed", "ecosystem", pv.Ecosystem, "name", pv.Name, "version", pv.Version, "error", err) + } + // pv.String keeps the namespace separator, which MakeString escapes for + // docker names. + if m.proxy != nil && m.proxy.Denylist.Denied(pv.String()) { + fail(fmt.Errorf("%w: %s", handler.ErrVersionDenied, pv)) + return + } + ref, platforms, err := imageRef(pv) + if err != nil { + fail(err) + return + } + result, err := m.Images.MirrorImage(ctx, ref, platforms) + for _, warning := range result.Warnings { + m.logger.Warn("mirror warning", "ecosystem", pv.Ecosystem, "name", pv.Name, "version", pv.Version, "warning", warning) + } + tracker.bytes.Add(result.Bytes) + if err != nil { + fail(err) + return + } + if result.Fetched == 0 { + tracker.skipped.Add(1) + m.logger.Debug("already cached", "ecosystem", pv.Ecosystem, "name", pv.Name, "version", pv.Version) + return + } + tracker.completed.Add(1) + m.logger.Info("mirrored", "ecosystem", pv.Ecosystem, "name", pv.Name, "version", pv.Version, "size", result.Bytes) +} diff --git a/internal/mirror/image_test.go b/internal/mirror/image_test.go new file mode 100644 index 0000000..339e86c --- /dev/null +++ b/internal/mirror/image_test.go @@ -0,0 +1,174 @@ +package mirror + +import ( + "context" + "errors" + "reflect" + "strings" + "sync" + "testing" + + "github.com/git-pkgs/proxy/internal/denylist" + "github.com/git-pkgs/proxy/internal/handler" +) + +func TestImageRefFromPURL(t *testing.T) { + const digest = "sha256:1111111111111111111111111111111111111111111111111111111111111111" + tests := []struct { + purl string + platforms []string + wantRef handler.ImageRef + wantPlatforms []handler.Platform + wantErr string + }{ + { + purl: "pkg:docker/library/alpine@3.20", + wantRef: handler.ImageRef{Name: "library/alpine", References: []string{"3.20"}}, + }, + { + purl: "pkg:docker/alpine@3.20?repository_url=ghcr.io", + platforms: []string{"linux/arm64/v8"}, + wantRef: handler.ImageRef{Registry: "ghcr.io", Name: "alpine", References: []string{"3.20"}}, + wantPlatforms: []handler.Platform{{OS: "linux", Architecture: "arm64", Variant: "v8"}}, + }, + { + purl: "pkg:docker/app@1.0?repository_url=https://registry.example/team", + wantRef: handler.ImageRef{Registry: "registry.example", Name: "team/app", References: []string{"1.0"}}, + }, + { + purl: "pkg:oci/debian@" + digest + "?repository_url=docker.io/library/debian&tag=bookworm&arch=arm64", + wantRef: handler.ImageRef{Registry: "docker.io", Name: "library/debian", References: []string{digest, "bookworm"}}, + wantPlatforms: []handler.Platform{{Architecture: "arm64"}}, + }, + { + purl: "pkg:oci/debian@" + digest + "?repository_url=docker.io/library/debian&arch=arm64", + platforms: []string{"linux/amd64"}, + wantRef: handler.ImageRef{Registry: "docker.io", Name: "library/debian", References: []string{digest}}, + wantPlatforms: []handler.Platform{{OS: "linux", Architecture: "amd64"}}, + }, + { + purl: "pkg:oci/debian@" + digest, + wantErr: "repository_url", + }, + } + for _, tt := range tests { + t.Run(tt.purl, func(t *testing.T) { + var items []PackageVersion + source := &PURLSource{PURLs: []string{tt.purl}, Platforms: tt.platforms} + if err := source.Enumerate(context.Background(), func(pv PackageVersion) error { + items = append(items, pv) + return nil + }); err != nil { + t.Fatalf("Enumerate: %v", err) + } + if len(items) != 1 { + t.Fatalf("items = %d, want 1", len(items)) + } + ref, platforms, err := imageRef(items[0]) + if tt.wantErr != "" { + if err == nil || !strings.Contains(err.Error(), tt.wantErr) { + t.Fatalf("err = %v, want %q", err, tt.wantErr) + } + return + } + if err != nil { + t.Fatalf("imageRef: %v", err) + } + if !reflect.DeepEqual(ref, tt.wantRef) { + t.Errorf("ref = %+v, want %+v", ref, tt.wantRef) + } + if !reflect.DeepEqual(platforms, tt.wantPlatforms) { + t.Errorf("platforms = %+v, want %+v", platforms, tt.wantPlatforms) + } + }) + } +} + +func TestUnversionedImagePURLIsNotExpanded(t *testing.T) { + var items []PackageVersion + source := &PURLSource{PURLs: []string{"pkg:docker/library/alpine"}} + if err := source.Enumerate(context.Background(), func(pv PackageVersion) error { + items = append(items, pv) + return nil + }); err != nil { + t.Fatalf("Enumerate: %v", err) + } + if len(items) != 1 || items[0].Version != "" { + t.Fatalf("items = %+v, want one unversioned item", items) + } +} + +type fakeImages struct { + mu sync.Mutex + refs []handler.ImageRef + result handler.ImageResult + err error +} + +func (f *fakeImages) MirrorImage(_ context.Context, ref handler.ImageRef, _ []handler.Platform) (handler.ImageResult, error) { + f.mu.Lock() + defer f.mu.Unlock() + f.refs = append(f.refs, ref) + return f.result, f.err +} + +func TestMirrorRunCountsImages(t *testing.T) { + tests := []struct { + name string + result handler.ImageResult + err error + wantCompleted int64 + wantSkipped int64 + wantFailed int64 + }{ + {name: "fetched", result: handler.ImageResult{Fetched: 3, Bytes: 42}, wantCompleted: 1}, + {name: "cached", result: handler.ImageResult{}, wantSkipped: 1}, + {name: "failed", err: errors.New("boom"), wantFailed: 1}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + m := setupTestMirror(t, 1) + images := &fakeImages{result: tt.result, err: tt.err} + m.Images = images + progress, err := m.Run(context.Background(), &PURLSource{PURLs: []string{"pkg:docker/library/alpine@3.20"}}) + if err != nil { + t.Fatalf("Run: %v", err) + } + if progress.Completed != tt.wantCompleted || progress.Skipped != tt.wantSkipped || progress.Failed != tt.wantFailed { + t.Errorf("progress = %+v", progress) + } + if progress.Bytes != tt.result.Bytes { + t.Errorf("bytes = %d, want %d", progress.Bytes, tt.result.Bytes) + } + if len(images.refs) != 1 || images.refs[0].Name != "library/alpine" { + t.Errorf("refs = %+v", images.refs) + } + }) + } +} + +func TestMirrorRunDeniedImage(t *testing.T) { + m := setupTestMirror(t, 1) + policy, err := denylist.New([]string{"pkg:docker/library/alpine@3.20"}) + if err != nil { + t.Fatal(err) + } + m.proxy.Denylist = policy + images := &fakeImages{} + m.Images = images + + progress, err := m.Run(context.Background(), &PURLSource{PURLs: []string{"pkg:docker/library/alpine@3.20"}}) + if err != nil { + t.Fatalf("Run: %v", err) + } + if progress.Failed != 1 || len(images.refs) != 0 { + t.Errorf("denied image: failed = %d, mirror calls = %d", progress.Failed, len(images.refs)) + } +} + +func TestJobRequestRejectsInvalidPlatform(t *testing.T) { + js := &JobStore{} + if _, err := js.sourceFromRequest(JobRequest{PURLs: []string{"pkg:docker/library/alpine@3.20"}, Platforms: []string{"linux"}}); err == nil { + t.Error("invalid platform was accepted") + } +} diff --git a/internal/mirror/job.go b/internal/mirror/job.go index 70d7003..3d82966 100644 --- a/internal/mirror/job.go +++ b/internal/mirror/job.go @@ -39,6 +39,9 @@ type JobRequest struct { PURLs []string `json:"purls,omitempty"` SBOM json.RawMessage `json:"sbom,omitempty"` Registry string `json:"registry,omitempty"` + // Platforms limit which platforms of container images are mirrored, + // as os/architecture[/variant]. + Platforms []string `json:"platforms,omitempty"` } // JobStore manages in-memory mirror jobs. @@ -191,13 +194,16 @@ func (js *JobStore) runJob(ctx context.Context, cancel context.CancelFunc, job * } func (js *JobStore) sourceFromRequest(req JobRequest) (Source, error) { //nolint:ireturn // interface return is the design + if _, err := ParsePlatforms(req.Platforms); err != nil { + return nil, err + } switch { case len(req.PURLs) > 0 && len(req.SBOM) > 0: return nil, fmt.Errorf("request must include only one of purls or sbom") case len(req.PURLs) > 0: - return &PURLSource{PURLs: req.PURLs}, nil + return &PURLSource{PURLs: req.PURLs, Platforms: req.Platforms}, nil case len(req.SBOM) > 0: - return &SBOMSource{Data: req.SBOM}, nil + return &SBOMSource{Data: req.SBOM, Platforms: req.Platforms}, nil case req.Registry != "": return nil, fmt.Errorf("registry mirroring is not yet implemented; use purls instead") default: diff --git a/internal/mirror/mirror.go b/internal/mirror/mirror.go index b52c4f6..fb98c11 100644 --- a/internal/mirror/mirror.go +++ b/internal/mirror/mirror.go @@ -22,6 +22,10 @@ type Mirror struct { storage storage.Storage logger *slog.Logger workers int + + // Images mirrors docker and oci items. Without it they fail like any + // other unsupported ecosystem. + Images ImageMirrorer } // New creates a new Mirror with the given dependencies. @@ -203,6 +207,10 @@ func (m *Mirror) RunDryRun(ctx context.Context, source Source) ([]PackageVersion } func (m *Mirror) mirrorOne(ctx context.Context, pv PackageVersion, tracker *progressTracker) { + if isImageEcosystem(pv.Ecosystem) && m.Images != nil { + m.mirrorImage(ctx, pv, tracker) + return + } result, err := m.proxy.GetOrFetchArtifact(ctx, pv.Ecosystem, pv.Name, pv.Version, "") if err != nil { tracker.failed.Add(1) diff --git a/internal/mirror/source.go b/internal/mirror/source.go index 4269264..6d3d096 100644 --- a/internal/mirror/source.go +++ b/internal/mirror/source.go @@ -20,6 +20,11 @@ type PackageVersion struct { Ecosystem string Name string Version string + // Qualifiers are the PURL qualifiers. Images use repository_url, tag and + // arch. + Qualifiers map[string]string + // Platforms limit which platforms of an image index are mirrored. + Platforms []string } func (pv PackageVersion) String() string { @@ -36,6 +41,8 @@ type Source interface { type PURLSource struct { PURLs []string RegClient *registries.Client + // Platforms limit which platforms of container images are mirrored. + Platforms []string } func (s *PURLSource) Enumerate(ctx context.Context, fn func(PackageVersion) error) error { @@ -56,8 +63,15 @@ func (s *PURLSource) Enumerate(ctx context.Context, fn func(PackageVersion) erro name = p.Namespace + "/" + p.Name } - if p.Version != "" { - if err := fn(PackageVersion{Ecosystem: ecosystem, Name: name, Version: p.Version}); err != nil { + if p.Version != "" || isImageEcosystem(ecosystem) { + // Images are never expanded to all tags; an unversioned image + // fails when it is mirrored. + pv := PackageVersion{Ecosystem: ecosystem, Name: name, Version: p.Version} + if isImageEcosystem(ecosystem) { + pv.Qualifiers = p.Qualifiers.Map() + pv.Platforms = s.Platforms + } + if err := fn(pv); err != nil { return err } continue @@ -98,6 +112,8 @@ type SBOMSource struct { Data []byte Name string RegClient *registries.Client + // Platforms limit which platforms of container images are mirrored. + Platforms []string } func (s *SBOMSource) Enumerate(ctx context.Context, fn func(PackageVersion) error) error { @@ -109,7 +125,7 @@ func (s *SBOMSource) Enumerate(ctx context.Context, fn func(PackageVersion) erro return fmt.Errorf("parsing SBOM: %w", err) } - inner := &PURLSource{PURLs: purls, RegClient: s.RegClient} + inner := &PURLSource{PURLs: purls, RegClient: s.RegClient, Platforms: s.Platforms} return inner.Enumerate(ctx, fn) } diff --git a/internal/server/mirror_proxy_test.go b/internal/server/mirror_proxy_test.go index 2df395a..129a913 100644 --- a/internal/server/mirror_proxy_test.go +++ b/internal/server/mirror_proxy_test.go @@ -40,7 +40,7 @@ func TestNewMirrorProxyFollowsBearerChallenge(t *testing.T) { registry := newTokenRegistry(t) cfg := config.Default() cfg.Upstream.AllowLoopback = true - proxy := NewMirrorProxy(cfg, nil, nil, slog.New(slog.NewTextHandler(io.Discard, nil))) + proxy, _ := NewMirrorProxy(cfg, nil, nil, slog.New(slog.NewTextHandler(io.Discard, nil))) resp, err := proxy.HTTPClient.Get(registry.URL + "/v2/library/app/manifests/1.0") if err != nil { @@ -54,7 +54,7 @@ func TestNewMirrorProxyFollowsBearerChallenge(t *testing.T) { func TestNewMirrorProxyRefusesLoopbackByDefault(t *testing.T) { registry := newTokenRegistry(t) - proxy := NewMirrorProxy(config.Default(), nil, nil, slog.New(slog.NewTextHandler(io.Discard, nil))) + proxy, _ := NewMirrorProxy(config.Default(), nil, nil, slog.New(slog.NewTextHandler(io.Discard, nil))) resp, err := proxy.HTTPClient.Get(registry.URL + "/v2/library/app/manifests/1.0") if err == nil { diff --git a/internal/server/server.go b/internal/server/server.go index de6204c..7391c91 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -338,6 +338,7 @@ func (s *Server) serve(listener net.Listener) error { // Mirror API endpoints (opt-in via mirror_api config or PROXY_MIRROR_API env) if s.cfg.MirrorAPI { mirrorSvc := mirror.New(proxy, s.db, s.storage, s.logger, 4) //nolint:mnd // default concurrency + mirrorSvc.Images = s.newContainerHandler(proxy) jobStore := mirror.NewJobStore(bgCtx, mirrorSvc) mirrorAPI := NewMirrorAPIHandler(jobStore) r.Post("/api/mirror", mirrorAPI.HandleCreate) @@ -421,14 +422,7 @@ func (s *Server) mountProtocolHandlers(r chi.Router, proxy *handler.Proxy) { juliaHandler := handler.NewJuliaHandlerWithUpstream(proxy, s.cfg.Upstream.Julia) swiftHandler := handler.NewSwiftHandler(proxy, s.cfg.BaseURL, s.cfg.Upstream.Swift) homebrewHandler := handler.NewHomebrewHandler(proxy, s.cfg.Upstream.HomebrewAPI) - containerHandler := handler.NewContainerHandlerWithRegistry( - proxy, - s.cfg.BaseURL, - s.cfg.Upstream.OCIDefault, - s.cfg.Upstream.OCI, - ) - containerHandler.SetMirroredRegistries(s.cfg.Upstream.OCIMirrors) - handler.RegisterHomebrewArtifacts(containerHandler, s.cfg.Upstream.HomebrewArtifact) + containerHandler := s.newContainerHandler(proxy) helmHandler := handler.NewHelmHandlerWithOCIRegistries( proxy, s.cfg.BaseURL, s.cfg.Upstream.Helm, s.cfg.Upstream.OCIDefault, s.cfg.Upstream.OCI) apkHandler := handler.NewAPKHandler(proxy, s.cfg.BaseURL, s.cfg.Upstream.APK) @@ -488,15 +482,30 @@ func (s *Server) newUpstreamClients(artifactTimeout time.Duration) (metadataClie } // NewMirrorProxy builds the proxy for the mirror command with the same -// upstream clients, SSRF guard and upstream authentication as serve. Artifact +// upstream clients, SSRF guard and upstream authentication as serve, and a +// container handler with the same OCI routes for mirroring images. Artifact // downloads have no overall time limit; the caller cancels through ctx. -func NewMirrorProxy(cfg *config.Config, db *database.DB, store storage.Storage, logger *slog.Logger) *handler.Proxy { +func NewMirrorProxy(cfg *config.Config, db *database.DB, store storage.Storage, logger *slog.Logger) (*handler.Proxy, *handler.ContainerHandler) { s := &Server{cfg: cfg, db: db, storage: store, logger: logger, ecr: newECRTokens(logger)} metadataClient, artifactClient := s.newUpstreamClients(0) proxy := handler.NewProxy(db, store, fetch.NewFetcher(fetch.WithHTTPClient(artifactClient)), fetch.NewResolver(), logger) proxy.HTTPClient = metadataClient proxy.AuthForURL = s.authForURL - return proxy + return proxy, s.newContainerHandler(proxy) +} + +// newContainerHandler builds the OCI handler with the configured default +// registry, named upstreams, mirrored registries and Homebrew routes. +func (s *Server) newContainerHandler(proxy *handler.Proxy) *handler.ContainerHandler { + containerHandler := handler.NewContainerHandlerWithRegistry( + proxy, + s.cfg.BaseURL, + s.cfg.Upstream.OCIDefault, + s.cfg.Upstream.OCI, + ) + containerHandler.SetMirroredRegistries(s.cfg.Upstream.OCIMirrors) + handler.RegisterHomebrewArtifacts(containerHandler, s.cfg.Upstream.HomebrewArtifact) + return containerHandler } func configureScanning(proxy *handler.Proxy, cfg config.ScanningConfig, baseURL string, logger *slog.Logger) (*scanner.Group, error) { From 729138d778324d67d7b000fa3995b047c6ce87df Mon Sep 17 00:00:00 2001 From: Christian Heim Date: Mon, 12 Oct 2026 09:42:38 +0200 Subject: [PATCH 198/633] Reject package requests with ? or # in the decoded path (#437) * Reject package requests whose decoded path contains '?' or '#', in one middleware in front of every ecosystem mount. Handlers build the upstream URL from the decoded path, so busybox.apk%3F/foo-9.9-r0.apk fetched busybox but cached it as foo; PyPI could be poisoned the same way, and the other ecosystems stored files under mangled names past the denylist and cooldown. * Let Composer archive downloads through the '?'/'#' guard: they resolve the upstream URL from metadata, and with #433 a branch like dev-fix/#42 is requested as dev-fix%2F%2342. Adversarial review finding F1. --- internal/server/middleware.go | 16 ++++ internal/server/query_in_path_test.go | 107 ++++++++++++++++++++++++++ internal/server/server.go | 1 + 3 files changed, 124 insertions(+) create mode 100644 internal/server/query_in_path_test.go diff --git a/internal/server/middleware.go b/internal/server/middleware.go index a5433ce..4653493 100644 --- a/internal/server/middleware.go +++ b/internal/server/middleware.go @@ -123,3 +123,19 @@ func requestEcosystem(path string) string { return "other" } } + +// rejectQueryInPath refuses requests whose decoded path contains '?' or '#'. +// Handlers build upstream URLs from the decoded path, where such a character +// would end the path: busybox.apk%3F/foo-9.9-r0.apk fetches busybox but is +// parsed and cached as foo, under the wrong PURL and past the denylist. +// Composer archives are exempt: their upstream URL comes from the package +// metadata, and git branch versions such as dev-fix/#42 may contain '#'. +func rejectQueryInPath(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if strings.ContainsAny(r.URL.Path, "?#") && !strings.HasPrefix(r.URL.Path, "/composer/files/") { + http.Error(w, "invalid path", http.StatusBadRequest) + return + } + next.ServeHTTP(w, r) + }) +} diff --git a/internal/server/query_in_path_test.go b/internal/server/query_in_path_test.go new file mode 100644 index 0000000..7c70a13 --- /dev/null +++ b/internal/server/query_in_path_test.go @@ -0,0 +1,107 @@ +package server + +import ( + "context" + "errors" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + + "github.com/git-pkgs/proxy/internal/config" + "github.com/git-pkgs/proxy/internal/database" + "github.com/git-pkgs/proxy/internal/handler" + "github.com/git-pkgs/proxy/internal/storage" + "github.com/git-pkgs/registries/fetch" + "github.com/go-chi/chi/v5" +) + +// countingTransport records every upstream request and fails it, so a test +// can tell whether a handler reached out at all. +type countingTransport struct{ hits atomic.Int32 } + +func (c *countingTransport) RoundTrip(*http.Request) (*http.Response, error) { + c.hits.Add(1) + return nil, errors.New("upstream contacted") +} + +// TestProtocolHandlersRejectQueryInPath sends a path with an encoded '?' or +// '#' to every mounted ecosystem. Without the guard, most of these fetch the +// file before the '?' and cache it under the name after it. +func TestProtocolHandlersRejectQueryInPath(t *testing.T) { + db, err := database.Create(t.TempDir() + "/test.db") + if err != nil { + t.Fatalf("creating database: %v", err) + } + t.Cleanup(func() { _ = db.Close() }) + store, err := storage.OpenBucket(context.Background(), "file://"+t.TempDir()) + if err != nil { + t.Fatalf("opening storage: %v", err) + } + + transport := &countingTransport{} + client := &http.Client{Transport: transport} + fetcher := fetch.NewFetcher(fetch.WithHTTPClient(client), fetch.WithMaxRetries(0)) + t.Cleanup(func() { _ = fetcher.Close() }) + proxy := handler.NewProxy(db, store, fetcher, fetch.NewResolver(), slog.New(slog.NewTextHandler(io.Discard, nil))) + proxy.HTTPClient = client + + cfg := config.Default() + cfg.BaseURL = "http://proxy.example" + cfg.Upstream.Generic = map[string]string{"github": "https://github.com"} + s := &Server{cfg: cfg} + r := chi.NewRouter() + s.mountProtocolHandlers(r, proxy) + + for _, target := range []string{ + "/apk/alpine/v3.22/main/x86_64/busybox-1.37.0-r12.apk%3F/foo-9.9-r0.apk", + "/apk/alpine/v3.22/main/x86_64/busybox-1.37.0-r12.apk%23/foo-9.9-r0.apk", + "/rpm/Packages/n/nginx-1.24.0-1.fc39.x86_64.rpm%3F/foo-9.9-1.fc39.x86_64.rpm", + "/debian/pool/main/c/curl/curl_8.5.0-2_amd64.deb%3F/foo_9.9-1_amd64.deb", + "/generic/github/owner/repo/releases/download/v1/tool.tar.gz%3F/other/repo/releases/download/v2/tool.tar.gz", + "/pypi/packages/ab/cd/ef/requests-2.31.0.tar.gz%3F/ab/cd/ef/foo-9.9.tar.gz", + "/maven/com/google/guava/guava/32.1.3-jre/guava-32.1.3-jre.jar%3F/org/evil/foo/9.9/foo-9.9.jar", + "/conda/conda-forge/linux-64/numpy-1.26.0-py312_0.conda%3F%2Ffoo-9.9-py_0.conda", + "/cran/src/contrib/ggplot2_3.4.0.tar.gz%3F%2Ffoo_9.9.tar.gz", + "/gem/gems/rails-7.0.0.gem%3F%2Ffoo-9.9.gem", + "/hex/tarballs/phoenix-1.7.10.tar%3F%2Ffoo-9.9.tar", + "/pub/packages/http/versions/0.13.0.tar.gz%3F1.2.0.tar.gz", + "/go/golang.org/x/text/@v/v0.3.0.zip%3F/@v/v0.3.8.zip", + "/nuget/v3-flatcontainer/newtonsoft.json/13.0.1/newtonsoft.json.13.0.1.nupkg%3F%2Ffoo.9.9.nupkg", + "/conan/v2/files/zlib/1.3/_/_/rev/recipe/conanfile.py%3F%2Fx", + "/composer/p2/monolog/monolog.json%3Fx", + "/npm/lodash%3F/-/lodash-4.17.21.tgz", + "/cargo/crates/serde%3F/1.0.0/download", + "/julia/registry/23338594-aafe-5451-b93e-139f81909106/abc%3F", + "/swift/apple/swift-nio/2.0.0%3F.zip", + "/homebrew/formula%3F.json", + "/helm/charts/nginx-1.0.0.tgz%3F", + "/v2/library/alpine/manifests/latest%3F", + "/gradle/cache/abc%3F", + } { + w := httptest.NewRecorder() + r.ServeHTTP(w, httptest.NewRequest(http.MethodGet, target, nil)) + if w.Code != http.StatusBadRequest { + t.Errorf("%s: status = %d, want 400", target, w.Code) + } + } + if n := transport.hits.Load(); n != 0 { + t.Errorf("upstream was contacted %d times", n) + } +} + +// TestComposerFilesAllowHashInVersion keeps Composer archive downloads of git +// branches with '#' in their name working: the version is escaped in the dist +// URL and the archive is resolved from metadata, not from the request path. +func TestComposerFilesAllowHashInVersion(t *testing.T) { + var reached atomic.Bool + next := http.HandlerFunc(func(http.ResponseWriter, *http.Request) { reached.Store(true) }) + + w := httptest.NewRecorder() + rejectQueryInPath(next).ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/composer/files/acme/lib/dev-fix%2F%2342/abc123.zip", nil)) + if w.Code == http.StatusBadRequest || !reached.Load() { + t.Errorf("status = %d, reached = %v; want the request passed on", w.Code, reached.Load()) + } +} diff --git a/internal/server/server.go b/internal/server/server.go index 380892a..68fdc2f 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -453,6 +453,7 @@ func (s *Server) mountProtocolHandlers(r chi.Router, proxy *handler.Proxy) { rpmHandler := handler.NewRPMHandlerWithUpstream(proxy, s.cfg.BaseURL, s.cfg.Upstream.RPM) genericHandler := handler.NewGenericHandler(proxy, s.cfg.Upstream.Generic) + r = r.With(rejectQueryInPath) r.Mount("/npm", http.StripPrefix("/npm", npmHandler.Routes())) r.Mount("/cargo", http.StripPrefix("/cargo", cargoHandler.Routes())) r.Mount("/gem", http.StripPrefix("/gem", gemHandler.Routes())) From 16ebeaa9759b203426f8973ee9352e3bfe26b735 Mon Sep 17 00:00:00 2001 From: Christian Heim Date: Mon, 12 Oct 2026 09:53:53 +0200 Subject: [PATCH 199/633] On Postgres, metadata fetched_at is now stored in UTC, so metadata_ttl no longer expires hours late when the proxy runs east of UTC. Entries written before this expire late one last time and are stored in UTC on their next refresh. (#435) --- internal/database/metadata_cache_test.go | 39 ++++++++++++++++++++++++ internal/database/queries.go | 9 +++++- 2 files changed, 47 insertions(+), 1 deletion(-) diff --git a/internal/database/metadata_cache_test.go b/internal/database/metadata_cache_test.go index 46a38a7..26e0a63 100644 --- a/internal/database/metadata_cache_test.go +++ b/internal/database/metadata_cache_test.go @@ -362,3 +362,42 @@ func TestMetadataCacheRoundTripsContentEncoding(t *testing.T) { t.Fatalf("content encoding round-trip failed: %#v", got) } } + +// On Postgres, fetched_at is a TIMESTAMP column without a zone. East of UTC +// it must still read back as the moment it was written, or metadata_ttl +// expires hours late. last_modified is an HTTP date in UTC and must keep it. +func TestMetadataCacheTimesOffUTC(t *testing.T) { + tokyo, err := time.LoadLocation("Asia/Tokyo") + if err != nil { + t.Skipf("time zone data unavailable: %v", err) + } + prev := time.Local + time.Local = tokyo + t.Cleanup(func() { time.Local = prev }) + + runWithBothDatabases(t, func(t *testing.T, db *DB) { + fetched := time.Now().Add(-90 * time.Minute) + modified := time.Date(2026, 10, 1, 12, 0, 0, 0, time.UTC) + entry := &MetadataCacheEntry{ + Ecosystem: testEcosystemNPM, + Name: "tz", + StoragePath: "_metadata/npm/tz/metadata", + LastModified: sql.NullTime{Time: modified, Valid: true}, + FetchedAt: sql.NullTime{Time: fetched, Valid: true}, + } + if err := db.UpsertMetadataCache(entry); err != nil { + t.Fatalf("UpsertMetadataCache() error = %v", err) + } + + got, err := db.GetMetadataCache(testEcosystemNPM, "tz") + if err != nil || got == nil { + t.Fatalf("GetMetadataCache() = %v, %v", got, err) + } + if diff := got.FetchedAt.Time.Sub(fetched).Abs(); diff > time.Second { + t.Errorf("fetched_at read back as %v, want %v", got.FetchedAt.Time, fetched) + } + if !got.LastModified.Time.Equal(modified) { + t.Errorf("last_modified read back as %v, want %v", got.LastModified.Time, modified) + } + }) +} diff --git a/internal/database/queries.go b/internal/database/queries.go index 9503022..5897d95 100644 --- a/internal/database/queries.go +++ b/internal/database/queries.go @@ -1122,6 +1122,13 @@ func (db *DB) GetMetadataCache(ecosystem, name string) (*MetadataCacheEntry, err func (db *DB) UpsertMetadataCache(entry *MetadataCacheEntry) error { now := time.Now() + // fetched_at is a TIMESTAMP without zone on Postgres, and lib/pq reads it + // back labelled UTC, so it has to be stored in UTC. last_modified already + // is, since it comes from an HTTP date. + fetchedAt := entry.FetchedAt + if fetchedAt.Valid { + fetchedAt.Time = fetchedAt.Time.UTC() + } var query string if db.dialect == DialectPostgres { @@ -1162,7 +1169,7 @@ func (db *DB) UpsertMetadataCache(entry *MetadataCacheEntry) error { _, err := db.Exec(query, entry.Ecosystem, entry.Name, entry.StoragePath, entry.ETag, entry.Link, - entry.ContentType, entry.ContentEncoding, entry.ContentDigest, entry.Size, entry.LastModified, entry.FetchedAt, now, now, + entry.ContentType, entry.ContentEncoding, entry.ContentDigest, entry.Size, entry.LastModified, fetchedAt, now, now, ) if err != nil { return fmt.Errorf("upserting metadata cache: %w", err) From dcf90cfec3c595ec2632051bbf75dfddb69a26e6 Mon Sep 17 00:00:00 2001 From: Christian Heim Date: Mon, 12 Oct 2026 09:56:57 +0200 Subject: [PATCH 200/633] Close the Composer download gaps left after #430 (#433) * GitLab Composer archives are now cached per commit and fetched for the commit the lock file pins, and branch names with a slash are escaped in dist URLs so the download route matches them. * Composer branch archives that an older proxy cached from another commit's URL are now discarded and fetched again, and a refetched artifact records the URL it actually came from. * When a GitLab commit is fetched under its own name, the archive older proxies cached under the shared archive.zip name is dropped. A test now moves the branch from A to B and checks that A keeps coming from the cache. --- internal/database/queries.go | 2 + internal/handler/composer.go | 76 ++++++++- internal/handler/composer_test.go | 260 +++++++++++++++++++++++++++--- 3 files changed, 317 insertions(+), 21 deletions(-) diff --git a/internal/database/queries.go b/internal/database/queries.go index 5897d95..40fd2ce 100644 --- a/internal/database/queries.go +++ b/internal/database/queries.go @@ -359,6 +359,7 @@ func (db *DB) upsertArtifactFrom(a *Artifact, previous sql.NullString) (bool, er created_at, updated_at) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12) ON CONFLICT(version_purl, filename) DO UPDATE SET + upstream_url = EXCLUDED.upstream_url, storage_path = EXCLUDED.storage_path, content_hash = EXCLUDED.content_hash, size = EXCLUDED.size, @@ -374,6 +375,7 @@ func (db *DB) upsertArtifactFrom(a *Artifact, previous sql.NullString) (bool, er created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(version_purl, filename) DO UPDATE SET + upstream_url = excluded.upstream_url, storage_path = excluded.storage_path, content_hash = excluded.content_hash, size = excluded.size, diff --git a/internal/handler/composer.go b/internal/handler/composer.go index 182fe7a..f7b40ae 100644 --- a/internal/handler/composer.go +++ b/internal/handler/composer.go @@ -8,6 +8,7 @@ import ( "fmt" "io" "net/http" + "net/url" "path" "regexp" "strings" @@ -372,7 +373,7 @@ func (h *ComposerHandler) proxyDistURL(packageName, version, upstreamURL, distTy return "", false } return fmt.Sprintf("%s/composer/files/%s/%s/%s/%s", - h.proxyURL, parts[0], parts[1], version, distFilename(upstreamURL, distType)), true + h.proxyURL, parts[0], parts[1], url.PathEscape(version), distFilename(upstreamURL, distType)), true } // distFilename returns the file name this proxy serves an archive under: the @@ -410,6 +411,16 @@ func upstreamDistURL(listedURL, distType, filename string, branch bool) (string, if filename == listed { return listedURL, true } + if commit, ok := shaQueryCommit(listed); ok { + requested, ok := shaQueryCommit(filename) + listedSha, requestedSha := "sha="+commit, "sha="+requested + idx := strings.LastIndex(listed, listedSha) + if !branch || !ok || idx < 0 || listed[:idx]+requestedSha+listed[idx+len(listedSha):] != filename { + return "", false + } + idx = strings.LastIndex(listedURL, listedSha) + return listedURL[:idx] + requestedSha + listedURL[idx+len(listedSha):], true + } ext := path.Ext(listed) commit := strings.TrimSuffix(listed, ext) requested, ok := strings.CutSuffix(filename, ext) @@ -420,6 +431,22 @@ func upstreamDistURL(listedURL, distType, filename string, branch bool) (string, return listedURL[:idx] + requested + listedURL[idx+len(commit):], true } +// shaQueryCommit returns the commit an archive name passes in a sha query +// parameter. GitLab names the archive of every commit archive.zip and picks +// the commit that way. +func shaQueryCommit(filename string) (string, bool) { + _, query, ok := strings.Cut(filename, "?") + if !ok { + return "", false + } + values, err := url.ParseQuery(query) + if err != nil { + return "", false + } + commit := values.Get("sha") + return commit, composerCommitPattern.MatchString(commit) +} + // composerFields holds one version's fields as Composer sees them after // expanding the minified format: raw values under their decoded names, in the // order the names first appeared. A removed field keeps its slot with a nil @@ -544,8 +571,7 @@ func (h *ComposerHandler) handleDownload(w http.ResponseWriter, r *http.Request) // The archive is cached under the file name from the path, so fetch the // one that name stands for. Compare it as sent, the way rewriteDist handed - // it out. A query such as GitLab's ?sha= is compared but is not part of - // the cache key. + // it out, query included. requested := path.Base(r.URL.EscapedPath()) if r.URL.RawQuery != "" { requested += "?" + r.URL.RawQuery @@ -564,16 +590,60 @@ func (h *ComposerHandler) handleDownload(w http.ResponseWriter, r *http.Request) "package", packageName, "version", version, "download_url", downloadURL) + // Archives picked by ?sha= share one file name, so the commit goes into + // the name they are cached under. Proxy versions before that cached every + // commit under the shared name, which is dropped once a commit is fetched. + var sharedFilename string + if commit, ok := shaQueryCommit(requested); ok { + sharedFilename = h.routes.cacheFilename(packageName, filename) + ext := path.Ext(filename) + filename = strings.TrimSuffix(filename, ext) + "-" + commit + ext + } cacheFilename := h.routes.cacheFilename(packageName, filename) + if commit := strings.TrimSuffix(filename, path.Ext(filename)); isDevVersion(version) && composerCommitPattern.MatchString(commit) { + h.discardOtherCommit(packageName, version, cacheFilename, commit) + } result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "composer", packageName, version, cacheFilename, downloadURL) if err != nil { h.proxy.serveArtifactError(w, err, "failed to fetch package") return } + if sharedFilename != "" && !result.Cached { + if err := h.proxy.ClearCachedArtifact("composer", packageName, version, sharedFilename); err != nil { + h.proxy.Logger.Warn("failed to discard composer archive cached under a shared name", + "package", packageName, "version", version, "filename", sharedFilename, "error", err) + } + } ServeArtifactRequest(w, r, result) } +// discardOtherCommit clears a cached branch archive named for commit that was +// fetched from another commit's URL. Proxy versions before the pinned-commit +// fix cached the branch head under whatever commit a lock file asked for. The +// record is only cleared while it still points at the object checked here. +func (h *ComposerHandler) discardOtherCommit(packageName, version, cacheFilename, commit string) { + _, versionPURL, err := packagePURLStrings("composer", packageName, version) + if err != nil { + return + } + record, err := h.proxy.DB.GetArtifact(versionPURL, cacheFilename) + if err != nil { + h.proxy.Logger.Warn("failed to read composer cache record", + "purl", versionPURL, "filename", cacheFilename, "error", err) + return + } + if record == nil || !record.StoragePath.Valid || strings.Contains(record.UpstreamURL, commit) { + return + } + h.proxy.Logger.Warn("cached composer archive is from another commit, discarding", + "purl", versionPURL, "filename", cacheFilename, "upstream_url", record.UpstreamURL) + if err := h.proxy.DB.DiscardArtifact(versionPURL, cacheFilename, record.StoragePath.String); err != nil { + h.proxy.Logger.Warn("failed to discard composer cache record", + "purl", versionPURL, "filename", cacheFilename, "error", err) + } +} + // isDevVersion reports whether a Composer version string refers to a // development (unstable, branch) version rather than a tagged release. // Composer formats these as either "dev-" (e.g. "dev-master") or diff --git a/internal/handler/composer_test.go b/internal/handler/composer_test.go index f47550e..ad3eb7a 100644 --- a/internal/handler/composer_test.go +++ b/internal/handler/composer_test.go @@ -9,6 +9,7 @@ import ( "net/http" "net/http/httptest" "strings" + "sync/atomic" "testing" "time" @@ -579,13 +580,15 @@ func cachedComposerFile(t *testing.T, proxy *Proxy, filename string) string { // TestComposerDownloadPinnedBranchCommit checks that a lock file pinning an // older commit of a branch gets that commit rather than the head the branch's // metadata lists now, and that the head is not cached under the older name. +// The second round is served from the cache. func TestComposerDownloadPinnedBranchCommit(t *testing.T) { tests := []struct { - name, distPath, filename string + name, distPath, filename, cached string }{ - {"github zipball", "/repos/v/p/zipball/%s", "%s.zip"}, - {"github zipball without .zip", "/repos/v/p/zipball/%s", "%s"}, - {"bitbucket archive", "/v/p/get/%s.zip", "%s.zip"}, + {"github zipball", "/repos/v/p/zipball/%s", "%s.zip", "%s.zip"}, + {"github zipball without .zip", "/repos/v/p/zipball/%s", "%s", "%s"}, + {"bitbucket archive", "/v/p/get/%s.zip", "%s.zip", "%s.zip"}, + {"gitlab archive", "/api/v4/projects/v%%2Fp/repository/archive.zip?sha=%s", "archive.zip?sha=%s", "archive-%s.zip"}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -593,24 +596,238 @@ func TestComposerDownloadPinnedBranchCommit(t *testing.T) { fmt.Sprintf(tt.distPath, composerCommitA): "commit A", fmt.Sprintf(tt.distPath, composerCommitB): "commit B", }) - for _, c := range []struct{ commit, want string }{ - {composerCommitA, "commit A"}, - {composerCommitB, "commit B"}, - } { - filename, want := fmt.Sprintf(tt.filename, c.commit), c.want - w := httptest.NewRecorder() - h.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/files/v/p/dev-master/"+filename, nil)) - if w.Code != http.StatusOK || w.Body.String() != want { - t.Errorf("%s: status %d, body %q, want 200 %q", filename, w.Code, w.Body.String(), want) - } - if got := cachedComposerFile(t, proxy, filename); got != want { - t.Errorf("%s cached as %q, want %q", filename, got, want) + for range 2 { + for _, c := range []struct{ commit, want string }{ + {composerCommitA, "commit A"}, + {composerCommitB, "commit B"}, + } { + filename, want := fmt.Sprintf(tt.filename, c.commit), c.want + w := httptest.NewRecorder() + h.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/files/v/p/dev-master/"+filename, nil)) + if w.Code != http.StatusOK || w.Body.String() != want { + t.Errorf("%s: status %d, body %q, want 200 %q", filename, w.Code, w.Body.String(), want) + } + cached := fmt.Sprintf(tt.cached, c.commit) + if got := cachedComposerFile(t, proxy, cached); got != want { + t.Errorf("%s cached as %q, want %q", cached, got, want) + } } } }) } } +// composerMovingBranch is an upstream whose dev-master head can move. It +// serves the archives of commits A and B under distPath and counts how often +// they are fetched. +type composerMovingBranch struct { + h http.Handler + proxy *Proxy + url string + head atomic.Value + fetches atomic.Int32 +} + +func setupComposerMovingBranch(t *testing.T, distPath string) *composerMovingBranch { + t.Helper() + b := &composerMovingBranch{} + b.head.Store(composerCommitA) + archives := map[string]string{ + fmt.Sprintf(distPath, composerCommitA): "commit A", + fmt.Sprintf(distPath, composerCommitB): "commit B", + } + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/p2/v/p.json" || r.URL.Path == "/p2/v/p~dev.json" { + dist := b.url + fmt.Sprintf(distPath, b.head.Load()) + _, _ = fmt.Fprintf(w, `{"packages":{"v/p":[{"version":"dev-master","dist":{"url":%q,"type":"zip"}}]}}`, dist) + return + } + body, ok := archives[r.URL.RequestURI()] + if !ok { + http.NotFound(w, r) + return + } + b.fetches.Add(1) + _, _ = io.WriteString(w, body) + })) + t.Cleanup(srv.Close) + b.url = srv.URL + + b.proxy, _, _, _ = setupTestProxy(t) + fetcher := fetch.NewFetcher(fetch.WithHTTPClient(srv.Client()), fetch.WithMaxRetries(0)) + b.proxy.Fetcher = fetcher + t.Cleanup(func() { _ = fetcher.Close() }) + b.h = NewComposerHandlerWithUpstreams(b.proxy, "http://proxy", srv.URL, srv.URL).Routes() + return b +} + +// TestComposerDownloadBranchMovesOn checks a lock file pinning A while the +// branch moves from A to B: A is cached while it is the head and still served +// from the cache afterwards, and B, the head a new lock file pins, comes from +// upstream. +func TestComposerDownloadBranchMovesOn(t *testing.T) { + tests := []struct { + name, distPath, filename, cached string + }{ + {"github zipball", "/repos/v/p/zipball/%s", "%s.zip", "%s.zip"}, + {"gitlab archive", "/api/v4/projects/v%%2Fp/repository/archive.zip?sha=%s", "archive.zip?sha=%s", "archive-%s.zip"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + b := setupComposerMovingBranch(t, tt.distPath) + get := func(commit, want string, wantFetches int32) { + t.Helper() + filename := fmt.Sprintf(tt.filename, commit) + w := httptest.NewRecorder() + b.h.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/files/v/p/dev-master/"+filename, nil)) + if w.Code != http.StatusOK || w.Body.String() != want { + t.Errorf("%s: status %d, body %q, want 200 %q", filename, w.Code, w.Body.String(), want) + } + if got := cachedComposerFile(t, b.proxy, fmt.Sprintf(tt.cached, commit)); got != want { + t.Errorf("%s cached as %q, want %q", filename, got, want) + } + if n := b.fetches.Load(); n != wantFetches { + t.Errorf("after %s: %d upstream fetches, want %d", filename, n, wantFetches) + } + } + get(composerCommitA, "commit A", 1) + b.head.Store(composerCommitB) + get(composerCommitA, "commit A", 1) + get(composerCommitB, "commit B", 2) + get(composerCommitB, "commit B", 2) + }) + } +} + +// TestComposerDownloadGitLabSharedName checks that an archive an older proxy +// cached under GitLab's shared archive.zip name is dropped once a commit is +// fetched under its own name. +func TestComposerDownloadGitLabSharedName(t *testing.T) { + distPath := "/api/v4/projects/v%%2Fp/repository/archive.zip?sha=%s" + b := setupComposerMovingBranch(t, distPath) + shared, err := b.proxy.GetOrFetchArtifactFromURL(context.Background(), "composer", "v/p", "dev-master", + "archive.zip", b.url+fmt.Sprintf(distPath, composerCommitB)) + if err != nil { + t.Fatalf("seeding shared entry: %v", err) + } + _ = shared.Reader.Close() + + w := httptest.NewRecorder() + b.h.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/files/v/p/dev-master/archive.zip?sha="+composerCommitA, nil)) + if w.Code != http.StatusOK || w.Body.String() != "commit A" { + t.Errorf("status %d, body %q, want 200 %q", w.Code, w.Body.String(), "commit A") + } + if got := cachedComposerFile(t, b.proxy, "archive.zip"); got != "" { + t.Errorf("archive.zip still cached as %q", got) + } +} + +// TestComposerDownloadCachedFromOtherCommit checks that an archive an older +// proxy cached under a commit's name but fetched from another commit's URL is +// fetched again, while a correct entry is served from the cache. +func TestComposerDownloadCachedFromOtherCommit(t *testing.T) { + zipball := "/repos/v/p/zipball/" + var fetches atomic.Int32 + var srv *httptest.Server + srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/p2/v/p.json", "/p2/v/p~dev.json": + _, _ = fmt.Fprintf(w, `{"packages":{"v/p":[{"version":"dev-master","dist":{"url":%q,"type":"zip"}}]}}`, srv.URL+zipball+composerCommitB) + case zipball + composerCommitA: + fetches.Add(1) + _, _ = io.WriteString(w, "commit A") + case zipball + composerCommitB: + fetches.Add(1) + _, _ = io.WriteString(w, "commit B") + default: + http.NotFound(w, r) + } + })) + t.Cleanup(srv.Close) + proxy, _, _, _ := setupTestProxy(t) + fetcher := fetch.NewFetcher(fetch.WithHTTPClient(srv.Client()), fetch.WithMaxRetries(0)) + proxy.Fetcher = fetcher + t.Cleanup(func() { _ = fetcher.Close() }) + h := NewComposerHandlerWithUpstreams(proxy, "http://proxy", srv.URL, srv.URL).Routes() + + // What proxy versions before the pinned-commit fix stored: the head + // under the name of the commit the lock file pins. + stale, err := proxy.GetOrFetchArtifactFromURL(context.Background(), "composer", "v/p", "dev-master", + composerCommitA+".zip", srv.URL+zipball+composerCommitB) + if err != nil { + t.Fatalf("seeding stale entry: %v", err) + } + _ = stale.Reader.Close() + fetches.Store(0) + + for range 2 { + w := httptest.NewRecorder() + h.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/files/v/p/dev-master/"+composerCommitA+".zip", nil)) + if w.Code != http.StatusOK || w.Body.String() != "commit A" { + t.Errorf("status %d, body %q, want 200 %q", w.Code, w.Body.String(), "commit A") + } + } + if got := cachedComposerFile(t, proxy, composerCommitA+".zip"); got != "commit A" { + t.Errorf("cached as %q, want %q", got, "commit A") + } + if n := fetches.Load(); n != 1 { + t.Errorf("upstream archive fetched %d times, want 1", n) + } +} + +// TestComposerDownloadGitLabTaggedRelease checks that a tagged release on +// GitLab is only served as the commit its metadata lists. +func TestComposerDownloadGitLabTaggedRelease(t *testing.T) { + archive := "/api/v4/projects/v%2Fp/repository/archive.zip?sha=" + h, _ := setupComposerBranch(t, archive+composerCommitB, map[string]string{ + archive + composerCommitA: "commit A", + archive + composerCommitB: "commit B", + }) + + w := httptest.NewRecorder() + h.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/files/v/p/1.0.0/archive.zip?sha="+composerCommitA, nil)) + if w.Code != http.StatusNotFound { + t.Errorf("other commit: status %d, body %q, want 404", w.Code, w.Body.String()) + } + + w = httptest.NewRecorder() + h.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/files/v/p/1.0.0/archive.zip?sha="+composerCommitB, nil)) + if w.Code != http.StatusOK || w.Body.String() != "commit B" { + t.Errorf("listed commit: status %d, body %q, want 200 %q", w.Code, w.Body.String(), "commit B") + } +} + +// TestComposerDownloadBranchWithSlash checks that a branch whose name holds a +// slash gets a dist URL the download route matches. +func TestComposerDownloadBranchWithSlash(t *testing.T) { + const version = "dev-feature/x" + var srv *httptest.Server + srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/zip/x.zip" { + _, _ = io.WriteString(w, "feature") + return + } + _, _ = fmt.Fprintf(w, `{"packages":{"v/p":[{"version":%q,"dist":{"url":%q,"type":"zip"}}]}}`, version, srv.URL+"/zip/x.zip") + })) + t.Cleanup(srv.Close) + proxy, _, _, _ := setupTestProxy(t) + fetcher := fetch.NewFetcher(fetch.WithHTTPClient(srv.Client()), fetch.WithMaxRetries(0)) + proxy.Fetcher = fetcher + t.Cleanup(func() { _ = fetcher.Close() }) + h := NewComposerHandlerWithUpstreams(proxy, "http://proxy", srv.URL, srv.URL) + + distURL, ok := h.proxyDistURL("v/p", version, srv.URL+"/zip/x.zip", "zip") + if want := "http://proxy/composer/files/v/p/dev-feature%2Fx/x.zip"; !ok || distURL != want { + t.Fatalf("dist URL = %q, want %q", distURL, want) + } + + w := httptest.NewRecorder() + h.Routes().ServeHTTP(w, httptest.NewRequest(http.MethodGet, strings.TrimPrefix(distURL, "http://proxy/composer"), nil)) + if w.Code != http.StatusOK || w.Body.String() != "feature" { + t.Errorf("status %d, body %q, want 200 %q", w.Code, w.Body.String(), "feature") + } +} + // TestComposerDownloadUnlistedFile checks that a file the metadata no longer // lists, and that names no commit to fetch instead, is a 404 rather than the // listed archive cached under the requested name. @@ -620,8 +837,8 @@ func TestComposerDownloadUnlistedFile(t *testing.T) { }{ {"plain name", "/dist/main.zip", "main.zip", "old.zip"}, {"escaped name", "/dist/p%2Bdev-main.zip", "p%2Bdev-main.zip", "old.zip"}, - {"gitlab archive", "/api/v4/projects/v%2Fp/repository/archive.zip?sha=" + composerCommitB, - "archive.zip?sha=" + composerCommitB, "archive.zip?sha=" + composerCommitA}, + {"gitlab archive with other query", "/api/v4/projects/v%2Fp/repository/archive.zip?sha=" + composerCommitB, + "archive.zip?sha=" + composerCommitB, "archive.zip?sha=" + composerCommitA + "&path=src"}, {"branch name for a commit", "/repos/v/p/zipball/" + composerCommitB, composerCommitB + ".zip", "master.zip"}, } for _, tt := range tests { @@ -676,6 +893,7 @@ func TestComposerDownloadTaggedReleaseCommit(t *testing.T) { func TestComposerUpstreamDistURL(t *testing.T) { const zipball = "https://api.github.com/repos/v/p/zipball/" + const gitlab = "https://gitlab.com/api/v4/projects/v%2Fp/repository/archive.zip?sha=" sha256A := strings.Repeat("a", 64) tests := []struct { name, listed, distType, filename, want string @@ -689,6 +907,9 @@ func TestComposerUpstreamDistURL(t *testing.T) { {"short hash", zipball + composerCommitB, "zip", "1111111.zip", ""}, {"upper-case hash", zipball + composerCommitB, "zip", strings.ToUpper(sha256A) + ".zip", ""}, {"other name", "https://example.com/dist/main.zip", "zip", "old.zip", ""}, + {"gitlab older commit", gitlab + composerCommitB, "zip", "archive.zip?sha=" + composerCommitA, gitlab + composerCommitA}, + {"gitlab short hash", gitlab + composerCommitB, "zip", "archive.zip?sha=1111111", ""}, + {"gitlab other archive", gitlab + composerCommitB, "zip", "other.zip?sha=" + composerCommitA, ""}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -701,6 +922,9 @@ func TestComposerUpstreamDistURL(t *testing.T) { if got, ok := upstreamDistURL(zipball+composerCommitB, "zip", composerCommitA+".zip", false); ok { t.Errorf("tagged release rebuilt to %q, want only the listed archive", got) } + if got, ok := upstreamDistURL(gitlab+composerCommitB, "zip", "archive.zip?sha="+composerCommitA, false); ok { + t.Errorf("tagged GitLab release rebuilt to %q, want only the listed archive", got) + } } func TestComposerRewriteMetadataCooldown(t *testing.T) { From b664530f889aa0a70405114ee6096089fd891d98 Mon Sep 17 00:00:00 2001 From: Christian Heim Date: Mon, 12 Oct 2026 10:06:01 +0200 Subject: [PATCH 201/633] Add age-based retention for cached artifacts (#428) * Add age-based retention for cached artifacts Cached artifacts stayed forever unless storage.max_size pushed them out. storage.retention adds a default, per-ecosystem and per-package duration after which an artifact nobody downloaded gets evicted. Ecosystems opt in one at a time: a handler registers with retention.Default, and until it does, configuring that ecosystem (or one of its packages) fails validation, and the default doesn't touch it. This commit registers none, so nothing changes yet. Startup logs which ecosystems the default covers and which it doesn't. An artifact counts as expired when both its fetch time and its last access are older than the cutoff. Checking only the last access would evict a refetched artifact again right away, since clearing a record keeps its old access time and a cache miss doesn't record a hit. The sweep scans artifacts in id windows, so no single query holds the SQLite connection for long, and checks each row against its own rule. Expired records are cleared and their files go through pending_deletes instead of being deleted directly, because a buffered hit can make an artifact that's being served right now look expired. Buffered hits are flushed before each sweep, and one sweep clears at most 10,000 artifacts. Since a big first sweep can queue a lot, reclaim now keeps working through due entries for up to 30 seconds per tick instead of stopping after 100, and pending_deletes gets an index on queued_at. Also: - proxy_artifacts_evicted_total{reason, ecosystem} counts LRU and retention evictions. - gradle.build_cache.max_age accepts "7d", as its comment always said. Refs #306 * Document storage retention Describes the new storage.retention block, the order rules apply in, that the default only covers ecosystems that support retention yet, and what to expect from the first sweep over an old cache. Also lists the new eviction counter and notes that gradle max_age takes days. Refs #306 * Fix retention timing on Postgres and tighten the sweep On Postgres, fetched_at and last_accessed_at are TIMESTAMP columns without a zone. They hold the local time the proxy wrote, but lib/pq reads them back labelled UTC. East of UTC that makes every artifact look hours younger than it is, so the sweep evicted late: two hours in Berlin, nine in Tokyo. The candidates' times are now read back as local time, and the database tests run on both SQLite and Postgres, with one that sets the local zone to Asia/Tokyo. A few smaller things: - When a delete fails, reclaim stops for the rest of the minute instead of trying every queued path. Otherwise a storage outage turns into thirty seconds of failed deletes and warnings every minute. - If the buffered hits can't be written, the sweep skips that round. Running on stale access times could clear something that is being downloaded right now. - DefaultCanonical only accepts a package key when rebuilding it gives the same PURL. For alpine and deb it would otherwise produce keys that never match. - sweep_interval has to be at least a minute. - New tests cover the eviction counter and an artifact that gets a hit between the scan and the clear. - The docs no longer say nothing changes without retention, and they point out that the example's ecosystem and package rules only pass validation once those ecosystems support retention. Refs #306 * Reject qualifiers and subpaths in retention package keys A rule names a whole package. Qualifiers like ?type=pom were dropped on the way to the stored PURL and the rule quietly covered every file of the package. Now such a key fails validation. Refs #306 * Say when a package key doesn't match how packages are stored A package rule for an ecosystem that supports retention, written in a form its packages aren't stored under (pkg:npm/babel/core without the @), failed with "retention is not supported for npm packages". Once the ecosystem is enabled that is no longer true. Such a key now fails with "does not match how npm packages are stored" and points to the package rule forms in the docs. Keys for ecosystems without retention support keep the old message. Refs #306 * Note that re-running the mirror keeps packages cached A mirror run over a package that is still cached goes through the normal cache lookup and records a download, so the package's age starts over. The docs only said that mirrored packages age from the time they were mirrored. Refs #306 * Say when an ecosystem takes no package rules Some ecosystems will support retention without package rules, because nothing in their stored records names a package the way a PURL would (conan, helm). A package key for one of them used to fail with "does not match how conan packages are stored; see the package rule forms", which sends people looking for a form that doesn't exist. It now says package rules aren't supported there and points to the ecosystem rule. Refs #306 * Use helm as the example of an ecosystem without package rules Conan gets package rules once its handler stores archives under the recipe name, so the config test that shows the refusal now uses helm, whose cache records name the repository, not the chart. Refs #306 * Say that each cached file ages on its own The retention docs said each artifact, each version of a package, ages on its own. A version can have several cached files (a jar and its pom, several wheels, Conan sources and binaries), and each of them ages separately. Refs #306 * Clear an expired artifact and queue its file in one transaction The sweep cleared the record and queued the file for deletion as two separate writes. If queueing failed, or the process stopped in between, the file stayed in storage with no record pointing at it and no queue entry, and no later sweep would find it again. Both writes now share a transaction, so a failed queue rolls the clear back and the next sweep retries. Refs #306 * Show artifact evictions on the analytics page proxy_artifacts_evicted_total had no place on /ui/analytics. The Runtime card now lists evicted artifacts by reason and ecosystem, without the failure tint, and the coverage test records an eviction so it doesn't depend on test order. Refs #306 * Store artifact times in UTC fetched_at and last_accessed_at are now written in UTC, and the retention cutoffs are compared in UTC, the same convention #434 uses for the metadata cache. lib/pq sends a time with its offset and a Postgres TIMESTAMP keeps only the wall clock, so UTC on write is what reads back correctly. That replaces reinterpreting the times as local on read. Rows written before this still hold local time. East of UTC they look younger by the offset until they are rewritten, so retention evicts them a few hours late once; west of UTC that much early. Refs #306 --------- Co-authored-by: t --- README.md | 5 +- config.example.yaml | 15 + docs/configuration.md | 55 ++- internal/config/config.go | 45 +- internal/config/retention.go | 158 +++++++ internal/config/retention_test.go | 167 +++++++ internal/database/hits.go | 3 +- internal/database/queries.go | 29 +- internal/database/retention.go | 128 ++++++ internal/database/retention_test.go | 396 ++++++++++++++++ internal/database/schema.go | 13 + internal/metrics/metrics.go | 19 + internal/retention/retention.go | 209 +++++++++ internal/retention/retention_test.go | 163 +++++++ internal/server/analytics_coverage_test.go | 2 + internal/server/analytics_runtime.go | 12 +- internal/server/analytics_runtime_test.go | 24 + internal/server/eviction.go | 1 + internal/server/reclaim.go | 38 +- internal/server/retention.go | 173 +++++++ internal/server/retention_test.go | 423 ++++++++++++++++++ internal/server/server.go | 1 + .../templates/components/runtime_metrics.html | 1 + 23 files changed, 2058 insertions(+), 22 deletions(-) create mode 100644 internal/config/retention.go create mode 100644 internal/config/retention_test.go create mode 100644 internal/database/retention.go create mode 100644 internal/database/retention_test.go create mode 100644 internal/retention/retention.go create mode 100644 internal/retention/retention_test.go create mode 100644 internal/server/retention.go create mode 100644 internal/server/retention_test.go diff --git a/README.md b/README.md index 06cd0ed..57e6e1a 100644 --- a/README.md +++ b/README.md @@ -741,6 +741,8 @@ base_url: "http://localhost:8080" storage: url: "file:///var/cache/proxy/artifacts" max_size: "10GB" # Optional: evict LRU when exceeded + retention: + default: "30d" # Optional: evict artifacts not downloaded for 30 days database: driver: "sqlite" @@ -1214,6 +1216,7 @@ The proxy exposes Prometheus metrics at `GET /metrics`. All metric names are pre | `proxy_cache_misses_total` | counter | `ecosystem` | Cache misses | | `proxy_cache_size_bytes` | gauge | | Total size of cached artifacts | | `proxy_cached_artifacts_total` | gauge | | Number of cached artifacts | +| `proxy_artifacts_evicted_total` | counter | `reason`, `ecosystem` | Artifacts evicted by the size limit (`lru`) or by age (`retention`) | | `proxy_upstream_fetch_duration_seconds` | histogram | `ecosystem` | Time spent fetching from upstream | | `proxy_upstream_errors_total` | counter | `ecosystem`, `error_type` | Upstream fetch failures | | `proxy_storage_operation_duration_seconds` | histogram | `operation` | Storage read/write latency | @@ -1276,7 +1279,7 @@ The same figures are available as JSON from `GET /stats`, which reports `downloa `ecosystem` means three slightly different things across `/metrics`, and queries that join across them need to know which. -**From the package record, normalized.** The six `proxy_ecosystem_*` gauges, `proxy_cache_hits_total`, `proxy_cache_misses_total`, `proxy_integrity_failures_total` and the scan metrics. Aliases collapse here: `gem` reads as `rubygems`, `composer` as `packagist`, `go` as `golang`. +**From the package record, normalized.** The six `proxy_ecosystem_*` gauges, `proxy_cache_hits_total`, `proxy_cache_misses_total`, `proxy_artifacts_evicted_total`, `proxy_integrity_failures_total` and the scan metrics. Aliases collapse here: `gem` reads as `rubygems`, `composer` as `packagist`, `go` as `golang`. **From the request path.** `proxy_requests_total`, `proxy_request_duration_seconds` and `proxy_response_bytes_total`. The names mostly coincide with the normalized ones -- these also report `rubygems`, `packagist` and `golang` -- but the Debian route reports `debian` where the package record says `deb`, and any path that is not a package endpoint reports `other`, which corresponds to no ecosystem at all. diff --git a/config.example.yaml b/config.example.yaml index d29b4ba..1893859 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -74,6 +74,21 @@ storage: # Empty or "0" means unlimited max_size: "" + # Evict artifacts that have not been served for longer than a duration + # ("30d", "72h"), independent of max_size. Resolution order: package rule, + # then ecosystem rule, then default. "0" or empty means never evict by age. + # The default only applies to ecosystems that support retention; naming an + # ecosystem or package that does not is a configuration error. See + # docs/configuration.md for the supported ecosystems. + retention: + default: "" + # ecosystems: + # npm: "30d" + # packages: + # "pkg:npm/lodash": "0" + # How often to look for expired artifacts; at least "1m". + sweep_interval: "10m" + # Store fetched artifacts. Set to false to stream every download from # upstream without storing it; metadata filtering, cooldown and the # denylist still apply. Useful when another cache sits in front of the diff --git a/docs/configuration.md b/docs/configuration.md index 675d45a..f30fa27 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -46,6 +46,8 @@ storage: | `storage.path` | `PROXY_STORAGE_PATH` | `-storage-path` | Local path (deprecated, use url) | | `storage.max_size` | `PROXY_STORAGE_MAX_SIZE` | - | Max cache size (e.g., "10GB") | | `storage.cache_artifacts` | `PROXY_STORAGE_CACHE_ARTIFACTS` | - | Store fetched artifacts (default: true); `false` streams them from upstream | +| `storage.retention.default` | `PROXY_STORAGE_RETENTION_DEFAULT` | - | Evict artifacts not served for this long (e.g. "30d"); see [Retention](#retention) | +| `storage.retention.sweep_interval` | `PROXY_STORAGE_RETENTION_SWEEP_INTERVAL` | - | How often expired artifacts are looked for (default: "10m", at least "1m") | `storage.max_size` counts cached artifacts only. An artifact replaced by a refetch stays in storage for at least an hour, or `storage.direct_serve_ttl` if longer, so requests already reading it can finish, and storage use can exceed the limit by what was replaced in that time. @@ -62,6 +64,57 @@ Every download is a fresh upstream fetch, and concurrent requests for the same a `cache_artifacts: false` cannot be combined with `scanning.enabled`, `storage.direct_serve` or `mirror_api`, which all need stored artifacts, and the `mirror` command refuses to run with it. +### Retention + +Retention evicts cached artifacts that nobody has downloaded for a while. It works alongside `storage.max_size`: retention removes what has gone unused, and the size limit still applies to whatever is left. + +The `ecosystems` and `packages` entries below only pass validation once the ecosystems they name support retention; see the table further down. + +```yaml +storage: + retention: + default: "30d" + ecosystems: + npm: "14d" + maven: "0" + packages: + "pkg:npm/lodash": "0" + sweep_interval: "10m" +``` + +Each cached file ages on its own, counted from the last time the proxy served it, or from when it was fetched if it has not been served since. A newer release of the same package does not shorten the life of older versions. Durations take a `d` suffix for days, and `"0"` or an empty value means never evict by age. With no retention configured, artifacts stay until `max_size` evicts them. `sweep_interval` must be at least `1m`. + +A package rule wins over an ecosystem rule, which wins over `default`. Package rules are keyed by package PURL without a version. The ecosystems and package rules can only name ecosystems that support retention in the running version; anything else, including a typo, stops the proxy at startup with an error naming the setting. `default` applies only to supported ecosystems. Support is added one ecosystem at a time, so a later release can make an existing `default` cover more ecosystems. On startup the proxy logs which ecosystems retention covers (`covered`) and which ones `default` does not reach yet (`default_not_applied`). + +| Ecosystem key | Retention supported | Package rule form | +|---------------|---------------------|-------------------| +| `alpine` | no | - | +| `cargo` | no | - | +| `composer` | no | - | +| `conan` | no | - | +| `conda` | no | - | +| `cran` | no | - | +| `deb` | no | - | +| `gem` | no | - | +| `golang` | no | - | +| `helm` | no | - | +| `hex` | no | - | +| `julia` | no | - | +| `maven` | no | - | +| `npm` | no | - | +| `nuget` | no | - | +| `oci` | no | - | +| `pub` | no | - | +| `pypi` | no | - | +| `rpm` | no | - | +| `swift` | no | - | + +An evicted artifact is fetched again from upstream on the next request. Packages preloaded with `proxy mirror` age from the time they were mirrored, so an unused mirrored package is evicted like any other. Running `proxy mirror` again over a package that is still cached counts as a download and restarts its clock, so packages a scheduled mirror job keeps listing never age out. A registry that removes old releases (Linux distribution archives in particular) cannot serve an evicted version again, so set those ecosystems' rules with that in mind. + +Expired artifacts are not deleted right away. Their records are cleared and the files wait in the same queue a replaced artifact goes through, for at least an hour (or `storage.direct_serve_ttl` if longer), so downloads already in progress can finish. `storage.max_size` counts the space as free from the moment the record is cleared. Once that wait is over, each minute the queue is worked through for up to 30 seconds, pausing until the next minute after a delete fails. One sweep clears at most 10,000 artifacts, so the first sweep over a large, old cache spreads its evictions, and the downloads that bring popular artifacts back, over several intervals. + +With several proxies sharing one database and `database.hit_flush_interval` set, a proxy's sweep cannot see downloads another proxy has not written yet. Keep retention durations well above the flush interval. + ### Amazon S3 ```yaml @@ -520,7 +573,7 @@ gradle: |--------|-------------|-------------| | `gradle.build_cache.read_only` | `PROXY_GRADLE_BUILD_CACHE_READ_ONLY` | Disable PUT uploads and keep GET/HEAD read-only | | `gradle.build_cache.max_upload_size` | `PROXY_GRADLE_BUILD_CACHE_MAX_UPLOAD_SIZE` | Maximum accepted PUT body size (must be > 0) | -| `gradle.build_cache.max_age` | `PROXY_GRADLE_BUILD_CACHE_MAX_AGE` | Delete entries older than this duration (default `168h`, set `0` to disable) | +| `gradle.build_cache.max_age` | `PROXY_GRADLE_BUILD_CACHE_MAX_AGE` | Delete entries older than this duration (default `168h`, also accepts days such as `7d`; set `0` to disable) | | `gradle.build_cache.max_size` | `PROXY_GRADLE_BUILD_CACHE_MAX_SIZE` | Total size cap for `_gradle/http-build-cache`, deleting oldest first (`0` disables) | | `gradle.build_cache.sweep_interval` | `PROXY_GRADLE_BUILD_CACHE_SWEEP_INTERVAL` | Frequency for background eviction sweeps | diff --git a/internal/config/config.go b/internal/config/config.go index fc314bd..bf10cbe 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -74,6 +74,7 @@ import ( "time" "github.com/git-pkgs/proxy/internal/denylist" + "github.com/git-pkgs/proxy/internal/retention" "github.com/git-pkgs/purl" "gopkg.in/yaml.v3" ) @@ -407,6 +408,33 @@ type StorageConfig struct { // of the proxy. False is incompatible with scanning, direct_serve and // mirror_api, which all depend on stored artifacts. Default: true. CacheArtifacts bool `json:"cache_artifacts" yaml:"cache_artifacts"` + + // Retention evicts cached artifacts that have not been accessed for a + // configured time, per ecosystem and per package. + Retention RetentionConfig `json:"retention" yaml:"retention"` +} + +// RetentionConfig configures age-based eviction of cached artifacts. An +// artifact's age is the time since it was last served, or since it was +// fetched when it has not been served since. Durations accept a "d" suffix +// for days; "0" or empty means never evict by age. +type RetentionConfig struct { + // Default applies to every ecosystem that supports retention and has no + // rule of its own. + Default string `json:"default" yaml:"default"` + + // Ecosystems overrides the default per ecosystem key (e.g. "npm", "oci"). + // Configuring an ecosystem whose retention is not supported yet is an + // error. + Ecosystems map[string]string `json:"ecosystems" yaml:"ecosystems"` + + // Packages overrides the ecosystem rule for single packages, keyed by + // package PURL. + Packages map[string]string `json:"packages" yaml:"packages"` + + // SweepInterval is how often expired artifacts are looked for, at least + // one minute. Default: "10m". + SweepInterval string `json:"sweep_interval" yaml:"sweep_interval"` } // GradleConfig configures Gradle-specific features. @@ -895,6 +923,9 @@ func Default() *Config { Path: "./cache/artifacts", MaxSize: "", CacheArtifacts: true, + Retention: RetentionConfig{ + SweepInterval: defaultRetentionSweepIntervalStr, + }, }, Database: DatabaseConfig{ Driver: "sqlite", @@ -1034,6 +1065,8 @@ func (c *Config) LoadFromEnv() { setEnvString(&c.Storage.DirectServeTTL, "PROXY_STORAGE_DIRECT_SERVE_TTL") setEnvString(&c.Storage.DirectServeBaseURL, "PROXY_STORAGE_DIRECT_SERVE_BASE_URL") setEnvBool(&c.Storage.CacheArtifacts, "PROXY_STORAGE_CACHE_ARTIFACTS") + setEnvString(&c.Storage.Retention.Default, "PROXY_STORAGE_RETENTION_DEFAULT") + setEnvString(&c.Storage.Retention.SweepInterval, "PROXY_STORAGE_RETENTION_SWEEP_INTERVAL") setEnvString(&c.Database.Driver, "PROXY_DATABASE_DRIVER") setEnvString(&c.Database.Path, "PROXY_DATABASE_PATH") setEnvString(&c.Database.URL, "PROXY_DATABASE_URL") @@ -1212,6 +1245,10 @@ func (c *Config) validateComponents() error { return err } + if _, err := c.RetentionRules(retention.Default); err != nil { + return err + } + if _, err := denylist.New(c.Denylist.Packages); err != nil { return err } @@ -1262,7 +1299,7 @@ func (g *GradleBuildCacheConfig) Validate() error { } if g.MaxAge != "" && g.MaxAge != "0" { - if _, err := time.ParseDuration(g.MaxAge); err != nil { + if _, err := parseDayDuration(g.MaxAge); err != nil { return fmt.Errorf("invalid gradle.build_cache.max_age %q: %w", g.MaxAge, err) } } @@ -1297,6 +1334,10 @@ const ( defaultGradleBuildCacheSweepInterval = 10 * time.Minute defaultGradleMaxUploadSizeStr = "100MB" defaultGradleSweepIntervalStr = "10m" + defaultRetentionSweepInterval = 10 * time.Minute + defaultRetentionSweepIntervalStr = "10m" + minRetentionSweepInterval = time.Minute + maxDurationDays = 36500 defaultScanningTimeoutStr = "30s" ) @@ -1465,7 +1506,7 @@ func (c *Config) ParseGradleBuildCacheMaxAge() time.Duration { if c.Gradle.BuildCache.MaxAge == "" || c.Gradle.BuildCache.MaxAge == "0" { return 0 } - d, err := time.ParseDuration(c.Gradle.BuildCache.MaxAge) + d, err := parseDayDuration(c.Gradle.BuildCache.MaxAge) if err != nil || d <= 0 { return 0 } diff --git a/internal/config/retention.go b/internal/config/retention.go new file mode 100644 index 0000000..3467166 --- /dev/null +++ b/internal/config/retention.go @@ -0,0 +1,158 @@ +package config + +import ( + "fmt" + "math" + "sort" + "strconv" + "strings" + "time" + + "github.com/git-pkgs/cooldown" + "github.com/git-pkgs/proxy/internal/retention" + "github.com/git-pkgs/purl" +) + +// parseDayDuration parses a duration that may use a "d" suffix for days, as +// cooldown values do. Day counts that are not finite or exceed +// maxDurationDays are rejected, since converting them to a time.Duration +// would overflow. +func parseDayDuration(s string) (time.Duration, error) { + if num, ok := strings.CutSuffix(strings.TrimSpace(s), "d"); ok { + days, err := strconv.ParseFloat(num, 64) + if err == nil && (math.IsNaN(days) || math.IsInf(days, 0) || math.Abs(days) > maxDurationDays) { + return 0, fmt.Errorf("invalid duration %q: day count out of range", s) + } + } + return cooldown.ParseDuration(s) +} + +// parseRetentionDuration parses one retention value. field names the +// setting in error messages. +func parseRetentionDuration(field, value string) (time.Duration, error) { + d, err := parseDayDuration(value) + if err != nil { + return 0, fmt.Errorf("invalid %s: %w", field, err) + } + if d < 0 { + return 0, fmt.Errorf("invalid %s %q: must not be negative", field, value) + } + return d, nil +} + +// RetentionRules validates storage.retention against the ecosystems +// registered with reg and resolves it. An ecosystem or package that reg +// does not support is an error, so a rule is never silently ignored. +func (c *Config) RetentionRules(reg *retention.Registry) (retention.Rules, error) { + r := c.Storage.Retention + rules := retention.Rules{ + Ecosystems: map[string]time.Duration{}, + Packages: map[string]time.Duration{}, + } + + var err error + if rules.Default, err = parseRetentionDuration("storage.retention.default", r.Default); err != nil { + return retention.Rules{}, err + } + + if r.SweepInterval != "" { + d, err := parseDayDuration(r.SweepInterval) + if err != nil { + return retention.Rules{}, fmt.Errorf("invalid storage.retention.sweep_interval: %w", err) + } + if d < minRetentionSweepInterval { + return retention.Rules{}, fmt.Errorf("invalid storage.retention.sweep_interval %q: must be at least %s", + r.SweepInterval, minRetentionSweepInterval) + } + } + + for _, key := range sortedKeys(r.Ecosystems) { + field := "storage.retention.ecosystems." + key + if !retention.IsKnown(key) { + return retention.Rules{}, fmt.Errorf("invalid %s: unknown ecosystem %q (valid: %s)", + field, key, strings.Join(retention.KnownKeys, ", ")) + } + if _, ok := reg.Lookup(key); !ok { + return retention.Rules{}, fmt.Errorf("invalid %s: retention is not supported for %q in this version", field, key) + } + d, err := parseRetentionDuration(field, r.Ecosystems[key]) + if err != nil { + return retention.Rules{}, err + } + rules.Ecosystems[key] = d + } + + for _, key := range sortedKeys(r.Packages) { + field := fmt.Sprintf("storage.retention.packages[%q]", key) + canonical, err := retentionPackageKey(reg, key) + if err != nil { + return retention.Rules{}, fmt.Errorf("invalid %s: %w", field, err) + } + d, err := parseRetentionDuration(field, r.Packages[key]) + if err != nil { + return retention.Rules{}, err + } + if prev, ok := rules.Packages[canonical]; ok && prev != d { + return retention.Rules{}, fmt.Errorf("invalid %s: conflicts with another key for package %s", field, canonical) + } + rules.Packages[canonical] = d + } + + return rules, nil +} + +// retentionPackageKey returns the stored package PURL a configured package +// key stands for. +func retentionPackageKey(reg *retention.Registry, key string) (string, error) { + // A julia PURL needs a uuid qualifier to parse at all, and the julia + // handler stores none, so no key could ever match one of its packages. + if strings.HasPrefix(strings.ToLower(key), "pkg:julia/") { + return "", fmt.Errorf("package rules are not supported for julia") + } + p, err := purl.Parse(key) + if err != nil { + return "", fmt.Errorf("not a valid package PURL: %w", err) + } + if p.Version != "" { + return "", fmt.Errorf("package PURL must not carry a version") + } + // A rule names a whole package; qualifiers and subpaths would be + // dropped on the way to the stored PURL, so refuse them rather than + // let "?type=pom" quietly cover every file of the package. + if len(p.Qualifiers) > 0 || p.Subpath != "" { + return "", fmt.Errorf("package PURL must not carry qualifiers or a subpath") + } + canonical, ok := reg.Canonical(p) + if ok { + return canonical, nil + } + ecosystem, registered := reg.KeyForDBEcosystem(purl.PURLTypeToEcosystem(p.Type)) + if !registered { + return "", fmt.Errorf("retention is not supported for %s packages in this version", p.Type) + } + if spec, _ := reg.Lookup(ecosystem); spec.CanonicalPackage == nil { + return "", fmt.Errorf("package rules are not supported for %s; use storage.retention.ecosystems.%s", ecosystem, ecosystem) + } + return "", fmt.Errorf("does not match how %s packages are stored; see the package rule forms in docs/configuration.md", ecosystem) +} + +func sortedKeys(m map[string]string) []string { + keys := make([]string, 0, len(m)) + for k := range m { + keys = append(keys, k) + } + sort.Strings(keys) + return keys +} + +// ParseRetentionSweepInterval returns how often the retention sweep runs. +func (c *Config) ParseRetentionSweepInterval() time.Duration { + if c.Storage.Retention.SweepInterval == "" { + return defaultRetentionSweepInterval + } + d, err := parseDayDuration(c.Storage.Retention.SweepInterval) + if err != nil || d < minRetentionSweepInterval { + return defaultRetentionSweepInterval + } + return d +} diff --git a/internal/config/retention_test.go b/internal/config/retention_test.go new file mode 100644 index 0000000..e416301 --- /dev/null +++ b/internal/config/retention_test.go @@ -0,0 +1,167 @@ +package config + +import ( + "strings" + "testing" + "time" + + "github.com/git-pkgs/proxy/internal/retention" +) + +const retentionDay = 24 * time.Hour + +func retentionTestRegistry() *retention.Registry { + reg := retention.NewRegistry() + reg.Register(retention.Spec{Key: "npm", CanonicalPackage: retention.DefaultCanonical("npm")}) + reg.Register(retention.Spec{Key: "julia"}) + reg.Register(retention.Spec{Key: "helm"}) + return reg +} + +func TestRetentionRulesUnsetIsDisabled(t *testing.T) { + rules, err := Default().RetentionRules(retentionTestRegistry()) + if err != nil { + t.Fatalf("RetentionRules() error = %v", err) + } + if got := rules.MinPositive(); got != 0 { + t.Errorf("MinPositive() = %v, want 0 for an unset retention block", got) + } +} + +func TestRetentionRulesResolves(t *testing.T) { + cfg := Default() + cfg.Storage.Retention = RetentionConfig{ + Default: "30d", + Ecosystems: map[string]string{"npm": "14d"}, + Packages: map[string]string{ + "pkg:npm/lodash": "0", + "pkg:npm/@babel/core": "90d", + }, + SweepInterval: "5m", + } + + rules, err := cfg.RetentionRules(retentionTestRegistry()) + if err != nil { + t.Fatalf("RetentionRules() error = %v", err) + } + if rules.Default != 30*retentionDay { + t.Errorf("Default = %v, want 30d", rules.Default) + } + if got := rules.For("npm", "pkg:npm/left-pad"); got != 14*retentionDay { + t.Errorf("npm rule = %v, want 14d", got) + } + if got, ok := rules.Packages["pkg:npm/lodash"]; !ok || got != 0 { + t.Errorf("lodash rule = %v, %v, want 0, true", got, ok) + } + if got := rules.For("npm", "pkg:npm/%40babel/core"); got != 90*retentionDay { + t.Errorf("babel rule = %v, want 90d under its canonical key", got) + } + if got := cfg.ParseRetentionSweepInterval(); got != 5*time.Minute { + t.Errorf("ParseRetentionSweepInterval() = %v, want 5m", got) + } +} + +func TestRetentionRulesErrors(t *testing.T) { + tests := []struct { + name string + modify func(*RetentionConfig) + wantErr string + }{ + {"bad default", func(r *RetentionConfig) { r.Default = "7x" }, "storage.retention.default"}, + {"negative default", func(r *RetentionConfig) { r.Default = "-1d" }, "must not be negative"}, + {"NaN days", func(r *RetentionConfig) { r.Default = "NaNd" }, "out of range"}, + {"infinite days", func(r *RetentionConfig) { r.Default = "Infd" }, "out of range"}, + {"overflowing days", func(r *RetentionConfig) { r.Default = "1e9d" }, "out of range"}, + {"zero sweep interval", func(r *RetentionConfig) { r.SweepInterval = "0s" }, "must be at least 1m0s"}, + {"sweep interval below a minute", func(r *RetentionConfig) { r.SweepInterval = "30s" }, "must be at least 1m0s"}, + {"bad sweep interval", func(r *RetentionConfig) { r.SweepInterval = "often" }, "sweep_interval"}, + {"unknown ecosystem", func(r *RetentionConfig) { r.Ecosystems = map[string]string{"nmp": "1d"} }, "unknown ecosystem"}, + {"generic is not an ecosystem key", func(r *RetentionConfig) { r.Ecosystems = map[string]string{"generic": "1d"} }, "unknown ecosystem"}, + {"ecosystem not registered", func(r *RetentionConfig) { r.Ecosystems = map[string]string{"oci": "14d"} }, + `storage.retention.ecosystems.oci: retention is not supported for "oci"`}, + {"bad ecosystem duration", func(r *RetentionConfig) { r.Ecosystems = map[string]string{"npm": "soon"} }, "storage.retention.ecosystems.npm"}, + {"package of unregistered ecosystem", func(r *RetentionConfig) { r.Packages = map[string]string{"pkg:cargo/serde": "1d"} }, + "not supported for cargo packages"}, + {"julia package", func(r *RetentionConfig) { r.Packages = map[string]string{"pkg:julia/Example": "1d"} }, + "not supported for julia"}, + {"package of ecosystem without package rules", func(r *RetentionConfig) { r.Packages = map[string]string{"pkg:helm/nginx": "1d"} }, + "package rules are not supported for helm; use storage.retention.ecosystems.helm"}, + {"invalid package PURL", func(r *RetentionConfig) { r.Packages = map[string]string{"lodash": "1d"} }, "not a valid package PURL"}, + {"npm package without its scope marker", func(r *RetentionConfig) { r.Packages = map[string]string{"pkg:npm/babel/core": "1d"} }, + "does not match how npm packages are stored"}, + {"npm package with an encoded scope separator", func(r *RetentionConfig) { r.Packages = map[string]string{"pkg:npm/%40babel%2Fcore": "1d"} }, + "does not match how npm packages are stored"}, + {"versioned package PURL", func(r *RetentionConfig) { r.Packages = map[string]string{"pkg:npm/lodash@4.17.21": "1d"} }, + "must not carry a version"}, + {"conflicting package keys", func(r *RetentionConfig) { + r.Packages = map[string]string{"pkg:npm/%40babel/core": "1d", "pkg:npm/@babel/core": "2d"} + }, "conflicts with another key"}, + {"package PURL with qualifiers", func(r *RetentionConfig) { r.Packages = map[string]string{"pkg:npm/lodash?type=tgz": "1d"} }, + "must not carry qualifiers or a subpath"}, + {"package PURL with subpath", func(r *RetentionConfig) { r.Packages = map[string]string{"pkg:npm/lodash#lib": "1d"} }, + "must not carry qualifiers or a subpath"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cfg := Default() + tt.modify(&cfg.Storage.Retention) + _, err := cfg.RetentionRules(retentionTestRegistry()) + if err == nil { + t.Fatalf("RetentionRules() succeeded, want error containing %q", tt.wantErr) + } + if !strings.Contains(err.Error(), tt.wantErr) { + t.Errorf("RetentionRules() error = %q, want it to contain %q", err, tt.wantErr) + } + }) + } +} + +// Validate checks retention against the handlers' registry, which this +// package's tests leave empty, so any ecosystem rule fails while a bare +// default passes. +func TestValidateRetention(t *testing.T) { + cfg := Default() + cfg.Storage.Retention.Default = "30d" + if err := cfg.Validate(); err != nil { + t.Errorf("Validate() with only a default = %v, want nil", err) + } + + cfg.Storage.Retention.Ecosystems = map[string]string{"npm": "14d"} + err := cfg.Validate() + if err == nil || !strings.Contains(err.Error(), "not supported") { + t.Errorf("Validate() with an unsupported ecosystem = %v, want a not supported error", err) + } +} + +func TestRetentionEnvOverrides(t *testing.T) { + t.Setenv("PROXY_STORAGE_RETENTION_DEFAULT", "21d") + t.Setenv("PROXY_STORAGE_RETENTION_SWEEP_INTERVAL", "1h") + + cfg := Default() + cfg.LoadFromEnv() + + if cfg.Storage.Retention.Default != "21d" { + t.Errorf("Retention.Default = %q, want 21d", cfg.Storage.Retention.Default) + } + if got := cfg.ParseRetentionSweepInterval(); got != time.Hour { + t.Errorf("ParseRetentionSweepInterval() = %v, want 1h", got) + } +} + +func TestRetentionSweepIntervalDefault(t *testing.T) { + cfg := Default() + if got := cfg.ParseRetentionSweepInterval(); got != 10*time.Minute { + t.Errorf("ParseRetentionSweepInterval() = %v, want 10m", got) + } +} + +func TestGradleBuildCacheMaxAgeAcceptsDays(t *testing.T) { + cfg := Default() + cfg.Gradle.BuildCache.MaxAge = "7d" + if err := cfg.Validate(); err != nil { + t.Fatalf("Validate() with max_age 7d = %v, want nil", err) + } + if got := cfg.ParseGradleBuildCacheMaxAge(); got != 7*retentionDay { + t.Errorf("ParseGradleBuildCacheMaxAge() = %v, want 168h", got) + } +} diff --git a/internal/database/hits.go b/internal/database/hits.go index 249c663..4b5de0b 100644 --- a/internal/database/hits.go +++ b/internal/database/hits.go @@ -129,7 +129,8 @@ func (db *DB) writeHits(pending map[hitKey]hitEntry) error { }) for _, k := range keys { e := pending[k] - if _, err := stmt.Exec(e.count, e.last, e.last, e.last, e.last, k.versionPURL, k.filename); err != nil { + last := e.last.UTC() + if _, err := stmt.Exec(e.count, last, last, e.last, e.last, k.versionPURL, k.filename); err != nil { return err } } diff --git a/internal/database/queries.go b/internal/database/queries.go index 40fd2ce..2652bc1 100644 --- a/internal/database/queries.go +++ b/internal/database/queries.go @@ -322,6 +322,17 @@ const upsertArtifactAttempts = 5 // would remove it. The write applies only if the record still holds the path // read before it, so commits racing on one artifact each queue the path they // replaced. +// inUTC returns t in UTC. Artifacts store fetched_at and last_accessed_at in +// UTC: lib/pq sends a time with its offset, a Postgres TIMESTAMP column keeps +// only the wall clock, and lib/pq reads that back as UTC, so anything else +// comes back shifted by the local offset. +func inUTC(t sql.NullTime) sql.NullTime { + if t.Valid { + t.Time = t.Time.UTC() + } + return t +} + func (db *DB) UpsertArtifact(a *Artifact) error { for range upsertArtifactAttempts { var previous sql.NullString @@ -388,7 +399,7 @@ func (db *DB) upsertArtifactFrom(a *Artifact, previous sql.NullString) (bool, er res, err := db.Exec(query, a.VersionPURL, a.Filename, a.UpstreamURL, a.StoragePath, a.ContentHash, - a.Size, a.ContentType, a.FetchedAt, a.HitCount, a.LastAccessedAt, now, now, previous, + a.Size, a.ContentType, inUTC(a.FetchedAt), a.HitCount, inUTC(a.LastAccessedAt), now, now, previous, ) if err != nil { return false, fmt.Errorf("upserting artifact: %w", err) @@ -411,7 +422,7 @@ func (db *DB) RecordArtifactHit(versionPURL, filename string) error { SET hit_count = hit_count + 1, last_accessed_at = ?, updated_at = ? WHERE version_purl = ? AND filename = ? `) - _, err := db.Exec(query, now, now, versionPURL, filename) + _, err := db.Exec(query, now.UTC(), now, versionPURL, filename) return err } @@ -423,7 +434,7 @@ func (db *DB) MarkArtifactCached(versionPURL, filename, storagePath, contentHash fetched_at = ?, updated_at = ? WHERE version_purl = ? AND filename = ? `) - _, err := db.Exec(query, storagePath, contentHash, size, contentType, now, now, versionPURL, filename) + _, err := db.Exec(query, storagePath, contentHash, size, contentType, now.UTC(), now, versionPURL, filename) return err } @@ -499,14 +510,16 @@ func (db *DB) DiscardArtifact(versionPURL, filename, storagePath string) error { // Queueing a path again, as reclaim does after a failed delete, restarts its // grace period and moves it behind the rest. func (db *DB) QueuePendingDelete(path string) error { - query := db.Rebind(` - INSERT INTO pending_deletes (path, queued_at) VALUES (?, ?) - ON CONFLICT(path) DO UPDATE SET queued_at = excluded.queued_at - `) - _, err := db.Exec(query, path, time.Now().UTC()) + _, err := db.Exec(db.Rebind(queuePendingDeleteQuery), path, time.Now().UTC()) return err } +// queuePendingDeleteQuery queues a path, or moves an already queued one to +// the back of the queue. It takes the path and the time it is queued at. +const queuePendingDeleteQuery = ` + INSERT INTO pending_deletes (path, queued_at) VALUES (?, ?) + ON CONFLICT(path) DO UPDATE SET queued_at = excluded.queued_at` + // GetDuePendingDeletes returns up to limit paths queued before cutoff, oldest // first. A path a record points at again is skipped. func (db *DB) GetDuePendingDeletes(cutoff time.Time, limit int) ([]string, error) { diff --git a/internal/database/retention.go b/internal/database/retention.go new file mode 100644 index 0000000..cd6e11f --- /dev/null +++ b/internal/database/retention.go @@ -0,0 +1,128 @@ +package database + +import ( + "database/sql" + "errors" + "fmt" + "time" +) + +// RetentionCandidate is a cached artifact that has not been accessed since a +// cutoff, together with the package record it belongs to. +type RetentionCandidate struct { + ID int64 `db:"id"` + VersionPURL string `db:"version_purl"` + Filename string `db:"filename"` + StoragePath string `db:"storage_path"` + Size sql.NullInt64 `db:"size"` + FetchedAt sql.NullTime `db:"fetched_at"` + LastAccessedAt sql.NullTime `db:"last_accessed_at"` + Ecosystem string `db:"ecosystem"` + PackagePURL string `db:"package_purl"` +} + +// ExpiredBefore reports whether the artifact was neither fetched nor +// accessed at or after cutoff. Both are checked: clearing a record keeps its +// last access time, so a record fetched again after an eviction has a fresh +// fetch time next to an old access time. +func (c RetentionCandidate) ExpiredBefore(cutoff time.Time) bool { + if !c.FetchedAt.Valid || !c.FetchedAt.Time.Before(cutoff) { + return false + } + return !c.LastAccessedAt.Valid || c.LastAccessedAt.Time.Before(cutoff) +} + +// expiredCondition selects cached artifacts fetched before the cutoff and not +// accessed since. It takes the cutoff twice. +const expiredCondition = ` + storage_path IS NOT NULL + AND fetched_at < ? + AND (last_accessed_at IS NULL OR last_accessed_at < ?)` + +// MaxArtifactID returns the highest artifact id, or zero for an empty table. +func (db *DB) MaxArtifactID() (int64, error) { + var maxID sql.NullInt64 + if err := db.Get(&maxID, `SELECT MAX(id) FROM artifacts`); err != nil { + return 0, err + } + return maxID.Int64, nil +} + +// GetRetentionCandidates returns the cached artifacts with fromID < id <= toID +// that expired before cutoff. Bounding the scan by id keeps each query short, +// which matters on SQLite, where every request shares one connection. +func (db *DB) GetRetentionCandidates(fromID, toID int64, cutoff time.Time) ([]RetentionCandidate, error) { + var candidates []RetentionCandidate + query := db.Rebind(` + SELECT a.id, a.version_purl, a.filename, a.storage_path, a.size, + a.fetched_at, a.last_accessed_at, p.ecosystem, p.purl AS package_purl + FROM ( + SELECT * FROM artifacts + WHERE id > ? AND id <= ? AND` + expiredCondition + ` + ) a + JOIN versions v ON v.purl = a.version_purl + JOIN packages p ON p.purl = v.package_purl + ORDER BY a.id + `) + if err := db.Select(&candidates, query, fromID, toID, cutoff.UTC(), cutoff.UTC()); err != nil { + return nil, err + } + return candidates, nil +} + +// ClearExpiredArtifact marks an artifact uncached like ClearArtifactCache, +// but only while it is still expired before cutoff, so an artifact served +// between the scan and the clear stays cached. It queues storagePath for +// deletion in the same transaction: if queueing fails, the record keeps its +// path and a later sweep tries again, instead of the object staying in +// storage with nothing left pointing at it. +func (db *DB) ClearExpiredArtifact(versionPURL, filename, storagePath string, cutoff time.Time) (bool, error) { + tx, err := db.Beginx() + if err != nil { + return false, err + } + defer func() { _ = tx.Rollback() }() + + query := db.Rebind(` + UPDATE artifacts + SET storage_path = NULL, content_hash = NULL, size = NULL, + content_type = NULL, fetched_at = NULL, updated_at = ? + WHERE version_purl = ? AND filename = ? AND storage_path = ? AND` + expiredCondition) + res, err := tx.Exec(query, time.Now(), versionPURL, filename, storagePath, cutoff.UTC(), cutoff.UTC()) + if err != nil { + return false, err + } + n, err := res.RowsAffected() + if err != nil || n == 0 { + return false, err + } + if _, err := tx.Exec(db.Rebind(queuePendingDeleteQuery), storagePath, time.Now().UTC()); err != nil { + return false, fmt.Errorf("queueing %s for deletion: %w", storagePath, err) + } + return true, tx.Commit() +} + +// GetArtifactEcosystem returns the packages.ecosystem value of the package an +// artifact version belongs to, or "" when it has no package record. +func (db *DB) GetArtifactEcosystem(versionPURL string) (string, error) { + var ecosystem string + query := db.Rebind(` + SELECT p.ecosystem FROM versions v + JOIN packages p ON p.purl = v.package_purl + WHERE v.purl = ? + `) + err := db.Get(&ecosystem, query, versionPURL) + if errors.Is(err, sql.ErrNoRows) { + return "", nil + } + return ecosystem, err +} + +// FlushHits writes hits buffered by BatchHits now, so a caller about to read +// last_accessed_at sees them. +func (db *DB) FlushHits() error { + if db.hits == nil { + return nil + } + return db.flushHits() +} diff --git a/internal/database/retention_test.go b/internal/database/retention_test.go new file mode 100644 index 0000000..213ae22 --- /dev/null +++ b/internal/database/retention_test.go @@ -0,0 +1,396 @@ +package database + +import ( + "database/sql" + "slices" + "strings" + "testing" + "time" +) + +// seedRetentionArtifact stores a cached artifact for pkg:{ecosystem}/{name} +// with the given fetch and last access times; a zero accessed leaves the +// artifact never served. +// Artifacts store fetched_at and last_accessed_at in UTC, and the retention +// cutoffs are compared in UTC. In a zone east of UTC, any write or cutoff +// left in local time shifts an artifact by the offset on Postgres. +func TestArtifactTimesAreStoredInUTC(t *testing.T) { + tokyo, err := time.LoadLocation("Asia/Tokyo") + if err != nil { + t.Skipf("time zone data unavailable: %v", err) + } + prev := time.Local + time.Local = tokyo + t.Cleanup(func() { time.Local = prev }) + + runWithBothDatabases(t, func(t *testing.T, db *DB) { + now := time.Now() + old := now.Add(-48 * time.Hour) + seedRetentionArtifact(t, db, "npm", "stale", now.Add(-90*time.Minute), time.Time{}) + seedRetentionArtifact(t, db, "npm", "fresh", now.Add(-30*time.Minute), time.Time{}) + seedRetentionArtifact(t, db, "npm", "hit", old, time.Time{}) + if err := db.RecordArtifactHit("pkg:npm/hit@1.0.0", "hit.tgz"); err != nil { + t.Fatalf("RecordArtifactHit: %v", err) + } + seedRetentionArtifact(t, db, "npm", "batched", old, time.Time{}) + hits := map[hitKey]hitEntry{{"pkg:npm/batched@1.0.0", "batched.tgz"}: {count: 1, last: now.Add(-5 * time.Hour)}} + if err := db.writeHits(hits); err != nil { + t.Fatalf("writeHits: %v", err) + } + seedRetentionArtifact(t, db, "npm", "refetched", old, time.Time{}) + if err := db.MarkArtifactCached("pkg:npm/refetched@1.0.0", "refetched.tgz", "npm/refetched/1.0.0/refetched.tgz", "hash", 100, ""); err != nil { + t.Fatalf("MarkArtifactCached: %v", err) + } + + expired := func(cutoff time.Time) []string { + t.Helper() + candidates, err := db.GetRetentionCandidates(0, 100, cutoff) + if err != nil { + t.Fatalf("GetRetentionCandidates: %v", err) + } + var names []string + for _, c := range candidates { + if !c.ExpiredBefore(cutoff) { + t.Errorf("%s returned as a candidate but read back as not expired before %v", c.Filename, cutoff) + } + names = append(names, strings.TrimSuffix(c.Filename, ".tgz")) + } + slices.Sort(names) + return names + } + if got, want := expired(now.Add(-time.Hour)), []string{"batched", "stale"}; !slices.Equal(got, want) { + t.Errorf("expired an hour ago = %v, want %v", got, want) + } + if got, want := expired(now.Add(time.Hour)), []string{"batched", "fresh", "hit", "refetched", "stale"}; !slices.Equal(got, want) { + t.Errorf("expired an hour from now = %v, want %v", got, want) + } + cleared, err := db.ClearExpiredArtifact("pkg:npm/fresh@1.0.0", "fresh.tgz", "npm/fresh/1.0.0/fresh.tgz", now.Add(-time.Hour)) + if err != nil || cleared { + t.Errorf("ClearExpiredArtifact(fresh, an hour ago) = %v, %v, want false, nil", cleared, err) + } + }) +} + +func seedRetentionArtifact(t *testing.T, db *DB, ecosystem, name string, fetched, accessed time.Time) { + t.Helper() + pkgPURL := "pkg:" + ecosystem + "/" + name + versionPURL := pkgPURL + "@1.0.0" + if err := db.UpsertPackage(&Package{PURL: pkgPURL, Ecosystem: ecosystem, Name: name}); err != nil { + t.Fatalf("UpsertPackage: %v", err) + } + if err := db.UpsertVersion(&Version{PURL: versionPURL, PackagePURL: pkgPURL}); err != nil { + t.Fatalf("UpsertVersion: %v", err) + } + art := &Artifact{ + VersionPURL: versionPURL, + Filename: name + ".tgz", + UpstreamURL: "https://example.com/" + name + ".tgz", + StoragePath: sql.NullString{String: ecosystem + "/" + name + "/1.0.0/" + name + ".tgz", Valid: true}, + Size: sql.NullInt64{Int64: 100, Valid: true}, + FetchedAt: sql.NullTime{Time: fetched, Valid: true}, + } + if !accessed.IsZero() { + art.LastAccessedAt = sql.NullTime{Time: accessed, Valid: true} + } + if err := db.UpsertArtifact(art); err != nil { + t.Fatalf("UpsertArtifact: %v", err) + } +} + +func candidateNames(candidates []RetentionCandidate) map[string]RetentionCandidate { + byName := make(map[string]RetentionCandidate, len(candidates)) + for _, c := range candidates { + byName[c.Filename] = c + } + return byName +} + +func TestGetRetentionCandidates(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + now := time.Now() + cutoff := now.Add(-24 * time.Hour) + old := now.Add(-48 * time.Hour) + + seedRetentionArtifact(t, db, "npm", "stale", old, old) + seedRetentionArtifact(t, db, "npm", "never-served", old, time.Time{}) + seedRetentionArtifact(t, db, "npm", "recently-served", old, now) + // Fetched again after an eviction: the clear kept the old access time. + seedRetentionArtifact(t, db, "npm", "refetched", now, old) + seedRetentionArtifact(t, db, "gem", "other-ecosystem", old, old) + + maxID, err := db.MaxArtifactID() + if err != nil { + t.Fatalf("MaxArtifactID: %v", err) + } + candidates, err := db.GetRetentionCandidates(0, maxID, cutoff) + if err != nil { + t.Fatalf("GetRetentionCandidates: %v", err) + } + got := candidateNames(candidates) + + for _, want := range []string{"stale.tgz", "never-served.tgz", "other-ecosystem.tgz"} { + if _, ok := got[want]; !ok { + t.Errorf("%s missing from candidates", want) + } + } + for _, unwanted := range []string{"recently-served.tgz", "refetched.tgz"} { + if _, ok := got[unwanted]; ok { + t.Errorf("%s returned as a candidate", unwanted) + } + } + + stale := got["stale.tgz"] + if stale.Ecosystem != "npm" || stale.PackagePURL != "pkg:npm/stale" { + t.Errorf("stale candidate ecosystem/package = %q/%q, want npm/pkg:npm/stale", stale.Ecosystem, stale.PackagePURL) + } + if !stale.ExpiredBefore(cutoff) { + t.Error("stale candidate not expired before the cutoff") + } + if got["other-ecosystem.tgz"].Ecosystem != "gem" { + t.Errorf("other-ecosystem candidate ecosystem = %q, want gem", got["other-ecosystem.tgz"].Ecosystem) + } + }) +} + +func TestGetRetentionCandidatesWindow(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + old := time.Now().Add(-48 * time.Hour) + for _, name := range []string{"a", "b", "c"} { + seedRetentionArtifact(t, db, "npm", name, old, old) + } + cutoff := time.Now().Add(-time.Hour) + + first, err := db.GetRetentionCandidates(0, 1, cutoff) + if err != nil { + t.Fatalf("GetRetentionCandidates: %v", err) + } + rest, err := db.GetRetentionCandidates(1, 3, cutoff) + if err != nil { + t.Fatalf("GetRetentionCandidates: %v", err) + } + if len(first) != 1 || len(rest) != 2 { + t.Fatalf("window sizes = %d and %d, want 1 and 2", len(first), len(rest)) + } + if first[0].ID != 1 || rest[0].ID != 2 || rest[1].ID != 3 { + t.Errorf("window ids = %d, %d, %d, want 1, 2, 3", first[0].ID, rest[0].ID, rest[1].ID) + } + }) +} + +func TestClearExpiredArtifact(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + now := time.Now() + old := now.Add(-48 * time.Hour) + cutoff := now.Add(-24 * time.Hour) + seedRetentionArtifact(t, db, "npm", "stale", old, old) + seedRetentionArtifact(t, db, "npm", "served", old, old) + const stalePath = "npm/stale/1.0.0/stale.tgz" + + // A hit lands between the scan and the clear. + if err := db.RecordArtifactHit("pkg:npm/served@1.0.0", "served.tgz"); err != nil { + t.Fatalf("RecordArtifactHit: %v", err) + } + cleared, err := db.ClearExpiredArtifact("pkg:npm/served@1.0.0", "served.tgz", "npm/served/1.0.0/served.tgz", cutoff) + if err != nil || cleared { + t.Errorf("ClearExpiredArtifact(served) = %v, %v, want false, nil", cleared, err) + } + + cleared, err = db.ClearExpiredArtifact("pkg:npm/stale@1.0.0", "stale.tgz", "elsewhere", cutoff) + if err != nil || cleared { + t.Errorf("ClearExpiredArtifact(moved record) = %v, %v, want false, nil", cleared, err) + } + + cleared, err = db.ClearExpiredArtifact("pkg:npm/stale@1.0.0", "stale.tgz", stalePath, cutoff) + if err != nil || !cleared { + t.Fatalf("ClearExpiredArtifact(stale) = %v, %v, want true, nil", cleared, err) + } + art, err := db.GetArtifact("pkg:npm/stale@1.0.0", "stale.tgz") + if err != nil { + t.Fatalf("GetArtifact: %v", err) + } + if art.StoragePath.Valid || art.FetchedAt.Valid { + t.Error("stale artifact still cached after the clear") + } + if !art.LastAccessedAt.Valid { + t.Error("clear dropped last_accessed_at") + } + var queued int + if err := db.Get(&queued, db.Rebind(`SELECT COUNT(*) FROM pending_deletes WHERE path = ?`), stalePath); err != nil || queued != 1 { + t.Errorf("queued deletes for %s = %d, %v, want 1", stalePath, queued, err) + } + }) +} + +// When the object can't be queued for deletion, the clear is rolled back: +// the record keeps pointing at the object, so a later sweep retries it +// rather than leaving a file in storage that nothing refers to. +func TestClearExpiredArtifactRollsBackWhenQueueingFails(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + now := time.Now() + old := now.Add(-48 * time.Hour) + seedRetentionArtifact(t, db, "npm", "stale", old, old) + if _, err := db.Exec(`DROP TABLE pending_deletes`); err != nil { + t.Fatalf("dropping pending_deletes: %v", err) + } + + cleared, err := db.ClearExpiredArtifact("pkg:npm/stale@1.0.0", "stale.tgz", "npm/stale/1.0.0/stale.tgz", now.Add(-24*time.Hour)) + if err == nil || cleared { + t.Fatalf("ClearExpiredArtifact = %v, %v, want false and an error", cleared, err) + } + art, err := db.GetArtifact("pkg:npm/stale@1.0.0", "stale.tgz") + if err != nil { + t.Fatalf("GetArtifact: %v", err) + } + if !art.StoragePath.Valid || !art.FetchedAt.Valid { + t.Error("record cleared although its object couldn't be queued for deletion") + } + }) +} + +func TestFlushHitsWritesBufferedHits(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + old := time.Now().Add(-48 * time.Hour) + seedRetentionArtifact(t, db, "npm", "buffered", old, old) + db.BatchHits(time.Hour, nil) + + if err := db.RecordArtifactHit("pkg:npm/buffered@1.0.0", "buffered.tgz"); err != nil { + t.Fatalf("RecordArtifactHit: %v", err) + } + if err := db.FlushHits(); err != nil { + t.Fatalf("FlushHits: %v", err) + } + art, err := db.GetArtifact("pkg:npm/buffered@1.0.0", "buffered.tgz") + if err != nil { + t.Fatalf("GetArtifact: %v", err) + } + if !art.LastAccessedAt.Time.After(old) { + t.Errorf("last_accessed_at = %v after FlushHits, want the buffered hit", art.LastAccessedAt.Time) + } + }) +} + +func TestFlushHitsWithoutBatching(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + if err := db.FlushHits(); err != nil { + t.Errorf("FlushHits without batching = %v, want nil", err) + } + }) +} + +func TestMaxArtifactIDEmpty(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + maxID, err := db.MaxArtifactID() + if err != nil || maxID != 0 { + t.Errorf("MaxArtifactID() = %d, %v, want 0, nil", maxID, err) + } + }) +} + +func TestGetArtifactEcosystem(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + seedRetentionArtifact(t, db, "rubygems", "rails", time.Now(), time.Time{}) + + got, err := db.GetArtifactEcosystem("pkg:rubygems/rails@1.0.0") + if err != nil || got != "rubygems" { + t.Errorf("GetArtifactEcosystem() = %q, %v, want rubygems, nil", got, err) + } + got, err = db.GetArtifactEcosystem("pkg:npm/missing@1.0.0") + if err != nil || got != "" { + t.Errorf("GetArtifactEcosystem(missing) = %q, %v, want empty, nil", got, err) + } + }) +} + +func TestPendingDeletesQueuedAtIndex(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + indexQuery := `SELECT COUNT(*) FROM sqlite_master WHERE type = 'index' AND name = 'idx_pending_deletes_queued_at'` + if db.Dialect() == DialectPostgres { + indexQuery = `SELECT COUNT(*) FROM pg_indexes WHERE indexname = 'idx_pending_deletes_queued_at'` + } + hasIndex := func() bool { + t.Helper() + var count int + if err := db.Get(&count, indexQuery); err != nil { + t.Fatalf("querying indexes: %v", err) + } + return count == 1 + } + + if !hasIndex() { + t.Error("fresh schema has no idx_pending_deletes_queued_at") + } + if _, err := db.Exec(`DROP INDEX idx_pending_deletes_queued_at`); err != nil { + t.Fatalf("dropping index: %v", err) + } + if err := migrateAddPendingDeletesQueuedAtIndex(db); err != nil { + t.Fatalf("migration: %v", err) + } + if err := migrateAddPendingDeletesQueuedAtIndex(db); err != nil { + t.Fatalf("migration is not idempotent: %v", err) + } + if !hasIndex() { + t.Error("migration did not create idx_pending_deletes_queued_at") + } + }) +} + +func TestRetentionCandidateExpiredBefore(t *testing.T) { + now := time.Now() + cutoff := now.Add(-24 * time.Hour) + old := sql.NullTime{Time: now.Add(-48 * time.Hour), Valid: true} + recent := sql.NullTime{Time: now, Valid: true} + + tests := []struct { + name string + fetched, accessed sql.NullTime + want bool + }{ + {"fetched and accessed before", old, old, true}, + {"never served", old, sql.NullTime{}, true}, + {"accessed since", old, recent, false}, + {"fetched again since", recent, old, false}, + {"not cached", sql.NullTime{}, old, false}, + } + for _, tt := range tests { + c := RetentionCandidate{FetchedAt: tt.fetched, LastAccessedAt: tt.accessed} + if got := c.ExpiredBefore(cutoff); got != tt.want { + t.Errorf("%s: ExpiredBefore() = %v, want %v", tt.name, got, tt.want) + } + } +} + +// A TIMESTAMP column on Postgres holds the local wall clock time without a +// zone. Read back, the candidate's times must still compare correctly +// against a cutoff east of UTC, where a time labelled UTC would look hours +// newer than it is. +func TestGetRetentionCandidatesTimezone(t *testing.T) { + tokyo, err := time.LoadLocation("Asia/Tokyo") + if err != nil { + t.Skipf("time zone data unavailable: %v", err) + } + prev := time.Local + time.Local = tokyo + t.Cleanup(func() { time.Local = prev }) + + runWithBothDatabases(t, func(t *testing.T, db *DB) { + now := time.Now() + fetched := now.Add(-90 * time.Minute) + seedRetentionArtifact(t, db, "npm", "tz", fetched, time.Time{}) + cutoff := now.Add(-60 * time.Minute) + + candidates, err := db.GetRetentionCandidates(0, 10, cutoff) + if err != nil { + t.Fatalf("GetRetentionCandidates: %v", err) + } + if len(candidates) != 1 { + t.Fatalf("got %d candidates, want 1", len(candidates)) + } + c := candidates[0] + if !c.ExpiredBefore(cutoff) { + t.Errorf("candidate fetched at %v not expired before %v", c.FetchedAt.Time, cutoff) + } + if diff := c.FetchedAt.Time.Sub(fetched).Abs(); diff > time.Second { + t.Errorf("fetched_at read back as %v, want %v", c.FetchedAt.Time, fetched) + } + }) +} diff --git a/internal/database/schema.go b/internal/database/schema.go index eb45789..4aef75c 100644 --- a/internal/database/schema.go +++ b/internal/database/schema.go @@ -82,6 +82,7 @@ CREATE TABLE IF NOT EXISTS pending_deletes ( path TEXT NOT NULL PRIMARY KEY, queued_at DATETIME NOT NULL ); +CREATE INDEX IF NOT EXISTS idx_pending_deletes_queued_at ON pending_deletes(queued_at); CREATE TABLE IF NOT EXISTS vulnerabilities ( id INTEGER PRIMARY KEY, @@ -190,6 +191,7 @@ CREATE TABLE IF NOT EXISTS pending_deletes ( path TEXT NOT NULL PRIMARY KEY, queued_at TIMESTAMP NOT NULL ); +CREATE INDEX IF NOT EXISTS idx_pending_deletes_queued_at ON pending_deletes(queued_at); CREATE TABLE IF NOT EXISTS vulnerabilities ( id SERIAL PRIMARY KEY, @@ -379,6 +381,7 @@ var migrations = []migration{ {"007_add_metadata_link", migrateAddMetadataLink}, {"008_add_metadata_content_encoding", migrateAddMetadataContentEncoding}, {"009_add_pending_deletes", migrateAddPendingDeletes}, + {"010_add_pending_deletes_queued_at_index", migrateAddPendingDeletesQueuedAtIndex}, } // isTableNotFound returns true if the error indicates a missing table. @@ -668,6 +671,16 @@ func migrateAddPendingDeletes(db *DB) error { return nil } +// migrateAddPendingDeletesQueuedAtIndex indexes the column reclaim orders +// the queue by, which grows large after a retention sweep. +func migrateAddPendingDeletesQueuedAtIndex(db *DB) error { + _, err := db.Exec(`CREATE INDEX IF NOT EXISTS idx_pending_deletes_queued_at ON pending_deletes(queued_at)`) + if err != nil { + return fmt.Errorf("creating pending_deletes queued_at index: %w", err) + } + return nil +} + // EnsureMetadataCacheTable creates the metadata_cache table if it doesn't exist. func (db *DB) EnsureMetadataCacheTable() error { has, err := db.HasTable("metadata_cache") diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go index 1958165..23dca2c 100644 --- a/internal/metrics/metrics.go +++ b/internal/metrics/metrics.go @@ -62,6 +62,14 @@ var ( }, ) + ArtifactsEvicted = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "proxy_artifacts_evicted_total", + Help: "Total number of cached artifacts evicted, by reason (lru or retention) and ecosystem", + }, + []string{"reason", "ecosystem"}, + ) + // Upstream metrics UpstreamFetchDuration = prometheus.NewHistogramVec( prometheus.HistogramOpts{ @@ -250,6 +258,7 @@ func init() { CacheMisses, CacheSize, CachedArtifacts, + ArtifactsEvicted, UpstreamFetchDuration, UpstreamErrors, CircuitBreakerState, @@ -309,6 +318,16 @@ func RecordCacheHit(ecosystem string) { CacheHits.WithLabelValues(purl.NormalizeEcosystem(ecosystem)).Inc() } +// RecordArtifactEvicted counts one evicted artifact. ecosystem is the +// package record's value, normalized like the cache hit counter; an artifact +// without a package record counts as "other". +func RecordArtifactEvicted(reason, ecosystem string) { + if ecosystem == "" { + ecosystem = "other" + } + ArtifactsEvicted.WithLabelValues(reason, purl.NormalizeEcosystem(ecosystem)).Inc() +} + // RecordCacheMiss increments cache miss counter. func RecordCacheMiss(ecosystem string) { CacheMisses.WithLabelValues(purl.NormalizeEcosystem(ecosystem)).Inc() diff --git a/internal/retention/retention.go b/internal/retention/retention.go new file mode 100644 index 0000000..c26ef4a --- /dev/null +++ b/internal/retention/retention.go @@ -0,0 +1,209 @@ +// Package retention holds the per-ecosystem rules for evicting cached +// artifacts that have not been accessed for a configured time. +// +// An ecosystem takes part only once its handler registers a Spec with +// Default, usually from an init function in the handler's own file. Until +// then configuring retention for it is a validation error, and the global +// default does not apply to it. +package retention + +import ( + "fmt" + "slices" + "sort" + "sync" + "time" + + "github.com/git-pkgs/purl" +) + +// KnownKeys lists the ecosystem keys storage.retention.ecosystems accepts. +// They are the literals the handlers pass when they fetch an artifact. A key +// outside this list is a typo; a key in it whose handler has not registered +// yet is not supported by this version. +var KnownKeys = []string{ + "alpine", "cargo", "composer", "conan", "conda", "cran", "deb", "gem", + "golang", "helm", "hex", "julia", "maven", "npm", "nuget", "oci", "pub", + "pypi", "rpm", "swift", +} + +// IsKnown reports whether key is one of KnownKeys. +func IsKnown(key string) bool { + return slices.Contains(KnownKeys, key) +} + +// Spec describes how one ecosystem takes part in retention. +type Spec struct { + // Key is the ecosystem's config key, one of KnownKeys. + Key string + + // ExtraDBEcosystems lists packages.ecosystem values besides the ones + // derived from Key that belong to this ecosystem. + ExtraDBEcosystems []string + + // CanonicalPackage turns a package PURL from the config into the + // package PURL the handler stores in the cache, reporting false when the + // PURL does not belong to this ecosystem. Nil means the ecosystem accepts + // no package overrides. + CanonicalPackage func(p *purl.PURL) (string, bool) +} + +// dbEcosystems returns every packages.ecosystem value that belongs to the +// spec: the key, its PURL type and the ecosystem name that type maps back +// to, the same spellings git-pkgs matches. The proxy's mirror command and +// git-pkgs store "rubygems" where the gem handler stores "gem", for example. +func (s Spec) dbEcosystems() []string { + purlType := purl.EcosystemToPURLType(s.Key) + values := []string{s.Key, purlType, purl.PURLTypeToEcosystem(purlType)} + values = append(values, s.ExtraDBEcosystems...) + slices.Sort(values) + return slices.Compact(values) +} + +// DefaultCanonical returns a CanonicalPackage for ecosystems that store the +// package PURL exactly as purl.MakePURLString builds it from the PURL's full +// name. It accepts a PURL only when building it back that way gives the same +// package, so for an ecosystem that stores a different form, such as one +// with a default namespace, it refuses the key instead of producing one +// that never matches. Such ecosystems need their own CanonicalPackage. +func DefaultCanonical(key string) func(p *purl.PURL) (string, bool) { + purlType := purl.EcosystemToPURLType(key) + return func(p *purl.PURL) (string, bool) { + if p == nil || p.Type != purlType { + return "", false + } + canonical := purl.MakePURLString(key, p.FullName(), "") + if canonical == "" { + return "", false + } + given := purl.New(p.Type, p.Namespace, p.Name, "", nil).String() + return canonical, canonical == given + } +} + +// Registry maps ecosystem keys to their specs. +type Registry struct { + mu sync.RWMutex + specs map[string]Spec + byDB map[string]string +} + +// NewRegistry returns an empty registry. +func NewRegistry() *Registry { + return &Registry{specs: map[string]Spec{}, byDB: map[string]string{}} +} + +// Default is the registry the handlers register with. +var Default = NewRegistry() + +// Register adds s. It panics when the key is not one of KnownKeys, is +// already registered, or claims a packages.ecosystem value another spec +// claims, since each of those is a programming error. +func (r *Registry) Register(s Spec) { + r.mu.Lock() + defer r.mu.Unlock() + + if !IsKnown(s.Key) { + panic(fmt.Sprintf("retention: unknown ecosystem key %q", s.Key)) + } + if _, ok := r.specs[s.Key]; ok { + panic(fmt.Sprintf("retention: ecosystem %q registered twice", s.Key)) + } + values := s.dbEcosystems() + for _, v := range values { + if owner, ok := r.byDB[v]; ok { + panic(fmt.Sprintf("retention: ecosystem value %q claimed by both %q and %q", v, owner, s.Key)) + } + } + r.specs[s.Key] = s + for _, v := range values { + r.byDB[v] = s.Key + } +} + +// Lookup returns the spec registered for key. +func (r *Registry) Lookup(key string) (Spec, bool) { + r.mu.RLock() + defer r.mu.RUnlock() + s, ok := r.specs[key] + return s, ok +} + +// Keys returns the registered keys in sorted order. +func (r *Registry) Keys() []string { + r.mu.RLock() + defer r.mu.RUnlock() + keys := make([]string, 0, len(r.specs)) + for k := range r.specs { + keys = append(keys, k) + } + sort.Strings(keys) + return keys +} + +// KeyForDBEcosystem returns the registered key a packages.ecosystem value +// belongs to. +func (r *Registry) KeyForDBEcosystem(ecosystem string) (string, bool) { + r.mu.RLock() + defer r.mu.RUnlock() + key, ok := r.byDB[ecosystem] + return key, ok +} + +// Canonical returns the stored package PURL a configured package PURL +// stands for, asking the registered specs in key order. +func (r *Registry) Canonical(p *purl.PURL) (string, bool) { + for _, key := range r.Keys() { + s, _ := r.Lookup(key) + if s.CanonicalPackage == nil { + continue + } + if canonical, ok := s.CanonicalPackage(p); ok { + return canonical, true + } + } + return "", false +} + +// Rules are resolved retention durations. A duration of zero means an +// artifact is never evicted by age. +type Rules struct { + // Default applies to registered ecosystems without their own rule. + Default time.Duration + // Ecosystems holds rules keyed by registered ecosystem key. + Ecosystems map[string]time.Duration + // Packages holds rules keyed by the package PURL stored in the cache. + Packages map[string]time.Duration +} + +// For returns the retention for an artifact of the given ecosystem key and +// stored package PURL: the package rule, else the ecosystem rule, else the +// default. +func (r Rules) For(key, packagePURL string) time.Duration { + if d, ok := r.Packages[packagePURL]; ok { + return d + } + if d, ok := r.Ecosystems[key]; ok { + return d + } + return r.Default +} + +// MinPositive returns the shortest non-zero duration among all rules, or +// zero when every rule is zero. +func (r Rules) MinPositive() time.Duration { + minimum := time.Duration(0) + consider := func(d time.Duration) { + if d > 0 && (minimum == 0 || d < minimum) { + minimum = d + } + } + consider(r.Default) + for _, d := range r.Ecosystems { + consider(d) + } + for _, d := range r.Packages { + consider(d) + } + return minimum +} diff --git a/internal/retention/retention_test.go b/internal/retention/retention_test.go new file mode 100644 index 0000000..9fa67b7 --- /dev/null +++ b/internal/retention/retention_test.go @@ -0,0 +1,163 @@ +package retention + +import ( + "slices" + "testing" + "time" + + "github.com/git-pkgs/purl" +) + +func TestSpecDBEcosystems(t *testing.T) { + tests := []struct { + key string + want []string + }{ + {"npm", []string{"npm"}}, + {"gem", []string{"gem", "rubygems"}}, + {"composer", []string{"composer", "packagist"}}, + {"alpine", []string{"alpine", "apk"}}, + {"golang", []string{"golang"}}, + {"deb", []string{"deb"}}, + } + for _, tt := range tests { + t.Run(tt.key, func(t *testing.T) { + got := Spec{Key: tt.key}.dbEcosystems() + if !slices.Equal(got, tt.want) { + t.Errorf("dbEcosystems() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestRegistryRegister(t *testing.T) { + reg := NewRegistry() + reg.Register(Spec{Key: "gem"}) + reg.Register(Spec{Key: "npm"}) + + if got := reg.Keys(); !slices.Equal(got, []string{"gem", "npm"}) { + t.Errorf("Keys() = %v, want [gem npm]", got) + } + for _, value := range []string{"gem", "rubygems"} { + if key, ok := reg.KeyForDBEcosystem(value); !ok || key != "gem" { + t.Errorf("KeyForDBEcosystem(%q) = %q, %v, want gem, true", value, key, ok) + } + } + if _, ok := reg.KeyForDBEcosystem("pypi"); ok { + t.Error("KeyForDBEcosystem(pypi) found a key for an unregistered ecosystem") + } + if _, ok := reg.Lookup("pypi"); ok { + t.Error("Lookup(pypi) found an unregistered ecosystem") + } +} + +func TestRegistryRegisterPanics(t *testing.T) { + tests := []struct { + name string + specs []Spec + }{ + {"unknown key", []Spec{{Key: "generic"}}}, + {"duplicate key", []Spec{{Key: "npm"}, {Key: "npm"}}}, + {"overlapping ecosystem value", []Spec{{Key: "gem"}, {Key: "npm", ExtraDBEcosystems: []string{"rubygems"}}}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + reg := NewRegistry() + defer func() { + if recover() == nil { + t.Error("Register did not panic") + } + }() + for _, s := range tt.specs { + reg.Register(s) + } + }) + } +} + +func mustParse(t *testing.T, s string) *purl.PURL { + t.Helper() + p, err := purl.Parse(s) + if err != nil { + t.Fatalf("parsing %q: %v", s, err) + } + return p +} + +func TestDefaultCanonical(t *testing.T) { + canonical := DefaultCanonical("npm") + + got, ok := canonical(mustParse(t, "pkg:npm/%40babel/core")) + if !ok || got != "pkg:npm/%40babel/core" { + t.Errorf("scoped package = %q, %v, want pkg:npm/%%40babel/core, true", got, ok) + } + if got, ok := canonical(mustParse(t, "pkg:pypi/requests")); ok { + t.Errorf("other type accepted as %q", got) + } + if _, ok := canonical(nil); ok { + t.Error("nil PURL accepted") + } + + // alpine stores pkg:apk/alpine/{name} and deb stores pkg:deb/{name}, so + // building their keys back from the full name gives a different package. + // The default refuses them rather than produce keys that never match. + for _, tt := range []struct{ key, purl string }{ + {"alpine", "pkg:apk/alpine/curl"}, + {"alpine", "pkg:apk/curl"}, + {"deb", "pkg:deb/debian/curl"}, + } { + if got, ok := DefaultCanonical(tt.key)(mustParse(t, tt.purl)); ok { + t.Errorf("DefaultCanonical(%q) accepted %s as %q", tt.key, tt.purl, got) + } + } +} + +func TestRegistryCanonical(t *testing.T) { + reg := NewRegistry() + reg.Register(Spec{Key: "julia"}) + reg.Register(Spec{Key: "npm", CanonicalPackage: DefaultCanonical("npm")}) + + if got, ok := reg.Canonical(mustParse(t, "pkg:npm/lodash")); !ok || got != "pkg:npm/lodash" { + t.Errorf("Canonical(npm) = %q, %v, want pkg:npm/lodash, true", got, ok) + } + if got, ok := reg.Canonical(mustParse(t, "pkg:cargo/serde")); ok { + t.Errorf("Canonical(cargo) = %q for an unregistered ecosystem", got) + } +} + +func TestRulesFor(t *testing.T) { + rules := Rules{ + Default: 30 * 24 * time.Hour, + Ecosystems: map[string]time.Duration{"npm": 14 * 24 * time.Hour, "maven": 0}, + Packages: map[string]time.Duration{"pkg:npm/lodash": 0, "pkg:maven/a/b": 90 * 24 * time.Hour}, + } + tests := []struct { + key, pkg string + want time.Duration + }{ + {"npm", "pkg:npm/lodash", 0}, + {"npm", "pkg:npm/left-pad", 14 * 24 * time.Hour}, + {"maven", "pkg:maven/a/b", 90 * 24 * time.Hour}, + {"maven", "pkg:maven/c/d", 0}, + {"cargo", "pkg:cargo/serde", 30 * 24 * time.Hour}, + } + for _, tt := range tests { + if got := rules.For(tt.key, tt.pkg); got != tt.want { + t.Errorf("For(%q, %q) = %v, want %v", tt.key, tt.pkg, got, tt.want) + } + } +} + +func TestRulesMinPositive(t *testing.T) { + if got := (Rules{}).MinPositive(); got != 0 { + t.Errorf("empty rules MinPositive() = %v, want 0", got) + } + rules := Rules{ + Default: 0, + Ecosystems: map[string]time.Duration{"npm": 14 * time.Hour, "maven": 0}, + Packages: map[string]time.Duration{"pkg:npm/lodash": 2 * time.Hour}, + } + if got := rules.MinPositive(); got != 2*time.Hour { + t.Errorf("MinPositive() = %v, want 2h", got) + } +} diff --git a/internal/server/analytics_coverage_test.go b/internal/server/analytics_coverage_test.go index 5517dea..2c4ac63 100644 --- a/internal/server/analytics_coverage_test.go +++ b/internal/server/analytics_coverage_test.go @@ -44,6 +44,7 @@ var metricSurface = map[string]string{ "proxy_health_probe_failures_total": "Runtime: Health probe failures", "proxy_circuit_breaker_state": "Runtime: Circuit breakers", "proxy_circuit_breaker_trips_total": "Runtime: Circuit breaker trips", + "proxy_artifacts_evicted_total": "Runtime: Evicted artifacts", "proxy_scan_duration_seconds": "Runtime: Pre-cache scanning -- Scans", "proxy_scan_blocked_total": "Runtime: Pre-cache scanning -- Artifacts blocked", "proxy_scan_errors_total": "Runtime: Pre-cache scanning -- Scan errors", @@ -70,6 +71,7 @@ func TestEveryMetricIsSurfaced(t *testing.T) { metrics.RecordScanResult("npm", "clamav", false, 0) metrics.RecordScanError("npm", "clamav", "timeout") metrics.UpdateCacheStats(1, 1) + metrics.RecordArtifactEvicted("retention", "npm") metrics.UpdateEcosystemStats([]metrics.EcosystemStats{{Ecosystem: "npm", DownloadedBytes: 1}}) snap, err := metrics.Gather() diff --git a/internal/server/analytics_runtime.go b/internal/server/analytics_runtime.go index 2b77730..dad3d19 100644 --- a/internal/server/analytics_runtime.go +++ b/internal/server/analytics_runtime.go @@ -32,6 +32,7 @@ type RuntimeView struct { StorageErrors []LabelledCount IntegrityFailures []LabelledCount ProbeFailures []LabelledCount + Evictions []LabelledCount Breakers []BreakerRow BreakerTrips []LabelledCount @@ -112,6 +113,7 @@ func runtimeView(snap *metrics.Snapshot) RuntimeView { v.StorageErrors = failureRows(snap, "proxy_storage_errors_total", "operation") v.IntegrityFailures = failureRows(snap, "proxy_integrity_failures_total", "ecosystem") v.ProbeFailures = failureRows(snap, "proxy_health_probe_failures_total", "step") + v.Evictions = countRows(snap, "proxy_artifacts_evicted_total", "reason", "ecosystem") v.Breakers = breakerRows(snap) v.BreakerTrips = failureRows(snap, "proxy_circuit_breaker_trips_total", "registry") @@ -202,6 +204,15 @@ func statusClasses(snap *metrics.Snapshot) []LabelledCount { // that has never fired carries no information, and a wall of zeroes buries the // rows that matter. func failureRows(snap *metrics.Snapshot, name string, labels ...string) []LabelledCount { + rows := countRows(snap, name, labels...) + for i := range rows { + rows[i].Bad = true + } + return rows +} + +// countRows renders every non-zero series of a counter as a row. +func countRows(snap *metrics.Snapshot, name string, labels ...string) []LabelledCount { var rows []LabelledCount for _, s := range snap.Samples(name) { if s.Value == 0 { @@ -210,7 +221,6 @@ func failureRows(snap *metrics.Snapshot, name string, labels ...string) []Labell rows = append(rows, LabelledCount{ Label: joinLabels(s, labels...), Count: formatCount(int64(s.Value)), - Bad: true, }) } return rows diff --git a/internal/server/analytics_runtime_test.go b/internal/server/analytics_runtime_test.go index 6cab558..422c02d 100644 --- a/internal/server/analytics_runtime_test.go +++ b/internal/server/analytics_runtime_test.go @@ -128,6 +128,30 @@ func TestRuntimeViewShapesRegistry(t *testing.T) { } } +// Evictions are routine housekeeping, not failures: they reach the page by +// reason and ecosystem without the failure tint. +func TestRuntimeViewShowsEvictions(t *testing.T) { + reg := prometheus.NewRegistry() + evicted := prometheus.NewCounterVec( + prometheus.CounterOpts{Name: "proxy_artifacts_evicted_total", Help: "t"}, []string{"reason", "ecosystem"}) + reg.MustRegister(evicted) + evicted.WithLabelValues("retention", "npm").Add(4) + evicted.WithLabelValues("lru", "cargo") + + snap, err := metrics.GatherFrom(reg) + if err != nil { + t.Fatalf("GatherFrom: %v", err) + } + v := runtimeView(snap) + + if len(v.Evictions) != 1 { + t.Fatalf("Evictions = %+v, want only the non-zero row", v.Evictions) + } + if got := v.Evictions[0]; got.Label != "retention · npm" || got.Count != "4" || got.Bad { + t.Errorf("eviction row = %+v, want retention · npm = 4 without the failure tint", got) + } +} + // A counter sitting at zero carries no information and would bury the rows // that do, so it is left out. func TestRuntimeViewDropsZeroCounters(t *testing.T) { diff --git a/internal/server/eviction.go b/internal/server/eviction.go index f9a706a..7fe108a 100644 --- a/internal/server/eviction.go +++ b/internal/server/eviction.go @@ -141,6 +141,7 @@ func evictBatch(ctx context.Context, db *database.DB, store storage.Storage, log freed += art.Size.Int64 } cleared++ + recordLRUEviction(db, logger, art.VersionPURL) } return cleared, freed diff --git a/internal/server/reclaim.go b/internal/server/reclaim.go index 72a4e8f..aa986a1 100644 --- a/internal/server/reclaim.go +++ b/internal/server/reclaim.go @@ -17,6 +17,10 @@ const ( reclaimInterval = 1 * time.Minute reclaimBatch = 100 reclaimMinGrace = 1 * time.Hour + // reclaimTickBudget bounds how long one tick keeps deleting batches, so + // a long queue, such as one left by a retention sweep, drains faster than + // one batch per interval without one tick running on indefinitely. + reclaimTickBudget = 30 * time.Second ) func (s *Server) startReclaimLoop(ctx context.Context) { @@ -30,29 +34,46 @@ func (s *Server) startReclaimLoop(ctx context.Context) { case <-ctx.Done(): return case <-ticker.C: - reclaimStorage(ctx, s.db, s.storage, s.logger, time.Now().Add(-grace)) + reclaimDue(ctx, s.db, s.storage, s.logger, time.Now().Add(-grace), reclaimTickBudget) } } } -// reclaimStorage deletes up to one batch of objects queued before cutoff. A -// delete that fails is queued again, behind the rest, so objects the backend -// keeps refusing cannot fill every batch. -func reclaimStorage(ctx context.Context, db *database.DB, store storage.Storage, logger *slog.Logger, cutoff time.Time) { +// reclaimDue deletes batches of objects queued before cutoff until a batch +// comes back short, a delete fails or budget has passed. Stopping on a +// failed delete keeps a storage outage to one batch of attempts per tick. +func reclaimDue(ctx context.Context, db *database.DB, store storage.Storage, logger *slog.Logger, + cutoff time.Time, budget time.Duration) { + start := time.Now() + for ctx.Err() == nil && time.Since(start) < budget { + listed, failed := reclaimStorage(ctx, db, store, logger, cutoff) + if listed < reclaimBatch || failed > 0 { + return + } + } +} + +// reclaimStorage deletes up to one batch of objects queued before cutoff and +// reports how many it listed and how many deletes failed. A delete that +// fails is queued again, behind the rest, so objects the backend keeps +// refusing cannot fill every batch. +func reclaimStorage(ctx context.Context, db *database.DB, store storage.Storage, logger *slog.Logger, + cutoff time.Time) (listed, failed int) { paths, err := db.GetDuePendingDeletes(cutoff, reclaimBatch) if err != nil { logger.Warn("reclaim: failed to list pending deletes", "error", err) - return + return 0, 0 } for _, path := range paths { if ctx.Err() != nil { - return + return len(paths), failed } if err := store.Delete(ctx, path); err != nil { if ctx.Err() != nil { - return + return len(paths), failed } + failed++ logger.Warn("reclaim: failed to delete object, will retry", "path", path, "error", err) if err := db.QueuePendingDelete(path); err != nil { logger.Warn("reclaim: failed to requeue object", "path", path, "error", err) @@ -63,4 +84,5 @@ func reclaimStorage(ctx context.Context, db *database.DB, store storage.Storage, logger.Warn("reclaim: failed to dequeue deleted object", "path", path, "error", err) } } + return len(paths), failed } diff --git a/internal/server/retention.go b/internal/server/retention.go new file mode 100644 index 0000000..660716d --- /dev/null +++ b/internal/server/retention.go @@ -0,0 +1,173 @@ +package server + +import ( + "context" + "log/slog" + "slices" + "time" + + "github.com/git-pkgs/proxy/internal/database" + "github.com/git-pkgs/proxy/internal/metrics" + "github.com/git-pkgs/proxy/internal/retention" +) + +// retentionWindow is how many artifact ids one query scans. +const retentionWindow = 5000 + +// retentionMaxClears caps how many artifacts one sweep clears, so a first +// sweep over a large, old cache spreads its deletes and the refetches that +// follow over several intervals. A variable so tests can lower it. +var retentionMaxClears = 10000 + +func (s *Server) startRetentionLoop(ctx context.Context) { + reg := retention.Default + rules, err := s.cfg.RetentionRules(reg) + if err != nil { + // Validate already rejected an invalid config at startup. + s.logger.Error("retention: invalid configuration, sweep disabled", "error", err) + return + } + if rules.MinPositive() == 0 { + return + } + + covered := reg.Keys() + if len(covered) == 0 { + // Only a default can be set without any registered ecosystem; + // validation rejects every other rule. + s.logger.Info("retention: no ecosystem supports retention in this version, sweep disabled", + "default_not_applied", notCovered(covered)) + return + } + s.logger.Info("retention enabled", + "covered", covered, + "default_not_applied", notCovered(covered), + "sweep_interval", s.cfg.ParseRetentionSweepInterval()) + + ticker := time.NewTicker(s.cfg.ParseRetentionSweepInterval()) + defer ticker.Stop() + + sweepRetention(ctx, s.db, s.logger, reg, rules, time.Now()) + + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + sweepRetention(ctx, s.db, s.logger, reg, rules, time.Now()) + } + } +} + +// notCovered returns the known ecosystem keys that are not in covered. +func notCovered(covered []string) []string { + var missing []string + for _, key := range retention.KnownKeys { + if !slices.Contains(covered, key) { + missing = append(missing, key) + } + } + return missing +} + +// sweepRetention clears cached artifacts that have outlived their retention +// as of now and queues their objects for deletion. Objects are not deleted +// directly: a hit still buffered when the scan ran can make an artifact that +// is being served look expired, and the queue's grace period covers that. +func sweepRetention(ctx context.Context, db *database.DB, logger *slog.Logger, + reg *retention.Registry, rules retention.Rules, now time.Time) { + minimum := rules.MinPositive() + if minimum == 0 { + return + } + + // Without the buffered hits written, an artifact being served right now + // can look expired, so skip this sweep rather than clear it. + if err := db.FlushHits(); err != nil { + logger.Warn("retention: failed to write buffered hits, skipping this sweep", "error", err) + return + } + + maxID, err := db.MaxArtifactID() + if err != nil { + logger.Warn("retention: failed to read artifact ids", "error", err) + return + } + + // Rows that are not expired under the shortest rule cannot be expired + // under any, so the query uses that cutoff and each row is then checked + // against its own rule. + scanCutoff := now.Add(-minimum) + cleared := 0 + freed := int64(0) + capped := false + + for from := int64(0); from < maxID && !capped; from += retentionWindow { + if ctx.Err() != nil { + return + } + candidates, err := db.GetRetentionCandidates(from, from+retentionWindow, scanCutoff) + if err != nil { + logger.Warn("retention: failed to scan artifacts", "from_id", from, "error", err) + return + } + for _, c := range candidates { + if ctx.Err() != nil { + return + } + if cleared >= retentionMaxClears { + capped = true + break + } + if !clearIfExpired(db, logger, reg, rules, now, c) { + continue + } + metrics.RecordArtifactEvicted("retention", c.Ecosystem) + cleared++ + if c.Size.Valid { + freed += c.Size.Int64 + } + } + } + + if cleared > 0 { + logger.Info("retention: sweep completed", + "cleared", cleared, "freed_bytes", freed, "capped", capped) + } +} + +// clearIfExpired clears one candidate when its own rule says it has expired +// and queues its object for deletion, reporting whether it did. +func clearIfExpired(db *database.DB, logger *slog.Logger, reg *retention.Registry, + rules retention.Rules, now time.Time, c database.RetentionCandidate) bool { + key, ok := reg.KeyForDBEcosystem(c.Ecosystem) + if !ok { + return false + } + d := rules.For(key, c.PackagePURL) + if d <= 0 { + return false + } + cutoff := now.Add(-d) + if !c.ExpiredBefore(cutoff) { + return false + } + + cleared, err := db.ClearExpiredArtifact(c.VersionPURL, c.Filename, c.StoragePath, cutoff) + if err != nil { + logger.Warn("retention: failed to clear artifact record", + "version_purl", c.VersionPURL, "filename", c.Filename, "error", err) + return false + } + return cleared +} + +// recordLRUEviction counts an artifact the size limit evicted under the +// ecosystem of its package record. +func recordLRUEviction(db *database.DB, logger *slog.Logger, versionPURL string) { + ecosystem, err := db.GetArtifactEcosystem(versionPURL) + if err != nil { + logger.Debug("eviction: failed to look up ecosystem for metrics", "version_purl", versionPURL, "error", err) + } + metrics.RecordArtifactEvicted("lru", ecosystem) +} diff --git a/internal/server/retention_test.go b/internal/server/retention_test.go new file mode 100644 index 0000000..4ffd399 --- /dev/null +++ b/internal/server/retention_test.go @@ -0,0 +1,423 @@ +package server + +import ( + "context" + "database/sql" + "fmt" + "slices" + "strings" + "testing" + "time" + + "github.com/git-pkgs/proxy/internal/database" + "github.com/git-pkgs/proxy/internal/metrics" + "github.com/git-pkgs/proxy/internal/retention" + "github.com/git-pkgs/proxy/internal/storage" + "github.com/prometheus/client_golang/prometheus/testutil" +) + +const retentionDay = 24 * time.Hour + +// retentionRegistry registers npm and gem the way their handlers would, in a +// registry of the test's own so the handlers' registrations stay untouched. +func retentionRegistry() *retention.Registry { + reg := retention.NewRegistry() + reg.Register(retention.Spec{Key: "npm", CanonicalPackage: retention.DefaultCanonical("npm")}) + reg.Register(retention.Spec{Key: "gem", CanonicalPackage: retention.DefaultCanonical("gem")}) + return reg +} + +// seedAgedArtifact stores a cached artifact of pkg:{ecosystem}/{name}, with +// the package record's ecosystem set to dbEcosystem, fetched and last +// accessed at the given times. A zero accessed leaves it never served. +func seedAgedArtifact(t *testing.T, db *database.DB, store storage.Storage, dbEcosystem, purlType, name string, + fetched, accessed time.Time) string { + t.Helper() + pkgPURL := "pkg:" + purlType + "/" + name + versionPURL := pkgPURL + "@1.0.0" + filename := name + "-1.0.0.tgz" + if err := db.UpsertPackage(&database.Package{PURL: pkgPURL, Ecosystem: dbEcosystem, Name: name}); err != nil { + t.Fatalf("UpsertPackage: %v", err) + } + if err := db.UpsertVersion(&database.Version{PURL: versionPURL, PackagePURL: pkgPURL}); err != nil { + t.Fatalf("UpsertVersion: %v", err) + } + path := storage.ArtifactPath(dbEcosystem, "", name, "1.0.0", filename) + size, hash, err := store.Store(context.Background(), path, strings.NewReader("artifact")) + if err != nil { + t.Fatalf("Store: %v", err) + } + art := &database.Artifact{ + VersionPURL: versionPURL, + Filename: filename, + UpstreamURL: "https://example.com/" + filename, + StoragePath: sql.NullString{String: path, Valid: true}, + ContentHash: sql.NullString{String: hash, Valid: true}, + Size: sql.NullInt64{Int64: size, Valid: true}, + FetchedAt: sql.NullTime{Time: fetched, Valid: true}, + } + if !accessed.IsZero() { + art.LastAccessedAt = sql.NullTime{Time: accessed, Valid: true} + } + if err := db.UpsertArtifact(art); err != nil { + t.Fatalf("UpsertArtifact: %v", err) + } + return path +} + +func isCached(t *testing.T, db *database.DB, purlType, name string) bool { + t.Helper() + art, err := db.GetArtifact("pkg:"+purlType+"/"+name+"@1.0.0", name+"-1.0.0.tgz") + if err != nil { + t.Fatalf("GetArtifact: %v", err) + } + return art.StoragePath.Valid +} + +func TestSweepRetentionAppliesEachRule(t *testing.T) { + db, store := setupEvictionTest(t) + now := time.Now() + ago := func(days int) time.Time { return now.Add(-time.Duration(days) * retentionDay) } + + rules := retention.Rules{ + Default: 30 * retentionDay, + Ecosystems: map[string]time.Duration{"npm": 14 * retentionDay}, + Packages: map[string]time.Duration{ + "pkg:npm/pinned": 0, + "pkg:npm/long-lived": 90 * retentionDay, + "pkg:npm/long-lived-expired": 90 * retentionDay, + "pkg:gem/short": 2 * retentionDay, + }, + } + + cases := []struct { + dbEco, purlType, name string + fetched, accessed time.Time + wantCached bool + }{ + {"npm", "npm", "stale", ago(60), ago(20), false}, + {"npm", "npm", "fresh", ago(60), ago(5), true}, + {"npm", "npm", "never-served", ago(20), time.Time{}, false}, + {"npm", "npm", "pinned", ago(400), ago(400), true}, + {"npm", "npm", "long-lived", ago(60), ago(60), true}, + {"npm", "npm", "long-lived-expired", ago(100), ago(100), false}, + {"gem", "gem", "default-applies", ago(40), ago(40), false}, + {"gem", "gem", "within-default", ago(40), ago(20), true}, + {"rubygems", "gem", "mirrored", ago(40), time.Time{}, false}, + {"gem", "gem", "short", ago(3), ago(3), false}, + {"pypi", "pypi", "unregistered", ago(400), ago(400), true}, + } + for _, c := range cases { + seedAgedArtifact(t, db, store, c.dbEco, c.purlType, c.name, c.fetched, c.accessed) + } + sweepRetention(context.Background(), db, discardLogger(), retentionRegistry(), rules, now) + + for _, c := range cases { + if got := isCached(t, db, c.purlType, c.name); got != c.wantCached { + t.Errorf("%s cached = %v, want %v", c.name, got, c.wantCached) + } + } +} + +func TestSweepRetentionQueuesInsteadOfDeleting(t *testing.T) { + db, store := setupEvictionTest(t) + now := time.Now() + old := now.Add(-30 * retentionDay) + path := seedAgedArtifact(t, db, store, "npm", "npm", "stale", old, old) + + rules := retention.Rules{Ecosystems: map[string]time.Duration{"npm": retentionDay}} + sweepRetention(context.Background(), db, discardLogger(), retentionRegistry(), rules, now) + + if isCached(t, db, "npm", "stale") { + t.Fatal("stale artifact still cached") + } + if !objectExists(t, store, path) { + t.Error("object deleted directly instead of waiting in the queue") + } + if got := queuedPaths(t, db); !slices.Equal(got, []string{path}) { + t.Errorf("queue = %v, want [%s]", got, path) + } +} + +// An artifact fetched again after an eviction keeps the access time of its +// earlier copy and must not be cleared again by the next sweep. +func TestSweepRetentionKeepsRefetchedArtifact(t *testing.T) { + db, store := setupEvictionTest(t) + now := time.Now() + old := now.Add(-30 * retentionDay) + seedAgedArtifact(t, db, store, "npm", "npm", "popular", old, old) + rules := retention.Rules{Ecosystems: map[string]time.Duration{"npm": retentionDay}} + + sweepRetention(context.Background(), db, discardLogger(), retentionRegistry(), rules, now) + if isCached(t, db, "npm", "popular") { + t.Fatal("first sweep kept the expired artifact") + } + + // Refetch: the upsert sets a new fetch time but leaves last_accessed_at. + seedAgedArtifact(t, db, store, "npm", "npm", "popular", now, time.Time{}) + art, err := db.GetArtifact("pkg:npm/popular@1.0.0", "popular-1.0.0.tgz") + if err != nil { + t.Fatalf("GetArtifact: %v", err) + } + if !art.LastAccessedAt.Valid || art.LastAccessedAt.Time.After(old.Add(time.Minute)) { + t.Fatalf("test setup: last_accessed_at = %v, want the old access kept", art.LastAccessedAt) + } + + sweepRetention(context.Background(), db, discardLogger(), retentionRegistry(), rules, now.Add(time.Hour)) + if !isCached(t, db, "npm", "popular") { + t.Error("refetched artifact cleared again by the next sweep") + } +} + +func TestSweepRetentionFlushesBufferedHits(t *testing.T) { + db, store := setupEvictionTest(t) + now := time.Now() + old := now.Add(-30 * retentionDay) + seedAgedArtifact(t, db, store, "npm", "npm", "busy", old, old) + db.BatchHits(time.Hour, nil) + if err := db.RecordArtifactHit("pkg:npm/busy@1.0.0", "busy-1.0.0.tgz"); err != nil { + t.Fatalf("RecordArtifactHit: %v", err) + } + + rules := retention.Rules{Ecosystems: map[string]time.Duration{"npm": retentionDay}} + sweepRetention(context.Background(), db, discardLogger(), retentionRegistry(), rules, time.Now()) + + if !isCached(t, db, "npm", "busy") { + t.Error("artifact with a buffered hit was cleared") + } +} + +func TestSweepRetentionCapsClearsPerSweep(t *testing.T) { + db, store := setupEvictionTest(t) + prev := retentionMaxClears + retentionMaxClears = 2 + t.Cleanup(func() { retentionMaxClears = prev }) + + now := time.Now() + old := now.Add(-30 * retentionDay) + names := []string{"a", "b", "c"} + for _, name := range names { + seedAgedArtifact(t, db, store, "npm", "npm", name, old, old) + } + rules := retention.Rules{Ecosystems: map[string]time.Duration{"npm": retentionDay}} + + countCached := func() int { + n := 0 + for _, name := range names { + if isCached(t, db, "npm", name) { + n++ + } + } + return n + } + + sweepRetention(context.Background(), db, discardLogger(), retentionRegistry(), rules, now) + if got := countCached(); got != 1 { + t.Fatalf("cached after the first sweep = %d, want 1 (cap of 2)", got) + } + sweepRetention(context.Background(), db, discardLogger(), retentionRegistry(), rules, now) + if got := countCached(); got != 0 { + t.Errorf("cached after the second sweep = %d, want 0", got) + } +} + +func TestSweepRetentionStopsOnCancel(t *testing.T) { + db, store := setupEvictionTest(t) + now := time.Now() + old := now.Add(-30 * retentionDay) + seedAgedArtifact(t, db, store, "npm", "npm", "stale", old, old) + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + rules := retention.Rules{Ecosystems: map[string]time.Duration{"npm": retentionDay}} + sweepRetention(ctx, db, discardLogger(), retentionRegistry(), rules, now) + + if !isCached(t, db, "npm", "stale") { + t.Error("cancelled sweep cleared an artifact") + } +} + +func TestSweepRetentionWithoutRulesDoesNothing(t *testing.T) { + db, store := setupEvictionTest(t) + old := time.Now().Add(-400 * retentionDay) + seedAgedArtifact(t, db, store, "npm", "npm", "ancient", old, old) + + sweepRetention(context.Background(), db, discardLogger(), retentionRegistry(), retention.Rules{}, time.Now()) + + if !isCached(t, db, "npm", "ancient") { + t.Error("sweep without rules cleared an artifact") + } +} + +func TestNotCovered(t *testing.T) { + got := notCovered([]string{"npm", "pypi"}) + if slices.Contains(got, "npm") || slices.Contains(got, "pypi") { + t.Errorf("notCovered = %v, includes a covered key", got) + } + if len(got) != len(retention.KnownKeys)-2 { + t.Errorf("notCovered returned %d keys, want %d", len(got), len(retention.KnownKeys)-2) + } +} + +func TestReclaimDueDrainsSeveralBatches(t *testing.T) { + db, store := setupEvictionTest(t) + const queued = reclaimBatch*2 + 10 + for i := range queued { + storeQueued(t, db, store, fmt.Sprintf("npm/old/1.0.0/%d/old.tgz", i)) + } + + reclaimDue(context.Background(), db, store, discardLogger(), time.Now().Add(time.Hour), time.Minute) + + remaining, err := db.GetDuePendingDeletes(time.Now().Add(time.Hour), queued) + if err != nil { + t.Fatalf("GetDuePendingDeletes: %v", err) + } + if len(remaining) != 0 { + t.Errorf("%d objects still queued, want the whole queue reclaimed in one tick", len(remaining)) + } +} + +func TestReclaimDueRespectsBudget(t *testing.T) { + db, store := setupEvictionTest(t) + storeQueued(t, db, store, "npm/old/1.0.0/a/old.tgz") + + reclaimDue(context.Background(), db, store, discardLogger(), time.Now().Add(time.Hour), 0) + + if got := queuedPaths(t, db); len(got) != 1 { + t.Errorf("queue = %v, want the entry kept with no budget", got) + } +} + +// A hit that lands after the scan but before the clear keeps the artifact: +// the clear re-checks the artifact against its own rule's cutoff. +func TestClearIfExpiredRechecksAgainstItsRule(t *testing.T) { + db, store := setupEvictionTest(t) + now := time.Now() + old := now.Add(-30 * retentionDay) + seedAgedArtifact(t, db, store, "npm", "npm", "racing", old, old) + rules := retention.Rules{Ecosystems: map[string]time.Duration{"npm": retentionDay}} + + candidates, err := db.GetRetentionCandidates(0, 10, now.Add(-retentionDay)) + if err != nil || len(candidates) != 1 { + t.Fatalf("GetRetentionCandidates = %v, %v, want one candidate", candidates, err) + } + if err := db.RecordArtifactHit("pkg:npm/racing@1.0.0", "racing-1.0.0.tgz"); err != nil { + t.Fatalf("RecordArtifactHit: %v", err) + } + + if clearIfExpired(db, discardLogger(), retentionRegistry(), rules, now.Add(time.Minute), candidates[0]) { + t.Error("cleared an artifact served after the scan") + } + if !isCached(t, db, "npm", "racing") { + t.Error("artifact served after the scan is no longer cached") + } +} + +// When buffered hits cannot be written, the sweep must not run on the stale +// access times they would have replaced. +func TestSweepRetentionSkipsWhenHitsCannotBeWritten(t *testing.T) { + db, store := setupEvictionTest(t) + now := time.Now() + old := now.Add(-30 * retentionDay) + seedAgedArtifact(t, db, store, "npm", "npm", "busy", old, old) + db.BatchHits(time.Hour, nil) + if err := db.RecordArtifactHit("pkg:npm/busy@1.0.0", "busy-1.0.0.tgz"); err != nil { + t.Fatalf("RecordArtifactHit: %v", err) + } + // Refuse hit writes only; clearing a record leaves hit_count alone. + if _, err := db.Exec(`CREATE TRIGGER refuse_hits BEFORE UPDATE ON artifacts + WHEN NEW.hit_count > OLD.hit_count BEGIN SELECT RAISE(ABORT, 'hits refused'); END`); err != nil { + t.Fatalf("creating trigger: %v", err) + } + + rules := retention.Rules{Ecosystems: map[string]time.Duration{"npm": retentionDay}} + sweepRetention(context.Background(), db, discardLogger(), retentionRegistry(), rules, now) + + if !isCached(t, db, "npm", "busy") { + t.Error("sweep cleared an artifact whose buffered hit could not be written") + } + if _, err := db.Exec(`DROP TRIGGER refuse_hits`); err != nil { + t.Fatalf("dropping trigger: %v", err) + } +} + +// When the object can't be queued for deletion, the sweep leaves the record +// cached and counts nothing; the next sweep clears it. +func TestSweepRetentionRetriesWhenQueueingFails(t *testing.T) { + db, store := setupEvictionTest(t) + now := time.Now() + old := now.Add(-30 * retentionDay) + path := seedAgedArtifact(t, db, store, "npm", "npm", "stuck", old, old) + if _, err := db.Exec(`CREATE TRIGGER refuse_queue BEFORE INSERT ON pending_deletes + BEGIN SELECT RAISE(ABORT, 'queue refused'); END`); err != nil { + t.Fatalf("creating trigger: %v", err) + } + evicted := func() float64 { + return testutil.ToFloat64(metrics.ArtifactsEvicted.WithLabelValues("retention", "npm")) + } + before := evicted() + + rules := retention.Rules{Ecosystems: map[string]time.Duration{"npm": retentionDay}} + sweepRetention(context.Background(), db, discardLogger(), retentionRegistry(), rules, now) + if !isCached(t, db, "npm", "stuck") { + t.Fatal("sweep cleared an artifact whose object couldn't be queued for deletion") + } + if got := evicted() - before; got != 0 { + t.Errorf("evictions counted on a failed queue = %v, want 0", got) + } + + if _, err := db.Exec(`DROP TRIGGER refuse_queue`); err != nil { + t.Fatalf("dropping trigger: %v", err) + } + sweepRetention(context.Background(), db, discardLogger(), retentionRegistry(), rules, now) + if isCached(t, db, "npm", "stuck") { + t.Error("next sweep left the artifact cached") + } + var queued []string + if err := db.Select(&queued, `SELECT path FROM pending_deletes`); err != nil { + t.Fatalf("reading pending_deletes: %v", err) + } + if len(queued) != 1 || queued[0] != path { + t.Errorf("queued paths = %v, want [%s]", queued, path) + } + if got := evicted() - before; got != 1 { + t.Errorf("evictions counted after the retry = %v, want 1", got) + } +} + +func TestEvictionsAreCounted(t *testing.T) { + db, store := setupEvictionTest(t) + ctx := context.Background() + now := time.Now() + old := now.Add(-30 * retentionDay) + + retained := testutil.ToFloat64(metrics.ArtifactsEvicted.WithLabelValues("retention", "rubygems")) + seedAgedArtifact(t, db, store, "gem", "gem", "expired", old, old) + rules := retention.Rules{Ecosystems: map[string]time.Duration{"gem": retentionDay}} + sweepRetention(ctx, db, discardLogger(), retentionRegistry(), rules, now) + if got := testutil.ToFloat64(metrics.ArtifactsEvicted.WithLabelValues("retention", "rubygems")) - retained; got != 1 { + t.Errorf("retention evictions counted = %v, want 1", got) + } + + lru := testutil.ToFloat64(metrics.ArtifactsEvicted.WithLabelValues("lru", "npm")) + seedArtifact(t, ctx, db, store, "big", 500, now) + evictLRU(ctx, db, store, discardLogger(), 100) + if got := testutil.ToFloat64(metrics.ArtifactsEvicted.WithLabelValues("lru", "npm")) - lru; got != 1 { + t.Errorf("lru evictions counted = %v, want 1", got) + } +} + +func TestReclaimDueStopsAfterAFailedDelete(t *testing.T) { + db, store := setupEvictionTest(t) + queued := reclaimBatch * 2 + for i := range queued { + storeQueued(t, db, store, fmt.Sprintf("npm/old/1.0.0/%d/old.tgz", i)) + } + + undeletable := &undeletableStorage{Storage: store} + reclaimDue(context.Background(), db, undeletable, discardLogger(), time.Now().Add(time.Hour), time.Minute) + + if got := undeletable.deletes.Load(); got != reclaimBatch { + t.Errorf("delete attempts = %d, want one batch of %d during an outage", got, reclaimBatch) + } +} diff --git a/internal/server/server.go b/internal/server/server.go index 68fdc2f..78377f9 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -374,6 +374,7 @@ func (s *Server) serve(listener net.Listener) error { "database", s.cfg.Database.String()) go s.updateCacheStatsMetrics() go s.startEvictionLoop(bgCtx) + go s.startRetentionLoop(bgCtx) go s.startReclaimLoop(bgCtx) if listener != nil { diff --git a/internal/server/templates/components/runtime_metrics.html b/internal/server/templates/components/runtime_metrics.html index 03fc9a0..ff0f67c 100644 --- a/internal/server/templates/components/runtime_metrics.html +++ b/internal/server/templates/components/runtime_metrics.html @@ -60,6 +60,7 @@ {{template "runtime_counts" dict "Title" "Storage errors" "Rows" .StorageErrors "Empty" "None"}} {{template "runtime_counts" dict "Title" "Integrity failures" "Rows" .IntegrityFailures "Empty" "None"}} {{template "runtime_counts" dict "Title" "Health probe failures" "Rows" .ProbeFailures "Empty" "None"}} + {{template "runtime_counts" dict "Title" "Evicted artifacts" "Rows" .Evictions "Empty" "None"}} {{template "runtime_counts" dict "Title" "Circuit breaker trips" "Rows" .BreakerTrips "Empty" "None"}}
From 0c495fd90620c77bd871afa1aebb94a30782823e Mon Sep 17 00:00:00 2001 From: Christian Heim Date: Mon, 12 Oct 2026 10:08:01 +0200 Subject: [PATCH 202/633] Proxy the OCI referrers API in the container handler (#427) * Proxy the OCI referrers API in the container handler GET /v2/{name}/referrers/{digest} used to fall through to a plain 404, so clients only ever saw referrers stored under the sha256- tag schema. Registries that keep SBOMs and signatures behind the referrers API looked empty through the proxy. The index is now fetched from the upstream that the repository routes to and cached like the tag list: fresh within the metadata TTL, revalidated with If-None-Match, and served stale when the upstream is down. artifactType is not forwarded, so one cached row covers every filter and clients filter the full list themselves, which they do when OCI-Filters-Applied is absent. An upstream 404 is passed through untouched, so clients still fall back to the tag schema for registries like GHCR. When the upstream cannot answer and nothing is cached, the proxy keeps answering 404 as before rather than failing the request. Closes #144 * Cover absolute upstream links in the referrers link test The second page now answers with an absolute Link pointing back at the upstream host, and the test checks that it comes back rewritten to the proxy. That was missing so far, and it also gives the upstream variable a reason to be declared before the handler closure, which staticcheck complained about. * Read cached referrers through a small shared loader loadContainerReferrers was a line-for-line copy of loadContainerTags with the types renamed, which dupl flags in CI. The part that opens the cache row and reads the body now lives in loadContainerMetadata next to storeContainerMetadata, and the referrers loader only maps the row onto its struct. The tag and manifest loaders are left as they are. * Make the manifest routing test notice which handler answered For foo/manifests/referrers/ the manifest handler and the referrers handler send the very same upstream path, so the test passed even with the referrers case moved in front of the manifest case. The fake upstream now rejects requests that only accept an image index, which is what the referrers handler sends. * Check that a cached referrers row links back to whoever reads it A repository can be reached as upstream// and, when the default registry is the same host, as plain . Both end up in the same cache row, so the pagination Link has to be rewritten when the row is served, not when it is stored. The link test now reads the row through both paths; with a store-time rewrite the second path would get the first one's link. * Make the referrers ETag test prove the revalidation Without If-None-Match the fake upstream just sends the full index again, so the old assertions held either way. The test now counts the 304s the upstream sent and expects exactly one. * Reword which tools use the referrers API in the README The cosign mention was too broad. cosign reaches the referrers API for its newer signature bundles, while the old .sig tags keep going through the normal manifest path. Also rewraps the paragraph, one line had run past the others. * Check that a 304 restarts the referrers cache TTL The ETag test now backdates the cached row instead of running with a zero TTL. After the upstream answers 304, a third request has to come from the cache. Before, the test would also have passed if the 304 branch forgot to bump fetchedAt and every request kept revalidating. * Test the referrers Content-Type fallback None of the fake upstreams left out Content-Type, so the default to the image index type was never exercised. oras-go rejects anything else, so it is worth pinning, both for the first fetch and for the cached copy. * Test referrers digests with extra characters around them The invalid-request table only had digests that were too short or used an unknown algorithm, so it would not notice if the digest pattern lost its anchors. Two cases now put junk before and after an otherwise valid sha256 digest. * Reuse the manifest cache key for referrers The referrers handler had its own key function, but it did exactly what containerManifestCacheKey already does with four strings. Referrers rows sit in their own oci-referrers ecosystem anyway, so they can't collide with manifests. * Drop the separate struct for cached referrers A cached referrers response carries the same fields as a cached tag list, and the freshness check was a straight copy too. So the referrers code just uses cachedContainerTags and containerTagsFresh now. Loading, storing and writing stay on their own, since the ecosystem and the Link handling are different. * Route referrers requests by ns like the other OCI endpoints Now that containerd ns support is on main, referrers requests have to go through registryForRequest too, otherwise a _default mirror would look every subject up on Docker Hub. ns only picks the registry, so it is dropped from the forwarded query and the cache key, and the next-page Link keeps it, the same way tag lists do. This also follows the new rewriteContainerTagsLink signature, which takes the namespace. * Without a cached index, referrers now pass upstream errors through and answer 502 when the upstream can't be reached or sends garbage, so clients no longer read an outage as a registry without the referrers API. Stale cache is still served when there is one. * The tag list and manifest routes now only match paths that end in their endpoint, so a repository with tags/list or manifests in its name can still reach the referrers endpoint. --- README.md | 11 + internal/handler/container.go | 9 +- internal/handler/container_metadata.go | 22 + internal/handler/container_referrers.go | 219 ++++++++ internal/handler/container_referrers_test.go | 502 +++++++++++++++++++ 5 files changed, 760 insertions(+), 3 deletions(-) create mode 100644 internal/handler/container_referrers.go create mode 100644 internal/handler/container_referrers_test.go diff --git a/README.md b/README.md index 57e6e1a..39907b2 100644 --- a/README.md +++ b/README.md @@ -451,6 +451,17 @@ Or pull images directly: docker pull localhost:8080/library/nginx:latest ``` +SBOMs, signatures and attestations attached to an image are found through the +OCI 1.1 referrers API (`GET /v2/{name}/referrers/{digest}`). `oras discover` +and `notation verify` use it, and so does cosign for its newer signature +bundles. The proxy forwards these requests and caches the returned index with +the same TTL as other metadata, so they keep working offline. It does not +filter by `artifactType`; it returns the full list and the client filters it, +as the spec allows. Registries without the referrers API (GHCR, for example) +answer 404, and clients then look up the `sha256-` tag instead, which +the proxy serves like any other manifest. The proxy is pull-only, so attaching +new artifacts through it is not supported. + #### containerd (Kubernetes, k3s, nerdctl) containerd mirrors send the original registry host in an `ns` query diff --git a/internal/handler/container.go b/internal/handler/container.go index 63afd68..202400d 100644 --- a/internal/handler/container.go +++ b/internal/handler/container.go @@ -324,12 +324,15 @@ func (h *ContainerHandler) Routes() http.Handler { case strings.HasSuffix(path, "/blobs/"+r.URL.Query().Get("digest")) || strings.Contains(path, "/blobs/sha256:"): // Blob download: GET /v2/{name}/blobs/{digest} h.handleBlobDownload(w, r, path) - case strings.Contains(path, "/manifests/"): + case manifestPathPattern.MatchString(path): // Manifest: GET /v2/{name}/manifests/{reference} h.handleManifest(w, r, path) - case strings.Contains(path, "/tags/list"): + case tagsListPathPattern.MatchString(path): // Tags list: GET /v2/{name}/tags/list h.handleTagsList(w, r, path) + case referrersPathPattern.MatchString(path): + // Referrers: GET /v2/{name}/referrers/{digest} + h.handleReferrers(w, r, path) default: http.Error(w, "not found", http.StatusNotFound) } @@ -656,7 +659,7 @@ func (h *ContainerHandler) parseBlobPath(path string) (name, digest string) { } // manifestPathPattern matches manifest paths: {name}/manifests/{reference} -var manifestPathPattern = regexp.MustCompile(`^(.+)/manifests/(.+)$`) +var manifestPathPattern = regexp.MustCompile(`^(.+)/manifests/([^/]+)$`) // parseManifestPath extracts repository name and reference from a manifest path. func (h *ContainerHandler) parseManifestPath(path string) (name, reference string) { diff --git a/internal/handler/container_metadata.go b/internal/handler/container_metadata.go index edc78ad..1168ebd 100644 --- a/internal/handler/container_metadata.go +++ b/internal/handler/container_metadata.go @@ -37,3 +37,25 @@ func (h *ContainerHandler) storeContainerMetadata(ctx context.Context, ecosystem } return size, nil } + +// loadContainerMetadata returns a cached metadata row and its body, or nil +// when nothing usable is cached. +func (h *ContainerHandler) loadContainerMetadata(ctx context.Context, ecosystem, cacheKey string) (*database.MetadataCacheEntry, []byte, error) { + if h.proxy.DB == nil || h.proxy.Storage == nil { + return nil, nil, nil + } + entry, err := h.proxy.DB.GetMetadataCache(ecosystem, cacheKey) + if err != nil || entry == nil { + return nil, nil, err + } + reader, err := h.proxy.Storage.Open(ctx, entry.StoragePath) + if err != nil { + return nil, nil, nil + } + defer func() { _ = reader.Close() }() + body, err := h.proxy.ReadMetadata(reader) + if err != nil { + return nil, nil, err + } + return entry, body, nil +} diff --git a/internal/handler/container_referrers.go b/internal/handler/container_referrers.go new file mode 100644 index 0000000..d03e859 --- /dev/null +++ b/internal/handler/container_referrers.go @@ -0,0 +1,219 @@ +package handler + +import ( + "cmp" + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "regexp" + "strconv" + "strings" + "time" +) + +const ( + containerReferrersCacheEcosystem = "oci-referrers" + containerReferrersMediaType = "application/vnd.oci.image.index.v1+json" + referrersMatchCount = 3 // full match + name + digest +) + +var ( + // referrersPathPattern matches referrers paths: {name}/referrers/{digest} + referrersPathPattern = regexp.MustCompile(`^(.+)/referrers/([^/]+)$`) + referrersDigestPattern = regexp.MustCompile(`^(sha256:[a-f0-9]{64}|sha512:[a-f0-9]{128})$`) +) + +// parseReferrersPath extracts repository name and subject digest from a +// referrers path. +func (h *ContainerHandler) parseReferrersPath(path string) (name, digest string) { + matches := referrersPathPattern.FindStringSubmatch(path) + if len(matches) != referrersMatchCount { + return "", "" + } + return matches[1], matches[2] +} + +// handleReferrers serves the OCI 1.1 referrers API. +// Path format: {name}/referrers/{digest} +func (h *ContainerHandler) handleReferrers(w http.ResponseWriter, r *http.Request, path string) { + if r.Method != http.MethodGet { + http.Error(w, "method not allowed", http.StatusMethodNotAllowed) + return + } + + name, digest := h.parseReferrersPath(path) + if name == "" || !referrersDigestPattern.MatchString(digest) { + h.containerError(w, http.StatusBadRequest, "DIGEST_INVALID", "invalid referrers digest") + return + } + + registryURL, upstreamName, _, ok := h.registryForRequest(r, name) + if !ok { + h.containerError(w, http.StatusNotFound, "NAME_UNKNOWN", "unknown upstream registry") + return + } + + h.proxy.Logger.Info("container referrers request", "name", upstreamName, "digest", digest) + h.serveReferrers(w, r, registryURL, upstreamName, digest) +} + +func (h *ContainerHandler) serveReferrers(w http.ResponseWriter, r *http.Request, registryURL, name, digest string) { + // artifactType is never forwarded, so the upstream returns the full index + // and one row serves every filter. Without OCI-Filters-Applied in the + // response, clients filter the index themselves, as the spec requires. + // ns only picks the registry, so it is neither forwarded nor part of the + // cache identity. + query := r.URL.Query() + query.Del("artifactType") + query.Del(namespaceQueryParam) + // Same identity shape as manifests (registry, name, reference, variant); + // the oci-referrers ecosystem keeps the rows apart. + cacheKey := h.containerManifestCacheKey(registryURL, name, digest, query.Encode()) + cached, err := h.loadContainerReferrers(r.Context(), cacheKey) + if err != nil { + h.proxy.Logger.Warn("failed to read cached container referrers", "error", err) + cached = nil + } + if cached != nil && h.containerTagsFresh(cached) { + h.writeContainerReferrers(w, r, registryURL, cached, false) + return + } + + upstreamURL := fmt.Sprintf("%s/v2/%s/referrers/%s", registryURL, name, digest) + if encoded := query.Encode(); encoded != "" { + upstreamURL += "?" + encoded + } + req, err := http.NewRequestWithContext(r.Context(), http.MethodGet, upstreamURL, nil) + if err != nil { + h.containerError(w, http.StatusInternalServerError, "INTERNAL_ERROR", "failed to create request") + return + } + req.Header.Set("Accept", containerReferrersMediaType) + if cached != nil && cached.etag != "" { + req.Header.Set("If-None-Match", cached.etag) + } + + resp, err := h.proxy.HTTPClient.Do(req) + if err != nil { + h.serveStaleReferrersOrError(w, r, registryURL, cached, err) + return + } + defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode == http.StatusNotModified && cached != nil { + cached.fetchedAt = time.Now() + if err := h.storeContainerReferrers(r.Context(), cacheKey, cached); err != nil { + h.proxy.Logger.Warn("failed to refresh cached container referrers", "error", err) + } + h.writeContainerReferrers(w, r, registryURL, cached, false) + return + } + if resp.StatusCode != http.StatusOK { + if shouldServeStaleManifest(resp.StatusCode) && cached != nil { + h.serveStaleReferrersOrError(w, r, registryURL, cached, + fmt.Errorf("upstream returned status %d", resp.StatusCode)) + return + } + // Upstream errors reach the client as sent. Only a 404 tells it the + // registry has no referrers API, so it falls back to the sha256- + // tag schema served as manifests. + h.proxy.relayResponse(w, r, resp, copyContainerReferrersHeaders) + return + } + + body, err := h.proxy.ReadMetadata(resp.Body) + if err != nil { + h.serveStaleReferrersOrError(w, r, registryURL, cached, fmt.Errorf("reading referrers: %w", err)) + return + } + if !json.Valid(body) { + h.serveStaleReferrersOrError(w, r, registryURL, cached, errors.New("upstream referrers response is not JSON")) + return + } + referrers := &cachedContainerTags{ + body: body, + contentType: resp.Header.Get(headerContentType), + etag: resp.Header.Get(headerETag), + // Stored as sent upstream and rewritten when served, so a row shared + // by several client paths to the same registry links back correctly. + link: strings.Join(resp.Header.Values("Link"), ", "), + size: int64(len(body)), + fetchedAt: time.Now(), + } + if referrers.contentType == "" { + referrers.contentType = containerReferrersMediaType + } + if err := h.storeContainerReferrers(r.Context(), cacheKey, referrers); err != nil { + h.proxy.Logger.Warn("failed to cache container referrers", "error", err) + } + h.writeContainerReferrers(w, r, registryURL, referrers, false) +} + +// serveStaleReferrersOrError serves a cached index when the upstream cannot +// answer. Without one it returns 502 rather than 404, so clients do not take +// a failed fetch as a registry without the referrers API. +func (h *ContainerHandler) serveStaleReferrersOrError(w http.ResponseWriter, r *http.Request, registryURL string, cached *cachedContainerTags, err error) { + if cached != nil { + h.proxy.Logger.Warn("upstream referrers fetch failed, serving stale cache", "error", err) + h.writeContainerReferrers(w, r, registryURL, cached, true) + return + } + h.proxy.Logger.Error("failed to fetch container referrers", "error", err) + h.containerError(w, http.StatusBadGateway, "INTERNAL_ERROR", "failed to fetch from upstream") +} + +// copyContainerReferrersHeaders also keeps Retry-After, since uncached 429 +// and 5xx answers are relayed here instead of turning into a 502. +func copyContainerReferrersHeaders(destination, source http.Header) { + copyContainerTagsHeaders(destination, source) + if value := source.Get("Retry-After"); value != "" { + destination.Set("Retry-After", value) + } +} + +func (h *ContainerHandler) loadContainerReferrers(ctx context.Context, cacheKey string) (*cachedContainerTags, error) { + entry, body, err := h.loadContainerMetadata(ctx, containerReferrersCacheEcosystem, cacheKey) + if err != nil || entry == nil { + return nil, err + } + referrers := &cachedContainerTags{ + body: body, + contentType: cmp.Or(entry.ContentType.String, containerReferrersMediaType), + etag: entry.ETag.String, + link: entry.Link.String, + size: int64(len(body)), + fetchedAt: entry.FetchedAt.Time, + } + if entry.Size.Valid { + referrers.size = entry.Size.Int64 + } + return referrers, nil +} + +func (h *ContainerHandler) storeContainerReferrers(ctx context.Context, cacheKey string, referrers *cachedContainerTags) error { + size, err := h.storeContainerMetadata(ctx, containerReferrersCacheEcosystem, cacheKey, referrers.body, + referrers.etag, referrers.link, referrers.contentType, "", time.Time{}, referrers.fetchedAt) + if err != nil { + return fmt.Errorf("storing referrers: %w", err) + } + referrers.size = size + return nil +} + +func (h *ContainerHandler) writeContainerReferrers(w http.ResponseWriter, r *http.Request, registryURL string, referrers *cachedContainerTags, stale bool) { + w.Header().Set(headerContentType, referrers.contentType) + w.Header().Set(headerContentLength, strconv.FormatInt(referrers.size, 10)) + if referrers.etag != "" { + w.Header().Set(headerETag, referrers.etag) + } + if link := h.rewriteContainerTagsLink(referrers.link, registryURL, r.URL.Path, r.URL.Query().Get(namespaceQueryParam)); link != "" { + w.Header().Set("Link", link) + } + if stale { + w.Header().Set("Warning", containerStaleWarning) + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write(referrers.body) +} diff --git a/internal/handler/container_referrers_test.go b/internal/handler/container_referrers_test.go new file mode 100644 index 0000000..75f1cf9 --- /dev/null +++ b/internal/handler/container_referrers_test.go @@ -0,0 +1,502 @@ +package handler + +import ( + "io" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "testing" + "time" +) + +const testReferrersIndex = `{"schemaVersion":2,"mediaType":"application/vnd.oci.image.index.v1+json","manifests":[` + + `{"mediaType":"application/vnd.oci.image.manifest.v1+json","digest":"sha256:1111111111111111111111111111111111111111111111111111111111111111","size":10,"artifactType":"application/spdx+json"},` + + `{"mediaType":"application/vnd.oci.image.manifest.v1+json","digest":"sha256:2222222222222222222222222222222222222222222222222222222222222222","size":10,"artifactType":"application/vnd.cncf.notary.signature"}]}` + +var testReferrersSubject = "sha256:" + sha256Hex("subject manifest") + +func newReferrersTestHandler(t *testing.T, upstream *httptest.Server) (*ContainerHandler, *Proxy) { + t.Helper() + proxy, _, _, _ := setupTestProxy(t) + proxy.HTTPClient = upstream.Client() + return &ContainerHandler{proxy: proxy, registryURL: upstream.URL, proxyURL: "http://proxy.example.test"}, proxy +} + +func serveReferrersRequest(h *ContainerHandler, method, target string) *httptest.ResponseRecorder { + recorder := httptest.NewRecorder() + h.Routes().ServeHTTP(recorder, httptest.NewRequest(method, target, nil)) + return recorder +} + +func TestContainerHandler_ReferrersCachesIndexForEveryFilter(t *testing.T) { + upstreamAvailable := true + upstreamRequests := 0 + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + upstreamRequests++ + if r.URL.Path != "/v2/library/nginx/referrers/"+testReferrersSubject { + http.NotFound(w, r) + return + } + if r.URL.RawQuery != "" { + t.Errorf("upstream query = %q, want artifactType stripped", r.URL.RawQuery) + } + if got := r.Header.Get("Accept"); got != containerReferrersMediaType { + t.Errorf("upstream Accept = %q, want %q", got, containerReferrersMediaType) + } + if !upstreamAvailable { + http.Error(w, "upstream unavailable", http.StatusServiceUnavailable) + return + } + w.Header().Set("Content-Type", containerReferrersMediaType) + w.Header().Set("OCI-Filters-Applied", "artifactType") + _, _ = io.WriteString(w, testReferrersIndex) + })) + defer upstream.Close() + + h, proxy := newReferrersTestHandler(t, upstream) + proxy.MetadataTTL = time.Hour + base := "/library/nginx/referrers/" + testReferrersSubject + + first := serveReferrersRequest(h, http.MethodGet, base+"?artifactType=application/spdx%2Bjson") + if first.Code != http.StatusOK { + t.Fatalf("first status = %d, want 200: %s", first.Code, first.Body.String()) + } + if first.Body.String() != testReferrersIndex { + t.Errorf("first body = %q, want upstream index", first.Body.String()) + } + if got := first.Header().Get("Content-Type"); got != containerReferrersMediaType { + t.Errorf("first Content-Type = %q, want %q", got, containerReferrersMediaType) + } + if got := first.Header().Get("OCI-Filters-Applied"); got != "" { + t.Errorf("OCI-Filters-Applied = %q, want absent so clients filter", got) + } + + other := serveReferrersRequest(h, http.MethodGet, base+"?artifactType=application/vnd.cncf.notary.signature") + if other.Code != http.StatusOK || other.Body.String() != testReferrersIndex { + t.Fatalf("other filter = %d %q, want cached full index", other.Code, other.Body.String()) + } + if upstreamRequests != 1 { + t.Fatalf("upstream requests after fresh hit = %d, want 1", upstreamRequests) + } + + proxy.MetadataTTL = 0 + upstreamAvailable = false + stale := serveReferrersRequest(h, http.MethodGet, base) + if stale.Code != http.StatusOK || stale.Body.String() != testReferrersIndex { + t.Fatalf("stale = %d %q, want cached index", stale.Code, stale.Body.String()) + } + if got := stale.Header().Get("Content-Type"); got != containerReferrersMediaType { + t.Errorf("stale Content-Type = %q, want %q", got, containerReferrersMediaType) + } + if got := stale.Header().Get("Warning"); got != containerStaleWarning { + t.Errorf("stale Warning = %q, want stale warning", got) + } + if upstreamRequests != 2 { + t.Errorf("upstream requests after stale fallback = %d, want 2", upstreamRequests) + } +} + +func TestContainerHandler_ReferrersRevalidatesWithETag(t *testing.T) { + upstreamRequests := 0 + notModified := 0 + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + upstreamRequests++ + if r.Header.Get("If-None-Match") == `"referrers-etag"` { + notModified++ + w.WriteHeader(http.StatusNotModified) + return + } + w.Header().Set("Content-Type", containerReferrersMediaType) + w.Header().Set("ETag", `"referrers-etag"`) + _, _ = io.WriteString(w, testReferrersIndex) + })) + defer upstream.Close() + + h, proxy := newReferrersTestHandler(t, upstream) + proxy.MetadataTTL = time.Hour + target := "/library/nginx/referrers/" + testReferrersSubject + + if first := serveReferrersRequest(h, http.MethodGet, target); first.Code != http.StatusOK { + t.Fatalf("first status = %d, want 200", first.Code) + } + // Age the row past the TTL so the next request has to revalidate. + cacheKey := h.containerManifestCacheKey(upstream.URL, "library/nginx", testReferrersSubject, "") + entry, err := proxy.DB.GetMetadataCache(containerReferrersCacheEcosystem, cacheKey) + if err != nil || entry == nil { + t.Fatalf("cached row = %v, %v, want one", entry, err) + } + entry.FetchedAt.Time = time.Now().Add(-2 * time.Hour) + if err := proxy.DB.UpsertMetadataCache(entry); err != nil { + t.Fatalf("ageing cached row: %v", err) + } + + second := serveReferrersRequest(h, http.MethodGet, target) + if second.Code != http.StatusOK || second.Body.String() != testReferrersIndex { + t.Fatalf("revalidated = %d %q, want cached index", second.Code, second.Body.String()) + } + if got := second.Header().Get("Warning"); got != "" { + t.Errorf("Warning = %q, want none after 304", got) + } + + // The 304 starts the TTL again, so this one stays in the cache. + if third := serveReferrersRequest(h, http.MethodGet, target); third.Code != http.StatusOK { + t.Fatalf("third status = %d, want 200", third.Code) + } + if upstreamRequests != 2 || notModified != 1 { + t.Errorf("upstream requests = %d with %d answered 304, want 2 with 1", upstreamRequests, notModified) + } +} + +func TestContainerHandler_ReferrersDefaultsMissingContentType(t *testing.T) { + upstreamRequests := 0 + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + upstreamRequests++ + // A nil value stops net/http from sniffing a Content-Type. + w.Header()["Content-Type"] = nil + _, _ = io.WriteString(w, testReferrersIndex) + })) + defer upstream.Close() + + h, proxy := newReferrersTestHandler(t, upstream) + proxy.MetadataTTL = time.Hour + target := "/library/nginx/referrers/" + testReferrersSubject + + // oras-go only accepts the exact image index type, both fresh and cached. + for _, label := range []string{"fetched", "cached"} { + got := serveReferrersRequest(h, http.MethodGet, target) + if got.Code != http.StatusOK { + t.Fatalf("%s status = %d, want 200", label, got.Code) + } + if ct := got.Header().Get("Content-Type"); ct != containerReferrersMediaType { + t.Errorf("%s Content-Type = %q, want %q", label, ct, containerReferrersMediaType) + } + } + if upstreamRequests != 1 { + t.Errorf("upstream requests = %d, want 1", upstreamRequests) + } +} + +func TestContainerHandler_ReferrersRelaysUpstreamErrorsWithoutCaching(t *testing.T) { + tests := []struct { + name string + status int + header string + value string + wantError string + }{ + {name: "no referrers API", status: http.StatusNotFound, header: "Content-Type", value: "application/json", wantError: "NOT_FOUND"}, + {name: "repository unknown", status: http.StatusNotFound, header: "Content-Type", value: "application/json", wantError: "NAME_UNKNOWN"}, + {name: "auth challenge", status: http.StatusUnauthorized, header: "WWW-Authenticate", value: `Bearer realm="https://auth.example.test/token"`, wantError: "UNAUTHORIZED"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + upstreamRequests := 0 + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + upstreamRequests++ + w.Header().Set(tt.header, tt.value) + w.WriteHeader(tt.status) + _, _ = io.WriteString(w, `{"errors":[{"code":"`+tt.wantError+`"}]}`) + })) + defer upstream.Close() + + h, proxy := newReferrersTestHandler(t, upstream) + proxy.MetadataTTL = time.Hour + target := "/library/nginx/referrers/" + testReferrersSubject + for range 2 { + got := serveReferrersRequest(h, http.MethodGet, target) + if got.Code != tt.status { + t.Fatalf("status = %d, want %d", got.Code, tt.status) + } + if !strings.Contains(got.Body.String(), tt.wantError) { + t.Errorf("body = %q, want upstream error %s", got.Body.String(), tt.wantError) + } + if got.Header().Get(tt.header) != tt.value { + t.Errorf("%s = %q, want %q", tt.header, got.Header().Get(tt.header), tt.value) + } + } + if upstreamRequests != 2 { + t.Errorf("upstream requests = %d, want 2 (error must not be cached)", upstreamRequests) + } + }) + } +} + +func TestContainerHandler_ReferrersReportsUpstreamFailureWithoutCache(t *testing.T) { + tests := []struct { + name string + handler http.HandlerFunc + wantStatus int + wantRetryAfter string + }{ + {name: "upstream 503", wantStatus: http.StatusServiceUnavailable, wantRetryAfter: "120", handler: func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Retry-After", "120") + http.Error(w, "unavailable", http.StatusServiceUnavailable) + }}, + {name: "upstream 429", wantStatus: http.StatusTooManyRequests, wantRetryAfter: "60", handler: func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Retry-After", "60") + http.Error(w, "slow down", http.StatusTooManyRequests) + }}, + {name: "upstream not JSON", wantStatus: http.StatusBadGateway, handler: func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "text/html") + _, _ = io.WriteString(w, "captive portal") + }}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + upstreamRequests := 0 + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + upstreamRequests++ + tt.handler(w, r) + })) + defer upstream.Close() + + h, proxy := newReferrersTestHandler(t, upstream) + proxy.MetadataTTL = time.Hour + target := "/library/nginx/referrers/" + testReferrersSubject + for range 2 { + got := serveReferrersRequest(h, http.MethodGet, target) + if got.Code != tt.wantStatus { + t.Fatalf("status = %d, want %d: %s", got.Code, tt.wantStatus, got.Body.String()) + } + if got.Header().Get("Retry-After") != tt.wantRetryAfter { + t.Errorf("Retry-After = %q, want %q", got.Header().Get("Retry-After"), tt.wantRetryAfter) + } + } + if upstreamRequests != 2 { + t.Errorf("upstream requests = %d, want 2 (failure must not be cached)", upstreamRequests) + } + }) + } + + t.Run("upstream unreachable", func(t *testing.T) { + upstream := httptest.NewServer(http.NotFoundHandler()) + h, _ := newReferrersTestHandler(t, upstream) + upstream.Close() + got := serveReferrersRequest(h, http.MethodGet, "/library/nginx/referrers/"+testReferrersSubject) + if got.Code != http.StatusBadGateway { + t.Fatalf("status = %d, want 502 so clients keep using the referrers API: %s", got.Code, got.Body.String()) + } + }) +} + +func TestContainerHandler_ReferrersNamedRegistryRewritesLink(t *testing.T) { + upstreamRequests := 0 + var upstream *httptest.Server + upstream = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + upstreamRequests++ + if r.URL.Path != "/v2/owner/img/referrers/"+testReferrersSubject { + http.NotFound(w, r) + return + } + w.Header().Set("Content-Type", containerReferrersMediaType) + switch r.URL.Query().Get("last") { + case "": + w.Header().Add("Link", `; rel="next"`) + w.Header().Add("Link", `; rel="help"`) + case "abc": + w.Header().Set("Link", `<`+upstream.URL+`/v2/owner/img/referrers/`+testReferrersSubject+`?last=def>; rel="next"`) + } + _, _ = io.WriteString(w, testReferrersIndex) + })) + defer upstream.Close() + + proxy, _, _, _ := setupTestProxy(t) + proxy.HTTPClient = upstream.Client() + proxy.MetadataTTL = time.Hour + // The default registry is the same upstream, so upstream/test/owner/img + // and owner/img share one cache row. + h := NewContainerHandlerWithRegistry(proxy, "http://proxy.example.test", upstream.URL, map[string]string{"test": upstream.URL}) + routes := http.StripPrefix("/v2", h.Routes()) + + first := httptest.NewRecorder() + routes.ServeHTTP(first, httptest.NewRequest(http.MethodGet, + "/v2/upstream/test/owner/img/referrers/"+testReferrersSubject+"?artifactType=application/spdx%2Bjson", nil)) + if first.Code != http.StatusOK { + t.Fatalf("first page status = %d, want 200: %s", first.Code, first.Body.String()) + } + wantLink := `; rel="next", ; rel="help"` + if got := first.Header().Get("Link"); got != wantLink { + t.Fatalf("Link = %q, want %q", got, wantLink) + } + + shared := httptest.NewRecorder() + routes.ServeHTTP(shared, httptest.NewRequest(http.MethodGet, "/v2/owner/img/referrers/"+testReferrersSubject, nil)) + if shared.Code != http.StatusOK { + t.Fatalf("shared row status = %d, want 200: %s", shared.Code, shared.Body.String()) + } + wantSharedLink := `; rel="next", ; rel="help"` + if got := shared.Header().Get("Link"); got != wantSharedLink { + t.Errorf("shared row Link = %q, want it rewritten for the second path %q", got, wantSharedLink) + } + if upstreamRequests != 1 { + t.Fatalf("upstream requests after shared row hit = %d, want 1", upstreamRequests) + } + + nextURL := strings.TrimPrefix(strings.SplitN(first.Header().Get("Link"), ">", 2)[0], "<") + next := httptest.NewRecorder() + routes.ServeHTTP(next, httptest.NewRequest(http.MethodGet, nextURL, nil)) + if next.Code != http.StatusOK { + t.Fatalf("next page status = %d, want 200: %s", next.Code, next.Body.String()) + } + wantNextLink := `; rel="next"` + if got := next.Header().Get("Link"); got != wantNextLink { + t.Errorf("next page Link = %q, want absolute upstream link rewritten to %q", got, wantNextLink) + } + if upstreamRequests != 2 { + t.Errorf("upstream requests = %d, want 2 (pages are separate rows)", upstreamRequests) + } +} + +func TestContainerHandler_ReferrersRejectsInvalidRequests(t *testing.T) { + upstreamRequests := 0 + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + upstreamRequests++ + w.WriteHeader(http.StatusInternalServerError) + })) + defer upstream.Close() + + proxy, _, _, _ := setupTestProxy(t) + proxy.HTTPClient = upstream.Client() + h := NewContainerHandlerWithRegistry(proxy, "http://proxy.example.test", upstream.URL) + + tests := []struct { + name string + method string + path string + wantStatus int + wantCode string + }{ + {name: "post", method: http.MethodPost, path: "/library/nginx/referrers/" + testReferrersSubject, wantStatus: http.StatusMethodNotAllowed}, + {name: "short digest", method: http.MethodGet, path: "/library/nginx/referrers/sha256:abc", wantStatus: http.StatusBadRequest, wantCode: "DIGEST_INVALID"}, + {name: "trailing characters", method: http.MethodGet, path: "/library/nginx/referrers/" + testReferrersSubject + "x", wantStatus: http.StatusBadRequest, wantCode: "DIGEST_INVALID"}, + {name: "leading characters", method: http.MethodGet, path: "/library/nginx/referrers/x" + testReferrersSubject, wantStatus: http.StatusBadRequest, wantCode: "DIGEST_INVALID"}, + {name: "unknown algorithm", method: http.MethodGet, path: "/library/nginx/referrers/md5:" + strings.Repeat("a", 32), wantStatus: http.StatusBadRequest, wantCode: "DIGEST_INVALID"}, + {name: "unknown named upstream", method: http.MethodGet, path: "/upstream/missing/owner/img/referrers/" + testReferrersSubject, wantStatus: http.StatusNotFound, wantCode: "NAME_UNKNOWN"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := serveReferrersRequest(h, tt.method, tt.path) + if got.Code != tt.wantStatus { + t.Fatalf("status = %d, want %d: %s", got.Code, tt.wantStatus, got.Body.String()) + } + if tt.wantCode != "" && !strings.Contains(got.Body.String(), tt.wantCode) { + t.Errorf("body = %q, want %s", got.Body.String(), tt.wantCode) + } + }) + } + if upstreamRequests != 0 { + t.Errorf("upstream requests = %d, want 0", upstreamRequests) + } +} + +func TestContainerHandler_ReferrersRouteKeepsManifestPaths(t *testing.T) { + tagSchema := "sha256-" + strings.TrimPrefix(testReferrersSubject, "sha256:") + var upstreamPaths []string + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + upstreamPaths = append(upstreamPaths, r.URL.Path) + // Both handlers build the same upstream path here, so tell them apart + // by Accept: only the referrers handler asks for the index type alone. + if r.Header.Get("Accept") == containerReferrersMediaType { + http.Error(w, "reached the referrers handler", http.StatusBadRequest) + return + } + w.Header().Set("Content-Type", containerReferrersMediaType) + _, _ = io.WriteString(w, testReferrersIndex) + })) + defer upstream.Close() + + h, _ := newReferrersTestHandler(t, upstream) + for _, path := range []string{ + // A repository may contain a "referrers" component; manifest paths + // must keep reaching the manifest handler. + "/foo/referrers/app/manifests/latest", + // Clients without a referrers API read the tag schema as a manifest. + "/library/nginx/manifests/" + tagSchema, + } { + if got := serveReferrersRequest(h, http.MethodGet, path); got.Code != http.StatusOK { + t.Fatalf("GET %s status = %d, want 200: %s", path, got.Code, got.Body.String()) + } + } + want := []string{ + "/v2/foo/referrers/app/manifests/latest", + "/v2/library/nginx/manifests/" + tagSchema, + } + if strings.Join(upstreamPaths, "\n") != strings.Join(want, "\n") { + t.Errorf("upstream paths = %q, want %q", upstreamPaths, want) + } +} + +func TestContainerHandler_ReferrersRouteAllowsEndpointNamesInRepository(t *testing.T) { + var upstreamPaths []string + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + upstreamPaths = append(upstreamPaths, r.URL.Path) + // The manifest handler would forward the same path, so only answer + // the referrers handler, which asks for the index type alone. + if r.Header.Get("Accept") != containerReferrersMediaType { + http.Error(w, "reached the manifest handler", http.StatusBadRequest) + return + } + w.Header().Set("Content-Type", containerReferrersMediaType) + _, _ = io.WriteString(w, testReferrersIndex) + })) + defer upstream.Close() + + h, _ := newReferrersTestHandler(t, upstream) + for _, name := range []string{"owner/tags/list", "owner/manifests/app"} { + upstreamPaths = nil + got := serveReferrersRequest(h, http.MethodGet, "/"+name+"/referrers/"+testReferrersSubject) + if got.Code != http.StatusOK { + t.Fatalf("%s: status = %d, want 200: %s", name, got.Code, got.Body.String()) + } + want := "/v2/" + name + "/referrers/" + testReferrersSubject + if len(upstreamPaths) != 1 || upstreamPaths[0] != want { + t.Errorf("%s: upstream paths = %q, want [%q]", name, upstreamPaths, want) + } + } +} + +func TestContainerHandler_ReferrersFollowsNamespace(t *testing.T) { + var upstreamQueries []string + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/v2/owner/img/referrers/"+testReferrersSubject { + http.NotFound(w, r) + return + } + upstreamQueries = append(upstreamQueries, r.URL.RawQuery) + w.Header().Set("Content-Type", containerReferrersMediaType) + w.Header().Set("Link", `; rel="next"`) + _, _ = io.WriteString(w, testReferrersIndex) + })) + defer upstream.Close() + + proxy, _, _, _ := setupTestProxy(t) + proxy.HTTPClient = upstream.Client() + proxy.MetadataTTL = time.Hour + h := NewContainerHandler(proxy, "http://proxy.example.test", map[string]string{"test": upstream.URL}) + routes := http.StripPrefix("/v2", h.Routes()) + host := strings.TrimPrefix(upstream.URL, "http://") + target := "/v2/owner/img/referrers/" + testReferrersSubject + + withNS := httptest.NewRecorder() + routes.ServeHTTP(withNS, httptest.NewRequest(http.MethodGet, target+"?ns="+url.QueryEscape(host), nil)) + if withNS.Code != http.StatusOK { + t.Fatalf("ns request status = %d, want 200: %s", withNS.Code, withNS.Body.String()) + } + wantLink := `; rel="next"` + if got := withNS.Header().Get("Link"); got != wantLink { + t.Errorf("Link = %q, want next page kept on the ns route %q", got, wantLink) + } + + // The prefix route reaches the same registry and shares the row. + prefixed := httptest.NewRecorder() + routes.ServeHTTP(prefixed, httptest.NewRequest(http.MethodGet, "/v2/upstream/test/owner/img/referrers/"+testReferrersSubject, nil)) + if prefixed.Code != http.StatusOK { + t.Fatalf("prefix request status = %d, want 200: %s", prefixed.Code, prefixed.Body.String()) + } + if len(upstreamQueries) != 1 || upstreamQueries[0] != "" { + t.Errorf("upstream queries = %q, want one request without ns", upstreamQueries) + } +} From d2a069f6f2746f6bd5190e2feef0123542702a12 Mon Sep 17 00:00:00 2001 From: Giles Westwood Date: Mon, 12 Oct 2026 09:12:16 +0100 Subject: [PATCH 203/633] Cache configured version-pinned paths of generic upstreams (#424) * Cache configured version-pinned paths of generic upstreams The generic handler only knew GitHub release asset paths were immutable, so any other download, such as Maven from archive.apache.org, went through the metadata route: streamed uncached by default, or kept as metadata and never listed as a package. upstream.generic_artifacts maps a generic upstream to regular expressions for its version-pinned paths. A pattern must match the whole path after /generic/{name}/ and has version and file named groups, plus an optional name group. Matching downloads take the release asset route: cached in the artifact cache, served while the upstream is down, and listed under the generic ecosystem. Other paths are unchanged. * Fix generic artifact cache identity and query forwarding * Serve generic release assets cached under the unhashed key Release assets cached before hashed keys used {repository}/{asset} and became unreachable after an upgrade, so an offline upstream returned 502. Serve such an entry when its recorded upstream URL matches the request. --------- Co-authored-by: Giles Westwood <201443+gilesw@users.noreply.github.com> --- README.md | 18 +++ config.example.yaml | 9 ++ docs/configuration.md | 23 +++- internal/config/config.go | 48 ++++++- internal/config/config_test.go | 34 +++++ internal/handler/generic.go | 138 ++++++++++++++++--- internal/handler/generic_test.go | 224 +++++++++++++++++++++++++++++++ internal/server/server.go | 2 +- 8 files changed, 470 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index 39907b2..020916c 100644 --- a/README.md +++ b/README.md @@ -702,6 +702,24 @@ Commit a `mise.lock` and install with `mise install --locked` so pinned installs need no API call at all. Add a bearer token for `https://api.github.com` under `upstream.auth` if the fleet exceeds GitHub's anonymous rate limit. +Tools mise downloads from elsewhere, such as Maven from `archive.apache.org`, +can be cached the same way by declaring their version-pinned paths in +`upstream.generic_artifacts` (see [configuration](docs/configuration.md)): + +```yaml +upstream: + generic: + apache: "https://archive.apache.org" + generic_artifacts: + apache: + - 'dist/(?Pmaven)/maven-3/(?P[^/]+)/binaries/(?P[^/]+)' +``` + +```toml +[settings.url_replacements] +"https://archive.apache.org/" = "http://localhost:8080/generic/apache/" +``` + ## Configuration The proxy can be configured via: diff --git a/config.example.yaml b/config.example.yaml index 1893859..f91b7d3 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -274,6 +274,15 @@ upstream: # generic: # github: "https://github.com" # github-api: "https://api.github.com" + # apache: "https://archive.apache.org" + + # Version-pinned paths of generic upstreams, cached immutably like GitHub + # release assets. Each pattern must match the whole path after + # /generic/{name}/ and have `version` and `file` named groups; an optional + # `name` group names the package. + # generic_artifacts: + # apache: + # - 'dist/(?Pmaven)/maven-3/(?P[^/]+)/binaries/(?P[^/]+)' # Authentication for upstream registries # Keys are absolute URL scopes. Scheme, host, effective port, and path diff --git a/docs/configuration.md b/docs/configuration.md index f30fa27..6b17452 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -359,7 +359,28 @@ Only configured upstreams are reachable, so this is not an open HTTP proxy. Paths shaped like `{owner}/{repo}/releases/download/{tag}/{asset}` are version-pinned GitHub release assets: they are stored in the artifact cache and served from it without revalidation, including while the upstream is -down. Every other path is served through the metadata cache (`cache_metadata` +down. + +Other upstreams can declare their own version-pinned paths in +`generic_artifacts`. Each pattern is a regular expression that must match the +whole path after `/generic/{name}/`, with `version` and `file` named groups +(single path segments) and an optional `name` group for the package name, +which otherwise is the upstream name. Matching downloads are cached like +release assets and listed under the `generic` ecosystem: + +```yaml +upstream: + generic: + apache: "https://archive.apache.org" + generic_artifacts: + apache: + - 'dist/(?Pmaven)/maven-3/(?P[^/]+)/binaries/(?P[^/]+)' +``` + +Only list paths whose content never changes for a given version: a match is +never revalidated. + +Every other path is served through the metadata cache (`cache_metadata` must be enabled for offline fallback): fresh within `metadata_ttl`, then revalidated with the upstream's `ETag`/`Last-Modified`, and served stale with a `Warning: 110` header when the upstream fails, refuses or rate-limits the diff --git a/internal/config/config.go b/internal/config/config.go index bf10cbe..25ee4d7 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -68,6 +68,7 @@ import ( "os" "path" "path/filepath" + "regexp" "sort" "strconv" "strings" @@ -688,11 +689,21 @@ type UpstreamConfig struct { // Generic maps names to plain HTTP upstream base URLs, served at // /generic/{name}/. The remaining request path and query string are // appended to the upstream URL. GitHub release asset paths - // ({owner}/{repo}/releases/download/{tag}/{asset}) are cached in the - // artifact cache; everything else goes through the metadata cache. + // ({owner}/{repo}/releases/download/{tag}/{asset}) and paths matching + // GenericArtifacts are cached in the artifact cache; everything else + // goes through the metadata cache. // Example: {"github": "https://github.com", "github-api": "https://api.github.com"}. Generic map[string]string `json:"generic" yaml:"generic"` + // GenericArtifacts maps a Generic upstream name to regular expressions for + // its version-pinned download paths. A request path after + // /generic/{name}/ that one of them matches in full is cached in the + // artifact cache like a GitHub release asset. Each pattern needs `version` + // and `file` named groups; an optional `name` group names the package, + // which otherwise is the upstream name. + // Example: {"apache": ["dist/(?Pmaven)/maven-3/(?P[^/]+)/binaries/(?P[^/]+)"]}. + GenericArtifacts map[string][]string `json:"generic_artifacts" yaml:"generic_artifacts"` + // Auth configures authentication for upstream registries. // Keys are absolute URL scopes matched by scheme, host, effective port, // and path-segment prefix. @@ -754,6 +765,9 @@ func (u *UpstreamConfig) Validate() error { if err := validateNamedUpstreams("upstream.generic", u.Generic); err != nil { return err } + if err := u.validateGenericArtifacts(); err != nil { + return err + } if err := validateNamedUpstreams("upstream.debian_repositories", u.DebianRepositories); err != nil { return err } @@ -768,6 +782,36 @@ func (u *UpstreamConfig) Validate() error { return nil } +func (u *UpstreamConfig) validateGenericArtifacts() error { + for name, patterns := range u.GenericArtifacts { + if _, found := u.Generic[name]; !found { + return fmt.Errorf("invalid upstream.generic_artifacts name %q: no upstream.generic entry", name) + } + for _, pattern := range patterns { + if _, err := CompileGenericArtifactPattern(pattern); err != nil { + return fmt.Errorf("invalid upstream.generic_artifacts.%s pattern %q: %w", name, pattern, err) + } + } + } + return nil +} + +// CompileGenericArtifactPattern compiles an upstream.generic_artifacts pattern +// so that it only matches a whole path, and checks it has the `version` and +// `file` groups. +func CompileGenericArtifactPattern(pattern string) (*regexp.Regexp, error) { + re, err := regexp.Compile(`^(?:` + pattern + `)$`) + if err != nil { + return nil, err + } + for _, group := range []string{"version", "file"} { + if re.SubexpIndex(group) < 0 { + return nil, fmt.Errorf("missing (?P<%s>...) group", group) + } + } + return re, nil +} + // debianReservedRepositoryNames are the upstream.debian archive's own root // paths, which a repository of the same name would shadow. var debianReservedRepositoryNames = []string{"pool", "dists"} diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 5566afd..0f7a78f 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -1382,6 +1382,40 @@ func TestValidateNamedUpstreams(t *testing.T) { }, wantErr: true, }, + { + name: "generic artifact patterns", + modify: func(cfg *Config) { + cfg.Upstream.Generic = map[string]string{"apache": "https://archive.apache.org"} + cfg.Upstream.GenericArtifacts = map[string][]string{ + "apache": {`dist/(?Pmaven)/maven-3/(?P[^/]+)/binaries/(?P[^/]+)`}, + } + }, + }, + { + name: "generic artifact patterns for an unknown upstream", + modify: func(cfg *Config) { + cfg.Upstream.GenericArtifacts = map[string][]string{ + "apache": {`(?P[^/]+)/(?P[^/]+)`}, + } + }, + wantErr: true, + }, + { + name: "generic artifact pattern does not compile", + modify: func(cfg *Config) { + cfg.Upstream.Generic = map[string]string{"apache": "https://archive.apache.org"} + cfg.Upstream.GenericArtifacts = map[string][]string{"apache": {`(?P[^/]+/(?P[^/]+)`}} + }, + wantErr: true, + }, + { + name: "generic artifact pattern without a file group", + modify: func(cfg *Config) { + cfg.Upstream.Generic = map[string]string{"apache": "https://archive.apache.org"} + cfg.Upstream.GenericArtifacts = map[string][]string{"apache": {`(?P[^/]+)/[^/]+`}} + }, + wantErr: true, + }, { name: "generic upstream URL is not absolute", modify: func(cfg *Config) { diff --git a/internal/handler/generic.go b/internal/handler/generic.go index a2458b2..67ae94a 100644 --- a/internal/handler/generic.go +++ b/internal/handler/generic.go @@ -1,11 +1,15 @@ package handler import ( + "context" "crypto/sha256" "encoding/hex" + "fmt" "net/http" "regexp" "strings" + + "github.com/git-pkgs/proxy/internal/config" ) const ( @@ -33,7 +37,8 @@ var githubReleaseAssetPattern = regexp.MustCompile(`^([^/]+)/([^/]+)/releases/do // fixed URL shapes, such as mise's aqua backend fetching GitHub release // assets, and is pointed at by URL-rewriting settings on the client. // -// Release-asset paths ({owner}/{repo}/releases/download/{tag}/{asset}) are +// Release-asset paths ({owner}/{repo}/releases/download/{tag}/{asset}), and +// paths matching the upstream's configured artifact patterns, are // version-pinned and cached in the shared artifact cache, so they keep being // served when the upstream is unreachable. Every other path is served through // the metadata cache: fresh within the metadata TTL, revalidated with the @@ -41,19 +46,36 @@ var githubReleaseAssetPattern = regexp.MustCompile(`^([^/]+)/([^/]+)/releases/do // or refuses the request. That covers API responses such as // api.github.com/repos/{owner}/{repo}/releases/tags/{tag}. type GenericHandler struct { - proxy *Proxy - repositories map[string]string + proxy *Proxy + repositories map[string]string + artifactPatterns map[string][]*regexp.Regexp } // NewGenericHandler creates a generic HTTP download proxy handler. -func NewGenericHandler(proxy *Proxy, repositories map[string]string) *GenericHandler { +// artifactPatterns maps upstream names to upstream.generic_artifacts patterns; +// a pattern that does not compile is logged and skipped. +func NewGenericHandler(proxy *Proxy, repositories map[string]string, artifactPatterns ...map[string][]string) *GenericHandler { h := &GenericHandler{ - proxy: proxy, - repositories: make(map[string]string, len(repositories)), + proxy: proxy, + repositories: make(map[string]string, len(repositories)), + artifactPatterns: make(map[string][]*regexp.Regexp), } for name, upstreamURL := range repositories { h.repositories[name] = strings.TrimSuffix(upstreamURL, "/") } + if len(artifactPatterns) > 0 { + for name, patterns := range artifactPatterns[0] { + for _, pattern := range patterns { + re, err := config.CompileGenericArtifactPattern(pattern) + if err != nil { + proxy.Logger.Warn("skipping generic artifact pattern", + "repository", name, "pattern", pattern, "error", err) + continue + } + h.artifactPatterns[name] = append(h.artifactPatterns[name], re) + } + } + } return h } @@ -80,8 +102,8 @@ func (h *GenericHandler) Routes() http.Handler { return } - if asset, ok := parseGitHubReleaseAsset(rest); ok { - h.handleReleaseAsset(w, r, repository, upstreamURL, rest, asset) + if artifact, ok := h.parseArtifact(repository, rest); ok { + h.handleArtifact(w, r, repository, upstreamURL, rest, artifact) return } @@ -89,6 +111,54 @@ func (h *GenericHandler) Routes() http.Handler { }) } +// genericArtifact is the identity of a version-pinned download. +type genericArtifact struct { + name string + version string + filename string +} + +// parseArtifact returns the identity of a version-pinned download path: one +// matching the upstream's artifact patterns, else a GitHub release asset. +func (h *GenericHandler) parseArtifact(repository, path string) (genericArtifact, bool) { + for _, pattern := range h.artifactPatterns[repository] { + if artifact, ok := matchArtifactPattern(pattern, repository, path); ok { + return artifact, true + } + } + if asset, ok := parseGitHubReleaseAsset(path); ok { + return genericArtifact{ + name: asset.owner + "/" + asset.repo, + version: asset.tag, + filename: asset.filename, + }, true + } + return genericArtifact{}, false +} + +// matchArtifactPattern extracts the artifact identity from a path matched by +// pattern. The version and file must be single, non-empty path segments; the +// name defaults to the upstream name. +func matchArtifactPattern(pattern *regexp.Regexp, repository, path string) (genericArtifact, bool) { + matches := pattern.FindStringSubmatch(path) + if matches == nil { + return genericArtifact{}, false + } + artifact := genericArtifact{ + name: repository, + version: matches[pattern.SubexpIndex("version")], + filename: matches[pattern.SubexpIndex("file")], + } + if i := pattern.SubexpIndex("name"); i >= 0 && matches[i] != "" { + artifact.name = matches[i] + } + if artifact.version == "" || artifact.filename == "" || + strings.Contains(artifact.version, "/") || strings.Contains(artifact.filename, "/") { + return genericArtifact{}, false + } + return artifact, true +} + // githubReleaseAsset is the identity of a version-pinned release download. type githubReleaseAsset struct { owner string @@ -112,21 +182,26 @@ func parseGitHubReleaseAsset(path string) (githubReleaseAsset, bool) { }, true } -// handleReleaseAsset fetches and caches a version-pinned release asset in the -// artifact cache. The configured upstream name is part of the cache identity -// so two upstreams serving the same path never share bytes. -func (h *GenericHandler) handleReleaseAsset(w http.ResponseWriter, r *http.Request, repository, upstreamURL, path string, asset githubReleaseAsset) { - name := asset.owner + "/" + asset.repo +// handleArtifact fetches and caches a version-pinned download in the artifact +// cache. The upstream, full path and query identify the cached bytes, since +// distinct downloads can have the same captured name, version and filename. +func (h *GenericHandler) handleArtifact(w http.ResponseWriter, r *http.Request, repository, upstreamURL, path string, artifact genericArtifact) { downloadURL := upstreamURL + "/" + path - cacheFilename := repository + "/" + asset.filename + if r.URL.RawQuery != "" { + downloadURL += "?" + r.URL.RawQuery + } + cacheFilename := repository + "/" + h.cacheKey(repository, upstreamURL, path, r.URL.RawQuery) + "/" + artifact.filename - h.proxy.Logger.Info("generic release asset download", - "repository", repository, "name", name, "version", asset.tag, "filename", asset.filename) + h.proxy.Logger.Info("generic artifact download", + "repository", repository, "name", artifact.name, "version", artifact.version, "filename", artifact.filename) - result, err := h.proxy.GetOrFetchArtifactFromURL( - r.Context(), genericEcosystem, name, asset.tag, cacheFilename, downloadURL) + result, err := h.legacyArtifact(r.Context(), repository, downloadURL, artifact) + if err == nil && result == nil { + result, err = h.proxy.GetOrFetchArtifactFromURL( + r.Context(), genericEcosystem, artifact.name, artifact.version, cacheFilename, downloadURL) + } if err != nil { - h.proxy.serveArtifactError(w, err, "failed to fetch release asset") + h.proxy.serveArtifactError(w, err, "failed to fetch artifact") return } @@ -136,6 +211,25 @@ func (h *GenericHandler) handleReleaseAsset(w http.ResponseWriter, r *http.Reque ServeArtifactRequest(w, r, result) } +// legacyArtifact returns the artifact cached under the unhashed +// {repository}/{filename} key, provided its recorded upstream URL is +// downloadURL, so a different path or query sharing that key is not served. +func (h *GenericHandler) legacyArtifact(ctx context.Context, repository, downloadURL string, artifact genericArtifact) (*CacheResult, error) { + pkgPURL, versionPURL, err := packagePURLStrings(genericEcosystem, artifact.name, artifact.version) + if err != nil { + return nil, err + } + filename := repository + "/" + artifact.filename + record, err := h.proxy.DB.GetArtifact(versionPURL, filename) + if err != nil { + return nil, fmt.Errorf("checking artifact cache: %w", err) + } + if record == nil || record.UpstreamURL != downloadURL { + return nil, nil + } + return h.proxy.checkCache(ctx, pkgPURL, versionPURL, filename) +} + // handleMetadata serves any other path through the metadata cache. The query // string is forwarded and is part of the cache identity. A fixed Accept header // keeps all clients on one cached representation. @@ -146,14 +240,14 @@ func (h *GenericHandler) handleMetadata(w http.ResponseWriter, r *http.Request, } h.proxy.ProxyCached(w, r, target, genericEcosystem, - h.metadataCacheKey(repository, upstreamURL, path, r.URL.RawQuery), genericAcceptAny) + h.cacheKey(repository, upstreamURL, path, r.URL.RawQuery), genericAcceptAny) } -// metadataCacheKey derives the metadata cache key from the upstream name, its +// cacheKey derives the cache key from the upstream name, its // URL, the request path and query. Hashing the identity keeps distinct // upstreams from sharing entries and drops cached entries when an upstream is // repointed, mirroring APKHandler.metadataCacheKey. -func (h *GenericHandler) metadataCacheKey(repository, upstreamURL, path, query string) string { +func (h *GenericHandler) cacheKey(repository, upstreamURL, path, query string) string { identity := repository + "\x00" + upstreamURL + "\x00" + path + "\x00" + query digest := sha256.Sum256([]byte(identity)) return hex.EncodeToString(digest[:]) diff --git a/internal/handler/generic_test.go b/internal/handler/generic_test.go index e67a6b1..3e45fb9 100644 --- a/internal/handler/generic_test.go +++ b/internal/handler/generic_test.go @@ -1,13 +1,16 @@ package handler import ( + "context" "net/http" "net/http/httptest" + "regexp" "strings" "sync/atomic" "testing" "time" + "github.com/git-pkgs/proxy/internal/config" upstreamhttp "github.com/git-pkgs/proxy/internal/httpclient" "github.com/git-pkgs/registries/fetch" ) @@ -51,6 +54,179 @@ func TestParseGitHubReleaseAsset(t *testing.T) { } } +const testMavenPattern = `dist/(?Pmaven)/maven-3/(?P[^/]+)/binaries/(?P[^/]+)` + +func TestMatchArtifactPattern(t *testing.T) { + named, err := config.CompileGenericArtifactPattern(testMavenPattern) + if err != nil { + t.Fatal(err) + } + unnamed, err := config.CompileGenericArtifactPattern(`dist/v(?P[^/]+)/(?P.+)`) + if err != nil { + t.Fatal(err) + } + + tests := []struct { + name string + pattern *regexp.Regexp + path string + want genericArtifact + ok bool + }{ + { + "name group", + named, + "dist/maven/maven-3/3.9.9/binaries/apache-maven-3.9.9-bin.tar.gz", + genericArtifact{name: "maven", version: "3.9.9", filename: "apache-maven-3.9.9-bin.tar.gz"}, + true, + }, + { + "name defaults to upstream", + unnamed, + "dist/v20.15.0/node-v20.15.0-linux-x64.tar.xz", + genericArtifact{name: "apache", version: "20.15.0", filename: "node-v20.15.0-linux-x64.tar.xz"}, + true, + }, + // Patterns match whole paths only. + {"prefix only", named, "dist/maven/maven-3/3.9.9/binaries/apache-maven-3.9.9-bin.tar.gz/extra", genericArtifact{}, false}, + {"suffix only", named, "mirror/dist/maven/maven-3/3.9.9/binaries/apache-maven-3.9.9-bin.tar.gz", genericArtifact{}, false}, + {"directory listing", named, "dist/maven/maven-3/3.9.9/binaries/", genericArtifact{}, false}, + // A file group spanning segments would nest storage paths. + {"file with slash", unnamed, "dist/v20.15.0/win-x64/node.exe", genericArtifact{}, false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, ok := matchArtifactPattern(tt.pattern, "apache", tt.path) + if ok != tt.ok || got != tt.want { + t.Errorf("matchArtifactPattern(%q) = (%+v, %v), want (%+v, %v)", tt.path, got, ok, tt.want, tt.ok) + } + }) + } +} + +func TestGenericHandler_ArtifactPatternIsCachedAndOtherPathsAreNot(t *testing.T) { + const ( + artifactPath = "/dist/maven/maven-3/3.9.9/binaries/apache-maven-3.9.9-bin.tar.gz" + listingPath = "/dist/maven/maven-3/3.9.9/binaries/" + ) + tarball := []byte("maven tarball bytes") + var available atomic.Bool + available.Store(true) + var artifactRequests, listingRequests atomic.Int32 + + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if !available.Load() { + http.Error(w, "unavailable", http.StatusServiceUnavailable) + return + } + switch r.URL.Path { + case artifactPath: + artifactRequests.Add(1) + _, _ = w.Write(tarball) + case listingPath: + listingRequests.Add(1) + _, _ = w.Write([]byte("listing")) + default: + http.NotFound(w, r) + } + })) + defer upstream.Close() + + proxy, db, store, _ := setupTestProxy(t) + store.seekable = true + proxy.HTTPClient = upstream.Client() + fetcher := fetch.NewFetcher(fetch.WithHTTPClient(upstream.Client()), fetch.WithMaxRetries(0)) + proxy.Fetcher = fetcher + t.Cleanup(func() { _ = fetcher.Close() }) + + h := NewGenericHandler(proxy, + map[string]string{"apache": upstream.URL}, + map[string][]string{"apache": {testMavenPattern}}) + + w := serveGenericRequest(h, "/apache"+artifactPath) + if w.Code != http.StatusOK || w.Body.String() != string(tarball) { + t.Fatalf("status = %d, body = %q, want 200 %q", w.Code, w.Body.String(), tarball) + } + + pkg, err := db.GetPackageByEcosystemName(genericEcosystem, "maven") + if err != nil || pkg == nil { + t.Fatalf("package generic/maven = %v, %v; want a cached package", pkg, err) + } + versions, err := db.GetVersionsByPackagePURL(pkg.PURL) + if err != nil || len(versions) != 1 || !strings.HasSuffix(versions[0].PURL, "@3.9.9") { + t.Fatalf("versions = %+v, %v; want one @3.9.9", versions, err) + } + + available.Store(false) + w = serveGenericRequest(h, "/apache"+artifactPath) + if w.Code != http.StatusOK || w.Body.String() != string(tarball) { + t.Fatalf("cached: status = %d, body = %q, want 200 %q", w.Code, w.Body.String(), tarball) + } + if got := artifactRequests.Load(); got != 1 { + t.Errorf("artifact upstream requests = %d, want 1", got) + } + + // Paths the pattern does not match keep going through the metadata route. + available.Store(true) + for range 2 { + serveGenericRequest(h, "/apache"+listingPath) + } + if got := listingRequests.Load(); got != 2 { + t.Errorf("listing upstream requests = %d, want 2 with metadata caching off", got) + } +} + +func TestGenericHandler_ConfiguredArtifactsKeepDistinctPathsAndQueries(t *testing.T) { + for _, tt := range []struct { + name string + targets []string + }{ + {"platform paths", []string{ + "/releases/tool/1.0.0/linux/tool.tar.gz", + "/releases/tool/1.0.0/darwin/tool.tar.gz", + }}, + {"query parameters", []string{ + "/releases/tool/1.0.0/linux/tool.tar.gz?token=example&variant=a%2Bb", + "/releases/tool/1.0.0/linux/tool.tar.gz?token=other&variant=a+b", + "/releases/tool/1.0.0/linux/tool.tar.gz", + }}, + } { + t.Run(tt.name, func(t *testing.T) { + var requests atomic.Int32 + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests.Add(1) + // Echo the exact target so dropped or changed queries fail too. + _, _ = w.Write([]byte(r.URL.RequestURI())) + })) + defer upstream.Close() + + proxy, _, store, _ := setupTestProxy(t) + store.seekable = true + fetcher := fetch.NewFetcher(fetch.WithHTTPClient(upstream.Client()), fetch.WithMaxRetries(0)) + proxy.Fetcher = fetcher + t.Cleanup(func() { _ = fetcher.Close() }) + h := NewGenericHandler(proxy, + map[string]string{"downloads": upstream.URL}, + map[string][]string{"downloads": {`releases/(?Ptool)/(?P[^/]+)/(linux|darwin)/(?P[^/]+)`}}) + + for pass := range 2 { + for _, target := range tt.targets { + w := serveGenericRequest(h, "/downloads"+target) + if w.Code != http.StatusOK || w.Body.String() != target { + t.Fatalf("pass %d, target %q: status = %d, body = %q; want 200 %q", + pass, target, w.Code, w.Body.String(), target) + } + } + upstream.Close() + } + if got := requests.Load(); got != int32(len(tt.targets)) { + t.Errorf("upstream requests = %d, want %d", got, len(tt.targets)) + } + }) + } +} + func TestGenericHandler_RejectsUnknownUpstreamAndBadPaths(t *testing.T) { h := NewGenericHandler(testProxy(), map[string]string{"github": "https://github.com"}) @@ -160,6 +336,54 @@ func TestGenericHandler_ReleaseAssetIsCachedAndServedWhenUpstreamDown(t *testing } } +func TestGenericHandler_ServesReleaseAssetCachedUnderUnhashedKey(t *testing.T) { + asset := []byte("jq binary bytes") + var available atomic.Bool + available.Store(true) + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if !available.Load() { + http.Error(w, "unavailable", http.StatusServiceUnavailable) + return + } + _, _ = w.Write(asset) + })) + defer upstream.Close() + + proxy, _, store, _ := setupTestProxy(t) + store.seekable = true + fetcher := fetch.NewFetcher(fetch.WithHTTPClient(upstream.Client()), fetch.WithMaxRetries(0)) + proxy.Fetcher = fetcher + t.Cleanup(func() { _ = fetcher.Close() }) + + // Cache the asset the way releases before hashed keys did. + result, err := proxy.GetOrFetchArtifactFromURL(context.Background(), genericEcosystem, + "jqlang/jq", "jq-1.7.1", "github/jq-linux-amd64", upstream.URL+testReleaseAssetPath) + if err != nil { + t.Fatalf("priming legacy entry: %v", err) + } + if result.Reader != nil { + _ = result.Reader.Close() + } + available.Store(false) + + h := NewGenericHandler(proxy, map[string]string{"github": upstream.URL}) + w := serveGenericRequest(h, "/github"+testReleaseAssetPath) + if w.Code != http.StatusOK || w.Body.String() != string(asset) { + t.Fatalf("legacy entry: status = %d, body = %q; want 200 %q", w.Code, w.Body.String(), asset) + } + + w = serveGenericRequest(h, "/github"+testReleaseAssetPath+"?x=1") + if w.Code != http.StatusBadGateway { + t.Errorf("with query: status = %d, want 502", w.Code) + } + + repointed := NewGenericHandler(proxy, map[string]string{"github": upstream.URL + "/mirror"}) + w = serveGenericRequest(repointed, "/github"+testReleaseAssetPath) + if w.Code != http.StatusBadGateway { + t.Errorf("repointed upstream: status = %d, want 502", w.Code) + } +} + func TestGenericHandler_ReleaseAssetNotFoundIsNotCached(t *testing.T) { upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { http.NotFound(w, r) diff --git a/internal/server/server.go b/internal/server/server.go index 78377f9..841f30d 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -452,7 +452,7 @@ func (s *Server) mountProtocolHandlers(r chi.Router, proxy *handler.Proxy) { s.cfg.Upstream.DebianRepositories, ) rpmHandler := handler.NewRPMHandlerWithUpstream(proxy, s.cfg.BaseURL, s.cfg.Upstream.RPM) - genericHandler := handler.NewGenericHandler(proxy, s.cfg.Upstream.Generic) + genericHandler := handler.NewGenericHandler(proxy, s.cfg.Upstream.Generic, s.cfg.Upstream.GenericArtifacts) r = r.With(rejectQueryInPath) r.Mount("/npm", http.StripPrefix("/npm", npmHandler.Routes())) From 1d88be53abcbbc22e285c50e2aa9a2bc5600e9c9 Mon Sep 17 00:00:00 2001 From: Christian Heim Date: Sun, 11 Oct 2026 01:07:07 +0200 Subject: [PATCH 204/633] Enable storage retention for alpine Packages are stored as pkg:apk/alpine/{name} whichever configured repository they came from, so a package rule covers the package in all of them. Rules may leave out the alpine namespace (pkg:apk/curl); another namespace could never match and is refused. The docs row warns that Alpine mirrors only keep the current build, so an evicted older one can't come back. The mirror can't fetch apk packages, so there's no mirror test. Refs #306 --- docs/configuration.md | 2 +- internal/handler/apk.go | 21 +++ internal/handler/apk_test.go | 30 ++++ internal/server/retention_alpine_test.go | 195 +++++++++++++++++++++++ 4 files changed, 247 insertions(+), 1 deletion(-) create mode 100644 internal/server/retention_alpine_test.go diff --git a/docs/configuration.md b/docs/configuration.md index 6b17452..d4c2148 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -88,7 +88,7 @@ A package rule wins over an ecosystem rule, which wins over `default`. Package r | Ecosystem key | Retention supported | Package rule form | |---------------|---------------------|-------------------| -| `alpine` | no | - | +| `alpine` | yes | `pkg:apk/curl` or `pkg:apk/alpine/curl`; case doesn't matter. A rule covers the package in every configured APK repository. Alpine mirrors keep only the current build of each package per release, so an evicted older one can't be fetched again. | | `cargo` | no | - | | `composer` | no | - | | `conan` | no | - | diff --git a/internal/handler/apk.go b/internal/handler/apk.go index e0b4b31..6e18e74 100644 --- a/internal/handler/apk.go +++ b/internal/handler/apk.go @@ -6,6 +6,9 @@ import ( "net/http" "regexp" "strings" + + "github.com/git-pkgs/proxy/internal/retention" + "github.com/git-pkgs/purl" ) const ( @@ -18,6 +21,24 @@ const ( apkMatchCount = 3 // full match + name + version ) +// Packages are stored as pkg:apk/alpine/{name}, the namespace purl gives +// every apk package, whichever configured repository they came from. Package +// rules may leave the namespace out. +func init() { + retention.Default.Register(retention.Spec{Key: apkEcosystem, CanonicalPackage: apkRetentionKey}) +} + +// apkRetentionKey maps pkg:apk/{name} and pkg:apk/alpine/{name} to the stored +// package PURL. Another namespace, or a name with a slash, can never match a +// stored package and is refused. +func apkRetentionKey(p *purl.PURL) (string, bool) { + if p == nil || p.Type != "apk" || (p.Namespace != "" && p.Namespace != "alpine") || strings.Contains(p.Name, "/") { + return "", false + } + canonical := purl.MakePURLString(apkEcosystem, p.Name, "") + return canonical, canonical != "" +} + // APKHandler handles Alpine APK repository protocol requests. Each configured // upstream repository is mounted at /apk/{repository}/ and the remaining path // mirrors the upstream layout ({release}/{repo}/{arch}/{file}). diff --git a/internal/handler/apk_test.go b/internal/handler/apk_test.go index 77e1da5..61c874e 100644 --- a/internal/handler/apk_test.go +++ b/internal/handler/apk_test.go @@ -8,6 +8,7 @@ import ( "testing" "time" + "github.com/git-pkgs/purl" "github.com/git-pkgs/registries/fetch" ) @@ -360,3 +361,32 @@ func serveAPKRequest(h *APKHandler, target string) *httptest.ResponseRecorder { h.Routes().ServeHTTP(w, httptest.NewRequest(http.MethodGet, target, nil)) return w } + +func TestAPKRetentionKey(t *testing.T) { + tests := []struct { + key string + want string + ok bool + }{ + {"pkg:apk/curl", "pkg:apk/alpine/curl", true}, + {"pkg:apk/alpine/curl", "pkg:apk/alpine/curl", true}, + {"pkg:apk/Alpine/Curl", "pkg:apk/alpine/curl", true}, + {"pkg:apk/alpine/libstdc++", "pkg:apk/alpine/libstdc%2B%2B", true}, + {"pkg:apk/wolfi/curl", "", false}, + {"pkg:apk/alpine/main%2Fcurl", "", false}, + {"pkg:deb/alpine/curl", "", false}, + } + for _, tt := range tests { + p, err := purl.Parse(tt.key) + if err != nil { + t.Fatalf("Parse(%s): %v", tt.key, err) + } + got, ok := apkRetentionKey(p) + if ok != tt.ok || (ok && got != tt.want) { + t.Errorf("apkRetentionKey(%s) = %q, %v, want %q, %v", tt.key, got, ok, tt.want, tt.ok) + } + } + if _, ok := apkRetentionKey(nil); ok { + t.Error("apkRetentionKey(nil) accepted") + } +} diff --git a/internal/server/retention_alpine_test.go b/internal/server/retention_alpine_test.go new file mode 100644 index 0000000..4118b44 --- /dev/null +++ b/internal/server/retention_alpine_test.go @@ -0,0 +1,195 @@ +package server + +import ( + "context" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/git-pkgs/proxy/internal/config" + "github.com/git-pkgs/proxy/internal/database" + "github.com/git-pkgs/proxy/internal/handler" + "github.com/git-pkgs/proxy/internal/retention" + "github.com/git-pkgs/registries/fetch" +) + +const ( + alpineCurlPath = "/v3.22/main/x86_64/curl-8.14.1-r1.apk" + alpineBusyboxPath = "/v3.22/main/x86_64/busybox-1.37.0-r12.apk" +) + +// alpineRetentionUpstream serves package files the way an Alpine mirror +// does, counting downloads. +func alpineRetentionUpstream(t *testing.T) (*httptest.Server, *atomic.Int64) { + t.Helper() + var downloads atomic.Int64 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case alpineCurlPath, alpineBusyboxPath: + downloads.Add(1) + _, _ = io.WriteString(w, "apk bytes") + default: + http.NotFound(w, r) + } + })) + t.Cleanup(srv.Close) + return srv, &downloads +} + +// newAlpineRetentionProxy mounts the upstream twice, as the default "alpine" +// repository and as a named "edge" repository. +func newAlpineRetentionProxy(t *testing.T, upstream *httptest.Server) (*httptest.Server, *database.DB) { + t.Helper() + db, store := setupEvictionTest(t) + // The upstream is a plain-HTTP loopback server; the default fetcher + // refuses loopback, a stock client reaches it. + fetcher := fetch.NewFetcher(fetch.WithHTTPClient(upstream.Client()), fetch.WithMaxRetries(0)) + t.Cleanup(func() { _ = fetcher.Close() }) + proxy := handler.NewProxy(db, store, fetcher, fetch.NewResolver(), discardLogger()) + proxy.HTTPClient = upstream.Client() + h := handler.NewAPKHandler(proxy, "http://proxy.test", map[string]string{"alpine": upstream.URL, "edge": upstream.URL}) + srv := httptest.NewServer(h.Routes()) + t.Cleanup(srv.Close) + return srv, db +} + +func alpineGet(t *testing.T, url string) { + t.Helper() + resp, err := http.Get(url) + if err != nil { + t.Fatalf("GET %s: %v", url, err) + } + body, _ := io.ReadAll(resp.Body) + _ = resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Fatalf("GET %s = %d %s, want 200", url, resp.StatusCode, body) + } +} + +// alpineArtifact returns the record of the file cached under filename, +// which for apk is the repository-qualified request path. +func alpineArtifact(t *testing.T, db *database.DB, filename string) database.Artifact { + t.Helper() + var art database.Artifact + if err := db.Get(&art, db.Rebind(`SELECT * FROM artifacts WHERE filename = ?`), filename); err != nil { + t.Fatalf("reading artifact %s: %v", filename, err) + } + return art +} + +func alpineBackdate(t *testing.T, db *database.DB, filename string, fetched, accessed time.Time) { + t.Helper() + query := db.Rebind(`UPDATE artifacts SET fetched_at = ?, last_accessed_at = ? WHERE filename = ?`) + if _, err := db.Exec(query, fetched, accessed, filename); err != nil { + t.Fatalf("backdating %s: %v", filename, err) + } +} + +func TestAlpineRegistersForRetention(t *testing.T) { + if _, ok := retention.Default.Lookup("alpine"); !ok { + t.Fatal("alpine is not registered for retention") + } + for _, value := range []string{"alpine", "apk"} { + if key, ok := retention.Default.KeyForDBEcosystem(value); !ok || key != "alpine" { + t.Errorf("KeyForDBEcosystem(%s) = %q, %v, want alpine, true", value, key, ok) + } + } +} + +// A package served from the cache stays while it is used, is evicted once +// its rule runs out, comes back from upstream on the next download, and +// stays after that even with the evicted copy's old access time on the +// record. +func TestAlpineRetentionEvictsAndRefetches(t *testing.T) { + upstream, downloads := alpineRetentionUpstream(t) + srv, db := newAlpineRetentionProxy(t, upstream) + const file = "alpine" + alpineCurlPath + + alpineGet(t, srv.URL+"/alpine"+alpineCurlPath) + alpineGet(t, srv.URL+"/alpine"+alpineCurlPath) + if got := downloads.Load(); got != 1 { + t.Fatalf("upstream downloads = %d, want 1", got) + } + if !alpineArtifact(t, db, file).LastAccessedAt.Valid { + t.Fatal("cache hit did not record an access") + } + + rules := retention.Rules{Ecosystems: map[string]time.Duration{"alpine": retentionDay}} + now := time.Now() + sweepRetention(context.Background(), db, discardLogger(), retention.Default, rules, now) + if !alpineArtifact(t, db, file).StoragePath.Valid { + t.Fatal("package used today was evicted") + } + + old := now.Add(-2 * retentionDay) + alpineBackdate(t, db, file, old, old) + sweepRetention(context.Background(), db, discardLogger(), retention.Default, rules, now) + if alpineArtifact(t, db, file).StoragePath.Valid { + t.Fatal("package unused for two days kept under a one-day rule") + } + + alpineGet(t, srv.URL+"/alpine"+alpineCurlPath) + if got := downloads.Load(); got != 2 { + t.Errorf("upstream downloads = %d, want 2 after the eviction", got) + } + art := alpineArtifact(t, db, file) + alpineBackdate(t, db, file, art.FetchedAt.Time, now.Add(-30*retentionDay)) + sweepRetention(context.Background(), db, discardLogger(), retention.Default, rules, time.Now()) + if !alpineArtifact(t, db, file).StoragePath.Valid { + t.Error("refetched package evicted again by the next sweep") + } +} + +// A key in another namespace parses as an apk PURL but can never match a +// stored package, so it is refused at validation. Both documented forms are +// accepted. +func TestAlpineRetentionRefusesOtherNamespaces(t *testing.T) { + cfg := config.Default() + cfg.Storage.Retention.Packages = map[string]string{"pkg:apk/curl": "0", "pkg:apk/alpine/busybox": "0"} + if _, err := cfg.RetentionRules(retention.Default); err != nil { + t.Fatalf("RetentionRules: %v", err) + } + cfg.Storage.Retention.Packages = map[string]string{"pkg:apk/wolfi/curl": "0"} + _, err := cfg.RetentionRules(retention.Default) + if err == nil || !strings.Contains(err.Error(), "does not match how alpine packages are stored") { + t.Errorf("RetentionRules(pkg:apk/wolfi/curl) error = %v, want the key refused", err) + } +} + +// A package rule reaches the package in every configured repository, in +// either documented form. +func TestAlpineRetentionPackageRules(t *testing.T) { + upstream, _ := alpineRetentionUpstream(t) + srv, db := newAlpineRetentionProxy(t, upstream) + files := map[string]bool{ + "alpine" + alpineCurlPath: true, + "edge" + alpineCurlPath: true, + "alpine" + alpineBusyboxPath: false, + } + + for _, key := range []string{"pkg:apk/curl", "pkg:apk/alpine/curl"} { + old := time.Now().Add(-30 * retentionDay) + for file := range files { + alpineGet(t, srv.URL+"/"+file) + alpineBackdate(t, db, file, old, old) + } + cfg := config.Default() + cfg.Storage.Retention.Ecosystems = map[string]string{"alpine": "1d"} + cfg.Storage.Retention.Packages = map[string]string{key: "0"} + rules, err := cfg.RetentionRules(retention.Default) + if err != nil { + t.Fatalf("RetentionRules(%s): %v", key, err) + } + sweepRetention(context.Background(), db, discardLogger(), retention.Default, rules, time.Now()) + + for file, want := range files { + if got := alpineArtifact(t, db, file).StoragePath.Valid; got != want { + t.Errorf("with %s: %s cached = %v, want %v", key, file, got, want) + } + } + } +} From 42b4303b716e5026ae8e3674b70046bc6f243a76 Mon Sep 17 00:00:00 2001 From: Christian Heim Date: Sun, 11 Oct 2026 01:50:16 +0200 Subject: [PATCH 205/633] Enable storage retention for cran Source, archived and binary packages are stored under pkg:cran/{name}, the name from the filename with its case kept, and purl keeps cran names as they are. Package rules use the default mapping and match the name as published, so pkg:cran/rcpp doesn't cover Rcpp. The mirror can't fetch cran packages, so there's no mirror test. Refs #306 --- docs/configuration.md | 2 +- internal/handler/cran.go | 10 ++ internal/server/retention_cran_test.go | 198 +++++++++++++++++++++++++ 3 files changed, 209 insertions(+), 1 deletion(-) create mode 100644 internal/server/retention_cran_test.go diff --git a/docs/configuration.md b/docs/configuration.md index 6b17452..b710a2e 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -93,7 +93,7 @@ A package rule wins over an ecosystem rule, which wins over `default`. Package r | `composer` | no | - | | `conan` | no | - | | `conda` | no | - | -| `cran` | no | - | +| `cran` | yes | `pkg:cran/ggplot2`, the package name as published, case included (`pkg:cran/Rcpp`). A rule covers source and binary builds alike. | | `deb` | no | - | | `gem` | no | - | | `golang` | no | - | diff --git a/internal/handler/cran.go b/internal/handler/cran.go index 6a363a4..d0dbd52 100644 --- a/internal/handler/cran.go +++ b/internal/handler/cran.go @@ -3,12 +3,22 @@ package handler import ( "net/http" "strings" + + "github.com/git-pkgs/proxy/internal/retention" ) const ( cranUpstream = "https://cloud.r-project.org" ) +// Source and binary packages are stored under pkg:cran/{name}, the name from +// the filename with its case kept, whether they came from src/contrib, its +// Archive or a binary tree. purl keeps cran names as they are, so package +// rules use the default key mapping. +func init() { + retention.Default.Register(retention.Spec{Key: "cran", CanonicalPackage: retention.DefaultCanonical("cran")}) +} + // CRANHandler handles CRAN (R) registry protocol requests. type CRANHandler struct { proxy *Proxy diff --git a/internal/server/retention_cran_test.go b/internal/server/retention_cran_test.go new file mode 100644 index 0000000..afed21e --- /dev/null +++ b/internal/server/retention_cran_test.go @@ -0,0 +1,198 @@ +package server + +import ( + "context" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/git-pkgs/proxy/internal/config" + "github.com/git-pkgs/proxy/internal/database" + "github.com/git-pkgs/proxy/internal/handler" + "github.com/git-pkgs/proxy/internal/retention" + "github.com/git-pkgs/registries/fetch" +) + +const ( + cranRcppSource = "/src/contrib/Rcpp_1.0.13.tar.gz" + cranRcppArchive = "/src/contrib/Archive/Rcpp/Rcpp_1.0.12.tar.gz" + cranRcppBinary = "/bin/windows/contrib/4.4/Rcpp_1.0.13.zip" + cranGgplotSource = "/src/contrib/ggplot2_3.5.1.tar.gz" +) + +// cranRetentionUpstream serves source, archived and binary packages the way +// a CRAN mirror does, counting downloads. +func cranRetentionUpstream(t *testing.T) (*httptest.Server, *atomic.Int64) { + t.Helper() + var downloads atomic.Int64 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case cranRcppSource, cranRcppArchive, cranRcppBinary, cranGgplotSource: + downloads.Add(1) + _, _ = io.WriteString(w, "cran package bytes") + default: + http.NotFound(w, r) + } + })) + t.Cleanup(srv.Close) + return srv, &downloads +} + +func newCRANRetentionProxy(t *testing.T, upstream *httptest.Server) (*httptest.Server, *database.DB) { + t.Helper() + db, store := setupEvictionTest(t) + // The upstream is a plain-HTTP loopback server; the default fetcher + // refuses loopback, a stock client reaches it. + fetcher := fetch.NewFetcher(fetch.WithHTTPClient(upstream.Client()), fetch.WithMaxRetries(0)) + t.Cleanup(func() { _ = fetcher.Close() }) + proxy := handler.NewProxy(db, store, fetcher, fetch.NewResolver(), discardLogger()) + proxy.HTTPClient = upstream.Client() + h := handler.NewCRANHandlerWithUpstream(proxy, "http://proxy.test", upstream.URL) + srv := httptest.NewServer(h.Routes()) + t.Cleanup(srv.Close) + return srv, db +} + +func cranGet(t *testing.T, url string) { + t.Helper() + resp, err := http.Get(url) + if err != nil { + t.Fatalf("GET %s: %v", url, err) + } + body, _ := io.ReadAll(resp.Body) + _ = resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Fatalf("GET %s = %d %s, want 200", url, resp.StatusCode, body) + } +} + +// cranArtifact returns the record of the file named after the last path +// segment of path. +func cranArtifact(t *testing.T, db *database.DB, path string) database.Artifact { + t.Helper() + filename := path[strings.LastIndex(path, "/")+1:] + var art database.Artifact + if err := db.Get(&art, db.Rebind(`SELECT * FROM artifacts WHERE filename = ?`), filename); err != nil { + t.Fatalf("reading artifact %s: %v", filename, err) + } + return art +} + +func cranBackdate(t *testing.T, db *database.DB, path string, fetched, accessed time.Time) { + t.Helper() + filename := path[strings.LastIndex(path, "/")+1:] + query := db.Rebind(`UPDATE artifacts SET fetched_at = ?, last_accessed_at = ? WHERE filename = ?`) + if _, err := db.Exec(query, fetched, accessed, filename); err != nil { + t.Fatalf("backdating %s: %v", filename, err) + } +} + +func TestCRANRegistersForRetention(t *testing.T) { + if _, ok := retention.Default.Lookup("cran"); !ok { + t.Fatal("cran is not registered for retention") + } + if key, ok := retention.Default.KeyForDBEcosystem("cran"); !ok || key != "cran" { + t.Errorf("KeyForDBEcosystem(cran) = %q, %v, want cran, true", key, ok) + } +} + +// A package served from the cache stays while it is used, is evicted once +// its rule runs out, comes back from upstream on the next download, and +// stays after that even with the evicted copy's old access time on the +// record. +func TestCRANRetentionEvictsAndRefetches(t *testing.T) { + upstream, downloads := cranRetentionUpstream(t) + srv, db := newCRANRetentionProxy(t, upstream) + + cranGet(t, srv.URL+cranGgplotSource) + cranGet(t, srv.URL+cranGgplotSource) + if got := downloads.Load(); got != 1 { + t.Fatalf("upstream downloads = %d, want 1", got) + } + if !cranArtifact(t, db, cranGgplotSource).LastAccessedAt.Valid { + t.Fatal("cache hit did not record an access") + } + + rules := retention.Rules{Ecosystems: map[string]time.Duration{"cran": retentionDay}} + now := time.Now() + sweepRetention(context.Background(), db, discardLogger(), retention.Default, rules, now) + if !cranArtifact(t, db, cranGgplotSource).StoragePath.Valid { + t.Fatal("package used today was evicted") + } + + old := now.Add(-2 * retentionDay) + cranBackdate(t, db, cranGgplotSource, old, old) + sweepRetention(context.Background(), db, discardLogger(), retention.Default, rules, now) + if cranArtifact(t, db, cranGgplotSource).StoragePath.Valid { + t.Fatal("package unused for two days kept under a one-day rule") + } + + cranGet(t, srv.URL+cranGgplotSource) + if got := downloads.Load(); got != 2 { + t.Errorf("upstream downloads = %d, want 2 after the eviction", got) + } + art := cranArtifact(t, db, cranGgplotSource) + cranBackdate(t, db, cranGgplotSource, art.FetchedAt.Time, now.Add(-30*retentionDay)) + sweepRetention(context.Background(), db, discardLogger(), retention.Default, rules, time.Now()) + if !cranArtifact(t, db, cranGgplotSource).StoragePath.Valid { + t.Error("refetched package evicted again by the next sweep") + } +} + +// A key with a namespace parses as a cran PURL but can never match a stored +// package, so it is refused at validation. A plain key is accepted. +func TestCRANRetentionRefusesNamespacedKeys(t *testing.T) { + cfg := config.Default() + cfg.Storage.Retention.Packages = map[string]string{"pkg:cran/Rcpp": "0"} + if _, err := cfg.RetentionRules(retention.Default); err != nil { + t.Fatalf("RetentionRules(pkg:cran/Rcpp): %v", err) + } + cfg.Storage.Retention.Packages = map[string]string{"pkg:cran/contrib/Rcpp": "0"} + _, err := cfg.RetentionRules(retention.Default) + if err == nil || !strings.Contains(err.Error(), "does not match how cran packages are stored") { + t.Errorf("RetentionRules(pkg:cran/contrib/Rcpp) error = %v, want the key refused", err) + } +} + +// A package rule reaches the package's current source, archived source and +// binary builds, and only under its published case. +func TestCRANRetentionPackageRules(t *testing.T) { + upstream, _ := cranRetentionUpstream(t) + srv, db := newCRANRetentionProxy(t, upstream) + rcpp := []string{cranRcppSource, cranRcppArchive, cranRcppBinary} + + for _, tt := range []struct { + key string + keepRcpp bool + }{ + {"pkg:cran/Rcpp", true}, + {"pkg:cran/rcpp", false}, + } { + old := time.Now().Add(-30 * retentionDay) + for _, path := range append([]string{cranGgplotSource}, rcpp...) { + cranGet(t, srv.URL+path) + cranBackdate(t, db, path, old, old) + } + cfg := config.Default() + cfg.Storage.Retention.Ecosystems = map[string]string{"cran": "1d"} + cfg.Storage.Retention.Packages = map[string]string{tt.key: "0"} + rules, err := cfg.RetentionRules(retention.Default) + if err != nil { + t.Fatalf("RetentionRules(%s): %v", tt.key, err) + } + sweepRetention(context.Background(), db, discardLogger(), retention.Default, rules, time.Now()) + + for _, path := range rcpp { + if got := cranArtifact(t, db, path).StoragePath.Valid; got != tt.keepRcpp { + t.Errorf("with %s: %s cached = %v, want %v", tt.key, path, got, tt.keepRcpp) + } + } + if cranArtifact(t, db, cranGgplotSource).StoragePath.Valid { + t.Errorf("with %s: ggplot2 kept past the cran rule", tt.key) + } + } +} From 5b9a5fca6bf9441f00725d53d460dad7470c52a5 Mon Sep 17 00:00:00 2001 From: Christian Heim Date: Mon, 12 Oct 2026 10:29:12 +0200 Subject: [PATCH 206/633] When the shared tag entry holds a representation the client does not accept, the lookup now goes on to the client's own per-Accept entry instead of giving up, so a Docker-only client still gets its cached manifest list after an OCI index was cached for the same tag. --- internal/handler/container_manifest.go | 22 +++++++------- .../handler/container_manifest_key_test.go | 29 +++++++++++++++++++ 2 files changed, 40 insertions(+), 11 deletions(-) diff --git a/internal/handler/container_manifest.go b/internal/handler/container_manifest.go index 4ff750e..9d8921b 100644 --- a/internal/handler/container_manifest.go +++ b/internal/handler/container_manifest.go @@ -191,12 +191,13 @@ func (h *ContainerHandler) loadContainerManifestForAccept(ctx context.Context, r h.proxy.Logger.Warn("failed to read cached container manifest", "error", err) return nil } - if cached != nil { - if containerManifestCacheCompatible(accept, cached) { - return cached - } - return nil + if cached != nil && containerManifestCacheCompatible(accept, cached) { + return cached } + // The shared entry may hold a representation this client does not accept, + // for example an OCI index for a client that only accepts Docker manifest + // lists. Its own per-Accept entry is still written, so look there. + sharedEntryExists := cached != nil for _, legacyCacheKey := range h.containerManifestLegacyCacheKeys(registryURL, name, reference, accept, cacheAccept, cacheKey) { cached, err = h.loadContainerManifest(ctx, legacyCacheKey) @@ -204,14 +205,13 @@ func (h *ContainerHandler) loadContainerManifestForAccept(ctx context.Context, r h.proxy.Logger.Warn("failed to read legacy cached container manifest", "error", err) return nil } - if cached == nil { + if cached == nil || !containerManifestCacheCompatible(accept, cached) { continue } - if !containerManifestCacheCompatible(accept, cached) { - return nil - } - if err := h.storeContainerManifest(ctx, cacheKey, cached); err != nil { - h.proxy.Logger.Warn("failed to migrate cached container manifest", "error", err) + if !sharedEntryExists { + if err := h.storeContainerManifest(ctx, cacheKey, cached); err != nil { + h.proxy.Logger.Warn("failed to migrate cached container manifest", "error", err) + } } return cached } diff --git a/internal/handler/container_manifest_key_test.go b/internal/handler/container_manifest_key_test.go index e339ced..62ceac8 100644 --- a/internal/handler/container_manifest_key_test.go +++ b/internal/handler/container_manifest_key_test.go @@ -163,3 +163,32 @@ func TestContainerHandler_IndexlessClientKeepsOwnTagEntry(t *testing.T) { t.Errorf("offline index-less body = %q, want the converted manifest", response.Body.String()) } } + +func TestContainerHandler_IncompatibleSharedEntryFallsBackToAcceptEntry(t *testing.T) { + list := `{"schemaVersion":2,"mediaType":"application/vnd.docker.distribution.manifest.list.v2+json","manifests":[]}` + index := `{"schemaVersion":2,"mediaType":"application/vnd.oci.image.index.v1+json","manifests":[]}` + registry := newFakeManifestRegistry(t) + registry.set("/v2/library/app/manifests/latest", dockerManifestListMediaType, list) + h := newManifestKeyTestHandler(t, registry) + + const dockerOnly = "application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.docker.distribution.manifest.v2+json" + if response := getManifest(t, h, "/library/app/manifests/latest", dockerOnly); response.Body.String() != list { + t.Fatalf("Docker-only pull body = %q, want the manifest list", response.Body.String()) + } + registry.set("/v2/library/app/manifests/latest", ociImageIndexMediaType, index) + if response := getManifest(t, h, "/library/app/manifests/latest", ociImageIndexMediaType); response.Body.String() != index { + t.Fatalf("OCI pull body = %q, want the index", response.Body.String()) + } + registry.offline.Store(true) + + response := getManifest(t, h, "/library/app/manifests/latest", dockerOnly) + if response.Code != http.StatusOK { + t.Fatalf("offline Docker-only pull status = %d, want 200: %s", response.Code, response.Body.String()) + } + if response.Body.String() != list { + t.Errorf("offline Docker-only body = %q, want the manifest list", response.Body.String()) + } + if response := getManifest(t, h, "/library/app/manifests/latest", ociImageIndexMediaType); response.Body.String() != index { + t.Errorf("offline OCI body = %q, want the index still in the shared entry", response.Body.String()) + } +} From 67fd1ad6c1ee128455cd7a770c8e1f45c0a4d024 Mon Sep 17 00:00:00 2001 From: Christian Heim Date: Mon, 12 Oct 2026 10:36:51 +0200 Subject: [PATCH 207/633] The mirror now resolves a tag without caching it, mirrors the image it points to, and stores the tag only once that image is complete, so a failed or interrupted mirror keeps the previously cached tag instead of pointing it at an image with missing layers. --- internal/handler/container_mirror.go | 87 ++++++++++++++++++++++- internal/handler/container_mirror_test.go | 38 ++++++++++ 2 files changed, 124 insertions(+), 1 deletion(-) diff --git a/internal/handler/container_mirror.go b/internal/handler/container_mirror.go index f9e3d49..aa0c764 100644 --- a/internal/handler/container_mirror.go +++ b/internal/handler/container_mirror.go @@ -11,6 +11,7 @@ import ( "regexp" "slices" "strings" + "time" ) const ( @@ -160,7 +161,11 @@ func (h *ContainerHandler) MirrorImage(ctx context.Context, ref ImageRef, platfo } var digests []string for _, reference := range ref.References { - digest, err := m.mirrorManifest(ctx, reference, 0) + mirror := m.mirrorTag + if mirrorDigestPattern.MatchString(reference) { + mirror = func(ctx context.Context, digest string) (string, error) { return m.mirrorManifest(ctx, digest, 0) } + } + digest, err := mirror(ctx, reference) if err != nil { return m.result, fmt.Errorf("%s:%s: %w", ref.Name, reference, err) } @@ -197,6 +202,86 @@ func normalizeImageRef(ref ImageRef, defaultRegistryURL string) (ImageRef, error return ref, nil } +// errTagUpstreamUnavailable marks tag lookups that failed because the +// upstream could not be reached or answered with 429 or 5xx. +var errTagUpstreamUnavailable = errors.New("upstream unavailable") + +// mirrorTag resolves a tag upstream without caching it, mirrors the manifest +// it names by digest, and stores the tag only after that manifest and +// everything it references are cached. A failed mirror leaves the previously +// cached tag as it was. If the upstream is unreachable, the cached tag is +// mirrored as it is. +func (m *imageMirror) mirrorTag(ctx context.Context, tag string) (string, error) { + accept := containerManifestAccept(&http.Request{Header: http.Header{}}) + cacheAccept := normalizeContainerManifestAccept(accept) + before := m.h.loadContainerManifestForAccept(ctx, m.registryURL, m.upstreamName, tag, accept, cacheAccept) + + manifest, err := m.fetchTagManifest(ctx, tag, accept) + if err != nil { + if before == nil || !errors.Is(err, errTagUpstreamUnavailable) { + return "", fmt.Errorf("manifest %s: %w", tag, err) + } + m.result.Warnings = append(m.result.Warnings, fmt.Sprintf("manifest %s served from stale cache, upstream unreachable", tag)) + _, err := m.mirrorManifest(ctx, before.contentDigest, 0) + return before.contentDigest, err + } + + // The digest entry is immutable, so it can be stored right away; the + // walk below then reads it from cache instead of fetching it again. + digest := manifest.contentDigest + if before == nil || before.contentDigest != digest { + m.result.Fetched++ + } + m.h.storeContainerManifestForAccept(ctx, m.registryURL, m.upstreamName, digest, accept, cacheAccept, manifest) + if _, err := m.mirrorManifest(ctx, digest, 0); err != nil { + return "", err + } + m.h.storeContainerManifestForAccept(ctx, m.registryURL, m.upstreamName, tag, accept, cacheAccept, manifest) + return digest, nil +} + +// fetchTagManifest fetches and verifies the manifest a tag points to without +// caching it. +func (m *imageMirror) fetchTagManifest(ctx context.Context, tag, accept string) (*cachedContainerManifest, error) { + upstreamURL := fmt.Sprintf("%s/v2/%s/manifests/%s", m.registryURL, m.upstreamName, tag) + request, err := http.NewRequestWithContext(ctx, http.MethodGet, upstreamURL, nil) + if err != nil { + return nil, err + } + request.Header.Set("Accept", accept) + response, err := m.h.proxy.HTTPClient.Do(request) + if err != nil { + if ctx.Err() != nil { + return nil, ctx.Err() + } + return nil, fmt.Errorf("%w: %w", errTagUpstreamUnavailable, err) + } + defer func() { _ = response.Body.Close() }() + if response.StatusCode != http.StatusOK { + if shouldServeStaleManifest(response.StatusCode) { + return nil, fmt.Errorf("%w: status %d", errTagUpstreamUnavailable, response.StatusCode) + } + return nil, fmt.Errorf("upstream status %d", response.StatusCode) + } + body, err := m.h.proxy.ReadMetadata(response.Body) + if err != nil { + return nil, fmt.Errorf("reading manifest: %w", err) + } + digest := sha256Digest(body) + if contentDigest := response.Header.Get("Docker-Content-Digest"); strings.HasPrefix(contentDigest, "sha256:") && contentDigest != digest { + return nil, fmt.Errorf("digest verification failed: upstream says %s, body is %s", contentDigest, digest) + } + return &cachedContainerManifest{ + body: body, + contentType: response.Header.Get(headerContentType), + contentDigest: digest, + etag: response.Header.Get(headerETag), + size: int64(len(body)), + lastModified: parseHTTPTime(response.Header.Get(headerLastModified)), + fetchedAt: time.Now(), + }, nil +} + // mirrorManifest stores the manifest for reference through the handler's own // manifest route, then follows what it references. It returns the manifest // digest. diff --git a/internal/handler/container_mirror_test.go b/internal/handler/container_mirror_test.go index fc4e8a6..ebd32b6 100644 --- a/internal/handler/container_mirror_test.go +++ b/internal/handler/container_mirror_test.go @@ -293,3 +293,41 @@ func TestNormalizeImageRef(t *testing.T) { }) } } + +func TestMirrorImageKeepsPreviousTagWhenMovedTagIsIncomplete(t *testing.T) { + registry, image := newFakeImageRegistry(t) + proxy := newMirrorTestProxy(t, registry) + h := NewContainerHandlerWithRegistry(proxy, "http://proxy.example", registry.server.URL) + ref := ImageRef{Name: "team/app", References: []string{"single"}} + + if _, err := h.MirrorImage(context.Background(), ref, nil); err != nil { + t.Fatalf("first MirrorImage: %v", err) + } + previous := registry.manifests["/v2/team/app/manifests/single"] + + // Move the tag to an image whose layer the registry cannot serve. + broken, err := json.Marshal(map[string]any{ + "schemaVersion": 2, "mediaType": ociManifestMediaType, + "config": map[string]any{"mediaType": "application/vnd.oci.image.config.v1+json", "digest": image.singleLayer}, + "layers": []map[string]any{{"mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": sha256Digest([]byte("missing layer"))}}, + }) + if err != nil { + t.Fatal(err) + } + brokenDoc := fakeManifest{contentType: ociManifestMediaType, body: string(broken)} + registry.manifests["/v2/team/app/manifests/single"] = brokenDoc + registry.manifests["/v2/team/app/manifests/"+sha256Digest(broken)] = brokenDoc + + if _, err := h.MirrorImage(context.Background(), ref, nil); err == nil { + t.Fatal("mirroring the incomplete image did not fail") + } + registry.offline.Store(true) + + response := pull(t, h, "/team/app/manifests/single", dockerManifestAccept) + if response.Code != http.StatusOK { + t.Fatalf("offline tag status = %d: %s", response.Code, response.Body.String()) + } + if response.Body.String() != previous.body { + t.Errorf("offline tag points to %s, want the previously mirrored image", sha256Digest(response.Body.Bytes())) + } +} From 086abebe6eff924276cec5e5ab55fb2250bb1a72 Mon Sep 17 00:00:00 2001 From: Christian Heim Date: Mon, 12 Oct 2026 10:59:22 +0200 Subject: [PATCH 208/633] A database refresh now fetches the signature as well and stores it for that exact database content, keyed by its digest. A signature requested later, after the upstream has moved on, is still the one for the database the client got, and a refresh that returns the same bytes keeps the cached signature, so it is served offline too. The database is fetched once more after its signature to catch a mirror that swapped both in between. A missing signature is remembered per database, so official repos answer 404 without going upstream. --- README.md | 6 +- internal/handler/arch.go | 210 +++++++++++++++++++++++++++++----- internal/handler/arch_test.go | 134 +++++++++++++++++----- 3 files changed, 291 insertions(+), 59 deletions(-) diff --git a/README.md b/README.md index ad47805..2f839cc 100644 --- a/README.md +++ b/README.md @@ -684,9 +684,9 @@ pacman -Syu Repository databases (`core.db`, `core.files` and their `.tar.*` forms), database signatures, packages and package signatures are served byte-for-byte unchanged, so pacman's signature checks keep working. Databases use the -metadata cache (`metadata_ttl`, stale fallback); a database signature is -refreshed together with its database rather than on its own TTL, and a -signature that belongs to an older database is never served in its place. Enable `cache_metadata` (or set `PROXY_CACHE_METADATA=true`) so +metadata cache (`metadata_ttl`, stale fallback). Each database refresh also +fetches its signature and stores it for exactly that database, so a signature +is only ever served with the database it signs. Enable `cache_metadata` (or set `PROXY_CACHE_METADATA=true`) so `pacman -Sy` keeps working while the upstream is down; without it databases are fetched fresh on every request. Packages and their `.sig` files are stored in the shared artifact cache and remain available when the upstream is unreachable. diff --git a/internal/handler/arch.go b/internal/handler/arch.go index a38fbc1..f3505a4 100644 --- a/internal/handler/arch.go +++ b/internal/handler/arch.go @@ -8,6 +8,9 @@ import ( "net/http" "regexp" "strings" + + "github.com/git-pkgs/proxy/internal/database" + "github.com/git-pkgs/proxy/internal/metrics" ) const ( @@ -75,7 +78,7 @@ func (h *ArchHandler) Routes() http.Handler { switch { case isArchDatabase(rest): - h.handleMetadata(w, r, repository, upstreamURL, rest) + h.handleDatabase(w, r, repository, upstreamURL, rest) case isArchDatabaseSignature(rest): h.handleDatabaseSignature(w, r, repository, upstreamURL, rest) case isArchPackage(rest) || isArchPackageSignature(rest): @@ -161,51 +164,204 @@ func (h *ArchHandler) handleMetadata(w http.ResponseWriter, r *http.Request, rep h.metadataCacheKey(repository, upstreamURL, path), archAccept) } -// handleDatabaseSignature lets a database's .sig follow the database instead -// of its own TTL, since pacman rejects a mismatched pair. A signature older -// than the cached database is fetched again and never served as a fallback. -func (h *ArchHandler) handleDatabaseSignature(w http.ResponseWriter, r *http.Request, repository, upstreamURL, path string) { +// archDatabase names a database and its signature in the cache. +type archDatabase struct { + dbKey, dbURL string + sigKey, sigURL string +} + +func (h *ArchHandler) database(repository, upstreamURL, path string) archDatabase { + return archDatabase{ + dbKey: h.metadataCacheKey(repository, upstreamURL, path), + dbURL: upstreamURL + "/" + path, + sigKey: h.metadataCacheKey(repository, upstreamURL, path+archSignatureSuffix), + sigURL: upstreamURL + "/" + path + archSignatureSuffix, + } +} + +func (d archDatabase) coalesceKey(kind string) string { + return metadataCoalesceKey(archEcosystem, d.dbKey, d.dbURL, archAccept, "identity", false) + "\x00" + kind +} + +// handleDatabase serves a database. Each refresh fetches the signature too, so +// a signature requested later still matches the database the client got. +func (h *ArchHandler) handleDatabase(w http.ResponseWriter, r *http.Request, repository, upstreamURL, path string) { p := h.proxy - sigKey := h.metadataCacheKey(repository, upstreamURL, path) if !p.CacheMetadata || p.DB == nil { h.handleMetadata(w, r, repository, upstreamURL, path) return } + d := h.database(repository, upstreamURL, path) + if _, hit := p.cachedMetadataState(r.Context(), archEcosystem, d.dbKey, nil); hit != nil { + metrics.RecordCacheHit(archEcosystem) + h.writeMetadata(w, r, d.dbKey, *hit) + return + } + p.recordMetadataCacheMiss(archEcosystem) + res, _ := p.coalesceMetadata(r.Context(), d.coalesceKey("database"), func(ctx context.Context) metadataResult { + entry, hit := p.cachedMetadataState(ctx, archEcosystem, d.dbKey, nil) + if hit != nil { + return *hit + } + res, _ := h.refresh(ctx, d, entry) + return res + }) + h.writeMetadata(w, r, d.dbKey, res) +} - dbKey := h.metadataCacheKey(repository, upstreamURL, strings.TrimSuffix(path, archSignatureSuffix)) - dbEntry, _ := p.DB.GetMetadataCache(archEcosystem, dbKey) - sigEntry, _ := p.DB.GetMetadataCache(archEcosystem, sigKey) - if dbEntry == nil || sigEntry == nil || !dbEntry.FetchedAt.Valid || !sigEntry.FetchedAt.Valid { +// handleDatabaseSignature serves the signature stored for the cached +// database's content, never one fetched for another version of it. +func (h *ArchHandler) handleDatabaseSignature(w http.ResponseWriter, r *http.Request, repository, upstreamURL, path string) { + p := h.proxy + if !p.CacheMetadata || p.DB == nil { h.handleMetadata(w, r, repository, upstreamURL, path) return } - - if !sigEntry.FetchedAt.Time.Before(dbEntry.FetchedAt.Time) { - body, contentType, err := p.readCachedMetadata(r.Context(), sigEntry, nil) - if err == nil { - p.writeMetadataCachedResponseWithEncoding(w, r, archEcosystem, sigKey, body, contentType, sigEntry.ContentEncoding.String) - return - } - // The cached bytes are gone; refetch below. + d := h.database(repository, upstreamURL, strings.TrimSuffix(path, archSignatureSuffix)) + if res, ok := h.cachedSignature(r.Context(), d); ok { + metrics.RecordCacheHit(archEcosystem) + h.writeMetadata(w, r, d.sigKey, res) + return } - - signatureURL := upstreamURL + "/" + path - // Own key: a regular fetch would return the cached copy within its TTL. - key := metadataCoalesceKey(archEcosystem, sigKey, signatureURL, archAccept, "identity", false) + "\x00follow-database" - res, _ := p.coalesceMetadata(r.Context(), key, func(fetchCtx context.Context) metadataResult { - // No entry: a 304 or stale fallback would return the old signature. - return p.fetchMetadataFromUpstream(fetchCtx, archEcosystem, sigKey, signatureURL, archAccept, "identity", nil, nil) + p.recordMetadataCacheMiss(archEcosystem) + // Nothing stored for this database yet, e.g. the signature fetch failed + // during the last refresh: refresh both. + res, _ := p.coalesceMetadata(r.Context(), d.coalesceKey("signature"), func(ctx context.Context) metadataResult { + if res, ok := h.cachedSignature(ctx, d); ok { + return res + } + entry, _ := p.DB.GetMetadataCache(archEcosystem, d.dbKey) + if _, err := h.refresh(ctx, d, entry); err != nil { + return metadataResult{err: err} + } + if res, ok := h.cachedSignature(ctx, d); ok { + return res + } + return metadataResult{err: errArchSignatureUnpaired} }) + h.writeMetadata(w, r, d.sigKey, res) +} + +var errArchSignatureUnpaired = errors.New("database signature could not be paired with the database") + +// archPairAttempts bounds refetches while the mirror replaces the database. +const archPairAttempts = 3 + +// refresh fetches the database, then its signature, and stores the signature +// for the database's content. It returns the database and why no signature +// could be stored, if so. +func (h *ArchHandler) refresh(ctx context.Context, d archDatabase, entry *database.MetadataCacheEntry) (metadataResult, error) { + p := h.proxy + res := p.fetchMetadataFromUpstream(ctx, archEcosystem, d.dbKey, d.dbURL, archAccept, "identity", nil, entry) + if res.err != nil { + return res, res.err + } + for range archPairAttempts { + current, _ := p.DB.GetMetadataCache(archEcosystem, d.dbKey) + if current == nil || !current.ContentDigest.Valid { + return res, errArchSignatureUnpaired + } + dbDigest := current.ContentDigest.String + if _, ok := h.storedSignature(ctx, d, dbDigest); ok { + return res, nil + } + + sig, err := p.fetchUpstreamMetadata(ctx, d.sigURL, nil, archAccept, "identity") + if errors.Is(err, ErrUpstreamNotFound) { + h.storeSignature(ctx, d, dbDigest, nil) + return res, err + } + if err != nil { + return res, err + } + + // The mirror may have replaced the database in the meantime. + again, err := p.fetchUpstreamMetadata(ctx, d.dbURL, current, archAccept, "identity") + if err != nil { + return res, err + } + if archDigest(again.body) == dbDigest { + h.storeSignature(ctx, d, dbDigest, sig) + return res, nil + } + p.cacheMetadataBlob(ctx, archEcosystem, d.dbKey, metadataStoragePath(archEcosystem, d.dbKey), again) + res = metadataResult{body: again.body, contentType: again.contentType, contentEncoding: again.contentEncoding} + } + return res, errArchSignatureUnpaired +} + +// cachedSignature returns the signature stored for the cached database. +func (h *ArchHandler) cachedSignature(ctx context.Context, d archDatabase) (metadataResult, bool) { + current, _ := h.proxy.DB.GetMetadataCache(archEcosystem, d.dbKey) + if current == nil || !current.ContentDigest.Valid { + return metadataResult{}, false + } + return h.storedSignature(ctx, d, current.ContentDigest.String) +} + +// storedSignature returns the signature stored for dbDigest. The storage +// path names the digest, so a refresh with unchanged bytes keeps it. +func (h *ArchHandler) storedSignature(ctx context.Context, d archDatabase, dbDigest string) (metadataResult, bool) { + entry, _ := h.proxy.DB.GetMetadataCache(archEcosystem, d.sigKey) + if entry == nil { + return metadataResult{}, false + } + switch entry.StoragePath { + case archMissingSignaturePath(d, dbDigest): + return metadataResult{err: ErrUpstreamNotFound}, true + case archSignaturePath(d, dbDigest): + body, contentType, err := h.proxy.readCachedMetadata(ctx, entry, nil) + if err != nil { + return metadataResult{}, false + } + return metadataResult{body: body, contentType: contentType, contentEncoding: entry.ContentEncoding.String}, true + } + return metadataResult{}, false +} + +// storeSignature stores sig for dbDigest, or that it is missing when sig is +// nil, and drops the one stored for an earlier database. +func (h *ArchHandler) storeSignature(ctx context.Context, d archDatabase, dbDigest string, sig *upstreamMetadata) { + p := h.proxy + path := archSignaturePath(d, dbDigest) + if sig == nil { + path = archMissingSignaturePath(d, dbDigest) + sig = &upstreamMetadata{contentType: archPackageMediaType} + } + previous, _ := p.DB.GetMetadataCache(archEcosystem, d.sigKey) + p.cacheMetadataBlob(ctx, archEcosystem, d.sigKey, path, sig) + if previous == nil || previous.StoragePath == path { + return + } + if current, _ := p.DB.GetMetadataCache(archEcosystem, d.sigKey); current != nil && current.StoragePath == path { + _ = p.Storage.Delete(ctx, previous.StoragePath) + } +} + +func archSignaturePath(d archDatabase, dbDigest string) string { + return "_metadata/" + archEcosystem + "/" + d.sigKey + "/" + strings.TrimPrefix(dbDigest, "sha256:") +} + +func archMissingSignaturePath(d archDatabase, dbDigest string) string { + return archSignaturePath(d, dbDigest) + "-missing" +} + +func archDigest(body []byte) string { + digest := sha256.Sum256(body) + return "sha256:" + hex.EncodeToString(digest[:]) +} + +func (h *ArchHandler) writeMetadata(w http.ResponseWriter, r *http.Request, cacheKey string, res metadataResult) { if res.err != nil { if errors.Is(res.err, ErrUpstreamNotFound) { http.Error(w, "not found", http.StatusNotFound) return } - p.Logger.Error("metadata fetch failed", "error", res.err) + h.proxy.Logger.Error("metadata fetch failed", "error", res.err) http.Error(w, "failed to fetch from upstream", http.StatusBadGateway) return } - p.writeMetadataCachedResponseWithEncoding(w, r, archEcosystem, sigKey, res.body, res.contentType, res.contentEncoding) + h.proxy.writeMetadataCachedResponseWithEncoding(w, r, archEcosystem, cacheKey, res.body, res.contentType, res.contentEncoding) } // metadataCacheKey hashes repository, upstream URL and path, as APKHandler does. diff --git a/internal/handler/arch_test.go b/internal/handler/arch_test.go index 587ab34..1b71b3f 100644 --- a/internal/handler/arch_test.go +++ b/internal/handler/arch_test.go @@ -101,21 +101,23 @@ func TestArchHandler_UnknownRepositoryReturns404(t *testing.T) { } } -// archUpstream is a fake pacman mirror. Files can be swapped and the mirror -// taken offline; requests are counted per path. +// archUpstream is a fake pacman mirror. Files can be swapped, single paths +// broken and the mirror taken offline; requests are counted per path. type archUpstream struct { *httptest.Server mu sync.Mutex files map[string][]byte modified map[string]time.Time hits map[string]int + broken map[string]bool + served func(path string) // called after a file is served, without mu available atomic.Bool auth atomic.Value } func newArchUpstream(t *testing.T, files map[string][]byte) *archUpstream { t.Helper() - u := &archUpstream{files: files, modified: map[string]time.Time{}, hits: map[string]int{}} + u := &archUpstream{files: files, modified: map[string]time.Time{}, hits: map[string]int{}, broken: map[string]bool{}} u.available.Store(true) u.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if !u.available.Load() { @@ -126,8 +128,14 @@ func newArchUpstream(t *testing.T, files map[string][]byte) *archUpstream { u.mu.Lock() data, ok := u.files[r.URL.Path] modified := u.modified[r.URL.Path] + broken := u.broken[r.URL.Path] + served := u.served u.hits[r.URL.Path]++ u.mu.Unlock() + if broken { + http.Error(w, "broken", http.StatusInternalServerError) + return + } if !ok { http.NotFound(w, r) return @@ -137,6 +145,9 @@ func newArchUpstream(t *testing.T, files map[string][]byte) *archUpstream { } w.Header().Set("Content-Type", "application/octet-stream") _, _ = w.Write(data) + if served != nil { + served(r.URL.Path) + } })) t.Cleanup(u.Close) return u @@ -206,10 +217,14 @@ func TestArchHandler_DatabasesServedUnchanged(t *testing.T) { t.Errorf("Authorization = %q, want %q", got, "Bearer arch-token") } + // core.db is fetched twice on its first request: once more after its + // signature, to make sure the two belong together. + wantHits := map[string]int{"/core/os/x86_64/core.db": 2} for _, path := range paths { assertArchBody(t, serveArchRequest(h, http.MethodGet, "/archlinux"+path, nil), path+" cached", files[path]) - if got := upstream.hitCount(path); got != 1 { - t.Errorf("%s: upstream hits = %d, want 1 (second read from cache)", path, got) + want := max(wantHits[path], 1) + if got := upstream.hitCount(path); got != want { + t.Errorf("%s: upstream hits = %d, want %d (second read from cache)", path, got, want) } } @@ -228,7 +243,7 @@ func TestArchHandler_StaleDatabaseServedWhenUpstreamDown(t *testing.T) { h := NewArchHandler(proxy, "http://proxy.example", map[string]string{"archlinux": upstream.URL}) assertArchBody(t, serveArchRequest(h, http.MethodGet, "/archlinux/core/os/x86_64/core.db", nil), "seed", db) - ageMetadata(t, proxy, h, "archlinux", upstream.URL, "core/os/x86_64/core.db", 2*time.Hour) + ageMetadata(t, proxy, h, "archlinux", upstream.URL, "core/os/x86_64/core.db") upstream.available.Store(false) w := serveArchRequest(h, http.MethodGet, "/archlinux/core/os/x86_64/core.db", nil) @@ -260,8 +275,8 @@ func TestArchHandler_DatabaseIfModifiedSince(t *testing.T) { // TestArchHandler_MissingDatabaseSignature covers the official repositories, // which sign no databases: pacman with DatabaseOptional asks for core.db.sig -// and must get a 404 while the upstream is up. With the upstream down the -// answer is a 502, which pacman also treats as "no signature". +// and must get a 404. The database refresh found it missing, so the answer +// needs no upstream and holds offline too. func TestArchHandler_MissingDatabaseSignature(t *testing.T) { upstream := newArchUpstream(t, map[string][]byte{"/core/os/x86_64/core.db": []byte("core-db-bytes")}) proxy := newArchTestProxy(t, upstream) @@ -271,11 +286,14 @@ func TestArchHandler_MissingDatabaseSignature(t *testing.T) { if w := serveArchRequest(h, http.MethodGet, "/archlinux/core/os/x86_64/core.db.sig", nil); w.Code != http.StatusNotFound { t.Errorf("missing signature: status = %d, want 404", w.Code) } + if got := upstream.hitCount("/core/os/x86_64/core.db.sig"); got != 1 { + t.Errorf("signature upstream hits = %d, want 1 (asked once, with the database)", got) + } upstream.available.Store(false) assertArchBody(t, serveArchRequest(h, http.MethodGet, "/archlinux/core/os/x86_64/core.db", nil), "db offline", []byte("core-db-bytes")) - if w := serveArchRequest(h, http.MethodGet, "/archlinux/core/os/x86_64/core.db.sig", nil); w.Code != http.StatusBadGateway { - t.Errorf("missing signature offline: status = %d, want 502", w.Code) + if w := serveArchRequest(h, http.MethodGet, "/archlinux/core/os/x86_64/core.db.sig", nil); w.Code != http.StatusNotFound { + t.Errorf("missing signature offline: status = %d, want 404", w.Code) } } @@ -302,7 +320,7 @@ func TestArchHandler_DatabaseSignatureFollowsDatabase(t *testing.T) { // the signature does. upstream.set(dbPath, []byte("db v2")) upstream.set(sigPath, []byte("sig v2")) - ageMetadata(t, proxy, h, "custom", upstream.URL, dbPath[1:], 2*time.Hour) + ageMetadata(t, proxy, h, "custom", upstream.URL, dbPath[1:]) assertArchBody(t, serveArchRequest(h, http.MethodGet, "/custom"+dbPath, nil), "db v2", []byte("db v2")) assertArchBody(t, serveArchRequest(h, http.MethodGet, "/custom"+sigPath, nil), "sig after db refresh", []byte("sig v2")) @@ -312,11 +330,11 @@ func TestArchHandler_DatabaseSignatureFollowsDatabase(t *testing.T) { } } -// TestArchHandler_OutdatedSignatureNeverServed covers a signature refetch that -// fails after the database was refreshed. The cached signature belongs to -// the old database, and pacman rejects a present but invalid signature even -// with DatabaseOptional, so the proxy must answer 404 or 502 rather than fall -// back to it. +// TestArchHandler_OutdatedSignatureNeverServed covers a signature fetch that +// fails while the database is refreshed. The cached signature belongs to the +// old database, and pacman rejects a present but invalid signature even with +// DatabaseOptional, so the proxy must answer 404 or 502 rather than fall back +// to it. func TestArchHandler_OutdatedSignatureNeverServed(t *testing.T) { const dbPath, sigPath = "/custom/os/x86_64/custom.db", "/custom/os/x86_64/custom.db.sig" @@ -325,12 +343,8 @@ func TestArchHandler_OutdatedSignatureNeverServed(t *testing.T) { breakSig func(u *archUpstream) want int }{ - {"signature removed upstream", func(u *archUpstream) { - u.mu.Lock() - delete(u.files, sigPath) - u.mu.Unlock() - }, http.StatusNotFound}, - {"upstream down after database refresh", func(u *archUpstream) { u.available.Store(false) }, http.StatusBadGateway}, + {"signature removed upstream", func(u *archUpstream) { delete(u.files, sigPath) }, http.StatusNotFound}, + {"signature fetch fails", func(u *archUpstream) { u.broken[sigPath] = true }, http.StatusBadGateway}, } { t.Run(tt.name, func(t *testing.T) { upstream := newArchUpstream(t, map[string][]byte{dbPath: []byte("db v1"), sigPath: []byte("sig v1")}) @@ -340,11 +354,13 @@ func TestArchHandler_OutdatedSignatureNeverServed(t *testing.T) { assertArchBody(t, serveArchRequest(h, http.MethodGet, "/custom"+dbPath, nil), "db v1", []byte("db v1")) assertArchBody(t, serveArchRequest(h, http.MethodGet, "/custom"+sigPath, nil), "sig v1", []byte("sig v1")) - upstream.set(dbPath, []byte("db v2")) - ageMetadata(t, proxy, h, "custom", upstream.URL, dbPath[1:], 2*time.Hour) + upstream.mu.Lock() + upstream.files[dbPath] = []byte("db v2") + tt.breakSig(upstream) + upstream.mu.Unlock() + ageMetadata(t, proxy, h, "custom", upstream.URL, dbPath[1:]) assertArchBody(t, serveArchRequest(h, http.MethodGet, "/custom"+dbPath, nil), "db v2", []byte("db v2")) - tt.breakSig(upstream) for i := range 2 { w := serveArchRequest(h, http.MethodGet, "/custom"+sigPath, nil) if w.Code != tt.want { @@ -355,6 +371,67 @@ func TestArchHandler_OutdatedSignatureNeverServed(t *testing.T) { } } +// TestArchHandler_DelayedSignatureMatchesCachedDatabase covers a signature +// first requested after the upstream moved on: it must be the one for the +// database the proxy still serves, not the upstream's new one. +func TestArchHandler_DelayedSignatureMatchesCachedDatabase(t *testing.T) { + const dbPath, sigPath = "/custom/os/x86_64/custom.db", "/custom/os/x86_64/custom.db.sig" + upstream := newArchUpstream(t, map[string][]byte{dbPath: []byte("db v1"), sigPath: []byte("sig v1")}) + proxy := newArchTestProxy(t, upstream) + h := NewArchHandler(proxy, "http://proxy.example", map[string]string{"custom": upstream.URL}) + + assertArchBody(t, serveArchRequest(h, http.MethodGet, "/custom"+dbPath, nil), "db v1", []byte("db v1")) + + upstream.set(dbPath, []byte("db v2")) + upstream.set(sigPath, []byte("sig v2")) + + assertArchBody(t, serveArchRequest(h, http.MethodGet, "/custom"+dbPath, nil), "db within TTL", []byte("db v1")) + assertArchBody(t, serveArchRequest(h, http.MethodGet, "/custom"+sigPath, nil), "first signature request", []byte("sig v1")) +} + +// TestArchHandler_SignatureSurvivesUnchangedRefresh covers a database refresh +// that returns the same bytes: the cached signature still matches and must be +// served offline afterwards. +func TestArchHandler_SignatureSurvivesUnchangedRefresh(t *testing.T) { + const dbPath, sigPath = "/custom/os/x86_64/custom.db", "/custom/os/x86_64/custom.db.sig" + upstream := newArchUpstream(t, map[string][]byte{dbPath: []byte("db v1"), sigPath: []byte("sig v1")}) + proxy := newArchTestProxy(t, upstream) + h := NewArchHandler(proxy, "http://proxy.example", map[string]string{"custom": upstream.URL}) + + assertArchBody(t, serveArchRequest(h, http.MethodGet, "/custom"+dbPath, nil), "db", []byte("db v1")) + assertArchBody(t, serveArchRequest(h, http.MethodGet, "/custom"+sigPath, nil), "sig", []byte("sig v1")) + + ageMetadata(t, proxy, h, "custom", upstream.URL, dbPath[1:]) + assertArchBody(t, serveArchRequest(h, http.MethodGet, "/custom"+dbPath, nil), "db refreshed unchanged", []byte("db v1")) + + upstream.available.Store(false) + assertArchBody(t, serveArchRequest(h, http.MethodGet, "/custom"+sigPath, nil), "sig offline", []byte("sig v1")) + if got := upstream.hitCount(sigPath); got != 1 { + t.Errorf("signature upstream hits = %d, want 1", got) + } +} + +// TestArchHandler_DatabaseReplacedDuringRefresh covers a mirror that swaps +// database and signature between the proxy's two fetches. The proxy must +// notice and store the new pair, not the old database with the new signature. +func TestArchHandler_DatabaseReplacedDuringRefresh(t *testing.T) { + const dbPath, sigPath = "/custom/os/x86_64/custom.db", "/custom/os/x86_64/custom.db.sig" + upstream := newArchUpstream(t, map[string][]byte{dbPath: []byte("db v1"), sigPath: []byte("sig v1")}) + var swapped atomic.Bool + upstream.served = func(path string) { + if path == sigPath && swapped.CompareAndSwap(false, true) { + upstream.set(dbPath, []byte("db v2")) + upstream.set(sigPath, []byte("sig v2")) + } + } + proxy := newArchTestProxy(t, upstream) + h := NewArchHandler(proxy, "http://proxy.example", map[string]string{"custom": upstream.URL}) + + assertArchBody(t, serveArchRequest(h, http.MethodGet, "/custom"+dbPath, nil), "db", []byte("db v2")) + upstream.available.Store(false) + assertArchBody(t, serveArchRequest(h, http.MethodGet, "/custom"+sigPath, nil), "sig", []byte("sig v2")) +} + // TestArchHandler_RejectsQueryAndFragmentInPath covers a decoded '?' or '#' // in the path, which would otherwise fetch one file and cache it under the // name that follows. @@ -378,15 +455,14 @@ func TestArchHandler_RejectsQueryAndFragmentInPath(t *testing.T) { } } -// ageMetadata moves a cached metadata entry's fetch time into the past, as if -// it had been fetched age ago. -func ageMetadata(t *testing.T, proxy *Proxy, h *ArchHandler, repository, upstreamURL, path string, age time.Duration) { +// ageMetadata moves a metadata entry's fetch time past the one-hour TTL. +func ageMetadata(t *testing.T, proxy *Proxy, h *ArchHandler, repository, upstreamURL, path string) { t.Helper() entry, err := proxy.DB.GetMetadataCache(archEcosystem, h.metadataCacheKey(repository, upstreamURL, path)) if err != nil || entry == nil { t.Fatalf("metadata entry for %s: %v", path, err) } - entry.FetchedAt.Time = time.Now().Add(-age) + entry.FetchedAt.Time = time.Now().Add(-2 * time.Hour) if err := proxy.DB.UpsertMetadataCache(entry); err != nil { t.Fatalf("aging metadata entry: %v", err) } From 3b262f101bf04154823e9570b0c44630a09b5a1f Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Mon, 12 Oct 2026 09:41:42 +0200 Subject: [PATCH 209/633] Record the detail behind each failure counter - Failure recorders take a FailureDetail: PURL, filename, URL, storage path, error - Keep the last 20 failures per counter series in memory, reset on restart - Strip userinfo, query and fragment from URLs, including those inside error text - Pass the detail from upstream, storage, integrity, health probe and scan call sites - Package names and URLs stay out of Prometheus labels --- internal/handler/gradle.go | 8 +- internal/handler/handler.go | 30 +++-- internal/handler/handler_test.go | 10 ++ internal/metrics/failures.go | 119 ++++++++++++++++++++ internal/metrics/failures_test.go | 125 +++++++++++++++++++++ internal/metrics/metrics.go | 33 ++++-- internal/metrics/metrics_test.go | 8 +- internal/metrics/snapshot_test.go | 2 +- internal/scanner/group.go | 3 +- internal/server/analytics_coverage_test.go | 10 +- internal/server/health.go | 4 +- 11 files changed, 315 insertions(+), 37 deletions(-) create mode 100644 internal/metrics/failures.go create mode 100644 internal/metrics/failures_test.go diff --git a/internal/handler/gradle.go b/internal/handler/gradle.go index 9c74a6a..e251209 100644 --- a/internal/handler/gradle.go +++ b/internal/handler/gradle.go @@ -100,7 +100,7 @@ func (h *GradleBuildCacheHandler) handleGetOrHead(w http.ResponseWriter, r *http exists, err := h.proxy.Storage.Exists(r.Context(), storagePath) metrics.RecordStorageOperation("read", time.Since(existsStart)) if err != nil { - metrics.RecordStorageError("read") + metrics.RecordStorageError("read", metrics.FailureDetail{Path: storagePath, Err: err}) h.proxy.Logger.Error("failed to check gradle build cache entry", "key", key, "error", err) http.Error(w, "failed to read cache entry", http.StatusInternalServerError) return @@ -116,7 +116,7 @@ func (h *GradleBuildCacheHandler) handleGetOrHead(w http.ResponseWriter, r *http size, err := h.proxy.Storage.Size(r.Context(), storagePath) metrics.RecordStorageOperation("read", time.Since(sizeStart)) if err != nil { - metrics.RecordStorageError("read") + metrics.RecordStorageError("read", metrics.FailureDetail{Path: storagePath, Err: err}) } else if size >= 0 { w.Header().Set(headerContentLength, strconv.FormatInt(size, 10)) } @@ -134,7 +134,7 @@ func (h *GradleBuildCacheHandler) handleGetOrHead(w http.ResponseWriter, r *http http.NotFound(w, r) return } - metrics.RecordStorageError("read") + metrics.RecordStorageError("read", metrics.FailureDetail{Path: storagePath, Err: err}) h.proxy.Logger.Error("failed to open gradle build cache entry", "key", key, "error", err) http.Error(w, "failed to read cache entry", http.StatusInternalServerError) return @@ -165,7 +165,7 @@ func (h *GradleBuildCacheHandler) handlePut(w http.ResponseWriter, r *http.Reque return } - metrics.RecordStorageError("write") + metrics.RecordStorageError("write", metrics.FailureDetail{Path: storagePath, Err: err}) h.proxy.Logger.Error("failed to store gradle build cache entry", "key", key, "error", err) http.Error(w, "failed to write cache entry", http.StatusInternalServerError) return diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 449fcef..0c39532 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -367,7 +367,8 @@ func (p *Proxy) checkCache(ctx context.Context, pkgPURL, versionPURL, filename s reader, err := p.Storage.Open(ctx, artifact.StoragePath) metrics.RecordStorageOperation("read", time.Since(start)) if err != nil { - metrics.RecordStorageError("read") + metrics.RecordStorageError("read", metrics.FailureDetail{ + PURL: versionPURL, Filename: filename, Path: artifact.StoragePath, Err: err}) p.Logger.Warn("cached artifact missing from storage, will refetch", "path", artifact.StoragePath, "error", err) return nil, nil @@ -378,7 +379,8 @@ func (p *Proxy) checkCache(ctx context.Context, pkgPURL, versionPURL, filename s p.Logger.Error("cached artifact failed integrity check", "purl", versionPURL, "filename", filename, "path", artifact.StoragePath, "reason", reason) - metrics.RecordIntegrityFailure(purl.NormalizeEcosystem(artifact.Ecosystem)) + metrics.RecordIntegrityFailure(purl.NormalizeEcosystem(artifact.Ecosystem), metrics.FailureDetail{ + PURL: versionPURL, Filename: filename, Path: artifact.StoragePath, Err: errors.New(reason)}) if err := p.DB.DiscardArtifact(versionPURL, filename, artifact.StoragePath); err != nil { p.Logger.Warn("failed to clear corrupt artifact from cache", "error", err) } @@ -421,7 +423,8 @@ func (p *Proxy) rejectUnusableCacheRecord(artifact *database.CachedArtifact, ver p.Logger.Warn("cached artifact has unusable integrity metadata", "purl", versionPURL, "filename", filename, "path", artifact.StoragePath, "error", cause) - metrics.RecordIntegrityFailure(purl.NormalizeEcosystem(artifact.Ecosystem)) + metrics.RecordIntegrityFailure(purl.NormalizeEcosystem(artifact.Ecosystem), metrics.FailureDetail{ + PURL: versionPURL, Filename: filename, Path: artifact.StoragePath, Err: cause}) if err := p.DB.DiscardArtifact(versionPURL, filename, artifact.StoragePath); err != nil { p.Logger.Warn("failed to clear unusable artifact from cache", "error", err) } @@ -459,7 +462,8 @@ func (p *Proxy) fetchAndCache(ctx context.Context, ecosystem, name, version, fil if err != nil { metrics.RecordUpstreamFetch(ecosystem, fetchDuration) - metrics.RecordUpstreamError(ecosystem, "fetch_failed") + metrics.RecordUpstreamError(ecosystem, "fetch_failed", metrics.FailureDetail{ + PURL: versionPURL, Filename: filename, URL: info.URL, Err: err}) if errors.Is(err, fetch.ErrNotFound) { return artifacts.Artifact{}, "", ErrUpstreamNotFound } @@ -491,7 +495,8 @@ func (p *Proxy) storeArtifact(ctx context.Context, ecosystem, name, version, fil _ = artifact.Body.Close() metrics.RecordStorageOperation("write", time.Since(storeStart)) if err != nil { - metrics.RecordStorageError("write") + metrics.RecordStorageError("write", metrics.FailureDetail{ + PURL: versionPURL, Filename: filename, URL: upstreamURL, Path: storagePath, Err: err}) return artifacts.Artifact{}, "", fmt.Errorf("storing artifact: %w", err) } @@ -546,7 +551,8 @@ func (p *Proxy) openStoredArtifact(ctx context.Context, artifact artifacts.Artif metrics.RecordStorageOperation("read", time.Since(readStart)) if err != nil { - metrics.RecordStorageError("read") + metrics.RecordStorageError("read", metrics.FailureDetail{ + PURL: artifact.PURL, Filename: artifact.Filename, Path: storagePath, Err: err}) return nil, fmt.Errorf("opening cached artifact: %w", err) } @@ -1711,7 +1717,8 @@ func (p *Proxy) fetchAndCacheFromURL(ctx context.Context, ecosystem, name, versi artifact, err := p.Fetcher.FetchWithHeaders(ctx, downloadURL, headers) metrics.RecordUpstreamFetch(ecosystem, time.Since(fetchStart)) if err != nil { - metrics.RecordUpstreamError(ecosystem, "fetch_failed") + metrics.RecordUpstreamError(ecosystem, "fetch_failed", metrics.FailureDetail{ + PURL: versionPURL, Filename: filename, URL: downloadURL, Err: err}) if errors.Is(err, fetch.ErrNotFound) { return artifacts.Artifact{}, "", ErrUpstreamNotFound } @@ -1736,7 +1743,8 @@ func (p *Proxy) streamFromUpstream(ctx context.Context, ecosystem, name, version artifact, err := fetchArtifact(ctx) metrics.RecordUpstreamFetch(ecosystem, time.Since(fetchStart)) if err != nil { - metrics.RecordUpstreamError(ecosystem, "fetch_failed") + metrics.RecordUpstreamError(ecosystem, "fetch_failed", metrics.FailureDetail{ + PURL: versionPURL, Filename: filename, URL: upstreamURL, Err: err}) if errors.Is(err, fetch.ErrNotFound) { return nil, ErrUpstreamNotFound } @@ -1748,7 +1756,8 @@ func (p *Proxy) streamFromUpstream(ctx context.Context, ecosystem, name, version onError: func(read int64, err error) { p.Logger.Warn("streaming artifact from upstream failed", "purl", versionPURL, "filename", filename, "url", upstreamURL, "bytes", read, "error", err) - metrics.RecordUpstreamError(ecosystem, "stream_failed") + metrics.RecordUpstreamError(ecosystem, "stream_failed", metrics.FailureDetail{ + PURL: versionPURL, Filename: filename, URL: upstreamURL, Err: err}) }, } result := &CacheResult{ @@ -1773,7 +1782,8 @@ func (p *Proxy) streamFromUpstream(ctx context.Context, ecosystem, name, version result.Reader, err = checks.wrapFailOnMismatch(body, func(reason string) { p.Logger.Error("streamed artifact failed integrity check", "purl", versionPURL, "filename", filename, "url", upstreamURL, "reason", reason) - metrics.RecordIntegrityFailure(purl.NormalizeEcosystem(ecosystem)) + metrics.RecordIntegrityFailure(purl.NormalizeEcosystem(ecosystem), metrics.FailureDetail{ + PURL: versionPURL, Filename: filename, URL: upstreamURL, Err: errors.New(reason)}) }) if err != nil { _ = artifact.Body.Close() diff --git a/internal/handler/handler_test.go b/internal/handler/handler_test.go index 4c05cbd..2dc9a75 100644 --- a/internal/handler/handler_test.go +++ b/internal/handler/handler_test.go @@ -1005,6 +1005,11 @@ func TestGetOrFetchArtifactFromURL_FetchError(t *testing.T) { if diff := testutil.ToFloat64(metrics.UpstreamErrors.WithLabelValues("pypi", "fetch_failed")) - errorsBefore; diff != 1 { t.Errorf("upstream errors delta = %.0f, want 1", diff) } + recent := metrics.RecentFailures("proxy_upstream_errors_total", map[string]string{"ecosystem": "pypi", "error_type": "fetch_failed"}) + if len(recent) == 0 || recent[0].PURL != "pkg:pypi/fail@1.0.0" || + recent[0].URL != "https://pypi.org/files/fail-1.0.0.tar.gz" || recent[0].Error != "connection refused" { + t.Errorf("latest upstream failure = %+v, want the failed package, its URL and the error", recent) + } } func TestGetOrFetchArtifactFromURL_StoreError(t *testing.T) { @@ -1026,6 +1031,11 @@ func TestGetOrFetchArtifactFromURL_StoreError(t *testing.T) { if diff := testutil.ToFloat64(metrics.StorageErrors.WithLabelValues("write")) - errorsBefore; diff != 1 { t.Errorf("storage errors delta = %.0f, want 1", diff) } + recent := metrics.RecentFailures("proxy_storage_errors_total", map[string]string{"operation": "write"}) + if len(recent) == 0 || recent[0].PURL != "pkg:pypi/fail@1.0.0" || recent[0].Filename != "fail-1.0.0.tar.gz" || + recent[0].Path == "" || recent[0].Error != "disk full" { + t.Errorf("latest storage failure = %+v, want the failed package, its storage path and the error", recent) + } } func TestServeArtifact(t *testing.T) { diff --git a/internal/metrics/failures.go b/internal/metrics/failures.go new file mode 100644 index 0000000..e1e207b --- /dev/null +++ b/internal/metrics/failures.go @@ -0,0 +1,119 @@ +package metrics + +import ( + "net/url" + "regexp" + "sync" + "time" +) + +// recentFailuresPerSeries bounds how many failures are kept for each series of +// a failure counter. The series set is already bounded by the counter's +// labels, so this caps the whole store. +const recentFailuresPerSeries = 20 + +// FailureDetail is what a failure counter's call site knows beyond the +// counter's labels: which artifact failed, where it came from or was going, +// and why. Every field is optional. +// +// None of it becomes a metric label. Package names and URLs are unbounded, so +// they are kept in a small in-memory store for /ui/analytics instead. +type FailureDetail struct { + PURL string + Filename string + URL string + Path string + Err error +} + +// Failure is one recorded failure, with credentials already stripped from the +// URL and the error text. +type Failure struct { + At time.Time + PURL string + Filename string + URL string + Path string + Error string +} + +// recentFailures holds the latest failures per counter series. It lives in +// process memory only, so like the counters themselves it starts empty when +// the proxy starts. +var recentFailures = struct { + sync.Mutex + series map[string][]Failure +}{series: make(map[string][]Failure)} + +// RecentFailures returns the latest failures recorded for one series of a +// failure counter, newest first, or nil when the series has none. +func RecentFailures(name string, labels map[string]string) []Failure { + recentFailures.Lock() + defer recentFailures.Unlock() + + stored := recentFailures.series[failureKey(name, labels)] + if len(stored) == 0 { + return nil + } + return append([]Failure(nil), stored...) +} + +func recordFailure(name string, labels map[string]string, detail FailureDetail) { + failure := Failure{ + At: time.Now(), + PURL: detail.PURL, + Filename: detail.Filename, + URL: redactURL(detail.URL), + Path: detail.Path, + } + if detail.Err != nil { + failure.Error = redactURLsIn(detail.Err.Error()) + } + + key := failureKey(name, labels) + + recentFailures.Lock() + defer recentFailures.Unlock() + + stored := append([]Failure{failure}, recentFailures.series[key]...) + if len(stored) > recentFailuresPerSeries { + stored = stored[:recentFailuresPerSeries] + } + recentFailures.series[key] = stored +} + +func resetRecentFailures() { + recentFailures.Lock() + defer recentFailures.Unlock() + recentFailures.series = make(map[string][]Failure) +} + +func failureKey(name string, labels map[string]string) string { + return name + "{" + labelKey(labels) + "}" +} + +var urlInText = regexp.MustCompile(`[a-zA-Z][a-zA-Z0-9+.-]*://[^\s"'<>]+`) + +// redactURLsIn strips credentials from every URL embedded in text, such as the +// one net/http quotes in a failed request's error. +func redactURLsIn(text string) string { + return urlInText.ReplaceAllStringFunc(text, redactURL) +} + +// redactURL drops the userinfo, query and fragment, which is where registry +// tokens and signed-URL signatures live. +func redactURL(raw string) string { + if raw == "" { + return "" + } + u, err := url.Parse(raw) + if err != nil { + return "(unparseable URL)" + } + u.User = nil + u.RawQuery = "" + u.ForceQuery = false + u.Fragment = "" + u.RawFragment = "" + return u.String() +} diff --git a/internal/metrics/failures_test.go b/internal/metrics/failures_test.go new file mode 100644 index 0000000..5e4b60d --- /dev/null +++ b/internal/metrics/failures_test.go @@ -0,0 +1,125 @@ +package metrics + +import ( + "errors" + "fmt" + "strings" + "testing" + + "github.com/prometheus/client_golang/prometheus/testutil" +) + +func TestRecentFailuresNewestFirst(t *testing.T) { + t.Cleanup(resetRecentFailures) + + RecordUpstreamError("npm", "fetch_failed", FailureDetail{PURL: "pkg:npm/left-pad@1.0.0"}) + RecordUpstreamError("npm", "fetch_failed", FailureDetail{PURL: "pkg:npm/right-pad@2.0.0"}) + + got := RecentFailures("proxy_upstream_errors_total", map[string]string{"ecosystem": "npm", "error_type": "fetch_failed"}) + if len(got) != 2 { + t.Fatalf("got %d failures, want 2", len(got)) + } + if got[0].PURL != "pkg:npm/right-pad@2.0.0" || got[1].PURL != "pkg:npm/left-pad@1.0.0" { + t.Errorf("failures not newest first: %q, %q", got[0].PURL, got[1].PURL) + } + if got[0].At.IsZero() { + t.Error("failure has no timestamp") + } +} + +func TestRecentFailuresAreBoundedPerSeries(t *testing.T) { + t.Cleanup(resetRecentFailures) + + for i := range recentFailuresPerSeries + 5 { + RecordStorageError("write", FailureDetail{Path: fmt.Sprintf("npm/pkg-%d.tgz", i)}) + } + + got := RecentFailures("proxy_storage_errors_total", map[string]string{"operation": "write"}) + if len(got) != recentFailuresPerSeries { + t.Fatalf("got %d failures, want the cap of %d", len(got), recentFailuresPerSeries) + } + if want := fmt.Sprintf("npm/pkg-%d.tgz", recentFailuresPerSeries+4); got[0].Path != want { + t.Errorf("newest failure = %q, want %q", got[0].Path, want) + } +} + +func TestRecentFailuresAreKeptPerSeries(t *testing.T) { + t.Cleanup(resetRecentFailures) + + RecordIntegrityFailure("npm", FailureDetail{PURL: "pkg:npm/a@1"}) + RecordIntegrityFailure("pypi", FailureDetail{PURL: "pkg:pypi/b@1"}) + + npm := RecentFailures("proxy_integrity_failures_total", map[string]string{"ecosystem": "npm"}) + if len(npm) != 1 || npm[0].PURL != "pkg:npm/a@1" { + t.Errorf("npm series = %+v, want only pkg:npm/a@1", npm) + } + if got := RecentFailures("proxy_integrity_failures_total", map[string]string{"ecosystem": "deb"}); got != nil { + t.Errorf("unseen series = %+v, want nil", got) + } +} + +func TestRecordFailureStillCountsTheMetric(t *testing.T) { + t.Cleanup(resetRecentFailures) + + before := testutil.ToFloat64(HealthProbeFailures.WithLabelValues("read")) + RecordHealthProbeFailure("read", FailureDetail{Err: errors.New("disk gone")}) + if diff := testutil.ToFloat64(HealthProbeFailures.WithLabelValues("read")) - before; diff != 1 { + t.Errorf("counter delta = %.0f, want 1", diff) + } + + got := RecentFailures("proxy_health_probe_failures_total", map[string]string{"step": "read"}) + if len(got) != 1 || got[0].Error != "disk gone" { + t.Errorf("failures = %+v, want one carrying the error text", got) + } +} + +func TestScanErrorRecordsDetail(t *testing.T) { + t.Cleanup(resetRecentFailures) + + RecordScanError("NPM", "clamav", "timeout", FailureDetail{PURL: "pkg:npm/a@1", Filename: "a-1.tgz"}) + + got := RecentFailures("proxy_scan_errors_total", + map[string]string{"ecosystem": "npm", "scanner": "clamav", "error_type": "timeout"}) + if len(got) != 1 || got[0].Filename != "a-1.tgz" { + t.Errorf("failures = %+v, want one keyed by the normalised ecosystem", got) + } +} + +// The analytics page is unauthenticated, and private registry URLs carry +// credentials in the userinfo or the query string. Neither may be kept. +func TestRecentFailuresStripCredentials(t *testing.T) { + t.Cleanup(resetRecentFailures) + + RecordUpstreamError("npm", "fetch_failed", FailureDetail{ + URL: "https://user:s3cret@registry.example.com/pkg/-/pkg-1.0.0.tgz?token=abc#frag", + Err: errors.New(`Get "https://user:s3cret@registry.example.com/pkg?sig=xyz": connection refused`), + }) + + got := RecentFailures("proxy_upstream_errors_total", map[string]string{"ecosystem": "npm", "error_type": "fetch_failed"}) + if len(got) != 1 { + t.Fatalf("got %d failures, want 1", len(got)) + } + if want := "https://registry.example.com/pkg/-/pkg-1.0.0.tgz"; got[0].URL != want { + t.Errorf("URL = %q, want %q", got[0].URL, want) + } + if want := `Get "https://registry.example.com/pkg": connection refused`; got[0].Error != want { + t.Errorf("Error = %q, want %q", got[0].Error, want) + } + for _, secret := range []string{"s3cret", "token", "sig=", "user:"} { + if strings.Contains(got[0].URL+got[0].Error, secret) { + t.Errorf("recorded failure still contains %q", secret) + } + } +} + +func TestRecentFailuresReturnsACopy(t *testing.T) { + t.Cleanup(resetRecentFailures) + + RecordStorageError("read", FailureDetail{Path: "a"}) + labels := map[string]string{"operation": "read"} + RecentFailures("proxy_storage_errors_total", labels)[0].Path = "mutated" + + if got := RecentFailures("proxy_storage_errors_total", labels)[0].Path; got != "a" { + t.Errorf("stored failure was mutated through the returned slice: %q", got) + } +} diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go index 23dca2c..7567ff8 100644 --- a/internal/metrics/metrics.go +++ b/internal/metrics/metrics.go @@ -338,9 +338,12 @@ func RecordUpstreamFetch(ecosystem string, duration time.Duration) { UpstreamFetchDuration.WithLabelValues(ecosystem).Observe(duration.Seconds()) } -// RecordUpstreamError increments upstream error counter. -func RecordUpstreamError(ecosystem, errorType string) { +// RecordUpstreamError increments upstream error counter and keeps the detail +// for /ui/analytics. +func RecordUpstreamError(ecosystem, errorType string, detail FailureDetail) { UpstreamErrors.WithLabelValues(ecosystem, errorType).Inc() + recordFailure("proxy_upstream_errors_total", + map[string]string{"ecosystem": ecosystem, "error_type": errorType}, detail) } // RecordStorageOperation tracks storage operation duration. @@ -348,20 +351,26 @@ func RecordStorageOperation(operation string, duration time.Duration) { StorageOperationDuration.WithLabelValues(operation).Observe(duration.Seconds()) } -// RecordIntegrityFailure increments the integrity failure counter. -func RecordIntegrityFailure(ecosystem string) { +// RecordIntegrityFailure increments the integrity failure counter and keeps +// the detail for /ui/analytics. +func RecordIntegrityFailure(ecosystem string, detail FailureDetail) { IntegrityFailures.WithLabelValues(ecosystem).Inc() + recordFailure("proxy_integrity_failures_total", map[string]string{"ecosystem": ecosystem}, detail) } -// RecordHealthProbeFailure increments the health probe failure counter. +// RecordHealthProbeFailure increments the health probe failure counter and +// keeps the detail for /ui/analytics. // step is one of: "write", "size", "read", "verify", "delete". -func RecordHealthProbeFailure(step string) { +func RecordHealthProbeFailure(step string, detail FailureDetail) { HealthProbeFailures.WithLabelValues(step).Inc() + recordFailure("proxy_health_probe_failures_total", map[string]string{"step": step}, detail) } -// RecordStorageError increments storage error counter. -func RecordStorageError(operation string) { +// RecordStorageError increments storage error counter and keeps the detail +// for /ui/analytics. +func RecordStorageError(operation string, detail FailureDetail) { StorageErrors.WithLabelValues(operation).Inc() + recordFailure("proxy_storage_errors_total", map[string]string{"operation": operation}, detail) } // RecordScanResult tracks a completed pre-cache scan call. @@ -375,8 +384,12 @@ func RecordScanResult(ecosystem, scannerName string, allowed bool, duration time // RecordScanError increments the scan error counter. // errorType is one of: "error" (scanner call failed), "timeout", "cancelled". -func RecordScanError(ecosystem, scannerName, errorType string) { - ScanErrors.WithLabelValues(purl.NormalizeEcosystem(ecosystem), scannerName, errorType).Inc() +// The detail is kept for /ui/analytics. +func RecordScanError(ecosystem, scannerName, errorType string, detail FailureDetail) { + ecosystem = purl.NormalizeEcosystem(ecosystem) + ScanErrors.WithLabelValues(ecosystem, scannerName, errorType).Inc() + recordFailure("proxy_scan_errors_total", + map[string]string{"ecosystem": ecosystem, "scanner": scannerName, "error_type": errorType}, detail) } // UpdateCacheStats updates cache size and artifact count gauges. diff --git a/internal/metrics/metrics_test.go b/internal/metrics/metrics_test.go index a445467..60b7ed5 100644 --- a/internal/metrics/metrics_test.go +++ b/internal/metrics/metrics_test.go @@ -32,8 +32,8 @@ func TestRecordCacheOperations(t *testing.T) { func TestRecordUpstreamOperations(t *testing.T) { RecordUpstreamFetch("npm", 500*time.Millisecond) RecordUpstreamFetch("pypi", 1*time.Second) - RecordUpstreamError("npm", "fetch_failed") - RecordUpstreamError("pypi", "not_found") + RecordUpstreamError("npm", "fetch_failed", FailureDetail{}) + RecordUpstreamError("pypi", "not_found", FailureDetail{}) // No panics = success } @@ -41,8 +41,8 @@ func TestRecordUpstreamOperations(t *testing.T) { func TestRecordStorageOperations(t *testing.T) { RecordStorageOperation("read", 10*time.Millisecond) RecordStorageOperation("write", 50*time.Millisecond) - RecordStorageError("read") - RecordStorageError("write") + RecordStorageError("read", FailureDetail{}) + RecordStorageError("write", FailureDetail{}) // No panics = success } diff --git a/internal/metrics/snapshot_test.go b/internal/metrics/snapshot_test.go index dd695f7..fcefad3 100644 --- a/internal/metrics/snapshot_test.go +++ b/internal/metrics/snapshot_test.go @@ -125,7 +125,7 @@ func TestSnapshotMissingMetric(t *testing.T) { // come back. This is the check that a newly added metric is reachable by the UI. func TestGatherSeesRegisteredMetrics(t *testing.T) { RecordCacheHit("npm") - RecordScanError("npm", "clamav", "timeout") + RecordScanError("npm", "clamav", "timeout", FailureDetail{}) snap, err := Gather() if err != nil { diff --git a/internal/scanner/group.go b/internal/scanner/group.go index ed0be57..a31d324 100644 --- a/internal/scanner/group.go +++ b/internal/scanner/group.go @@ -184,7 +184,8 @@ func (g *Group) evaluate(scanCtx context.Context, req Request, e entry) (finding // timed out or failed on its own. errType = "cancelled" } - metrics.RecordScanError(req.Ecosystem, e.scanner.Name(), errType) + metrics.RecordScanError(req.Ecosystem, e.scanner.Name(), errType, metrics.FailureDetail{ + PURL: req.PURL, Filename: req.Filename, Err: err}) if errType == "cancelled" { // Another scanner already decided the verdict and cancelled diff --git a/internal/server/analytics_coverage_test.go b/internal/server/analytics_coverage_test.go index 2c4ac63..0a5cf1e 100644 --- a/internal/server/analytics_coverage_test.go +++ b/internal/server/analytics_coverage_test.go @@ -61,15 +61,15 @@ func TestEveryMetricIsSurfaced(t *testing.T) { metrics.RecordCacheHit("npm") metrics.RecordCacheMiss("npm") metrics.RecordUpstreamFetch("npm", 0) - metrics.RecordUpstreamError("npm", "fetch_failed") + metrics.RecordUpstreamError("npm", "fetch_failed", metrics.FailureDetail{}) metrics.RecordStorageOperation("get", 0) - metrics.RecordStorageError("get") - metrics.RecordIntegrityFailure("npm") - metrics.RecordHealthProbeFailure("write") + metrics.RecordStorageError("get", metrics.FailureDetail{}) + metrics.RecordIntegrityFailure("npm", metrics.FailureDetail{}) + metrics.RecordHealthProbeFailure("write", metrics.FailureDetail{}) metrics.UpdateCircuitBreakerState("example.test", 0) metrics.RecordCircuitBreakerTrip("example.test") metrics.RecordScanResult("npm", "clamav", false, 0) - metrics.RecordScanError("npm", "clamav", "timeout") + metrics.RecordScanError("npm", "clamav", "timeout", metrics.FailureDetail{}) metrics.UpdateCacheStats(1, 1) metrics.RecordArtifactEvicted("retention", "npm") metrics.UpdateEcosystemStats([]metrics.EcosystemStats{{Ecosystem: "npm", DownloadedBytes: 1}}) diff --git a/internal/server/health.go b/internal/server/health.go index 9533bd5..fd49cf1 100644 --- a/internal/server/health.go +++ b/internal/server/health.go @@ -169,9 +169,9 @@ func (c *healthCache) Check() error { if err != nil { var pe *probeError if errors.As(err, &pe) { - metrics.RecordHealthProbeFailure(pe.step) + metrics.RecordHealthProbeFailure(pe.step, metrics.FailureDetail{Err: pe.err}) } else { - metrics.RecordHealthProbeFailure("unknown") + metrics.RecordHealthProbeFailure("unknown", metrics.FailureDetail{Err: err}) } } From 496f272fedbc75e1fb027114c91bcda5e5a2c5b6 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Mon, 12 Oct 2026 09:41:42 +0200 Subject: [PATCH 210/633] List recent failures under each failure count on /ui/analytics - Add ui_failure_details / PROXY_UI_FAILURE_DETAILS, off by default - Each failure row expands to its latest failures, newest first - Show a note naming the config key when failures exist and the flag is off - Document the flag in README, configuration reference and example config --- README.md | 8 ++ config.example.yaml | 7 ++ docs/configuration.md | 1 + internal/config/config.go | 9 ++ internal/config/config_test.go | 4 + internal/server/analytics.go | 2 +- internal/server/analytics_failures_test.go | 105 ++++++++++++++++++ internal/server/analytics_runtime.go | 78 ++++++++++--- internal/server/analytics_runtime_test.go | 10 +- .../templates/components/runtime_metrics.html | 38 ++++++- 10 files changed, 241 insertions(+), 21 deletions(-) create mode 100644 internal/server/analytics_failures_test.go diff --git a/README.md b/README.md index 020916c..e7c90b9 100644 --- a/README.md +++ b/README.md @@ -1330,6 +1330,14 @@ Set `ui_request_sources: true` and both appear on `/ui/analytics` under **Runtim **Why addresses are not Prometheus labels.** Client tool names are exported as `proxy_client_requests_total{client}` because they come from a closed set. Addresses are not exported at all: the caller set is unbounded and outside the proxy's control, and every new address would create a time series that lives forever in your TSDB. The same goes for anything job-scoped -- a pipeline ID must never become a label. Per-address and per-request detail belongs in the access log, which records `remote_ip`, `user_agent`, `client`, `ecosystem` and `bytes` on every line as JSONL, ready for `jq`, Loki or whatever you ship logs to. +### Failure details + +The failure counts under **Runtime** on `/ui/analytics` -- upstream errors, storage errors, integrity failures, health probe failures and scan errors -- say how often something failed, not what. Set `ui_failure_details: true` and each count gets a **Latest** list underneath it, newest first: the package URL and filename, the upstream URL or storage path, the error, and how long ago it happened. + +The list keeps the last 20 failures per row, in memory only, so it starts empty when the proxy starts, like the counts above it. Credentials, query strings and fragments are stripped from every URL before it is kept, including URLs inside error messages, since that is where registry tokens and signed-URL signatures live. + +The flag defaults to off for the same reason as `ui_request_sources`: package names and upstream hosts say what your users depend on, and `/ui` is not authenticated. Package names and URLs are never Prometheus labels; the same detail is in the proxy's log. + ### Grafana dashboard A ready-made dashboard lives at [`deploy/grafana/git-pkgs-proxy.json`](deploy/grafana/git-pkgs-proxy.json). Import it via **Dashboards -> New -> Import** and pick your Prometheus data source when prompted; it has no hardcoded data source UID. diff --git a/config.example.yaml b/config.example.yaml index f91b7d3..d7ea935 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -42,6 +42,13 @@ base_url: "http://localhost:8080" # otherwise read the addresses of your build fleet. # ui_request_sources: false +# List the latest failures behind each failure count on /ui/analytics (upstream, +# storage, integrity, health probe and scan errors): the package, its upstream +# URL or storage path, and the error. URLs lose credentials and query strings. +# +# Off by default, for the same reason as ui_request_sources. +# ui_failure_details: false + # Artifact storage configuration storage: # Storage backend URL diff --git a/docs/configuration.md b/docs/configuration.md index 6b17452..51afb8a 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -21,6 +21,7 @@ See `config.example.yaml` in the repository root for a complete example. | `ui_base_url` | `PROXY_UI_URL` | - | (defaults to `base_url`) | Public URL where the web UI is reached. Set separately when the UI lives behind a different hostname than package endpoints (e.g. public domain vs Docker network alias). Used for canonical/og:url tags and the install guide banner. The proxy still serves package endpoints on the same listener, so any reverse proxy fronting the UI publicly should restrict the public route to `PathPrefix(/ui)` to avoid exposing package endpoints. | | `trust_forwarded_for` | `PROXY_TRUST_FORWARDED_FOR` | - | `false` | Attribute requests to the leftmost `X-Forwarded-For` entry instead of the TCP peer address, in the structured log, the access log and the request-source table on `/ui/analytics`. | | `ui_request_sources` | `PROXY_UI_REQUEST_SOURCES` | - | `false` | Show the request-source table on `/ui/analytics`, which reports caller addresses, the tool each ran and how much each pulled. | +| `ui_failure_details` | `PROXY_UI_FAILURE_DETAILS` | - | `false` | List the latest failures behind each failure count on `/ui/analytics`: the package, its upstream URL or storage path, and the error. URLs are shown without credentials or query strings. | ## Storage diff --git a/internal/config/config.go b/internal/config/config.go index 25ee4d7..b3e1ac9 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -119,6 +119,14 @@ type Config struct { // is enabled /ui exposes what is cached rather than who called. UIRequestSources bool `json:"ui_request_sources" yaml:"ui_request_sources"` + // UIFailureDetails lists the latest failures behind each failure counter on + // /ui/analytics: the package, its upstream URL or storage path, and the + // error. URLs are shown without credentials or query strings. + // + // Off by default, for the same reason as UIRequestSources: package names and + // upstream hosts say what the proxy's users depend on. + UIFailureDetails bool `json:"ui_failure_details" yaml:"ui_failure_details"` + // Storage configures artifact storage. Storage StorageConfig `json:"storage" yaml:"storage"` @@ -1102,6 +1110,7 @@ func (c *Config) LoadFromEnv() { setEnvString(&c.UIBaseURL, "PROXY_UI_URL") setEnvBool(&c.TrustForwardedFor, "PROXY_TRUST_FORWARDED_FOR") setEnvBool(&c.UIRequestSources, "PROXY_UI_REQUEST_SOURCES") + setEnvBool(&c.UIFailureDetails, "PROXY_UI_FAILURE_DETAILS") setEnvString(&c.Storage.URL, "PROXY_STORAGE_URL") setEnvString(&c.Storage.Path, "PROXY_STORAGE_PATH") setEnvString(&c.Storage.MaxSize, "PROXY_STORAGE_MAX_SIZE") diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 0f7a78f..491514e 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -463,6 +463,7 @@ func TestLoadFromEnv(t *testing.T) { t.Setenv("PROXY_ACCESS_LOG_PATH", "/tmp/proxy-access.jsonl") t.Setenv("PROXY_TRUST_FORWARDED_FOR", "true") t.Setenv("PROXY_UI_REQUEST_SOURCES", "true") + t.Setenv("PROXY_UI_FAILURE_DETAILS", "true") t.Setenv("PROXY_UPSTREAM_ALLOW_PRIVATE_HOSTS", "registry.internal, 10.0.0.12") t.Setenv("PROXY_UPSTREAM_ALLOW_LOOPBACK", "true") t.Setenv("PROXY_GRADLE_BUILD_CACHE_READ_ONLY", "true") @@ -500,6 +501,9 @@ func TestLoadFromEnv(t *testing.T) { if !cfg.UIRequestSources { t.Error("UIRequestSources = false, want true from the environment") } + if !cfg.UIFailureDetails { + t.Error("UIFailureDetails = false, want true from the environment") + } if got := strings.Join(cfg.Upstream.AllowPrivateHosts, ","); got != "registry.internal,10.0.0.12" { t.Errorf("Upstream.AllowPrivateHosts = %q, want %q", got, "registry.internal,10.0.0.12") } diff --git a/internal/server/analytics.go b/internal/server/analytics.go index eaf1948..b7b4a83 100644 --- a/internal/server/analytics.go +++ b/internal/server/analytics.go @@ -171,7 +171,7 @@ func (s *Server) handleAnalytics(w http.ResponseWriter, r *http.Request) { if err != nil { s.logger.Error("failed to gather runtime metrics", "error", err) } else { - data.Runtime = runtimeView(snap) + data.Runtime = runtimeView(snap, s.cfg != nil && s.cfg.UIFailureDetails) if s.cfg != nil && s.cfg.UIRequestSources { data.Runtime.SourcesOn = true data.Runtime.Sources = s.sources.Top(analyticsTopSources) diff --git a/internal/server/analytics_failures_test.go b/internal/server/analytics_failures_test.go new file mode 100644 index 0000000..4f60724 --- /dev/null +++ b/internal/server/analytics_failures_test.go @@ -0,0 +1,105 @@ +package server + +import ( + "errors" + "strings" + "testing" + + "github.com/git-pkgs/proxy/internal/metrics" + "github.com/prometheus/client_golang/prometheus" +) + +func storageErrorSnapshot(t *testing.T, operation string) *metrics.Snapshot { + t.Helper() + reg := prometheus.NewRegistry() + errs := prometheus.NewCounterVec( + prometheus.CounterOpts{Name: "proxy_storage_errors_total", Help: "t"}, []string{"operation"}) + reg.MustRegister(errs) + errs.WithLabelValues(operation).Inc() + + snap, err := metrics.GatherFrom(reg) + if err != nil { + t.Fatalf("GatherFrom: %v", err) + } + return snap +} + +func TestRuntimeViewAttachesRecentFailuresWhenEnabled(t *testing.T) { + const operation = "test-attach" + metrics.RecordStorageError(operation, metrics.FailureDetail{ + PURL: "pkg:npm/left-pad@1.0.0", Filename: "left-pad-1.0.0.tgz", Path: "npm/left-pad", Err: errors.New("disk full")}) + + v := runtimeView(storageErrorSnapshot(t, operation), true) + + if !v.FailureDetailsOn { + t.Error("FailureDetailsOn = false, want true") + } + if len(v.StorageErrors) != 1 || len(v.StorageErrors[0].Recent) != 1 { + t.Fatalf("StorageErrors = %+v, want one row with one recent failure", v.StorageErrors) + } + got := v.StorageErrors[0].Recent[0] + if got.PURL != "pkg:npm/left-pad@1.0.0" || got.Filename != "left-pad-1.0.0.tgz" || + got.Path != "npm/left-pad" || got.Error != "disk full" || got.Age == "" { + t.Errorf("recent failure = %+v", got) + } +} + +func TestRuntimeViewOmitsRecentFailuresWhenDisabled(t *testing.T) { + const operation = "test-omit" + metrics.RecordStorageError(operation, metrics.FailureDetail{PURL: "pkg:npm/left-pad@1.0.0"}) + + v := runtimeView(storageErrorSnapshot(t, operation), false) + + if v.FailureDetailsOn { + t.Error("FailureDetailsOn = true, want false") + } + if len(v.StorageErrors) != 1 || v.StorageErrors[0].Recent != nil { + t.Errorf("StorageErrors = %+v, want the count without recent failures", v.StorageErrors) + } +} + +// Package names and upstream URLs say what the proxy's users depend on, so like +// the caller table they are published only when asked for. +func TestAnalyticsPageHidesFailureDetailsByDefault(t *testing.T) { + ts := newTestServer(t) + defer ts.close() + + if ts.server.cfg.UIFailureDetails { + t.Fatal("UIFailureDetails defaults to true; it must default off") + } + metrics.RecordUpstreamError("npm", "fetch_failed", metrics.FailureDetail{PURL: "pkg:npm/hidden-by-default@1.0.0"}) + + body := ts.getOK(t, "/ui/analytics") + if strings.Contains(body, "hidden-by-default") { + t.Error("a failed package rendered with ui_failure_details off") + } + if !strings.Contains(body, "ui_failure_details") { + t.Error("the off-state note naming the config key did not render") + } +} + +func TestAnalyticsPageShowsFailureDetailsWhenEnabled(t *testing.T) { + ts := newTestServer(t) + defer ts.close() + + ts.server.cfg.UIFailureDetails = true + metrics.RecordUpstreamError("npm", "fetch_failed", metrics.FailureDetail{ + PURL: "pkg:npm/shown-when-enabled@1.0.0", + URL: "https://registry.example.com/shown-when-enabled.tgz?token=secret", + Err: errors.New("upstream returned 503"), + }) + + body := ts.getOK(t, "/ui/analytics") + for _, want := range []string{ + "pkg:npm/shown-when-enabled@1.0.0", + "https://registry.example.com/shown-when-enabled.tgz", + "upstream returned 503", + } { + if !strings.Contains(body, want) { + t.Errorf("rendered page missing %q", want) + } + } + if strings.Contains(body, "token=secret") { + t.Error("rendered page leaks the URL's query string") + } +} diff --git a/internal/server/analytics_runtime.go b/internal/server/analytics_runtime.go index dad3d19..8e5d0d1 100644 --- a/internal/server/analytics_runtime.go +++ b/internal/server/analytics_runtime.go @@ -50,6 +50,12 @@ type RuntimeView struct { // SourcesOn gates the caller table. The page is unauthenticated, so the // addresses behind it are published only when asked for. SourcesOn bool + // FailureDetailsOn gates the recent failures listed under each failure + // row, for the same reason as SourcesOn. + FailureDetailsOn bool + // HasFailures is set when any failure counter has fired, which is when the + // note about turning failure details on is worth showing. + HasFailures bool } // ClientRow is one client tool's share of requests and bytes. @@ -65,6 +71,21 @@ type LabelledCount struct { Count string // Bad marks a failure row, which the template tints. Bad bool + // Recent lists the latest failures behind the count, newest first. + Recent []FailureRow + // series is the counter series the row was rendered from, for looking up + // what was recorded against it. + series map[string]string +} + +// FailureRow is one recorded failure behind a failure counter. +type FailureRow struct { + Age string + PURL string + Filename string + URL string + Path string + Error string } // LabelledStat is a histogram series rendered as a row: how many observations, @@ -82,13 +103,17 @@ type BreakerRow struct { Open bool } -// runtimeView shapes a registry snapshot for the analytics page. -func runtimeView(snap *metrics.Snapshot) RuntimeView { +// runtimeView shapes a registry snapshot for the analytics page. With +// failureDetails set, each failure row carries the latest failures behind it. +func runtimeView(snap *metrics.Snapshot, failureDetails bool) RuntimeView { if snap == nil { return RuntimeView{} } - v := RuntimeView{Available: true} + v := RuntimeView{Available: true, FailureDetailsOn: failureDetails} + failures := func(name string, labels ...string) []LabelledCount { + return failureRows(snap, name, failureDetails, labels...) + } requests := snap.Sum("proxy_requests_total") v.Requests = formatCount(int64(requests)) @@ -107,21 +132,23 @@ func runtimeView(snap *metrics.Snapshot) RuntimeView { v.UpstreamFetches = formatCount(int64(snap.Count("proxy_upstream_fetch_duration_seconds"))) v.UpstreamFetchMean = formatDuration(snap.Mean("proxy_upstream_fetch_duration_seconds")) - v.UpstreamErrors = failureRows(snap, "proxy_upstream_errors_total", "ecosystem", "error_type") + v.UpstreamErrors = failures("proxy_upstream_errors_total", "ecosystem", "error_type") v.StorageOps = histogramRows(snap, "proxy_storage_operation_duration_seconds", "operation") - v.StorageErrors = failureRows(snap, "proxy_storage_errors_total", "operation") - v.IntegrityFailures = failureRows(snap, "proxy_integrity_failures_total", "ecosystem") - v.ProbeFailures = failureRows(snap, "proxy_health_probe_failures_total", "step") + v.StorageErrors = failures("proxy_storage_errors_total", "operation") + v.IntegrityFailures = failures("proxy_integrity_failures_total", "ecosystem") + v.ProbeFailures = failures("proxy_health_probe_failures_total", "step") v.Evictions = countRows(snap, "proxy_artifacts_evicted_total", "reason", "ecosystem") v.Breakers = breakerRows(snap) - v.BreakerTrips = failureRows(snap, "proxy_circuit_breaker_trips_total", "registry") + v.BreakerTrips = failures("proxy_circuit_breaker_trips_total", "registry") v.Scans = histogramRows(snap, "proxy_scan_duration_seconds", "ecosystem", "scanner") - v.ScansBlocked = failureRows(snap, "proxy_scan_blocked_total", "ecosystem", "scanner") - v.ScanErrors = failureRows(snap, "proxy_scan_errors_total", "ecosystem", "scanner", "error_type") + v.ScansBlocked = failures("proxy_scan_blocked_total", "ecosystem", "scanner") + v.ScanErrors = failures("proxy_scan_errors_total", "ecosystem", "scanner", "error_type") v.ScanningOn = len(v.Scans) > 0 || len(v.ScansBlocked) > 0 || len(v.ScanErrors) > 0 + v.HasFailures = len(v.UpstreamErrors) > 0 || len(v.StorageErrors) > 0 || len(v.IntegrityFailures) > 0 || + len(v.ProbeFailures) > 0 || len(v.ScanErrors) > 0 v.ResponseBytes = formatSize(int64(snap.Sum("proxy_response_bytes_total"))) v.Clients = clientRows(snap) @@ -202,11 +229,15 @@ func statusClasses(snap *metrics.Snapshot) []LabelledCount { // failureRows renders every non-zero series of a failure counter, keyed by the // joined values of the given labels. Zero-valued series are dropped: a counter // that has never fired carries no information, and a wall of zeroes buries the -// rows that matter. -func failureRows(snap *metrics.Snapshot, name string, labels ...string) []LabelledCount { +// rows that matter. With withRecent set, each row carries the latest failures +// recorded for its series. +func failureRows(snap *metrics.Snapshot, name string, withRecent bool, labels ...string) []LabelledCount { rows := countRows(snap, name, labels...) for i := range rows { rows[i].Bad = true + if withRecent { + rows[i].Recent = recentFailureRows(metrics.RecentFailures(name, rows[i].series)) + } } return rows } @@ -219,8 +250,27 @@ func countRows(snap *metrics.Snapshot, name string, labels ...string) []Labelled continue } rows = append(rows, LabelledCount{ - Label: joinLabels(s, labels...), - Count: formatCount(int64(s.Value)), + Label: joinLabels(s, labels...), + Count: formatCount(int64(s.Value)), + series: s.Labels, + }) + } + return rows +} + +func recentFailureRows(failures []metrics.Failure) []FailureRow { + if len(failures) == 0 { + return nil + } + rows := make([]FailureRow, 0, len(failures)) + for _, f := range failures { + rows = append(rows, FailureRow{ + Age: formatTimeAgo(f.At), + PURL: f.PURL, + Filename: f.Filename, + URL: f.URL, + Path: f.Path, + Error: f.Error, }) } return rows diff --git a/internal/server/analytics_runtime_test.go b/internal/server/analytics_runtime_test.go index 422c02d..5bff0f3 100644 --- a/internal/server/analytics_runtime_test.go +++ b/internal/server/analytics_runtime_test.go @@ -27,7 +27,7 @@ func TestFormatDuration(t *testing.T) { } func TestRuntimeViewNilSnapshot(t *testing.T) { - if v := runtimeView(nil); v.Available { + if v := runtimeView(nil, false); v.Available { t.Error("a nil snapshot must not report itself as available") } } @@ -63,7 +63,7 @@ func TestRuntimeViewShapesRegistry(t *testing.T) { if err != nil { t.Fatalf("GatherFrom: %v", err) } - v := runtimeView(snap) + v := runtimeView(snap, false) if !v.Available { t.Fatal("expected the view to be available") @@ -142,7 +142,7 @@ func TestRuntimeViewShowsEvictions(t *testing.T) { if err != nil { t.Fatalf("GatherFrom: %v", err) } - v := runtimeView(snap) + v := runtimeView(snap, false) if len(v.Evictions) != 1 { t.Fatalf("Evictions = %+v, want only the non-zero row", v.Evictions) @@ -168,7 +168,7 @@ func TestRuntimeViewDropsZeroCounters(t *testing.T) { if err != nil { t.Fatalf("GatherFrom: %v", err) } - v := runtimeView(snap) + v := runtimeView(snap, false) if len(v.StorageErrors) != 1 { t.Fatalf("expected only the non-zero row, got %+v", v.StorageErrors) @@ -185,7 +185,7 @@ func TestRuntimeViewEmptyRegistry(t *testing.T) { if err != nil { t.Fatalf("GatherFrom: %v", err) } - v := runtimeView(snap) + v := runtimeView(snap, false) if !v.Available { t.Error("an empty registry is still a valid snapshot") diff --git a/internal/server/templates/components/runtime_metrics.html b/internal/server/templates/components/runtime_metrics.html index ff0f67c..9804b47 100644 --- a/internal/server/templates/components/runtime_metrics.html +++ b/internal/server/templates/components/runtime_metrics.html @@ -92,6 +92,14 @@
+ {{if and .HasFailures (not .FailureDetailsOn)}} +

+ Per-failure detail is off. Set ui_failure_details: true + to list the packages, URLs and errors behind each failure count here, once + /ui is behind authentication; the same detail is in the log. +

+ {{end}} +
@@ -191,9 +199,14 @@ {{if .Rows}}
{{range .Rows}} -
+
{{.Label}}
{{.Count}}
+ {{if .Recent}} +
+ {{template "recent_failures" .Recent}} +
+ {{end}}
{{end}}
@@ -203,6 +216,29 @@
{{end}} +{{/* recent_failures lists the latest failures behind a failure count, newest first. */}} +{{define "recent_failures"}} +
+ + Latest {{len .}} + +
    + {{range .}} +
  • +
    + {{if .PURL}}{{.PURL}}{{else if .Path}}{{.Path}}{{else if .URL}}{{.URL}}{{else}}-{{end}} + {{.Age}} +
    + {{if .Filename}}
    {{.Filename}}
    {{end}} + {{if and .URL (or .PURL .Path)}}
    {{.URL}}
    {{end}} + {{if and .Path .PURL}}
    {{.Path}}
    {{end}} + {{if .Error}}
    {{.Error}}
    {{end}} +
    • + {{end}} +
+
+{{end}} + {{/* runtime_stats renders a histogram list: how many observations, and their mean. */}} {{define "runtime_stats"}}
From 90989eb03056b6fd85687f25ef3e34daeef21b8f Mon Sep 17 00:00:00 2001 From: Christian Heim Date: Mon, 12 Oct 2026 13:58:09 +0200 Subject: [PATCH 211/633] Enable storage retention for maven (#444) Package rules use pkg:maven/{group}/{artifact} with the case as published. Keys without a group or with the group written as a path can never match and are refused. Files from the Gradle Plugin Portal fallback fall under the maven rule; the Gradle build cache isn't affected. The mirror can't fetch Maven packages today (it passes group/artifact where the resolver wants group:artifact), so there's no mirror test. Refs #306 --- docs/configuration.md | 2 +- internal/handler/maven.go | 21 +++ internal/server/retention_maven_test.go | 211 ++++++++++++++++++++++++ 3 files changed, 233 insertions(+), 1 deletion(-) create mode 100644 internal/server/retention_maven_test.go diff --git a/docs/configuration.md b/docs/configuration.md index 6b17452..3a4e480 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -100,7 +100,7 @@ A package rule wins over an ecosystem rule, which wins over `default`. Package r | `helm` | no | - | | `hex` | no | - | | `julia` | no | - | -| `maven` | no | - | +| `maven` | yes | `pkg:maven/com.google.guava/guava`, group and artifact as published, case included. Gradle build cache entries are not covered; they use `gradle.build_cache.max_age`. | | `npm` | no | - | | `nuget` | no | - | | `oci` | no | - | diff --git a/internal/handler/maven.go b/internal/handler/maven.go index 850cec3..700c107 100644 --- a/internal/handler/maven.go +++ b/internal/handler/maven.go @@ -6,6 +6,9 @@ import ( "net/http" "path" "strings" + + "github.com/git-pkgs/proxy/internal/retention" + "github.com/git-pkgs/purl" ) const ( @@ -14,6 +17,24 @@ const ( minMavenParts = 4 // group path segments + artifact + version + filename ) +// Maven files are stored under pkg:maven/{group}/{artifact}, built from the +// group:artifact name, which is what the default key mapping produces from +// a package rule. Names keep their case, as Maven coordinates do. +func init() { + retention.Default.Register(retention.Spec{Key: "maven", CanonicalPackage: mavenRetentionKey}) +} + +// mavenRetentionKey maps a package rule to the stored package PURL. A key +// without a group, or with the group written as a path (com/google/guava) +// instead of dotted, parses fine but can never match a stored package, so +// it is refused rather than accepted as a rule that does nothing. +func mavenRetentionKey(p *purl.PURL) (string, bool) { + if p == nil || p.Namespace == "" || strings.Contains(p.Namespace, "/") { + return "", false + } + return retention.DefaultCanonical("maven")(p) +} + // MavenHandler handles Maven repository protocol requests. type MavenHandler struct { proxy *Proxy diff --git a/internal/server/retention_maven_test.go b/internal/server/retention_maven_test.go new file mode 100644 index 0000000..61824f5 --- /dev/null +++ b/internal/server/retention_maven_test.go @@ -0,0 +1,211 @@ +package server + +import ( + "context" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/git-pkgs/proxy/internal/config" + "github.com/git-pkgs/proxy/internal/database" + "github.com/git-pkgs/proxy/internal/handler" + "github.com/git-pkgs/proxy/internal/retention" + "github.com/git-pkgs/registries/fetch" +) + +const ( + mavenGuavaJar = "/com/google/guava/guava/32.1.3-jre/guava-32.1.3-jre.jar" + mavenGuavaPom = "/com/google/guava/guava/32.1.3-jre/guava-32.1.3-jre.pom" + mavenLang3Jar = "/org/apache/commons/commons-lang3/3.14.0/commons-lang3-3.14.0.jar" + // mavenPluginJar exists only on the plugin portal, so it is fetched + // through the handler's fallback. + mavenPluginJar = "/com/example/plugin/example-plugin/1.0/example-plugin-1.0.jar" +) + +// mavenRetentionUpstreams returns a Maven Central stand-in and a plugin +// portal stand-in, counting downloads from either. +func mavenRetentionUpstreams(t *testing.T) (central, portal *httptest.Server, downloads *atomic.Int64) { + t.Helper() + downloads = &atomic.Int64{} + serve := func(paths ...string) *httptest.Server { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + for _, p := range paths { + if r.URL.Path == p { + downloads.Add(1) + _, _ = io.WriteString(w, "maven file bytes") + return + } + } + http.NotFound(w, r) + })) + t.Cleanup(srv.Close) + return srv + } + central = serve(mavenGuavaJar, mavenGuavaPom, mavenLang3Jar) + portal = serve(mavenPluginJar) + return central, portal, downloads +} + +func newMavenRetentionProxy(t *testing.T, central, portal *httptest.Server) (*httptest.Server, *database.DB) { + t.Helper() + db, store := setupEvictionTest(t) + // Both stand-ins are plain-HTTP loopback servers; the default fetcher + // refuses loopback, a stock client reaches either. + fetcher := fetch.NewFetcher(fetch.WithHTTPClient(central.Client()), fetch.WithMaxRetries(0)) + t.Cleanup(func() { _ = fetcher.Close() }) + proxy := handler.NewProxy(db, store, fetcher, fetch.NewResolver(), discardLogger()) + proxy.HTTPClient = central.Client() + h := handler.NewMavenHandler(proxy, "http://proxy.test", central.URL, portal.URL) + srv := httptest.NewServer(h.Routes()) + t.Cleanup(srv.Close) + return srv, db +} + +func mavenGet(t *testing.T, url string) { + t.Helper() + resp, err := http.Get(url) + if err != nil { + t.Fatalf("GET %s: %v", url, err) + } + body, _ := io.ReadAll(resp.Body) + _ = resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Fatalf("GET %s = %d %s, want 200", url, resp.StatusCode, body) + } +} + +// mavenArtifact returns the record of the file named filename. +func mavenArtifact(t *testing.T, db *database.DB, filename string) database.Artifact { + t.Helper() + var art database.Artifact + if err := db.Get(&art, db.Rebind(`SELECT * FROM artifacts WHERE filename = ?`), filename); err != nil { + t.Fatalf("reading artifact %s: %v", filename, err) + } + return art +} + +func mavenBackdate(t *testing.T, db *database.DB, filename string, fetched, accessed time.Time) { + t.Helper() + query := db.Rebind(`UPDATE artifacts SET fetched_at = ?, last_accessed_at = ? WHERE filename = ?`) + if _, err := db.Exec(query, fetched, accessed, filename); err != nil { + t.Fatalf("backdating %s: %v", filename, err) + } +} + +func TestMavenRegistersForRetention(t *testing.T) { + if _, ok := retention.Default.Lookup("maven"); !ok { + t.Fatal("maven is not registered for retention") + } + if key, ok := retention.Default.KeyForDBEcosystem("maven"); !ok || key != "maven" { + t.Errorf("KeyForDBEcosystem(maven) = %q, %v, want maven, true", key, ok) + } +} + +// A Maven file served from the cache stays while it is used, is evicted +// once its rule runs out, comes back from upstream on the next download, and +// stays after that even with the evicted copy's old access time on the +// record. The jar and the pom of one version age independently. +func TestMavenRetentionEvictsAndRefetches(t *testing.T) { + central, portal, downloads := mavenRetentionUpstreams(t) + srv, db := newMavenRetentionProxy(t, central, portal) + const jar, pom = "guava-32.1.3-jre.jar", "guava-32.1.3-jre.pom" + + mavenGet(t, srv.URL+mavenGuavaJar) + mavenGet(t, srv.URL+mavenGuavaJar) + mavenGet(t, srv.URL+mavenGuavaPom) + if got := downloads.Load(); got != 2 { + t.Fatalf("upstream downloads = %d, want 2 (jar once, pom once)", got) + } + if !mavenArtifact(t, db, jar).LastAccessedAt.Valid { + t.Fatal("cache hit did not record an access") + } + + rules := retention.Rules{Ecosystems: map[string]time.Duration{"maven": retentionDay}} + now := time.Now() + sweepRetention(context.Background(), db, discardLogger(), retention.Default, rules, now) + if !mavenArtifact(t, db, jar).StoragePath.Valid { + t.Fatal("jar used today was evicted") + } + + // Only the pom goes unused for two days. + old := now.Add(-2 * retentionDay) + mavenBackdate(t, db, pom, old, old) + sweepRetention(context.Background(), db, discardLogger(), retention.Default, rules, now) + if mavenArtifact(t, db, pom).StoragePath.Valid { + t.Fatal("pom unused for two days kept under a one-day rule") + } + if !mavenArtifact(t, db, jar).StoragePath.Valid { + t.Fatal("jar of the same version evicted along with the pom") + } + + mavenGet(t, srv.URL+mavenGuavaPom) + if got := downloads.Load(); got != 3 { + t.Errorf("upstream downloads = %d, want 3 after the pom's eviction", got) + } + art := mavenArtifact(t, db, pom) + mavenBackdate(t, db, pom, art.FetchedAt.Time, now.Add(-30*retentionDay)) + sweepRetention(context.Background(), db, discardLogger(), retention.Default, rules, time.Now()) + if !mavenArtifact(t, db, pom).StoragePath.Valid { + t.Error("refetched pom evicted again by the next sweep") + } +} + +// Keys that parse as Maven PURLs but can never match a stored package are +// refused at validation instead of becoming rules that do nothing. +func TestMavenRetentionRefusesUnmatchableKeys(t *testing.T) { + for _, key := range []string{"pkg:maven/com/google/guava/guava", "pkg:maven/guava"} { + cfg := config.Default() + cfg.Storage.Retention.Packages = map[string]string{key: "0"} + _, err := cfg.RetentionRules(retention.Default) + if err == nil || !strings.Contains(err.Error(), "does not match how maven packages are stored") { + t.Errorf("RetentionRules(%s) error = %v, want the key refused", key, err) + } + } +} + +// A package rule written as a Maven PURL reaches the stored package, and a +// file fetched through the Gradle Plugin Portal fallback falls under the +// maven rule like one from Maven Central. +func TestMavenRetentionPackageRuleAndPluginPortal(t *testing.T) { + central, portal, _ := mavenRetentionUpstreams(t) + srv, db := newMavenRetentionProxy(t, central, portal) + mavenGet(t, srv.URL+mavenGuavaJar) + mavenGet(t, srv.URL+mavenLang3Jar) + mavenGet(t, srv.URL+mavenPluginJar) + + var stored string + jar := mavenArtifact(t, db, "guava-32.1.3-jre.jar") + if err := db.Get(&stored, db.Rebind(`SELECT package_purl FROM versions WHERE purl = ?`), jar.VersionPURL); err != nil { + t.Fatalf("reading stored package PURL: %v", err) + } + if stored != "pkg:maven/com.google.guava/guava" { + t.Errorf("stored package PURL = %q, want pkg:maven/com.google.guava/guava", stored) + } + + old := time.Now().Add(-30 * retentionDay) + for _, filename := range []string{"guava-32.1.3-jre.jar", "commons-lang3-3.14.0.jar", "example-plugin-1.0.jar"} { + mavenBackdate(t, db, filename, old, old) + } + cfg := config.Default() + cfg.Storage.Retention.Ecosystems = map[string]string{"maven": "1d"} + cfg.Storage.Retention.Packages = map[string]string{"pkg:maven/com.google.guava/guava": "0"} + rules, err := cfg.RetentionRules(retention.Default) + if err != nil { + t.Fatalf("RetentionRules: %v", err) + } + sweepRetention(context.Background(), db, discardLogger(), retention.Default, rules, time.Now()) + + if !mavenArtifact(t, db, "guava-32.1.3-jre.jar").StoragePath.Valid { + t.Error("guava evicted despite its never-evict rule") + } + if mavenArtifact(t, db, "commons-lang3-3.14.0.jar").StoragePath.Valid { + t.Error("commons-lang3 kept past the maven rule") + } + if mavenArtifact(t, db, "example-plugin-1.0.jar").StoragePath.Valid { + t.Error("plugin portal artifact kept past the maven rule") + } +} From b40a3f067a810dcef6f89220a77ab8dbae35b1e7 Mon Sep 17 00:00:00 2001 From: Christian Heim Date: Mon, 12 Oct 2026 14:00:57 +0200 Subject: [PATCH 212/633] Enable storage retention for oci (#443) Blobs are stored under the repository name of the pull, which purl writes as one name (pkg:oci/library%2Fnginx). Package rules write it as a path, pkg:oci/library/nginx, and get mapped to that form; keys with an empty segment, a tag or digest, or a bare upstream/{name} prefix are refused. Named upstreams keep their prefix, also when containerd reaches them through ns. Homebrew bottles and Helm charts pulled over OCI fall under oci too, bottles under Homebrew's image name (openssl@3 is homebrew/core/openssl/3). Manifests live in the metadata cache and aren't touched. Refs #306 --- docs/configuration.md | 2 +- internal/handler/container.go | 35 ++++ internal/handler/container_test.go | 35 ++++ internal/server/retention_oci_test.go | 236 ++++++++++++++++++++++++++ 4 files changed, 307 insertions(+), 1 deletion(-) create mode 100644 internal/server/retention_oci_test.go diff --git a/docs/configuration.md b/docs/configuration.md index 3a4e480..1c7887a 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -103,7 +103,7 @@ A package rule wins over an ecosystem rule, which wins over `default`. Package r | `maven` | yes | `pkg:maven/com.google.guava/guava`, group and artifact as published, case included. Gradle build cache entries are not covered; they use `gradle.build_cache.max_age`. | | `npm` | no | - | | `nuget` | no | - | -| `oci` | no | - | +| `oci` | yes | `pkg:oci/library/nginx`, the repository name of the pull. A named upstream keeps its prefix, also when containerd reaches it through `ns`: `ghcr.io/owner/app` pulled that way is `pkg:oci/upstream/ghcr/owner/app`. Homebrew bottles use Homebrew's image name, with `@` written as `/` and `+` as `x` (`openssl@3` is `pkg:oci/homebrew/core/openssl/3`). Helm charts pulled over OCI are stored like images. Only blobs are evicted; manifests stay in the metadata cache. | | `pub` | no | - | | `pypi` | no | - | | `rpm` | no | - | diff --git a/internal/handler/container.go b/internal/handler/container.go index 202400d..49ef3b2 100644 --- a/internal/handler/container.go +++ b/internal/handler/container.go @@ -12,6 +12,9 @@ import ( "slices" "strings" "sync" + + "github.com/git-pkgs/proxy/internal/retention" + "github.com/git-pkgs/purl" ) const ( @@ -57,6 +60,38 @@ func isDockerHubKey(key string) bool { return slices.Contains(dockerHubNamespaceKeys, key) } +// Blobs are stored under the repository name the client asks for, which +// purl writes as one name, pkg:oci/library%2Fnginx. Named upstreams keep +// their upstream/{name}/ prefix, also when containerd reaches them through +// ns. Homebrew bottles are stored under Homebrew's image name, which writes +// openssl@3 as homebrew/core/openssl/3, and Helm charts pulled over OCI go +// through here like images. +// Package rules write the repository name as a path, pkg:oci/library/nginx, +// and are mapped to that form. +func init() { + retention.Default.Register(retention.Spec{Key: "oci", CanonicalPackage: ociRetentionKey}) +} + +// ociRetentionKey maps a package rule to the stored package PURL. It refuses +// a key with an empty path segment, a segment with ':' or '@' (a tag or +// digest from an image reference), or an upstream/{name} prefix without a +// repository, since no blob is ever stored under such a name. +func ociRetentionKey(p *purl.PURL) (string, bool) { + if p == nil { + return "", false + } + name := p.Name + if p.Namespace != "" { + name = p.Namespace + "/" + p.Name + } + segments := strings.Split(name, "/") + if slices.Contains(segments, "") || strings.ContainsAny(name, ":@") || + (segments[0] == "upstream" && len(segments) < registrySelectorParts) { + return "", false + } + return retention.DefaultCanonical("oci")(purl.New(p.Type, "", name, "", nil)) +} + // ContainerHandler handles OCI/Docker container registry protocol requests. // It implements the OCI Distribution Spec for pulling images. // Reference: https://github.com/opencontainers/distribution-spec/blob/main/spec.md diff --git a/internal/handler/container_test.go b/internal/handler/container_test.go index bd237cf..1681792 100644 --- a/internal/handler/container_test.go +++ b/internal/handler/container_test.go @@ -11,6 +11,7 @@ import ( "time" upstreamhttp "github.com/git-pkgs/proxy/internal/httpclient" + "github.com/git-pkgs/purl" "github.com/git-pkgs/registries/fetch" ) @@ -1336,3 +1337,37 @@ func TestContainerHandler_Routes_VersionCheck(t *testing.T) { t.Errorf("Docker-Distribution-Api-Version = %q, want %q", got, "registry/2.0") } } + +func TestOCIRetentionKey(t *testing.T) { + tests := []struct { + key string + want string + ok bool + }{ + {"pkg:oci/library/nginx", "pkg:oci/library%2Fnginx", true}, + {"pkg:oci/library%2Fnginx", "pkg:oci/library%2Fnginx", true}, + {"pkg:oci/nginx", "pkg:oci/nginx", true}, + {"pkg:oci/upstream/ghcr/owner/app", "pkg:oci/upstream%2Fghcr%2Fowner%2Fapp", true}, + {"pkg:oci/homebrew/core/wget", "pkg:oci/homebrew%2Fcore%2Fwget", true}, + {"pkg:oci/upstream/ghcr", "", false}, + {"pkg:oci/library%2F%2Fnginx", "", false}, + {"pkg:oci/%2Fnginx", "", false}, + {"pkg:oci/library/nginx:latest", "", false}, + {"pkg:oci/homebrew/core/openssl%403", "", false}, + {"pkg:oci/homebrew/core/openssl/3", "pkg:oci/homebrew%2Fcore%2Fopenssl%2F3", true}, + {"pkg:npm/library/nginx", "", false}, + } + for _, tt := range tests { + p, err := purl.Parse(tt.key) + if err != nil { + t.Fatalf("Parse(%s): %v", tt.key, err) + } + got, ok := ociRetentionKey(p) + if ok != tt.ok || (ok && got != tt.want) { + t.Errorf("ociRetentionKey(%s) = %q, %v, want %q, %v", tt.key, got, ok, tt.want, tt.ok) + } + } + if _, ok := ociRetentionKey(nil); ok { + t.Error("ociRetentionKey(nil) accepted") + } +} diff --git a/internal/server/retention_oci_test.go b/internal/server/retention_oci_test.go new file mode 100644 index 0000000..b99b28c --- /dev/null +++ b/internal/server/retention_oci_test.go @@ -0,0 +1,236 @@ +package server + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "io" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/git-pkgs/proxy/internal/config" + "github.com/git-pkgs/proxy/internal/database" + "github.com/git-pkgs/proxy/internal/handler" + "github.com/git-pkgs/proxy/internal/retention" + "github.com/git-pkgs/purl" + "github.com/git-pkgs/registries/fetch" +) + +// ociBlob is a blob of one repository, addressed by its digest. +type ociBlob struct { + repository string + content string +} + +func (b ociBlob) digest() string { + sum := sha256.Sum256([]byte(b.content)) + return "sha256:" + hex.EncodeToString(sum[:]) +} + +var ( + ociNginxBlob = ociBlob{"library/nginx", "nginx layer"} + ociWgetBlob = ociBlob{"homebrew/core/wget", "wget bottle"} + ociAppBlob = ociBlob{"owner/app", "app layer"} +) + +// ociRetentionRegistry serves the given blobs the way a registry does, +// counting downloads. +func ociRetentionRegistry(t *testing.T, downloads *atomic.Int64, blobs ...ociBlob) *httptest.Server { + t.Helper() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + for _, b := range blobs { + if r.URL.Path == "/v2/"+b.repository+"/blobs/"+b.digest() { + downloads.Add(1) + _, _ = io.WriteString(w, b.content) + return + } + } + http.NotFound(w, r) + })) + t.Cleanup(srv.Close) + return srv +} + +// newOCIRetentionProxy serves a default registry holding nginx and the +// Homebrew bottle, and a named upstream "ghcr" holding owner/app. +func newOCIRetentionProxy(t *testing.T) (srv *httptest.Server, db *database.DB, ghcrHost string, downloads *atomic.Int64) { + t.Helper() + downloads = &atomic.Int64{} + defaultRegistry := ociRetentionRegistry(t, downloads, ociNginxBlob, ociWgetBlob) + ghcr := ociRetentionRegistry(t, downloads, ociAppBlob) + + db, store := setupEvictionTest(t) + // Both registries are plain-HTTP loopback servers; the default fetcher + // refuses loopback, a stock client reaches either. + fetcher := fetch.NewFetcher(fetch.WithHTTPClient(defaultRegistry.Client()), fetch.WithMaxRetries(0)) + t.Cleanup(func() { _ = fetcher.Close() }) + proxy := handler.NewProxy(db, store, fetcher, fetch.NewResolver(), discardLogger()) + proxy.HTTPClient = defaultRegistry.Client() + h := handler.NewContainerHandlerWithRegistry(proxy, "http://proxy.test", defaultRegistry.URL, map[string]string{"ghcr": ghcr.URL}) + handler.RegisterHomebrewArtifacts(h, defaultRegistry.URL) + srv = httptest.NewServer(h.Routes()) + t.Cleanup(srv.Close) + + u, err := url.Parse(ghcr.URL) + if err != nil { + t.Fatal(err) + } + return srv, db, u.Host, downloads +} + +func ociGet(t *testing.T, url string) { + t.Helper() + resp, err := http.Get(url) + if err != nil { + t.Fatalf("GET %s: %v", url, err) + } + body, _ := io.ReadAll(resp.Body) + _ = resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Fatalf("GET %s = %d %s, want 200", url, resp.StatusCode, body) + } +} + +// ociArtifact returns the record of the blob stored under cacheName. +func ociArtifact(t *testing.T, db *database.DB, cacheName string, b ociBlob) database.Artifact { + t.Helper() + art, err := db.GetArtifact(purl.MakePURLString("oci", cacheName, b.digest()), b.digest()) + if err != nil || art == nil { + t.Fatalf("reading blob of %s: %v", cacheName, err) + } + return *art +} + +func ociBackdate(t *testing.T, db *database.DB, b ociBlob, fetched, accessed time.Time) { + t.Helper() + query := db.Rebind(`UPDATE artifacts SET fetched_at = ?, last_accessed_at = ? WHERE filename = ?`) + if _, err := db.Exec(query, fetched, accessed, b.digest()); err != nil { + t.Fatalf("backdating %s: %v", b.repository, err) + } +} + +func TestOCIRegistersForRetention(t *testing.T) { + if _, ok := retention.Default.Lookup("oci"); !ok { + t.Fatal("oci is not registered for retention") + } + if key, ok := retention.Default.KeyForDBEcosystem("oci"); !ok || key != "oci" { + t.Errorf("KeyForDBEcosystem(oci) = %q, %v, want oci, true", key, ok) + } +} + +// A blob served from the cache stays while it is used, is evicted once its +// rule runs out, comes back from the registry on the next pull, and stays +// after that even with the evicted copy's old access time on the record. +func TestOCIRetentionEvictsAndRefetches(t *testing.T) { + srv, db, _, downloads := newOCIRetentionProxy(t) + blobURL := srv.URL + "/library/nginx/blobs/" + ociNginxBlob.digest() + + ociGet(t, blobURL) + ociGet(t, blobURL) + if got := downloads.Load(); got != 1 { + t.Fatalf("registry downloads = %d, want 1", got) + } + if !ociArtifact(t, db, "library/nginx", ociNginxBlob).LastAccessedAt.Valid { + t.Fatal("cache hit did not record an access") + } + + rules := retention.Rules{Ecosystems: map[string]time.Duration{"oci": retentionDay}} + now := time.Now() + sweepRetention(context.Background(), db, discardLogger(), retention.Default, rules, now) + if !ociArtifact(t, db, "library/nginx", ociNginxBlob).StoragePath.Valid { + t.Fatal("blob used today was evicted") + } + + old := now.Add(-2 * retentionDay) + ociBackdate(t, db, ociNginxBlob, old, old) + sweepRetention(context.Background(), db, discardLogger(), retention.Default, rules, now) + if ociArtifact(t, db, "library/nginx", ociNginxBlob).StoragePath.Valid { + t.Fatal("blob unused for two days kept under a one-day rule") + } + + ociGet(t, blobURL) + if got := downloads.Load(); got != 2 { + t.Errorf("registry downloads = %d, want 2 after the eviction", got) + } + art := ociArtifact(t, db, "library/nginx", ociNginxBlob) + ociBackdate(t, db, ociNginxBlob, art.FetchedAt.Time, now.Add(-30*retentionDay)) + sweepRetention(context.Background(), db, discardLogger(), retention.Default, rules, time.Now()) + if !ociArtifact(t, db, "library/nginx", ociNginxBlob).StoragePath.Valid { + t.Error("refetched blob evicted again by the next sweep") + } +} + +// Keys that parse as OCI PURLs but name no repository a blob is stored +// under are refused at validation. A path-style key is accepted. +func TestOCIRetentionRefusesUnmatchableKeys(t *testing.T) { + cfg := config.Default() + cfg.Storage.Retention.Packages = map[string]string{"pkg:oci/library/nginx": "0"} + if _, err := cfg.RetentionRules(retention.Default); err != nil { + t.Fatalf("RetentionRules(pkg:oci/library/nginx): %v", err) + } + for _, key := range []string{"pkg:oci/upstream/ghcr", "pkg:oci/library%2F%2Fnginx"} { + cfg.Storage.Retention.Packages = map[string]string{key: "0"} + _, err := cfg.RetentionRules(retention.Default) + if err == nil || !strings.Contains(err.Error(), "does not match how oci packages are stored") { + t.Errorf("RetentionRules(%s) error = %v, want the key refused", key, err) + } + } +} + +// Package rules reach a Docker Hub repository, a Homebrew bottle and a +// named upstream's repository, the last also when containerd pulls it +// through ns instead of the upstream/ prefix. +func TestOCIRetentionPackageRules(t *testing.T) { + srv, db, ghcrHost, _ := newOCIRetentionProxy(t) + pull := func() { + ociGet(t, srv.URL+"/library/nginx/blobs/"+ociNginxBlob.digest()) + ociGet(t, srv.URL+"/homebrew/core/wget/blobs/"+ociWgetBlob.digest()) + ociGet(t, srv.URL+"/owner/app/blobs/"+ociAppBlob.digest()+"?ns="+ghcrHost) + } + pull() + + var stored string + art := ociArtifact(t, db, "upstream/ghcr/owner/app", ociAppBlob) + if err := db.Get(&stored, db.Rebind(`SELECT package_purl FROM versions WHERE purl = ?`), art.VersionPURL); err != nil { + t.Fatalf("reading stored package PURL: %v", err) + } + if stored != "pkg:oci/upstream%2Fghcr%2Fowner%2Fapp" { + t.Errorf("stored package PURL = %q, want pkg:oci/upstream%%2Fghcr%%2Fowner%%2Fapp", stored) + } + + blobs := []struct { + cacheName string + blob ociBlob + }{ + {"library/nginx", ociNginxBlob}, + {"homebrew/core/wget", ociWgetBlob}, + {"upstream/ghcr/owner/app", ociAppBlob}, + } + for _, kept := range []string{"pkg:oci/library/nginx", "pkg:oci/homebrew/core/wget", "pkg:oci/upstream/ghcr/owner/app"} { + pull() + old := time.Now().Add(-30 * retentionDay) + for _, b := range blobs { + ociBackdate(t, db, b.blob, old, old) + } + cfg := config.Default() + cfg.Storage.Retention.Ecosystems = map[string]string{"oci": "1d"} + cfg.Storage.Retention.Packages = map[string]string{kept: "0"} + rules, err := cfg.RetentionRules(retention.Default) + if err != nil { + t.Fatalf("RetentionRules(%s): %v", kept, err) + } + sweepRetention(context.Background(), db, discardLogger(), retention.Default, rules, time.Now()) + + for _, b := range blobs { + want := kept == "pkg:oci/"+b.cacheName + if got := ociArtifact(t, db, b.cacheName, b.blob).StoragePath.Valid; got != want { + t.Errorf("with %s: %s cached = %v, want %v", kept, b.cacheName, got, want) + } + } + } +} From c61cb51a0ea48aa6b2cb0e266e10cedd60231a0c Mon Sep 17 00:00:00 2001 From: Christian Heim Date: Mon, 12 Oct 2026 14:04:16 +0200 Subject: [PATCH 213/633] Enable storage retention for golang (#442) Module paths are stored lowercased, so package rules match whatever case they're written in. A major version like /v2 is its own module and needs its own rule. Keys in the !-escaped form from proxy URLs and the module cache are refused, since they could never match. Refs #306 --- docs/configuration.md | 2 +- internal/handler/go.go | 12 + internal/server/retention_golang_test.go | 265 +++++++++++++++++++++++ 3 files changed, 278 insertions(+), 1 deletion(-) create mode 100644 internal/server/retention_golang_test.go diff --git a/docs/configuration.md b/docs/configuration.md index 1c7887a..4446df2 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -96,7 +96,7 @@ A package rule wins over an ecosystem rule, which wins over `default`. Package r | `cran` | no | - | | `deb` | no | - | | `gem` | no | - | -| `golang` | no | - | +| `golang` | yes | `pkg:golang/github.com/stretchr/testify`, the module path as in `go.mod`; case doesn't matter. A major version like `/v2` (or `.v3` on gopkg.in) is a module of its own and needs its own rule. The `!`-escaped form from proxy URLs and the module cache (`github.com/!azure/...`) is refused. | | `helm` | no | - | | `hex` | no | - | | `julia` | no | - | diff --git a/internal/handler/go.go b/internal/handler/go.go index fb85587..353edbb 100644 --- a/internal/handler/go.go +++ b/internal/handler/go.go @@ -6,6 +6,7 @@ import ( "net/http" "strings" + "github.com/git-pkgs/proxy/internal/retention" "github.com/git-pkgs/registries/fetch" ) @@ -14,6 +15,17 @@ const ( asciiCaseOffset = 32 // difference between lowercase and uppercase ASCII letters ) +// Module zips are stored under pkg:golang/{module path}, which the PURL +// rules lowercase, so a package rule covers a module whatever case either +// is written in. A major version suffix (/v2) is part of the module path +// and so a module of its own. A key in the escaped form proxy URLs and the +// module cache use (github.com/!azure) does not survive the default +// mapping's round trip and is refused; the handler stores decoded paths, +// so such a key could never match. +func init() { + retention.Default.Register(retention.Spec{Key: "golang", CanonicalPackage: retention.DefaultCanonical("golang")}) +} + // GoHandler handles Go module proxy protocol requests. type GoHandler struct { proxy *Proxy diff --git a/internal/server/retention_golang_test.go b/internal/server/retention_golang_test.go new file mode 100644 index 0000000..50c8dde --- /dev/null +++ b/internal/server/retention_golang_test.go @@ -0,0 +1,265 @@ +package server + +import ( + "context" + "io" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/git-pkgs/proxy/internal/config" + "github.com/git-pkgs/proxy/internal/database" + "github.com/git-pkgs/proxy/internal/handler" + "github.com/git-pkgs/proxy/internal/mirror" + "github.com/git-pkgs/proxy/internal/retention" + "github.com/git-pkgs/registries/fetch" +) + +const ( + golangTestifyZip = "/github.com/stretchr/testify/@v/v1.8.4.zip" + // golangTOMLZip is github.com/BurntSushi/toml in the escaped form the + // go command requests it in. + golangTOMLZip = "/github.com/!burnt!sushi/toml/@v/v1.3.2.zip" + golangBackoffZip = "/github.com/cenkalti/backoff/v4/@v/v4.3.0.zip" +) + +// golangRetentionUpstream serves module zips the way proxy.golang.org does, +// counting downloads. +func golangRetentionUpstream(t *testing.T) (*httptest.Server, *atomic.Int64) { + t.Helper() + var downloads atomic.Int64 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case golangTestifyZip, golangTOMLZip, golangBackoffZip: + downloads.Add(1) + _, _ = io.WriteString(w, "module zip bytes") + default: + http.NotFound(w, r) + } + })) + t.Cleanup(srv.Close) + return srv, &downloads +} + +func newGolangRetentionProxy(t *testing.T, upstream *httptest.Server) (*httptest.Server, *database.DB) { + t.Helper() + db, store := setupEvictionTest(t) + // The upstream is a plain-HTTP loopback server; the default fetcher + // refuses loopback, a stock client reaches it. + fetcher := fetch.NewFetcher(fetch.WithHTTPClient(upstream.Client()), fetch.WithMaxRetries(0)) + t.Cleanup(func() { _ = fetcher.Close() }) + proxy := handler.NewProxy(db, store, fetcher, fetch.NewResolver(), discardLogger()) + proxy.HTTPClient = upstream.Client() + h := handler.NewGoHandlerWithUpstream(proxy, "http://proxy.test", upstream.URL) + srv := httptest.NewServer(h.Routes()) + t.Cleanup(srv.Close) + return srv, db +} + +func golangGet(t *testing.T, url string) { + t.Helper() + resp, err := http.Get(url) + if err != nil { + t.Fatalf("GET %s: %v", url, err) + } + body, _ := io.ReadAll(resp.Body) + _ = resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Fatalf("GET %s = %d %s, want 200", url, resp.StatusCode, body) + } +} + +// golangArtifact returns the record of the zip named filename. +func golangArtifact(t *testing.T, db *database.DB, filename string) database.Artifact { + t.Helper() + var art database.Artifact + if err := db.Get(&art, db.Rebind(`SELECT * FROM artifacts WHERE filename = ?`), filename); err != nil { + t.Fatalf("reading artifact %s: %v", filename, err) + } + return art +} + +func golangBackdate(t *testing.T, db *database.DB, filename string, fetched, accessed time.Time) { + t.Helper() + query := db.Rebind(`UPDATE artifacts SET fetched_at = ?, last_accessed_at = ? WHERE filename = ?`) + if _, err := db.Exec(query, fetched, accessed, filename); err != nil { + t.Fatalf("backdating %s: %v", filename, err) + } +} + +func TestGolangRegistersForRetention(t *testing.T) { + if _, ok := retention.Default.Lookup("golang"); !ok { + t.Fatal("golang is not registered for retention") + } + if key, ok := retention.Default.KeyForDBEcosystem("golang"); !ok || key != "golang" { + t.Errorf("KeyForDBEcosystem(golang) = %q, %v, want golang, true", key, ok) + } +} + +// A module zip served from the cache stays while it is used, is evicted +// once its rule runs out, comes back from upstream on the next download, and +// stays after that even with the evicted copy's old access time on the +// record. +func TestGolangRetentionEvictsAndRefetches(t *testing.T) { + upstream, downloads := golangRetentionUpstream(t) + srv, db := newGolangRetentionProxy(t, upstream) + const zip = "testify@v1.8.4.zip" + + golangGet(t, srv.URL+golangTestifyZip) + golangGet(t, srv.URL+golangTestifyZip) + if got := downloads.Load(); got != 1 { + t.Fatalf("upstream downloads = %d, want 1", got) + } + if !golangArtifact(t, db, zip).LastAccessedAt.Valid { + t.Fatal("cache hit did not record an access") + } + + rules := retention.Rules{Ecosystems: map[string]time.Duration{"golang": retentionDay}} + now := time.Now() + sweepRetention(context.Background(), db, discardLogger(), retention.Default, rules, now) + if !golangArtifact(t, db, zip).StoragePath.Valid { + t.Fatal("zip used today was evicted") + } + + old := now.Add(-2 * retentionDay) + golangBackdate(t, db, zip, old, old) + sweepRetention(context.Background(), db, discardLogger(), retention.Default, rules, now) + if golangArtifact(t, db, zip).StoragePath.Valid { + t.Fatal("zip unused for two days kept under a one-day rule") + } + + golangGet(t, srv.URL+golangTestifyZip) + if got := downloads.Load(); got != 2 { + t.Errorf("upstream downloads = %d, want 2 after the eviction", got) + } + art := golangArtifact(t, db, zip) + golangBackdate(t, db, zip, art.FetchedAt.Time, now.Add(-30*retentionDay)) + sweepRetention(context.Background(), db, discardLogger(), retention.Default, rules, time.Now()) + if !golangArtifact(t, db, zip).StoragePath.Valid { + t.Error("refetched zip evicted again by the next sweep") + } +} + +// Keys in the escaped form, written out or percent-encoded, parse as Go +// PURLs but can never match a stored module, so they are refused at +// validation instead of becoming rules that do nothing. The same module +// written as in go.mod is accepted. +func TestGolangRetentionRefusesEscapedKeys(t *testing.T) { + cfg := config.Default() + cfg.Storage.Retention.Packages = map[string]string{"pkg:golang/github.com/BurntSushi/toml": "0"} + if _, err := cfg.RetentionRules(retention.Default); err != nil { + t.Fatalf("RetentionRules(pkg:golang/github.com/BurntSushi/toml): %v", err) + } + for _, key := range []string{"pkg:golang/github.com/!burnt!sushi/toml", "pkg:golang/github.com/%21burnt%21sushi/toml"} { + cfg := config.Default() + cfg.Storage.Retention.Packages = map[string]string{key: "0"} + _, err := cfg.RetentionRules(retention.Default) + if err == nil || !strings.Contains(err.Error(), "does not match how golang packages are stored") { + t.Errorf("RetentionRules(%s) error = %v, want the key refused", key, err) + } + } +} + +// A package rule reaches a module whose path has capitals, whatever case +// the rule is written in, and a rule for a module does not cover its /v4 +// major version. +func TestGolangRetentionPackageRules(t *testing.T) { + upstream, _ := golangRetentionUpstream(t) + srv, db := newGolangRetentionProxy(t, upstream) + golangGet(t, srv.URL+golangTOMLZip) + golangGet(t, srv.URL+golangTestifyZip) + golangGet(t, srv.URL+golangBackoffZip) + const toml, testify, backoff = "toml@v1.3.2.zip", "testify@v1.8.4.zip", "v4@v4.3.0.zip" + + var stored string + zip := golangArtifact(t, db, toml) + if err := db.Get(&stored, db.Rebind(`SELECT package_purl FROM versions WHERE purl = ?`), zip.VersionPURL); err != nil { + t.Fatalf("reading stored package PURL: %v", err) + } + if stored != "pkg:golang/github.com/burntsushi/toml" { + t.Errorf("stored package PURL = %q, want pkg:golang/github.com/burntsushi/toml", stored) + } + + for _, keys := range []map[string]string{ + {"pkg:golang/github.com/BurntSushi/toml": "0", "pkg:golang/github.com/cenkalti/backoff": "0"}, + {"pkg:golang/github.com/burntsushi/toml": "0", "pkg:golang/github.com/cenkalti/backoff": "0"}, + } { + old := time.Now().Add(-30 * retentionDay) + for _, filename := range []string{toml, testify, backoff} { + golangBackdate(t, db, filename, old, old) + } + cfg := config.Default() + cfg.Storage.Retention.Ecosystems = map[string]string{"golang": "1d"} + cfg.Storage.Retention.Packages = keys + rules, err := cfg.RetentionRules(retention.Default) + if err != nil { + t.Fatalf("RetentionRules(%v): %v", keys, err) + } + sweepRetention(context.Background(), db, discardLogger(), retention.Default, rules, time.Now()) + + if !golangArtifact(t, db, toml).StoragePath.Valid { + t.Errorf("toml evicted despite its never-evict rule %v", keys) + } + if golangArtifact(t, db, testify).StoragePath.Valid { + t.Error("testify kept past the golang rule") + } + if golangArtifact(t, db, backoff).StoragePath.Valid { + t.Error("backoff/v4 kept by the rule for backoff") + } + golangGet(t, srv.URL+golangTestifyZip) + golangGet(t, srv.URL+golangBackoffZip) + } +} + +// golangRewriteTransport sends every request to the test upstream, so the +// mirror command's fixed proxy.golang.org URLs reach it. +type golangRewriteTransport struct { + target *url.URL + base http.RoundTripper +} + +func (r golangRewriteTransport) RoundTrip(req *http.Request) (*http.Response, error) { + req = req.Clone(req.Context()) + req.URL.Scheme = r.target.Scheme + req.URL.Host = r.target.Host + req.Host = r.target.Host + return r.base.RoundTrip(req) +} + +// A module preloaded with the mirror command falls under the golang rule. +func TestGolangRetentionCoversMirroredModules(t *testing.T) { + upstream, downloads := golangRetentionUpstream(t) + target, err := url.Parse(upstream.URL) + if err != nil { + t.Fatal(err) + } + db, store := setupEvictionTest(t) + client := &http.Client{Transport: golangRewriteTransport{target: target, base: upstream.Client().Transport}} + fetcher := fetch.NewFetcher(fetch.WithHTTPClient(client), fetch.WithMaxRetries(0)) + t.Cleanup(func() { _ = fetcher.Close() }) + proxy := handler.NewProxy(db, store, fetcher, fetch.NewResolver(), discardLogger()) + + m := mirror.New(proxy, db, store, discardLogger(), 1) + progress, err := m.Run(context.Background(), &mirror.PURLSource{PURLs: []string{"pkg:golang/github.com/stretchr/testify@v1.8.4"}}) + if err != nil || progress.Failed != 0 || downloads.Load() != 1 { + t.Fatalf("mirror run = %+v, %v, downloads %d; want one successful download", progress, err, downloads.Load()) + } + + const zip = "testify@v1.8.4.zip" + if golangArtifact(t, db, zip).LastAccessedAt.Valid { + t.Fatal("mirrored module has an access recorded before anyone downloaded it") + } + query := db.Rebind(`UPDATE artifacts SET fetched_at = ? WHERE filename = ?`) + if _, err := db.Exec(query, time.Now().Add(-30*retentionDay), zip); err != nil { + t.Fatalf("backdating %s: %v", zip, err) + } + rules := retention.Rules{Ecosystems: map[string]time.Duration{"golang": retentionDay}} + sweepRetention(context.Background(), db, discardLogger(), retention.Default, rules, time.Now()) + if golangArtifact(t, db, zip).StoragePath.Valid { + t.Error("never-served mirrored module kept past the golang rule") + } +} From 18f410d021f05c7f9f79996f09e0e897929c319e Mon Sep 17 00:00:00 2001 From: Christian Heim Date: Mon, 12 Oct 2026 14:06:37 +0200 Subject: [PATCH 214/633] Enable storage retention for pypi (#441) * Enable storage retention for pypi PyPI names are stored the way purl normalizes them, lowercased and with _ turned into -, so Typing_Extensions and typing-extensions hit the same package. The mirror can't fetch PyPI files today, so there's no mirror test. Refs #306 * Correct the pypi package rule note Dots in PyPI names aren't normalized, so zope.interface and zope_interface files end up under two packages. The docs now say to add a rule for each. Refs #306 * Mention repeated separators in the pypi rule note Runs like "__" aren't collapsed either, so a__b is stored as a--b. Refs #306 --- docs/configuration.md | 2 +- internal/handler/pypi.go | 11 ++ internal/server/retention_pypi_test.go | 195 +++++++++++++++++++++++++ 3 files changed, 207 insertions(+), 1 deletion(-) create mode 100644 internal/server/retention_pypi_test.go diff --git a/docs/configuration.md b/docs/configuration.md index 4446df2..e5388cf 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -105,7 +105,7 @@ A package rule wins over an ecosystem rule, which wins over `default`. Package r | `nuget` | no | - | | `oci` | yes | `pkg:oci/library/nginx`, the repository name of the pull. A named upstream keeps its prefix, also when containerd reaches it through `ns`: `ghcr.io/owner/app` pulled that way is `pkg:oci/upstream/ghcr/owner/app`. Homebrew bottles use Homebrew's image name, with `@` written as `/` and `+` as `x` (`openssl@3` is `pkg:oci/homebrew/core/openssl/3`). Helm charts pulled over OCI are stored like images. Only blobs are evicted; manifests stay in the metadata cache. | | `pub` | no | - | -| `pypi` | no | - | +| `pypi` | yes | `pkg:pypi/requests`; case and single `_`/`-` don't matter. Dots and repeated separators are kept as written, so a project like `zope.interface` can be stored as `zope.interface` and `zope-interface`; add a rule for each. | | `rpm` | no | - | | `swift` | no | - | diff --git a/internal/handler/pypi.go b/internal/handler/pypi.go index e37fddb..f4cef06 100644 --- a/internal/handler/pypi.go +++ b/internal/handler/pypi.go @@ -12,6 +12,8 @@ import ( "strconv" "strings" "time" + + "github.com/git-pkgs/proxy/internal/retention" ) const ( @@ -45,6 +47,15 @@ type PyPIHandler struct { proxyURL string } +// PyPI files are stored under pkg:pypi/{name}, the name taken from the +// filename and normalized the way purl does it: lowercased, "_" turned into +// "-". Package rules use the default key mapping, so pkg:pypi/Typing_Extensions +// matches typing-extensions. Dots and runs of separators are kept, so files +// named zope.interface-* and zope_interface-* end up under two packages. +func init() { + retention.Default.Register(retention.Spec{Key: "pypi", CanonicalPackage: retention.DefaultCanonical("pypi")}) +} + // NewPyPIHandler creates a new PyPI protocol handler. func NewPyPIHandler(proxy *Proxy, proxyURL string) *PyPIHandler { return NewPyPIHandlerWithUpstreams(proxy, proxyURL, "", "") diff --git a/internal/server/retention_pypi_test.go b/internal/server/retention_pypi_test.go new file mode 100644 index 0000000..c51e1ed --- /dev/null +++ b/internal/server/retention_pypi_test.go @@ -0,0 +1,195 @@ +package server + +import ( + "context" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/git-pkgs/proxy/internal/config" + "github.com/git-pkgs/proxy/internal/database" + "github.com/git-pkgs/proxy/internal/handler" + "github.com/git-pkgs/proxy/internal/retention" + "github.com/git-pkgs/registries/fetch" +) + +const ( + // pypiProxyPrefix is the route the proxy rewrites download links to: it + // keeps the upstream path, /packages/... included, after /packages. + pypiProxyPrefix = "/packages" + pypiWheelPath = "/packages/ab/cd/0123/Typing_Extensions-4.0.0-py3-none-any.whl" + pypiSdistPath = "/packages/ef/01/4567/requests-2.31.0.tar.gz" +) + +// pypiRetentionUpstream serves a wheel and an sdist, counting downloads. +func pypiRetentionUpstream(t *testing.T) (*httptest.Server, *atomic.Int64) { + t.Helper() + var downloads atomic.Int64 + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case pypiWheelPath, pypiSdistPath: + downloads.Add(1) + _, _ = io.WriteString(w, "distribution bytes") + default: + http.NotFound(w, r) + } + })) + t.Cleanup(upstream.Close) + return upstream, &downloads +} + +func newPyPIRetentionProxy(t *testing.T, upstream *httptest.Server) (*httptest.Server, *database.DB) { + t.Helper() + db, store := setupEvictionTest(t) + // The upstream's own client reaches the loopback test server, which the + // default fetcher refuses. + fetcher := fetch.NewFetcher(fetch.WithHTTPClient(upstream.Client()), fetch.WithMaxRetries(0)) + t.Cleanup(func() { _ = fetcher.Close() }) + proxy := handler.NewProxy(db, store, fetcher, fetch.NewResolver(), discardLogger()) + proxy.HTTPClient = upstream.Client() + h := handler.NewPyPIHandlerWithUpstreams(proxy, "http://proxy.test", upstream.URL, upstream.URL) + srv := httptest.NewServer(h.Routes()) + t.Cleanup(srv.Close) + return srv, db +} + +func pypiGet(t *testing.T, url string) { + t.Helper() + resp, err := http.Get(url) + if err != nil { + t.Fatalf("GET %s: %v", url, err) + } + body, _ := io.ReadAll(resp.Body) + _ = resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Fatalf("GET %s = %d %s, want 200", url, resp.StatusCode, body) + } +} + +// pypiArtifact returns the record of the file named filename. +func pypiArtifact(t *testing.T, db *database.DB, filename string) database.Artifact { + t.Helper() + var art database.Artifact + err := db.Get(&art, db.Rebind(`SELECT * FROM artifacts WHERE filename = ?`), filename) + if err != nil { + t.Fatalf("reading artifact %s: %v", filename, err) + } + return art +} + +func pypiBackdate(t *testing.T, db *database.DB, filename string, fetched, accessed time.Time) { + t.Helper() + query := db.Rebind(`UPDATE artifacts SET fetched_at = ?, last_accessed_at = ? WHERE filename = ?`) + if _, err := db.Exec(query, fetched, accessed, filename); err != nil { + t.Fatalf("backdating %s: %v", filename, err) + } +} + +func TestPyPIRegistersForRetention(t *testing.T) { + if _, ok := retention.Default.Lookup("pypi"); !ok { + t.Fatal("pypi is not registered for retention") + } + if key, ok := retention.Default.KeyForDBEcosystem("pypi"); !ok || key != "pypi" { + t.Errorf("KeyForDBEcosystem(pypi) = %q, %v, want pypi, true", key, ok) + } +} + +// A PyPI file served from the cache stays while it is being used, is +// evicted once its rule runs out, comes back from upstream on the next +// download, and the fresh fetch keeps it even though the evicted copy's old +// access time is still on the record. +func TestPyPIRetentionEvictsAndRefetches(t *testing.T) { + upstream, downloads := pypiRetentionUpstream(t) + srv, db := newPyPIRetentionProxy(t, upstream) + const filename = "requests-2.31.0.tar.gz" + + pypiGet(t, srv.URL+pypiProxyPrefix+pypiSdistPath) + pypiGet(t, srv.URL+pypiProxyPrefix+pypiSdistPath) + if got := downloads.Load(); got != 1 { + t.Fatalf("upstream downloads = %d, want 1 (second from cache)", got) + } + if art := pypiArtifact(t, db, filename); !art.LastAccessedAt.Valid { + t.Fatal("cache hit did not record an access") + } + + rules := retention.Rules{Ecosystems: map[string]time.Duration{"pypi": retentionDay}} + now := time.Now() + sweepRetention(context.Background(), db, discardLogger(), retention.Default, rules, now) + if !pypiArtifact(t, db, filename).StoragePath.Valid { + t.Fatal("file used today was evicted") + } + + sweepRetention(context.Background(), db, discardLogger(), retention.Default, rules, now.Add(2*retentionDay)) + if pypiArtifact(t, db, filename).StoragePath.Valid { + t.Fatal("file unused for two days kept under a one-day rule") + } + + pypiGet(t, srv.URL+pypiProxyPrefix+pypiSdistPath) + if got := downloads.Load(); got != 2 { + t.Errorf("upstream downloads = %d, want 2 after eviction", got) + } + old := time.Now().Add(-30 * retentionDay) + art := pypiArtifact(t, db, filename) + pypiBackdate(t, db, filename, art.FetchedAt.Time, old) + sweepRetention(context.Background(), db, discardLogger(), retention.Default, rules, time.Now()) + if !pypiArtifact(t, db, filename).StoragePath.Valid { + t.Error("refetched file evicted again by the next sweep") + } +} + +// Wheel names use underscores and the publisher's casing; a package rule +// written in any of the spellings PyPI treats as the same project has to +// reach the stored package. +func TestPyPIRetentionPackageRuleMatchesStoredPackage(t *testing.T) { + upstream, _ := pypiRetentionUpstream(t) + srv, db := newPyPIRetentionProxy(t, upstream) + pypiGet(t, srv.URL+pypiProxyPrefix+pypiWheelPath) + pypiGet(t, srv.URL+pypiProxyPrefix+pypiSdistPath) + + wheel := pypiArtifact(t, db, "Typing_Extensions-4.0.0-py3-none-any.whl") + var stored string + if err := db.Get(&stored, db.Rebind(`SELECT package_purl FROM versions WHERE purl = ?`), wheel.VersionPURL); err != nil { + t.Fatalf("reading stored package PURL: %v", err) + } + + for _, key := range []string{"pkg:pypi/typing-extensions", "pkg:pypi/Typing_Extensions", "pkg:pypi/typing_extensions"} { + cfg := config.Default() + cfg.Storage.Retention.Packages = map[string]string{key: "0"} + rules, err := cfg.RetentionRules(retention.Default) + if err != nil { + t.Fatalf("RetentionRules(%s): %v", key, err) + } + if _, ok := rules.Packages[stored]; !ok { + t.Errorf("rule %s resolved to %v, want the stored package %s", key, rules.Packages, stored) + } + } + + // With both files long unused, the "0" rule keeps the wheel while the + // ecosystem rule evicts the requests sdist. + old := time.Now().Add(-30 * retentionDay) + for _, filename := range []string{"Typing_Extensions-4.0.0-py3-none-any.whl", "requests-2.31.0.tar.gz"} { + pypiBackdate(t, db, filename, old, old) + } + cfg := config.Default() + cfg.Storage.Retention.Ecosystems = map[string]string{"pypi": "1d"} + cfg.Storage.Retention.Packages = map[string]string{"pkg:pypi/Typing_Extensions": "0"} + rules, err := cfg.RetentionRules(retention.Default) + if err != nil { + t.Fatalf("RetentionRules: %v", err) + } + sweepRetention(context.Background(), db, discardLogger(), retention.Default, rules, time.Now()) + + if !pypiArtifact(t, db, "Typing_Extensions-4.0.0-py3-none-any.whl").StoragePath.Valid { + t.Error("typing-extensions evicted despite its never-evict rule") + } + if pypiArtifact(t, db, "requests-2.31.0.tar.gz").StoragePath.Valid { + t.Error("requests kept past the pypi rule") + } + if !strings.HasPrefix(stored, "pkg:pypi/") { + t.Errorf("stored package PURL = %q, want a pkg:pypi PURL", stored) + } +} From 4b8678c2c5797e7204a0b9a866baf71d9f13b576 Mon Sep 17 00:00:00 2001 From: Christian Heim Date: Mon, 12 Oct 2026 14:21:38 +0200 Subject: [PATCH 215/633] Enable storage retention for npm (#440) * Enable storage retention for npm npm stores packages under the PURL purl builds from the full name, so package rules use the default mapping. Both pkg:npm/@babel/core and pkg:npm/%40babel/core work. Refs #306 * Tighten npm retention tests and docs The docs now say npm package rules match the name exactly, so pkg:npm/jsonstream doesn't cover JSONStream. The refetch, package rule and mirror tests were weaker than their names and now check what they claim. Refs #306 --------- Co-authored-by: Andrew Nesbitt --- docs/configuration.md | 4 +- internal/handler/npm.go | 8 + internal/server/retention_npm_test.go | 236 ++++++++++++++++++++++++++ 3 files changed, 246 insertions(+), 2 deletions(-) create mode 100644 internal/server/retention_npm_test.go diff --git a/docs/configuration.md b/docs/configuration.md index e5388cf..41ba4d0 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -84,7 +84,7 @@ storage: Each cached file ages on its own, counted from the last time the proxy served it, or from when it was fetched if it has not been served since. A newer release of the same package does not shorten the life of older versions. Durations take a `d` suffix for days, and `"0"` or an empty value means never evict by age. With no retention configured, artifacts stay until `max_size` evicts them. `sweep_interval` must be at least `1m`. -A package rule wins over an ecosystem rule, which wins over `default`. Package rules are keyed by package PURL without a version. The ecosystems and package rules can only name ecosystems that support retention in the running version; anything else, including a typo, stops the proxy at startup with an error naming the setting. `default` applies only to supported ecosystems. Support is added one ecosystem at a time, so a later release can make an existing `default` cover more ecosystems. On startup the proxy logs which ecosystems retention covers (`covered`) and which ones `default` does not reach yet (`default_not_applied`). +A package rule wins over an ecosystem rule, which wins over `default`. Package rules are keyed by package PURL without a version, written in one of the forms the table below lists for the ecosystem. The ecosystems and package rules can only name ecosystems that support retention in the running version; anything else, including a typo, stops the proxy at startup with an error naming the setting. `default` applies only to supported ecosystems. Support is added one ecosystem at a time, so a later release can make an existing `default` cover more ecosystems. On startup the proxy logs which ecosystems retention covers (`covered`) and which ones `default` does not reach yet (`default_not_applied`). | Ecosystem key | Retention supported | Package rule form | |---------------|---------------------|-------------------| @@ -101,7 +101,7 @@ A package rule wins over an ecosystem rule, which wins over `default`. Package r | `hex` | no | - | | `julia` | no | - | | `maven` | yes | `pkg:maven/com.google.guava/guava`, group and artifact as published, case included. Gradle build cache entries are not covered; they use `gradle.build_cache.max_age`. | -| `npm` | no | - | +| `npm` | yes | `pkg:npm/lodash`, `pkg:npm/@babel/core`; the name has to match exactly, case included, so `pkg:npm/jsonstream` doesn't cover `JSONStream`. | | `nuget` | no | - | | `oci` | yes | `pkg:oci/library/nginx`, the repository name of the pull. A named upstream keeps its prefix, also when containerd reaches it through `ns`: `ghcr.io/owner/app` pulled that way is `pkg:oci/upstream/ghcr/owner/app`. Homebrew bottles use Homebrew's image name, with `@` written as `/` and `+` as `x` (`openssl@3` is `pkg:oci/homebrew/core/openssl/3`). Helm charts pulled over OCI are stored like images. Only blobs are evicted; manifests stay in the metadata cache. | | `pub` | no | - | diff --git a/internal/handler/npm.go b/internal/handler/npm.go index 5c8fbd7..3ed9beb 100644 --- a/internal/handler/npm.go +++ b/internal/handler/npm.go @@ -14,6 +14,8 @@ import ( "strings" "sync" "time" + + "github.com/git-pkgs/proxy/internal/retention" ) const ( @@ -33,6 +35,12 @@ const ( npmSecurityMaxBody = 16 << 20 ) +// npm tarballs are stored under pkg:npm/{name} as purl builds it, so package +// rules use the default key mapping. +func init() { + retention.Default.Register(retention.Spec{Key: "npm", CanonicalPackage: retention.DefaultCanonical("npm")}) +} + // NPMHandler handles npm registry protocol requests. type NPMHandler struct { proxy *Proxy diff --git a/internal/server/retention_npm_test.go b/internal/server/retention_npm_test.go new file mode 100644 index 0000000..36c4fef --- /dev/null +++ b/internal/server/retention_npm_test.go @@ -0,0 +1,236 @@ +package server + +import ( + "context" + "io" + "net/http" + "net/http/httptest" + "net/url" + "sync/atomic" + "testing" + "time" + + "github.com/git-pkgs/proxy/internal/config" + "github.com/git-pkgs/proxy/internal/database" + "github.com/git-pkgs/proxy/internal/handler" + "github.com/git-pkgs/proxy/internal/mirror" + "github.com/git-pkgs/proxy/internal/retention" + "github.com/git-pkgs/registries/fetch" +) + +// npmRetentionUpstream serves a packument and a tarball for lodash and +// @babel/core, counting tarball downloads. +func npmRetentionUpstream(t *testing.T) (*httptest.Server, *atomic.Int64) { + t.Helper() + var tarballs atomic.Int64 + var upstream *httptest.Server + upstream = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/lodash", "/@babel%2fcore", "/@babel%2Fcore", "/@babel/core": + name := "lodash" + tarball := upstream.URL + "/lodash/-/lodash-1.0.0.tgz" + if r.URL.Path != "/lodash" { + name = "@babel/core" + tarball = upstream.URL + "/@babel/core/-/core-1.0.0.tgz" + } + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, `{"name":"`+name+`","versions":{"1.0.0":{"name":"`+name+ + `","version":"1.0.0","dist":{"tarball":"`+tarball+`"}}}}`) + case "/lodash/-/lodash-1.0.0.tgz", "/@babel/core/-/core-1.0.0.tgz": + tarballs.Add(1) + _, _ = io.WriteString(w, "tarball bytes") + default: + http.NotFound(w, r) + } + })) + t.Cleanup(upstream.Close) + return upstream, &tarballs +} + +func newNPMRetentionProxy(t *testing.T, upstream *httptest.Server) (*httptest.Server, *database.DB) { + t.Helper() + db, store := setupEvictionTest(t) + // The upstream's own client reaches the loopback test server, which the + // default fetcher refuses. + fetcher := fetch.NewFetcher(fetch.WithHTTPClient(upstream.Client()), fetch.WithMaxRetries(0)) + proxy := handler.NewProxy(db, store, fetcher, fetch.NewResolver(), discardLogger()) + proxy.HTTPClient = upstream.Client() + srv := httptest.NewServer(handler.NewNPMHandler(proxy, "http://proxy.test", upstream.URL).Routes()) + t.Cleanup(srv.Close) + return srv, db +} + +func getOK(t *testing.T, url string) { + t.Helper() + resp, err := http.Get(url) + if err != nil { + t.Fatalf("GET %s: %v", url, err) + } + body, _ := io.ReadAll(resp.Body) + _ = resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Fatalf("GET %s = %d %s, want 200", url, resp.StatusCode, body) + } +} + +func TestNPMRegistersForRetention(t *testing.T) { + if _, ok := retention.Default.Lookup("npm"); !ok { + t.Fatal("npm is not registered for retention") + } + if key, ok := retention.Default.KeyForDBEcosystem("npm"); !ok || key != "npm" { + t.Errorf("KeyForDBEcosystem(npm) = %q, %v, want npm, true", key, ok) + } +} + +// An npm tarball served from the cache stays while it is being used, is +// evicted once its rule runs out, and comes back from upstream on the next +// download. +func TestNPMRetentionEvictsAndRefetches(t *testing.T) { + upstream, tarballs := npmRetentionUpstream(t) + srv, db := newNPMRetentionProxy(t, upstream) + const versionPURL, filename = "pkg:npm/lodash@1.0.0", "lodash-1.0.0.tgz" + tarballURL := srv.URL + "/lodash/-/" + filename + + getOK(t, tarballURL) + getOK(t, tarballURL) + if got := tarballs.Load(); got != 1 { + t.Fatalf("upstream tarball downloads = %d, want 1 (second from cache)", got) + } + art, err := db.GetArtifact(versionPURL, filename) + if err != nil || !art.LastAccessedAt.Valid { + t.Fatalf("cache hit did not record an access: %+v, %v", art, err) + } + path := art.StoragePath.String + + rules := retention.Rules{Ecosystems: map[string]time.Duration{"npm": retentionDay}} + now := time.Now() + sweepRetention(context.Background(), db, discardLogger(), retention.Default, rules, now) + if !isCached(t, db, "npm", "lodash") { + t.Fatal("artifact used today was evicted") + } + + sweepRetention(context.Background(), db, discardLogger(), retention.Default, rules, now.Add(2*retentionDay)) + if isCached(t, db, "npm", "lodash") { + t.Fatal("artifact unused for two days kept under a one-day rule") + } + if got := queuedPaths(t, db); len(got) != 1 || got[0] != path { + t.Errorf("queue = %v, want [%s]", got, path) + } + + getOK(t, tarballURL) + if got := tarballs.Load(); got != 2 { + t.Errorf("upstream tarball downloads = %d, want 2 after eviction", got) + } + // The refetched record keeps the access time of the evicted copy; age it + // past the rule so only the fresh fetch time can keep it. + backdate(t, db, "last_accessed_at", versionPURL, time.Now().Add(-30*retentionDay)) + sweepRetention(context.Background(), db, discardLogger(), retention.Default, rules, time.Now()) + if !isCached(t, db, "npm", "lodash") { + t.Error("refetched artifact evicted again by the next sweep") + } +} + +// backdate sets one timestamp column of an artifact's records. +func backdate(t *testing.T, db *database.DB, column, versionPURL string, at time.Time) { + t.Helper() + query := db.Rebind(`UPDATE artifacts SET ` + column + ` = ? WHERE version_purl = ?`) + if _, err := db.Exec(query, at, versionPURL); err != nil { + t.Fatalf("backdating %s: %v", column, err) + } +} + +// A package rule written the way people usually write npm PURLs has to name +// the package PURL the handler stores, scoped packages included. +func TestNPMRetentionPackageRuleMatchesStoredPackage(t *testing.T) { + upstream, _ := npmRetentionUpstream(t) + srv, db := newNPMRetentionProxy(t, upstream) + getOK(t, srv.URL+"/@babel/core/-/core-1.0.0.tgz") + + var stored string + if err := db.Get(&stored, `SELECT package_purl FROM versions WHERE purl LIKE 'pkg:npm/%core@1.0.0'`); err != nil { + t.Fatalf("reading stored package PURL: %v", err) + } + + for _, key := range []string{"pkg:npm/@babel/core", "pkg:npm/%40babel/core"} { + cfg := config.Default() + cfg.Storage.Retention.Packages = map[string]string{key: "0"} + rules, err := cfg.RetentionRules(retention.Default) + if err != nil { + t.Fatalf("RetentionRules(%s): %v", key, err) + } + if _, ok := rules.Packages[stored]; !ok { + t.Errorf("rule %s resolved to %v, want the stored package %s", key, rules.Packages, stored) + } + } + + // With both packages long unused, the "0" rule keeps @babel/core while + // the ecosystem rule evicts lodash. + getOK(t, srv.URL+"/lodash/-/lodash-1.0.0.tgz") + old := time.Now().Add(-30 * retentionDay) + for _, versionPURL := range []string{"pkg:npm/lodash@1.0.0", "pkg:npm/%40babel/core@1.0.0"} { + backdate(t, db, "fetched_at", versionPURL, old) + backdate(t, db, "last_accessed_at", versionPURL, old) + } + cfg := config.Default() + cfg.Storage.Retention.Ecosystems = map[string]string{"npm": "1d"} + cfg.Storage.Retention.Packages = map[string]string{"pkg:npm/@babel/core": "0"} + rules, err := cfg.RetentionRules(retention.Default) + if err != nil { + t.Fatalf("RetentionRules: %v", err) + } + sweepRetention(context.Background(), db, discardLogger(), retention.Default, rules, time.Now()) + + babel, err := db.GetArtifact("pkg:npm/%40babel/core@1.0.0", "core-1.0.0.tgz") + if err != nil { + t.Fatalf("GetArtifact(@babel/core): %v", err) + } + if !babel.StoragePath.Valid { + t.Error("@babel/core evicted despite its never-evict rule") + } + if isCached(t, db, "npm", "lodash") { + t.Error("lodash kept past the npm rule") + } +} + +// rewriteToUpstream sends every request to the test upstream, so the mirror +// command's fixed registry.npmjs.org tarball URLs reach it. +type rewriteToUpstream struct { + target *url.URL + base http.RoundTripper +} + +func (r rewriteToUpstream) RoundTrip(req *http.Request) (*http.Response, error) { + req = req.Clone(req.Context()) + req.URL.Scheme = r.target.Scheme + req.URL.Host = r.target.Host + req.Host = r.target.Host + return r.base.RoundTrip(req) +} + +// A package preloaded with the mirror command falls under the npm rule: +// the mirror records it under its own ecosystem spelling and filename. +func TestNPMRetentionCoversMirroredPackages(t *testing.T) { + upstream, tarballs := npmRetentionUpstream(t) + target, err := url.Parse(upstream.URL) + if err != nil { + t.Fatal(err) + } + db, store := setupEvictionTest(t) + client := &http.Client{Transport: rewriteToUpstream{target: target, base: upstream.Client().Transport}} + fetcher := fetch.NewFetcher(fetch.WithHTTPClient(client), fetch.WithMaxRetries(0)) + t.Cleanup(func() { _ = fetcher.Close() }) + proxy := handler.NewProxy(db, store, fetcher, fetch.NewResolver(), discardLogger()) + + m := mirror.New(proxy, db, store, discardLogger(), 1) + progress, err := m.Run(context.Background(), &mirror.PURLSource{PURLs: []string{"pkg:npm/lodash@1.0.0"}}) + if err != nil || progress.Failed != 0 || tarballs.Load() != 1 { + t.Fatalf("mirror run = %+v, %v, tarball downloads %d; want one successful download", progress, err, tarballs.Load()) + } + + backdate(t, db, "fetched_at", "pkg:npm/lodash@1.0.0", time.Now().Add(-30*retentionDay)) + rules := retention.Rules{Ecosystems: map[string]time.Duration{"npm": retentionDay}} + sweepRetention(context.Background(), db, discardLogger(), retention.Default, rules, time.Now()) + if isCached(t, db, "npm", "lodash") { + t.Error("never-served mirrored npm artifact kept past its rule") + } +} From 6e334b4c4220f9d969da5430d4f9aaafe37690bd Mon Sep 17 00:00:00 2001 From: Ondrej Kokes Date: Mon, 12 Oct 2026 14:45:34 +0200 Subject: [PATCH 216/633] npm: version lookup returns 502 (#419) * npm: version lookup returns 502 Looking up a specific version will trigger a 502 instead of return the version metadata. * scope only not handled explicitly * honor cooldown in version lookups tested (set default cooldown to 7d and 8.0.7 is released four days ago or something): ``` $ curl -sS 'http://localhost:8080/npm/@babel%2Fcore/8.0.7' | jq { "error": "package not found" } ``` * package not found -> version not found * gocognit --- internal/handler/handler.go | 8 ++ internal/handler/npm.go | 188 ++++++++++++++++++++++-- internal/handler/npm_test.go | 267 +++++++++++++++++++++++++++++++++++ 3 files changed, 454 insertions(+), 9 deletions(-) diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 449fcef..036e2a5 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -1030,6 +1030,14 @@ var ErrArtifactBlocked = errors.New("artifact blocked by security scan") // ErrVersionDenied indicates an operator explicitly denied this package version. var ErrVersionDenied = errors.New("package version blocked by denylist") +// ErrVersionNotFound indicates the requested version is not available through +// the proxy, including when cooldown or other filtering removed it. +var ErrVersionNotFound = errors.New("version not found") + +// ErrPackageNotFound indicates the requested package metadata is not available +// through the proxy or upstream registry. +var ErrPackageNotFound = errors.New("package not found") + // serveArtifactError writes response for a failed fetch: // 404 when upstream reports artifact missing, 403 when a security scan or // denylist blocked the artifact, 502 otherwise. diff --git a/internal/handler/npm.go b/internal/handler/npm.go index 3ed9beb..9ce667b 100644 --- a/internal/handler/npm.go +++ b/internal/handler/npm.go @@ -183,16 +183,25 @@ func (h *NPMHandler) sourceFor(packageName string) (registryURL, cacheKey string // handlePackageMetadata proxies package metadata from upstream and rewrites tarball URLs. func (h *NPMHandler) handlePackageMetadata(w http.ResponseWriter, r *http.Request) { - packageName := h.extractPackageName(r) + packagePath := h.extractPackageName(r) + if packagePath == "" || containsPathTraversal(packagePath) { + JSONError(w, http.StatusBadRequest, "invalid package name") + return + } + packageName, registryPath := npmMetadataPath(packagePath) if packageName == "" { JSONError(w, http.StatusBadRequest, "invalid package name") return } - h.proxy.Logger.Info("npm metadata request", "package", packageName) + h.proxy.Logger.Info("npm metadata request", "package", packageName, "path", packagePath) registryURL, cacheKey := h.sourceFor(packageName) - upstreamURL := fmt.Sprintf("%s/%s", registryURL, url.PathEscape(packageName)) + // A version or dist-tag is its own path segment. Escaping the whole path + // turns "lodash/4.17.21" into "lodash%2F4.17.21", which the registry + // rejects with 405. + upstreamURL := registryURL + "/" + registryPath + metadataCacheKey := npmMetadataCacheKey(cacheKey, packagePath, packageName) // Prefer the smaller abbreviated packument format but include application/json // as a fallback so upstreams that reject the abbreviated type (e.g. JFrog @@ -206,17 +215,27 @@ func (h *NPMHandler) handlePackageMetadata(w http.ResponseWriter, r *http.Reques accept = contentTypeJSON } - if rewritten, ok := h.proxy.storedRewrite("npm", cacheKey, h.proxyURL, cacheKey); ok { + if rewritten, ok := h.proxy.storedRewrite("npm", metadataCacheKey, h.proxyURL, metadataCacheKey); ok { w.Header().Set(headerContentType, contentTypeJSON) w.WriteHeader(http.StatusOK) _, _ = w.Write(rewritten) return } - body, _, err := h.proxy.FetchOrCacheMetadata(r.Context(), "npm", cacheKey, upstreamURL, accept) + specifier, versionLookup := npmVersionSpecifier(packagePath, packageName) + if versionLookup && h.npmVersionMetadataUsesFilteredPackument(packageName) { + h.handleFilteredNPMVersionMetadata(w, r, packageName, specifier, accept, registryURL, cacheKey, metadataCacheKey) + return + } + + body, _, err := h.proxy.FetchOrCacheMetadata(r.Context(), "npm", metadataCacheKey, upstreamURL, accept) if err != nil { if errors.Is(err, ErrUpstreamNotFound) { - JSONError(w, http.StatusNotFound, "package not found") + if versionLookup { + JSONError(w, http.StatusNotFound, ErrVersionNotFound.Error()) + } else { + JSONError(w, http.StatusNotFound, ErrPackageNotFound.Error()) + } return } h.proxy.Logger.Error("failed to fetch npm metadata", "error", err) @@ -224,13 +243,21 @@ func (h *NPMHandler) handlePackageMetadata(w http.ResponseWriter, r *http.Reques return } - rewritten, err := h.proxy.cachedRewrite(r.Context(), "npm", h.proxyURL, cacheKey, body, func(b []byte) ([]byte, error) { + rewritten, err := h.proxy.cachedRewrite(r.Context(), "npm", h.proxyURL, metadataCacheKey, body, func(b []byte) ([]byte, error) { return h.rewriteMetadata(packageName, b) }) if err != nil { if r.Context().Err() != nil { return // the client left while waiting on a shared rewrite } + if errors.Is(err, ErrVersionDenied) { + JSONError(w, http.StatusForbidden, err.Error()) + return + } + if errors.Is(err, ErrVersionNotFound) { + JSONError(w, http.StatusNotFound, ErrVersionNotFound.Error()) + return + } if len(h.proxy.Denylist.Versions(canonicalPackagePURL("npm", packageName))) != 0 { JSONError(w, http.StatusBadGateway, "failed to filter package metadata") return @@ -248,6 +275,112 @@ func (h *NPMHandler) handlePackageMetadata(w http.ResponseWriter, r *http.Reques _, _ = w.Write(rewritten) } +// handleFilteredNPMVersionMetadata serves /pkg/ and /pkg/ from the +// filtered packument so cooldown and dist-tags match the root metadata response. +func (h *NPMHandler) handleFilteredNPMVersionMetadata(w http.ResponseWriter, r *http.Request, packageName, specifier, accept, registryURL, cacheKey, metadataCacheKey string) { + _, packumentPath := npmMetadataPath(packageName) + packument, _, err := h.proxy.FetchOrCacheMetadata(r.Context(), "npm", cacheKey, registryURL+"/"+packumentPath, accept) + if err != nil { + if errors.Is(err, ErrUpstreamNotFound) { + JSONError(w, http.StatusNotFound, ErrPackageNotFound.Error()) + return + } + h.proxy.Logger.Error("failed to fetch npm metadata", "error", err) + JSONError(w, http.StatusBadGateway, "failed to fetch from upstream") + return + } + + rewritten, err := h.proxy.cachedRewrite(r.Context(), "npm", h.proxyURL, metadataCacheKey, packument, func(b []byte) ([]byte, error) { + return h.rewriteNPMVersionFromPackument(packageName, specifier, b) + }) + if err != nil { + if r.Context().Err() != nil { + return + } + if errors.Is(err, ErrVersionDenied) { + JSONError(w, http.StatusForbidden, err.Error()) + return + } + if errors.Is(err, ErrVersionNotFound) { + JSONError(w, http.StatusNotFound, ErrVersionNotFound.Error()) + return + } + if len(h.proxy.Denylist.Versions(canonicalPackagePURL("npm", packageName))) != 0 { + JSONError(w, http.StatusBadGateway, "failed to filter package metadata") + return + } + h.proxy.Logger.Warn("failed to rewrite npm version metadata", "error", err) + JSONError(w, http.StatusBadGateway, "failed to filter package metadata") + return + } + + w.Header().Set(headerContentType, contentTypeJSON) + w.WriteHeader(http.StatusOK) + _, _ = w.Write(rewritten) +} + +// npmMetadataCacheKey qualifies routed-package metadata cache entries with the +// requested version or dist-tag when the client asks for a single version. +func npmMetadataCacheKey(cacheKey, packagePath, packageName string) string { + specifier, versionLookup := npmVersionSpecifier(packagePath, packageName) + if !versionLookup { + return cacheKey + } + return cacheKey + "/" + specifier +} + +// npmVersionSpecifier returns the version or dist-tag segment after the package +// name in a metadata path, for example "4.17.21" or "latest". +func npmVersionSpecifier(packagePath, packageName string) (string, bool) { + if packagePath == packageName { + return "", false + } + specifier, ok := strings.CutPrefix(packagePath, packageName+"/") + return specifier, ok +} + +func (h *NPMHandler) npmVersionMetadataUsesFilteredPackument(packageName string) bool { + if h.proxy.Cooldown != nil && h.proxy.Cooldown.Enabled() { + return true + } + if h.proxy.Denylist != nil { + return len(h.proxy.Denylist.Versions(canonicalPackagePURL("npm", packageName))) != 0 + } + return false +} + +// rewriteNPMVersionFromPackument picks one version entry out of an already +// filtered packument and rewrites its tarball URL. +func (h *NPMHandler) rewriteNPMVersionFromPackument(packageName, specifier string, packument []byte) ([]byte, error) { + filtered, err := h.rewriteMetadata(packageName, packument) + if err != nil { + return nil, err + } + + target := specifier + if resolved, ok, err := lookupJSONString(filtered, "dist-tags", specifier); err != nil { + return nil, err + } else if ok { + target = resolved + } + + if h.proxy.versionDenied("npm", packageName, target) { + return nil, ErrVersionDenied + } + + versionRaw, err := lookupJSON(filtered, "versions", target) + if err != nil { + return nil, err + } + if len(versionRaw) == 0 { + return nil, ErrVersionNotFound + } + + var out bytes.Buffer + h.writeNPMVersion(&out, packageName, target, versionRaw) + return out.Bytes(), nil +} + // rewriteMetadata rewrites tarball URLs in npm package metadata to point at this proxy. // If cooldown is enabled, versions published too recently are filtered out. // @@ -266,6 +399,18 @@ func (h *NPMHandler) rewriteMetadata(packageName string, body []byte) ([]byte, e versionsRaw := doc.value(doc.versionsAt) if len(versionsRaw) == 0 || versionsRaw[0] != '{' { + version, ok, err := lookupJSONString(body, "version") + if err != nil { + return nil, err + } + if ok { + if h.proxy.versionDenied("npm", packageName, version) { + return nil, ErrVersionDenied + } + var out bytes.Buffer + h.writeNPMVersion(&out, packageName, version, body) + return out.Bytes(), nil + } if len(h.proxy.Denylist.Versions(canonicalPackagePURL("npm", packageName))) != 0 { return nil, errors.New("npm metadata has no versions object") } @@ -620,7 +765,7 @@ func (h *NPMHandler) handleDownload(w http.ResponseWriter, r *http.Request) { if h.versionInCooldown(packageName, version, metadata) { h.proxy.Logger.Info("cooldown: withholding npm tarball", "package", packageName, "version", version) - JSONError(w, http.StatusNotFound, "version not found") + JSONError(w, http.StatusNotFound, ErrVersionNotFound.Error()) return } @@ -628,7 +773,7 @@ func (h *NPMHandler) handleDownload(w http.ResponseWriter, r *http.Request) { if err != nil { switch { case errors.Is(err, ErrUpstreamNotFound): - JSONError(w, http.StatusNotFound, "package not found") + JSONError(w, http.StatusNotFound, ErrPackageNotFound.Error()) case errors.Is(err, ErrArtifactBlocked), errors.Is(err, ErrVersionDenied): JSONError(w, http.StatusForbidden, err.Error()) default: @@ -754,6 +899,31 @@ func (h *NPMHandler) versionInCooldown(packageName, version string, metadata fun return !h.proxy.Cooldown.IsAllowed("npm", canonicalPackagePURL("npm", packageName), publishedAt) } +// npmMetadataPath returns the package name and the encoded registry path. +// Examples: lodash → lodash; lodash/4.17.21 → lodash/4.17.21; +// @babel/core → @babel%2Fcore; @babel/core/7.23.0 → @babel%2Fcore/7.23.0. +func npmMetadataPath(packagePath string) (packageName, registryPath string) { + // GET /npm/ has no package segment; handlePackageMetadata rejects that earlier. + if packagePath == "" { + return "", "" + } + if !strings.Contains(packagePath, "/") { + return packagePath, url.PathEscape(packagePath) + } + if strings.HasPrefix(packagePath, "@") { + scope, rest, _ := strings.Cut(packagePath, "/") + name, version, ok := strings.Cut(rest, "/") + packageName = scope + "/" + name + registryPath = url.PathEscape(scope) + "%2F" + url.PathEscape(name) + if ok { + registryPath += "/" + url.PathEscape(version) + } + return packageName, registryPath + } + name, version, _ := strings.Cut(packagePath, "/") + return name, url.PathEscape(name) + "/" + url.PathEscape(version) +} + func escapeNPMDownloadPackage(packageName string) string { scope, name, scoped := strings.Cut(packageName, "/") if scoped && strings.HasPrefix(scope, "@") && len(scope) > 1 && name != "" && !strings.Contains(name, "/") { diff --git a/internal/handler/npm_test.go b/internal/handler/npm_test.go index 5d8378f..5f45feb 100644 --- a/internal/handler/npm_test.go +++ b/internal/handler/npm_test.go @@ -1047,3 +1047,270 @@ func TestNPMTarballStillRoutesToDownload(t *testing.T) { t.Fatalf("status = %d, want %d; body: %s", w.Code, http.StatusOK, w.Body.String()) } } + +func TestNPMMetadataPath(t *testing.T) { + tests := []struct { + path, wantName, wantRegistry string + }{ + {"", "", ""}, + {"lodash", "lodash", "lodash"}, + {"lodash/4.17.21", "lodash", "lodash/4.17.21"}, + {"lodash/latest", "lodash", "lodash/latest"}, + {"@babel/core", "@babel/core", "@babel%2Fcore"}, + {"@babel/core/7.23.0", "@babel/core", "@babel%2Fcore/7.23.0"}, + } + for _, tt := range tests { + name, registry := npmMetadataPath(tt.path) + if name != tt.wantName || registry != tt.wantRegistry { + t.Errorf("npmMetadataPath(%q) = (%q, %q), want (%q, %q)", + tt.path, name, registry, tt.wantName, tt.wantRegistry) + } + } +} + +func TestNPMVersionMetadataKeepsVersionSegment(t *testing.T) { + tests := []struct { + request, upstreamPath, wantTarball string + body string + }{ + { + request: "/lodash/4.17.21", + upstreamPath: "/lodash/4.17.21", + wantTarball: "http://proxy.local/npm/lodash/-/lodash-4.17.21.tgz", + body: `{ + "name": "lodash", + "version": "4.17.21", + "dist": {"tarball": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz"} + }`, + }, + { + request: "/@babel/core/7.23.0", + upstreamPath: "/@babel%2Fcore/7.23.0", + wantTarball: "http://proxy.local/npm/@babel%2Fcore/-/core-7.23.0.tgz", + body: `{ + "name": "@babel/core", + "version": "7.23.0", + "dist": {"tarball": "https://registry.npmjs.org/@babel/core/-/core-7.23.0.tgz"} + }`, + }, + } + for _, tt := range tests { + t.Run(tt.request, func(t *testing.T) { + var gotPath string + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotPath = r.URL.EscapedPath() + if gotPath != tt.upstreamPath { + w.WriteHeader(http.StatusMethodNotAllowed) + return + } + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, tt.body) + })) + defer upstream.Close() + + h := &NPMHandler{proxy: testProxy(), upstreamURL: upstream.URL, proxyURL: "http://proxy.local"} + w := httptest.NewRecorder() + h.handlePackageMetadata(w, httptest.NewRequest(http.MethodGet, tt.request, nil)) + + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want 200; body: %s", w.Code, w.Body.String()) + } + if gotPath != tt.upstreamPath { + t.Fatalf("upstream path = %q, want %q", gotPath, tt.upstreamPath) + } + var result map[string]any + if err := json.Unmarshal(w.Body.Bytes(), &result); err != nil { + t.Fatalf("parse response: %v", err) + } + if result["dist"].(map[string]any)["tarball"] != tt.wantTarball { + t.Errorf("tarball = %v", result["dist"].(map[string]any)["tarball"]) + } + }) + } +} + +func TestNPMMetadataRejectsEmptyPath(t *testing.T) { + h := NewNPMHandler(testProxy(), "http://proxy.local", "http://unused.test") + w := httptest.NewRecorder() + h.Routes().ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/", nil)) + if w.Code != http.StatusBadRequest { + t.Fatalf("status = %d, want 400; body: %s", w.Code, w.Body.String()) + } +} + +func TestNPMVersionMetadataRejectsTraversal(t *testing.T) { + called := false + upstream := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { + called = true + })) + defer upstream.Close() + + h := &NPMHandler{proxy: testProxy(), upstreamURL: upstream.URL, proxyURL: "http://proxy.local"} + req := httptest.NewRequest(http.MethodGet, "/lodash/x", nil) + req.URL.Path = "/lodash/.." + w := httptest.NewRecorder() + h.handlePackageMetadata(w, req) + + if w.Code != http.StatusBadRequest { + t.Fatalf("status = %d, want 400; body: %s", w.Code, w.Body.String()) + } + if called { + t.Fatal("upstream contacted for traversal path") + } +} + +func TestNPMVersionMetadataCooldown(t *testing.T) { + proxyURL, upstreamPaths := npmCooldownMetadataHarness(t) + + resp, err := http.Get(proxyURL + "/demo") + if err != nil { + t.Fatalf("packument request failed: %v", err) + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode != http.StatusOK { + t.Fatalf("packument status = %d, want 200", resp.StatusCode) + } + var packument map[string]any + if err := json.NewDecoder(resp.Body).Decode(&packument); err != nil { + t.Fatalf("decode packument: %v", err) + } + distTags := packument["dist-tags"].(map[string]any) + if distTags["latest"] != testVersion100 { + t.Fatalf("dist-tags.latest = %v, want %s", distTags["latest"], testVersion100) + } + if _, ok := packument["versions"].(map[string]any)["2.0.0"]; ok { + t.Fatal("packument should not include version 2.0.0") + } + + for _, tt := range []struct { + path string + wantStatus int + wantVersion string + wantError string + }{ + {"/demo/latest", http.StatusOK, testVersion100, ""}, + {"/demo/1.0.0", http.StatusOK, testVersion100, ""}, + {"/demo/2.0.0", http.StatusNotFound, "", "version not found"}, + } { + t.Run(tt.path, func(t *testing.T) { + testNPMVersionMetadataCooldownPath(t, proxyURL, upstreamPaths, tt.path, tt.wantStatus, tt.wantVersion, tt.wantError) + }) + } +} + +// testNPMVersionMetadataCooldownPath is a subtest helper so the parent test stays +// under gocognit's complexity limit. +func testNPMVersionMetadataCooldownPath(t *testing.T, proxyURL string, upstreamPaths *[]string, path string, wantStatus int, wantVersion, wantError string) { + t.Helper() + before := len(*upstreamPaths) + resp, err := http.Get(proxyURL + path) + if err != nil { + t.Fatalf("request failed: %v", err) + } + defer func() { _ = resp.Body.Close() }() + body := readNPMTestBody(t, resp.Body) + if resp.StatusCode != wantStatus { + t.Fatalf("status = %d, want %d; body: %s", resp.StatusCode, wantStatus, body) + } + for _, upstreamPath := range (*upstreamPaths)[before:] { + if upstreamPath == path { + t.Fatalf("upstream must not be called for version path %s when answering from the filtered packument", path) + } + } + if wantError != "" { + if !strings.Contains(body, wantError) { + t.Fatalf("error body should mention %q, got %s", wantError, body) + } + return + } + var versionDoc map[string]any + if err := json.Unmarshal([]byte(body), &versionDoc); err != nil { + t.Fatalf("decode response: %v", err) + } + if versionDoc["version"] != wantVersion { + t.Fatalf("version = %v, want %s", versionDoc["version"], wantVersion) + } + wantTarball := "http://proxy.test/npm/demo/-/demo-" + wantVersion + ".tgz" + if versionDoc["dist"].(map[string]any)["tarball"] != wantTarball { + t.Fatalf("tarball = %v, want %s", versionDoc["dist"].(map[string]any)["tarball"], wantTarball) + } +} + +func npmCooldownMetadataHarness(t *testing.T) (proxyURL string, upstreamPaths *[]string) { + t.Helper() + now := time.Now() + old := now.Add(-30 * 24 * time.Hour).Format(time.RFC3339) + recent := now.Add(-1 * time.Hour).Format(time.RFC3339) + packument := `{ + "name": "demo", + "dist-tags": {"latest": "2.0.0"}, + "time": {"1.0.0": "` + old + `", "2.0.0": "` + recent + `"}, + "versions": { + "1.0.0": { + "name": "demo", + "version": "1.0.0", + "dist": {"tarball": "https://registry.npmjs.org/demo/-/demo-1.0.0.tgz"} + }, + "2.0.0": { + "name": "demo", + "version": "2.0.0", + "dist": {"tarball": "https://registry.npmjs.org/demo/-/demo-2.0.0.tgz"} + } + } + }` + version200 := `{"name":"demo","version":"2.0.0","dist":{"tarball":"https://registry.npmjs.org/demo/-/demo-2.0.0.tgz"}}` + version100 := `{"name":"demo","version":"1.0.0","dist":{"tarball":"https://registry.npmjs.org/demo/-/demo-1.0.0.tgz"}}` + + var paths []string + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + paths = append(paths, r.URL.Path) + w.Header().Set("Content-Type", contentTypeJSON) + switch r.URL.Path { + case "/demo": + _, _ = io.WriteString(w, packument) + case "/demo/latest", "/demo/2.0.0": + _, _ = io.WriteString(w, version200) + case "/demo/1.0.0": + _, _ = io.WriteString(w, version100) + default: + http.NotFound(w, r) + } + })) + t.Cleanup(upstream.Close) + + proxy, _, _, _ := setupTestProxy(t) + proxy.HTTPClient = upstream.Client() + proxy.Cooldown = &cooldown.Config{Default: "7d"} + srv := httptest.NewServer(NewNPMHandler(proxy, "http://proxy.test", upstream.URL).Routes()) + t.Cleanup(srv.Close) + return srv.URL, &paths +} + +func readNPMTestBody(t *testing.T, r io.Reader) string { + b, err := io.ReadAll(r) + if err != nil { + t.Fatalf("read body: %v", err) + } + return string(b) +} + +func TestNPMVersionMetadataDenied(t *testing.T) { + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = io.WriteString(w, `{ + "name": "lodash", + "version": "4.17.21", + "dist": {"tarball": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz"} + }`) + })) + defer upstream.Close() + + proxy := testProxy() + setTestDenylist(t, proxy, "pkg:npm/lodash@4.17.21") + h := &NPMHandler{proxy: proxy, upstreamURL: upstream.URL, proxyURL: "http://proxy.local"} + + w := httptest.NewRecorder() + h.handlePackageMetadata(w, httptest.NewRequest(http.MethodGet, "/lodash/4.17.21", nil)) + if w.Code != http.StatusForbidden { + t.Fatalf("status = %d, want 403; body: %s", w.Code, w.Body.String()) + } +} From 4f680ddbd0b1381d29465aa3b7d0f0971607a589 Mon Sep 17 00:00:00 2001 From: simon chrzanowski Date: Thu, 8 Oct 2026 13:26:55 +0200 Subject: [PATCH 217/633] Send ETag and Last-Modified with rewritten npm and Composer metadata - Rewritten metadata went out with only Content-Type, so pnpm, npm and Composer could never revalidate and downloaded every document in full - ETag is the SHA-256 of the rewritten body, computed once per rewrite and kept with the rewrite cache entry; exact under cooldown as well - Last-Modified is the upstream Last-Modified, never earlier than the process start, since a restart can change the rewrite; omitted with cooldown on - Answer If-None-Match, and If-Modified-Since when there is no If-None-Match, with 304 on the rewrite fast path and on the fetch path --- internal/handler/composer.go | 10 +- internal/handler/metadata_validators_test.go | 253 +++++++++++++++++++ internal/handler/npm.go | 14 +- internal/handler/npm_rewrite_test.go | 2 +- internal/handler/rewrite_cache.go | 186 ++++++++++---- internal/handler/rewrite_cache_test.go | 32 +-- 6 files changed, 421 insertions(+), 76 deletions(-) create mode 100644 internal/handler/metadata_validators_test.go diff --git a/internal/handler/composer.go b/internal/handler/composer.go index f7b40ae..e85ddc7 100644 --- a/internal/handler/composer.go +++ b/internal/handler/composer.go @@ -128,9 +128,8 @@ func (h *ComposerHandler) handlePackageMetadata(w http.ResponseWriter, r *http.R repoURL, cacheKey := h.sourceFor(packageName) upstreamURL := fmt.Sprintf("%s/p2/%s/%s.json", repoURL, vendor, pkg) - if rewritten, ok := h.proxy.storedRewrite("composer", cacheKey, h.proxyURL, cacheKey); ok { - w.Header().Set(headerContentType, "application/json") - _, _ = w.Write(rewritten) + if stored, ok := h.proxy.storedRewrite("composer", cacheKey, h.proxyURL, cacheKey); ok { + serveRewrittenMetadata(w, r, stored) return } @@ -145,7 +144,7 @@ func (h *ComposerHandler) handlePackageMetadata(w http.ResponseWriter, r *http.R return } - rewritten, err := h.proxy.cachedRewrite(r.Context(), "composer", h.proxyURL, cacheKey, body, h.rewriteMetadata) + rewritten, etag, err := h.proxy.cachedRewrite(r.Context(), "composer", h.proxyURL, cacheKey, body, h.rewriteMetadata) if err != nil { if r.Context().Err() != nil { return // the client left while waiting on a shared rewrite @@ -156,8 +155,7 @@ func (h *ComposerHandler) handlePackageMetadata(w http.ResponseWriter, r *http.R return } - w.Header().Set(headerContentType, "application/json") - _, _ = w.Write(rewritten) + serveRewrittenMetadata(w, r, h.proxy.rewrittenMetadataFor("composer", packageName, rewritten, etag)) } // rewriteMetadata rewrites dist URLs in Composer metadata to point at this proxy. diff --git a/internal/handler/metadata_validators_test.go b/internal/handler/metadata_validators_test.go new file mode 100644 index 0000000..79f49ca --- /dev/null +++ b/internal/handler/metadata_validators_test.go @@ -0,0 +1,253 @@ +package handler + +import ( + "net/http" + "net/http/httptest" + "sync" + "testing" + "time" + + "github.com/git-pkgs/cooldown" +) + +// validatorUpstream serves one npm packument and one Composer document whose +// bodies a test can change, with a fixed Last-Modified. +type validatorUpstream struct { + *httptest.Server + mu sync.Mutex + npm string + composer string + lastModified time.Time +} + +func newValidatorUpstream(t *testing.T) *validatorUpstream { + t.Helper() + u := &validatorUpstream{ + npm: `{"name":"left-pad","versions":{"1.3.0":{"dist":{"tarball":"https://registry.npmjs.org/left-pad/-/left-pad-1.3.0.tgz"}}}}`, + composer: `{"packages":{"vendor/pkg":[{"version":"1.0.0","dist":{"type":"zip","url":"https://example.com/pkg-1.0.0.zip"}}]}}`, + lastModified: time.Date(2020, 1, 2, 3, 4, 5, 0, time.UTC), + } + u.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + u.mu.Lock() + defer u.mu.Unlock() + w.Header().Set("Content-Type", "application/json") + w.Header().Set("Last-Modified", u.lastModified.Format(http.TimeFormat)) + switch r.URL.Path { + case "/left-pad": + _, _ = w.Write([]byte(u.npm)) + case "/p2/vendor/pkg.json": + _, _ = w.Write([]byte(u.composer)) + default: + http.NotFound(w, r) + } + })) + t.Cleanup(u.Close) + return u +} + +func (u *validatorUpstream) set(npm, composer string) { + u.mu.Lock() + defer u.mu.Unlock() + u.npm, u.composer = npm, composer +} + +type validatorCase struct { + ecosystem, path string + handler func(*Proxy, *validatorUpstream) http.Handler +} + +var validatorCases = []validatorCase{ + {"npm", "/left-pad", func(p *Proxy, u *validatorUpstream) http.Handler { + return NewNPMHandler(p, "http://proxy.example", u.URL).Routes() + }}, + {"composer", "/p2/vendor/pkg.json", func(p *Proxy, u *validatorUpstream) http.Handler { + return NewComposerHandlerWithUpstreams(p, "http://proxy.example", u.URL, u.URL).Routes() + }}, +} + +func validatorProxy(t *testing.T, u *validatorUpstream, ttl time.Duration) *Proxy { + t.Helper() + proxy, _, _, _ := setupTestProxy(t) + proxy.HTTPClient = u.Client() + proxy.CacheMetadata = true + proxy.MetadataTTL = ttl + proxy.SetMetadataRewriteCacheSize(1 << 20) + return proxy +} + +func serve(handler http.Handler, path string, header http.Header) *httptest.ResponseRecorder { + req := httptest.NewRequest(http.MethodGet, path, nil) + for k, v := range header { + req.Header[k] = v + } + rec := httptest.NewRecorder() + handler.ServeHTTP(rec, req) + return rec +} + +// TestRewrittenMetadataETag checks that rewritten metadata carries an ETag +// that clients can revalidate with, on the first fetch, on the rewrite fast +// path and when the TTL forces a revalidation. +func TestRewrittenMetadataETag(t *testing.T) { + for _, c := range validatorCases { + for _, ttl := range []time.Duration{time.Hour, 0} { + t.Run(c.ecosystem+"/ttl="+ttl.String(), func(t *testing.T) { + u := newValidatorUpstream(t) + handler := c.handler(validatorProxy(t, u, ttl), u) + + first := serve(handler, c.path, nil) + etag := first.Header().Get("ETag") + if first.Code != http.StatusOK || etag == "" { + t.Fatalf("first response: status %d, ETag %q; want 200 with an ETag", first.Code, etag) + } + for _, ifNoneMatch := range []string{etag, "W/" + etag, `"other", ` + etag} { + expectNotModified(t, serve(handler, c.path, http.Header{"If-None-Match": {ifNoneMatch}}), etag) + } + if rec := serve(handler, c.path, nil); rec.Header().Get("ETag") != etag || rec.Body.String() != first.Body.String() { + t.Errorf("repeat response: ETag %q, want %q, and the same body", rec.Header().Get("ETag"), etag) + } + }) + } + } +} + +func expectNotModified(t *testing.T, rec *httptest.ResponseRecorder, etag string) { + t.Helper() + if rec.Code != http.StatusNotModified || rec.Body.Len() != 0 { + t.Errorf("status %d with %d bytes, want an empty 304", rec.Code, rec.Body.Len()) + } + if got := rec.Header().Get("ETag"); got != etag { + t.Errorf("304 ETag = %q, want %q", got, etag) + } +} + +// TestRewrittenMetadataETagFollowsUpstream checks that a new upstream +// document gets a new ETag, so a client's old copy no longer matches. +func TestRewrittenMetadataETagFollowsUpstream(t *testing.T) { + for _, c := range validatorCases { + t.Run(c.ecosystem, func(t *testing.T) { + u := newValidatorUpstream(t) + handler := c.handler(validatorProxy(t, u, 0), u) + etag := serve(handler, c.path, nil).Header().Get("ETag") + + u.set( + `{"name":"left-pad","versions":{"1.3.1":{"dist":{"tarball":"https://registry.npmjs.org/left-pad/-/left-pad-1.3.1.tgz"}}}}`, + `{"packages":{"vendor/pkg":[{"version":"1.0.1","dist":{"type":"zip","url":"https://example.com/pkg-1.0.1.zip"}}]}}`, + ) + rec := serve(handler, c.path, http.Header{"If-None-Match": {etag}}) + if rec.Code != http.StatusOK || rec.Header().Get("ETag") == etag { + t.Errorf("status %d, ETag %q; want 200 with a new ETag", rec.Code, rec.Header().Get("ETag")) + } + }) + } +} + +// TestRewrittenMetadataLastModified checks the validator Composer revalidates +// with. It is the upstream Last-Modified, but never earlier than the start of +// this process: a restart can change the rewrite without upstream changing. +func TestRewrittenMetadataLastModified(t *testing.T) { + for _, c := range validatorCases { + for _, ttl := range []time.Duration{time.Hour, 0} { + t.Run(c.ecosystem+"/ttl="+ttl.String(), func(t *testing.T) { + u := newValidatorUpstream(t) + handler := c.handler(validatorProxy(t, u, ttl), u) + + first := serve(handler, c.path, nil) + lastModified, err := http.ParseTime(first.Header().Get("Last-Modified")) + if err != nil { + t.Fatalf("Last-Modified %q: %v", first.Header().Get("Last-Modified"), err) + } + if lastModified.Before(processStartedAt.Truncate(time.Second)) { + t.Errorf("Last-Modified %v is before the process started at %v", lastModified, processStartedAt) + } + + current := serve(handler, c.path, http.Header{"If-Modified-Since": {first.Header().Get("Last-Modified")}}) + if current.Code != http.StatusNotModified || current.Body.Len() != 0 { + t.Errorf("If-Modified-Since Last-Modified: status %d with %d bytes, want an empty 304", current.Code, current.Body.Len()) + } + + older := lastModified.Add(-time.Hour).Format(http.TimeFormat) + if rec := serve(handler, c.path, http.Header{"If-Modified-Since": {older}}); rec.Code != http.StatusOK { + t.Errorf("If-Modified-Since an hour earlier: status %d, want 200", rec.Code) + } + + // If-None-Match takes precedence over If-Modified-Since. + rec := serve(handler, c.path, http.Header{ + "If-None-Match": {`"other"`}, + "If-Modified-Since": {first.Header().Get("Last-Modified")}, + }) + if rec.Code != http.StatusOK { + t.Errorf("mismatched If-None-Match with a current If-Modified-Since: status %d, want 200", rec.Code) + } + }) + } + } +} + +// TestRewrittenMetadataValidatorsWithCooldown checks that with cooldown on, +// rewritten metadata keeps an exact ETag but has no Last-Modified: cooldown +// changes the rewrite as versions age in, while upstream stays the same. +func TestRewrittenMetadataValidatorsWithCooldown(t *testing.T) { + for _, c := range validatorCases { + t.Run(c.ecosystem, func(t *testing.T) { + u := newValidatorUpstream(t) + proxy := validatorProxy(t, u, time.Hour) + proxy.Cooldown = &cooldown.Config{Default: "3d"} + handler := c.handler(proxy, u) + + first := serve(handler, c.path, nil) + etag := first.Header().Get("ETag") + if etag == "" { + t.Fatal("no ETag with cooldown on") + } + if lm := first.Header().Get("Last-Modified"); lm != "" { + t.Errorf("Last-Modified = %q with cooldown on, want none", lm) + } + if rec := serve(handler, c.path, http.Header{"If-None-Match": {etag}}); rec.Code != http.StatusNotModified { + t.Errorf("If-None-Match: status %d, want 304", rec.Code) + } + future := time.Now().Add(time.Hour).Format(http.TimeFormat) + if rec := serve(handler, c.path, http.Header{"If-Modified-Since": {future}}); rec.Code != http.StatusOK { + t.Errorf("If-Modified-Since: status %d, want 200", rec.Code) + } + }) + } +} + +// TestUnrewrittenMetadataHasNoValidators checks that metadata proxied as it +// is, because rewriting it failed, does not claim the rewrite's validators. +func TestUnrewrittenMetadataHasNoValidators(t *testing.T) { + u := newValidatorUpstream(t) + u.set(`{"name":"left-pad","versions":`, `{"packages":`) + for _, c := range validatorCases { + t.Run(c.ecosystem, func(t *testing.T) { + rec := serve(c.handler(validatorProxy(t, u, time.Hour), u), c.path, nil) + if rec.Code != http.StatusOK { + t.Fatalf("status %d: %s", rec.Code, rec.Body.String()) + } + if etag, lm := rec.Header().Get("ETag"), rec.Header().Get("Last-Modified"); etag != "" || lm != "" { + t.Errorf("ETag %q, Last-Modified %q; want neither on unrewritten metadata", etag, lm) + } + }) + } +} + +func TestRewriteCacheETag(t *testing.T) { + c := newRewriteCache(1 << 20) + exclaim := func(b []byte) ([]byte, error) { return []byte(string(b) + "!"), nil } + key := rewriteCacheKey("npm", "http://proxy", "demo", []byte("doc")) + + out, etag, err := c.rewrite(t.Context(), key, []byte("doc"), exclaim) + if err != nil || string(out) != "doc!" { + t.Fatalf("rewrite = %q, %v", out, err) + } + if want := metadataETag([]byte("doc!")); etag != want { + t.Errorf("rewrite ETag = %q, want %q", etag, want) + } + if _, cachedETag, ok := c.get(key); !ok || cachedETag != etag { + t.Errorf("get ETag = %q, %v; want %q", cachedETag, ok, etag) + } + if metadataETag([]byte("other")) == etag { + t.Error("different output has the same ETag") + } +} diff --git a/internal/handler/npm.go b/internal/handler/npm.go index 9ce667b..b3d81ef 100644 --- a/internal/handler/npm.go +++ b/internal/handler/npm.go @@ -215,10 +215,8 @@ func (h *NPMHandler) handlePackageMetadata(w http.ResponseWriter, r *http.Reques accept = contentTypeJSON } - if rewritten, ok := h.proxy.storedRewrite("npm", metadataCacheKey, h.proxyURL, metadataCacheKey); ok { - w.Header().Set(headerContentType, contentTypeJSON) - w.WriteHeader(http.StatusOK) - _, _ = w.Write(rewritten) + if stored, ok := h.proxy.storedRewrite("npm", metadataCacheKey, h.proxyURL, metadataCacheKey); ok { + serveRewrittenMetadata(w, r, stored) return } @@ -243,7 +241,7 @@ func (h *NPMHandler) handlePackageMetadata(w http.ResponseWriter, r *http.Reques return } - rewritten, err := h.proxy.cachedRewrite(r.Context(), "npm", h.proxyURL, metadataCacheKey, body, func(b []byte) ([]byte, error) { + rewritten, etag, err := h.proxy.cachedRewrite(r.Context(), "npm", h.proxyURL, metadataCacheKey, body, func(b []byte) ([]byte, error) { return h.rewriteMetadata(packageName, b) }) if err != nil { @@ -270,9 +268,7 @@ func (h *NPMHandler) handlePackageMetadata(w http.ResponseWriter, r *http.Reques return } - w.Header().Set(headerContentType, contentTypeJSON) - w.WriteHeader(http.StatusOK) - _, _ = w.Write(rewritten) + serveRewrittenMetadata(w, r, h.proxy.rewrittenMetadataFor("npm", metadataCacheKey, rewritten, etag)) } // handleFilteredNPMVersionMetadata serves /pkg/ and /pkg/ from the @@ -290,7 +286,7 @@ func (h *NPMHandler) handleFilteredNPMVersionMetadata(w http.ResponseWriter, r * return } - rewritten, err := h.proxy.cachedRewrite(r.Context(), "npm", h.proxyURL, metadataCacheKey, packument, func(b []byte) ([]byte, error) { + rewritten, _, err := h.proxy.cachedRewrite(r.Context(), "npm", h.proxyURL, metadataCacheKey, packument, func(b []byte) ([]byte, error) { return h.rewriteNPMVersionFromPackument(packageName, specifier, b) }) if err != nil { diff --git a/internal/handler/npm_rewrite_test.go b/internal/handler/npm_rewrite_test.go index 6503ae1..d9a787e 100644 --- a/internal/handler/npm_rewrite_test.go +++ b/internal/handler/npm_rewrite_test.go @@ -247,7 +247,7 @@ func TestNPMStoredRewriteSkippedWhenStale(t *testing.T) { body := []byte(`{}`) proxy.cacheMetadataBlob(t.Context(), "npm", "demo", metadataStoragePath("npm", "demo"), &upstreamMetadata{body: body}) key := rewriteCacheKey("npm", "http://proxy", "demo", body) - if _, err := proxy.rewrites.rewrite(t.Context(), key, body, func(b []byte) ([]byte, error) { return b, nil }); err != nil { + if _, _, err := proxy.rewrites.rewrite(t.Context(), key, body, func(b []byte) ([]byte, error) { return b, nil }); err != nil { t.Fatal(err) } if _, ok := proxy.storedRewrite("npm", "demo", "http://proxy", "demo"); !ok { diff --git a/internal/handler/rewrite_cache.go b/internal/handler/rewrite_cache.go index 23096d6..63f7448 100644 --- a/internal/handler/rewrite_cache.go +++ b/internal/handler/rewrite_cache.go @@ -6,6 +6,7 @@ import ( "crypto/sha256" "encoding/hex" "errors" + "net/http" "strings" "sync" "time" @@ -26,6 +27,9 @@ import ( // are rewritten again rather than served stale. Everything else a rewrite // depends on is fixed for the life of the process: the proxy URL is part of // the key, and the denylist is loaded at startup. +// +// Each entry keeps the ETag of its output, so clients can revalidate their +// copy without the output being hashed again on every request. type rewriteCache struct { maxBytes int64 @@ -37,19 +41,40 @@ type rewriteCache struct { } type rewriteEntry struct { - key string - out []byte + key string + out []byte + etag string } -// inflightRewrite is one rewrite that concurrent callers share. out and err -// are written before done closes and read only after, so the close is the +// inflightRewrite is one rewrite that concurrent callers share. out, etag and +// err are written before done closes and read only after, so the close is the // handoff. type inflightRewrite struct { done chan struct{} out []byte + etag string err error } +// processStartedAt bounds the Last-Modified of rewritten metadata from below. +// A rewrite depends on configuration read at startup as well as on the +// upstream document, so a restart can change it while upstream stays the same. +var processStartedAt = time.Now() + +// rewrittenMetadata is a rewritten metadata document with the validators a +// client revalidates its copy with. lastModified is zero when unknown. +type rewrittenMetadata struct { + body []byte + etag string + lastModified time.Time +} + +// metadataETag returns the strong ETag of a rewritten document. +func metadataETag(out []byte) string { + sum := sha256.Sum256(out) + return `"` + hex.EncodeToString(sum[:]) + `"` +} + // errSharedRewriteAbandoned is what waiters see if the caller running a // shared rewrite panicked out of it. var errSharedRewriteAbandoned = errors.New("shared metadata rewrite did not complete") @@ -81,44 +106,46 @@ func rewriteCacheKeyForDigest(ecosystem, proxyURL, name, digest string) string { return strings.Join([]string{ecosystem, proxyURL, name, digest}, "\x00") } -// get returns the cached rewrite for key, if there is one. -func (c *rewriteCache) get(key string) ([]byte, bool) { +// get returns the cached rewrite for key and its ETag, if there is one. +func (c *rewriteCache) get(key string) ([]byte, string, bool) { if c == nil { - return nil, false + return nil, "", false } c.mu.Lock() defer c.mu.Unlock() el, ok := c.entries[key] if !ok { - return nil, false + return nil, "", false } c.order.MoveToFront(el) - return el.Value.(*rewriteEntry).out, true + e := el.Value.(*rewriteEntry) + return e.out, e.etag, true } -// rewrite returns rewrite(in), from the cache when it can. Callers must treat -// the returned bytes as read-only: a cached result is shared. A caller waiting -// on another's rewrite leaves when ctx ends; the rewrite itself always runs -// to completion, so the result is cached for the next request. -func (c *rewriteCache) rewrite(ctx context.Context, key string, in []byte, rewrite func([]byte) ([]byte, error)) ([]byte, error) { +// rewrite returns rewrite(in) and its ETag, from the cache when it can. +// Callers must treat the returned bytes as read-only: a cached result is +// shared. A caller waiting on another's rewrite leaves when ctx ends; the +// rewrite itself always runs to completion, so the result is cached for the +// next request. +func (c *rewriteCache) rewrite(ctx context.Context, key string, in []byte, rewrite func([]byte) ([]byte, error)) ([]byte, string, error) { if c == nil { - return rewrite(in) + return rewriteWithETag(in, rewrite) } c.mu.Lock() if el, ok := c.entries[key]; ok { c.order.MoveToFront(el) - out := el.Value.(*rewriteEntry).out + e := el.Value.(*rewriteEntry) c.mu.Unlock() - return out, nil + return e.out, e.etag, nil } if f, ok := c.inFlight[key]; ok { c.mu.Unlock() select { case <-ctx.Done(): - return nil, ctx.Err() + return nil, "", ctx.Err() case <-f.done: - return f.out, f.err + return f.out, f.etag, f.err } } f := &inflightRewrite{done: make(chan struct{}), err: errSharedRewriteAbandoned} @@ -129,24 +156,33 @@ func (c *rewriteCache) rewrite(ctx context.Context, key string, in []byte, rewri c.mu.Lock() delete(c.inFlight, key) if f.err == nil { - c.add(key, f.out) + c.add(&rewriteEntry{key: key, out: f.out, etag: f.etag}) } c.mu.Unlock() close(f.done) }() - f.out, f.err = rewrite(in) - return f.out, f.err + f.out, f.etag, f.err = rewriteWithETag(in, rewrite) + return f.out, f.etag, f.err } -// add stores out under key and evicts least recently used entries until the -// cache is back under its limit. Output larger than the whole cache is not -// stored. Called with mu held. -func (c *rewriteCache) add(key string, out []byte) { - n := int64(len(out)) +// rewriteWithETag runs rewrite and computes the ETag of what it returns. +func rewriteWithETag(in []byte, rewrite func([]byte) ([]byte, error)) ([]byte, string, error) { + out, err := rewrite(in) + if err != nil { + return nil, "", err + } + return out, metadataETag(out), nil +} + +// add stores e and evicts least recently used entries until the cache is +// back under its limit. Output larger than the whole cache is not stored. +// Called with mu held. +func (c *rewriteCache) add(e *rewriteEntry) { + n := int64(len(e.out)) if n > c.maxBytes { return } - c.entries[key] = c.order.PushFront(&rewriteEntry{key: key, out: out}) + c.entries[e.key] = c.order.PushFront(e) c.size += n for c.size > c.maxBytes { oldest := c.order.Back() @@ -168,41 +204,103 @@ func (c *rewriteCache) len() int { } // cachedRewrite rewrites a metadata document through the proxy's rewrite -// cache. Cooldown filtering depends on the current time, so with cooldown on -// a cached rewrite could keep hiding a version past its cooldown; those -// rewrites always run. -func (p *Proxy) cachedRewrite(ctx context.Context, ecosystem, proxyURL, name string, in []byte, rewrite func([]byte) ([]byte, error)) ([]byte, error) { - if p.rewrites == nil || (p.Cooldown != nil && p.Cooldown.Enabled()) { - return rewrite(in) +// cache and returns it with its ETag. Cooldown filtering depends on the +// current time, so with cooldown on a cached rewrite could keep hiding a +// version past its cooldown; those rewrites always run. +func (p *Proxy) cachedRewrite(ctx context.Context, ecosystem, proxyURL, name string, in []byte, rewrite func([]byte) ([]byte, error)) ([]byte, string, error) { + if p.rewrites == nil || p.cooldownEnabled() { + return rewriteWithETag(in, rewrite) } return p.rewrites.rewrite(ctx, rewriteCacheKey(ecosystem, proxyURL, name, in), in, rewrite) } +func (p *Proxy) cooldownEnabled() bool { + return p.Cooldown != nil && p.Cooldown.Enabled() +} + +// rewriteLastModified returns the Last-Modified of a rewrite of metadata that +// upstream last modified at upstream, or zero when there is none to give. +// With cooldown on there is none: versions leaving cooldown change the +// rewrite while upstream stays the same. +func (p *Proxy) rewriteLastModified(upstream time.Time) time.Time { + if upstream.IsZero() || p.cooldownEnabled() { + return time.Time{} + } + if upstream.Before(processStartedAt) { + return processStartedAt + } + return upstream +} + +// rewrittenMetadataFor returns a fresh rewrite of the metadata cached for +// ecosystem and cacheKey with its validators. +func (p *Proxy) rewrittenMetadataFor(ecosystem, cacheKey string, out []byte, etag string) rewrittenMetadata { + return rewrittenMetadata{ + body: out, + etag: etag, + lastModified: p.rewriteLastModified(p.lookupCachedMeta(ecosystem, cacheKey).lastModified), + } +} + // storedRewrite returns the cached rewrite of the metadata stored for // ecosystem and cacheKey, when that metadata is still within its TTL and a // rewrite of it is cached. It reads only the cache row, never the stored // document, so a repeated request for a large packument costs a database // lookup instead of reading and hashing the whole document again. When it // reports false the caller takes the usual fetch and rewrite path. -func (p *Proxy) storedRewrite(ecosystem, cacheKey, proxyURL, name string) ([]byte, bool) { - if p.rewrites == nil || (p.Cooldown != nil && p.Cooldown.Enabled()) { - return nil, false +func (p *Proxy) storedRewrite(ecosystem, cacheKey, proxyURL, name string) (rewrittenMetadata, bool) { + if p.rewrites == nil || p.cooldownEnabled() { + return rewrittenMetadata{}, false } if !p.CacheMetadata || p.DB == nil || p.MetadataTTL <= 0 { - return nil, false + return rewrittenMetadata{}, false } entry, err := p.DB.GetMetadataCache(ecosystem, cacheKey) if err != nil || entry == nil || !entry.ContentDigest.Valid || entry.ContentEncoding.String != "" { - return nil, false + return rewrittenMetadata{}, false } if !entry.FetchedAt.Valid || time.Since(entry.FetchedAt.Time) >= p.MetadataTTL { - return nil, false + return rewrittenMetadata{}, false } - out, ok := p.rewrites.get(rewriteCacheKeyForDigest(ecosystem, proxyURL, name, entry.ContentDigest.String)) - if ok { - metrics.RecordCacheHit(ecosystem) + out, etag, ok := p.rewrites.get(rewriteCacheKeyForDigest(ecosystem, proxyURL, name, entry.ContentDigest.String)) + if !ok { + return rewrittenMetadata{}, false } - return out, ok + metrics.RecordCacheHit(ecosystem) + var upstreamModified time.Time + if entry.LastModified.Valid { + upstreamModified = entry.LastModified.Time + } + return rewrittenMetadata{body: out, etag: etag, lastModified: p.rewriteLastModified(upstreamModified)}, true +} + +// serveRewrittenMetadata writes rewritten metadata with its validators, or +// 304 Not Modified when they show the client's copy is current. +func serveRewrittenMetadata(w http.ResponseWriter, r *http.Request, m rewrittenMetadata) { + w.Header().Set(headerETag, m.etag) + if !m.lastModified.IsZero() { + w.Header().Set(headerLastModified, m.lastModified.UTC().Format(http.TimeFormat)) + } + if notModified(r, m.etag, m.lastModified) { + w.WriteHeader(http.StatusNotModified) + return + } + w.Header().Set(headerContentType, contentTypeJSON) + w.WriteHeader(http.StatusOK) + _, _ = w.Write(m.body) +} + +// notModified reports whether a conditional request's validators match. +// If-Modified-Since only counts when the request has no If-None-Match. +func notModified(r *http.Request, etag string, lastModified time.Time) bool { + if header := r.Header.Get("If-None-Match"); header != "" { + return ifNoneMatchHits(header, etag) + } + if lastModified.IsZero() { + return false + } + since, err := http.ParseTime(r.Header.Get("If-Modified-Since")) + return err == nil && !lastModified.Truncate(time.Second).After(since) } // SetMetadataRewriteCacheSize enables the cache of rewritten metadata with diff --git a/internal/handler/rewrite_cache_test.go b/internal/handler/rewrite_cache_test.go index 0edb94f..1581ed7 100644 --- a/internal/handler/rewrite_cache_test.go +++ b/internal/handler/rewrite_cache_test.go @@ -37,7 +37,7 @@ func TestRewriteCache_RepeatServesCachedRewrite(t *testing.T) { key := rewriteCacheKey("npm", "http://proxy", "left-pad", []byte("doc")) for range 3 { - out, err := c.rewrite(context.Background(), key, []byte("doc"), r.fn) + out, _, err := c.rewrite(context.Background(), key, []byte("doc"), r.fn) if err != nil || string(out) != "DOC" { t.Fatalf("rewrite = %q, %v", out, err) } @@ -53,7 +53,7 @@ func TestRewriteCache_NewUpstreamBytesRewriteAgain(t *testing.T) { for _, doc := range []string{"v1", "v2"} { key := rewriteCacheKey("npm", "http://proxy", "left-pad", []byte(doc)) - out, err := c.rewrite(context.Background(), key, []byte(doc), r.fn) + out, _, err := c.rewrite(context.Background(), key, []byte(doc), r.fn) if err != nil || string(out) != strings.ToUpper(doc) { t.Fatalf("rewrite(%s) = %q, %v", doc, out, err) } @@ -75,7 +75,7 @@ func TestRewriteCache_ConcurrentRequestsShareRewrite(t *testing.T) { outs := make(chan string, n) run := func() { - out, err := c.rewrite(context.Background(), key, []byte("doc"), r.fn) + out, _, err := c.rewrite(context.Background(), key, []byte("doc"), r.fn) if err != nil { t.Error(err) } @@ -108,20 +108,20 @@ func TestRewriteCache_WaiterLeavingKeepsRewrite(t *testing.T) { first := make(chan error, 1) go func() { - _, err := c.rewrite(context.Background(), key, []byte("doc"), r.fn) + _, _, err := c.rewrite(context.Background(), key, []byte("doc"), r.fn) first <- err }() <-r.entered ctx, cancel := context.WithCancel(context.Background()) cancel() - if _, err := c.rewrite(ctx, key, []byte("doc"), r.fn); !errors.Is(err, context.Canceled) { + if _, _, err := c.rewrite(ctx, key, []byte("doc"), r.fn); !errors.Is(err, context.Canceled) { t.Errorf("waiter err = %v, want context.Canceled", err) } close(r.release) if err := <-first; err != nil { t.Fatal(err) } - if out, err := c.rewrite(context.Background(), key, []byte("doc"), r.fn); err != nil || string(out) != "DOC" { + if out, _, err := c.rewrite(context.Background(), key, []byte("doc"), r.fn); err != nil || string(out) != "DOC" { t.Errorf("cached rewrite = %q, %v", out, err) } if got := r.calls.Load(); got != 1 { @@ -135,18 +135,18 @@ func TestRewriteCache_EvictsLeastRecentlyUsed(t *testing.T) { keyFor := func(doc string) string { return rewriteCacheKey("npm", "http://proxy", doc, []byte(doc)) } for _, doc := range []string{"aaa", "bbb"} { - _, _ = c.rewrite(context.Background(), keyFor(doc), []byte(doc), r.fn) + _, _, _ = c.rewrite(context.Background(), keyFor(doc), []byte(doc), r.fn) } - _, _ = c.rewrite(context.Background(), keyFor("aaa"), []byte("aaa"), r.fn) // aaa is now the most recent - _, _ = c.rewrite(context.Background(), keyFor("ccc"), []byte("ccc"), r.fn) // evicts bbb + _, _, _ = c.rewrite(context.Background(), keyFor("aaa"), []byte("aaa"), r.fn) // aaa is now the most recent + _, _, _ = c.rewrite(context.Background(), keyFor("ccc"), []byte("ccc"), r.fn) // evicts bbb before := r.calls.Load() - _, _ = c.rewrite(context.Background(), keyFor("aaa"), []byte("aaa"), r.fn) - _, _ = c.rewrite(context.Background(), keyFor("ccc"), []byte("ccc"), r.fn) + _, _, _ = c.rewrite(context.Background(), keyFor("aaa"), []byte("aaa"), r.fn) + _, _, _ = c.rewrite(context.Background(), keyFor("ccc"), []byte("ccc"), r.fn) if got := r.calls.Load() - before; got != 0 { t.Errorf("aaa and ccc rewritten %d times, want both still cached", got) } - _, _ = c.rewrite(context.Background(), keyFor("bbb"), []byte("bbb"), r.fn) + _, _, _ = c.rewrite(context.Background(), keyFor("bbb"), []byte("bbb"), r.fn) if got := r.calls.Load() - before; got != 1 { t.Errorf("bbb should have been evicted and rewritten") } @@ -157,7 +157,7 @@ func TestRewriteCache_OutputLargerThanCacheIsNotStored(t *testing.T) { var r countingRewrite key := rewriteCacheKey("npm", "http://proxy", "big", []byte("big")) - _, _ = c.rewrite(context.Background(), key, []byte("big"), r.fn) + _, _, _ = c.rewrite(context.Background(), key, []byte("big"), r.fn) if c.len() != 0 { t.Errorf("cached %d entries, want 0", c.len()) } @@ -170,7 +170,7 @@ func TestRewriteCache_ErrorsAreNotCached(t *testing.T) { key := rewriteCacheKey("npm", "http://proxy", "broken", []byte("doc")) for range 2 { - if _, err := c.rewrite(context.Background(), key, []byte("doc"), failing); err == nil { + if _, _, err := c.rewrite(context.Background(), key, []byte("doc"), failing); err == nil { t.Fatal("expected the rewrite error") } } @@ -185,7 +185,7 @@ func TestCachedRewrite_DisabledRewritesEveryTime(t *testing.T) { var r countingRewrite for range 2 { - _, _ = proxy.cachedRewrite(context.Background(), "npm", "http://proxy", "left-pad", []byte("doc"), r.fn) + _, _, _ = proxy.cachedRewrite(context.Background(), "npm", "http://proxy", "left-pad", []byte("doc"), r.fn) } if got := r.calls.Load(); got != 2 { t.Errorf("rewrites = %d, want 2", got) @@ -202,7 +202,7 @@ func TestCachedRewrite_CooldownBypassesCache(t *testing.T) { var r countingRewrite for range 2 { - _, _ = proxy.cachedRewrite(context.Background(), "npm", "http://proxy", "left-pad", []byte("doc"), r.fn) + _, _, _ = proxy.cachedRewrite(context.Background(), "npm", "http://proxy", "left-pad", []byte("doc"), r.fn) } if got := r.calls.Load(); got != 2 { t.Errorf("rewrites = %d, want 2", got) From bf7d17d39a45edfd2fae3af6de5f992edded5924 Mon Sep 17 00:00:00 2001 From: simon chrzanowski Date: Fri, 9 Oct 2026 08:10:41 +0200 Subject: [PATCH 218/633] Derive cooldown metadata ETags from the rewrite's inputs - With cooldown on every rewrite runs, and hashing each rewritten body for its ETag made every npm metadata request about 20% more expensive - Hash what fixes the output instead: the upstream digest, proxy URL, package name, the versions cooldown and the denylist kept, and a per-process nonce standing for the startup configuration - Carry the stored document's digest and upstream Last-Modified with the metadata lookup result, so neither is read from a second cache row lookup that a concurrent refresh could have changed - Fall back to hashing the body when the digest is unknown, and keep the body hash with cooldown off, where it is computed once per cached rewrite --- internal/handler/composer.go | 34 ++++--- internal/handler/handler.go | 75 ++++++++++++---- internal/handler/metadata_validators_test.go | 93 ++++++++++++++++++++ internal/handler/npm.go | 42 +++++---- internal/handler/rewrite_cache.go | 69 ++++++++++++--- internal/handler/rewrite_cache_test.go | 9 +- 6 files changed, 259 insertions(+), 63 deletions(-) diff --git a/internal/handler/composer.go b/internal/handler/composer.go index e85ddc7..4c79953 100644 --- a/internal/handler/composer.go +++ b/internal/handler/composer.go @@ -133,8 +133,8 @@ func (h *ComposerHandler) handlePackageMetadata(w http.ResponseWriter, r *http.R return } - body, _, err := h.proxy.FetchOrCacheMetadata(r.Context(), "composer", cacheKey, upstreamURL) - if err != nil { + upstream := h.proxy.fetchMetadataDocument(r.Context(), "composer", cacheKey, upstreamURL, contentTypeJSON) + if err := upstream.err; err != nil { if errors.Is(err, ErrUpstreamNotFound) { http.Error(w, "not found", http.StatusNotFound) return @@ -144,18 +144,18 @@ func (h *ComposerHandler) handlePackageMetadata(w http.ResponseWriter, r *http.R return } - rewritten, etag, err := h.proxy.cachedRewrite(r.Context(), "composer", h.proxyURL, cacheKey, body, h.rewriteMetadata) + rewritten, etag, err := h.proxy.cachedRewrite(r.Context(), "composer", h.proxyURL, cacheKey, upstream.body, upstream.digest, h.rewriteMetadataKeeping) if err != nil { if r.Context().Err() != nil { return // the client left while waiting on a shared rewrite } h.proxy.Logger.Warn("failed to rewrite metadata, proxying original", "error", err) w.Header().Set(headerContentType, "application/json") - _, _ = w.Write(body) + _, _ = w.Write(upstream.body) return } - serveRewrittenMetadata(w, r, h.proxy.rewrittenMetadataFor("composer", packageName, rewritten, etag)) + serveRewrittenMetadata(w, r, h.proxy.rewrittenMetadataFor(rewritten, etag, upstream.lastModified)) } // rewriteMetadata rewrites dist URLs in Composer metadata to point at this proxy. @@ -168,16 +168,24 @@ func (h *ComposerHandler) handlePackageMetadata(w http.ResponseWriter, r *http.R // of the original bytes: only dist values are written fresh, and minified // metadata stays minified. func (h *ComposerHandler) rewriteMetadata(body []byte) ([]byte, error) { + out, _, err := h.rewriteMetadataKeeping(body) + return out, err +} + +// rewriteMetadataKeeping is rewriteMetadata that also reports the versions +// it kept, each as its package name and version. +func (h *ComposerHandler) rewriteMetadataKeeping(body []byte) ([]byte, []string, error) { if !json.Valid(body) { - return nil, errors.New("composer metadata is not valid JSON") + return nil, nil, errors.New("composer metadata is not valid JSON") } format, _, err := lookupJSONString(body, "minified") if err != nil { - return nil, err + return nil, nil, err } minified := format == composerMinified var out bytes.Buffer + var kept []string out.Grow(len(body) + len(body)/8) err = rewriteJSONMembers(&out, body, func(out *bytes.Buffer, m jsonMember) (bool, error) { packages := m.value(body) @@ -193,24 +201,25 @@ func (h *ComposerHandler) rewriteMetadata(body []byte) ([]byte, error) { if err != nil { return false, err } - return true, h.writeVersions(out, packageName, versions, minified) + return true, h.writeVersions(out, packageName, versions, minified, &kept) }) }) if err != nil { - return nil, err + return nil, nil, err } - return out.Bytes(), nil + return out.Bytes(), kept, nil } // writeVersions writes one package's version list without the versions in -// cooldown and with each dist pointing at this proxy. +// cooldown and with each dist pointing at this proxy, and appends each version +// it writes to kept as its package name and version. // // Minified lists stay minified. Each version written carries the fields that // differ from what the client has expanded so far, which is the upstream // entry itself unless filtered versions were dropped before it; then it also // carries the changes those versions made. Every version gets its own dist, // since the proxied URL names the version. -func (h *ComposerHandler) writeVersions(out *bytes.Buffer, packageName string, versions []byte, minified bool) error { +func (h *ComposerHandler) writeVersions(out *bytes.Buffer, packageName string, versions []byte, minified bool, kept *[]string) error { packagePURL := canonicalPackagePURL("composer", packageName) upstream, written := newComposerFields(), newComposerFields() devReset, first := false, true @@ -252,6 +261,7 @@ func (h *ComposerHandler) writeVersions(out *bytes.Buffer, packageName string, v disableDefaultNotification(upstream) } h.writeVersion(out, packageName, version, upstream, written) + *kept = append(*kept, packageName+"\x00"+version) return nil }) out.WriteByte(']') diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 036e2a5..6f7f700 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -1086,24 +1086,35 @@ func (p *Proxy) FetchOrCacheMetadata(ctx context.Context, ecosystem, cacheKey, u // joined a fetch gets its own validation error directly. It must not modify the // body, which joined callers share. func (p *Proxy) fetchOrCacheMetadata(ctx context.Context, ecosystem, cacheKey, upstreamURL, acceptEncoding string, validate func([]byte) error, acceptHeaders ...string) ([]byte, string, string, error) { + accept := contentTypeJSON + if len(acceptHeaders) > 0 && acceptHeaders[0] != "" { + accept = acceptHeaders[0] + } + res := p.lookupMetadata(ctx, ecosystem, cacheKey, upstreamURL, accept, acceptEncoding, validate) + return res.body, res.contentType, res.contentEncoding, res.err +} + +// fetchMetadataDocument is FetchOrCacheMetadata for callers that rewrite the +// document and need to know which one they got: the result carries its +// digest and upstream Last-Modified along with the body. +func (p *Proxy) fetchMetadataDocument(ctx context.Context, ecosystem, cacheKey, upstreamURL, accept string) metadataResult { + return p.lookupMetadata(ctx, ecosystem, cacheKey, upstreamURL, accept, "", nil) +} + +// lookupMetadata implements fetchOrCacheMetadata and fetchMetadataDocument. +func (p *Proxy) lookupMetadata(ctx context.Context, ecosystem, cacheKey, upstreamURL, accept, acceptEncoding string, validate func([]byte) error) metadataResult { if containsPathTraversal(cacheKey) { - return nil, "", "", fmt.Errorf("invalid cache key: %q", cacheKey) + return metadataResult{err: fmt.Errorf("invalid cache key: %q", cacheKey)} } // Serve from cache if within TTL (skip upstream entirely) if _, hit := p.cachedMetadataState(ctx, ecosystem, cacheKey, validate); hit != nil { metrics.RecordCacheHit(ecosystem) - return hit.body, hit.contentType, hit.contentEncoding, nil + return *hit } p.recordMetadataCacheMiss(ecosystem) - accept := contentTypeJSON - if len(acceptHeaders) > 0 && acceptHeaders[0] != "" { - accept = acceptHeaders[0] - } - - res := p.coalescedMetadataMiss(ctx, ecosystem, cacheKey, upstreamURL, accept, acceptEncoding, validate) - return res.body, res.contentType, res.contentEncoding, res.err + return p.coalescedMetadataMiss(ctx, ecosystem, cacheKey, upstreamURL, accept, acceptEncoding, validate) } // coalescedMetadataMiss handles a metadata cache miss, sharing one upstream @@ -1144,7 +1155,8 @@ func (p *Proxy) cachedMetadataState(ctx context.Context, ecosystem, cacheKey str if entry != nil && p.MetadataTTL > 0 && entry.FetchedAt.Valid && time.Since(entry.FetchedAt.Time) < p.MetadataTTL { data, ct, err := p.readCachedMetadata(ctx, entry, validate) if err == nil { - return entry, &metadataResult{body: data, contentType: ct, contentEncoding: entry.ContentEncoding.String} + hit := metadataResult{body: data, contentType: ct, contentEncoding: entry.ContentEncoding.String}.describedBy(entry) + return entry, &hit } if validate != nil { return nil, nil @@ -1167,10 +1179,11 @@ func (p *Proxy) fetchMetadataFromUpstream(ctx context.Context, ecosystem, cacheK err = validate(meta.body) } if err == nil { + var digest string if p.CacheMetadata { - p.cacheMetadataBlob(ctx, ecosystem, cacheKey, metadataStoragePath(ecosystem, cacheKey), meta) + digest = p.cacheMetadataBlob(ctx, ecosystem, cacheKey, metadataStoragePath(ecosystem, cacheKey), meta) } - return metadataResult{body: meta.body, contentType: meta.contentType, contentEncoding: meta.contentEncoding} + return metadataResult{body: meta.body, contentType: meta.contentType, contentEncoding: meta.contentEncoding, digest: digest, lastModified: meta.lastModified} } // Upstream failed -- fall back to cache if available @@ -1193,7 +1206,7 @@ func (p *Proxy) fetchMetadataFromUpstream(ctx context.Context, ecosystem, cacheK p.Logger.Info("serving metadata from cache", "ecosystem", ecosystem, "key", cacheKey) - return metadataResult{body: data, contentType: ct, contentEncoding: entry.ContentEncoding.String} + return metadataResult{body: data, contentType: ct, contentEncoding: entry.ContentEncoding.String}.describedBy(entry) } // metadataResult is what a metadata lookup hands back. A shared fetch gives @@ -1202,7 +1215,23 @@ type metadataResult struct { body []byte contentType string contentEncoding string - err error + // digest identifies body when the metadata cache stored it, and is + // empty otherwise. lastModified is upstream's, zero when unknown. + digest string + lastModified time.Time + err error +} + +// describedBy returns res with the digest and Last-Modified that entry, the +// cache row its body was read for, records. +func (res metadataResult) describedBy(entry *database.MetadataCacheEntry) metadataResult { + if entry.ContentDigest.Valid { + res.digest = entry.ContentDigest.String + } + if entry.LastModified.Valid { + res.lastModified = entry.LastModified.Time + } + return res } // inflightMetadata is one metadata fetch that concurrent callers share. res is @@ -1385,16 +1414,22 @@ func (p *Proxy) fetchUpstreamMetadata(ctx context.Context, upstreamURL string, e return meta, nil } -// cacheMetadataBlob stores metadata bytes in storage and updates the database. -func (p *Proxy) cacheMetadataBlob(ctx context.Context, ecosystem, cacheKey, storagePath string, meta *upstreamMetadata) { +// cacheMetadataBlob stores metadata bytes in storage and updates the +// database. It returns the digest it recorded for the bytes, or "" when they +// were not cached. +func (p *Proxy) cacheMetadataBlob(ctx context.Context, ecosystem, cacheKey, storagePath string, meta *upstreamMetadata) string { if p.DB == nil || p.Storage == nil { - return + return "" } size, hash, err := p.Storage.Store(ctx, storagePath, bytes.NewReader(meta.body)) if err != nil { p.Logger.Warn("failed to cache metadata", "ecosystem", ecosystem, "key", cacheKey, "error", err) - return + return "" + } + var digest string + if hash != "" { + digest = "sha256:" + hash } err = p.DB.UpsertMetadataCache(&database.MetadataCacheEntry{ @@ -1406,7 +1441,7 @@ func (p *Proxy) cacheMetadataBlob(ctx context.Context, ecosystem, cacheKey, stor ContentEncoding: sql.NullString{String: meta.contentEncoding, Valid: meta.contentEncoding != ""}, // The digest identifies the stored bytes, so a rewrite cached for them // can be found without reading them back (see storedRewrite). - ContentDigest: sql.NullString{String: "sha256:" + hash, Valid: hash != ""}, + ContentDigest: sql.NullString{String: digest, Valid: digest != ""}, Size: sql.NullInt64{Int64: size, Valid: true}, LastModified: sql.NullTime{Time: meta.lastModified, Valid: !meta.lastModified.IsZero()}, FetchedAt: sql.NullTime{Time: time.Now(), Valid: true}, @@ -1420,7 +1455,9 @@ func (p *Proxy) cacheMetadataBlob(ctx context.Context, ecosystem, cacheKey, stor if delErr := p.Storage.Delete(ctx, storagePath); delErr != nil { p.Logger.Warn("failed to discard metadata blob", "ecosystem", ecosystem, "key", cacheKey, "error", delErr) } + return "" } + return digest } // currentMetadataEntry re-reads the metadata cache row and returns it, or diff --git a/internal/handler/metadata_validators_test.go b/internal/handler/metadata_validators_test.go index 79f49ca..00213e1 100644 --- a/internal/handler/metadata_validators_test.go +++ b/internal/handler/metadata_validators_test.go @@ -1,8 +1,10 @@ package handler import ( + "fmt" "net/http" "net/http/httptest" + "strings" "sync" "testing" "time" @@ -251,3 +253,94 @@ func TestRewriteCacheETag(t *testing.T) { t.Error("different output has the same ETag") } } + +// cooldownUpstream sets documents with one version published long ago and +// one published two hours ago, so a cooldown of a day hides the recent one +// and a cooldown of an hour keeps it. +func cooldownUpstream(t *testing.T) *validatorUpstream { + t.Helper() + recent := time.Now().Add(-2 * time.Hour).UTC().Format(time.RFC3339) + u := newValidatorUpstream(t) + u.set( + `{"name":"left-pad","dist-tags":{"latest":"1.4.0"},"versions":{`+ + `"1.3.0":{"dist":{"tarball":"https://registry.npmjs.org/left-pad/-/left-pad-1.3.0.tgz"}},`+ + `"1.4.0":{"dist":{"tarball":"https://registry.npmjs.org/left-pad/-/left-pad-1.4.0.tgz"}}},`+ + `"time":{"1.3.0":"2016-01-01T00:00:00Z","1.4.0":"`+recent+`"}}`, + `{"packages":{"vendor/pkg":[`+ + `{"version":"1.0.0","time":"2016-01-01T00:00:00Z","dist":{"type":"zip","url":"https://example.com/pkg-1.0.0.zip"}},`+ + `{"version":"1.1.0","time":"`+recent+`","dist":{"type":"zip","url":"https://example.com/pkg-1.1.0.zip"}}]}}`, + ) + return u +} + +// TestRewrittenMetadataETagWithCooldownFollowsKeptVersions checks that with +// cooldown on, the ETag changes when a version leaves cooldown although the +// upstream document stays the same, and comes back when the same versions do. +func TestRewrittenMetadataETagWithCooldownFollowsKeptVersions(t *testing.T) { + for _, c := range validatorCases { + t.Run(c.ecosystem, func(t *testing.T) { + u := cooldownUpstream(t) + proxy := validatorProxy(t, u, time.Hour) + handler := c.handler(proxy, u) + + proxy.Cooldown = &cooldown.Config{Default: "1d"} + etag := serve(handler, c.path, nil).Header().Get("ETag") + + proxy.Cooldown = &cooldown.Config{Default: "1h"} + rec := serve(handler, c.path, http.Header{"If-None-Match": {etag}}) + if rec.Code != http.StatusOK || rec.Header().Get("ETag") == etag { + t.Errorf("after a version left cooldown: status %d, ETag %q; want 200 with a new ETag", rec.Code, rec.Header().Get("ETag")) + } + + proxy.Cooldown = &cooldown.Config{Default: "1d"} + expectNotModified(t, serve(handler, c.path, http.Header{"If-None-Match": {etag}}), etag) + }) + } +} + +// TestRewrittenMetadataETagWithCooldownComesFromInputs checks that with +// cooldown on, the ETag is derived from what fixes the rewrite rather than +// from the rewritten body, when the metadata cache knows the upstream +// digest, and falls back to the body when it does not. +func TestRewrittenMetadataETagWithCooldownComesFromInputs(t *testing.T) { + for _, c := range validatorCases { + for _, cacheMetadata := range []bool{true, false} { + t.Run(fmt.Sprintf("%s/cacheMetadata=%t", c.ecosystem, cacheMetadata), func(t *testing.T) { + u := cooldownUpstream(t) + proxy := validatorProxy(t, u, time.Hour) + proxy.CacheMetadata = cacheMetadata + proxy.Cooldown = &cooldown.Config{Default: "1d"} + handler := c.handler(proxy, u) + + rec := serve(handler, c.path, nil) + bodyETag := metadataETag(rec.Body.Bytes()) + if got := rec.Header().Get("ETag"); (got == bodyETag) == cacheMetadata { + t.Errorf("ETag %q, body hash %q; want them to differ only with a known digest", got, bodyETag) + } + etag := rec.Header().Get("ETag") + expectNotModified(t, serve(handler, c.path, http.Header{"If-None-Match": {etag}}), etag) + }) + } + } +} + +func TestKeptVersionsETag(t *testing.T) { + base := keptVersionsETag("sha256:abc", "http://proxy", "left-pad", []string{"1.0.0", "1.1.0"}) + if got := keptVersionsETag("sha256:abc", "http://proxy", "left-pad", []string{"1.1.0", "1.0.0"}); got != base { + t.Errorf("kept versions in another order: ETag %q, want %q", got, base) + } + for name, etag := range map[string]string{ + "digest": keptVersionsETag("sha256:abd", "http://proxy", "left-pad", []string{"1.0.0", "1.1.0"}), + "proxy URL": keptVersionsETag("sha256:abc", "http://other", "left-pad", []string{"1.0.0", "1.1.0"}), + "name": keptVersionsETag("sha256:abc", "http://proxy", "right-pad", []string{"1.0.0", "1.1.0"}), + "kept versions": keptVersionsETag("sha256:abc", "http://proxy", "left-pad", []string{"1.0.0"}), + "field bounds": keptVersionsETag("sha256:abc", "http://proxy", "left-pad", []string{"1.0.0\x001.1.0"}), + } { + if etag == base { + t.Errorf("a different %s gives the same ETag", name) + } + } + if !strings.HasPrefix(base, `"`) || !strings.HasSuffix(base, `"`) { + t.Errorf("ETag %q is not a quoted strong validator", base) + } +} diff --git a/internal/handler/npm.go b/internal/handler/npm.go index b3d81ef..f819a6e 100644 --- a/internal/handler/npm.go +++ b/internal/handler/npm.go @@ -10,6 +10,7 @@ import ( "net/http" "net/url" "reflect" + "slices" "sort" "strings" "sync" @@ -226,8 +227,8 @@ func (h *NPMHandler) handlePackageMetadata(w http.ResponseWriter, r *http.Reques return } - body, _, err := h.proxy.FetchOrCacheMetadata(r.Context(), "npm", metadataCacheKey, upstreamURL, accept) - if err != nil { + upstream := h.proxy.fetchMetadataDocument(r.Context(), "npm", metadataCacheKey, upstreamURL, accept) + if err := upstream.err; err != nil { if errors.Is(err, ErrUpstreamNotFound) { if versionLookup { JSONError(w, http.StatusNotFound, ErrVersionNotFound.Error()) @@ -241,8 +242,8 @@ func (h *NPMHandler) handlePackageMetadata(w http.ResponseWriter, r *http.Reques return } - rewritten, etag, err := h.proxy.cachedRewrite(r.Context(), "npm", h.proxyURL, metadataCacheKey, body, func(b []byte) ([]byte, error) { - return h.rewriteMetadata(packageName, b) + rewritten, etag, err := h.proxy.cachedRewrite(r.Context(), "npm", h.proxyURL, metadataCacheKey, upstream.body, upstream.digest, func(b []byte) ([]byte, []string, error) { + return h.rewriteMetadataKeeping(packageName, b) }) if err != nil { if r.Context().Err() != nil { @@ -264,11 +265,11 @@ func (h *NPMHandler) handlePackageMetadata(w http.ResponseWriter, r *http.Reques h.proxy.Logger.Warn("failed to rewrite metadata, proxying original", "error", err) w.Header().Set(headerContentType, contentTypeJSON) w.WriteHeader(http.StatusOK) - _, _ = w.Write(body) + _, _ = w.Write(upstream.body) return } - serveRewrittenMetadata(w, r, h.proxy.rewrittenMetadataFor("npm", metadataCacheKey, rewritten, etag)) + serveRewrittenMetadata(w, r, h.proxy.rewrittenMetadataFor(rewritten, etag, upstream.lastModified)) } // handleFilteredNPMVersionMetadata serves /pkg/ and /pkg/ from the @@ -286,8 +287,9 @@ func (h *NPMHandler) handleFilteredNPMVersionMetadata(w http.ResponseWriter, r * return } - rewritten, _, err := h.proxy.cachedRewrite(r.Context(), "npm", h.proxyURL, metadataCacheKey, packument, func(b []byte) ([]byte, error) { - return h.rewriteNPMVersionFromPackument(packageName, specifier, b) + rewritten, _, err := h.proxy.cachedRewrite(r.Context(), "npm", h.proxyURL, metadataCacheKey, packument, "", func(b []byte) ([]byte, []string, error) { + out, err := h.rewriteNPMVersionFromPackument(packageName, specifier, b) + return out, nil, err }) if err != nil { if r.Context().Err() != nil { @@ -388,29 +390,36 @@ func (h *NPMHandler) rewriteNPMVersionFromPackument(packageName, specifier strin // map and dist-tags when filtering changed them, are written fresh. Everything // else reaches the client byte for byte as upstream sent it. func (h *NPMHandler) rewriteMetadata(packageName string, body []byte) ([]byte, error) { + out, _, err := h.rewriteMetadataKeeping(packageName, body) + return out, err +} + +// rewriteMetadataKeeping is rewriteMetadata that also reports the versions it +// kept. +func (h *NPMHandler) rewriteMetadataKeeping(packageName string, body []byte) ([]byte, []string, error) { doc, err := indexNPMPackument(body) if err != nil { - return nil, err + return nil, nil, err } versionsRaw := doc.value(doc.versionsAt) if len(versionsRaw) == 0 || versionsRaw[0] != '{' { version, ok, err := lookupJSONString(body, "version") if err != nil { - return nil, err + return nil, nil, err } if ok { if h.proxy.versionDenied("npm", packageName, version) { - return nil, ErrVersionDenied + return nil, nil, ErrVersionDenied } var out bytes.Buffer h.writeNPMVersion(&out, packageName, version, body) - return out.Bytes(), nil + return out.Bytes(), []string{version}, nil } if len(h.proxy.Denylist.Versions(canonicalPackagePURL("npm", packageName))) != 0 { - return nil, errors.New("npm metadata has no versions object") + return nil, nil, errors.New("npm metadata has no versions object") } - return body, nil // No versions to rewrite + return body, nil, nil // No versions to rewrite } // The filters only look at which versions exist, so they run on a map of @@ -421,7 +430,7 @@ func (h *NPMHandler) rewriteMetadata(packageName string, body []byte) ([]byte, e versions[version] = nil return err }); err != nil { - return nil, err + return nil, nil, err } metadata := map[string]any{} timeMap := doc.decodeObject(doc.timeAt, metadata, "time") @@ -434,7 +443,7 @@ func (h *NPMHandler) rewriteMetadata(packageName string, body []byte) ([]byte, e // Untouched members are copied as they are; the three the filters // handled are written from their filtered state, and only when it changed. - return doc.write(func(out *bytes.Buffer, i int) error { + out, err := doc.write(func(out *bytes.Buffer, i int) error { switch { case i == doc.versionsAt: return h.writeNPMVersions(out, packageName, versionsRaw, versions) @@ -452,6 +461,7 @@ func (h *NPMHandler) rewriteMetadata(packageName string, body []byte) ([]byte, e return nil } }) + return out, slices.Collect(maps.Keys(versions)), err } // npmPackument is a packument's top-level members as offsets into its bytes, diff --git a/internal/handler/rewrite_cache.go b/internal/handler/rewrite_cache.go index 63f7448..853e49a 100644 --- a/internal/handler/rewrite_cache.go +++ b/internal/handler/rewrite_cache.go @@ -3,10 +3,14 @@ package handler import ( "container/list" "context" + "crypto/rand" "crypto/sha256" + "encoding/binary" "encoding/hex" "errors" + "io" "net/http" + "slices" "strings" "sync" "time" @@ -75,6 +79,33 @@ func metadataETag(out []byte) string { return `"` + hex.EncodeToString(sum[:]) + `"` } +// rewriteConfigNonce stands for the configuration a rewrite reads at startup, +// such as the denylist and cooldown rules, in ETags derived from a rewrite's +// inputs. It changes on every start, so no ETag outlives the configuration it +// was derived under. +var rewriteConfigNonce = rand.Text() + +// keptVersionsETag returns the strong ETag of a rewrite from what fixes its +// output: the upstream document's digest, the proxy URL and package name its +// download URLs carry, the versions cooldown and the denylist kept, and the +// startup configuration. Hashing these costs far less than hashing the +// rewritten document. Each field is length-prefixed, so different inputs never +// hash the same bytes. +func keptVersionsETag(digest, proxyURL, name string, kept []string) string { + h := sha256.New() + var length []byte + for _, field := range append([]string{rewriteConfigNonce, digest, proxyURL, name}, slices.Sorted(slices.Values(kept))...) { + length = binary.AppendUvarint(length[:0], uint64(len(field))) + _, _ = h.Write(length) + _, _ = io.WriteString(h, field) + } + return `"` + hex.EncodeToString(h.Sum(nil)) + `"` +} + +// keepingRewrite rewrites a metadata document and reports the versions it +// kept. Together with the document they fix the rewrite's output. +type keepingRewrite func([]byte) ([]byte, []string, error) + // errSharedRewriteAbandoned is what waiters see if the caller running a // shared rewrite panicked out of it. var errSharedRewriteAbandoned = errors.New("shared metadata rewrite did not complete") @@ -204,14 +235,28 @@ func (c *rewriteCache) len() int { } // cachedRewrite rewrites a metadata document through the proxy's rewrite -// cache and returns it with its ETag. Cooldown filtering depends on the +// cache and returns it with its ETag. digest identifies in when the metadata +// cache stored it, and is empty otherwise. Cooldown filtering depends on the // current time, so with cooldown on a cached rewrite could keep hiding a -// version past its cooldown; those rewrites always run. -func (p *Proxy) cachedRewrite(ctx context.Context, ecosystem, proxyURL, name string, in []byte, rewrite func([]byte) ([]byte, error)) ([]byte, string, error) { - if p.rewrites == nil || p.cooldownEnabled() { - return rewriteWithETag(in, rewrite) +// version past its cooldown; those rewrites always run, and when digest is +// known their ETag comes from their inputs rather than from hashing each +// output again. +func (p *Proxy) cachedRewrite(ctx context.Context, ecosystem, proxyURL, name string, in []byte, digest string, rewrite keepingRewrite) ([]byte, string, error) { + if p.cooldownEnabled() && digest != "" { + out, kept, err := rewrite(in) + if err != nil { + return nil, "", err + } + return out, keptVersionsETag(digest, proxyURL, name, kept), nil } - return p.rewrites.rewrite(ctx, rewriteCacheKey(ecosystem, proxyURL, name, in), in, rewrite) + rewriteOnly := func(b []byte) ([]byte, error) { + out, _, err := rewrite(b) + return out, err + } + if p.rewrites == nil || p.cooldownEnabled() { + return rewriteWithETag(in, rewriteOnly) + } + return p.rewrites.rewrite(ctx, rewriteCacheKey(ecosystem, proxyURL, name, in), in, rewriteOnly) } func (p *Proxy) cooldownEnabled() bool { @@ -232,14 +277,10 @@ func (p *Proxy) rewriteLastModified(upstream time.Time) time.Time { return upstream } -// rewrittenMetadataFor returns a fresh rewrite of the metadata cached for -// ecosystem and cacheKey with its validators. -func (p *Proxy) rewrittenMetadataFor(ecosystem, cacheKey string, out []byte, etag string) rewrittenMetadata { - return rewrittenMetadata{ - body: out, - etag: etag, - lastModified: p.rewriteLastModified(p.lookupCachedMeta(ecosystem, cacheKey).lastModified), - } +// rewrittenMetadataFor returns a fresh rewrite of metadata that upstream last +// modified at upstreamModified, with its validators. +func (p *Proxy) rewrittenMetadataFor(out []byte, etag string, upstreamModified time.Time) rewrittenMetadata { + return rewrittenMetadata{body: out, etag: etag, lastModified: p.rewriteLastModified(upstreamModified)} } // storedRewrite returns the cached rewrite of the metadata stored for diff --git a/internal/handler/rewrite_cache_test.go b/internal/handler/rewrite_cache_test.go index 1581ed7..733ba31 100644 --- a/internal/handler/rewrite_cache_test.go +++ b/internal/handler/rewrite_cache_test.go @@ -31,6 +31,11 @@ func (r *countingRewrite) fn(in []byte) ([]byte, error) { return []byte(strings.ToUpper(string(in))), nil } +func (r *countingRewrite) keeping(in []byte) ([]byte, []string, error) { + out, err := r.fn(in) + return out, nil, err +} + func TestRewriteCache_RepeatServesCachedRewrite(t *testing.T) { c := newRewriteCache(1 << 20) var r countingRewrite @@ -185,7 +190,7 @@ func TestCachedRewrite_DisabledRewritesEveryTime(t *testing.T) { var r countingRewrite for range 2 { - _, _, _ = proxy.cachedRewrite(context.Background(), "npm", "http://proxy", "left-pad", []byte("doc"), r.fn) + _, _, _ = proxy.cachedRewrite(context.Background(), "npm", "http://proxy", "left-pad", []byte("doc"), "", r.keeping) } if got := r.calls.Load(); got != 2 { t.Errorf("rewrites = %d, want 2", got) @@ -202,7 +207,7 @@ func TestCachedRewrite_CooldownBypassesCache(t *testing.T) { var r countingRewrite for range 2 { - _, _, _ = proxy.cachedRewrite(context.Background(), "npm", "http://proxy", "left-pad", []byte("doc"), r.fn) + _, _, _ = proxy.cachedRewrite(context.Background(), "npm", "http://proxy", "left-pad", []byte("doc"), "", r.keeping) } if got := r.calls.Load(); got != 2 { t.Errorf("rewrites = %d, want 2", got) From 8903a8c9c98003badb932de32a1f492a3a42ac90 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sat, 10 Oct 2026 13:47:39 +0200 Subject: [PATCH 219/633] Date rewritten metadata by when the rewrite was made Last-Modified was upstream's date clamped to the process start, so different documents could share a date: after a restart, a cached document and the newer one a refresh fetched were both dated at startup, and a client holding the first got 304 for the second. - Date each rewrite by when the rewrite cache first made it, and keep that date while the entry stays cached - Give every new rewrite of a package a date later than the last one it got, even within the same second or after its entry was evicted, so no two representations share a date; dates are remembered only until the clock passes them - Send no Last-Modified when the rewrite cache is off, since nothing records when an uncached rewrite was made; cooldown still sends none - Drop the upstream Last-Modified from metadata lookups, which nothing uses now --- internal/handler/composer.go | 4 +- internal/handler/handler.go | 18 +- internal/handler/metadata_validators_test.go | 178 +++++++++++++++++-- internal/handler/npm.go | 8 +- internal/handler/rewrite_cache.go | 158 ++++++++-------- internal/handler/rewrite_cache_test.go | 4 +- 6 files changed, 265 insertions(+), 105 deletions(-) diff --git a/internal/handler/composer.go b/internal/handler/composer.go index 4c79953..b348024 100644 --- a/internal/handler/composer.go +++ b/internal/handler/composer.go @@ -144,7 +144,7 @@ func (h *ComposerHandler) handlePackageMetadata(w http.ResponseWriter, r *http.R return } - rewritten, etag, err := h.proxy.cachedRewrite(r.Context(), "composer", h.proxyURL, cacheKey, upstream.body, upstream.digest, h.rewriteMetadataKeeping) + rewritten, err := h.proxy.cachedRewrite(r.Context(), "composer", h.proxyURL, cacheKey, upstream.body, upstream.digest, h.rewriteMetadataKeeping) if err != nil { if r.Context().Err() != nil { return // the client left while waiting on a shared rewrite @@ -155,7 +155,7 @@ func (h *ComposerHandler) handlePackageMetadata(w http.ResponseWriter, r *http.R return } - serveRewrittenMetadata(w, r, h.proxy.rewrittenMetadataFor(rewritten, etag, upstream.lastModified)) + serveRewrittenMetadata(w, r, rewritten) } // rewriteMetadata rewrites dist URLs in Composer metadata to point at this proxy. diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 6f7f700..56d4cdd 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -1096,7 +1096,7 @@ func (p *Proxy) fetchOrCacheMetadata(ctx context.Context, ecosystem, cacheKey, u // fetchMetadataDocument is FetchOrCacheMetadata for callers that rewrite the // document and need to know which one they got: the result carries its -// digest and upstream Last-Modified along with the body. +// digest along with the body. func (p *Proxy) fetchMetadataDocument(ctx context.Context, ecosystem, cacheKey, upstreamURL, accept string) metadataResult { return p.lookupMetadata(ctx, ecosystem, cacheKey, upstreamURL, accept, "", nil) } @@ -1183,7 +1183,7 @@ func (p *Proxy) fetchMetadataFromUpstream(ctx context.Context, ecosystem, cacheK if p.CacheMetadata { digest = p.cacheMetadataBlob(ctx, ecosystem, cacheKey, metadataStoragePath(ecosystem, cacheKey), meta) } - return metadataResult{body: meta.body, contentType: meta.contentType, contentEncoding: meta.contentEncoding, digest: digest, lastModified: meta.lastModified} + return metadataResult{body: meta.body, contentType: meta.contentType, contentEncoding: meta.contentEncoding, digest: digest} } // Upstream failed -- fall back to cache if available @@ -1216,21 +1216,17 @@ type metadataResult struct { contentType string contentEncoding string // digest identifies body when the metadata cache stored it, and is - // empty otherwise. lastModified is upstream's, zero when unknown. - digest string - lastModified time.Time - err error + // empty otherwise. + digest string + err error } -// describedBy returns res with the digest and Last-Modified that entry, the -// cache row its body was read for, records. +// describedBy returns res with the digest that entry, the cache row its body +// was read for, records. func (res metadataResult) describedBy(entry *database.MetadataCacheEntry) metadataResult { if entry.ContentDigest.Valid { res.digest = entry.ContentDigest.String } - if entry.LastModified.Valid { - res.lastModified = entry.LastModified.Time - } return res } diff --git a/internal/handler/metadata_validators_test.go b/internal/handler/metadata_validators_test.go index 00213e1..5db4308 100644 --- a/internal/handler/metadata_validators_test.go +++ b/internal/handler/metadata_validators_test.go @@ -10,6 +10,7 @@ import ( "time" "github.com/git-pkgs/cooldown" + "github.com/git-pkgs/proxy/internal/database" ) // validatorUpstream serves one npm packument and one Composer document whose @@ -53,28 +54,54 @@ func (u *validatorUpstream) set(npm, composer string) { u.npm, u.composer = npm, composer } +func (u *validatorUpstream) setLastModified(lastModified time.Time) { + u.mu.Lock() + defer u.mu.Unlock() + u.lastModified = lastModified +} + type validatorCase struct { - ecosystem, path string - handler func(*Proxy, *validatorUpstream) http.Handler + ecosystem, name, path string + handler func(*Proxy, *validatorUpstream) http.Handler } var validatorCases = []validatorCase{ - {"npm", "/left-pad", func(p *Proxy, u *validatorUpstream) http.Handler { + {"npm", "left-pad", "/left-pad", func(p *Proxy, u *validatorUpstream) http.Handler { return NewNPMHandler(p, "http://proxy.example", u.URL).Routes() }}, - {"composer", "/p2/vendor/pkg.json", func(p *Proxy, u *validatorUpstream) http.Handler { + {"composer", "vendor/pkg", "/p2/vendor/pkg.json", func(p *Proxy, u *validatorUpstream) http.Handler { return NewComposerHandlerWithUpstreams(p, "http://proxy.example", u.URL, u.URL).Routes() }}, } func validatorProxy(t *testing.T, u *validatorUpstream, ttl time.Duration) *Proxy { t.Helper() - proxy, _, _, _ := setupTestProxy(t) + proxy, _ := validatorProxyWithDB(t, u, ttl) + return proxy +} + +func validatorProxyWithDB(t *testing.T, u *validatorUpstream, ttl time.Duration) (*Proxy, *database.DB) { + t.Helper() + proxy, db, _, _ := setupTestProxy(t) proxy.HTTPClient = u.Client() proxy.CacheMetadata = true proxy.MetadataTTL = ttl proxy.SetMetadataRewriteCacheSize(1 << 20) - return proxy + return proxy, db +} + +// expireMetadata ages a metadata cache row past any TTL, so the next request +// refreshes it from upstream. +func expireMetadata(t *testing.T, db *database.DB, ecosystem, name string) { + t.Helper() + entry, err := db.GetMetadataCache(ecosystem, name) + if err != nil || entry == nil { + t.Fatalf("metadata row %s/%s = %v, %v", ecosystem, name, entry, err) + } + entry.FetchedAt.Time = time.Now().Add(-48 * time.Hour) + if err := db.UpsertMetadataCache(entry); err != nil { + t.Fatal(err) + } } func serve(handler http.Handler, path string, header http.Header) *httptest.ResponseRecorder { @@ -145,8 +172,9 @@ func TestRewrittenMetadataETagFollowsUpstream(t *testing.T) { } // TestRewrittenMetadataLastModified checks the validator Composer revalidates -// with. It is the upstream Last-Modified, but never earlier than the start of -// this process: a restart can change the rewrite without upstream changing. +// with. It is when this process made the rewrite, not upstream's date: the +// rewrite also depends on configuration read at startup, so the same upstream +// document can be rewritten differently after a restart. func TestRewrittenMetadataLastModified(t *testing.T) { for _, c := range validatorCases { for _, ttl := range []time.Duration{time.Hour, 0} { @@ -154,13 +182,14 @@ func TestRewrittenMetadataLastModified(t *testing.T) { u := newValidatorUpstream(t) handler := c.handler(validatorProxy(t, u, ttl), u) + before := time.Now().Truncate(time.Second) first := serve(handler, c.path, nil) lastModified, err := http.ParseTime(first.Header().Get("Last-Modified")) if err != nil { t.Fatalf("Last-Modified %q: %v", first.Header().Get("Last-Modified"), err) } - if lastModified.Before(processStartedAt.Truncate(time.Second)) { - t.Errorf("Last-Modified %v is before the process started at %v", lastModified, processStartedAt) + if lastModified.Before(before) || lastModified.After(time.Now()) { + t.Errorf("Last-Modified %v, want the time of the rewrite, at or after %v", lastModified, before) } current := serve(handler, c.path, http.Header{"If-Modified-Since": {first.Header().Get("Last-Modified")}}) @@ -186,6 +215,50 @@ func TestRewrittenMetadataLastModified(t *testing.T) { } } +// TestRewrittenMetadataLastModifiedFollowsRefresh is a regression test for a +// false 304. Upstream dates both documents before this process started, the +// cached one and the newer one a refresh fetches, and neither may give the +// newer document a Last-Modified a client holding the older one already has: +// If-Modified-Since from the first response must fetch the new body. +func TestRewrittenMetadataLastModifiedFollowsRefresh(t *testing.T) { + for _, c := range validatorCases { + for _, ttl := range []time.Duration{time.Hour, 0} { + t.Run(c.ecosystem+"/ttl="+ttl.String(), func(t *testing.T) { + u := newValidatorUpstream(t) + proxy, db := validatorProxyWithDB(t, u, ttl) + handler := c.handler(proxy, u) + + first := serve(handler, c.path, nil) + firstModified := first.Header().Get("Last-Modified") + if first.Code != http.StatusOK || firstModified == "" { + t.Fatalf("first response: status %d, Last-Modified %q; want 200 with a Last-Modified", first.Code, firstModified) + } + + u.set( + `{"name":"left-pad","versions":{"1.3.1":{"dist":{"tarball":"https://registry.npmjs.org/left-pad/-/left-pad-1.3.1.tgz"}}}}`, + `{"packages":{"vendor/pkg":[{"version":"1.0.1","dist":{"type":"zip","url":"https://example.com/pkg-1.0.1.zip"}}]}}`, + ) + u.setLastModified(u.lastModified.Add(24 * time.Hour)) + expireMetadata(t, db, c.ecosystem, c.name) + + rec := serve(handler, c.path, http.Header{"If-Modified-Since": {firstModified}}) + if rec.Code != http.StatusOK || rec.Body.String() == first.Body.String() { + t.Fatalf("If-Modified-Since from before the refresh: status %d, body changed %t; want 200 with the new body", + rec.Code, rec.Body.String() != first.Body.String()) + } + was, _ := http.ParseTime(firstModified) + now, err := http.ParseTime(rec.Header().Get("Last-Modified")) + if err != nil || !now.After(was) { + t.Errorf("Last-Modified after the refresh = %q, want later than %q", rec.Header().Get("Last-Modified"), firstModified) + } + if rec := serve(handler, c.path, http.Header{"If-Modified-Since": {rec.Header().Get("Last-Modified")}}); rec.Code != http.StatusNotModified { + t.Errorf("If-Modified-Since the new Last-Modified: status %d, want 304", rec.Code) + } + }) + } + } +} + // TestRewrittenMetadataValidatorsWithCooldown checks that with cooldown on, // rewritten metadata keeps an exact ETag but has no Last-Modified: cooldown // changes the rewrite as versions age in, while upstream stays the same. @@ -243,17 +316,92 @@ func TestRewriteCacheETag(t *testing.T) { if err != nil || string(out) != "doc!" { t.Fatalf("rewrite = %q, %v", out, err) } - if want := metadataETag([]byte("doc!")); etag != want { - t.Errorf("rewrite ETag = %q, want %q", etag, want) + if want := metadataETag([]byte("doc!")); etag.etag != want { + t.Errorf("rewrite ETag = %q, want %q", etag.etag, want) } - if _, cachedETag, ok := c.get(key); !ok || cachedETag != etag { - t.Errorf("get ETag = %q, %v; want %q", cachedETag, ok, etag) + if _, cached, ok := c.get(key); !ok || cached != etag { + t.Errorf("get validators = %+v, %v; want %+v", cached, ok, etag) } - if metadataETag([]byte("other")) == etag { + if metadataETag([]byte("other")) == etag.etag { t.Error("different output has the same ETag") } } +// TestRewriteCacheLastModified checks the Last-Modified the rewrite cache +// gives each rewrite: the second it was made, kept while it stays cached, and +// always later than any earlier rewrite of the same package, so that a client +// holding one representation never gets 304 for another. +func TestRewriteCacheLastModified(t *testing.T) { + c := newRewriteCache(8) + clock := time.Date(2026, 10, 10, 12, 0, 0, 400_000_000, time.UTC) + c.now = func() time.Time { return clock } + second := clock.Truncate(time.Second) + identity := func(b []byte) ([]byte, error) { return b, nil } + rewrite := func(name, doc string) time.Time { + t.Helper() + _, v, err := c.rewrite(t.Context(), rewriteCacheKey("npm", "http://proxy", name, []byte(doc)), []byte(doc), identity) + if err != nil { + t.Fatal(err) + } + return v.lastModified + } + + if got := rewrite("demo", "a"); !got.Equal(second) { + t.Errorf("first rewrite: Last-Modified %v, want %v", got, second) + } + clock = clock.Add(300 * time.Millisecond) + if got := rewrite("demo", "a"); !got.Equal(second) { + t.Errorf("cached rewrite: Last-Modified %v, want %v as before", got, second) + } + + // A new document in the same second must not share the date. + b := rewrite("demo", "b") + if !b.After(second) { + t.Errorf("new document in the same second: Last-Modified %v, want later than %v", b, second) + } + // Another package is not held back by it. + if got := rewrite("other", "x"); !got.Equal(second) { + t.Errorf("other package: Last-Modified %v, want %v", got, second) + } + + // Evicting what came before does not let a later rewrite reuse its date. + for _, doc := range []string{"ccc", "ddd", "eee"} { + rewrite("filler", doc) + } + if _, _, ok := c.get(rewriteCacheKey("npm", "http://proxy", "demo", []byte("b"))); ok { + t.Fatal("expected b to be evicted") + } + if got := rewrite("demo", "b"); !got.After(b) { + t.Errorf("rewrite after eviction: Last-Modified %v, want later than %v", got, b) + } + + // Once the clock moves on, dates follow it again instead of drifting ahead. + clock = clock.Add(10 * time.Second) + if got, want := rewrite("demo", "f"), clock.Truncate(time.Second); !got.Equal(want) { + t.Errorf("later rewrite: Last-Modified %v, want %v", got, want) + } +} + +// TestRewrittenMetadataLastModifiedNeedsRewriteCache checks that without the +// rewrite cache there is no Last-Modified: nothing records when a rewrite was +// first made, so no date can be relied on to change with it. +func TestRewrittenMetadataLastModifiedNeedsRewriteCache(t *testing.T) { + for _, c := range validatorCases { + t.Run(c.ecosystem, func(t *testing.T) { + u := newValidatorUpstream(t) + proxy := validatorProxy(t, u, time.Hour) + proxy.SetMetadataRewriteCacheSize(0) + rec := serve(c.handler(proxy, u), c.path, nil) + if rec.Code != http.StatusOK || rec.Header().Get("ETag") == "" { + t.Fatalf("status %d, ETag %q; want 200 with an ETag", rec.Code, rec.Header().Get("ETag")) + } + if lm := rec.Header().Get("Last-Modified"); lm != "" { + t.Errorf("Last-Modified = %q without the rewrite cache, want none", lm) + } + }) + } +} + // cooldownUpstream sets documents with one version published long ago and // one published two hours ago, so a cooldown of a day hides the recent one // and a cooldown of an hour keeps it. diff --git a/internal/handler/npm.go b/internal/handler/npm.go index f819a6e..96ab10e 100644 --- a/internal/handler/npm.go +++ b/internal/handler/npm.go @@ -242,7 +242,7 @@ func (h *NPMHandler) handlePackageMetadata(w http.ResponseWriter, r *http.Reques return } - rewritten, etag, err := h.proxy.cachedRewrite(r.Context(), "npm", h.proxyURL, metadataCacheKey, upstream.body, upstream.digest, func(b []byte) ([]byte, []string, error) { + rewritten, err := h.proxy.cachedRewrite(r.Context(), "npm", h.proxyURL, metadataCacheKey, upstream.body, upstream.digest, func(b []byte) ([]byte, []string, error) { return h.rewriteMetadataKeeping(packageName, b) }) if err != nil { @@ -269,7 +269,7 @@ func (h *NPMHandler) handlePackageMetadata(w http.ResponseWriter, r *http.Reques return } - serveRewrittenMetadata(w, r, h.proxy.rewrittenMetadataFor(rewritten, etag, upstream.lastModified)) + serveRewrittenMetadata(w, r, rewritten) } // handleFilteredNPMVersionMetadata serves /pkg/ and /pkg/ from the @@ -287,7 +287,7 @@ func (h *NPMHandler) handleFilteredNPMVersionMetadata(w http.ResponseWriter, r * return } - rewritten, _, err := h.proxy.cachedRewrite(r.Context(), "npm", h.proxyURL, metadataCacheKey, packument, "", func(b []byte) ([]byte, []string, error) { + rewritten, err := h.proxy.cachedRewrite(r.Context(), "npm", h.proxyURL, metadataCacheKey, packument, "", func(b []byte) ([]byte, []string, error) { out, err := h.rewriteNPMVersionFromPackument(packageName, specifier, b) return out, nil, err }) @@ -314,7 +314,7 @@ func (h *NPMHandler) handleFilteredNPMVersionMetadata(w http.ResponseWriter, r * w.Header().Set(headerContentType, contentTypeJSON) w.WriteHeader(http.StatusOK) - _, _ = w.Write(rewritten) + _, _ = w.Write(rewritten.body) } // npmMetadataCacheKey qualifies routed-package metadata cache entries with the diff --git a/internal/handler/rewrite_cache.go b/internal/handler/rewrite_cache.go index 853e49a..5fc0b71 100644 --- a/internal/handler/rewrite_cache.go +++ b/internal/handler/rewrite_cache.go @@ -32,47 +32,51 @@ import ( // depends on is fixed for the life of the process: the proxy URL is part of // the key, and the denylist is loaded at startup. // -// Each entry keeps the ETag of its output, so clients can revalidate their -// copy without the output being hashed again on every request. +// Each entry keeps the validators of its output, so clients can revalidate +// their copy without the output being hashed again on every request. type rewriteCache struct { maxBytes int64 + now func() time.Time mu sync.Mutex size int64 order *list.List // most recently used at the front entries map[string]*list.Element inFlight map[string]*inflightRewrite + // issued holds the Last-Modified last given to a rewrite of each package, + // keyed by its cache key without the digest, until the clock passes it. + issued map[string]time.Time } type rewriteEntry struct { - key string - out []byte - etag string + key string + out []byte + metadataValidators } -// inflightRewrite is one rewrite that concurrent callers share. out, etag and +// inflightRewrite is one rewrite that concurrent callers share. out, v and // err are written before done closes and read only after, so the close is the // handoff. type inflightRewrite struct { done chan struct{} out []byte - etag string + v metadataValidators err error } -// processStartedAt bounds the Last-Modified of rewritten metadata from below. -// A rewrite depends on configuration read at startup as well as on the -// upstream document, so a restart can change it while upstream stays the same. -var processStartedAt = time.Now() - -// rewrittenMetadata is a rewritten metadata document with the validators a -// client revalidates its copy with. lastModified is zero when unknown. -type rewrittenMetadata struct { - body []byte +// metadataValidators are what a client revalidates its copy of rewritten +// metadata with. lastModified is zero when there is none to give. +type metadataValidators struct { etag string lastModified time.Time } +// rewrittenMetadata is a rewritten metadata document with its validators. +type rewrittenMetadata struct { + body []byte + metadataValidators +} + // metadataETag returns the strong ETag of a rewritten document. func metadataETag(out []byte) string { sum := sha256.Sum256(out) @@ -118,9 +122,11 @@ func newRewriteCache(maxBytes int64) *rewriteCache { } return &rewriteCache{ maxBytes: maxBytes, + now: time.Now, order: list.New(), entries: make(map[string]*list.Element), inFlight: make(map[string]*inflightRewrite), + issued: make(map[string]time.Time), } } @@ -132,33 +138,34 @@ func rewriteCacheKey(ecosystem, proxyURL, name string, in []byte) string { } // rewriteCacheKeyForDigest is rewriteCacheKey for upstream bytes known only by -// their digest, in the form the metadata cache records it. +// their digest, in the form the metadata cache records it. The digest comes +// last, so the key without it names the package the rewrite is for. func rewriteCacheKeyForDigest(ecosystem, proxyURL, name, digest string) string { return strings.Join([]string{ecosystem, proxyURL, name, digest}, "\x00") } -// get returns the cached rewrite for key and its ETag, if there is one. -func (c *rewriteCache) get(key string) ([]byte, string, bool) { +// get returns the cached rewrite for key and its validators, if there is one. +func (c *rewriteCache) get(key string) ([]byte, metadataValidators, bool) { if c == nil { - return nil, "", false + return nil, metadataValidators{}, false } c.mu.Lock() defer c.mu.Unlock() el, ok := c.entries[key] if !ok { - return nil, "", false + return nil, metadataValidators{}, false } c.order.MoveToFront(el) e := el.Value.(*rewriteEntry) - return e.out, e.etag, true + return e.out, e.metadataValidators, true } -// rewrite returns rewrite(in) and its ETag, from the cache when it can. +// rewrite returns rewrite(in) and its validators, from the cache when it can. // Callers must treat the returned bytes as read-only: a cached result is // shared. A caller waiting on another's rewrite leaves when ctx ends; the // rewrite itself always runs to completion, so the result is cached for the // next request. -func (c *rewriteCache) rewrite(ctx context.Context, key string, in []byte, rewrite func([]byte) ([]byte, error)) ([]byte, string, error) { +func (c *rewriteCache) rewrite(ctx context.Context, key string, in []byte, rewrite func([]byte) ([]byte, error)) (out []byte, v metadataValidators, err error) { if c == nil { return rewriteWithETag(in, rewrite) } @@ -168,15 +175,15 @@ func (c *rewriteCache) rewrite(ctx context.Context, key string, in []byte, rewri c.order.MoveToFront(el) e := el.Value.(*rewriteEntry) c.mu.Unlock() - return e.out, e.etag, nil + return e.out, e.metadataValidators, nil } if f, ok := c.inFlight[key]; ok { c.mu.Unlock() select { case <-ctx.Done(): - return nil, "", ctx.Err() + return nil, metadataValidators{}, ctx.Err() case <-f.done: - return f.out, f.etag, f.err + return f.out, f.v, f.err } } f := &inflightRewrite{done: make(chan struct{}), err: errSharedRewriteAbandoned} @@ -187,22 +194,50 @@ func (c *rewriteCache) rewrite(ctx context.Context, key string, in []byte, rewri c.mu.Lock() delete(c.inFlight, key) if f.err == nil { - c.add(&rewriteEntry{key: key, out: f.out, etag: f.etag}) + f.v.lastModified = c.issueLastModified(key) + c.add(&rewriteEntry{key: key, out: f.out, metadataValidators: f.v}) } c.mu.Unlock() close(f.done) + out, v, err = f.out, f.v, f.err }() - f.out, f.etag, f.err = rewriteWithETag(in, rewrite) - return f.out, f.etag, f.err + f.out, f.v, f.err = rewriteWithETag(in, rewrite) + return f.out, f.v, f.err } -// rewriteWithETag runs rewrite and computes the ETag of what it returns. -func rewriteWithETag(in []byte, rewrite func([]byte) ([]byte, error)) ([]byte, string, error) { +// issueLastModified returns the Last-Modified of a rewrite that has just been +// made for key: the current second, or a second after the date last given to +// a rewrite of the same package when that is no earlier. Upstream's own date +// cannot serve, since the rewrite also depends on configuration read at +// startup, and two documents can share a second. A strictly later date for +// every new rewrite means a client holding one rewrite never gets 304 for +// another, even after its entry was evicted. Dates are remembered only until +// the clock passes them; after that the clock alone keeps them increasing. +// Called with mu held. +func (c *rewriteCache) issueLastModified(key string) time.Time { + now := c.now().Truncate(time.Second) + for pkg, issued := range c.issued { + if issued.Before(now) { + delete(c.issued, pkg) + } + } + pkg := key[:strings.LastIndexByte(key, 0)] + lastModified := now + if issued, ok := c.issued[pkg]; ok && !issued.Before(now) { + lastModified = issued.Add(time.Second) + } + c.issued[pkg] = lastModified + return lastModified +} + +// rewriteWithETag runs rewrite and computes the ETag of what it returns. It +// gives no Last-Modified: nothing records when an uncached rewrite was made. +func rewriteWithETag(in []byte, rewrite func([]byte) ([]byte, error)) ([]byte, metadataValidators, error) { out, err := rewrite(in) if err != nil { - return nil, "", err + return nil, metadataValidators{}, err } - return out, metadataETag(out), nil + return out, metadataValidators{etag: metadataETag(out)}, nil } // add stores e and evicts least recently used entries until the cache is @@ -235,54 +270,39 @@ func (c *rewriteCache) len() int { } // cachedRewrite rewrites a metadata document through the proxy's rewrite -// cache and returns it with its ETag. digest identifies in when the metadata -// cache stored it, and is empty otherwise. Cooldown filtering depends on the -// current time, so with cooldown on a cached rewrite could keep hiding a -// version past its cooldown; those rewrites always run, and when digest is -// known their ETag comes from their inputs rather than from hashing each -// output again. -func (p *Proxy) cachedRewrite(ctx context.Context, ecosystem, proxyURL, name string, in []byte, digest string, rewrite keepingRewrite) ([]byte, string, error) { +// cache and returns it with its validators. digest identifies in when the +// metadata cache stored it, and is empty otherwise. Cooldown filtering depends +// on the current time, so with cooldown on a cached rewrite could keep hiding +// a version past its cooldown; those rewrites always run, with no +// Last-Modified, and when digest is known their ETag comes from their inputs +// rather than from hashing each output again. +func (p *Proxy) cachedRewrite(ctx context.Context, ecosystem, proxyURL, name string, in []byte, digest string, rewrite keepingRewrite) (rewrittenMetadata, error) { if p.cooldownEnabled() && digest != "" { out, kept, err := rewrite(in) if err != nil { - return nil, "", err + return rewrittenMetadata{}, err } - return out, keptVersionsETag(digest, proxyURL, name, kept), nil + return rewrittenMetadata{out, metadataValidators{etag: keptVersionsETag(digest, proxyURL, name, kept)}}, nil } rewriteOnly := func(b []byte) ([]byte, error) { out, _, err := rewrite(b) return out, err } + var out []byte + var v metadataValidators + var err error if p.rewrites == nil || p.cooldownEnabled() { - return rewriteWithETag(in, rewriteOnly) + out, v, err = rewriteWithETag(in, rewriteOnly) + } else { + out, v, err = p.rewrites.rewrite(ctx, rewriteCacheKey(ecosystem, proxyURL, name, in), in, rewriteOnly) } - return p.rewrites.rewrite(ctx, rewriteCacheKey(ecosystem, proxyURL, name, in), in, rewriteOnly) + return rewrittenMetadata{out, v}, err } func (p *Proxy) cooldownEnabled() bool { return p.Cooldown != nil && p.Cooldown.Enabled() } -// rewriteLastModified returns the Last-Modified of a rewrite of metadata that -// upstream last modified at upstream, or zero when there is none to give. -// With cooldown on there is none: versions leaving cooldown change the -// rewrite while upstream stays the same. -func (p *Proxy) rewriteLastModified(upstream time.Time) time.Time { - if upstream.IsZero() || p.cooldownEnabled() { - return time.Time{} - } - if upstream.Before(processStartedAt) { - return processStartedAt - } - return upstream -} - -// rewrittenMetadataFor returns a fresh rewrite of metadata that upstream last -// modified at upstreamModified, with its validators. -func (p *Proxy) rewrittenMetadataFor(out []byte, etag string, upstreamModified time.Time) rewrittenMetadata { - return rewrittenMetadata{body: out, etag: etag, lastModified: p.rewriteLastModified(upstreamModified)} -} - // storedRewrite returns the cached rewrite of the metadata stored for // ecosystem and cacheKey, when that metadata is still within its TTL and a // rewrite of it is cached. It reads only the cache row, never the stored @@ -303,16 +323,12 @@ func (p *Proxy) storedRewrite(ecosystem, cacheKey, proxyURL, name string) (rewri if !entry.FetchedAt.Valid || time.Since(entry.FetchedAt.Time) >= p.MetadataTTL { return rewrittenMetadata{}, false } - out, etag, ok := p.rewrites.get(rewriteCacheKeyForDigest(ecosystem, proxyURL, name, entry.ContentDigest.String)) + out, v, ok := p.rewrites.get(rewriteCacheKeyForDigest(ecosystem, proxyURL, name, entry.ContentDigest.String)) if !ok { return rewrittenMetadata{}, false } metrics.RecordCacheHit(ecosystem) - var upstreamModified time.Time - if entry.LastModified.Valid { - upstreamModified = entry.LastModified.Time - } - return rewrittenMetadata{body: out, etag: etag, lastModified: p.rewriteLastModified(upstreamModified)}, true + return rewrittenMetadata{out, v}, true } // serveRewrittenMetadata writes rewritten metadata with its validators, or diff --git a/internal/handler/rewrite_cache_test.go b/internal/handler/rewrite_cache_test.go index 733ba31..473f80a 100644 --- a/internal/handler/rewrite_cache_test.go +++ b/internal/handler/rewrite_cache_test.go @@ -190,7 +190,7 @@ func TestCachedRewrite_DisabledRewritesEveryTime(t *testing.T) { var r countingRewrite for range 2 { - _, _, _ = proxy.cachedRewrite(context.Background(), "npm", "http://proxy", "left-pad", []byte("doc"), "", r.keeping) + _, _ = proxy.cachedRewrite(context.Background(), "npm", "http://proxy", "left-pad", []byte("doc"), "", r.keeping) } if got := r.calls.Load(); got != 2 { t.Errorf("rewrites = %d, want 2", got) @@ -207,7 +207,7 @@ func TestCachedRewrite_CooldownBypassesCache(t *testing.T) { var r countingRewrite for range 2 { - _, _, _ = proxy.cachedRewrite(context.Background(), "npm", "http://proxy", "left-pad", []byte("doc"), "", r.keeping) + _, _ = proxy.cachedRewrite(context.Background(), "npm", "http://proxy", "left-pad", []byte("doc"), "", r.keeping) } if got := r.calls.Load(); got != 2 { t.Errorf("rewrites = %d, want 2", got) From cff4b5188d3129260520af6f4795b20f66561559 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Mon, 12 Oct 2026 07:07:03 +0200 Subject: [PATCH 220/633] Date rewritten metadata by when its package started serving it Two false 304s remained. A rewrite kept the date it was made, so when upstream went from A to B and back to A, A's cached rewrite came back with a date earlier than B's and a client holding B got 304. And a second rewrite in the same second was dated a second ahead of the clock, a date only that process remembered: after a restart a changed document could be dated earlier than one the client held. - Record which rewrite each package is serving and since when, and date a rewrite by the second its package switched to it, including back to one still cached - Never date ahead of the clock: a rewrite switched to in a second that already dated another for the package goes undated until the next second - Date nothing in the second the cache starts, the last second the previous process may have dated a rewrite in - Drop a package's record when its current rewrite is evicted, so records stay bounded by the cache; forgetting one dated in the current second holds back dates for new packages until the next second --- internal/handler/metadata_validators_test.go | 269 +++++++++++++++---- internal/handler/rewrite_cache.go | 162 +++++++---- 2 files changed, 338 insertions(+), 93 deletions(-) diff --git a/internal/handler/metadata_validators_test.go b/internal/handler/metadata_validators_test.go index 5db4308..8e9f90b 100644 --- a/internal/handler/metadata_validators_test.go +++ b/internal/handler/metadata_validators_test.go @@ -76,18 +76,61 @@ var validatorCases = []validatorCase{ func validatorProxy(t *testing.T, u *validatorUpstream, ttl time.Duration) *Proxy { t.Helper() - proxy, _ := validatorProxyWithDB(t, u, ttl) + proxy, _, _ := validatorProxyWithDB(t, u, ttl) return proxy } -func validatorProxyWithDB(t *testing.T, u *validatorUpstream, ttl time.Duration) (*Proxy, *database.DB) { +// validatorProxyWithDB returns a proxy whose rewrite cache runs on a clock +// the test moves, starting a second after the process start it models. +func validatorProxyWithDB(t *testing.T, u *validatorUpstream, ttl time.Duration) (*Proxy, *database.DB, *testClock) { t.Helper() proxy, db, _, _ := setupTestProxy(t) proxy.HTTPClient = u.Client() proxy.CacheMetadata = true proxy.MetadataTTL = ttl + clock := &testClock{t: time.Date(2026, 10, 10, 12, 0, 0, 0, time.UTC)} + restartRewriteCache(proxy, clock) + clock.advance(time.Second) + return proxy, db, clock +} + +// restartRewriteCache gives proxy a new, empty rewrite cache started at the +// clock's current time, as a restart of the process would. +func restartRewriteCache(proxy *Proxy, clock *testClock) { proxy.SetMetadataRewriteCacheSize(1 << 20) - return proxy, db + proxy.rewrites.useClock(clock.now) +} + +// useClock makes the cache read time from now, as if it was made at now(). +func (c *rewriteCache) useClock(now func() time.Time) { + c.now = now + c.forgotten = now().Truncate(time.Second) +} + +type testClock struct { + mu sync.Mutex + t time.Time +} + +func (c *testClock) now() time.Time { + c.mu.Lock() + defer c.mu.Unlock() + return c.t +} + +func (c *testClock) advance(d time.Duration) { + c.mu.Lock() + defer c.mu.Unlock() + c.t = c.t.Add(d) +} + +// npmV and composerV are upstream documents with a single version v. +func npmV(v string) string { + return `{"name":"left-pad","versions":{"` + v + `":{"dist":{"tarball":"https://registry.npmjs.org/left-pad/-/left-pad-` + v + `.tgz"}}}}` +} + +func composerV(v string) string { + return `{"packages":{"vendor/pkg":[{"version":"` + v + `","dist":{"type":"zip","url":"https://example.com/pkg-` + v + `.zip"}}]}}` } // expireMetadata ages a metadata cache row past any TTL, so the next request @@ -172,25 +215,28 @@ func TestRewrittenMetadataETagFollowsUpstream(t *testing.T) { } // TestRewrittenMetadataLastModified checks the validator Composer revalidates -// with. It is when this process made the rewrite, not upstream's date: the -// rewrite also depends on configuration read at startup, so the same upstream -// document can be rewritten differently after a restart. +// with. It is when this process started serving the rewrite, not upstream's +// date: the rewrite also depends on configuration read at startup, so the +// same upstream document can be rewritten differently after a restart. It +// stays the same while the rewrite does. func TestRewrittenMetadataLastModified(t *testing.T) { for _, c := range validatorCases { for _, ttl := range []time.Duration{time.Hour, 0} { t.Run(c.ecosystem+"/ttl="+ttl.String(), func(t *testing.T) { u := newValidatorUpstream(t) - handler := c.handler(validatorProxy(t, u, ttl), u) + proxy, _, clock := validatorProxyWithDB(t, u, ttl) + handler := c.handler(proxy, u) - before := time.Now().Truncate(time.Second) + clock.advance(400 * time.Millisecond) first := serve(handler, c.path, nil) lastModified, err := http.ParseTime(first.Header().Get("Last-Modified")) if err != nil { t.Fatalf("Last-Modified %q: %v", first.Header().Get("Last-Modified"), err) } - if lastModified.Before(before) || lastModified.After(time.Now()) { - t.Errorf("Last-Modified %v, want the time of the rewrite, at or after %v", lastModified, before) + if want := clock.now().Truncate(time.Second); !lastModified.Equal(want) { + t.Errorf("Last-Modified %v, want %v, the second of the rewrite", lastModified, want) } + clock.advance(time.Hour) current := serve(handler, c.path, http.Header{"If-Modified-Since": {first.Header().Get("Last-Modified")}}) if current.Code != http.StatusNotModified || current.Body.Len() != 0 { @@ -225,7 +271,7 @@ func TestRewrittenMetadataLastModifiedFollowsRefresh(t *testing.T) { for _, ttl := range []time.Duration{time.Hour, 0} { t.Run(c.ecosystem+"/ttl="+ttl.String(), func(t *testing.T) { u := newValidatorUpstream(t) - proxy, db := validatorProxyWithDB(t, u, ttl) + proxy, db, clock := validatorProxyWithDB(t, u, ttl) handler := c.handler(proxy, u) first := serve(handler, c.path, nil) @@ -234,10 +280,8 @@ func TestRewrittenMetadataLastModifiedFollowsRefresh(t *testing.T) { t.Fatalf("first response: status %d, Last-Modified %q; want 200 with a Last-Modified", first.Code, firstModified) } - u.set( - `{"name":"left-pad","versions":{"1.3.1":{"dist":{"tarball":"https://registry.npmjs.org/left-pad/-/left-pad-1.3.1.tgz"}}}}`, - `{"packages":{"vendor/pkg":[{"version":"1.0.1","dist":{"type":"zip","url":"https://example.com/pkg-1.0.1.zip"}}]}}`, - ) + clock.advance(time.Minute) + u.set(npmV("1.3.1"), composerV("1.0.1")) u.setLastModified(u.lastModified.Add(24 * time.Hour)) expireMetadata(t, db, c.ecosystem, c.name) @@ -259,6 +303,121 @@ func TestRewrittenMetadataLastModifiedFollowsRefresh(t *testing.T) { } } +// TestRewrittenMetadataLastModifiedReturningDocument is a regression test +// for a false 304. Upstream serves A, then B, then A again; A's cached +// rewrite must not bring back the date it was first served with, which is +// earlier than B's, or a client holding B gets 304 for A. +func TestRewrittenMetadataLastModifiedReturningDocument(t *testing.T) { + for _, c := range validatorCases { + for _, ttl := range []time.Duration{time.Hour, 0} { + t.Run(c.ecosystem+"/ttl="+ttl.String(), func(t *testing.T) { + u := newValidatorUpstream(t) + proxy, db, clock := validatorProxyWithDB(t, u, ttl) + handler := c.handler(proxy, u) + change := func(v string) { + clock.advance(10 * time.Second) + u.set(npmV(v), composerV(v)) + if ttl > 0 { + expireMetadata(t, db, c.ecosystem, c.name) + } + } + + u.set(npmV("1.0.0"), composerV("1.0.0")) + a := serve(handler, c.path, nil) + change("2.0.0") + b := serve(handler, c.path, nil) + if b.Body.String() == a.Body.String() || b.Header().Get("Last-Modified") == "" { + t.Fatalf("B: Last-Modified %q, body changed %t; want a new body with a date", b.Header().Get("Last-Modified"), b.Body.String() != a.Body.String()) + } + change("1.0.0") + + rec := serve(handler, c.path, http.Header{"If-Modified-Since": {b.Header().Get("Last-Modified")}}) + if rec.Code != http.StatusOK || rec.Body.String() != a.Body.String() { + t.Fatalf("A again, If-Modified-Since B's date: status %d, body is A %t; want 200 with A", rec.Code, rec.Body.String() == a.Body.String()) + } + was, _ := http.ParseTime(b.Header().Get("Last-Modified")) + if now, err := http.ParseTime(rec.Header().Get("Last-Modified")); err != nil || !now.After(was) { + t.Errorf("A again: Last-Modified %q, want later than B's %q", rec.Header().Get("Last-Modified"), b.Header().Get("Last-Modified")) + } + }) + } + } +} + +// TestRewrittenMetadataLastModifiedSameSecond checks that a document that +// replaces another within the second the other was dated gets no date of +// its own until a later second, rather than one ahead of the clock. The +// clock-time date it would share is one a client may already hold. +func TestRewrittenMetadataLastModifiedSameSecond(t *testing.T) { + for _, c := range validatorCases { + t.Run(c.ecosystem, func(t *testing.T) { + u := newValidatorUpstream(t) + proxy, _, clock := validatorProxyWithDB(t, u, 0) + handler := c.handler(proxy, u) + + a := serve(handler, c.path, nil) + aModified := a.Header().Get("Last-Modified") + u.set(npmV("2.0.0"), composerV("2.0.0")) + for i, header := range []http.Header{nil, {"If-Modified-Since": {aModified}}} { + rec := serve(handler, c.path, header) + if rec.Code != http.StatusOK || rec.Body.String() == a.Body.String() { + t.Fatalf("request %d for B: status %d; want 200 with B", i, rec.Code) + } + if lm := rec.Header().Get("Last-Modified"); lm != "" { + t.Errorf("request %d for B in A's second: Last-Modified %q, want none", i, lm) + } + } + + clock.advance(time.Second) + rec := serve(handler, c.path, nil) + lastModified, err := http.ParseTime(rec.Header().Get("Last-Modified")) + if want := clock.now().Truncate(time.Second); err != nil || !lastModified.Equal(want) { + t.Errorf("B a second later: Last-Modified %q, want %v", rec.Header().Get("Last-Modified"), want) + } + }) + } +} + +// TestRewrittenMetadataLastModifiedAcrossRestart is a regression test for a +// false 304 after a restart. Dates are never ahead of the clock, and the +// restarted process dates nothing in the second it starts, the last in which +// the previous process may have dated a document. +func TestRewrittenMetadataLastModifiedAcrossRestart(t *testing.T) { + for _, c := range validatorCases { + t.Run(c.ecosystem, func(t *testing.T) { + u := newValidatorUpstream(t) + proxy, _, clock := validatorProxyWithDB(t, u, 0) + handler := c.handler(proxy, u) + + // Two documents in one second; the client keeps the latest date. + latest := serve(handler, c.path, nil).Header().Get("Last-Modified") + u.set(npmV("2.0.0"), composerV("2.0.0")) + if lm := serve(handler, c.path, nil).Header().Get("Last-Modified"); lm != "" { + latest = lm + } + if held, _ := http.ParseTime(latest); held.After(clock.now()) { + t.Errorf("Last-Modified %q is ahead of the clock at %v", latest, clock.now()) + } + + restartRewriteCache(proxy, clock) + u.set(npmV("3.0.0"), composerV("3.0.0")) + rec := serve(handler, c.path, http.Header{"If-Modified-Since": {latest}}) + if rec.Code != http.StatusOK || !strings.Contains(rec.Body.String(), "3.0.0") { + t.Fatalf("changed document after a restart, If-Modified-Since %q: status %d; want 200 with it", latest, rec.Code) + } + if lm := rec.Header().Get("Last-Modified"); lm != "" { + t.Errorf("Last-Modified %q in the second the process started, want none", lm) + } + + clock.advance(time.Second) + rec = serve(handler, c.path, http.Header{"If-Modified-Since": {latest}}) + if rec.Code != http.StatusOK || rec.Header().Get("Last-Modified") == "" { + t.Errorf("a second after the restart: status %d, Last-Modified %q; want 200 with a date", rec.Code, rec.Header().Get("Last-Modified")) + } + }) + } +} + // TestRewrittenMetadataValidatorsWithCooldown checks that with cooldown on, // rewritten metadata keeps an exact ETag but has no Last-Modified: cooldown // changes the rewrite as versions age in, while upstream stays the same. @@ -328,58 +487,76 @@ func TestRewriteCacheETag(t *testing.T) { } // TestRewriteCacheLastModified checks the Last-Modified the rewrite cache -// gives each rewrite: the second it was made, kept while it stays cached, and -// always later than any earlier rewrite of the same package, so that a client -// holding one representation never gets 304 for another. +// serves each rewrite with: the second the package started serving it, so a +// client holding one representation never gets 304 for another. A date is +// never ahead of the clock, so none is given in a second that may already +// have dated another representation of the package. func TestRewriteCacheLastModified(t *testing.T) { + clock := &testClock{t: time.Date(2026, 10, 10, 12, 0, 0, 400_000_000, time.UTC)} c := newRewriteCache(8) - clock := time.Date(2026, 10, 10, 12, 0, 0, 400_000_000, time.UTC) - c.now = func() time.Time { return clock } - second := clock.Truncate(time.Second) + c.useClock(clock.now) + second := func() time.Time { return clock.now().Truncate(time.Second) } identity := func(b []byte) ([]byte, error) { return b, nil } + key := func(name, doc string) string { return rewriteCacheKey("npm", "http://proxy", name, []byte(doc)) } rewrite := func(name, doc string) time.Time { t.Helper() - _, v, err := c.rewrite(t.Context(), rewriteCacheKey("npm", "http://proxy", name, []byte(doc)), []byte(doc), identity) + _, v, err := c.rewrite(t.Context(), key(name, doc), []byte(doc), identity) if err != nil { t.Fatal(err) } return v.lastModified } - - if got := rewrite("demo", "a"); !got.Equal(second) { - t.Errorf("first rewrite: Last-Modified %v, want %v", got, second) + expect := func(what string, got, want time.Time) { + t.Helper() + if !got.Equal(want) { + t.Errorf("%s: Last-Modified %v, want %v", what, got, want) + } } - clock = clock.Add(300 * time.Millisecond) - if got := rewrite("demo", "a"); !got.Equal(second) { - t.Errorf("cached rewrite: Last-Modified %v, want %v as before", got, second) + var undated time.Time + + expect("in the second the cache started", rewrite("demo", "a"), undated) + clock.advance(time.Second) + a := second() + expect("a second later", rewrite("demo", "a"), a) + clock.advance(300 * time.Millisecond) + if _, v, ok := c.get(key("demo", "a")); !ok || !v.lastModified.Equal(a) { + t.Errorf("get: Last-Modified %v, %v; want %v", v.lastModified, ok, a) } - // A new document in the same second must not share the date. - b := rewrite("demo", "b") - if !b.After(second) { - t.Errorf("new document in the same second: Last-Modified %v, want later than %v", b, second) - } + // A new document in the second a was dated gets no date, and neither + // does a, back in the same second, until a later one. + expect("new document in the same second", rewrite("demo", "b"), undated) + expect("a back in the same second", rewrite("demo", "a"), undated) // Another package is not held back by it. - if got := rewrite("other", "x"); !got.Equal(second) { - t.Errorf("other package: Last-Modified %v, want %v", got, second) - } + expect("other package", rewrite("other", "x"), a) + clock.advance(time.Second) + expect("a in the next second", rewrite("demo", "a"), second()) - // Evicting what came before does not let a later rewrite reuse its date. + // A cached document coming back is dated when it comes back. + clock.advance(10 * time.Second) + b := second() + expect("b", rewrite("demo", "b"), b) + clock.advance(10 * time.Second) + expect("a after b", rewrite("demo", "a"), second()) + + // Evicting the package's current document forgets the package. Its next + // document can be dated at once unless the forgotten one was dated in + // this second. + clock.advance(10 * time.Second) for _, doc := range []string{"ccc", "ddd", "eee"} { rewrite("filler", doc) } - if _, _, ok := c.get(rewriteCacheKey("npm", "http://proxy", "demo", []byte("b"))); ok { - t.Fatal("expected b to be evicted") + if _, _, ok := c.get(key("demo", "a")); ok { + t.Fatal("expected a to be evicted") } - if got := rewrite("demo", "b"); !got.After(b) { - t.Errorf("rewrite after eviction: Last-Modified %v, want later than %v", got, b) + if _, ok := c.serving[rewritePackage(key("demo", "a"))]; ok { + t.Error("the package is still recorded after its rewrite was evicted") } - - // Once the clock moves on, dates follow it again instead of drifting ahead. - clock = clock.Add(10 * time.Second) - if got, want := rewrite("demo", "f"), clock.Truncate(time.Second); !got.Equal(want) { - t.Errorf("later rewrite: Last-Modified %v, want %v", got, want) + expect("after a clean eviction", rewrite("demo", "f"), second()) + for _, doc := range []string{"ggg", "hhh", "iii"} { + rewrite("filler", doc) } + expect("after an eviction in the second f was dated", rewrite("demo", "j"), undated) } // TestRewrittenMetadataLastModifiedNeedsRewriteCache checks that without the diff --git a/internal/handler/rewrite_cache.go b/internal/handler/rewrite_cache.go index 5fc0b71..540a13f 100644 --- a/internal/handler/rewrite_cache.go +++ b/internal/handler/rewrite_cache.go @@ -32,8 +32,9 @@ import ( // depends on is fixed for the life of the process: the proxy URL is part of // the key, and the denylist is loaded at startup. // -// Each entry keeps the validators of its output, so clients can revalidate -// their copy without the output being hashed again on every request. +// Each entry keeps the ETag of its output, so clients can revalidate their +// copy without the output being hashed again on every request. The cache also +// dates what it serves for each package; see lastModified. type rewriteCache struct { maxBytes int64 now func() time.Time @@ -43,24 +44,38 @@ type rewriteCache struct { order *list.List // most recently used at the front entries map[string]*list.Element inFlight map[string]*inflightRewrite - // issued holds the Last-Modified last given to a rewrite of each package, - // keyed by its cache key without the digest, until the clock passes it. - issued map[string]time.Time + // serving holds the rewrite each package is serving, keyed by its cache + // key without the digest, while that rewrite is cached. + serving map[string]servedRewrite + // forgotten is the latest second in which the cache may have dated a + // rewrite of a package it no longer has in serving: the second it started, + // as the previous process may have dated one then, or a later one. + forgotten time.Time +} + +// servedRewrite is the rewrite a package is serving and since when. +type servedRewrite struct { + etag string + // switched is the second the package started serving it. + switched time.Time + // since is its Last-Modified: the first second, from switched on, that + // can have dated no other rewrite of the package. Zero until there is one. + since time.Time } type rewriteEntry struct { - key string - out []byte - metadataValidators + key string + out []byte + etag string } -// inflightRewrite is one rewrite that concurrent callers share. out, v and -// err are written before done closes and read only after, so the close is the -// handoff. +// inflightRewrite is one rewrite that concurrent callers share. out, etag +// and err are written before done closes and read only after, so the close is +// the handoff. type inflightRewrite struct { done chan struct{} out []byte - v metadataValidators + etag string err error } @@ -121,12 +136,13 @@ func newRewriteCache(maxBytes int64) *rewriteCache { return nil } return &rewriteCache{ - maxBytes: maxBytes, - now: time.Now, - order: list.New(), - entries: make(map[string]*list.Element), - inFlight: make(map[string]*inflightRewrite), - issued: make(map[string]time.Time), + maxBytes: maxBytes, + now: time.Now, + order: list.New(), + entries: make(map[string]*list.Element), + inFlight: make(map[string]*inflightRewrite), + serving: make(map[string]servedRewrite), + forgotten: time.Now().Truncate(time.Second), } } @@ -157,7 +173,7 @@ func (c *rewriteCache) get(key string) ([]byte, metadataValidators, bool) { } c.order.MoveToFront(el) e := el.Value.(*rewriteEntry) - return e.out, e.metadataValidators, true + return e.out, c.validators(e), true } // rewrite returns rewrite(in) and its validators, from the cache when it can. @@ -173,9 +189,9 @@ func (c *rewriteCache) rewrite(ctx context.Context, key string, in []byte, rewri c.mu.Lock() if el, ok := c.entries[key]; ok { c.order.MoveToFront(el) - e := el.Value.(*rewriteEntry) + v := c.validators(el.Value.(*rewriteEntry)) c.mu.Unlock() - return e.out, e.metadataValidators, nil + return el.Value.(*rewriteEntry).out, v, nil } if f, ok := c.inFlight[key]; ok { c.mu.Unlock() @@ -183,7 +199,7 @@ func (c *rewriteCache) rewrite(ctx context.Context, key string, in []byte, rewri case <-ctx.Done(): return nil, metadataValidators{}, ctx.Err() case <-f.done: - return f.out, f.v, f.err + return f.out, c.servedValidators(key, f.etag), f.err } } f := &inflightRewrite{done: make(chan struct{}), err: errSharedRewriteAbandoned} @@ -194,44 +210,95 @@ func (c *rewriteCache) rewrite(ctx context.Context, key string, in []byte, rewri c.mu.Lock() delete(c.inFlight, key) if f.err == nil { - f.v.lastModified = c.issueLastModified(key) - c.add(&rewriteEntry{key: key, out: f.out, metadataValidators: f.v}) + c.add(&rewriteEntry{key: key, out: f.out, etag: f.etag}) } c.mu.Unlock() close(f.done) - out, v, err = f.out, f.v, f.err + out, v, err = f.out, c.servedValidators(key, f.etag), f.err }() - f.out, f.v, f.err = rewriteWithETag(in, rewrite) - return f.out, f.v, f.err + var fv metadataValidators + f.out, fv, f.err = rewriteWithETag(in, rewrite) + f.etag = fv.etag + return f.out, fv, f.err } -// issueLastModified returns the Last-Modified of a rewrite that has just been -// made for key: the current second, or a second after the date last given to -// a rewrite of the same package when that is no earlier. Upstream's own date -// cannot serve, since the rewrite also depends on configuration read at -// startup, and two documents can share a second. A strictly later date for -// every new rewrite means a client holding one rewrite never gets 304 for -// another, even after its entry was evicted. Dates are remembered only until -// the clock passes them; after that the clock alone keeps them increasing. -// Called with mu held. -func (c *rewriteCache) issueLastModified(key string) time.Time { +// servedValidators returns the validators to serve the rewrite made for key +// with, if it was cached, and its bare ETag otherwise. +func (c *rewriteCache) servedValidators(key, etag string) metadataValidators { + c.mu.Lock() + defer c.mu.Unlock() + if el, ok := c.entries[key]; ok && el.Value.(*rewriteEntry).etag == etag { + return c.validators(el.Value.(*rewriteEntry)) + } + return metadataValidators{etag: etag} +} + +// validators returns the validators to serve cached rewrite e with, and +// records that its package is serving it. Called with mu held. +func (c *rewriteCache) validators(e *rewriteEntry) metadataValidators { + return metadataValidators{etag: e.etag, lastModified: c.lastModified(rewritePackage(e.key), e.etag)} +} + +// rewritePackage returns the part of a rewrite cache key that names the +// package, without the digest of the upstream document. +func rewritePackage(key string) string { + return key[:strings.LastIndexByte(key, 0)] +} + +// lastModified returns the Last-Modified to serve the rewrite of pkg with +// the given ETag with, or zero for none, and records that pkg is serving it. +// +// The date is the second pkg started serving the rewrite, which is later than +// any date given to a rewrite pkg served before, so a client holding one never +// gets 304 for another. Upstream's date cannot serve, since the rewrite also +// depends on configuration read at startup, and a rewrite cached earlier and +// served again keeps no date of its own. Dates are never ahead of the clock, +// so they hold across a restart. When pkg switches rewrites in a second that +// already dated one, the new one shares that second with a date a client may +// hold, so it goes undated until the next second. Called with mu held. +func (c *rewriteCache) lastModified(pkg, etag string) time.Time { now := c.now().Truncate(time.Second) - for pkg, issued := range c.issued { - if issued.Before(now) { - delete(c.issued, pkg) + cur, ok := c.serving[pkg] + switch { + case ok && cur.etag == etag: + if cur.since.IsZero() && now.After(cur.switched) { + cur.since = now } + case ok: + since := now + if !cur.switched.Before(now) || !cur.since.Before(now) { + since = time.Time{} + } + cur = servedRewrite{etag: etag, switched: now, since: since} + default: + since := now + if !now.After(c.forgotten) { + since = time.Time{} + } + cur = servedRewrite{etag: etag, switched: now, since: since} } - pkg := key[:strings.LastIndexByte(key, 0)] - lastModified := now - if issued, ok := c.issued[pkg]; ok && !issued.Before(now) { - lastModified = issued.Add(time.Second) + c.serving[pkg] = cur + return cur.since +} + +// forget drops what serving records for e's package when e is the rewrite +// the package is serving, so the map holds only packages with a cached +// rewrite. Called with mu held, as e is evicted. +func (c *rewriteCache) forget(e *rewriteEntry) { + pkg := rewritePackage(e.key) + cur, ok := c.serving[pkg] + if !ok || cur.etag != e.etag { + return + } + delete(c.serving, pkg) + now := c.now().Truncate(time.Second) + if !cur.switched.Before(now) || !cur.since.Before(now) { + c.forgotten = now } - c.issued[pkg] = lastModified - return lastModified } // rewriteWithETag runs rewrite and computes the ETag of what it returns. It -// gives no Last-Modified: nothing records when an uncached rewrite was made. +// gives no Last-Modified: nothing records when an uncached rewrite was served. func rewriteWithETag(in []byte, rewrite func([]byte) ([]byte, error)) ([]byte, metadataValidators, error) { out, err := rewrite(in) if err != nil { @@ -255,6 +322,7 @@ func (c *rewriteCache) add(e *rewriteEntry) { e := oldest.Value.(*rewriteEntry) c.order.Remove(oldest) delete(c.entries, e.key) + c.forget(e) c.size -= int64(len(e.out)) } } From 599b2e1473d7a43861a5b9563ad2ddd83f5cc83f Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Mon, 12 Oct 2026 12:52:02 +0200 Subject: [PATCH 221/633] Add a metadata_dates table for rewritten metadata dates Replicas sharing a database need one place to agree on the Last-Modified of rewritten metadata, which process memory cannot give them. - Add metadata_dates, keyed by ecosystem, proxy URL and package, holding the ETag being dated and its date in Unix seconds, so no zone is involved - Insert a package's first date only if no replica recorded one first - Swap a date only if it is still the one read, so of two replicas switching a package at once only one succeeds - Create it in both schemas and in migration 011 --- docs/architecture.md | 11 ++ internal/database/database_test.go | 2 +- internal/database/metadata_dates.go | 77 ++++++++++++++ internal/database/metadata_dates_test.go | 128 +++++++++++++++++++++++ internal/database/schema.go | 40 +++++++ 5 files changed, 257 insertions(+), 1 deletion(-) create mode 100644 internal/database/metadata_dates.go create mode 100644 internal/database/metadata_dates_test.go diff --git a/docs/architecture.md b/docs/architecture.md index 0d37a60..c96ae33 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -189,8 +189,19 @@ metadata_cache ( updated_at DATETIME ) -- indexes: (ecosystem, name) unique + +metadata_dates ( + ecosystem TEXT NOT NULL, + proxy_url TEXT NOT NULL, -- each proxy URL rewrites to its own download URLs + name TEXT NOT NULL, + etag TEXT NOT NULL, -- the rewritten representation being dated + last_modified INTEGER NOT NULL, -- Unix seconds; BIGINT on Postgres + PRIMARY KEY (ecosystem, proxy_url, name) +) ``` +`metadata_dates` holds the `Last-Modified` served with rewritten npm and Composer metadata. A package's date is the second it started serving its current representation, and a new representation is only dated in a later second. Keeping it in the database means replicas sharing one serve the same date, and never give one representation's date to another, so `If-Modified-Since` cannot return 304 for a body the client does not have. + On PostgreSQL, `INTEGER PRIMARY KEY` becomes `SERIAL`, `DATETIME` becomes `TIMESTAMP`, `INTEGER DEFAULT 0` booleans become `BOOLEAN DEFAULT FALSE`, and size/count columns use `BIGINT`. The `MigrateSchema()` function handles backward compatibility with older git-pkgs databases by running named migrations that add missing columns and tables. See [migrations.md](migrations.md) for how to add new schema changes. diff --git a/internal/database/database_test.go b/internal/database/database_test.go index 13f6828..f09a86a 100644 --- a/internal/database/database_test.go +++ b/internal/database/database_test.go @@ -613,7 +613,7 @@ func createTestPostgresDB(t *testing.T) *DB { // Drop and recreate every table CreateSchema creates for clean test state; // leftover migration records make the next CreateSchema fail on the // migrations primary key. - tables := []string{"artifacts", "pending_deletes", "versions", "packages", "vulnerabilities", "metadata_cache", "migrations", "schema_info"} + tables := []string{"artifacts", "pending_deletes", "versions", "packages", "vulnerabilities", "metadata_cache", "metadata_dates", "migrations", "schema_info"} for _, table := range tables { _, _ = db.Exec("DROP TABLE IF EXISTS " + table + " CASCADE") } diff --git a/internal/database/metadata_dates.go b/internal/database/metadata_dates.go new file mode 100644 index 0000000..f705f29 --- /dev/null +++ b/internal/database/metadata_dates.go @@ -0,0 +1,77 @@ +package database + +import ( + "database/sql" + "errors" + "fmt" + "time" +) + +// MetadataDate is the Last-Modified a package's rewritten metadata is served +// with, and the ETag of the rewrite it dates. +// +// Rewritten metadata has no date of its own, so the proxy assigns one when a +// package starts serving a new rewrite. Keeping it in the database rather than +// in process memory is what lets replicas sharing the database agree on it: a +// date one replica gave out is never given by another to a different rewrite. +type MetadataDate struct { + ETag string + LastModified time.Time +} + +type metadataDateRow struct { + ETag string `db:"etag"` + LastModified int64 `db:"last_modified"` +} + +// GetMetadataDate returns the date recorded for a package's metadata as +// rewritten for proxyURL, or nil when there is none. +func (db *DB) GetMetadataDate(ecosystem, proxyURL, name string) (*MetadataDate, error) { + var row metadataDateRow + err := db.Get(&row, db.Rebind(` + SELECT etag, last_modified FROM metadata_dates + WHERE ecosystem = ? AND proxy_url = ? AND name = ? + `), ecosystem, proxyURL, name) + if errors.Is(err, sql.ErrNoRows) { + return nil, nil + } + if err != nil { + return nil, fmt.Errorf("getting metadata date: %w", err) + } + return &MetadataDate{ETag: row.ETag, LastModified: time.Unix(row.LastModified, 0).UTC()}, nil +} + +// InsertMetadataDate records the first date of a package's metadata and +// reports whether it did; it leaves a date another replica recorded first. +func (db *DB) InsertMetadataDate(ecosystem, proxyURL, name string, date MetadataDate) (bool, error) { + result, err := db.Exec(db.Rebind(` + INSERT INTO metadata_dates (ecosystem, proxy_url, name, etag, last_modified) + VALUES (?, ?, ?, ?, ?) + ON CONFLICT (ecosystem, proxy_url, name) DO NOTHING + `), ecosystem, proxyURL, name, date.ETag, date.LastModified.Unix()) + if err != nil { + return false, fmt.Errorf("inserting metadata date: %w", err) + } + return affectedOne(result) +} + +// SwapMetadataDate replaces a package's date with next if it is still old, +// and reports whether it did; it leaves a date another replica swapped in. +func (db *DB) SwapMetadataDate(ecosystem, proxyURL, name string, old, next MetadataDate) (bool, error) { + result, err := db.Exec(db.Rebind(` + UPDATE metadata_dates SET etag = ?, last_modified = ? + WHERE ecosystem = ? AND proxy_url = ? AND name = ? AND etag = ? AND last_modified = ? + `), next.ETag, next.LastModified.Unix(), ecosystem, proxyURL, name, old.ETag, old.LastModified.Unix()) + if err != nil { + return false, fmt.Errorf("swapping metadata date: %w", err) + } + return affectedOne(result) +} + +func affectedOne(result sql.Result) (bool, error) { + n, err := result.RowsAffected() + if err != nil { + return false, fmt.Errorf("counting affected rows: %w", err) + } + return n == 1, nil +} diff --git a/internal/database/metadata_dates_test.go b/internal/database/metadata_dates_test.go new file mode 100644 index 0000000..864723e --- /dev/null +++ b/internal/database/metadata_dates_test.go @@ -0,0 +1,128 @@ +package database + +import ( + "path/filepath" + "testing" + "time" +) + +const testProxyURL = "http://proxy.example" + +func TestMetadataDateMissing(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + got, err := db.GetMetadataDate(testEcosystemNPM, testProxyURL, "missing") + if err != nil || got != nil { + t.Errorf("GetMetadataDate() = %+v, %v; want nil, nil", got, err) + } + }) +} + +func TestMetadataDateInsertOnce(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + first := MetadataDate{ETag: `"a"`, LastModified: time.Date(2026, 10, 10, 12, 0, 1, 0, time.UTC)} + second := MetadataDate{ETag: `"b"`, LastModified: first.LastModified.Add(time.Second)} + + inserted, err := db.InsertMetadataDate(testEcosystemNPM, testProxyURL, "left-pad", first) + if err != nil || !inserted { + t.Fatalf("first InsertMetadataDate() = %v, %v; want true", inserted, err) + } + inserted, err = db.InsertMetadataDate(testEcosystemNPM, testProxyURL, "left-pad", second) + if err != nil || inserted { + t.Fatalf("second InsertMetadataDate() = %v, %v; want false, the row exists", inserted, err) + } + + got, err := db.GetMetadataDate(testEcosystemNPM, testProxyURL, "left-pad") + if err != nil || got == nil || *got != first { + t.Errorf("GetMetadataDate() = %+v, %v; want %+v", got, err, first) + } + }) +} + +// A package's date is kept per proxy URL: each one rewrites the document to +// different download URLs, so each serves a different representation. +func TestMetadataDateKeyedByProxyURL(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + date := MetadataDate{ETag: `"a"`, LastModified: time.Date(2026, 10, 10, 12, 0, 1, 0, time.UTC)} + if _, err := db.InsertMetadataDate(testEcosystemNPM, testProxyURL, "left-pad", date); err != nil { + t.Fatal(err) + } + inserted, err := db.InsertMetadataDate(testEcosystemNPM, "http://other.example", "left-pad", date) + if err != nil || !inserted { + t.Errorf("InsertMetadataDate() for another proxy URL = %v, %v; want true", inserted, err) + } + }) +} + +// SwapMetadataDate is a compare-and-swap: it only replaces the date it was +// given, so of two replicas swapping from the same date only one succeeds. +func TestMetadataDateSwap(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + a := MetadataDate{ETag: `"a"`, LastModified: time.Date(2026, 10, 10, 12, 0, 1, 0, time.UTC)} + b := MetadataDate{ETag: `"b"`, LastModified: a.LastModified.Add(time.Second)} + c := MetadataDate{ETag: `"c"`, LastModified: b.LastModified.Add(time.Second)} + if _, err := db.InsertMetadataDate(testEcosystemNPM, testProxyURL, "left-pad", a); err != nil { + t.Fatal(err) + } + + swapped, err := db.SwapMetadataDate(testEcosystemNPM, testProxyURL, "left-pad", a, b) + if err != nil || !swapped { + t.Fatalf("SwapMetadataDate(a, b) = %v, %v; want true", swapped, err) + } + swapped, err = db.SwapMetadataDate(testEcosystemNPM, testProxyURL, "left-pad", a, c) + if err != nil || swapped { + t.Fatalf("SwapMetadataDate(a, c) after a was replaced = %v, %v; want false", swapped, err) + } + + got, err := db.GetMetadataDate(testEcosystemNPM, testProxyURL, "left-pad") + if err != nil || got == nil || *got != b { + t.Errorf("GetMetadataDate() = %+v, %v; want %+v", got, err, b) + } + }) +} + +// Dates are whole seconds in UTC whatever the local zone, so every replica +// reads back the date another one wrote. +func TestMetadataDateIsUTCSeconds(t *testing.T) { + tokyo, err := time.LoadLocation("Asia/Tokyo") + if err != nil { + t.Skipf("time zone data unavailable: %v", err) + } + prev := time.Local + time.Local = tokyo + t.Cleanup(func() { time.Local = prev }) + + runWithBothDatabases(t, func(t *testing.T, db *DB) { + written := time.Date(2026, 10, 10, 21, 0, 1, 700_000_000, tokyo) + if _, err := db.InsertMetadataDate(testEcosystemNPM, testProxyURL, "tz", MetadataDate{ETag: `"a"`, LastModified: written}); err != nil { + t.Fatal(err) + } + got, err := db.GetMetadataDate(testEcosystemNPM, testProxyURL, "tz") + if err != nil || got == nil { + t.Fatalf("GetMetadataDate() = %+v, %v", got, err) + } + if want := time.Date(2026, 10, 10, 12, 0, 1, 0, time.UTC); !got.LastModified.Equal(want) || got.LastModified.Location() != time.UTC { + t.Errorf("LastModified read back as %v, want %v", got.LastModified, want) + } + }) +} + +func TestMetadataDatesTableCreatedByMigration(t *testing.T) { + db, err := Create(filepath.Join(t.TempDir(), "test.db")) + if err != nil { + t.Fatalf("Create failed: %v", err) + } + defer func() { _ = db.Close() }() + if _, err := db.Exec("DROP TABLE metadata_dates"); err != nil { + t.Fatal(err) + } + if _, err := db.Exec("DELETE FROM migrations WHERE name = '011_add_metadata_dates'"); err != nil { + t.Fatal(err) + } + + if err := db.MigrateSchema(); err != nil { + t.Fatalf("MigrateSchema() error = %v", err) + } + if has, err := db.HasTable("metadata_dates"); err != nil || !has { + t.Errorf("HasTable(metadata_dates) = %v, %v; want true after migration", has, err) + } +} diff --git a/internal/database/schema.go b/internal/database/schema.go index 4aef75c..b8d3f93 100644 --- a/internal/database/schema.go +++ b/internal/database/schema.go @@ -119,6 +119,15 @@ CREATE TABLE IF NOT EXISTS metadata_cache ( ); CREATE UNIQUE INDEX IF NOT EXISTS idx_metadata_eco_name ON metadata_cache(ecosystem, name); +CREATE TABLE IF NOT EXISTS metadata_dates ( + ecosystem TEXT NOT NULL, + proxy_url TEXT NOT NULL, + name TEXT NOT NULL, + etag TEXT NOT NULL, + last_modified INTEGER NOT NULL, + PRIMARY KEY (ecosystem, proxy_url, name) +); + CREATE TABLE IF NOT EXISTS migrations ( name TEXT NOT NULL PRIMARY KEY, applied_at DATETIME NOT NULL @@ -228,6 +237,15 @@ CREATE TABLE IF NOT EXISTS metadata_cache ( ); CREATE UNIQUE INDEX IF NOT EXISTS idx_metadata_eco_name ON metadata_cache(ecosystem, name); +CREATE TABLE IF NOT EXISTS metadata_dates ( + ecosystem TEXT NOT NULL, + proxy_url TEXT NOT NULL, + name TEXT NOT NULL, + etag TEXT NOT NULL, + last_modified BIGINT NOT NULL, + PRIMARY KEY (ecosystem, proxy_url, name) +); + CREATE TABLE IF NOT EXISTS migrations ( name TEXT NOT NULL PRIMARY KEY, applied_at TIMESTAMP NOT NULL @@ -382,6 +400,7 @@ var migrations = []migration{ {"008_add_metadata_content_encoding", migrateAddMetadataContentEncoding}, {"009_add_pending_deletes", migrateAddPendingDeletes}, {"010_add_pending_deletes_queued_at_index", migrateAddPendingDeletesQueuedAtIndex}, + {"011_add_metadata_dates", migrateAddMetadataDates}, } // isTableNotFound returns true if the error indicates a missing table. @@ -681,6 +700,27 @@ func migrateAddPendingDeletesQueuedAtIndex(db *DB) error { return nil } +// migrateAddMetadataDates creates the table of Last-Modified dates for +// rewritten metadata, which replicas sharing the database agree on. +func migrateAddMetadataDates(db *DB) error { + seconds := "INTEGER" + if db.dialect == DialectPostgres { + seconds = "BIGINT" + } + query := fmt.Sprintf(`CREATE TABLE IF NOT EXISTS metadata_dates ( + ecosystem TEXT NOT NULL, + proxy_url TEXT NOT NULL, + name TEXT NOT NULL, + etag TEXT NOT NULL, + last_modified %s NOT NULL, + PRIMARY KEY (ecosystem, proxy_url, name) + )`, seconds) + if _, err := db.Exec(query); err != nil { + return fmt.Errorf("creating metadata_dates table: %w", err) + } + return nil +} + // EnsureMetadataCacheTable creates the metadata_cache table if it doesn't exist. func (db *DB) EnsureMetadataCacheTable() error { has, err := db.HasTable("metadata_cache") From 054c45c5a76be02c5f9099344ecb8da967ba5b76 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Mon, 12 Oct 2026 12:52:02 +0200 Subject: [PATCH 222/633] Share the dates of rewritten metadata across replicas Dates were kept in each process's rewrite cache, so replicas sharing a database and storage dated independently: one dated a document, upstream changed in that second, and another gave the new document the same date, so a client holding it got 304 and kept the old document. - Date from metadata_dates instead of process memory: a package's date is the second it started serving its current representation, recorded once for every replica - Switch a package to another representation only in a second later than its recorded date, and serve it undated until then - Date nothing in the second the process started, so dates clients hold from before this change, or from the process it replaced, stay older - Serve no date ahead of the local clock, recorded by a replica whose clock is ahead - Date rewrites without the rewrite cache too; the cache keeps only ETags - Add two-replica, pre-fix date and clock skew regression tests --- internal/handler/handler.go | 7 + internal/handler/metadata_dates.go | 83 ++++++ internal/handler/metadata_validators_test.go | 275 ++++++++++++------- internal/handler/rewrite_cache.go | 136 ++------- 4 files changed, 291 insertions(+), 210 deletions(-) create mode 100644 internal/handler/metadata_dates.go diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 56d4cdd..e867961 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -206,6 +206,11 @@ type Proxy struct { // rewrites caches metadata documents after their handler rewrites them. // Nil leaves every request to rewrite its own copy. rewrites *rewriteCache + + // now is the clock rewritten metadata is dated by, and started the time + // the process started; see metadataDate. + now func() time.Time + started time.Time } // CooldownPolicy supplies version-age filtering and package-specific durations. @@ -230,6 +235,8 @@ func NewProxy(db *database.DB, store storage.Storage, fetcher fetch.FetcherInter HTTPClient: &http.Client{ Timeout: defaultHTTPTimeout, }, + now: time.Now, + started: time.Now(), } } diff --git a/internal/handler/metadata_dates.go b/internal/handler/metadata_dates.go new file mode 100644 index 0000000..fca5a1f --- /dev/null +++ b/internal/handler/metadata_dates.go @@ -0,0 +1,83 @@ +package handler + +import ( + "time" + + "github.com/git-pkgs/proxy/internal/database" +) + +// metadataDateAttempts bounds how often metadataDate retries when other +// replicas keep changing a package's date between its read and its write. +const metadataDateAttempts = 3 + +// metadataDate returns the Last-Modified to serve a package's metadata with, +// as rewritten for proxyURL into the representation with the given ETag, or +// zero for none. +// +// The date is the second the package started serving that representation, +// recorded in the database so that every replica sharing it serves the same +// date. A switch to another representation is dated only in a second later +// than the recorded date, and then with that second, so a date is never given +// to two representations of a package: a client holding one never gets 304 +// for another, whichever replica it asks. Until such a second the +// representation goes undated, and If-Modified-Since is not evaluated. +// +// A package's first date is likewise later than the second this process +// started, as the process it replaced may have dated the package then without +// recording it. A date ahead of this process's clock, recorded by a replica +// whose clock is ahead, is not served until the clock catches up. Without a +// database, or when it fails, nothing is dated. +func (p *Proxy) metadataDate(ecosystem, proxyURL, name, etag string) time.Time { + if p.DB == nil || etag == "" { + return time.Time{} + } + now := p.clock().Truncate(time.Second) + for range metadataDateAttempts { + date, settled, err := p.claimMetadataDate(ecosystem, proxyURL, name, etag, now) + if err != nil { + p.Logger.Warn("failed to date rewritten metadata", "ecosystem", ecosystem, "name", name, "error", err) + return time.Time{} + } + if settled { + if date.After(now) { + return time.Time{} + } + return date + } + } + return time.Time{} +} + +// claimMetadataDate makes one attempt at recording the date of etag for a +// package at now. It reports false when another replica changed the date +// first, and the attempt should be made again. +func (p *Proxy) claimMetadataDate(ecosystem, proxyURL, name, etag string, now time.Time) (time.Time, bool, error) { + current, err := p.DB.GetMetadataDate(ecosystem, proxyURL, name) + if err != nil { + return time.Time{}, false, err + } + next := database.MetadataDate{ETag: etag, LastModified: now} + switch { + case current == nil: + if !now.After(p.started.Truncate(time.Second)) { + return time.Time{}, true, nil + } + inserted, err := p.DB.InsertMetadataDate(ecosystem, proxyURL, name, next) + return now, inserted, err + case current.ETag == etag: + return current.LastModified, true, nil + case !now.After(current.LastModified): + return time.Time{}, true, nil + default: + swapped, err := p.DB.SwapMetadataDate(ecosystem, proxyURL, name, *current, next) + return now, swapped, err + } +} + +// clock reads the time metadata is dated by. +func (p *Proxy) clock() time.Time { + if p.now == nil { + return time.Now() + } + return p.now() +} diff --git a/internal/handler/metadata_validators_test.go b/internal/handler/metadata_validators_test.go index 8e9f90b..a057e5e 100644 --- a/internal/handler/metadata_validators_test.go +++ b/internal/handler/metadata_validators_test.go @@ -80,8 +80,8 @@ func validatorProxy(t *testing.T, u *validatorUpstream, ttl time.Duration) *Prox return proxy } -// validatorProxyWithDB returns a proxy whose rewrite cache runs on a clock -// the test moves, starting a second after the process start it models. +// validatorProxyWithDB returns a proxy that dates metadata by a clock the +// test moves, starting a second after the process start it models. func validatorProxyWithDB(t *testing.T, u *validatorUpstream, ttl time.Duration) (*Proxy, *database.DB, *testClock) { t.Helper() proxy, db, _, _ := setupTestProxy(t) @@ -89,22 +89,32 @@ func validatorProxyWithDB(t *testing.T, u *validatorUpstream, ttl time.Duration) proxy.CacheMetadata = true proxy.MetadataTTL = ttl clock := &testClock{t: time.Date(2026, 10, 10, 12, 0, 0, 0, time.UTC)} - restartRewriteCache(proxy, clock) + restartProxy(proxy, clock) clock.advance(time.Second) return proxy, db, clock } -// restartRewriteCache gives proxy a new, empty rewrite cache started at the -// clock's current time, as a restart of the process would. -func restartRewriteCache(proxy *Proxy, clock *testClock) { +// restartProxy gives proxy a new, empty rewrite cache and makes it date +// metadata by clock from the clock's current time on, as a restart of the +// process would. What the database holds survives. +func restartProxy(proxy *Proxy, clock *testClock) { proxy.SetMetadataRewriteCacheSize(1 << 20) - proxy.rewrites.useClock(clock.now) + proxy.now = clock.now + proxy.started = clock.now() } -// useClock makes the cache read time from now, as if it was made at now(). -func (c *rewriteCache) useClock(now func() time.Time) { - c.now = now - c.forgotten = now().Truncate(time.Second) +// validatorReplica returns a second proxy process sharing proxy's database +// and storage, as another replica behind the same load balancer would, with +// its own rewrite cache and its own clock. +func validatorReplica(proxy *Proxy, clock *testClock) *Proxy { + replica := NewProxy(proxy.DB, proxy.Storage, proxy.Fetcher, proxy.Resolver, proxy.Logger) + replica.HTTPClient = proxy.HTTPClient + replica.CacheMetadata = proxy.CacheMetadata + replica.MetadataTTL = proxy.MetadataTTL + replica.SetMetadataRewriteCacheSize(1 << 20) + replica.now = clock.now + replica.started = proxy.started + return replica } type testClock struct { @@ -399,7 +409,7 @@ func TestRewrittenMetadataLastModifiedAcrossRestart(t *testing.T) { t.Errorf("Last-Modified %q is ahead of the clock at %v", latest, clock.now()) } - restartRewriteCache(proxy, clock) + restartProxy(proxy, clock) u.set(npmV("3.0.0"), composerV("3.0.0")) rec := serve(handler, c.path, http.Header{"If-Modified-Since": {latest}}) if rec.Code != http.StatusOK || !strings.Contains(rec.Body.String(), "3.0.0") { @@ -486,94 +496,169 @@ func TestRewriteCacheETag(t *testing.T) { } } -// TestRewriteCacheLastModified checks the Last-Modified the rewrite cache -// serves each rewrite with: the second the package started serving it, so a -// client holding one representation never gets 304 for another. A date is -// never ahead of the clock, so none is given in a second that may already -// have dated another representation of the package. -func TestRewriteCacheLastModified(t *testing.T) { - clock := &testClock{t: time.Date(2026, 10, 10, 12, 0, 0, 400_000_000, time.UTC)} - c := newRewriteCache(8) - c.useClock(clock.now) - second := func() time.Time { return clock.now().Truncate(time.Second) } - identity := func(b []byte) ([]byte, error) { return b, nil } - key := func(name, doc string) string { return rewriteCacheKey("npm", "http://proxy", name, []byte(doc)) } - rewrite := func(name, doc string) time.Time { - t.Helper() - _, v, err := c.rewrite(t.Context(), key(name, doc), []byte(doc), identity) - if err != nil { - t.Fatal(err) - } - return v.lastModified - } - expect := func(what string, got, want time.Time) { - t.Helper() - if !got.Equal(want) { - t.Errorf("%s: Last-Modified %v, want %v", what, got, want) - } - } - var undated time.Time - - expect("in the second the cache started", rewrite("demo", "a"), undated) - clock.advance(time.Second) - a := second() - expect("a second later", rewrite("demo", "a"), a) - clock.advance(300 * time.Millisecond) - if _, v, ok := c.get(key("demo", "a")); !ok || !v.lastModified.Equal(a) { - t.Errorf("get: Last-Modified %v, %v; want %v", v.lastModified, ok, a) - } - - // A new document in the second a was dated gets no date, and neither - // does a, back in the same second, until a later one. - expect("new document in the same second", rewrite("demo", "b"), undated) - expect("a back in the same second", rewrite("demo", "a"), undated) - // Another package is not held back by it. - expect("other package", rewrite("other", "x"), a) - clock.advance(time.Second) - expect("a in the next second", rewrite("demo", "a"), second()) - - // A cached document coming back is dated when it comes back. - clock.advance(10 * time.Second) - b := second() - expect("b", rewrite("demo", "b"), b) - clock.advance(10 * time.Second) - expect("a after b", rewrite("demo", "a"), second()) - - // Evicting the package's current document forgets the package. Its next - // document can be dated at once unless the forgotten one was dated in - // this second. - clock.advance(10 * time.Second) - for _, doc := range []string{"ccc", "ddd", "eee"} { - rewrite("filler", doc) - } - if _, _, ok := c.get(key("demo", "a")); ok { - t.Fatal("expected a to be evicted") - } - if _, ok := c.serving[rewritePackage(key("demo", "a"))]; ok { - t.Error("the package is still recorded after its rewrite was evicted") - } - expect("after a clean eviction", rewrite("demo", "f"), second()) - for _, doc := range []string{"ggg", "hhh", "iii"} { - rewrite("filler", doc) - } - expect("after an eviction in the second f was dated", rewrite("demo", "j"), undated) -} - -// TestRewrittenMetadataLastModifiedNeedsRewriteCache checks that without the -// rewrite cache there is no Last-Modified: nothing records when a rewrite was -// first made, so no date can be relied on to change with it. -func TestRewrittenMetadataLastModifiedNeedsRewriteCache(t *testing.T) { +// TestRewrittenMetadataLastModifiedWithoutRewriteCache checks that dates do +// not depend on the rewrite cache: the database records them. +func TestRewrittenMetadataLastModifiedWithoutRewriteCache(t *testing.T) { for _, c := range validatorCases { t.Run(c.ecosystem, func(t *testing.T) { u := newValidatorUpstream(t) - proxy := validatorProxy(t, u, time.Hour) + proxy, _, clock := validatorProxyWithDB(t, u, time.Hour) proxy.SetMetadataRewriteCacheSize(0) - rec := serve(c.handler(proxy, u), c.path, nil) - if rec.Code != http.StatusOK || rec.Header().Get("ETag") == "" { - t.Fatalf("status %d, ETag %q; want 200 with an ETag", rec.Code, rec.Header().Get("ETag")) + handler := c.handler(proxy, u) + + first := serve(handler, c.path, nil) + if first.Code != http.StatusOK || first.Header().Get("ETag") == "" { + t.Fatalf("status %d, ETag %q; want 200 with an ETag", first.Code, first.Header().Get("ETag")) } - if lm := rec.Header().Get("Last-Modified"); lm != "" { - t.Errorf("Last-Modified = %q without the rewrite cache, want none", lm) + lastModified, err := http.ParseTime(first.Header().Get("Last-Modified")) + if want := clock.now().Truncate(time.Second); err != nil || !lastModified.Equal(want) { + t.Errorf("Last-Modified %q without the rewrite cache, want %v", first.Header().Get("Last-Modified"), want) + } + clock.advance(time.Minute) + if rec := serve(handler, c.path, http.Header{"If-Modified-Since": {first.Header().Get("Last-Modified")}}); rec.Code != http.StatusNotModified { + t.Errorf("If-Modified-Since Last-Modified: status %d, want 304", rec.Code) + } + }) + } +} + +// TestRewrittenMetadataLastModifiedAcrossReplicas is a regression test for a +// false 304 between replicas sharing a database and storage. One replica +// dates a document, upstream changes in that second, and a client holding +// the date asks another replica: the date must not suppress the new body. +// Once a second has passed, both replicas date the new document alike. +func TestRewrittenMetadataLastModifiedAcrossReplicas(t *testing.T) { + for _, c := range validatorCases { + t.Run(c.ecosystem, func(t *testing.T) { + u := newValidatorUpstream(t) + a, _, clock := validatorProxyWithDB(t, u, 0) + b := validatorReplica(a, clock) + replicas := map[string]http.Handler{"A": c.handler(a, u), "B": c.handler(b, u)} + + first := serve(replicas["A"], c.path, nil) + held := first.Header().Get("Last-Modified") + if first.Code != http.StatusOK || held == "" { + t.Fatalf("replica A: status %d, Last-Modified %q; want 200 with a date", first.Code, held) + } + u.set(npmV("2.0.0"), composerV("2.0.0")) + for _, name := range []string{"B", "A"} { + expectChangedUndated(t, name, serve(replicas[name], c.path, http.Header{"If-Modified-Since": {held}}), first) + } + + clock.advance(time.Second) + shared := expectSharedDate(t, replicas, c.path, held) + for name, handler := range replicas { + if rec := serve(handler, c.path, http.Header{"If-Modified-Since": {shared}}); rec.Code != http.StatusNotModified { + t.Errorf("replica %s, If-Modified-Since the shared date: status %d, want 304", name, rec.Code) + } + if rec := serve(handler, c.path, http.Header{"If-Modified-Since": {held}}); rec.Code != http.StatusOK { + t.Errorf("replica %s, If-Modified-Since the old date: status %d, want 200", name, rec.Code) + } + } + }) + } +} + +// expectChangedUndated checks that a replica answered a conditional request +// with a body and ETag other than first's, and no date: the second first was +// dated in may not date another representation. +func expectChangedUndated(t *testing.T, replica string, rec *httptest.ResponseRecorder, first *httptest.ResponseRecorder) { + t.Helper() + if rec.Code != http.StatusOK || rec.Body.String() == first.Body.String() { + t.Fatalf("replica %s after upstream changed: status %d; want 200 with the new body", replica, rec.Code) + } + if rec.Header().Get("ETag") == first.Header().Get("ETag") { + t.Errorf("replica %s: the new body has the old ETag", replica) + } + if lm := rec.Header().Get("Last-Modified"); lm != "" { + t.Errorf("replica %s: Last-Modified %q in the second the old body was dated, want none", replica, lm) + } +} + +// expectSharedDate checks that every replica serves the same Last-Modified, +// later than held, and returns it. +func expectSharedDate(t *testing.T, replicas map[string]http.Handler, path, held string) string { + t.Helper() + was, _ := http.ParseTime(held) + shared := "" + for name, handler := range replicas { + lm := serve(handler, path, nil).Header().Get("Last-Modified") + if now, err := http.ParseTime(lm); err != nil || !now.After(was) { + t.Fatalf("replica %s: Last-Modified %q, want later than %q", name, lm, held) + } + if shared != "" && lm != shared { + t.Fatalf("replica %s: Last-Modified %q, another replica serves %q", name, lm, shared) + } + shared = lm + } + return shared +} + +// TestRewrittenMetadataLastModifiedFromBeforeTheFix checks that dates clients +// hold from before dates were shared cannot suppress a body. Those were never +// later than the clock of the process that gave them, which stopped before +// this one started, so no date is given in the second this process started +// and every later one is later than any of them. +func TestRewrittenMetadataLastModifiedFromBeforeTheFix(t *testing.T) { + for _, c := range validatorCases { + t.Run(c.ecosystem, func(t *testing.T) { + u := newValidatorUpstream(t) + a, _, clock := validatorProxyWithDB(t, u, 0) + restartProxy(a, clock) + b := validatorReplica(a, clock) + held := clock.now().Truncate(time.Second).Format(http.TimeFormat) + + for name, handler := range map[string]http.Handler{"A": c.handler(a, u), "B": c.handler(b, u)} { + rec := serve(handler, c.path, http.Header{"If-Modified-Since": {held}}) + if rec.Code != http.StatusOK || rec.Header().Get("Last-Modified") != "" { + t.Errorf("replica %s in its first second, If-Modified-Since %q: status %d, Last-Modified %q; want 200 without a date", + name, held, rec.Code, rec.Header().Get("Last-Modified")) + } + } + + clock.advance(time.Second) + for name, handler := range map[string]http.Handler{"A": c.handler(a, u), "B": c.handler(b, u)} { + rec := serve(handler, c.path, http.Header{"If-Modified-Since": {held}}) + was, _ := http.ParseTime(held) + now, err := http.ParseTime(rec.Header().Get("Last-Modified")) + if rec.Code != http.StatusOK || err != nil || !now.After(was) { + t.Errorf("replica %s a second later, If-Modified-Since %q: status %d, Last-Modified %q; want 200 with a later date", + name, held, rec.Code, rec.Header().Get("Last-Modified")) + } + } + }) + } +} + +// TestRewrittenMetadataLastModifiedWithClockSkew checks that a replica whose +// clock is behind gives no date ahead of its own clock, neither for a new +// document nor for one another replica dated. +func TestRewrittenMetadataLastModifiedWithClockSkew(t *testing.T) { + for _, c := range validatorCases { + t.Run(c.ecosystem, func(t *testing.T) { + u := newValidatorUpstream(t) + a, _, clock := validatorProxyWithDB(t, u, 0) + behind := &testClock{t: clock.now().Add(-2 * time.Second)} + b := validatorReplica(a, behind) + handlerA, handlerB := c.handler(a, u), c.handler(b, u) + + held := serve(handlerA, c.path, nil).Header().Get("Last-Modified") + u.set(npmV("2.0.0"), composerV("2.0.0")) + if rec := serve(handlerB, c.path, http.Header{"If-Modified-Since": {held}}); rec.Code != http.StatusOK || rec.Header().Get("Last-Modified") != "" { + t.Fatalf("lagging replica, new document: status %d, Last-Modified %q; want 200 without a date", rec.Code, rec.Header().Get("Last-Modified")) + } + + clock.advance(time.Second) + dated := serve(handlerA, c.path, nil).Header().Get("Last-Modified") + if dated == "" { + t.Fatal("replica A gave no date a second later") + } + if lm := serve(handlerB, c.path, nil).Header().Get("Last-Modified"); lm != "" { + t.Errorf("lagging replica: Last-Modified %q ahead of its clock at %v, want none", lm, behind.now()) + } + behind.advance(3 * time.Second) + if lm := serve(handlerB, c.path, nil).Header().Get("Last-Modified"); lm != dated { + t.Errorf("lagging replica once its clock caught up: Last-Modified %q, want %q", lm, dated) } }) } diff --git a/internal/handler/rewrite_cache.go b/internal/handler/rewrite_cache.go index 540a13f..7348bbe 100644 --- a/internal/handler/rewrite_cache.go +++ b/internal/handler/rewrite_cache.go @@ -33,34 +33,17 @@ import ( // the key, and the denylist is loaded at startup. // // Each entry keeps the ETag of its output, so clients can revalidate their -// copy without the output being hashed again on every request. The cache also -// dates what it serves for each package; see lastModified. +// copy without the output being hashed again on every request. Dates are not +// the cache's to give: they must agree across replicas, so the database holds +// them (see metadataDate). type rewriteCache struct { maxBytes int64 - now func() time.Time mu sync.Mutex size int64 order *list.List // most recently used at the front entries map[string]*list.Element inFlight map[string]*inflightRewrite - // serving holds the rewrite each package is serving, keyed by its cache - // key without the digest, while that rewrite is cached. - serving map[string]servedRewrite - // forgotten is the latest second in which the cache may have dated a - // rewrite of a package it no longer has in serving: the second it started, - // as the previous process may have dated one then, or a later one. - forgotten time.Time -} - -// servedRewrite is the rewrite a package is serving and since when. -type servedRewrite struct { - etag string - // switched is the second the package started serving it. - switched time.Time - // since is its Last-Modified: the first second, from switched on, that - // can have dated no other rewrite of the package. Zero until there is one. - since time.Time } type rewriteEntry struct { @@ -136,13 +119,10 @@ func newRewriteCache(maxBytes int64) *rewriteCache { return nil } return &rewriteCache{ - maxBytes: maxBytes, - now: time.Now, - order: list.New(), - entries: make(map[string]*list.Element), - inFlight: make(map[string]*inflightRewrite), - serving: make(map[string]servedRewrite), - forgotten: time.Now().Truncate(time.Second), + maxBytes: maxBytes, + order: list.New(), + entries: make(map[string]*list.Element), + inFlight: make(map[string]*inflightRewrite), } } @@ -154,13 +134,12 @@ func rewriteCacheKey(ecosystem, proxyURL, name string, in []byte) string { } // rewriteCacheKeyForDigest is rewriteCacheKey for upstream bytes known only by -// their digest, in the form the metadata cache records it. The digest comes -// last, so the key without it names the package the rewrite is for. +// their digest, in the form the metadata cache records it. func rewriteCacheKeyForDigest(ecosystem, proxyURL, name, digest string) string { return strings.Join([]string{ecosystem, proxyURL, name, digest}, "\x00") } -// get returns the cached rewrite for key and its validators, if there is one. +// get returns the cached rewrite for key and its ETag, if there is one. func (c *rewriteCache) get(key string) ([]byte, metadataValidators, bool) { if c == nil { return nil, metadataValidators{}, false @@ -173,10 +152,10 @@ func (c *rewriteCache) get(key string) ([]byte, metadataValidators, bool) { } c.order.MoveToFront(el) e := el.Value.(*rewriteEntry) - return e.out, c.validators(e), true + return e.out, metadataValidators{etag: e.etag}, true } -// rewrite returns rewrite(in) and its validators, from the cache when it can. +// rewrite returns rewrite(in) and its ETag, from the cache when it can. // Callers must treat the returned bytes as read-only: a cached result is // shared. A caller waiting on another's rewrite leaves when ctx ends; the // rewrite itself always runs to completion, so the result is cached for the @@ -189,9 +168,9 @@ func (c *rewriteCache) rewrite(ctx context.Context, key string, in []byte, rewri c.mu.Lock() if el, ok := c.entries[key]; ok { c.order.MoveToFront(el) - v := c.validators(el.Value.(*rewriteEntry)) + e := el.Value.(*rewriteEntry) c.mu.Unlock() - return el.Value.(*rewriteEntry).out, v, nil + return e.out, metadataValidators{etag: e.etag}, nil } if f, ok := c.inFlight[key]; ok { c.mu.Unlock() @@ -199,7 +178,7 @@ func (c *rewriteCache) rewrite(ctx context.Context, key string, in []byte, rewri case <-ctx.Done(): return nil, metadataValidators{}, ctx.Err() case <-f.done: - return f.out, c.servedValidators(key, f.etag), f.err + return f.out, metadataValidators{etag: f.etag}, f.err } } f := &inflightRewrite{done: make(chan struct{}), err: errSharedRewriteAbandoned} @@ -214,7 +193,6 @@ func (c *rewriteCache) rewrite(ctx context.Context, key string, in []byte, rewri } c.mu.Unlock() close(f.done) - out, v, err = f.out, c.servedValidators(key, f.etag), f.err }() var fv metadataValidators f.out, fv, f.err = rewriteWithETag(in, rewrite) @@ -222,83 +200,7 @@ func (c *rewriteCache) rewrite(ctx context.Context, key string, in []byte, rewri return f.out, fv, f.err } -// servedValidators returns the validators to serve the rewrite made for key -// with, if it was cached, and its bare ETag otherwise. -func (c *rewriteCache) servedValidators(key, etag string) metadataValidators { - c.mu.Lock() - defer c.mu.Unlock() - if el, ok := c.entries[key]; ok && el.Value.(*rewriteEntry).etag == etag { - return c.validators(el.Value.(*rewriteEntry)) - } - return metadataValidators{etag: etag} -} - -// validators returns the validators to serve cached rewrite e with, and -// records that its package is serving it. Called with mu held. -func (c *rewriteCache) validators(e *rewriteEntry) metadataValidators { - return metadataValidators{etag: e.etag, lastModified: c.lastModified(rewritePackage(e.key), e.etag)} -} - -// rewritePackage returns the part of a rewrite cache key that names the -// package, without the digest of the upstream document. -func rewritePackage(key string) string { - return key[:strings.LastIndexByte(key, 0)] -} - -// lastModified returns the Last-Modified to serve the rewrite of pkg with -// the given ETag with, or zero for none, and records that pkg is serving it. -// -// The date is the second pkg started serving the rewrite, which is later than -// any date given to a rewrite pkg served before, so a client holding one never -// gets 304 for another. Upstream's date cannot serve, since the rewrite also -// depends on configuration read at startup, and a rewrite cached earlier and -// served again keeps no date of its own. Dates are never ahead of the clock, -// so they hold across a restart. When pkg switches rewrites in a second that -// already dated one, the new one shares that second with a date a client may -// hold, so it goes undated until the next second. Called with mu held. -func (c *rewriteCache) lastModified(pkg, etag string) time.Time { - now := c.now().Truncate(time.Second) - cur, ok := c.serving[pkg] - switch { - case ok && cur.etag == etag: - if cur.since.IsZero() && now.After(cur.switched) { - cur.since = now - } - case ok: - since := now - if !cur.switched.Before(now) || !cur.since.Before(now) { - since = time.Time{} - } - cur = servedRewrite{etag: etag, switched: now, since: since} - default: - since := now - if !now.After(c.forgotten) { - since = time.Time{} - } - cur = servedRewrite{etag: etag, switched: now, since: since} - } - c.serving[pkg] = cur - return cur.since -} - -// forget drops what serving records for e's package when e is the rewrite -// the package is serving, so the map holds only packages with a cached -// rewrite. Called with mu held, as e is evicted. -func (c *rewriteCache) forget(e *rewriteEntry) { - pkg := rewritePackage(e.key) - cur, ok := c.serving[pkg] - if !ok || cur.etag != e.etag { - return - } - delete(c.serving, pkg) - now := c.now().Truncate(time.Second) - if !cur.switched.Before(now) || !cur.since.Before(now) { - c.forgotten = now - } -} - -// rewriteWithETag runs rewrite and computes the ETag of what it returns. It -// gives no Last-Modified: nothing records when an uncached rewrite was served. +// rewriteWithETag runs rewrite and computes the ETag of what it returns. func rewriteWithETag(in []byte, rewrite func([]byte) ([]byte, error)) ([]byte, metadataValidators, error) { out, err := rewrite(in) if err != nil { @@ -322,7 +224,6 @@ func (c *rewriteCache) add(e *rewriteEntry) { e := oldest.Value.(*rewriteEntry) c.order.Remove(oldest) delete(c.entries, e.key) - c.forget(e) c.size -= int64(len(e.out)) } } @@ -343,7 +244,8 @@ func (c *rewriteCache) len() int { // on the current time, so with cooldown on a cached rewrite could keep hiding // a version past its cooldown; those rewrites always run, with no // Last-Modified, and when digest is known their ETag comes from their inputs -// rather than from hashing each output again. +// rather than from hashing each output again. Their ETag also changes with +// every restart, so replicas would never agree on a date for them. func (p *Proxy) cachedRewrite(ctx context.Context, ecosystem, proxyURL, name string, in []byte, digest string, rewrite keepingRewrite) (rewrittenMetadata, error) { if p.cooldownEnabled() && digest != "" { out, kept, err := rewrite(in) @@ -364,6 +266,9 @@ func (p *Proxy) cachedRewrite(ctx context.Context, ecosystem, proxyURL, name str } else { out, v, err = p.rewrites.rewrite(ctx, rewriteCacheKey(ecosystem, proxyURL, name, in), in, rewriteOnly) } + if err == nil && !p.cooldownEnabled() { + v.lastModified = p.metadataDate(ecosystem, proxyURL, name, v.etag) + } return rewrittenMetadata{out, v}, err } @@ -395,6 +300,7 @@ func (p *Proxy) storedRewrite(ecosystem, cacheKey, proxyURL, name string) (rewri if !ok { return rewrittenMetadata{}, false } + v.lastModified = p.metadataDate(ecosystem, proxyURL, name, v.etag) metrics.RecordCacheHit(ecosystem) return rewrittenMetadata{out, v}, true } From f3505ad557e15b98cc77ac0e4cf164a22185e014 Mon Sep 17 00:00:00 2001 From: Christian Heim Date: Sun, 11 Oct 2026 01:55:16 +0200 Subject: [PATCH 223/633] Enable storage retention for helm Helm takes no package rules: charts are cached under the configured repository's name and the chart digest, not under the chart name, so a key like pkg:helm/nginx would name a repository. The helm rule and the default cover charts from every HTTP repository; charts pulled over OCI go through the container handler and fall under oci. Refs #306 --- docs/configuration.md | 2 +- internal/handler/helm.go | 11 ++ internal/server/retention_helm_test.go | 207 +++++++++++++++++++++++++ 3 files changed, 219 insertions(+), 1 deletion(-) create mode 100644 internal/server/retention_helm_test.go diff --git a/docs/configuration.md b/docs/configuration.md index 41ba4d0..5e35376 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -97,7 +97,7 @@ A package rule wins over an ecosystem rule, which wins over `default`. Package r | `deb` | no | - | | `gem` | no | - | | `golang` | yes | `pkg:golang/github.com/stretchr/testify`, the module path as in `go.mod`; case doesn't matter. A major version like `/v2` (or `.v3` on gopkg.in) is a module of its own and needs its own rule. The `!`-escaped form from proxy URLs and the module cache (`github.com/!azure/...`) is refused. | -| `helm` | no | - | +| `helm` | yes | No package rules: charts are cached per repository and digest, not under the chart name. The `helm` rule and `default` cover charts from HTTP repositories; charts pulled over OCI fall under `oci`. | | `hex` | no | - | | `julia` | no | - | | `maven` | yes | `pkg:maven/com.google.guava/guava`, group and artifact as published, case included. Gradle build cache entries are not covered; they use `gradle.build_cache.max_age`. | diff --git a/internal/handler/helm.go b/internal/handler/helm.go index 31ef901..cddc943 100644 --- a/internal/handler/helm.go +++ b/internal/handler/helm.go @@ -12,6 +12,8 @@ import ( "time" "gopkg.in/yaml.v3" + + "github.com/git-pkgs/proxy/internal/retention" ) const ( @@ -20,6 +22,15 @@ const ( sha256HexLength = 64 ) +// Helm takes no package rules: charts are cached under the configured +// repository's name and the chart digest, not under the chart name, so a key +// like pkg:helm/nginx would name a repository. The helm rule and the default +// cover charts from HTTP repositories; charts pulled over OCI go through the +// container handler and fall under oci. +func init() { + retention.Default.Register(retention.Spec{Key: "helm"}) +} + // HelmHandler serves read-only HTTP Helm chart repositories. Each configured // repository is mounted at /helm/{repository}/. type HelmHandler struct { diff --git a/internal/server/retention_helm_test.go b/internal/server/retention_helm_test.go new file mode 100644 index 0000000..41a837d --- /dev/null +++ b/internal/server/retention_helm_test.go @@ -0,0 +1,207 @@ +package server + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/git-pkgs/proxy/internal/config" + "github.com/git-pkgs/proxy/internal/database" + "github.com/git-pkgs/proxy/internal/handler" + "github.com/git-pkgs/proxy/internal/retention" + "github.com/git-pkgs/registries/fetch" +) + +// helmChart is a chart archive of a Helm repository, addressed by digest. +type helmChart struct { + filename string + content string +} + +func (c helmChart) digest() string { + sum := sha256.Sum256([]byte(c.content)) + return hex.EncodeToString(sum[:]) +} + +var ( + helmNginxChart = helmChart{"nginx-15.0.0.tgz", "nginx chart"} + helmRedisChart = helmChart{"redis-18.0.0.tgz", "redis chart"} +) + +// helmRetentionUpstream serves an index.yaml listing both charts and the +// chart archives the way an HTTP Helm repository does, counting downloads. +func helmRetentionUpstream(t *testing.T) (*httptest.Server, *atomic.Int64) { + t.Helper() + var downloads atomic.Int64 + charts := []helmChart{helmNginxChart, helmRedisChart} + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/index.yaml" { + var index strings.Builder + index.WriteString("apiVersion: v1\nentries:\n") + for _, c := range charts { + name := strings.SplitN(c.filename, "-", 2)[0] + _, _ = fmt.Fprintf(&index, " %s:\n - digest: %s\n urls: [%s]\n", name, c.digest(), c.filename) + } + _, _ = io.WriteString(w, index.String()) + return + } + for _, c := range charts { + if r.URL.Path == "/"+c.filename { + downloads.Add(1) + _, _ = io.WriteString(w, c.content) + return + } + } + http.NotFound(w, r) + })) + t.Cleanup(srv.Close) + return srv, &downloads +} + +// newHelmRetentionProxy mounts the upstream as the repositories "stable" +// and "internal". +func newHelmRetentionProxy(t *testing.T, upstream *httptest.Server) (*httptest.Server, *database.DB) { + t.Helper() + db, store := setupEvictionTest(t) + // The upstream is a plain-HTTP loopback server; the default fetcher + // refuses loopback, a stock client reaches it. + fetcher := fetch.NewFetcher(fetch.WithHTTPClient(upstream.Client()), fetch.WithMaxRetries(0)) + t.Cleanup(func() { _ = fetcher.Close() }) + proxy := handler.NewProxy(db, store, fetcher, fetch.NewResolver(), discardLogger()) + proxy.HTTPClient = upstream.Client() + h := handler.NewHelmHandler(proxy, "http://proxy.test", map[string]string{"stable": upstream.URL, "internal": upstream.URL}) + srv := httptest.NewServer(h.Routes()) + t.Cleanup(srv.Close) + return srv, db +} + +func helmGet(t *testing.T, srv *httptest.Server, repository string, c helmChart) { + t.Helper() + url := srv.URL + "/" + repository + "/charts/" + c.digest() + "/" + c.filename + resp, err := http.Get(url) + if err != nil { + t.Fatalf("GET %s: %v", url, err) + } + body, _ := io.ReadAll(resp.Body) + _ = resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Fatalf("GET %s = %d %s, want 200", url, resp.StatusCode, body) + } +} + +// helmArtifact returns the record of the chart cached for repository. +func helmArtifact(t *testing.T, db *database.DB, repository string, c helmChart) database.Artifact { + t.Helper() + var art database.Artifact + query := db.Rebind(`SELECT a.* FROM artifacts a JOIN versions v ON v.purl = a.version_purl + WHERE a.filename = ? AND v.package_purl = ?`) + if err := db.Get(&art, query, c.filename, "pkg:helm/"+repository); err != nil { + t.Fatalf("reading %s from %s: %v", c.filename, repository, err) + } + return art +} + +func helmBackdate(t *testing.T, db *database.DB, c helmChart, fetched, accessed time.Time) { + t.Helper() + query := db.Rebind(`UPDATE artifacts SET fetched_at = ?, last_accessed_at = ? WHERE filename = ?`) + if _, err := db.Exec(query, fetched, accessed, c.filename); err != nil { + t.Fatalf("backdating %s: %v", c.filename, err) + } +} + +func TestHelmRegistersForRetention(t *testing.T) { + spec, ok := retention.Default.Lookup("helm") + if !ok { + t.Fatal("helm is not registered for retention") + } + if spec.CanonicalPackage != nil { + t.Error("helm accepts package rules, but its records are keyed by repository, not chart") + } + if key, ok := retention.Default.KeyForDBEcosystem("helm"); !ok || key != "helm" { + t.Errorf("KeyForDBEcosystem(helm) = %q, %v, want helm, true", key, ok) + } +} + +// A chart served from the cache stays while it is used, is evicted once its +// rule runs out, comes back from upstream on the next download, and stays +// after that even with the evicted copy's old access time on the record. +func TestHelmRetentionEvictsAndRefetches(t *testing.T) { + upstream, downloads := helmRetentionUpstream(t) + srv, db := newHelmRetentionProxy(t, upstream) + + helmGet(t, srv, "stable", helmNginxChart) + helmGet(t, srv, "stable", helmNginxChart) + if got := downloads.Load(); got != 1 { + t.Fatalf("upstream downloads = %d, want 1", got) + } + if !helmArtifact(t, db, "stable", helmNginxChart).LastAccessedAt.Valid { + t.Fatal("cache hit did not record an access") + } + + rules := retention.Rules{Ecosystems: map[string]time.Duration{"helm": retentionDay}} + now := time.Now() + sweepRetention(context.Background(), db, discardLogger(), retention.Default, rules, now) + if !helmArtifact(t, db, "stable", helmNginxChart).StoragePath.Valid { + t.Fatal("chart used today was evicted") + } + + old := now.Add(-2 * retentionDay) + helmBackdate(t, db, helmNginxChart, old, old) + sweepRetention(context.Background(), db, discardLogger(), retention.Default, rules, now) + if helmArtifact(t, db, "stable", helmNginxChart).StoragePath.Valid { + t.Fatal("chart unused for two days kept under a one-day rule") + } + + helmGet(t, srv, "stable", helmNginxChart) + if got := downloads.Load(); got != 2 { + t.Errorf("upstream downloads = %d, want 2 after the eviction", got) + } + art := helmArtifact(t, db, "stable", helmNginxChart) + helmBackdate(t, db, helmNginxChart, art.FetchedAt.Time, now.Add(-30*retentionDay)) + sweepRetention(context.Background(), db, discardLogger(), retention.Default, rules, time.Now()) + if !helmArtifact(t, db, "stable", helmNginxChart).StoragePath.Valid { + t.Error("refetched chart evicted again by the next sweep") + } +} + +// Package rules for helm are refused, while the helm rule and the default +// both reach charts from every configured repository. +func TestHelmRetentionEcosystemRuleAndDefault(t *testing.T) { + cfg := config.Default() + cfg.Storage.Retention.Packages = map[string]string{"pkg:helm/nginx": "0"} + // Any refusal naming helm will do, except the one for ecosystems that + // aren't registered at all. + if _, err := cfg.RetentionRules(retention.Default); err == nil || !strings.Contains(err.Error(), "helm") || + strings.Contains(err.Error(), "in this version") { + t.Errorf("RetentionRules(pkg:helm/nginx) error = %v, want package rules refused", err) + } + + upstream, _ := helmRetentionUpstream(t) + srv, db := newHelmRetentionProxy(t, upstream) + for _, rules := range []retention.Rules{ + {Ecosystems: map[string]time.Duration{"helm": retentionDay}}, + {Default: retentionDay}, + } { + helmGet(t, srv, "stable", helmNginxChart) + helmGet(t, srv, "internal", helmRedisChart) + old := time.Now().Add(-30 * retentionDay) + helmBackdate(t, db, helmNginxChart, old, old) + helmBackdate(t, db, helmRedisChart, old, old) + sweepRetention(context.Background(), db, discardLogger(), retention.Default, rules, time.Now()) + + if helmArtifact(t, db, "stable", helmNginxChart).StoragePath.Valid { + t.Errorf("with %+v: nginx chart kept", rules) + } + if helmArtifact(t, db, "internal", helmRedisChart).StoragePath.Valid { + t.Errorf("with %+v: redis chart kept", rules) + } + } +} From 8f30c6d47808ffe19a78ee8747c500fbe5a29b6b Mon Sep 17 00:00:00 2001 From: Christian Heim Date: Sun, 11 Oct 2026 01:38:14 +0200 Subject: [PATCH 224/633] Enable storage retention for julia Julia takes no package rules, since its PURLs need a uuid qualifier the cache doesn't store; configuring one already fails validation. The julia rule and the default cover package tarballs, artifacts and registry snapshots alike. Everything there is content-addressed, so an evicted tarball comes back unchanged. Refs #306 --- docs/configuration.md | 2 +- internal/handler/julia.go | 9 ++ internal/server/retention_julia_test.go | 186 ++++++++++++++++++++++++ 3 files changed, 196 insertions(+), 1 deletion(-) create mode 100644 internal/server/retention_julia_test.go diff --git a/docs/configuration.md b/docs/configuration.md index 41ba4d0..43a3ab8 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -99,7 +99,7 @@ A package rule wins over an ecosystem rule, which wins over `default`. Package r | `golang` | yes | `pkg:golang/github.com/stretchr/testify`, the module path as in `go.mod`; case doesn't matter. A major version like `/v2` (or `.v3` on gopkg.in) is a module of its own and needs its own rule. The `!`-escaped form from proxy URLs and the module cache (`github.com/!azure/...`) is refused. | | `helm` | no | - | | `hex` | no | - | -| `julia` | no | - | +| `julia` | yes | No package rules: a julia PURL needs a `uuid` qualifier, which the cache doesn't store. The `julia` rule and `default` cover package tarballs, artifacts and registry snapshots alike. | | `maven` | yes | `pkg:maven/com.google.guava/guava`, group and artifact as published, case included. Gradle build cache entries are not covered; they use `gradle.build_cache.max_age`. | | `npm` | yes | `pkg:npm/lodash`, `pkg:npm/@babel/core`; the name has to match exactly, case included, so `pkg:npm/jsonstream` doesn't cover `JSONStream`. | | `nuget` | no | - | diff --git a/internal/handler/julia.go b/internal/handler/julia.go index 66a2245..749335b 100644 --- a/internal/handler/julia.go +++ b/internal/handler/julia.go @@ -14,6 +14,7 @@ import ( "sync" "github.com/BurntSushi/toml" + "github.com/git-pkgs/proxy/internal/retention" ) const ( @@ -28,6 +29,14 @@ var ( juliaUUIDPattern = regexp.MustCompile(`^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$`) ) +// Julia takes no package rules: its PURLs need a uuid qualifier the cache +// doesn't store. The julia rule and the default cover package tarballs, +// artifacts and registry snapshots alike; all are content-addressed, so an +// evicted one comes back unchanged. +func init() { + retention.Default.Register(retention.Spec{Key: "julia"}) +} + // JuliaHandler handles Julia Pkg server protocol requests. // // See https://pkgdocs.julialang.org/v1/registries/ and the PkgServer.jl diff --git a/internal/server/retention_julia_test.go b/internal/server/retention_julia_test.go new file mode 100644 index 0000000..5577ec7 --- /dev/null +++ b/internal/server/retention_julia_test.go @@ -0,0 +1,186 @@ +package server + +import ( + "context" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/git-pkgs/proxy/internal/config" + "github.com/git-pkgs/proxy/internal/database" + "github.com/git-pkgs/proxy/internal/handler" + "github.com/git-pkgs/proxy/internal/retention" + "github.com/git-pkgs/registries/fetch" +) + +const ( + juliaPackageUUID = "7876af07-990d-54b4-ab0e-23690620f79a" + juliaPackageHash = "46e44e2b1ab5b2e0b7a2b0b63c87a4b0a3f6b2a1" + juliaArtifactHsh = "0c61b4f2e0a4b0c3e8f1a2b3c4d5e6f708192a3b" + juliaRegistryUID = "23338594-aafe-5451-b93e-139f81909106" + juliaRegistryHsh = "9a1f0e2d3c4b5a69788796a5b4c3d2e1f0a9b8c7" +) + +// juliaRetentionUpstream serves a package tarball, an artifact and a +// registry snapshot the way a Pkg server does, counting downloads. The +// snapshot isn't a real registry, so the handler can't load names and files +// the package under its uuid. +func juliaRetentionUpstream(t *testing.T) (*httptest.Server, *atomic.Int64) { + t.Helper() + var downloads atomic.Int64 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/package/" + juliaPackageUUID + "/" + juliaPackageHash, "/artifact/" + juliaArtifactHsh, + "/registry/" + juliaRegistryUID + "/" + juliaRegistryHsh: + downloads.Add(1) + _, _ = io.WriteString(w, "tarball bytes") + default: + http.NotFound(w, r) + } + })) + t.Cleanup(srv.Close) + return srv, &downloads +} + +func newJuliaRetentionProxy(t *testing.T, upstream *httptest.Server) (*httptest.Server, *database.DB) { + t.Helper() + db, store := setupEvictionTest(t) + // The upstream is a plain-HTTP loopback server; the default fetcher + // refuses loopback, a stock client reaches it. + fetcher := fetch.NewFetcher(fetch.WithHTTPClient(upstream.Client()), fetch.WithMaxRetries(0)) + t.Cleanup(func() { _ = fetcher.Close() }) + proxy := handler.NewProxy(db, store, fetcher, fetch.NewResolver(), discardLogger()) + proxy.HTTPClient = upstream.Client() + h := handler.NewJuliaHandlerWithUpstream(proxy, upstream.URL) + srv := httptest.NewServer(h.Routes()) + t.Cleanup(srv.Close) + return srv, db +} + +func juliaGet(t *testing.T, url string) { + t.Helper() + resp, err := http.Get(url) + if err != nil { + t.Fatalf("GET %s: %v", url, err) + } + body, _ := io.ReadAll(resp.Body) + _ = resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Fatalf("GET %s = %d %s, want 200", url, resp.StatusCode, body) + } +} + +// juliaArtifact returns the record of the tarball named after hash. +func juliaArtifact(t *testing.T, db *database.DB, hash string) database.Artifact { + t.Helper() + var art database.Artifact + if err := db.Get(&art, db.Rebind(`SELECT * FROM artifacts WHERE filename = ?`), hash+".tar.gz"); err != nil { + t.Fatalf("reading artifact %s: %v", hash, err) + } + return art +} + +func juliaBackdate(t *testing.T, db *database.DB, hash string, fetched, accessed time.Time) { + t.Helper() + query := db.Rebind(`UPDATE artifacts SET fetched_at = ?, last_accessed_at = ? WHERE filename = ?`) + if _, err := db.Exec(query, fetched, accessed, hash+".tar.gz"); err != nil { + t.Fatalf("backdating %s: %v", hash, err) + } +} + +func TestJuliaRegistersForRetention(t *testing.T) { + spec, ok := retention.Default.Lookup("julia") + if !ok { + t.Fatal("julia is not registered for retention") + } + if spec.CanonicalPackage != nil { + t.Error("julia accepts package rules, but its PURLs can never match a stored package") + } + if key, ok := retention.Default.KeyForDBEcosystem("julia"); !ok || key != "julia" { + t.Errorf("KeyForDBEcosystem(julia) = %q, %v, want julia, true", key, ok) + } +} + +// An artifact served from the cache stays while it is used, is evicted once +// its rule runs out, comes back from upstream on the next download, and +// stays after that even with the evicted copy's old access time on the +// record. +func TestJuliaRetentionEvictsAndRefetches(t *testing.T) { + upstream, downloads := juliaRetentionUpstream(t) + srv, db := newJuliaRetentionProxy(t, upstream) + artifactURL := srv.URL + "/artifact/" + juliaArtifactHsh + + juliaGet(t, artifactURL) + juliaGet(t, artifactURL) + if got := downloads.Load(); got != 1 { + t.Fatalf("upstream downloads = %d, want 1", got) + } + if !juliaArtifact(t, db, juliaArtifactHsh).LastAccessedAt.Valid { + t.Fatal("cache hit did not record an access") + } + + rules := retention.Rules{Ecosystems: map[string]time.Duration{"julia": retentionDay}} + now := time.Now() + sweepRetention(context.Background(), db, discardLogger(), retention.Default, rules, now) + if !juliaArtifact(t, db, juliaArtifactHsh).StoragePath.Valid { + t.Fatal("artifact used today was evicted") + } + + old := now.Add(-2 * retentionDay) + juliaBackdate(t, db, juliaArtifactHsh, old, old) + sweepRetention(context.Background(), db, discardLogger(), retention.Default, rules, now) + if juliaArtifact(t, db, juliaArtifactHsh).StoragePath.Valid { + t.Fatal("artifact unused for two days kept under a one-day rule") + } + + juliaGet(t, artifactURL) + if got := downloads.Load(); got != 2 { + t.Errorf("upstream downloads = %d, want 2 after the eviction", got) + } + art := juliaArtifact(t, db, juliaArtifactHsh) + juliaBackdate(t, db, juliaArtifactHsh, art.FetchedAt.Time, now.Add(-30*retentionDay)) + sweepRetention(context.Background(), db, discardLogger(), retention.Default, rules, time.Now()) + if !juliaArtifact(t, db, juliaArtifactHsh).StoragePath.Valid { + t.Error("refetched artifact evicted again by the next sweep") + } +} + +// Package rules for julia are refused, while the julia rule and the default +// both reach package tarballs, artifacts and registry snapshots. +func TestJuliaRetentionEcosystemRuleAndDefault(t *testing.T) { + cfg := config.Default() + cfg.Storage.Retention.Packages = map[string]string{"pkg:julia/Example?uuid=" + juliaPackageUUID: "0"} + if _, err := cfg.RetentionRules(retention.Default); err == nil || !strings.Contains(err.Error(), "not supported for julia") { + t.Errorf("RetentionRules(julia package) error = %v, want package rules refused", err) + } + + upstream, _ := juliaRetentionUpstream(t) + srv, db := newJuliaRetentionProxy(t, upstream) + for _, rules := range []retention.Rules{ + {Ecosystems: map[string]time.Duration{"julia": retentionDay}}, + {Default: retentionDay}, + } { + juliaGet(t, srv.URL+"/package/"+juliaPackageUUID+"/"+juliaPackageHash) + juliaGet(t, srv.URL+"/artifact/"+juliaArtifactHsh) + juliaGet(t, srv.URL+"/registry/"+juliaRegistryUID+"/"+juliaRegistryHsh) + old := time.Now().Add(-30 * retentionDay) + juliaBackdate(t, db, juliaPackageHash, old, old) + juliaBackdate(t, db, juliaArtifactHsh, old, old) + juliaBackdate(t, db, juliaRegistryHsh, old, old) + sweepRetention(context.Background(), db, discardLogger(), retention.Default, rules, time.Now()) + + if juliaArtifact(t, db, juliaPackageHash).StoragePath.Valid { + t.Errorf("with %+v: package tarball kept", rules) + } + if juliaArtifact(t, db, juliaArtifactHsh).StoragePath.Valid { + t.Errorf("with %+v: artifact kept", rules) + } + if juliaArtifact(t, db, juliaRegistryHsh).StoragePath.Valid { + t.Errorf("with %+v: registry snapshot kept", rules) + } + } +} From 20855cae210505557be584d72435d3ec14893575 Mon Sep 17 00:00:00 2001 From: Christian Heim Date: Sat, 10 Oct 2026 23:51:55 +0200 Subject: [PATCH 225/633] Enable storage retention for gem Gems are stored under the name from the gem filename, and purl keeps gem names as they are, so package rules use the default mapping and match the name as published, case included. Packages that git-pkgs wrote as "rubygems" fall under the gem rules too. The filename parser splits platforms that end in a number (universal-darwin-22) at the wrong hyphen and files those gems under a package like sorbet-static-0.5.11002-universal-darwin, so package rules don't reach them; the docs row says so until the parser is fixed. The mirror can't fetch gems today (it asks the resolver for "rubygems", see #415), so there's no mirror test. Refs #306 --- docs/configuration.md | 2 +- internal/handler/gem.go | 12 ++ internal/server/retention_gem_test.go | 230 ++++++++++++++++++++++++++ 3 files changed, 243 insertions(+), 1 deletion(-) create mode 100644 internal/server/retention_gem_test.go diff --git a/docs/configuration.md b/docs/configuration.md index 41ba4d0..06e0b7c 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -95,7 +95,7 @@ A package rule wins over an ecosystem rule, which wins over `default`. Package r | `conda` | no | - | | `cran` | no | - | | `deb` | no | - | -| `gem` | no | - | +| `gem` | yes | `pkg:gem/rails`, the gem name as published, case included. Package rules don't reach gem files whose platform ends in a number (`universal-darwin-22`, `universal-java-17`) yet, which the proxy files under a wrong package name; the gem rule and `default` still cover them. | | `golang` | yes | `pkg:golang/github.com/stretchr/testify`, the module path as in `go.mod`; case doesn't matter. A major version like `/v2` (or `.v3` on gopkg.in) is a module of its own and needs its own rule. The `!`-escaped form from proxy URLs and the module cache (`github.com/!azure/...`) is refused. | | `helm` | no | - | | `hex` | no | - | diff --git a/internal/handler/gem.go b/internal/handler/gem.go index efe964b..93e9989 100644 --- a/internal/handler/gem.go +++ b/internal/handler/gem.go @@ -7,12 +7,24 @@ import ( "net/http" "strings" "time" + + "github.com/git-pkgs/proxy/internal/retention" ) const ( gemUpstream = "https://rubygems.org" ) +// Gems are stored under pkg:gem/{name} with the name from the gem filename, +// and purl keeps gem names as they are, so package rules use the default +// key mapping. parseGemFilename splits at the last hyphen before a digit, +// so a platform like universal-darwin-22 lands under a wrong package name +// that package rules don't reach. The spec also claims the "rubygems" +// spelling git-pkgs writes into packages.ecosystem. +func init() { + retention.Default.Register(retention.Spec{Key: "gem", CanonicalPackage: retention.DefaultCanonical("gem")}) +} + // GemHandler handles RubyGems registry protocol requests. type GemHandler struct { proxy *Proxy diff --git a/internal/server/retention_gem_test.go b/internal/server/retention_gem_test.go new file mode 100644 index 0000000..ebbc029 --- /dev/null +++ b/internal/server/retention_gem_test.go @@ -0,0 +1,230 @@ +package server + +import ( + "context" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/git-pkgs/proxy/internal/config" + "github.com/git-pkgs/proxy/internal/database" + "github.com/git-pkgs/proxy/internal/handler" + "github.com/git-pkgs/proxy/internal/retention" + "github.com/git-pkgs/registries/fetch" +) + +// gemRetentionUpstream serves gem files the way rubygems.org does, +// counting downloads. +func gemRetentionUpstream(t *testing.T) (*httptest.Server, *atomic.Int64) { + t.Helper() + var downloads atomic.Int64 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/gems/rack-3.0.8.gem", "/gems/RedCloth-4.3.3.gem": + downloads.Add(1) + _, _ = io.WriteString(w, "gem bytes") + default: + http.NotFound(w, r) + } + })) + t.Cleanup(srv.Close) + return srv, &downloads +} + +func newGemRetentionProxy(t *testing.T, upstream *httptest.Server) (*httptest.Server, *database.DB) { + t.Helper() + db, store := setupEvictionTest(t) + // The upstream is a plain-HTTP loopback server; the default fetcher + // refuses loopback, a stock client reaches it. + fetcher := fetch.NewFetcher(fetch.WithHTTPClient(upstream.Client()), fetch.WithMaxRetries(0)) + t.Cleanup(func() { _ = fetcher.Close() }) + proxy := handler.NewProxy(db, store, fetcher, fetch.NewResolver(), discardLogger()) + proxy.HTTPClient = upstream.Client() + h := handler.NewGemHandlerWithUpstream(proxy, "http://proxy.test", upstream.URL) + srv := httptest.NewServer(h.Routes()) + t.Cleanup(srv.Close) + return srv, db +} + +func gemGet(t *testing.T, url string) { + t.Helper() + resp, err := http.Get(url) + if err != nil { + t.Fatalf("GET %s: %v", url, err) + } + body, _ := io.ReadAll(resp.Body) + _ = resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Fatalf("GET %s = %d %s, want 200", url, resp.StatusCode, body) + } +} + +// gemArtifact returns the record of the gem file named filename. +func gemArtifact(t *testing.T, db *database.DB, filename string) database.Artifact { + t.Helper() + var art database.Artifact + if err := db.Get(&art, db.Rebind(`SELECT * FROM artifacts WHERE filename = ?`), filename); err != nil { + t.Fatalf("reading artifact %s: %v", filename, err) + } + return art +} + +func gemBackdate(t *testing.T, db *database.DB, filename string, fetched, accessed time.Time) { + t.Helper() + query := db.Rebind(`UPDATE artifacts SET fetched_at = ?, last_accessed_at = ? WHERE filename = ?`) + if _, err := db.Exec(query, fetched, accessed, filename); err != nil { + t.Fatalf("backdating %s: %v", filename, err) + } +} + +func TestGemRegistersForRetention(t *testing.T) { + if _, ok := retention.Default.Lookup("gem"); !ok { + t.Fatal("gem is not registered for retention") + } + for _, value := range []string{"gem", "rubygems"} { + if key, ok := retention.Default.KeyForDBEcosystem(value); !ok || key != "gem" { + t.Errorf("KeyForDBEcosystem(%s) = %q, %v, want gem, true", value, key, ok) + } + } +} + +// A gem served from the cache stays while it is used, is evicted once its +// rule runs out, comes back from upstream on the next download, and stays +// after that even with the evicted copy's old access time on the record. +func TestGemRetentionEvictsAndRefetches(t *testing.T) { + upstream, downloads := gemRetentionUpstream(t) + srv, db := newGemRetentionProxy(t, upstream) + const gem = "rack-3.0.8.gem" + + gemGet(t, srv.URL+"/gems/rack-3.0.8.gem") + gemGet(t, srv.URL+"/gems/rack-3.0.8.gem") + if got := downloads.Load(); got != 1 { + t.Fatalf("upstream downloads = %d, want 1", got) + } + if !gemArtifact(t, db, gem).LastAccessedAt.Valid { + t.Fatal("cache hit did not record an access") + } + + rules := retention.Rules{Ecosystems: map[string]time.Duration{"gem": retentionDay}} + now := time.Now() + sweepRetention(context.Background(), db, discardLogger(), retention.Default, rules, now) + if !gemArtifact(t, db, gem).StoragePath.Valid { + t.Fatal("gem used today was evicted") + } + + old := now.Add(-2 * retentionDay) + gemBackdate(t, db, gem, old, old) + sweepRetention(context.Background(), db, discardLogger(), retention.Default, rules, now) + if gemArtifact(t, db, gem).StoragePath.Valid { + t.Fatal("gem unused for two days kept under a one-day rule") + } + + gemGet(t, srv.URL+"/gems/rack-3.0.8.gem") + if got := downloads.Load(); got != 2 { + t.Errorf("upstream downloads = %d, want 2 after the eviction", got) + } + art := gemArtifact(t, db, gem) + gemBackdate(t, db, gem, art.FetchedAt.Time, now.Add(-30*retentionDay)) + sweepRetention(context.Background(), db, discardLogger(), retention.Default, rules, time.Now()) + if !gemArtifact(t, db, gem).StoragePath.Valid { + t.Error("refetched gem evicted again by the next sweep") + } +} + +// A key with a namespace parses as a gem PURL but can never match a +// stored gem, so it is refused at validation. A plain key is accepted. +func TestGemRetentionRefusesNamespacedKeys(t *testing.T) { + cfg := config.Default() + cfg.Storage.Retention.Packages = map[string]string{"pkg:gem/rack": "0"} + if _, err := cfg.RetentionRules(retention.Default); err != nil { + t.Fatalf("RetentionRules(pkg:gem/rack): %v", err) + } + cfg.Storage.Retention.Packages = map[string]string{"pkg:gem/rails/rack": "0"} + _, err := cfg.RetentionRules(retention.Default) + if err == nil || !strings.Contains(err.Error(), "does not match how gem packages are stored") { + t.Errorf("RetentionRules(pkg:gem/rails/rack) error = %v, want the key refused", err) + } +} + +// A package rule reaches a gem under its published name, case included; +// a rule with the name in another case covers nothing. +func TestGemRetentionPackageRules(t *testing.T) { + upstream, _ := gemRetentionUpstream(t) + srv, db := newGemRetentionProxy(t, upstream) + const redcloth, rack = "RedCloth-4.3.3.gem", "rack-3.0.8.gem" + + tests := []struct { + key string + keepRedCloth bool + }{ + {"pkg:gem/RedCloth", true}, + {"pkg:gem/redcloth", false}, + } + for _, tt := range tests { + gemGet(t, srv.URL+"/gems/RedCloth-4.3.3.gem") + gemGet(t, srv.URL+"/gems/rack-3.0.8.gem") + old := time.Now().Add(-30 * retentionDay) + gemBackdate(t, db, redcloth, old, old) + gemBackdate(t, db, rack, old, old) + + cfg := config.Default() + cfg.Storage.Retention.Ecosystems = map[string]string{"gem": "1d"} + cfg.Storage.Retention.Packages = map[string]string{tt.key: "0"} + rules, err := cfg.RetentionRules(retention.Default) + if err != nil { + t.Fatalf("RetentionRules(%s): %v", tt.key, err) + } + sweepRetention(context.Background(), db, discardLogger(), retention.Default, rules, time.Now()) + + if got := gemArtifact(t, db, redcloth).StoragePath.Valid; got != tt.keepRedCloth { + t.Errorf("with %s: RedCloth cached = %v, want %v", tt.key, got, tt.keepRedCloth) + } + if gemArtifact(t, db, rack).StoragePath.Valid { + t.Errorf("with %s: rack kept past the gem rule", tt.key) + } + } +} + +// git-pkgs writes gem packages with ecosystem "rubygems", and the record +// keeps the ecosystem of whoever wrote it first. A gem the proxy then caches +// under such a record still falls under the gem rule and its package rules. +func TestGemRetentionCoversRubygemsSpelling(t *testing.T) { + upstream, _ := gemRetentionUpstream(t) + srv, db := newGemRetentionProxy(t, upstream) + if err := db.UpsertPackage(&database.Package{PURL: "pkg:gem/rack", Ecosystem: "rubygems", Name: "rack"}); err != nil { + t.Fatalf("UpsertPackage: %v", err) + } + if err := db.UpsertPackage(&database.Package{PURL: "pkg:gem/RedCloth", Ecosystem: "rubygems", Name: "RedCloth"}); err != nil { + t.Fatalf("UpsertPackage: %v", err) + } + const redcloth, rack = "RedCloth-4.3.3.gem", "rack-3.0.8.gem" + gemGet(t, srv.URL+"/gems/rack-3.0.8.gem") + gemGet(t, srv.URL+"/gems/RedCloth-4.3.3.gem") + var ecosystem string + if err := db.Get(&ecosystem, db.Rebind(`SELECT ecosystem FROM packages WHERE purl = ?`), "pkg:gem/rack"); err != nil || ecosystem != "rubygems" { + t.Fatalf("rack package ecosystem = %q, %v, want rubygems", ecosystem, err) + } + + old := time.Now().Add(-30 * retentionDay) + gemBackdate(t, db, redcloth, old, old) + gemBackdate(t, db, rack, old, old) + cfg := config.Default() + cfg.Storage.Retention.Ecosystems = map[string]string{"gem": "1d"} + cfg.Storage.Retention.Packages = map[string]string{"pkg:gem/RedCloth": "0"} + rules, err := cfg.RetentionRules(retention.Default) + if err != nil { + t.Fatalf("RetentionRules: %v", err) + } + sweepRetention(context.Background(), db, discardLogger(), retention.Default, rules, time.Now()) + + if gemArtifact(t, db, rack).StoragePath.Valid { + t.Error("rack under a rubygems record kept past the gem rule") + } + if !gemArtifact(t, db, redcloth).StoragePath.Valid { + t.Error("RedCloth under a rubygems record evicted despite its never-evict rule") + } +} From 48edbb8ba031b06bc6b0639d3122a441497915e5 Mon Sep 17 00:00:00 2001 From: Christian Heim Date: Sat, 10 Oct 2026 23:15:44 +0200 Subject: [PATCH 226/633] Enable storage retention for pub Packages are stored under the name from the download path, and purl keeps pub names as they are, so package rules use the default mapping with the name as published. Refs #306 --- docs/configuration.md | 2 +- internal/handler/pub.go | 9 + internal/server/retention_pub_test.go | 234 ++++++++++++++++++++++++++ 3 files changed, 244 insertions(+), 1 deletion(-) create mode 100644 internal/server/retention_pub_test.go diff --git a/docs/configuration.md b/docs/configuration.md index 41ba4d0..2499d61 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -104,7 +104,7 @@ A package rule wins over an ecosystem rule, which wins over `default`. Package r | `npm` | yes | `pkg:npm/lodash`, `pkg:npm/@babel/core`; the name has to match exactly, case included, so `pkg:npm/jsonstream` doesn't cover `JSONStream`. | | `nuget` | no | - | | `oci` | yes | `pkg:oci/library/nginx`, the repository name of the pull. A named upstream keeps its prefix, also when containerd reaches it through `ns`: `ghcr.io/owner/app` pulled that way is `pkg:oci/upstream/ghcr/owner/app`. Homebrew bottles use Homebrew's image name, with `@` written as `/` and `+` as `x` (`openssl@3` is `pkg:oci/homebrew/core/openssl/3`). Helm charts pulled over OCI are stored like images. Only blobs are evicted; manifests stay in the metadata cache. | -| `pub` | no | - | +| `pub` | yes | `pkg:pub/http`, the package name as published. | | `pypi` | yes | `pkg:pypi/requests`; case and single `_`/`-` don't matter. Dots and repeated separators are kept as written, so a project like `zope.interface` can be stored as `zope.interface` and `zope-interface`; add a rule for each. | | `rpm` | no | - | | `swift` | no | - | diff --git a/internal/handler/pub.go b/internal/handler/pub.go index 6e967e3..e907e73 100644 --- a/internal/handler/pub.go +++ b/internal/handler/pub.go @@ -7,6 +7,8 @@ import ( "net/http" "strings" "time" + + "github.com/git-pkgs/proxy/internal/retention" ) const ( @@ -14,6 +16,13 @@ const ( pubPathParts = 2 // name + version in path split by /versions/ ) +// Packages are stored under pkg:pub/{name} with the name from the download +// path, and purl keeps pub names as they are, so package rules use the +// default key mapping. +func init() { + retention.Default.Register(retention.Spec{Key: "pub", CanonicalPackage: retention.DefaultCanonical("pub")}) +} + // PubHandler handles pub.dev registry protocol requests. type PubHandler struct { proxy *Proxy diff --git a/internal/server/retention_pub_test.go b/internal/server/retention_pub_test.go new file mode 100644 index 0000000..11f0625 --- /dev/null +++ b/internal/server/retention_pub_test.go @@ -0,0 +1,234 @@ +package server + +import ( + "context" + "io" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/git-pkgs/proxy/internal/config" + "github.com/git-pkgs/proxy/internal/database" + "github.com/git-pkgs/proxy/internal/handler" + "github.com/git-pkgs/proxy/internal/mirror" + "github.com/git-pkgs/proxy/internal/retention" + "github.com/git-pkgs/registries/fetch" +) + +const ( + pubHTTPPath = "/packages/http/versions/1.2.0.tar.gz" + pubPathPath = "/packages/path/versions/1.9.0.tar.gz" +) + +// pubRetentionUpstream serves package archives the way pub.dev does, +// counting downloads. +func pubRetentionUpstream(t *testing.T) (*httptest.Server, *atomic.Int64) { + t.Helper() + var downloads atomic.Int64 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case pubHTTPPath, pubPathPath: + downloads.Add(1) + _, _ = io.WriteString(w, "pub archive bytes") + default: + http.NotFound(w, r) + } + })) + t.Cleanup(srv.Close) + return srv, &downloads +} + +func newPubRetentionProxy(t *testing.T, upstream *httptest.Server) (*httptest.Server, *database.DB) { + t.Helper() + db, store := setupEvictionTest(t) + // The upstream is a plain-HTTP loopback server; the default fetcher + // refuses loopback, a stock client reaches it. + fetcher := fetch.NewFetcher(fetch.WithHTTPClient(upstream.Client()), fetch.WithMaxRetries(0)) + t.Cleanup(func() { _ = fetcher.Close() }) + proxy := handler.NewProxy(db, store, fetcher, fetch.NewResolver(), discardLogger()) + proxy.HTTPClient = upstream.Client() + h := handler.NewPubHandlerWithUpstream(proxy, "http://proxy.test", upstream.URL) + srv := httptest.NewServer(h.Routes()) + t.Cleanup(srv.Close) + return srv, db +} + +func pubGet(t *testing.T, url string) { + t.Helper() + resp, err := http.Get(url) + if err != nil { + t.Fatalf("GET %s: %v", url, err) + } + body, _ := io.ReadAll(resp.Body) + _ = resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Fatalf("GET %s = %d %s, want 200", url, resp.StatusCode, body) + } +} + +// pubArtifact returns the record of the archive named filename. +func pubArtifact(t *testing.T, db *database.DB, filename string) database.Artifact { + t.Helper() + var art database.Artifact + if err := db.Get(&art, db.Rebind(`SELECT * FROM artifacts WHERE filename = ?`), filename); err != nil { + t.Fatalf("reading artifact %s: %v", filename, err) + } + return art +} + +func pubBackdate(t *testing.T, db *database.DB, filename string, fetched, accessed time.Time) { + t.Helper() + query := db.Rebind(`UPDATE artifacts SET fetched_at = ?, last_accessed_at = ? WHERE filename = ?`) + if _, err := db.Exec(query, fetched, accessed, filename); err != nil { + t.Fatalf("backdating %s: %v", filename, err) + } +} + +func TestPubRegistersForRetention(t *testing.T) { + if _, ok := retention.Default.Lookup("pub"); !ok { + t.Fatal("pub is not registered for retention") + } + if key, ok := retention.Default.KeyForDBEcosystem("pub"); !ok || key != "pub" { + t.Errorf("KeyForDBEcosystem(pub) = %q, %v, want pub, true", key, ok) + } +} + +// An archive served from the cache stays while it is used, is evicted once +// its rule runs out, comes back from upstream on the next download, and +// stays after that even with the evicted copy's old access time on the +// record. +func TestPubRetentionEvictsAndRefetches(t *testing.T) { + upstream, downloads := pubRetentionUpstream(t) + srv, db := newPubRetentionProxy(t, upstream) + const archive = "http-1.2.0.tar.gz" + + pubGet(t, srv.URL+pubHTTPPath) + pubGet(t, srv.URL+pubHTTPPath) + if got := downloads.Load(); got != 1 { + t.Fatalf("upstream downloads = %d, want 1", got) + } + if !pubArtifact(t, db, archive).LastAccessedAt.Valid { + t.Fatal("cache hit did not record an access") + } + + rules := retention.Rules{Ecosystems: map[string]time.Duration{"pub": retentionDay}} + now := time.Now() + sweepRetention(context.Background(), db, discardLogger(), retention.Default, rules, now) + if !pubArtifact(t, db, archive).StoragePath.Valid { + t.Fatal("archive used today was evicted") + } + + old := now.Add(-2 * retentionDay) + pubBackdate(t, db, archive, old, old) + sweepRetention(context.Background(), db, discardLogger(), retention.Default, rules, now) + if pubArtifact(t, db, archive).StoragePath.Valid { + t.Fatal("archive unused for two days kept under a one-day rule") + } + + pubGet(t, srv.URL+pubHTTPPath) + if got := downloads.Load(); got != 2 { + t.Errorf("upstream downloads = %d, want 2 after the eviction", got) + } + art := pubArtifact(t, db, archive) + pubBackdate(t, db, archive, art.FetchedAt.Time, now.Add(-30*retentionDay)) + sweepRetention(context.Background(), db, discardLogger(), retention.Default, rules, time.Now()) + if !pubArtifact(t, db, archive).StoragePath.Valid { + t.Error("refetched archive evicted again by the next sweep") + } +} + +// A key with a namespace parses as a pub PURL but can never match a stored +// package, so it is refused at validation. A plain key is accepted. +func TestPubRetentionRefusesNamespacedKeys(t *testing.T) { + cfg := config.Default() + cfg.Storage.Retention.Packages = map[string]string{"pkg:pub/http": "0"} + if _, err := cfg.RetentionRules(retention.Default); err != nil { + t.Fatalf("RetentionRules(pkg:pub/http): %v", err) + } + cfg.Storage.Retention.Packages = map[string]string{"pkg:pub/dart-lang/http": "0"} + _, err := cfg.RetentionRules(retention.Default) + if err == nil || !strings.Contains(err.Error(), "does not match how pub packages are stored") { + t.Errorf("RetentionRules(pkg:pub/dart-lang/http) error = %v, want the key refused", err) + } +} + +// A package rule written as a pub PURL reaches the stored package. +func TestPubRetentionPackageRule(t *testing.T) { + upstream, _ := pubRetentionUpstream(t) + srv, db := newPubRetentionProxy(t, upstream) + const httpArchive, pathArchive = "http-1.2.0.tar.gz", "path-1.9.0.tar.gz" + pubGet(t, srv.URL+pubHTTPPath) + pubGet(t, srv.URL+pubPathPath) + old := time.Now().Add(-30 * retentionDay) + pubBackdate(t, db, httpArchive, old, old) + pubBackdate(t, db, pathArchive, old, old) + + cfg := config.Default() + cfg.Storage.Retention.Ecosystems = map[string]string{"pub": "1d"} + cfg.Storage.Retention.Packages = map[string]string{"pkg:pub/http": "0"} + rules, err := cfg.RetentionRules(retention.Default) + if err != nil { + t.Fatalf("RetentionRules: %v", err) + } + sweepRetention(context.Background(), db, discardLogger(), retention.Default, rules, time.Now()) + + if !pubArtifact(t, db, httpArchive).StoragePath.Valid { + t.Error("http evicted despite its never-evict rule") + } + if pubArtifact(t, db, pathArchive).StoragePath.Valid { + t.Error("path kept past the pub rule") + } +} + +// pubRewriteTransport sends every request to the test upstream, so the +// mirror command's fixed pub.dev URLs reach it. +type pubRewriteTransport struct { + target *url.URL + base http.RoundTripper +} + +func (r pubRewriteTransport) RoundTrip(req *http.Request) (*http.Response, error) { + req = req.Clone(req.Context()) + req.URL.Scheme = r.target.Scheme + req.URL.Host = r.target.Host + req.Host = r.target.Host + return r.base.RoundTrip(req) +} + +// A package preloaded with the mirror command falls under the pub rule. +func TestPubRetentionCoversMirroredPackages(t *testing.T) { + upstream, downloads := pubRetentionUpstream(t) + target, err := url.Parse(upstream.URL) + if err != nil { + t.Fatal(err) + } + db, store := setupEvictionTest(t) + client := &http.Client{Transport: pubRewriteTransport{target: target, base: upstream.Client().Transport}} + fetcher := fetch.NewFetcher(fetch.WithHTTPClient(client), fetch.WithMaxRetries(0)) + t.Cleanup(func() { _ = fetcher.Close() }) + proxy := handler.NewProxy(db, store, fetcher, fetch.NewResolver(), discardLogger()) + + m := mirror.New(proxy, db, store, discardLogger(), 1) + progress, err := m.Run(context.Background(), &mirror.PURLSource{PURLs: []string{"pkg:pub/http@1.2.0"}}) + if err != nil || progress.Failed != 0 || downloads.Load() != 1 { + t.Fatalf("mirror run = %+v, %v, downloads %d; want one successful download", progress, err, downloads.Load()) + } + + const archive = "http-1.2.0.tar.gz" + if pubArtifact(t, db, archive).LastAccessedAt.Valid { + t.Fatal("mirrored archive has an access recorded before anyone downloaded it") + } + query := db.Rebind(`UPDATE artifacts SET fetched_at = ? WHERE filename = ?`) + if _, err := db.Exec(query, time.Now().Add(-30*retentionDay), archive); err != nil { + t.Fatalf("backdating %s: %v", archive, err) + } + rules := retention.Rules{Ecosystems: map[string]time.Duration{"pub": retentionDay}} + sweepRetention(context.Background(), db, discardLogger(), retention.Default, rules, time.Now()) + if pubArtifact(t, db, archive).StoragePath.Valid { + t.Error("never-served mirrored archive kept past the pub rule") + } +} From 42f459e961c62ed6f3bf8806193e48260917fb56 Mon Sep 17 00:00:00 2001 From: Christian Heim Date: Sun, 11 Oct 2026 01:21:03 +0200 Subject: [PATCH 227/633] Enable storage retention for rpm Packages are stored as pkg:rpm/{name}, without a vendor and with the name's case kept. Package rules may name a vendor the way PURLs usually do (pkg:rpm/fedora/curl); it's dropped. The docs row warns that Fedora and EPEL update repositories only keep the latest build. The mirror can't fetch rpm packages, so there's no mirror test. Refs #306 --- docs/configuration.md | 2 +- internal/handler/rpm.go | 21 +++ internal/handler/rpm_test.go | 31 ++++ internal/server/retention_rpm_test.go | 195 ++++++++++++++++++++++++++ 4 files changed, 248 insertions(+), 1 deletion(-) create mode 100644 internal/server/retention_rpm_test.go diff --git a/docs/configuration.md b/docs/configuration.md index 41ba4d0..8d2c564 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -106,7 +106,7 @@ A package rule wins over an ecosystem rule, which wins over `default`. Package r | `oci` | yes | `pkg:oci/library/nginx`, the repository name of the pull. A named upstream keeps its prefix, also when containerd reaches it through `ns`: `ghcr.io/owner/app` pulled that way is `pkg:oci/upstream/ghcr/owner/app`. Homebrew bottles use Homebrew's image name, with `@` written as `/` and `+` as `x` (`openssl@3` is `pkg:oci/homebrew/core/openssl/3`). Helm charts pulled over OCI are stored like images. Only blobs are evicted; manifests stay in the metadata cache. | | `pub` | no | - | | `pypi` | yes | `pkg:pypi/requests`; case and single `_`/`-` don't matter. Dots and repeated separators are kept as written, so a project like `zope.interface` can be stored as `zope.interface` and `zope-interface`; add a rule for each. | -| `rpm` | no | - | +| `rpm` | yes | `pkg:rpm/curl`, the name as packaged, case included (`pkg:rpm/NetworkManager`). A vendor namespace such as `pkg:rpm/fedora/curl` is accepted and dropped. Fedora and EPEL update repositories keep only the latest build of a package, so an evicted older one may not be fetchable again. | | `swift` | no | - | An evicted artifact is fetched again from upstream on the next request. Packages preloaded with `proxy mirror` age from the time they were mirrored, so an unused mirrored package is evicted like any other. Running `proxy mirror` again over a package that is still cached counts as a download and restarts its clock, so packages a scheduled mirror job keeps listing never age out. A registry that removes old releases (Linux distribution archives in particular) cannot serve an evicted version again, so set those ecosystems' rules with that in mind. diff --git a/internal/handler/rpm.go b/internal/handler/rpm.go index 6b138cc..3321ccc 100644 --- a/internal/handler/rpm.go +++ b/internal/handler/rpm.go @@ -5,6 +5,9 @@ import ( "net/http" "regexp" "strings" + + "github.com/git-pkgs/proxy/internal/retention" + "github.com/git-pkgs/purl" ) const ( @@ -13,6 +16,24 @@ const ( rpmMatchCount = 5 // full match + name + version + release + arch ) +// Packages are stored as pkg:rpm/{name}, without a vendor and with the name's +// case kept. Package rules may name a vendor the way PURLs usually do, +// pkg:rpm/fedora/curl; it is dropped. +func init() { + retention.Default.Register(retention.Spec{Key: "rpm", CanonicalPackage: rpmRetentionKey}) +} + +// rpmRetentionKey maps a package rule to the stored package PURL. A name with +// a slash, or a namespace with more than one segment, can never match a stored +// package and is refused. +func rpmRetentionKey(p *purl.PURL) (string, bool) { + if p == nil || p.Type != "rpm" || strings.Contains(p.Namespace, "/") || strings.Contains(p.Name, "/") { + return "", false + } + canonical := purl.MakePURLString("rpm", p.Name, "") + return canonical, canonical != "" +} + // RPMHandler handles RPM/Yum repository protocol requests. // It proxies requests to upstream RPM repositories and caches .rpm packages. type RPMHandler struct { diff --git a/internal/handler/rpm_test.go b/internal/handler/rpm_test.go index 851a304..3526dbd 100644 --- a/internal/handler/rpm_test.go +++ b/internal/handler/rpm_test.go @@ -2,6 +2,8 @@ package handler import ( "testing" + + "github.com/git-pkgs/purl" ) func TestRPMHandler_parseRPMPath(t *testing.T) { @@ -21,3 +23,32 @@ func TestRPMHandler_Routes(t *testing.T) { h := NewRPMHandler(nil, "http://localhost:8080") assertRoutesBasics(t, h.Routes(), "/repodata/repomd.xml", "/releases/../../../etc/passwd") } + +func TestRPMRetentionKey(t *testing.T) { + tests := []struct { + key string + want string + ok bool + }{ + {"pkg:rpm/curl", "pkg:rpm/curl", true}, + {"pkg:rpm/fedora/curl", "pkg:rpm/curl", true}, + {"pkg:rpm/Fedora/NetworkManager", "pkg:rpm/NetworkManager", true}, + {"pkg:rpm/fedora/libstdc++", "pkg:rpm/libstdc%2B%2B", true}, + {"pkg:rpm/fedora/40/curl", "", false}, + {"pkg:rpm/fedora/Packages%2Fcurl", "", false}, + {"pkg:deb/fedora/curl", "", false}, + } + for _, tt := range tests { + p, err := purl.Parse(tt.key) + if err != nil { + t.Fatalf("Parse(%s): %v", tt.key, err) + } + got, ok := rpmRetentionKey(p) + if ok != tt.ok || (ok && got != tt.want) { + t.Errorf("rpmRetentionKey(%s) = %q, %v, want %q, %v", tt.key, got, ok, tt.want, tt.ok) + } + } + if _, ok := rpmRetentionKey(nil); ok { + t.Error("rpmRetentionKey(nil) accepted") + } +} diff --git a/internal/server/retention_rpm_test.go b/internal/server/retention_rpm_test.go new file mode 100644 index 0000000..8c0536b --- /dev/null +++ b/internal/server/retention_rpm_test.go @@ -0,0 +1,195 @@ +package server + +import ( + "context" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/git-pkgs/proxy/internal/config" + "github.com/git-pkgs/proxy/internal/database" + "github.com/git-pkgs/proxy/internal/handler" + "github.com/git-pkgs/proxy/internal/retention" + "github.com/git-pkgs/registries/fetch" +) + +const ( + rpmCurlPath = "/updates/40/Everything/x86_64/Packages/c/curl-8.6.0-10.fc40.x86_64.rpm" + rpmNMPath = "/updates/40/Everything/x86_64/Packages/n/NetworkManager-1.46.2-1.fc40.x86_64.rpm" + rpmCurlFile = "curl-8.6.0-10.fc40.x86_64.rpm" + rpmNMFile = "NetworkManager-1.46.2-1.fc40.x86_64.rpm" +) + +// rpmRetentionUpstream serves package files the way a Fedora mirror does, +// counting downloads. +func rpmRetentionUpstream(t *testing.T) (*httptest.Server, *atomic.Int64) { + t.Helper() + var downloads atomic.Int64 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case rpmCurlPath, rpmNMPath: + downloads.Add(1) + _, _ = io.WriteString(w, "rpm bytes") + default: + http.NotFound(w, r) + } + })) + t.Cleanup(srv.Close) + return srv, &downloads +} + +func newRPMRetentionProxy(t *testing.T, upstream *httptest.Server) (*httptest.Server, *database.DB) { + t.Helper() + db, store := setupEvictionTest(t) + // The upstream is a plain-HTTP loopback server; the default fetcher + // refuses loopback, a stock client reaches it. + fetcher := fetch.NewFetcher(fetch.WithHTTPClient(upstream.Client()), fetch.WithMaxRetries(0)) + t.Cleanup(func() { _ = fetcher.Close() }) + proxy := handler.NewProxy(db, store, fetcher, fetch.NewResolver(), discardLogger()) + proxy.HTTPClient = upstream.Client() + h := handler.NewRPMHandlerWithUpstream(proxy, "http://proxy.test", upstream.URL) + srv := httptest.NewServer(h.Routes()) + t.Cleanup(srv.Close) + return srv, db +} + +func rpmGet(t *testing.T, url string) { + t.Helper() + resp, err := http.Get(url) + if err != nil { + t.Fatalf("GET %s: %v", url, err) + } + body, _ := io.ReadAll(resp.Body) + _ = resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Fatalf("GET %s = %d %s, want 200", url, resp.StatusCode, body) + } +} + +// rpmArtifact returns the record of the package file named filename. +func rpmArtifact(t *testing.T, db *database.DB, filename string) database.Artifact { + t.Helper() + var art database.Artifact + if err := db.Get(&art, db.Rebind(`SELECT * FROM artifacts WHERE filename = ?`), filename); err != nil { + t.Fatalf("reading artifact %s: %v", filename, err) + } + return art +} + +func rpmBackdate(t *testing.T, db *database.DB, filename string, fetched, accessed time.Time) { + t.Helper() + query := db.Rebind(`UPDATE artifacts SET fetched_at = ?, last_accessed_at = ? WHERE filename = ?`) + if _, err := db.Exec(query, fetched, accessed, filename); err != nil { + t.Fatalf("backdating %s: %v", filename, err) + } +} + +func TestRPMRegistersForRetention(t *testing.T) { + if _, ok := retention.Default.Lookup("rpm"); !ok { + t.Fatal("rpm is not registered for retention") + } + if key, ok := retention.Default.KeyForDBEcosystem("rpm"); !ok || key != "rpm" { + t.Errorf("KeyForDBEcosystem(rpm) = %q, %v, want rpm, true", key, ok) + } +} + +// A package served from the cache stays while it is used, is evicted once +// its rule runs out, comes back from upstream on the next download, and +// stays after that even with the evicted copy's old access time on the +// record. +func TestRPMRetentionEvictsAndRefetches(t *testing.T) { + upstream, downloads := rpmRetentionUpstream(t) + srv, db := newRPMRetentionProxy(t, upstream) + + rpmGet(t, srv.URL+rpmCurlPath) + rpmGet(t, srv.URL+rpmCurlPath) + if got := downloads.Load(); got != 1 { + t.Fatalf("upstream downloads = %d, want 1", got) + } + if !rpmArtifact(t, db, rpmCurlFile).LastAccessedAt.Valid { + t.Fatal("cache hit did not record an access") + } + + rules := retention.Rules{Ecosystems: map[string]time.Duration{"rpm": retentionDay}} + now := time.Now() + sweepRetention(context.Background(), db, discardLogger(), retention.Default, rules, now) + if !rpmArtifact(t, db, rpmCurlFile).StoragePath.Valid { + t.Fatal("package used today was evicted") + } + + old := now.Add(-2 * retentionDay) + rpmBackdate(t, db, rpmCurlFile, old, old) + sweepRetention(context.Background(), db, discardLogger(), retention.Default, rules, now) + if rpmArtifact(t, db, rpmCurlFile).StoragePath.Valid { + t.Fatal("package unused for two days kept under a one-day rule") + } + + rpmGet(t, srv.URL+rpmCurlPath) + if got := downloads.Load(); got != 2 { + t.Errorf("upstream downloads = %d, want 2 after the eviction", got) + } + art := rpmArtifact(t, db, rpmCurlFile) + rpmBackdate(t, db, rpmCurlFile, art.FetchedAt.Time, now.Add(-30*retentionDay)) + sweepRetention(context.Background(), db, discardLogger(), retention.Default, rules, time.Now()) + if !rpmArtifact(t, db, rpmCurlFile).StoragePath.Valid { + t.Error("refetched package evicted again by the next sweep") + } +} + +// Keys that parse as rpm PURLs but can never match a stored package are +// refused at validation. Keys with and without a vendor are accepted. +func TestRPMRetentionRefusesUnmatchableKeys(t *testing.T) { + cfg := config.Default() + cfg.Storage.Retention.Packages = map[string]string{"pkg:rpm/curl": "0", "pkg:rpm/fedora/NetworkManager": "0"} + if _, err := cfg.RetentionRules(retention.Default); err != nil { + t.Fatalf("RetentionRules: %v", err) + } + cfg.Storage.Retention.Packages = map[string]string{"pkg:rpm/fedora/40/curl": "0"} + _, err := cfg.RetentionRules(retention.Default) + if err == nil || !strings.Contains(err.Error(), "does not match how rpm packages are stored") { + t.Errorf("RetentionRules(pkg:rpm/fedora/40/curl) error = %v, want the key refused", err) + } +} + +// A package rule reaches the package with or without a vendor, and matches +// the name's case: pkg:rpm/networkmanager doesn't cover NetworkManager. +func TestRPMRetentionPackageRules(t *testing.T) { + upstream, _ := rpmRetentionUpstream(t) + srv, db := newRPMRetentionProxy(t, upstream) + + tests := []struct { + key string + keepNM bool + }{ + {"pkg:rpm/NetworkManager", true}, + {"pkg:rpm/fedora/NetworkManager", true}, + {"pkg:rpm/networkmanager", false}, + } + for _, tt := range tests { + old := time.Now().Add(-30 * retentionDay) + for _, path := range []string{rpmCurlPath, rpmNMPath} { + rpmGet(t, srv.URL+path) + } + rpmBackdate(t, db, rpmCurlFile, old, old) + rpmBackdate(t, db, rpmNMFile, old, old) + cfg := config.Default() + cfg.Storage.Retention.Ecosystems = map[string]string{"rpm": "1d"} + cfg.Storage.Retention.Packages = map[string]string{tt.key: "0"} + rules, err := cfg.RetentionRules(retention.Default) + if err != nil { + t.Fatalf("RetentionRules(%s): %v", tt.key, err) + } + sweepRetention(context.Background(), db, discardLogger(), retention.Default, rules, time.Now()) + + if got := rpmArtifact(t, db, rpmNMFile).StoragePath.Valid; got != tt.keepNM { + t.Errorf("with %s: NetworkManager cached = %v, want %v", tt.key, got, tt.keepNM) + } + if rpmArtifact(t, db, rpmCurlFile).StoragePath.Valid { + t.Errorf("with %s: curl kept past the rpm rule", tt.key) + } + } +} From dcfb9a820df0e692b59f3f1c21a0bc6bcb6866e0 Mon Sep 17 00:00:00 2001 From: Christian Heim Date: Sat, 10 Oct 2026 22:55:09 +0200 Subject: [PATCH 228/633] Enable storage retention for nuget The handler stores NuGet ids lowercased, but purl keeps nuget names as written, so package rules get lowercased before they're matched. Case in a rule doesn't matter. The mirror stores ids as given, so a mixed-case PURL ends up as a copy clients never get. Until that's fixed, the docs ask for lowercase ids in mirror lists. Refs #306 --- docs/configuration.md | 2 +- internal/handler/nuget.go | 19 ++ internal/handler/nuget_test.go | 27 +++ internal/server/retention_nuget_test.go | 250 ++++++++++++++++++++++++ 4 files changed, 297 insertions(+), 1 deletion(-) create mode 100644 internal/server/retention_nuget_test.go diff --git a/docs/configuration.md b/docs/configuration.md index 41ba4d0..9766f30 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -102,7 +102,7 @@ A package rule wins over an ecosystem rule, which wins over `default`. Package r | `julia` | no | - | | `maven` | yes | `pkg:maven/com.google.guava/guava`, group and artifact as published, case included. Gradle build cache entries are not covered; they use `gradle.build_cache.max_age`. | | `npm` | yes | `pkg:npm/lodash`, `pkg:npm/@babel/core`; the name has to match exactly, case included, so `pkg:npm/jsonstream` doesn't cover `JSONStream`. | -| `nuget` | no | - | +| `nuget` | yes | `pkg:nuget/Newtonsoft.Json`; case doesn't matter. Give `proxy mirror` lowercase ids (`pkg:nuget/newtonsoft.json@13.0.3`): a mixed-case id is stored as a separate copy that clients are never served, so mirroring it doesn't keep their copy from aging out. | | `oci` | yes | `pkg:oci/library/nginx`, the repository name of the pull. A named upstream keeps its prefix, also when containerd reaches it through `ns`: `ghcr.io/owner/app` pulled that way is `pkg:oci/upstream/ghcr/owner/app`. Homebrew bottles use Homebrew's image name, with `@` written as `/` and `+` as `x` (`openssl@3` is `pkg:oci/homebrew/core/openssl/3`). Helm charts pulled over OCI are stored like images. Only blobs are evicted; manifests stay in the metadata cache. | | `pub` | no | - | | `pypi` | yes | `pkg:pypi/requests`; case and single `_`/`-` don't matter. Dots and repeated separators are kept as written, so a project like `zope.interface` can be stored as `zope.interface` and `zope-interface`; add a rule for each. | diff --git a/internal/handler/nuget.go b/internal/handler/nuget.go index 612f32e..1a3d0a1 100644 --- a/internal/handler/nuget.go +++ b/internal/handler/nuget.go @@ -6,6 +6,9 @@ import ( "fmt" "net/http" "strings" + + "github.com/git-pkgs/proxy/internal/retention" + "github.com/git-pkgs/purl" ) const ( @@ -13,6 +16,22 @@ const ( nugetSearchUpstream = "https://azuresearch-usnc.nuget.org" ) +// Packages are stored under pkg:nuget/{id} with the id lowercased, since +// NuGet ids are case-insensitive. purl keeps nuget names as written, so +// package rules are lowercased the same way before the default mapping. +func init() { + retention.Default.Register(retention.Spec{Key: "nuget", CanonicalPackage: nugetRetentionKey}) +} + +// nugetRetentionKey maps a package rule to the stored package PURL, so +// pkg:nuget/Newtonsoft.Json matches the stored pkg:nuget/newtonsoft.json. +func nugetRetentionKey(p *purl.PURL) (string, bool) { + if p == nil { + return "", false + } + return retention.DefaultCanonical("nuget")(purl.New(p.Type, p.Namespace, strings.ToLower(p.Name), "", nil)) +} + // NuGetHandler handles NuGet V3 API protocol requests. type NuGetHandler struct { proxy *Proxy diff --git a/internal/handler/nuget_test.go b/internal/handler/nuget_test.go index 710b8ea..5ae77dc 100644 --- a/internal/handler/nuget_test.go +++ b/internal/handler/nuget_test.go @@ -11,6 +11,7 @@ import ( "time" "github.com/git-pkgs/cooldown" + "github.com/git-pkgs/purl" ) func nugetTestProxy() *Proxy { @@ -1063,3 +1064,29 @@ func TestNuGetHandleRegistrationWithoutCooldown(t *testing.T) { t.Fatalf("status = %d, want %d", w.Code, http.StatusOK) } } + +func TestNuGetRetentionKey(t *testing.T) { + tests := []struct { + key string + want string + ok bool + }{ + {"pkg:nuget/Newtonsoft.Json", "pkg:nuget/newtonsoft.json", true}, + {"pkg:nuget/serilog", "pkg:nuget/serilog", true}, + {"pkg:nuget/serilog/serilog", "", false}, + {"pkg:npm/serilog", "", false}, + } + for _, tt := range tests { + p, err := purl.Parse(tt.key) + if err != nil { + t.Fatalf("Parse(%s): %v", tt.key, err) + } + got, ok := nugetRetentionKey(p) + if ok != tt.ok || (ok && got != tt.want) { + t.Errorf("nugetRetentionKey(%s) = %q, %v, want %q, %v", tt.key, got, ok, tt.want, tt.ok) + } + } + if _, ok := nugetRetentionKey(nil); ok { + t.Error("nugetRetentionKey(nil) accepted") + } +} diff --git a/internal/server/retention_nuget_test.go b/internal/server/retention_nuget_test.go new file mode 100644 index 0000000..44d45bb --- /dev/null +++ b/internal/server/retention_nuget_test.go @@ -0,0 +1,250 @@ +package server + +import ( + "context" + "io" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/git-pkgs/proxy/internal/config" + "github.com/git-pkgs/proxy/internal/database" + "github.com/git-pkgs/proxy/internal/handler" + "github.com/git-pkgs/proxy/internal/mirror" + "github.com/git-pkgs/proxy/internal/retention" + "github.com/git-pkgs/registries/fetch" +) + +const ( + nugetJSONPath = "/v3-flatcontainer/newtonsoft.json/13.0.3/newtonsoft.json.13.0.3.nupkg" + nugetSerilogPath = "/v3-flatcontainer/serilog/3.1.1/serilog.3.1.1.nupkg" +) + +// nugetRetentionUpstream serves packages the way the NuGet flat container +// does, counting downloads. +func nugetRetentionUpstream(t *testing.T) (*httptest.Server, *atomic.Int64) { + t.Helper() + var downloads atomic.Int64 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case nugetJSONPath, nugetSerilogPath: + downloads.Add(1) + _, _ = io.WriteString(w, "nupkg bytes") + default: + http.NotFound(w, r) + } + })) + t.Cleanup(srv.Close) + return srv, &downloads +} + +func newNuGetRetentionProxy(t *testing.T, upstream *httptest.Server) (*httptest.Server, *database.DB) { + t.Helper() + db, store := setupEvictionTest(t) + // The upstream is a plain-HTTP loopback server; the default fetcher + // refuses loopback, a stock client reaches it. + fetcher := fetch.NewFetcher(fetch.WithHTTPClient(upstream.Client()), fetch.WithMaxRetries(0)) + t.Cleanup(func() { _ = fetcher.Close() }) + proxy := handler.NewProxy(db, store, fetcher, fetch.NewResolver(), discardLogger()) + proxy.HTTPClient = upstream.Client() + h := handler.NewNuGetHandlerWithUpstreams(proxy, "http://proxy.test", upstream.URL, upstream.URL) + srv := httptest.NewServer(h.Routes()) + t.Cleanup(srv.Close) + return srv, db +} + +func nugetGet(t *testing.T, url string) { + t.Helper() + resp, err := http.Get(url) + if err != nil { + t.Fatalf("GET %s: %v", url, err) + } + body, _ := io.ReadAll(resp.Body) + _ = resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Fatalf("GET %s = %d %s, want 200", url, resp.StatusCode, body) + } +} + +// nugetArtifact returns the record of the package file named filename. +func nugetArtifact(t *testing.T, db *database.DB, filename string) database.Artifact { + t.Helper() + var art database.Artifact + if err := db.Get(&art, db.Rebind(`SELECT * FROM artifacts WHERE filename = ?`), filename); err != nil { + t.Fatalf("reading artifact %s: %v", filename, err) + } + return art +} + +func nugetBackdate(t *testing.T, db *database.DB, filename string, fetched, accessed time.Time) { + t.Helper() + query := db.Rebind(`UPDATE artifacts SET fetched_at = ?, last_accessed_at = ? WHERE filename = ?`) + if _, err := db.Exec(query, fetched, accessed, filename); err != nil { + t.Fatalf("backdating %s: %v", filename, err) + } +} + +func TestNuGetRegistersForRetention(t *testing.T) { + if _, ok := retention.Default.Lookup("nuget"); !ok { + t.Fatal("nuget is not registered for retention") + } + if key, ok := retention.Default.KeyForDBEcosystem("nuget"); !ok || key != "nuget" { + t.Errorf("KeyForDBEcosystem(nuget) = %q, %v, want nuget, true", key, ok) + } +} + +// A package served from the cache stays while it is used, is evicted once +// its rule runs out, comes back from upstream on the next download, and +// stays after that even with the evicted copy's old access time on the +// record. +func TestNuGetRetentionEvictsAndRefetches(t *testing.T) { + upstream, downloads := nugetRetentionUpstream(t) + srv, db := newNuGetRetentionProxy(t, upstream) + const nupkg = "serilog.3.1.1.nupkg" + + nugetGet(t, srv.URL+nugetSerilogPath) + nugetGet(t, srv.URL+nugetSerilogPath) + if got := downloads.Load(); got != 1 { + t.Fatalf("upstream downloads = %d, want 1", got) + } + if !nugetArtifact(t, db, nupkg).LastAccessedAt.Valid { + t.Fatal("cache hit did not record an access") + } + + rules := retention.Rules{Ecosystems: map[string]time.Duration{"nuget": retentionDay}} + now := time.Now() + sweepRetention(context.Background(), db, discardLogger(), retention.Default, rules, now) + if !nugetArtifact(t, db, nupkg).StoragePath.Valid { + t.Fatal("package used today was evicted") + } + + old := now.Add(-2 * retentionDay) + nugetBackdate(t, db, nupkg, old, old) + sweepRetention(context.Background(), db, discardLogger(), retention.Default, rules, now) + if nugetArtifact(t, db, nupkg).StoragePath.Valid { + t.Fatal("package unused for two days kept under a one-day rule") + } + + nugetGet(t, srv.URL+nugetSerilogPath) + if got := downloads.Load(); got != 2 { + t.Errorf("upstream downloads = %d, want 2 after the eviction", got) + } + art := nugetArtifact(t, db, nupkg) + nugetBackdate(t, db, nupkg, art.FetchedAt.Time, now.Add(-30*retentionDay)) + sweepRetention(context.Background(), db, discardLogger(), retention.Default, rules, time.Now()) + if !nugetArtifact(t, db, nupkg).StoragePath.Valid { + t.Error("refetched package evicted again by the next sweep") + } +} + +// A key with a namespace parses as a nuget PURL but can never match a +// stored package, so it is refused at validation. A plain key is accepted. +func TestNuGetRetentionRefusesNamespacedKeys(t *testing.T) { + cfg := config.Default() + cfg.Storage.Retention.Packages = map[string]string{"pkg:nuget/Serilog": "0"} + if _, err := cfg.RetentionRules(retention.Default); err != nil { + t.Fatalf("RetentionRules(pkg:nuget/Serilog): %v", err) + } + cfg.Storage.Retention.Packages = map[string]string{"pkg:nuget/serilog/serilog": "0"} + _, err := cfg.RetentionRules(retention.Default) + if err == nil || !strings.Contains(err.Error(), "does not match how nuget packages are stored") { + t.Errorf("RetentionRules(pkg:nuget/serilog/serilog) error = %v, want the key refused", err) + } +} + +// The handler stores package ids lowercased, and a package rule reaches the +// package whatever case its id is written in. +func TestNuGetRetentionPackageRules(t *testing.T) { + upstream, _ := nugetRetentionUpstream(t) + srv, db := newNuGetRetentionProxy(t, upstream) + const newtonsoft, serilog = "newtonsoft.json.13.0.3.nupkg", "serilog.3.1.1.nupkg" + + // The id in the URL is upper case here; nuget clients lowercase it, but + // the handler stores it lowercased either way. + nugetGet(t, srv.URL+"/v3-flatcontainer/Newtonsoft.Json/13.0.3/newtonsoft.json.13.0.3.nupkg") + var stored string + art := nugetArtifact(t, db, newtonsoft) + if err := db.Get(&stored, db.Rebind(`SELECT package_purl FROM versions WHERE purl = ?`), art.VersionPURL); err != nil { + t.Fatalf("reading stored package PURL: %v", err) + } + if stored != "pkg:nuget/newtonsoft.json" { + t.Errorf("stored package PURL = %q, want pkg:nuget/newtonsoft.json", stored) + } + + for _, key := range []string{"pkg:nuget/Newtonsoft.Json", "pkg:nuget/newtonsoft.json"} { + nugetGet(t, srv.URL+nugetJSONPath) + nugetGet(t, srv.URL+nugetSerilogPath) + old := time.Now().Add(-30 * retentionDay) + nugetBackdate(t, db, newtonsoft, old, old) + nugetBackdate(t, db, serilog, old, old) + + cfg := config.Default() + cfg.Storage.Retention.Ecosystems = map[string]string{"nuget": "1d"} + cfg.Storage.Retention.Packages = map[string]string{key: "0"} + rules, err := cfg.RetentionRules(retention.Default) + if err != nil { + t.Fatalf("RetentionRules(%s): %v", key, err) + } + sweepRetention(context.Background(), db, discardLogger(), retention.Default, rules, time.Now()) + + if !nugetArtifact(t, db, newtonsoft).StoragePath.Valid { + t.Errorf("with %s: Newtonsoft.Json evicted despite its never-evict rule", key) + } + if nugetArtifact(t, db, serilog).StoragePath.Valid { + t.Errorf("with %s: Serilog kept past the nuget rule", key) + } + } +} + +// nugetRewriteTransport sends every request to the test upstream, so the +// mirror command's fixed api.nuget.org URLs reach it. +type nugetRewriteTransport struct { + target *url.URL + base http.RoundTripper +} + +func (r nugetRewriteTransport) RoundTrip(req *http.Request) (*http.Response, error) { + req = req.Clone(req.Context()) + req.URL.Scheme = r.target.Scheme + req.URL.Host = r.target.Host + req.Host = r.target.Host + return r.base.RoundTrip(req) +} + +// A package preloaded with the mirror command falls under the nuget rule. +func TestNuGetRetentionCoversMirroredPackages(t *testing.T) { + upstream, downloads := nugetRetentionUpstream(t) + target, err := url.Parse(upstream.URL) + if err != nil { + t.Fatal(err) + } + db, store := setupEvictionTest(t) + client := &http.Client{Transport: nugetRewriteTransport{target: target, base: upstream.Client().Transport}} + fetcher := fetch.NewFetcher(fetch.WithHTTPClient(client), fetch.WithMaxRetries(0)) + t.Cleanup(func() { _ = fetcher.Close() }) + proxy := handler.NewProxy(db, store, fetcher, fetch.NewResolver(), discardLogger()) + + m := mirror.New(proxy, db, store, discardLogger(), 1) + progress, err := m.Run(context.Background(), &mirror.PURLSource{PURLs: []string{"pkg:nuget/serilog@3.1.1"}}) + if err != nil || progress.Failed != 0 || downloads.Load() != 1 { + t.Fatalf("mirror run = %+v, %v, downloads %d; want one successful download", progress, err, downloads.Load()) + } + + const nupkg = "serilog.3.1.1.nupkg" + if nugetArtifact(t, db, nupkg).LastAccessedAt.Valid { + t.Fatal("mirrored package has an access recorded before anyone downloaded it") + } + query := db.Rebind(`UPDATE artifacts SET fetched_at = ? WHERE filename = ?`) + if _, err := db.Exec(query, time.Now().Add(-30*retentionDay), nupkg); err != nil { + t.Fatalf("backdating %s: %v", nupkg, err) + } + rules := retention.Rules{Ecosystems: map[string]time.Duration{"nuget": retentionDay}} + sweepRetention(context.Background(), db, discardLogger(), retention.Default, rules, time.Now()) + if nugetArtifact(t, db, nupkg).StoragePath.Valid { + t.Error("never-served mirrored package kept past the nuget rule") + } +} From 9f424256460378f3d47bbb1f7ea20bb5ad347402 Mon Sep 17 00:00:00 2001 From: simon chrzanowski Date: Mon, 12 Oct 2026 16:11:49 +0200 Subject: [PATCH 229/633] Test upgrading cached metadata through the metadata_dates migration - Start from the schema before migration 011 on SQLite and PostgreSQL, with npm and Composer metadata cached, and check the rows and stored bytes survive the migration unchanged - After the upgrade, check two replicas serve the cached document with matching ETag and Last-Modified and answer conditional requests with 304 - Change the upstream document and check neither replica answers the old validators with 304, and both agree on the new ones - Give the PostgreSQL handler test a database of its own, so it shares no tables with the database package tests on the same server --- internal/database/metadata_dates_test.go | 71 +++++- .../handler/metadata_dates_upgrade_test.go | 234 ++++++++++++++++++ 2 files changed, 294 insertions(+), 11 deletions(-) create mode 100644 internal/handler/metadata_dates_upgrade_test.go diff --git a/internal/database/metadata_dates_test.go b/internal/database/metadata_dates_test.go index 864723e..66c178a 100644 --- a/internal/database/metadata_dates_test.go +++ b/internal/database/metadata_dates_test.go @@ -1,7 +1,8 @@ package database import ( - "path/filepath" + "database/sql" + "strings" "testing" "time" ) @@ -106,23 +107,71 @@ func TestMetadataDateIsUTCSeconds(t *testing.T) { }) } -func TestMetadataDatesTableCreatedByMigration(t *testing.T) { - db, err := Create(filepath.Join(t.TempDir(), "test.db")) +// TestMetadataDatesMigrationKeepsCachedMetadata upgrades a database from +// before migration 011 that holds cached npm and Composer metadata, and checks +// that the cached rows come through unchanged and dates can be recorded. +func TestMetadataDatesMigrationKeepsCachedMetadata(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + rollBackMetadataDates(t, db) + seeded := map[string]string{testEcosystemNPM: "left-pad", "composer": "vendor/pkg"} + before := make(map[string]*MetadataCacheEntry, len(seeded)) + for ecosystem, name := range seeded { + before[ecosystem] = seedMetadataCache(t, db, ecosystem, name) + } + + if err := db.MigrateSchema(); err != nil { + t.Fatalf("MigrateSchema() error = %v", err) + } + + for ecosystem, name := range seeded { + after, err := db.GetMetadataCache(ecosystem, name) + if err != nil || after == nil || *after != *before[ecosystem] { + t.Errorf("metadata row %s/%s after migration = %+v, %v; want %+v", ecosystem, name, after, err, before[ecosystem]) + } + date := MetadataDate{ETag: `"a"`, LastModified: time.Date(2026, 10, 10, 12, 0, 1, 0, time.UTC)} + if inserted, err := db.InsertMetadataDate(ecosystem, testProxyURL, name, date); err != nil || !inserted { + t.Errorf("InsertMetadataDate(%s/%s) after migration = %v, %v; want true", ecosystem, name, inserted, err) + } + } + }) +} + +// seedMetadataCache stores a metadata row as the metadata cache writes one +// and returns it as read back. +func seedMetadataCache(t *testing.T, db *DB, ecosystem, name string) *MetadataCacheEntry { + t.Helper() + err := db.UpsertMetadataCache(&MetadataCacheEntry{ + Ecosystem: ecosystem, + Name: name, + StoragePath: "_metadata/" + ecosystem + "/" + name + "/metadata", + ETag: sql.NullString{String: `"upstream"`, Valid: true}, + ContentType: sql.NullString{String: "application/json", Valid: true}, + ContentDigest: sql.NullString{String: "sha256:" + strings.Repeat("ab", 32), Valid: true}, + Size: sql.NullInt64{Int64: 123, Valid: true}, + LastModified: sql.NullTime{Time: time.Date(2020, 1, 2, 3, 4, 5, 0, time.UTC), Valid: true}, + FetchedAt: sql.NullTime{Time: time.Now().UTC().Truncate(time.Second), Valid: true}, + }) if err != nil { - t.Fatalf("Create failed: %v", err) + t.Fatal(err) } - defer func() { _ = db.Close() }() + entry, err := db.GetMetadataCache(ecosystem, name) + if err != nil || entry == nil { + t.Fatalf("GetMetadataCache(%s/%s) = %v, %v", ecosystem, name, entry, err) + } + return entry +} + +// rollBackMetadataDates returns db to the schema from before migration 011, +// which only added the metadata_dates table. +func rollBackMetadataDates(t *testing.T, db *DB) { + t.Helper() if _, err := db.Exec("DROP TABLE metadata_dates"); err != nil { t.Fatal(err) } if _, err := db.Exec("DELETE FROM migrations WHERE name = '011_add_metadata_dates'"); err != nil { t.Fatal(err) } - - if err := db.MigrateSchema(); err != nil { - t.Fatalf("MigrateSchema() error = %v", err) - } - if has, err := db.HasTable("metadata_dates"); err != nil || !has { - t.Errorf("HasTable(metadata_dates) = %v, %v; want true after migration", has, err) + if has, err := db.HasTable("metadata_dates"); err != nil || has { + t.Fatalf("HasTable(metadata_dates) after rollback = %v, %v; want false", has, err) } } diff --git a/internal/handler/metadata_dates_upgrade_test.go b/internal/handler/metadata_dates_upgrade_test.go new file mode 100644 index 0000000..4d1ca74 --- /dev/null +++ b/internal/handler/metadata_dates_upgrade_test.go @@ -0,0 +1,234 @@ +package handler + +import ( + "context" + "crypto/rand" + "encoding/hex" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "net/url" + "os" + "strings" + "testing" + "time" + + "github.com/git-pkgs/proxy/internal/database" + "github.com/git-pkgs/registries/fetch" +) + +// upgradeDatabases opens an empty database with the current schema for each +// dialect the proxy supports. A nil database means the dialect is unavailable. +var upgradeDatabases = []struct { + dialect string + open func(t *testing.T) *database.DB +}{ + {"sqlite", func(t *testing.T) *database.DB { + db, err := database.Create(t.TempDir() + "/test.db") + if err != nil { + t.Fatalf("Create failed: %v", err) + } + t.Cleanup(func() { _ = db.Close() }) + return db + }}, + {"postgres", openTestPostgres}, +} + +// openTestPostgres creates a database of its own on the server at +// PROXY_DATABASE_URL, so the test shares no tables with tests elsewhere that +// use the same server, and drops it afterwards. +func openTestPostgres(t *testing.T) *database.DB { + t.Helper() + serverURL := os.Getenv("PROXY_DATABASE_URL") + if serverURL == "" { + return nil + } + server, err := database.OpenPostgres(serverURL) + if err != nil { + t.Fatalf("OpenPostgres failed: %v", err) + } + t.Cleanup(func() { _ = server.Close() }) + + suffix := make([]byte, 8) + _, _ = rand.Read(suffix) + name := "proxy_upgrade_" + hex.EncodeToString(suffix) + if _, err := server.Exec("CREATE DATABASE " + name); err != nil { + t.Fatalf("creating database %s: %v", name, err) + } + t.Cleanup(func() { _, _ = server.Exec("DROP DATABASE IF EXISTS " + name + " WITH (FORCE)") }) + + dbURL, err := url.Parse(serverURL) + if err != nil { + t.Fatal(err) + } + dbURL.Path = "/" + name + db, err := database.OpenPostgres(dbURL.String()) + if err != nil { + t.Fatalf("OpenPostgres(%s) failed: %v", name, err) + } + t.Cleanup(func() { _ = db.Close() }) + if err := db.CreateSchema(); err != nil { + t.Fatalf("CreateSchema failed: %v", err) + } + return db +} + +// rollBackMetadataDates returns db to the schema from before migration 011, +// which only added the metadata_dates table. +func rollBackMetadataDates(t *testing.T, db *database.DB) { + t.Helper() + if _, err := db.Exec("DROP TABLE metadata_dates"); err != nil { + t.Fatal(err) + } + if _, err := db.Exec("DELETE FROM migrations WHERE name = '011_add_metadata_dates'"); err != nil { + t.Fatal(err) + } +} + +// upgradeProxy returns a proxy on db that dates metadata by a clock the test +// moves, a second after the process start it models. +func upgradeProxy(t *testing.T, db *database.DB, u *validatorUpstream) (*Proxy, *testClock) { + t.Helper() + proxy := NewProxy(db, newMockStorage(), &mockFetcher{}, fetch.NewResolver(), slog.New(slog.NewTextHandler(io.Discard, nil))) + proxy.HTTPClient = u.Client() + proxy.CacheMetadata = true + proxy.MetadataTTL = time.Hour + clock := &testClock{t: time.Date(2026, 10, 10, 12, 0, 0, 0, time.UTC)} + restartProxy(proxy, clock) + clock.advance(time.Second) + return proxy, clock +} + +// cachedMetadata is a metadata cache row and the bytes stored for it. +type cachedMetadata struct { + entry database.MetadataCacheEntry + body string +} + +func readCachedMetadata(t *testing.T, proxy *Proxy, ecosystem, name string) cachedMetadata { + t.Helper() + entry, err := proxy.DB.GetMetadataCache(ecosystem, name) + if err != nil || entry == nil { + t.Fatalf("metadata row %s/%s = %v, %v", ecosystem, name, entry, err) + } + r, err := proxy.Storage.Open(context.Background(), entry.StoragePath) + if err != nil { + t.Fatalf("opening stored metadata %s: %v", entry.StoragePath, err) + } + defer func() { _ = r.Close() }() + body, err := io.ReadAll(r) + if err != nil { + t.Fatal(err) + } + return cachedMetadata{entry: *entry, body: string(body)} +} + +// TestRewrittenMetadataAfterMetadataDatesMigration upgrades a database from +// before migration 011 that holds cached npm and Composer metadata. The cached +// rows and stored bytes must come through unchanged, and two replicas sharing +// the upgraded database must serve the cached document with validators that +// revalidate, then date a changed document alike. +func TestRewrittenMetadataAfterMetadataDatesMigration(t *testing.T) { + for _, d := range upgradeDatabases { + for _, c := range validatorCases { + t.Run(d.dialect+"/"+c.ecosystem, func(t *testing.T) { + db := d.open(t) + if db == nil { + t.Skip("PROXY_DATABASE_URL not set, skipping postgres test") + } + u := newValidatorUpstream(t) + a, clock := upgradeProxy(t, db, u) + upgradeWithCachedMetadata(t, a, u, c) + + replicas := map[string]http.Handler{"A": c.handler(a, u), "B": c.handler(validatorReplica(a, clock), u)} + cached := serve(replicas["A"], c.path, nil) + if cached.Code != http.StatusOK || strings.Contains(cached.Body.String(), "2.0.0") { + t.Fatalf("first request after the upgrade: status %d, body %s; want 200 with the document cached before it", cached.Code, cached.Body) + } + expectRevalidates(t, replicas, c.path, cached) + + expireMetadata(t, db, c.ecosystem, c.name) + clock.advance(time.Second) + changed := serve(replicas["A"], c.path, nil) + expectChangedDocument(t, cached, changed) + expectChangedForCachedValidators(t, replicas, c.path, cached, changed) + expectRevalidates(t, replicas, c.path, changed) + }) + } + } +} + +// upgradeWithCachedMetadata rolls proxy's database back to before migration +// 011, caches c's document as the proxy did then, moves upstream on to version +// 2.0.0 and migrates. The cached row and stored bytes must survive. +func upgradeWithCachedMetadata(t *testing.T, proxy *Proxy, u *validatorUpstream, c validatorCase) { + t.Helper() + rollBackMetadataDates(t, proxy.DB) + if _, _, err := proxy.FetchOrCacheMetadata(context.Background(), c.ecosystem, c.name, u.URL+c.path); err != nil { + t.Fatalf("seeding metadata before the upgrade: %v", err) + } + seeded := readCachedMetadata(t, proxy, c.ecosystem, c.name) + u.set(npmV("2.0.0"), composerV("2.0.0")) + + if err := proxy.DB.MigrateSchema(); err != nil { + t.Fatalf("MigrateSchema() error = %v", err) + } + if upgraded := readCachedMetadata(t, proxy, c.ecosystem, c.name); upgraded != seeded { + t.Fatalf("cached metadata after the upgrade = %+v, want %+v", upgraded, seeded) + } +} + +// expectChangedDocument checks that changed is the new upstream document, +// with an ETag of its own and a later date than cached. +func expectChangedDocument(t *testing.T, cached, changed *httptest.ResponseRecorder) { + t.Helper() + if changed.Code != http.StatusOK || !strings.Contains(changed.Body.String(), "2.0.0") { + t.Fatalf("after upstream changed: status %d, body %s; want 200 with the new document", changed.Code, changed.Body) + } + if changed.Header().Get("ETag") == cached.Header().Get("ETag") { + t.Error("the changed document has the cached document's ETag") + } + was, _ := http.ParseTime(cached.Header().Get("Last-Modified")) + if now, err := http.ParseTime(changed.Header().Get("Last-Modified")); err != nil || !now.After(was) { + t.Errorf("changed document Last-Modified %q, want later than %q", changed.Header().Get("Last-Modified"), cached.Header().Get("Last-Modified")) + } +} + +// expectChangedForCachedValidators checks that no replica answers a client +// holding the cached document's validators with 304 once it changed. +func expectChangedForCachedValidators(t *testing.T, replicas map[string]http.Handler, path string, cached, changed *httptest.ResponseRecorder) { + t.Helper() + held := []http.Header{ + {"If-None-Match": {cached.Header().Get("ETag")}}, + {"If-Modified-Since": {cached.Header().Get("Last-Modified")}}, + } + for name, handler := range replicas { + for _, header := range held { + if rec := serve(handler, path, header); rec.Code != http.StatusOK || rec.Body.String() != changed.Body.String() { + t.Errorf("replica %s, %v of the cached document: status %d; want 200 with the changed document", name, header, rec.Code) + } + } + } +} + +// expectRevalidates checks that every replica serves want's validators and +// answers a conditional request with either of them with 304. +func expectRevalidates(t *testing.T, replicas map[string]http.Handler, path string, want *httptest.ResponseRecorder) { + t.Helper() + etag, lastModified := want.Header().Get("ETag"), want.Header().Get("Last-Modified") + if etag == "" || lastModified == "" { + t.Fatalf("ETag %q, Last-Modified %q; want both", etag, lastModified) + } + for name, handler := range replicas { + rec := serve(handler, path, nil) + if rec.Header().Get("ETag") != etag || rec.Header().Get("Last-Modified") != lastModified { + t.Errorf("replica %s: ETag %q, Last-Modified %q; want %q, %q", name, rec.Header().Get("ETag"), rec.Header().Get("Last-Modified"), etag, lastModified) + } + for _, header := range []http.Header{{"If-None-Match": {etag}}, {"If-Modified-Since": {lastModified}}} { + if rec := serve(handler, path, header); rec.Code != http.StatusNotModified { + t.Errorf("replica %s, %v: status %d, want 304", name, header, rec.Code) + } + } + } +} From fdd0c814eb69403a5fe2a566ac15945d95836aab Mon Sep 17 00:00:00 2001 From: Giles Westwood <201443+gilesw@users.noreply.github.com> Date: Mon, 12 Oct 2026 15:39:23 +0100 Subject: [PATCH 230/633] Build in version-pinned paths for well-known download hosts Generic upstreams pointed at archive.apache.org, nodejs.org or dl.google.com/go now cache their versioned downloads as artifacts without any generic_artifacts patterns. Configured patterns add to the built-in ones. --- README.md | 18 ++++++++---- docs/configuration.md | 17 ++++++++--- internal/handler/generic.go | 45 ++++++++++++++++++++++------- internal/handler/generic_test.go | 49 ++++++++++++++++++++++++++++++++ 4 files changed, 108 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index 020916c..5b7d9d6 100644 --- a/README.md +++ b/README.md @@ -702,24 +702,30 @@ Commit a `mise.lock` and install with `mise install --locked` so pinned installs need no API call at all. Add a bearer token for `https://api.github.com` under `upstream.auth` if the fleet exceeds GitHub's anonymous rate limit. -Tools mise downloads from elsewhere, such as Maven from `archive.apache.org`, -can be cached the same way by declaring their version-pinned paths in -`upstream.generic_artifacts` (see [configuration](docs/configuration.md)): +Maven, mvnd, Node and Go downloads are cached the same way once their hosts +are configured as generic upstreams; their version-pinned paths are built in +(see [configuration](docs/configuration.md)): ```yaml upstream: generic: apache: "https://archive.apache.org" - generic_artifacts: - apache: - - 'dist/(?Pmaven)/maven-3/(?P[^/]+)/binaries/(?P[^/]+)' + nodejs: "https://nodejs.org" + golang: "https://dl.google.com/go" ``` ```toml +[settings] +node.mirror_url = "http://localhost:8080/generic/nodejs/dist/" +go.download_mirror = "http://localhost:8080/generic/golang" + [settings.url_replacements] "https://archive.apache.org/" = "http://localhost:8080/generic/apache/" ``` +Other download hosts need their version-pinned paths declared in +`upstream.generic_artifacts`. + ## Configuration The proxy can be configured via: diff --git a/docs/configuration.md b/docs/configuration.md index 41ba4d0..ce8aa30 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -361,8 +361,17 @@ version-pinned GitHub release assets: they are stored in the artifact cache and served from it without revalidation, including while the upstream is down. +Upstreams pointed at these hosts get version-pinned paths built in, whatever +the upstream is named: + +| Upstream URL | Cached as artifacts | Package name | +|---|---|---| +| `https://archive.apache.org` | `dist/{project dirs}/{version}/[binaries/]{file}`, where `{version}` starts with a digit | the project dirs, e.g. `maven/maven-3`, `maven/mvnd` | +| `https://nodejs.org` | `dist/v{version}/{file}` | the upstream name | +| `https://dl.google.com/go` | `go{version}.{os}-{arch}.tar.gz`, `.zip` and their `.sha256` | the upstream name | + Other upstreams can declare their own version-pinned paths in -`generic_artifacts`. Each pattern is a regular expression that must match the +`generic_artifacts`, which add to any built-in ones. Each pattern is a regular expression that must match the whole path after `/generic/{name}/`, with `version` and `file` named groups (single path segments) and an optional `name` group for the package name, which otherwise is the upstream name. Matching downloads are cached like @@ -371,10 +380,10 @@ release assets and listed under the `generic` ecosystem: ```yaml upstream: generic: - apache: "https://archive.apache.org" + eclipse: "https://download.eclipse.org" generic_artifacts: - apache: - - 'dist/(?Pmaven)/maven-3/(?P[^/]+)/binaries/(?P[^/]+)' + eclipse: + - 'jdtls/milestones/(?P[^/]+)/(?P[^/]+)' ``` Only list paths whose content never changes for a given version: a match is diff --git a/internal/handler/generic.go b/internal/handler/generic.go index 67ae94a..954e091 100644 --- a/internal/handler/generic.go +++ b/internal/handler/generic.go @@ -28,6 +28,23 @@ const ( // and served without revalidation once fetched. var githubReleaseAssetPattern = regexp.MustCompile(`^([^/]+)/([^/]+)/releases/download/([^/]+)/([^/]+)$`) +// builtinGenericArtifacts maps well-known download hosts to patterns for +// their version-pinned paths. They apply to any generic upstream configured +// with that base URL, before the upstream's own generic_artifacts. +var builtinGenericArtifacts = map[string][]string{ + // dist/{project dirs}/{version}/[binaries/]{file}, e.g. maven/maven-3 or maven/mvnd. + "https://archive.apache.org": { + `dist/(?P.+?)/(?P[0-9][^/]*)/(?:binaries/)?(?P[^/]+)`, + }, + "https://nodejs.org": { + `dist/(?Pv[0-9][^/]*)/(?P[^/]+)`, + }, + // The version is only in the filename: go1.23.2.linux-amd64.tar.gz. + "https://dl.google.com/go": { + `(?Pgo(?P[0-9][^/]*?)\.[a-z0-9]+-[a-z0-9]+\.(?:tar\.gz|zip)(?:\.sha256)?)`, + }, +} + // GenericHandler proxies plain HTTP downloads from configured upstream base // URLs. Each configured upstream is mounted at /generic/{name}/ and the // remaining request path (and query string) is appended to the upstream URL. @@ -38,7 +55,7 @@ var githubReleaseAssetPattern = regexp.MustCompile(`^([^/]+)/([^/]+)/releases/do // assets, and is pointed at by URL-rewriting settings on the client. // // Release-asset paths ({owner}/{repo}/releases/download/{tag}/{asset}), and -// paths matching the upstream's configured artifact patterns, are +// paths matching the upstream's built-in or configured artifact patterns, are // version-pinned and cached in the shared artifact cache, so they keep being // served when the upstream is unreachable. Every other path is served through // the metadata cache: fresh within the metadata TTL, revalidated with the @@ -61,24 +78,30 @@ func NewGenericHandler(proxy *Proxy, repositories map[string]string, artifactPat artifactPatterns: make(map[string][]*regexp.Regexp), } for name, upstreamURL := range repositories { - h.repositories[name] = strings.TrimSuffix(upstreamURL, "/") + upstreamURL = strings.TrimSuffix(upstreamURL, "/") + h.repositories[name] = upstreamURL + h.addArtifactPatterns(name, builtinGenericArtifacts[upstreamURL]) } if len(artifactPatterns) > 0 { for name, patterns := range artifactPatterns[0] { - for _, pattern := range patterns { - re, err := config.CompileGenericArtifactPattern(pattern) - if err != nil { - proxy.Logger.Warn("skipping generic artifact pattern", - "repository", name, "pattern", pattern, "error", err) - continue - } - h.artifactPatterns[name] = append(h.artifactPatterns[name], re) - } + h.addArtifactPatterns(name, patterns) } } return h } +func (h *GenericHandler) addArtifactPatterns(name string, patterns []string) { + for _, pattern := range patterns { + re, err := config.CompileGenericArtifactPattern(pattern) + if err != nil { + h.proxy.Logger.Warn("skipping generic artifact pattern", + "repository", name, "pattern", pattern, "error", err) + continue + } + h.artifactPatterns[name] = append(h.artifactPatterns[name], re) + } +} + // Routes returns the HTTP handler for generic download requests. // Mount this at /generic on your router. func (h *GenericHandler) Routes() http.Handler { diff --git a/internal/handler/generic_test.go b/internal/handler/generic_test.go index 3e45fb9..1292e8c 100644 --- a/internal/handler/generic_test.go +++ b/internal/handler/generic_test.go @@ -105,6 +105,55 @@ func TestMatchArtifactPattern(t *testing.T) { } } +func TestGenericHandler_BuiltinArtifactPatterns(t *testing.T) { + h := NewGenericHandler(testProxy(), map[string]string{ + "apache": "https://archive.apache.org/", + "node": "https://nodejs.org", + "golang": "https://dl.google.com/go", + "mirror": "https://mirror.example.com", + }, map[string][]string{ + "apache": {`extra/(?P[^/]+)/(?P[^/]+)`}, + }) + + tests := []struct { + repository string + path string + want genericArtifact + ok bool + }{ + {"apache", "dist/maven/maven-3/3.9.9/binaries/apache-maven-3.9.9-bin.tar.gz", + genericArtifact{name: "maven/maven-3", version: "3.9.9", filename: "apache-maven-3.9.9-bin.tar.gz"}, true}, + {"apache", "dist/maven/maven-3/3.9.9/binaries/apache-maven-3.9.9-bin.tar.gz.sha512", + genericArtifact{name: "maven/maven-3", version: "3.9.9", filename: "apache-maven-3.9.9-bin.tar.gz.sha512"}, true}, + {"apache", "dist/maven/mvnd/1.0.6/maven-mvnd-1.0.6-linux-amd64.tar.gz", + genericArtifact{name: "maven/mvnd", version: "1.0.6", filename: "maven-mvnd-1.0.6-linux-amd64.tar.gz"}, true}, + {"apache", "dist/maven/maven-3/", genericArtifact{}, false}, + {"apache", "dist/maven/maven-3/3.9.9/source/apache-maven-3.9.9-src.tar.gz", genericArtifact{}, false}, + {"apache", "extra/1.0/file.txt", genericArtifact{name: "apache", version: "1.0", filename: "file.txt"}, true}, + {"node", "dist/v22.11.0/node-v22.11.0-linux-x64.tar.gz", + genericArtifact{name: "node", version: "v22.11.0", filename: "node-v22.11.0-linux-x64.tar.gz"}, true}, + {"node", "dist/v22.11.0/SHASUMS256.txt", + genericArtifact{name: "node", version: "v22.11.0", filename: "SHASUMS256.txt"}, true}, + {"node", "dist/index.json", genericArtifact{}, false}, + {"golang", "go1.23.2.linux-amd64.tar.gz", + genericArtifact{name: "golang", version: "1.23.2", filename: "go1.23.2.linux-amd64.tar.gz"}, true}, + {"golang", "go1.23.2.darwin-arm64.tar.gz.sha256", + genericArtifact{name: "golang", version: "1.23.2", filename: "go1.23.2.darwin-arm64.tar.gz.sha256"}, true}, + {"golang", "go1.24rc1.windows-amd64.zip", + genericArtifact{name: "golang", version: "1.24rc1", filename: "go1.24rc1.windows-amd64.zip"}, true}, + {"mirror", "dist/maven/maven-3/3.9.9/binaries/apache-maven-3.9.9-bin.tar.gz", genericArtifact{}, false}, + } + + for _, tt := range tests { + t.Run(tt.repository+"/"+tt.path, func(t *testing.T) { + got, ok := h.parseArtifact(tt.repository, tt.path) + if ok != tt.ok || got != tt.want { + t.Errorf("parseArtifact(%q, %q) = (%+v, %v), want (%+v, %v)", tt.repository, tt.path, got, ok, tt.want, tt.ok) + } + }) + } +} + func TestGenericHandler_ArtifactPatternIsCachedAndOtherPathsAreNot(t *testing.T) { const ( artifactPath = "/dist/maven/maven-3/3.9.9/binaries/apache-maven-3.9.9-bin.tar.gz" From 59f0203b00e28b162451dc8316cda474f9373ff1 Mon Sep 17 00:00:00 2001 From: Christian Heim Date: Mon, 12 Oct 2026 17:08:17 +0200 Subject: [PATCH 231/633] When a signature has to be fetched on request because its fetch failed during the database refresh, it is now only served if it belongs to the database the proxy had cached at that point. If the upstream has published a new database in the meantime, the request fails with 502 instead of handing out a signature that cannot verify the database the client already has. The next sync gets the new pair. --- internal/handler/arch.go | 42 ++++++++++++++++++++++---------- internal/handler/arch_test.go | 45 +++++++++++++++++++++++++++++++++++ 2 files changed, 74 insertions(+), 13 deletions(-) diff --git a/internal/handler/arch.go b/internal/handler/arch.go index f3505a4..64fde48 100644 --- a/internal/handler/arch.go +++ b/internal/handler/arch.go @@ -224,20 +224,27 @@ func (h *ArchHandler) handleDatabaseSignature(w http.ResponseWriter, r *http.Req return } p.recordMetadataCacheMiss(archEcosystem) - // Nothing stored for this database yet, e.g. the signature fetch failed - // during the last refresh: refresh both. - res, _ := p.coalesceMetadata(r.Context(), d.coalesceKey("signature"), func(ctx context.Context) metadataResult { - if res, ok := h.cachedSignature(ctx, d); ok { + // Nothing stored for the cached database, e.g. its signature fetch failed + // during the last refresh. The client may already hold that database, so + // only a signature for it will do; if the refresh replaces it, fail. + entry, _ := p.DB.GetMetadataCache(archEcosystem, d.dbKey) + served := h.databaseDigest(d) + res, _ := p.coalesceMetadata(r.Context(), d.coalesceKey("signature\x00"+served), func(ctx context.Context) metadataResult { + if res, ok := h.storedSignature(ctx, d, served); ok { return res } - entry, _ := p.DB.GetMetadataCache(archEcosystem, d.dbKey) - if _, err := h.refresh(ctx, d, entry); err != nil { - return metadataResult{err: err} - } - if res, ok := h.cachedSignature(ctx, d); ok { + _, err := h.refresh(ctx, d, entry) + if served == "" { + if res, ok := h.cachedSignature(ctx, d); ok { + return res + } + } else if res, ok := h.storedSignature(ctx, d, served); ok { return res } - return metadataResult{err: errArchSignatureUnpaired} + if err == nil || h.databaseDigest(d) != served { + err = errArchSignatureUnpaired + } + return metadataResult{err: err} }) h.writeMetadata(w, r, d.sigKey, res) } @@ -292,11 +299,20 @@ func (h *ArchHandler) refresh(ctx context.Context, d archDatabase, entry *databa // cachedSignature returns the signature stored for the cached database. func (h *ArchHandler) cachedSignature(ctx context.Context, d archDatabase) (metadataResult, bool) { - current, _ := h.proxy.DB.GetMetadataCache(archEcosystem, d.dbKey) - if current == nil || !current.ContentDigest.Valid { + dbDigest := h.databaseDigest(d) + if dbDigest == "" { return metadataResult{}, false } - return h.storedSignature(ctx, d, current.ContentDigest.String) + return h.storedSignature(ctx, d, dbDigest) +} + +// databaseDigest returns the cached database's content digest, or "". +func (h *ArchHandler) databaseDigest(d archDatabase) string { + current, _ := h.proxy.DB.GetMetadataCache(archEcosystem, d.dbKey) + if current == nil || !current.ContentDigest.Valid { + return "" + } + return current.ContentDigest.String } // storedSignature returns the signature stored for dbDigest. The storage diff --git a/internal/handler/arch_test.go b/internal/handler/arch_test.go index 1b71b3f..2650f02 100644 --- a/internal/handler/arch_test.go +++ b/internal/handler/arch_test.go @@ -389,6 +389,51 @@ func TestArchHandler_DelayedSignatureMatchesCachedDatabase(t *testing.T) { assertArchBody(t, serveArchRequest(h, http.MethodGet, "/custom"+sigPath, nil), "first signature request", []byte("sig v1")) } +// TestArchHandler_RecoveredSignatureMatchesServedDatabase covers a signature +// whose fetch failed while the database was served. If the upstream has moved +// on by the time pacman asks for the signature, the new one cannot verify the +// database pacman holds, so the proxy must fail rather than serve it. +func TestArchHandler_RecoveredSignatureMatchesServedDatabase(t *testing.T) { + const dbPath, sigPath = "/custom/os/x86_64/custom.db", "/custom/os/x86_64/custom.db.sig" + upstream := newArchUpstream(t, map[string][]byte{dbPath: []byte("db v1"), sigPath: []byte("sig v1")}) + upstream.broken[sigPath] = true + proxy := newArchTestProxy(t, upstream) + h := NewArchHandler(proxy, "http://proxy.example", map[string]string{"custom": upstream.URL}) + + assertArchBody(t, serveArchRequest(h, http.MethodGet, "/custom"+dbPath, nil), "db v1", []byte("db v1")) + + upstream.mu.Lock() + upstream.files[dbPath] = []byte("db v2") + upstream.files[sigPath] = []byte("sig v2") + upstream.broken[sigPath] = false + upstream.mu.Unlock() + + if w := serveArchRequest(h, http.MethodGet, "/custom"+sigPath, nil); w.Code != http.StatusBadGateway { + t.Errorf("signature for db v1: status = %d, body = %q, want 502", w.Code, w.Body.String()) + } + + // The next sync gets the new pair. + assertArchBody(t, serveArchRequest(h, http.MethodGet, "/custom"+dbPath, nil), "db v2", []byte("db v2")) + assertArchBody(t, serveArchRequest(h, http.MethodGet, "/custom"+sigPath, nil), "sig v2", []byte("sig v2")) +} + +// TestArchHandler_RecoveredSignatureForUnchangedDatabase covers the same +// failed signature fetch when the database has not changed: the signature is +// fetched on request and served. +func TestArchHandler_RecoveredSignatureForUnchangedDatabase(t *testing.T) { + const dbPath, sigPath = "/custom/os/x86_64/custom.db", "/custom/os/x86_64/custom.db.sig" + upstream := newArchUpstream(t, map[string][]byte{dbPath: []byte("db v1"), sigPath: []byte("sig v1")}) + upstream.broken[sigPath] = true + proxy := newArchTestProxy(t, upstream) + h := NewArchHandler(proxy, "http://proxy.example", map[string]string{"custom": upstream.URL}) + + assertArchBody(t, serveArchRequest(h, http.MethodGet, "/custom"+dbPath, nil), "db v1", []byte("db v1")) + upstream.mu.Lock() + upstream.broken[sigPath] = false + upstream.mu.Unlock() + assertArchBody(t, serveArchRequest(h, http.MethodGet, "/custom"+sigPath, nil), "sig v1", []byte("sig v1")) +} + // TestArchHandler_SignatureSurvivesUnchangedRefresh covers a database refresh // that returns the same bytes: the cached signature still matches and must be // served offline afterwards. From 70ea9215c64a946ed40f03dc8ff09a42802aab1f Mon Sep 17 00:00:00 2001 From: Christian Heim Date: Sat, 10 Oct 2026 22:10:31 +0200 Subject: [PATCH 232/633] Enable storage retention for cargo Crates are stored under the name they were published with, and purl keeps cargo names as they are. Package rules have to match that name exactly, so pkg:cargo/inflector doesn't cover Inflector. Refs #306 --- docs/configuration.md | 2 +- internal/handler/cargo.go | 10 + internal/server/retention_cargo_test.go | 239 ++++++++++++++++++++++++ 3 files changed, 250 insertions(+), 1 deletion(-) create mode 100644 internal/server/retention_cargo_test.go diff --git a/docs/configuration.md b/docs/configuration.md index dcf52dc..bf92e13 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -89,7 +89,7 @@ A package rule wins over an ecosystem rule, which wins over `default`. Package r | Ecosystem key | Retention supported | Package rule form | |---------------|---------------------|-------------------| | `alpine` | yes | `pkg:apk/curl` or `pkg:apk/alpine/curl`; case doesn't matter. A rule covers the package in every configured APK repository. Alpine mirrors keep only the current build of each package per release, so an evicted older one can't be fetched again. | -| `cargo` | no | - | +| `cargo` | yes | `pkg:cargo/serde`, the crate name as published; case, `-` and `_` have to match, so `pkg:cargo/inflector` doesn't cover `Inflector`. | | `composer` | no | - | | `conan` | no | - | | `conda` | no | - | diff --git a/internal/handler/cargo.go b/internal/handler/cargo.go index f87819c..97d2f17 100644 --- a/internal/handler/cargo.go +++ b/internal/handler/cargo.go @@ -9,6 +9,8 @@ import ( "net/url" "strings" "time" + + "github.com/git-pkgs/proxy/internal/retention" ) const ( @@ -20,6 +22,14 @@ const ( cargoIndexLen3 = 3 ) +// Crates are stored under pkg:cargo/{name}, with the name cargo puts in the +// download URL, which is the name as published. purl keeps cargo names as +// they are, so package rules use the default key mapping and have to match +// that name exactly. +func init() { + retention.Default.Register(retention.Spec{Key: "cargo", CanonicalPackage: retention.DefaultCanonical("cargo")}) +} + // CargoHandler handles cargo registry protocol requests. type CargoHandler struct { proxy *Proxy diff --git a/internal/server/retention_cargo_test.go b/internal/server/retention_cargo_test.go new file mode 100644 index 0000000..d11c4fe --- /dev/null +++ b/internal/server/retention_cargo_test.go @@ -0,0 +1,239 @@ +package server + +import ( + "context" + "io" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/git-pkgs/proxy/internal/config" + "github.com/git-pkgs/proxy/internal/database" + "github.com/git-pkgs/proxy/internal/handler" + "github.com/git-pkgs/proxy/internal/mirror" + "github.com/git-pkgs/proxy/internal/retention" + "github.com/git-pkgs/registries/fetch" +) + +// cargoRetentionUpstream serves crate files the way static.crates.io does, +// counting downloads. +func cargoRetentionUpstream(t *testing.T) (*httptest.Server, *atomic.Int64) { + t.Helper() + var downloads atomic.Int64 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/crates/serde/serde-1.0.0.crate", "/crates/Inflector/Inflector-0.11.4.crate": + downloads.Add(1) + _, _ = io.WriteString(w, "crate bytes") + default: + http.NotFound(w, r) + } + })) + t.Cleanup(srv.Close) + return srv, &downloads +} + +func newCargoRetentionProxy(t *testing.T, upstream *httptest.Server) (*httptest.Server, *database.DB) { + t.Helper() + db, store := setupEvictionTest(t) + // The upstream is a plain-HTTP loopback server; the default fetcher + // refuses loopback, a stock client reaches it. + fetcher := fetch.NewFetcher(fetch.WithHTTPClient(upstream.Client()), fetch.WithMaxRetries(0)) + t.Cleanup(func() { _ = fetcher.Close() }) + proxy := handler.NewProxy(db, store, fetcher, fetch.NewResolver(), discardLogger()) + proxy.HTTPClient = upstream.Client() + h := handler.NewCargoHandler(proxy, "http://proxy.test", upstream.URL, upstream.URL+"/crates") + srv := httptest.NewServer(h.Routes()) + t.Cleanup(srv.Close) + return srv, db +} + +func cargoGet(t *testing.T, url string) { + t.Helper() + resp, err := http.Get(url) + if err != nil { + t.Fatalf("GET %s: %v", url, err) + } + body, _ := io.ReadAll(resp.Body) + _ = resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Fatalf("GET %s = %d %s, want 200", url, resp.StatusCode, body) + } +} + +// cargoArtifact returns the record of the crate file named filename. +func cargoArtifact(t *testing.T, db *database.DB, filename string) database.Artifact { + t.Helper() + var art database.Artifact + if err := db.Get(&art, db.Rebind(`SELECT * FROM artifacts WHERE filename = ?`), filename); err != nil { + t.Fatalf("reading artifact %s: %v", filename, err) + } + return art +} + +func cargoBackdate(t *testing.T, db *database.DB, filename string, fetched, accessed time.Time) { + t.Helper() + query := db.Rebind(`UPDATE artifacts SET fetched_at = ?, last_accessed_at = ? WHERE filename = ?`) + if _, err := db.Exec(query, fetched, accessed, filename); err != nil { + t.Fatalf("backdating %s: %v", filename, err) + } +} + +func TestCargoRegistersForRetention(t *testing.T) { + if _, ok := retention.Default.Lookup("cargo"); !ok { + t.Fatal("cargo is not registered for retention") + } + if key, ok := retention.Default.KeyForDBEcosystem("cargo"); !ok || key != "cargo" { + t.Errorf("KeyForDBEcosystem(cargo) = %q, %v, want cargo, true", key, ok) + } +} + +// A crate served from the cache stays while it is used, is evicted once its +// rule runs out, comes back from upstream on the next download, and stays +// after that even with the evicted copy's old access time on the record. +func TestCargoRetentionEvictsAndRefetches(t *testing.T) { + upstream, downloads := cargoRetentionUpstream(t) + srv, db := newCargoRetentionProxy(t, upstream) + const crate = "serde-1.0.0.crate" + + cargoGet(t, srv.URL+"/crates/serde/1.0.0/download") + cargoGet(t, srv.URL+"/crates/serde/1.0.0/download") + if got := downloads.Load(); got != 1 { + t.Fatalf("upstream downloads = %d, want 1", got) + } + if !cargoArtifact(t, db, crate).LastAccessedAt.Valid { + t.Fatal("cache hit did not record an access") + } + + rules := retention.Rules{Ecosystems: map[string]time.Duration{"cargo": retentionDay}} + now := time.Now() + sweepRetention(context.Background(), db, discardLogger(), retention.Default, rules, now) + if !cargoArtifact(t, db, crate).StoragePath.Valid { + t.Fatal("crate used today was evicted") + } + + old := now.Add(-2 * retentionDay) + cargoBackdate(t, db, crate, old, old) + sweepRetention(context.Background(), db, discardLogger(), retention.Default, rules, now) + if cargoArtifact(t, db, crate).StoragePath.Valid { + t.Fatal("crate unused for two days kept under a one-day rule") + } + + cargoGet(t, srv.URL+"/crates/serde/1.0.0/download") + if got := downloads.Load(); got != 2 { + t.Errorf("upstream downloads = %d, want 2 after the eviction", got) + } + art := cargoArtifact(t, db, crate) + cargoBackdate(t, db, crate, art.FetchedAt.Time, now.Add(-30*retentionDay)) + sweepRetention(context.Background(), db, discardLogger(), retention.Default, rules, time.Now()) + if !cargoArtifact(t, db, crate).StoragePath.Valid { + t.Error("refetched crate evicted again by the next sweep") + } +} + +// A key with a namespace parses as a cargo PURL but can never match a +// stored crate, so it is refused at validation. A plain key is accepted. +func TestCargoRetentionRefusesNamespacedKeys(t *testing.T) { + cfg := config.Default() + cfg.Storage.Retention.Packages = map[string]string{"pkg:cargo/serde": "0"} + if _, err := cfg.RetentionRules(retention.Default); err != nil { + t.Fatalf("RetentionRules(pkg:cargo/serde): %v", err) + } + cfg.Storage.Retention.Packages = map[string]string{"pkg:cargo/rust-lang/serde": "0"} + _, err := cfg.RetentionRules(retention.Default) + if err == nil || !strings.Contains(err.Error(), "does not match how cargo packages are stored") { + t.Errorf("RetentionRules(pkg:cargo/rust-lang/serde) error = %v, want the key refused", err) + } +} + +// A package rule reaches a crate under its published name, case included; +// a rule with the name in another case covers nothing. +func TestCargoRetentionPackageRules(t *testing.T) { + upstream, _ := cargoRetentionUpstream(t) + srv, db := newCargoRetentionProxy(t, upstream) + const inflector, serde = "Inflector-0.11.4.crate", "serde-1.0.0.crate" + + tests := []struct { + key string + keepInflector bool + }{ + {"pkg:cargo/Inflector", true}, + {"pkg:cargo/inflector", false}, + } + for _, tt := range tests { + cargoGet(t, srv.URL+"/crates/Inflector/0.11.4/download") + cargoGet(t, srv.URL+"/crates/serde/1.0.0/download") + old := time.Now().Add(-30 * retentionDay) + cargoBackdate(t, db, inflector, old, old) + cargoBackdate(t, db, serde, old, old) + + cfg := config.Default() + cfg.Storage.Retention.Ecosystems = map[string]string{"cargo": "1d"} + cfg.Storage.Retention.Packages = map[string]string{tt.key: "0"} + rules, err := cfg.RetentionRules(retention.Default) + if err != nil { + t.Fatalf("RetentionRules(%s): %v", tt.key, err) + } + sweepRetention(context.Background(), db, discardLogger(), retention.Default, rules, time.Now()) + + if got := cargoArtifact(t, db, inflector).StoragePath.Valid; got != tt.keepInflector { + t.Errorf("with %s: Inflector cached = %v, want %v", tt.key, got, tt.keepInflector) + } + if cargoArtifact(t, db, serde).StoragePath.Valid { + t.Errorf("with %s: serde kept past the cargo rule", tt.key) + } + } +} + +// cargoRewriteTransport sends every request to the test upstream, so the +// mirror command's fixed static.crates.io URLs reach it. +type cargoRewriteTransport struct { + target *url.URL + base http.RoundTripper +} + +func (r cargoRewriteTransport) RoundTrip(req *http.Request) (*http.Response, error) { + req = req.Clone(req.Context()) + req.URL.Scheme = r.target.Scheme + req.URL.Host = r.target.Host + req.Host = r.target.Host + return r.base.RoundTrip(req) +} + +// A crate preloaded with the mirror command falls under the cargo rule. +func TestCargoRetentionCoversMirroredCrates(t *testing.T) { + upstream, downloads := cargoRetentionUpstream(t) + target, err := url.Parse(upstream.URL) + if err != nil { + t.Fatal(err) + } + db, store := setupEvictionTest(t) + client := &http.Client{Transport: cargoRewriteTransport{target: target, base: upstream.Client().Transport}} + fetcher := fetch.NewFetcher(fetch.WithHTTPClient(client), fetch.WithMaxRetries(0)) + t.Cleanup(func() { _ = fetcher.Close() }) + proxy := handler.NewProxy(db, store, fetcher, fetch.NewResolver(), discardLogger()) + + m := mirror.New(proxy, db, store, discardLogger(), 1) + progress, err := m.Run(context.Background(), &mirror.PURLSource{PURLs: []string{"pkg:cargo/serde@1.0.0"}}) + if err != nil || progress.Failed != 0 || downloads.Load() != 1 { + t.Fatalf("mirror run = %+v, %v, downloads %d; want one successful download", progress, err, downloads.Load()) + } + + const crate = "serde-1.0.0.crate" + if cargoArtifact(t, db, crate).LastAccessedAt.Valid { + t.Fatal("mirrored crate has an access recorded before anyone downloaded it") + } + query := db.Rebind(`UPDATE artifacts SET fetched_at = ? WHERE filename = ?`) + if _, err := db.Exec(query, time.Now().Add(-30*retentionDay), crate); err != nil { + t.Fatalf("backdating %s: %v", crate, err) + } + rules := retention.Rules{Ecosystems: map[string]time.Duration{"cargo": retentionDay}} + sweepRetention(context.Background(), db, discardLogger(), retention.Default, rules, time.Now()) + if cargoArtifact(t, db, crate).StoragePath.Valid { + t.Error("never-served mirrored crate kept past the cargo rule") + } +} From 9c75e59ef4d5249ae1fb696fd51bc39531af0a6b Mon Sep 17 00:00:00 2001 From: Christian Heim Date: Sun, 11 Oct 2026 00:25:33 +0200 Subject: [PATCH 233/633] Enable storage retention for composer Archives are stored under pkg:composer/{vendor}/{package}, which purl lowercases, so case in a package rule doesn't matter. A key without a vendor could never match and is refused. Packages that git-pkgs wrote as "packagist" fall under the composer rules too. The mirror can't fetch composer packages today (its resolver has no composer support, and #415 would file them under packagist), so there's no mirror test. Refs #306 --- docs/configuration.md | 2 +- internal/handler/composer.go | 21 ++ internal/handler/composer_test.go | 28 +++ internal/server/retention_composer_test.go | 216 +++++++++++++++++++++ 4 files changed, 266 insertions(+), 1 deletion(-) create mode 100644 internal/server/retention_composer_test.go diff --git a/docs/configuration.md b/docs/configuration.md index bf92e13..f1966be 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -90,7 +90,7 @@ A package rule wins over an ecosystem rule, which wins over `default`. Package r |---------------|---------------------|-------------------| | `alpine` | yes | `pkg:apk/curl` or `pkg:apk/alpine/curl`; case doesn't matter. A rule covers the package in every configured APK repository. Alpine mirrors keep only the current build of each package per release, so an evicted older one can't be fetched again. | | `cargo` | yes | `pkg:cargo/serde`, the crate name as published; case, `-` and `_` have to match, so `pkg:cargo/inflector` doesn't cover `Inflector`. | -| `composer` | no | - | +| `composer` | yes | `pkg:composer/symfony/console`; case doesn't matter. A key without a vendor is refused. | | `conan` | no | - | | `conda` | no | - | | `cran` | no | - | diff --git a/internal/handler/composer.go b/internal/handler/composer.go index f7b40ae..4085906 100644 --- a/internal/handler/composer.go +++ b/internal/handler/composer.go @@ -13,6 +13,9 @@ import ( "regexp" "strings" "time" + + "github.com/git-pkgs/proxy/internal/retention" + "github.com/git-pkgs/purl" ) const ( @@ -27,6 +30,24 @@ const ( var composerUnsetJSON = []byte(`"` + composerUnset + `"`) +// Archives are stored under pkg:composer/{vendor}/{package}, which purl +// lowercases, so a package rule matches whatever case it is written in. The +// spec also claims the "packagist" spelling git-pkgs writes into +// packages.ecosystem. +func init() { + retention.Default.Register(retention.Spec{Key: "composer", CanonicalPackage: composerRetentionKey}) +} + +// composerRetentionKey maps a package rule to the stored package PURL. A +// key without a vendor parses fine but can never match, since every stored +// package has one, so it is refused. +func composerRetentionKey(p *purl.PURL) (string, bool) { + if p == nil || p.Namespace == "" { + return "", false + } + return retention.DefaultCanonical("composer")(p) +} + // ComposerHandler handles Composer/Packagist registry protocol requests. type ComposerHandler struct { proxy *Proxy diff --git a/internal/handler/composer_test.go b/internal/handler/composer_test.go index ad3eb7a..be672cc 100644 --- a/internal/handler/composer_test.go +++ b/internal/handler/composer_test.go @@ -14,6 +14,7 @@ import ( "time" "github.com/git-pkgs/cooldown" + "github.com/git-pkgs/purl" "github.com/git-pkgs/registries/fetch" ) @@ -979,3 +980,30 @@ func TestComposerRewriteMetadataCooldown(t *testing.T) { t.Errorf("expected version 5.0.0, got %v", v["version"]) } } + +func TestComposerRetentionKey(t *testing.T) { + tests := []struct { + key string + want string + ok bool + }{ + {"pkg:composer/symfony/console", "pkg:composer/symfony/console", true}, + {"pkg:composer/Symfony/Console", "pkg:composer/symfony/console", true}, + {"pkg:composer/console", "", false}, + {"pkg:composer/symfony/console/extra", "", false}, + {"pkg:npm/symfony/console", "", false}, + } + for _, tt := range tests { + p, err := purl.Parse(tt.key) + if err != nil { + t.Fatalf("Parse(%s): %v", tt.key, err) + } + got, ok := composerRetentionKey(p) + if ok != tt.ok || (ok && got != tt.want) { + t.Errorf("composerRetentionKey(%s) = %q, %v, want %q, %v", tt.key, got, ok, tt.want, tt.ok) + } + } + if _, ok := composerRetentionKey(nil); ok { + t.Error("composerRetentionKey(nil) accepted") + } +} diff --git a/internal/server/retention_composer_test.go b/internal/server/retention_composer_test.go new file mode 100644 index 0000000..d92ce99 --- /dev/null +++ b/internal/server/retention_composer_test.go @@ -0,0 +1,216 @@ +package server + +import ( + "context" + "fmt" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/git-pkgs/proxy/internal/config" + "github.com/git-pkgs/proxy/internal/database" + "github.com/git-pkgs/proxy/internal/handler" + "github.com/git-pkgs/proxy/internal/retention" + "github.com/git-pkgs/registries/fetch" +) + +const ( + composerConsolePath = "/files/symfony/console/v6.4.0/symfony-console-v6.4.0.zip" + composerMonologPath = "/files/monolog/monolog/3.5.0/monolog-monolog-3.5.0.zip" +) + +// composerRetentionUpstream serves Packagist metadata for symfony/console and +// monolog/monolog and their dist archives, counting archive downloads. +func composerRetentionUpstream(t *testing.T) (*httptest.Server, *atomic.Int64) { + t.Helper() + var downloads atomic.Int64 + var srv *httptest.Server + srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + meta := func(name, version string) { + _, _ = fmt.Fprintf(w, `{"packages":{%q:[{"name":%q,"version":%q,"dist":{"type":"zip","url":%q}}]}}`, + name, name, version, srv.URL+"/dist/"+strings.ReplaceAll(name, "/", "-")+"-"+version+".zip") + } + switch r.URL.Path { + case "/p2/symfony/console.json": + meta("symfony/console", "v6.4.0") + case "/p2/monolog/monolog.json": + meta("monolog/monolog", "3.5.0") + case "/dist/symfony-console-v6.4.0.zip", "/dist/monolog-monolog-3.5.0.zip": + downloads.Add(1) + _, _ = io.WriteString(w, "zip bytes") + default: + http.NotFound(w, r) + } + })) + t.Cleanup(srv.Close) + return srv, &downloads +} + +func newComposerRetentionProxy(t *testing.T, upstream *httptest.Server) (*httptest.Server, *database.DB) { + t.Helper() + db, store := setupEvictionTest(t) + // The upstream is a plain-HTTP loopback server; the default fetcher + // refuses loopback, a stock client reaches it. + fetcher := fetch.NewFetcher(fetch.WithHTTPClient(upstream.Client()), fetch.WithMaxRetries(0)) + t.Cleanup(func() { _ = fetcher.Close() }) + proxy := handler.NewProxy(db, store, fetcher, fetch.NewResolver(), discardLogger()) + proxy.HTTPClient = upstream.Client() + h := handler.NewComposerHandlerWithUpstreams(proxy, "http://proxy.test", upstream.URL, upstream.URL) + srv := httptest.NewServer(h.Routes()) + t.Cleanup(srv.Close) + return srv, db +} + +func composerGet(t *testing.T, url string) { + t.Helper() + resp, err := http.Get(url) + if err != nil { + t.Fatalf("GET %s: %v", url, err) + } + body, _ := io.ReadAll(resp.Body) + _ = resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Fatalf("GET %s = %d %s, want 200", url, resp.StatusCode, body) + } +} + +// composerArtifact returns the record of the archive named filename. +func composerArtifact(t *testing.T, db *database.DB, filename string) database.Artifact { + t.Helper() + var art database.Artifact + if err := db.Get(&art, db.Rebind(`SELECT * FROM artifacts WHERE filename = ?`), filename); err != nil { + t.Fatalf("reading artifact %s: %v", filename, err) + } + return art +} + +func composerBackdate(t *testing.T, db *database.DB, filename string, fetched, accessed time.Time) { + t.Helper() + query := db.Rebind(`UPDATE artifacts SET fetched_at = ?, last_accessed_at = ? WHERE filename = ?`) + if _, err := db.Exec(query, fetched, accessed, filename); err != nil { + t.Fatalf("backdating %s: %v", filename, err) + } +} + +func TestComposerRegistersForRetention(t *testing.T) { + if _, ok := retention.Default.Lookup("composer"); !ok { + t.Fatal("composer is not registered for retention") + } + for _, value := range []string{"composer", "packagist"} { + if key, ok := retention.Default.KeyForDBEcosystem(value); !ok || key != "composer" { + t.Errorf("KeyForDBEcosystem(%s) = %q, %v, want composer, true", value, key, ok) + } + } +} + +// An archive served from the cache stays while it is used, is evicted once +// its rule runs out, comes back from upstream on the next download, and +// stays after that even with the evicted copy's old access time on the +// record. +func TestComposerRetentionEvictsAndRefetches(t *testing.T) { + upstream, downloads := composerRetentionUpstream(t) + srv, db := newComposerRetentionProxy(t, upstream) + const archive = "symfony-console-v6.4.0.zip" + + composerGet(t, srv.URL+composerConsolePath) + composerGet(t, srv.URL+composerConsolePath) + if got := downloads.Load(); got != 1 { + t.Fatalf("upstream downloads = %d, want 1", got) + } + if !composerArtifact(t, db, archive).LastAccessedAt.Valid { + t.Fatal("cache hit did not record an access") + } + + rules := retention.Rules{Ecosystems: map[string]time.Duration{"composer": retentionDay}} + now := time.Now() + sweepRetention(context.Background(), db, discardLogger(), retention.Default, rules, now) + if !composerArtifact(t, db, archive).StoragePath.Valid { + t.Fatal("archive used today was evicted") + } + + old := now.Add(-2 * retentionDay) + composerBackdate(t, db, archive, old, old) + sweepRetention(context.Background(), db, discardLogger(), retention.Default, rules, now) + if composerArtifact(t, db, archive).StoragePath.Valid { + t.Fatal("archive unused for two days kept under a one-day rule") + } + + composerGet(t, srv.URL+composerConsolePath) + if got := downloads.Load(); got != 2 { + t.Errorf("upstream downloads = %d, want 2 after the eviction", got) + } + art := composerArtifact(t, db, archive) + composerBackdate(t, db, archive, art.FetchedAt.Time, now.Add(-30*retentionDay)) + sweepRetention(context.Background(), db, discardLogger(), retention.Default, rules, time.Now()) + if !composerArtifact(t, db, archive).StoragePath.Valid { + t.Error("refetched archive evicted again by the next sweep") + } +} + +// A key without a vendor parses as a composer PURL but can never match a +// stored package, so it is refused at validation. A key with a vendor is +// accepted. +func TestComposerRetentionRefusesKeysWithoutVendor(t *testing.T) { + cfg := config.Default() + cfg.Storage.Retention.Packages = map[string]string{"pkg:composer/symfony/console": "0"} + if _, err := cfg.RetentionRules(retention.Default); err != nil { + t.Fatalf("RetentionRules(pkg:composer/symfony/console): %v", err) + } + cfg.Storage.Retention.Packages = map[string]string{"pkg:composer/console": "0"} + _, err := cfg.RetentionRules(retention.Default) + if err == nil || !strings.Contains(err.Error(), "does not match how composer packages are stored") { + t.Errorf("RetentionRules(pkg:composer/console) error = %v, want the key refused", err) + } +} + +// A package rule reaches the stored package whatever case it is written in, +// also when git-pkgs created the package record as "packagist". +func TestComposerRetentionPackageRules(t *testing.T) { + upstream, _ := composerRetentionUpstream(t) + srv, db := newComposerRetentionProxy(t, upstream) + if err := db.UpsertPackage(&database.Package{PURL: "pkg:composer/monolog/monolog", Ecosystem: "packagist", Name: "monolog/monolog"}); err != nil { + t.Fatalf("UpsertPackage: %v", err) + } + const console, monolog = "symfony-console-v6.4.0.zip", "monolog-monolog-3.5.0.zip" + + tests := []struct { + keys map[string]string + keepConsole bool + keepMonolog bool + }{ + {map[string]string{"pkg:composer/Symfony/Console": "0"}, true, false}, + {map[string]string{"pkg:composer/monolog/monolog": "0"}, false, true}, + } + for _, tt := range tests { + composerGet(t, srv.URL+composerConsolePath) + composerGet(t, srv.URL+composerMonologPath) + old := time.Now().Add(-30 * retentionDay) + composerBackdate(t, db, console, old, old) + composerBackdate(t, db, monolog, old, old) + + cfg := config.Default() + cfg.Storage.Retention.Ecosystems = map[string]string{"composer": "1d"} + cfg.Storage.Retention.Packages = tt.keys + rules, err := cfg.RetentionRules(retention.Default) + if err != nil { + t.Fatalf("RetentionRules(%v): %v", tt.keys, err) + } + sweepRetention(context.Background(), db, discardLogger(), retention.Default, rules, time.Now()) + + if got := composerArtifact(t, db, console).StoragePath.Valid; got != tt.keepConsole { + t.Errorf("with %v: symfony/console cached = %v, want %v", tt.keys, got, tt.keepConsole) + } + if got := composerArtifact(t, db, monolog).StoragePath.Valid; got != tt.keepMonolog { + t.Errorf("with %v: monolog/monolog (packagist record) cached = %v, want %v", tt.keys, got, tt.keepMonolog) + } + } + + var ecosystem string + if err := db.Get(&ecosystem, db.Rebind(`SELECT ecosystem FROM packages WHERE purl = ?`), "pkg:composer/monolog/monolog"); err != nil || ecosystem != "packagist" { + t.Errorf("monolog package ecosystem = %q, %v, want packagist", ecosystem, err) + } +} From ff4d5ca10d44e9df221954feae84043b4b7a1cb3 Mon Sep 17 00:00:00 2001 From: Christian Heim Date: Sun, 11 Oct 2026 18:01:15 +0200 Subject: [PATCH 234/633] Enable storage retention for conan Since #431 the handler stores Conan archives under pkg:conan/{name} with the version as the version and the revisions and any user/channel in the filename. purl keeps conan names as they are, so package rules use the default mapping, and pkg:conan/zlib covers every version, revision and user/channel of the recipe. The mirror can't fetch conan packages, so there's no mirror test. Refs #306 --- docs/configuration.md | 2 +- internal/handler/conan.go | 11 ++ internal/server/retention_conan_test.go | 215 ++++++++++++++++++++++++ 3 files changed, 227 insertions(+), 1 deletion(-) create mode 100644 internal/server/retention_conan_test.go diff --git a/docs/configuration.md b/docs/configuration.md index 9dea95a..987a5c7 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -91,7 +91,7 @@ A package rule wins over an ecosystem rule, which wins over `default`. Package r | `alpine` | yes | `pkg:apk/curl` or `pkg:apk/alpine/curl`; case doesn't matter. A rule covers the package in every configured APK repository. Alpine mirrors keep only the current build of each package per release, so an evicted older one can't be fetched again. | | `cargo` | yes | `pkg:cargo/serde`, the crate name as published; case, `-` and `_` have to match, so `pkg:cargo/inflector` doesn't cover `Inflector`. | | `composer` | yes | `pkg:composer/symfony/console`; case doesn't matter. A key without a vendor is refused. | -| `conan` | no | - | +| `conan` | yes | `pkg:conan/zlib`, the recipe name; a rule covers every version, revision and user/channel of it. | | `conda` | no | - | | `cran` | yes | `pkg:cran/ggplot2`, the package name as published, case included (`pkg:cran/Rcpp`). A rule covers source and binary builds alike. | | `deb` | no | - | diff --git a/internal/handler/conan.go b/internal/handler/conan.go index d4040e2..6d88b43 100644 --- a/internal/handler/conan.go +++ b/internal/handler/conan.go @@ -3,12 +3,23 @@ package handler import ( "net/http" "strings" + + "github.com/git-pkgs/proxy/internal/retention" ) const ( conanUpstream = "https://center.conan.io" ) +// Recipe and package archives are stored under pkg:conan/{name}, with the +// version as the version and the revisions and any user/channel in the +// filename, and purl keeps conan names as they are. A package rule therefore +// covers every version, revision and user/channel of a recipe, and uses the +// default key mapping. +func init() { + retention.Default.Register(retention.Spec{Key: "conan", CanonicalPackage: retention.DefaultCanonical("conan")}) +} + // ConanHandler handles Conan registry protocol requests. type ConanHandler struct { proxy *Proxy diff --git a/internal/server/retention_conan_test.go b/internal/server/retention_conan_test.go new file mode 100644 index 0000000..17607bc --- /dev/null +++ b/internal/server/retention_conan_test.go @@ -0,0 +1,215 @@ +package server + +import ( + "context" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/git-pkgs/proxy/internal/config" + "github.com/git-pkgs/proxy/internal/database" + "github.com/git-pkgs/proxy/internal/handler" + "github.com/git-pkgs/proxy/internal/retention" + "github.com/git-pkgs/registries/fetch" +) + +const conanRecipeRevision = "b3b71bfe8dd07abc7b82ff2bd0eac021" + +// conanFile is an archive as the Conan 2 client requests it and the name +// the handler caches it under. +type conanFile struct { + path string + filename string +} + +var ( + conanZlibSources = conanFile{ + "/v2/conans/zlib/1.3.1/_/_/revisions/" + conanRecipeRevision + "/files/conan_sources.tgz", + conanRecipeRevision + "_conan_sources.tgz", + } + conanZlibPackage = conanFile{ + "/v2/conans/zlib/1.3.1/_/_/revisions/" + conanRecipeRevision + "/packages/0123abc/revisions/4567def/files/conan_package.tgz", + conanRecipeRevision + "_0123abc_4567def_conan_package.tgz", + } + conanZlibAcme = conanFile{ + "/v2/conans/zlib/1.2.13/acme/stable/revisions/" + conanRecipeRevision + "/files/conan_export.tgz", + "acme@stable_" + conanRecipeRevision + "_conan_export.tgz", + } + conanOpenSSLSources = conanFile{ + "/v2/conans/openssl/3.3.2/_/_/revisions/" + conanRecipeRevision + "/files/conan_sources.tgz", + "", + } +) + +// conanRetentionUpstream serves the archives the way a Conan 2 server +// does, counting downloads. +func conanRetentionUpstream(t *testing.T) (*httptest.Server, *atomic.Int64) { + t.Helper() + var downloads atomic.Int64 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + for _, f := range []conanFile{conanZlibSources, conanZlibPackage, conanZlibAcme, conanOpenSSLSources} { + if r.URL.Path == f.path { + downloads.Add(1) + _, _ = io.WriteString(w, "conan archive bytes") + return + } + } + http.NotFound(w, r) + })) + t.Cleanup(srv.Close) + return srv, &downloads +} + +func newConanRetentionProxy(t *testing.T, upstream *httptest.Server) (*httptest.Server, *database.DB) { + t.Helper() + db, store := setupEvictionTest(t) + // The upstream is a plain-HTTP loopback server; the default fetcher + // refuses loopback, a stock client reaches it. + fetcher := fetch.NewFetcher(fetch.WithHTTPClient(upstream.Client()), fetch.WithMaxRetries(0)) + t.Cleanup(func() { _ = fetcher.Close() }) + proxy := handler.NewProxy(db, store, fetcher, fetch.NewResolver(), discardLogger()) + proxy.HTTPClient = upstream.Client() + h := handler.NewConanHandlerWithUpstream(proxy, "http://proxy.test", upstream.URL) + srv := httptest.NewServer(h.Routes()) + t.Cleanup(srv.Close) + return srv, db +} + +func conanGet(t *testing.T, srv *httptest.Server, f conanFile) { + t.Helper() + resp, err := http.Get(srv.URL + f.path) + if err != nil { + t.Fatalf("GET %s: %v", f.path, err) + } + body, _ := io.ReadAll(resp.Body) + _ = resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Fatalf("GET %s = %d %s, want 200", f.path, resp.StatusCode, body) + } +} + +// conanArtifact returns the record of the archive of package that the +// handler cached under filename. +func conanArtifact(t *testing.T, db *database.DB, packagePURL, filename string) database.Artifact { + t.Helper() + var art database.Artifact + query := db.Rebind(`SELECT a.* FROM artifacts a JOIN versions v ON v.purl = a.version_purl + WHERE a.filename = ? AND v.package_purl = ?`) + if err := db.Get(&art, query, filename, packagePURL); err != nil { + t.Fatalf("reading %s of %s: %v", filename, packagePURL, err) + } + return art +} + +func conanBackdateAll(t *testing.T, db *database.DB, fetched, accessed time.Time) { + t.Helper() + query := db.Rebind(`UPDATE artifacts SET fetched_at = ?, last_accessed_at = ?`) + if _, err := db.Exec(query, fetched, accessed); err != nil { + t.Fatalf("backdating: %v", err) + } +} + +func TestConanRegistersForRetention(t *testing.T) { + if _, ok := retention.Default.Lookup("conan"); !ok { + t.Fatal("conan is not registered for retention") + } + if key, ok := retention.Default.KeyForDBEcosystem("conan"); !ok || key != "conan" { + t.Errorf("KeyForDBEcosystem(conan) = %q, %v, want conan, true", key, ok) + } +} + +// A package archive served from the cache stays while it is used, is +// evicted once its rule runs out, comes back from upstream on the next +// download, and stays after that even with the evicted copy's old access +// time on the record. +func TestConanRetentionEvictsAndRefetches(t *testing.T) { + upstream, downloads := conanRetentionUpstream(t) + srv, db := newConanRetentionProxy(t, upstream) + cached := func() database.Artifact { + return conanArtifact(t, db, "pkg:conan/zlib", conanZlibPackage.filename) + } + + conanGet(t, srv, conanZlibPackage) + conanGet(t, srv, conanZlibPackage) + if got := downloads.Load(); got != 1 { + t.Fatalf("upstream downloads = %d, want 1", got) + } + if !cached().LastAccessedAt.Valid { + t.Fatal("cache hit did not record an access") + } + + rules := retention.Rules{Ecosystems: map[string]time.Duration{"conan": retentionDay}} + now := time.Now() + sweepRetention(context.Background(), db, discardLogger(), retention.Default, rules, now) + if !cached().StoragePath.Valid { + t.Fatal("archive used today was evicted") + } + + old := now.Add(-2 * retentionDay) + conanBackdateAll(t, db, old, old) + sweepRetention(context.Background(), db, discardLogger(), retention.Default, rules, now) + if cached().StoragePath.Valid { + t.Fatal("archive unused for two days kept under a one-day rule") + } + + conanGet(t, srv, conanZlibPackage) + if got := downloads.Load(); got != 2 { + t.Errorf("upstream downloads = %d, want 2 after the eviction", got) + } + conanBackdateAll(t, db, cached().FetchedAt.Time, now.Add(-30*retentionDay)) + sweepRetention(context.Background(), db, discardLogger(), retention.Default, rules, time.Now()) + if !cached().StoragePath.Valid { + t.Error("refetched archive evicted again by the next sweep") + } +} + +// A key with a namespace parses as a conan PURL but can never match a +// stored recipe, so it is refused at validation. A plain key is accepted. +func TestConanRetentionRefusesNamespacedKeys(t *testing.T) { + cfg := config.Default() + cfg.Storage.Retention.Packages = map[string]string{"pkg:conan/zlib": "0"} + if _, err := cfg.RetentionRules(retention.Default); err != nil { + t.Fatalf("RetentionRules(pkg:conan/zlib): %v", err) + } + cfg.Storage.Retention.Packages = map[string]string{"pkg:conan/acme/zlib": "0"} + _, err := cfg.RetentionRules(retention.Default) + if err == nil || !strings.Contains(err.Error(), "does not match how conan packages are stored") { + t.Errorf("RetentionRules(pkg:conan/acme/zlib) error = %v, want the key refused", err) + } +} + +// A package rule covers a recipe's sources, its binary packages and its +// builds under another user/channel, while other recipes fall under the +// conan rule. +func TestConanRetentionPackageRule(t *testing.T) { + upstream, _ := conanRetentionUpstream(t) + srv, db := newConanRetentionProxy(t, upstream) + zlib := []conanFile{conanZlibSources, conanZlibPackage, conanZlibAcme} + for _, f := range append(zlib, conanOpenSSLSources) { + conanGet(t, srv, f) + } + old := time.Now().Add(-30 * retentionDay) + conanBackdateAll(t, db, old, old) + + cfg := config.Default() + cfg.Storage.Retention.Ecosystems = map[string]string{"conan": "1d"} + cfg.Storage.Retention.Packages = map[string]string{"pkg:conan/zlib": "0"} + rules, err := cfg.RetentionRules(retention.Default) + if err != nil { + t.Fatalf("RetentionRules: %v", err) + } + sweepRetention(context.Background(), db, discardLogger(), retention.Default, rules, time.Now()) + + for _, f := range zlib { + if !conanArtifact(t, db, "pkg:conan/zlib", f.filename).StoragePath.Valid { + t.Errorf("%s evicted despite the never-evict rule for zlib", f.filename) + } + } + if conanArtifact(t, db, "pkg:conan/openssl", conanRecipeRevision+"_conan_sources.tgz").StoragePath.Valid { + t.Error("openssl kept past the conan rule") + } +} From f8ca190274fbc5bfbda4d70d0158aef11e83b240 Mon Sep 17 00:00:00 2001 From: Christian Heim Date: Sun, 11 Oct 2026 01:34:46 +0200 Subject: [PATCH 235/633] Enable storage retention for swift Source archives are stored under the registry identity, pkg:generic/swift-registry/{scope}.{name}, lowercased. Package rules write the scope as the namespace (pkg:swift/apple/swift-nio) or use the stored form, in any case. A source repository key like pkg:swift/github.com/apple/swift-nio doesn't name a registry identity and is refused. The mirror can't fetch swift packages, so there's no mirror test. Refs #306 --- docs/configuration.md | 2 +- internal/handler/swift.go | 37 +++++ internal/handler/swift_test.go | 35 ++++ internal/server/retention_swift_test.go | 212 ++++++++++++++++++++++++ 4 files changed, 285 insertions(+), 1 deletion(-) create mode 100644 internal/server/retention_swift_test.go diff --git a/docs/configuration.md b/docs/configuration.md index fa55734..c4cd0e6 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -107,7 +107,7 @@ A package rule wins over an ecosystem rule, which wins over `default`. Package r | `pub` | yes | `pkg:pub/http`, the package name as published. | | `pypi` | yes | `pkg:pypi/requests`; case and single `_`/`-` don't matter. Dots and repeated separators are kept as written, so a project like `zope.interface` can be stored as `zope.interface` and `zope-interface`; add a rule for each. | | `rpm` | yes | `pkg:rpm/curl`, the name as packaged, case included (`pkg:rpm/NetworkManager`). A vendor namespace such as `pkg:rpm/fedora/curl` is accepted and dropped. Fedora and EPEL update repositories keep only the latest build of a package, so an evicted older one may not be fetchable again. | -| `swift` | no | - | +| `swift` | yes | `pkg:swift/apple/swift-nio`, with the registry scope as the namespace, or the stored form `pkg:generic/swift-registry/apple.swift-nio`; case doesn't matter. A source repository key like `pkg:swift/github.com/apple/swift-nio` is refused, since it doesn't name a registry identity. | An evicted artifact is fetched again from upstream on the next request. Packages preloaded with `proxy mirror` age from the time they were mirrored, so an unused mirrored package is evicted like any other. Running `proxy mirror` again over a package that is still cached counts as a download and restarts its clock, so packages a scheduled mirror job keeps listing never age out. A registry that removes old releases (Linux distribution archives in particular) cannot serve an evicted version again, so set those ecosystems' rules with that in mind. diff --git a/internal/handler/swift.go b/internal/handler/swift.go index 6cc7d78..bd46fab 100644 --- a/internal/handler/swift.go +++ b/internal/handler/swift.go @@ -15,6 +15,8 @@ import ( "github.com/git-pkgs/proxy/internal/config" "github.com/git-pkgs/proxy/internal/packageurl" + "github.com/git-pkgs/proxy/internal/retention" + "github.com/git-pkgs/purl" ) const ( @@ -26,6 +28,41 @@ const ( swiftMaxNameLength = 100 ) +// Source archives are stored under the registry identity, +// pkg:generic/swift-registry/{scope}.{name}, lowercased. Package rules write +// the scope as the namespace, pkg:swift/apple/swift-nio, or use the stored +// form. A source repository URL such as pkg:swift/github.com/apple/swift-nio +// doesn't name a registry identity and is refused. +func init() { + retention.Default.Register(retention.Spec{Key: "swift", CanonicalPackage: swiftRetentionKey}) +} + +// swiftRetentionKey maps a package rule to the stored package PURL, refusing +// keys that name no valid registry scope and package. MakeCacheStrings +// lowercases the identity the way the handler stores it. +func swiftRetentionKey(p *purl.PURL) (string, bool) { + if p == nil { + return "", false + } + var scope, name string + switch { + case p.Type == "swift": + scope, name = p.Namespace, p.Name + case p.Type == "generic" && strings.EqualFold(p.Namespace, "swift-registry"): + var found bool + if scope, name, found = strings.Cut(p.Name, "."); !found { + return "", false + } + default: + return "", false + } + if !validSwiftScope(scope) || !validSwiftPackageName(name) { + return "", false + } + packagePURL, _ := packageurl.MakeCacheStrings("swift", scope+"/"+name, "") + return packagePURL, packagePURL != "" +} + // SwiftHandler handles the read-only Swift Package Registry v1 protocol. type SwiftHandler struct { proxy *Proxy diff --git a/internal/handler/swift_test.go b/internal/handler/swift_test.go index c9df35a..199c712 100644 --- a/internal/handler/swift_test.go +++ b/internal/handler/swift_test.go @@ -14,6 +14,7 @@ import ( "testing" "github.com/git-pkgs/proxy/internal/packageurl" + "github.com/git-pkgs/purl" "github.com/git-pkgs/registries/fetch" ) @@ -548,3 +549,37 @@ func TestSwiftIdentifierValidation(t *testing.T) { }) } } + +func TestSwiftRetentionKey(t *testing.T) { + tests := []struct { + key string + want string + ok bool + }{ + {"pkg:swift/apple/swift-nio", "pkg:generic/swift-registry/apple.swift-nio", true}, + {"pkg:swift/Apple/Swift-NIO", "pkg:generic/swift-registry/apple.swift-nio", true}, + {"pkg:generic/swift-registry/apple.swift-nio", "pkg:generic/swift-registry/apple.swift-nio", true}, + {"pkg:generic/swift-registry/Apple.Swift-NIO", "pkg:generic/swift-registry/apple.swift-nio", true}, + {"pkg:generic/Swift-Registry/apple.swift-nio", "pkg:generic/swift-registry/apple.swift-nio", true}, + {"pkg:swift/github.com/apple/swift-nio", "", false}, + {"pkg:generic/swift-registry/swift-nio", "", false}, + {"pkg:generic/swift-registry/apple.swift.nio", "", false}, + {"pkg:swift/apple/-nio", "", false}, + {"pkg:generic/apple.swift-nio", "", false}, + {"pkg:generic/other/apple.swift-nio", "", false}, + {"pkg:npm/apple/swift-nio", "", false}, + } + for _, tt := range tests { + p, err := purl.Parse(tt.key) + if err != nil { + t.Fatalf("Parse(%s): %v", tt.key, err) + } + got, ok := swiftRetentionKey(p) + if ok != tt.ok || (ok && got != tt.want) { + t.Errorf("swiftRetentionKey(%s) = %q, %v, want %q, %v", tt.key, got, ok, tt.want, tt.ok) + } + } + if _, ok := swiftRetentionKey(nil); ok { + t.Error("swiftRetentionKey(nil) accepted") + } +} diff --git a/internal/server/retention_swift_test.go b/internal/server/retention_swift_test.go new file mode 100644 index 0000000..441320a --- /dev/null +++ b/internal/server/retention_swift_test.go @@ -0,0 +1,212 @@ +package server + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/git-pkgs/proxy/internal/config" + "github.com/git-pkgs/proxy/internal/database" + "github.com/git-pkgs/proxy/internal/handler" + "github.com/git-pkgs/proxy/internal/retention" + "github.com/git-pkgs/registries/fetch" +) + +// swiftRetentionUpstream serves release metadata and source archives for +// apple/swift-nio 2.65.0 and apple/swift-log 1.5.4 the way a Swift package +// registry does, counting archive downloads. +func swiftRetentionUpstream(t *testing.T) (*httptest.Server, *atomic.Int64) { + t.Helper() + var downloads atomic.Int64 + archives := map[string]string{ + "/apple/swift-nio/2.65.0": "swift-nio archive", + "/apple/swift-log/1.5.4": "swift-log archive", + } + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if archive, ok := archives[strings.TrimSuffix(r.URL.Path, ".zip")]; ok { + if strings.HasSuffix(r.URL.Path, ".zip") { + downloads.Add(1) + w.Header().Set("Content-Type", "application/zip") + _, _ = io.WriteString(w, archive) + return + } + sum := sha256.Sum256([]byte(archive)) + w.Header().Set("Content-Type", "application/json") + w.Header().Set("Content-Version", "1") + _, _ = fmt.Fprintf(w, `{"resources":[{"name":"source-archive","type":"application/zip","checksum":%q}]}`, hex.EncodeToString(sum[:])) + return + } + http.NotFound(w, r) + })) + t.Cleanup(srv.Close) + return srv, &downloads +} + +func newSwiftRetentionProxy(t *testing.T, upstream *httptest.Server) (*httptest.Server, *database.DB) { + t.Helper() + db, store := setupEvictionTest(t) + // The upstream is a plain-HTTP loopback server; the default fetcher + // refuses loopback, a stock client reaches it. + fetcher := fetch.NewFetcher(fetch.WithHTTPClient(upstream.Client()), fetch.WithMaxRetries(0)) + t.Cleanup(func() { _ = fetcher.Close() }) + proxy := handler.NewProxy(db, store, fetcher, fetch.NewResolver(), discardLogger()) + proxy.HTTPClient = upstream.Client() + h := handler.NewSwiftHandler(proxy, "http://proxy.test", upstream.URL) + srv := httptest.NewServer(h.Routes()) + t.Cleanup(srv.Close) + return srv, db +} + +func swiftGet(t *testing.T, url string) { + t.Helper() + req, err := http.NewRequest(http.MethodGet, url, nil) + if err != nil { + t.Fatal(err) + } + req.Header.Set("Accept", "application/vnd.swift.registry.v1+zip") + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatalf("GET %s: %v", url, err) + } + body, _ := io.ReadAll(resp.Body) + _ = resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Fatalf("GET %s = %d %s, want 200", url, resp.StatusCode, body) + } +} + +// swiftArtifact returns the record of the archive named filename. +func swiftArtifact(t *testing.T, db *database.DB, filename string) database.Artifact { + t.Helper() + var art database.Artifact + if err := db.Get(&art, db.Rebind(`SELECT * FROM artifacts WHERE filename = ?`), filename); err != nil { + t.Fatalf("reading artifact %s: %v", filename, err) + } + return art +} + +func swiftBackdate(t *testing.T, db *database.DB, filename string, fetched, accessed time.Time) { + t.Helper() + query := db.Rebind(`UPDATE artifacts SET fetched_at = ?, last_accessed_at = ? WHERE filename = ?`) + if _, err := db.Exec(query, fetched, accessed, filename); err != nil { + t.Fatalf("backdating %s: %v", filename, err) + } +} + +func TestSwiftRegistersForRetention(t *testing.T) { + if _, ok := retention.Default.Lookup("swift"); !ok { + t.Fatal("swift is not registered for retention") + } + if key, ok := retention.Default.KeyForDBEcosystem("swift"); !ok || key != "swift" { + t.Errorf("KeyForDBEcosystem(swift) = %q, %v, want swift, true", key, ok) + } +} + +// An archive served from the cache stays while it is used, is evicted once +// its rule runs out, comes back from upstream on the next download, and +// stays after that even with the evicted copy's old access time on the +// record. +func TestSwiftRetentionEvictsAndRefetches(t *testing.T) { + upstream, downloads := swiftRetentionUpstream(t) + srv, db := newSwiftRetentionProxy(t, upstream) + const archive = "swift-nio-2.65.0.zip" + + swiftGet(t, srv.URL+"/apple/swift-nio/2.65.0.zip") + swiftGet(t, srv.URL+"/apple/swift-nio/2.65.0.zip") + if got := downloads.Load(); got != 1 { + t.Fatalf("upstream downloads = %d, want 1", got) + } + if !swiftArtifact(t, db, archive).LastAccessedAt.Valid { + t.Fatal("cache hit did not record an access") + } + + rules := retention.Rules{Ecosystems: map[string]time.Duration{"swift": retentionDay}} + now := time.Now() + sweepRetention(context.Background(), db, discardLogger(), retention.Default, rules, now) + if !swiftArtifact(t, db, archive).StoragePath.Valid { + t.Fatal("archive used today was evicted") + } + + old := now.Add(-2 * retentionDay) + swiftBackdate(t, db, archive, old, old) + sweepRetention(context.Background(), db, discardLogger(), retention.Default, rules, now) + if swiftArtifact(t, db, archive).StoragePath.Valid { + t.Fatal("archive unused for two days kept under a one-day rule") + } + + swiftGet(t, srv.URL+"/apple/swift-nio/2.65.0.zip") + if got := downloads.Load(); got != 2 { + t.Errorf("upstream downloads = %d, want 2 after the eviction", got) + } + art := swiftArtifact(t, db, archive) + swiftBackdate(t, db, archive, art.FetchedAt.Time, now.Add(-30*retentionDay)) + sweepRetention(context.Background(), db, discardLogger(), retention.Default, rules, time.Now()) + if !swiftArtifact(t, db, archive).StoragePath.Valid { + t.Error("refetched archive evicted again by the next sweep") + } +} + +// A source repository key parses as a swift PURL but names no registry +// identity, so it is refused at validation. A scope/name key is accepted. +func TestSwiftRetentionRefusesRepositoryKeys(t *testing.T) { + cfg := config.Default() + cfg.Storage.Retention.Packages = map[string]string{"pkg:swift/apple/swift-nio": "0"} + if _, err := cfg.RetentionRules(retention.Default); err != nil { + t.Fatalf("RetentionRules(pkg:swift/apple/swift-nio): %v", err) + } + cfg.Storage.Retention.Packages = map[string]string{"pkg:swift/github.com/apple/swift-nio": "0"} + _, err := cfg.RetentionRules(retention.Default) + if err == nil || !strings.Contains(err.Error(), "does not match how swift packages are stored") { + t.Errorf("RetentionRules(pkg:swift/github.com/apple/swift-nio) error = %v, want the key refused", err) + } +} + +// A package rule reaches the stored archive in either documented form and +// whatever case it is written in, even when the client asked in another case. +func TestSwiftRetentionPackageRules(t *testing.T) { + upstream, _ := swiftRetentionUpstream(t) + srv, db := newSwiftRetentionProxy(t, upstream) + const nio, log = "swift-nio-2.65.0.zip", "swift-log-1.5.4.zip" + + swiftGet(t, srv.URL+"/Apple/Swift-NIO/2.65.0.zip") + var stored string + art := swiftArtifact(t, db, nio) + if err := db.Get(&stored, db.Rebind(`SELECT package_purl FROM versions WHERE purl = ?`), art.VersionPURL); err != nil { + t.Fatalf("reading stored package PURL: %v", err) + } + if stored != "pkg:generic/swift-registry/apple.swift-nio" { + t.Errorf("stored package PURL = %q, want pkg:generic/swift-registry/apple.swift-nio", stored) + } + + for _, key := range []string{"pkg:swift/Apple/Swift-NIO", "pkg:generic/swift-registry/apple.swift-nio"} { + swiftGet(t, srv.URL+"/apple/swift-nio/2.65.0.zip") + swiftGet(t, srv.URL+"/apple/swift-log/1.5.4.zip") + old := time.Now().Add(-30 * retentionDay) + swiftBackdate(t, db, nio, old, old) + swiftBackdate(t, db, log, old, old) + + cfg := config.Default() + cfg.Storage.Retention.Ecosystems = map[string]string{"swift": "1d"} + cfg.Storage.Retention.Packages = map[string]string{key: "0"} + rules, err := cfg.RetentionRules(retention.Default) + if err != nil { + t.Fatalf("RetentionRules(%s): %v", key, err) + } + sweepRetention(context.Background(), db, discardLogger(), retention.Default, rules, time.Now()) + + if !swiftArtifact(t, db, nio).StoragePath.Valid { + t.Errorf("with %s: swift-nio evicted despite its never-evict rule", key) + } + if swiftArtifact(t, db, log).StoragePath.Valid { + t.Errorf("with %s: swift-log kept past the swift rule", key) + } + } +} From b131db394d87f86fa0054df6c6d1bbec7fe4f776 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Mon, 12 Oct 2026 18:44:12 +0200 Subject: [PATCH 236/633] Isolate the recent failure history in tests The history is process-wide, and tests cleared it only after running, and not all of them. Records from earlier tests or an earlier -count repetition reached later assertions: TestRecordUpstreamOperations left a record for TestRecentFailuresNewestFirst, and the server tests never cleared it. - Export ResetRecentFailures so tests outside the metrics package can use it - Clear the history before and after every test that records or reads failures directly, in metrics, server and handler - Assert exactly one failure in the handler tests instead of looking only at the newest --- internal/handler/handler_test.go | 19 ++++++++++++++---- internal/metrics/failures.go | 5 ++++- internal/metrics/failures_test.go | 23 +++++++++++++++------- internal/metrics/metrics_test.go | 2 ++ internal/metrics/snapshot_test.go | 1 + internal/server/analytics_coverage_test.go | 1 + internal/server/analytics_failures_test.go | 13 ++++++++++++ 7 files changed, 52 insertions(+), 12 deletions(-) diff --git a/internal/handler/handler_test.go b/internal/handler/handler_test.go index 2dc9a75..ffc0d5f 100644 --- a/internal/handler/handler_test.go +++ b/internal/handler/handler_test.go @@ -990,7 +990,17 @@ func TestGetOrFetchArtifactFromURL_CacheMiss(t *testing.T) { } } +// isolateRecentFailures empties the process-wide failure history for the +// test and again when it ends, so neither earlier tests nor repeated runs +// reach its assertions. +func isolateRecentFailures(t *testing.T) { + t.Helper() + metrics.ResetRecentFailures() + t.Cleanup(metrics.ResetRecentFailures) +} + func TestGetOrFetchArtifactFromURL_FetchError(t *testing.T) { + isolateRecentFailures(t) proxy, _, _, fetcher := setupTestProxy(t) fetcher.fetchErr = errors.New("connection refused") errorsBefore := testutil.ToFloat64(metrics.UpstreamErrors.WithLabelValues("pypi", "fetch_failed")) @@ -1006,13 +1016,14 @@ func TestGetOrFetchArtifactFromURL_FetchError(t *testing.T) { t.Errorf("upstream errors delta = %.0f, want 1", diff) } recent := metrics.RecentFailures("proxy_upstream_errors_total", map[string]string{"ecosystem": "pypi", "error_type": "fetch_failed"}) - if len(recent) == 0 || recent[0].PURL != "pkg:pypi/fail@1.0.0" || + if len(recent) != 1 || recent[0].PURL != "pkg:pypi/fail@1.0.0" || recent[0].URL != "https://pypi.org/files/fail-1.0.0.tar.gz" || recent[0].Error != "connection refused" { - t.Errorf("latest upstream failure = %+v, want the failed package, its URL and the error", recent) + t.Errorf("upstream failures = %+v, want one: the failed package, its URL and the error", recent) } } func TestGetOrFetchArtifactFromURL_StoreError(t *testing.T) { + isolateRecentFailures(t) proxy, _, store, fetcher := setupTestProxy(t) store.storeErr = errors.New("disk full") errorsBefore := testutil.ToFloat64(metrics.StorageErrors.WithLabelValues("write")) @@ -1032,9 +1043,9 @@ func TestGetOrFetchArtifactFromURL_StoreError(t *testing.T) { t.Errorf("storage errors delta = %.0f, want 1", diff) } recent := metrics.RecentFailures("proxy_storage_errors_total", map[string]string{"operation": "write"}) - if len(recent) == 0 || recent[0].PURL != "pkg:pypi/fail@1.0.0" || recent[0].Filename != "fail-1.0.0.tar.gz" || + if len(recent) != 1 || recent[0].PURL != "pkg:pypi/fail@1.0.0" || recent[0].Filename != "fail-1.0.0.tar.gz" || recent[0].Path == "" || recent[0].Error != "disk full" { - t.Errorf("latest storage failure = %+v, want the failed package, its storage path and the error", recent) + t.Errorf("storage failures = %+v, want one: the failed package, its storage path and the error", recent) } } diff --git a/internal/metrics/failures.go b/internal/metrics/failures.go index e1e207b..425ba8e 100644 --- a/internal/metrics/failures.go +++ b/internal/metrics/failures.go @@ -82,7 +82,10 @@ func recordFailure(name string, labels map[string]string, detail FailureDetail) recentFailures.series[key] = stored } -func resetRecentFailures() { +// ResetRecentFailures empties the recent failure history. The history is +// process-wide, so tests that record or read failures call it to start from a +// known state and to leave none behind. +func ResetRecentFailures() { recentFailures.Lock() defer recentFailures.Unlock() recentFailures.series = make(map[string][]Failure) diff --git a/internal/metrics/failures_test.go b/internal/metrics/failures_test.go index 5e4b60d..3d6ab88 100644 --- a/internal/metrics/failures_test.go +++ b/internal/metrics/failures_test.go @@ -9,8 +9,17 @@ import ( "github.com/prometheus/client_golang/prometheus/testutil" ) +// isolateRecentFailures empties the process-wide failure history for the +// test and again when it ends, so neither earlier tests nor repeated runs +// reach its assertions. +func isolateRecentFailures(t *testing.T) { + t.Helper() + ResetRecentFailures() + t.Cleanup(ResetRecentFailures) +} + func TestRecentFailuresNewestFirst(t *testing.T) { - t.Cleanup(resetRecentFailures) + isolateRecentFailures(t) RecordUpstreamError("npm", "fetch_failed", FailureDetail{PURL: "pkg:npm/left-pad@1.0.0"}) RecordUpstreamError("npm", "fetch_failed", FailureDetail{PURL: "pkg:npm/right-pad@2.0.0"}) @@ -28,7 +37,7 @@ func TestRecentFailuresNewestFirst(t *testing.T) { } func TestRecentFailuresAreBoundedPerSeries(t *testing.T) { - t.Cleanup(resetRecentFailures) + isolateRecentFailures(t) for i := range recentFailuresPerSeries + 5 { RecordStorageError("write", FailureDetail{Path: fmt.Sprintf("npm/pkg-%d.tgz", i)}) @@ -44,7 +53,7 @@ func TestRecentFailuresAreBoundedPerSeries(t *testing.T) { } func TestRecentFailuresAreKeptPerSeries(t *testing.T) { - t.Cleanup(resetRecentFailures) + isolateRecentFailures(t) RecordIntegrityFailure("npm", FailureDetail{PURL: "pkg:npm/a@1"}) RecordIntegrityFailure("pypi", FailureDetail{PURL: "pkg:pypi/b@1"}) @@ -59,7 +68,7 @@ func TestRecentFailuresAreKeptPerSeries(t *testing.T) { } func TestRecordFailureStillCountsTheMetric(t *testing.T) { - t.Cleanup(resetRecentFailures) + isolateRecentFailures(t) before := testutil.ToFloat64(HealthProbeFailures.WithLabelValues("read")) RecordHealthProbeFailure("read", FailureDetail{Err: errors.New("disk gone")}) @@ -74,7 +83,7 @@ func TestRecordFailureStillCountsTheMetric(t *testing.T) { } func TestScanErrorRecordsDetail(t *testing.T) { - t.Cleanup(resetRecentFailures) + isolateRecentFailures(t) RecordScanError("NPM", "clamav", "timeout", FailureDetail{PURL: "pkg:npm/a@1", Filename: "a-1.tgz"}) @@ -88,7 +97,7 @@ func TestScanErrorRecordsDetail(t *testing.T) { // The analytics page is unauthenticated, and private registry URLs carry // credentials in the userinfo or the query string. Neither may be kept. func TestRecentFailuresStripCredentials(t *testing.T) { - t.Cleanup(resetRecentFailures) + isolateRecentFailures(t) RecordUpstreamError("npm", "fetch_failed", FailureDetail{ URL: "https://user:s3cret@registry.example.com/pkg/-/pkg-1.0.0.tgz?token=abc#frag", @@ -113,7 +122,7 @@ func TestRecentFailuresStripCredentials(t *testing.T) { } func TestRecentFailuresReturnsACopy(t *testing.T) { - t.Cleanup(resetRecentFailures) + isolateRecentFailures(t) RecordStorageError("read", FailureDetail{Path: "a"}) labels := map[string]string{"operation": "read"} diff --git a/internal/metrics/metrics_test.go b/internal/metrics/metrics_test.go index 60b7ed5..cca39c8 100644 --- a/internal/metrics/metrics_test.go +++ b/internal/metrics/metrics_test.go @@ -30,6 +30,7 @@ func TestRecordCacheOperations(t *testing.T) { } func TestRecordUpstreamOperations(t *testing.T) { + isolateRecentFailures(t) RecordUpstreamFetch("npm", 500*time.Millisecond) RecordUpstreamFetch("pypi", 1*time.Second) RecordUpstreamError("npm", "fetch_failed", FailureDetail{}) @@ -39,6 +40,7 @@ func TestRecordUpstreamOperations(t *testing.T) { } func TestRecordStorageOperations(t *testing.T) { + isolateRecentFailures(t) RecordStorageOperation("read", 10*time.Millisecond) RecordStorageOperation("write", 50*time.Millisecond) RecordStorageError("read", FailureDetail{}) diff --git a/internal/metrics/snapshot_test.go b/internal/metrics/snapshot_test.go index fcefad3..1a20906 100644 --- a/internal/metrics/snapshot_test.go +++ b/internal/metrics/snapshot_test.go @@ -124,6 +124,7 @@ func TestSnapshotMissingMetric(t *testing.T) { // Gather reads the real registry, so every metric this package registers must // come back. This is the check that a newly added metric is reachable by the UI. func TestGatherSeesRegisteredMetrics(t *testing.T) { + isolateRecentFailures(t) RecordCacheHit("npm") RecordScanError("npm", "clamav", "timeout", FailureDetail{}) diff --git a/internal/server/analytics_coverage_test.go b/internal/server/analytics_coverage_test.go index 0a5cf1e..12cc895 100644 --- a/internal/server/analytics_coverage_test.go +++ b/internal/server/analytics_coverage_test.go @@ -54,6 +54,7 @@ var metricSurface = map[string]string{ // on the analytics page. The page is meant to be a complete view of what // /metrics exposes, so a silently unsurfaced metric is a gap, not a detail. func TestEveryMetricIsSurfaced(t *testing.T) { + isolateRecentFailures(t) // Touch every metric family so it is present in the registry output, since // a vector with no observed label values gathers as nothing at all. metrics.RecordRequest("npm", 200, 0) diff --git a/internal/server/analytics_failures_test.go b/internal/server/analytics_failures_test.go index 4f60724..6f7856e 100644 --- a/internal/server/analytics_failures_test.go +++ b/internal/server/analytics_failures_test.go @@ -9,6 +9,15 @@ import ( "github.com/prometheus/client_golang/prometheus" ) +// isolateRecentFailures empties the process-wide failure history for the +// test and again when it ends, so neither earlier tests nor repeated runs +// reach its assertions. +func isolateRecentFailures(t *testing.T) { + t.Helper() + metrics.ResetRecentFailures() + t.Cleanup(metrics.ResetRecentFailures) +} + func storageErrorSnapshot(t *testing.T, operation string) *metrics.Snapshot { t.Helper() reg := prometheus.NewRegistry() @@ -25,6 +34,7 @@ func storageErrorSnapshot(t *testing.T, operation string) *metrics.Snapshot { } func TestRuntimeViewAttachesRecentFailuresWhenEnabled(t *testing.T) { + isolateRecentFailures(t) const operation = "test-attach" metrics.RecordStorageError(operation, metrics.FailureDetail{ PURL: "pkg:npm/left-pad@1.0.0", Filename: "left-pad-1.0.0.tgz", Path: "npm/left-pad", Err: errors.New("disk full")}) @@ -45,6 +55,7 @@ func TestRuntimeViewAttachesRecentFailuresWhenEnabled(t *testing.T) { } func TestRuntimeViewOmitsRecentFailuresWhenDisabled(t *testing.T) { + isolateRecentFailures(t) const operation = "test-omit" metrics.RecordStorageError(operation, metrics.FailureDetail{PURL: "pkg:npm/left-pad@1.0.0"}) @@ -61,6 +72,7 @@ func TestRuntimeViewOmitsRecentFailuresWhenDisabled(t *testing.T) { // Package names and upstream URLs say what the proxy's users depend on, so like // the caller table they are published only when asked for. func TestAnalyticsPageHidesFailureDetailsByDefault(t *testing.T) { + isolateRecentFailures(t) ts := newTestServer(t) defer ts.close() @@ -79,6 +91,7 @@ func TestAnalyticsPageHidesFailureDetailsByDefault(t *testing.T) { } func TestAnalyticsPageShowsFailureDetailsWhenEnabled(t *testing.T) { + isolateRecentFailures(t) ts := newTestServer(t) defer ts.close() From 680a2daba168ae8acfe6df5c5219c2b01e1bb1ec Mon Sep 17 00:00:00 2001 From: Christian Heim Date: Mon, 12 Oct 2026 19:55:10 +0200 Subject: [PATCH 237/633] Fix gem filenames whose platform ends in a number parseGemFilename split at the last hyphen before a digit, so sorbet-static-0.5.11002-universal-darwin-22.gem was stored as package sorbet-static-0.5.11002-universal-darwin. It now splits at the first one, which only misreads gem names with a - segment. The docs caveat is gone. Refs #306 --- docs/configuration.md | 2 +- internal/handler/gem.go | 20 ++++++++++---------- internal/handler/gem_test.go | 6 ++++++ 3 files changed, 17 insertions(+), 11 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 06e0b7c..36f4e79 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -95,7 +95,7 @@ A package rule wins over an ecosystem rule, which wins over `default`. Package r | `conda` | no | - | | `cran` | no | - | | `deb` | no | - | -| `gem` | yes | `pkg:gem/rails`, the gem name as published, case included. Package rules don't reach gem files whose platform ends in a number (`universal-darwin-22`, `universal-java-17`) yet, which the proxy files under a wrong package name; the gem rule and `default` still cover them. | +| `gem` | yes | `pkg:gem/rails`, the gem name as published, case included. | | `golang` | yes | `pkg:golang/github.com/stretchr/testify`, the module path as in `go.mod`; case doesn't matter. A major version like `/v2` (or `.v3` on gopkg.in) is a module of its own and needs its own rule. The `!`-escaped form from proxy URLs and the module cache (`github.com/!azure/...`) is refused. | | `helm` | no | - | | `hex` | no | - | diff --git a/internal/handler/gem.go b/internal/handler/gem.go index 93e9989..0657b6b 100644 --- a/internal/handler/gem.go +++ b/internal/handler/gem.go @@ -17,10 +17,8 @@ const ( // Gems are stored under pkg:gem/{name} with the name from the gem filename, // and purl keeps gem names as they are, so package rules use the default -// key mapping. parseGemFilename splits at the last hyphen before a digit, -// so a platform like universal-darwin-22 lands under a wrong package name -// that package rules don't reach. The spec also claims the "rubygems" -// spelling git-pkgs writes into packages.ecosystem. +// key mapping. The spec also claims the "rubygems" spelling git-pkgs writes +// into packages.ecosystem. func init() { retention.Default.Register(retention.Spec{Key: "gem", CanonicalPackage: retention.DefaultCanonical("gem")}) } @@ -89,14 +87,16 @@ func (h *GemHandler) handleDownload(w http.ResponseWriter, r *http.Request) { // parseGemFilename extracts name and version from a gem filename. // e.g., "rails-7.1.0.gem" -> ("rails", "7.1.0") // e.g., "aws-sdk-s3-1.142.0.gem" -> ("aws-sdk-s3", "1.142.0") +// e.g., "sorbet-static-0.5.11002-universal-darwin-22.gem" -> ("sorbet-static", "0.5.11002-universal-darwin-22") +// +// The version starts at the first hyphen followed by a digit. Platforms can +// end in a number (universal-java-17, arm64-darwin-23), so splitting at the +// last one would cut the platform; only gem names with a "-" segment +// are misread this way, and those are rare. func (h *GemHandler) parseGemFilename(filename string) (name, version string) { - // Remove .gem extension base := strings.TrimSuffix(filename, ".gem") - - // Find the last hyphen followed by a version number - // Version starts with a digit - for i := len(base) - 1; i >= 0; i-- { - if base[i] == '-' && i+1 < len(base) && base[i+1] >= '0' && base[i+1] <= '9' { + for i := 0; i+1 < len(base); i++ { + if base[i] == '-' && base[i+1] >= '0' && base[i+1] <= '9' { return base[:i], base[i+1:] } } diff --git a/internal/handler/gem_test.go b/internal/handler/gem_test.go index 7d90946..ce63deb 100644 --- a/internal/handler/gem_test.go +++ b/internal/handler/gem_test.go @@ -25,6 +25,12 @@ func TestGemParseFilename(t *testing.T) { {"aws-sdk-s3-1.142.0.gem", "aws-sdk-s3", "1.142.0"}, {"nokogiri-1.15.4-x86_64-linux.gem", "nokogiri", "1.15.4-x86_64-linux"}, {"activerecord-7.0.8.gem", "activerecord", "7.0.8"}, + {"sorbet-static-0.5.11002-universal-darwin-22.gem", "sorbet-static", "0.5.11002-universal-darwin-22"}, + {"jruby-openssl-0.14.2-universal-java-17.gem", "jruby-openssl", "0.14.2-universal-java-17"}, + {"google-protobuf-3.25.3-arm64-darwin-23.gem", "google-protobuf", "3.25.3-arm64-darwin-23"}, + {"nokogiri-1.16.0-x64-mingw-ucrt.gem", "nokogiri", "1.16.0-x64-mingw-ucrt"}, + {"rake-13.1.0.pre.1.gem", "rake", "13.1.0.pre.1"}, + {"no-version.gem", "", ""}, {"invalid", "", ""}, } From 5d7855ffc831bd30d6f37b4f8937c6c161eea87f Mon Sep 17 00:00:00 2001 From: Christian Heim Date: Mon, 12 Oct 2026 19:56:38 +0200 Subject: [PATCH 238/633] Store mirrored NuGet ids lowercased The mirror passed ids through as written, so pkg:nuget/Newtonsoft.Json@13.0.3 was stored under a copy no client is served while the handler kept serving its lowercased one. GetOrFetchArtifact now lowercases nuget names, and the docs no longer ask for lowercase ids in mirror lists. Refs #306 --- docs/configuration.md | 2 +- internal/handler/handler.go | 6 ++++ internal/server/retention_nuget_test.go | 37 +++++++++++++++++++++++++ 3 files changed, 44 insertions(+), 1 deletion(-) diff --git a/docs/configuration.md b/docs/configuration.md index 9766f30..5f39da5 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -102,7 +102,7 @@ A package rule wins over an ecosystem rule, which wins over `default`. Package r | `julia` | no | - | | `maven` | yes | `pkg:maven/com.google.guava/guava`, group and artifact as published, case included. Gradle build cache entries are not covered; they use `gradle.build_cache.max_age`. | | `npm` | yes | `pkg:npm/lodash`, `pkg:npm/@babel/core`; the name has to match exactly, case included, so `pkg:npm/jsonstream` doesn't cover `JSONStream`. | -| `nuget` | yes | `pkg:nuget/Newtonsoft.Json`; case doesn't matter. Give `proxy mirror` lowercase ids (`pkg:nuget/newtonsoft.json@13.0.3`): a mixed-case id is stored as a separate copy that clients are never served, so mirroring it doesn't keep their copy from aging out. | +| `nuget` | yes | `pkg:nuget/Newtonsoft.Json`; case doesn't matter. | | `oci` | yes | `pkg:oci/library/nginx`, the repository name of the pull. A named upstream keeps its prefix, also when containerd reaches it through `ns`: `ghcr.io/owner/app` pulled that way is `pkg:oci/upstream/ghcr/owner/app`. Homebrew bottles use Homebrew's image name, with `@` written as `/` and `+` as `x` (`openssl@3` is `pkg:oci/homebrew/core/openssl/3`). Helm charts pulled over OCI are stored like images. Only blobs are evicted; manifests stay in the metadata cache. | | `pub` | no | - | | `pypi` | yes | `pkg:pypi/requests`; case and single `_`/`-` don't matter. Dots and repeated separators are kept as written, so a project like `zope.interface` can be stored as `zope.interface` and `zope-interface`; add a rule for each. | diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 036e2a5..73c4d5dd 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -244,6 +244,12 @@ type CacheResult struct { // GetOrFetchArtifact retrieves an artifact from cache or fetches from upstream. func (p *Proxy) GetOrFetchArtifact(ctx context.Context, ecosystem, name, version, filename string) (*CacheResult, error) { + // NuGet ids are case-insensitive and the NuGet handler stores them + // lowercased, so a mirrored Newtonsoft.Json must land on the same record + // clients are served. + if ecosystem == "nuget" { + name = strings.ToLower(name) + } pkgPURL, versionPURL, err := packagePURLStrings(ecosystem, name, version) if err != nil { return nil, err diff --git a/internal/server/retention_nuget_test.go b/internal/server/retention_nuget_test.go index 44d45bb..db7daa8 100644 --- a/internal/server/retention_nuget_test.go +++ b/internal/server/retention_nuget_test.go @@ -248,3 +248,40 @@ func TestNuGetRetentionCoversMirroredPackages(t *testing.T) { t.Error("never-served mirrored package kept past the nuget rule") } } + +// Mirroring a mixed-case id fills the record clients are served, so their +// first download comes from the cache. +func TestNuGetMirrorStoresIDsLowercased(t *testing.T) { + upstream, downloads := nugetRetentionUpstream(t) + target, err := url.Parse(upstream.URL) + if err != nil { + t.Fatal(err) + } + db, store := setupEvictionTest(t) + client := &http.Client{Transport: nugetRewriteTransport{target: target, base: upstream.Client().Transport}} + fetcher := fetch.NewFetcher(fetch.WithHTTPClient(client), fetch.WithMaxRetries(0)) + t.Cleanup(func() { _ = fetcher.Close() }) + proxy := handler.NewProxy(db, store, fetcher, fetch.NewResolver(), discardLogger()) + proxy.HTTPClient = upstream.Client() + + m := mirror.New(proxy, db, store, discardLogger(), 1) + progress, err := m.Run(context.Background(), &mirror.PURLSource{PURLs: []string{"pkg:nuget/Serilog@3.1.1"}}) + if err != nil || progress.Failed != 0 || downloads.Load() != 1 { + t.Fatalf("mirror run = %+v, %v, downloads %d; want one successful download", progress, err, downloads.Load()) + } + + h := handler.NewNuGetHandlerWithUpstreams(proxy, "http://proxy.test", upstream.URL, upstream.URL) + srv := httptest.NewServer(h.Routes()) + t.Cleanup(srv.Close) + nugetGet(t, srv.URL+nugetSerilogPath) + if got := downloads.Load(); got != 1 { + t.Errorf("upstream downloads = %d, want 1: the client was not served the mirrored copy", got) + } + var n int + if err := db.Get(&n, `SELECT COUNT(*) FROM artifacts`); err != nil { + t.Fatal(err) + } + if n != 1 { + t.Errorf("artifacts = %d, want 1", n) + } +} From 743fbe50264660af73fe69536aebdd0e6ce37dba Mon Sep 17 00:00:00 2001 From: Christian Heim Date: Sat, 10 Oct 2026 23:28:13 +0200 Subject: [PATCH 239/633] Enable storage retention for hex Tarballs are stored under the name from the tarball filename, and purl keeps hex names as they are, so package rules use the default mapping with the name as published. The filename parser splits versions like 4.1.5-1 at the wrong hyphen and files them under a package like amqp_client-4.1.5, so package rules don't reach them; the docs row says so until the parser is fixed. Refs #306 --- docs/configuration.md | 2 +- internal/handler/hex.go | 10 ++ internal/server/retention_hex_test.go | 234 ++++++++++++++++++++++++++ 3 files changed, 245 insertions(+), 1 deletion(-) create mode 100644 internal/server/retention_hex_test.go diff --git a/docs/configuration.md b/docs/configuration.md index 97fb0f5..0a67c47 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -98,7 +98,7 @@ A package rule wins over an ecosystem rule, which wins over `default`. Package r | `gem` | no | - | | `golang` | yes | `pkg:golang/github.com/stretchr/testify`, the module path as in `go.mod`; case doesn't matter. A major version like `/v2` (or `.v3` on gopkg.in) is a module of its own and needs its own rule. The `!`-escaped form from proxy URLs and the module cache (`github.com/!azure/...`) is refused. | | `helm` | yes | No package rules: charts are cached per repository and digest, not under the chart name. The `helm` rule and `default` cover charts from HTTP repositories; charts pulled over OCI fall under `oci`. | -| `hex` | no | - | +| `hex` | yes | `pkg:hex/phoenix`, the package name as published. Package rules don't reach versions like `4.1.5-1` or `0.1.0-dev-10` yet, which the proxy files under a wrong package name; the hex rule and `default` still cover them. | | `julia` | yes | No package rules: a julia PURL needs a `uuid` qualifier, which the cache doesn't store. The `julia` rule and `default` cover package tarballs, artifacts and registry snapshots alike. | | `maven` | yes | `pkg:maven/com.google.guava/guava`, group and artifact as published, case included. Gradle build cache entries are not covered; they use `gradle.build_cache.max_age`. | | `npm` | yes | `pkg:npm/lodash`, `pkg:npm/@babel/core`; the name has to match exactly, case included, so `pkg:npm/jsonstream` doesn't cover `JSONStream`. | diff --git a/internal/handler/hex.go b/internal/handler/hex.go index c475e5a..adae0de 100644 --- a/internal/handler/hex.go +++ b/internal/handler/hex.go @@ -10,6 +10,7 @@ import ( "strings" "time" + "github.com/git-pkgs/proxy/internal/retention" "google.golang.org/protobuf/encoding/protowire" ) @@ -17,6 +18,15 @@ const ( hexUpstream = "https://repo.hex.pm" ) +// Tarballs are stored under pkg:hex/{name} with the name from the tarball +// filename, and purl keeps hex names as they are, so package rules use the +// default key mapping. parseTarballFilename splits at the last hyphen before +// a digit, so a version like 4.1.5-1 lands under a wrong package name that +// package rules don't reach. +func init() { + retention.Default.Register(retention.Spec{Key: "hex", CanonicalPackage: retention.DefaultCanonical("hex")}) +} + // HexHandler handles Hex.pm registry protocol requests. type HexHandler struct { proxy *Proxy diff --git a/internal/server/retention_hex_test.go b/internal/server/retention_hex_test.go new file mode 100644 index 0000000..4873dbb --- /dev/null +++ b/internal/server/retention_hex_test.go @@ -0,0 +1,234 @@ +package server + +import ( + "context" + "io" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/git-pkgs/proxy/internal/config" + "github.com/git-pkgs/proxy/internal/database" + "github.com/git-pkgs/proxy/internal/handler" + "github.com/git-pkgs/proxy/internal/mirror" + "github.com/git-pkgs/proxy/internal/retention" + "github.com/git-pkgs/registries/fetch" +) + +const ( + hexPhoenixPath = "/tarballs/phoenix-1.7.10.tar" + hexPlugPath = "/tarballs/plug-1.15.3.tar" +) + +// hexRetentionUpstream serves package tarballs the way repo.hex.pm does, +// counting downloads. +func hexRetentionUpstream(t *testing.T) (*httptest.Server, *atomic.Int64) { + t.Helper() + var downloads atomic.Int64 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case hexPhoenixPath, hexPlugPath: + downloads.Add(1) + _, _ = io.WriteString(w, "hex tarball bytes") + default: + http.NotFound(w, r) + } + })) + t.Cleanup(srv.Close) + return srv, &downloads +} + +func newHexRetentionProxy(t *testing.T, upstream *httptest.Server) (*httptest.Server, *database.DB) { + t.Helper() + db, store := setupEvictionTest(t) + // The upstream is a plain-HTTP loopback server; the default fetcher + // refuses loopback, a stock client reaches it. + fetcher := fetch.NewFetcher(fetch.WithHTTPClient(upstream.Client()), fetch.WithMaxRetries(0)) + t.Cleanup(func() { _ = fetcher.Close() }) + proxy := handler.NewProxy(db, store, fetcher, fetch.NewResolver(), discardLogger()) + proxy.HTTPClient = upstream.Client() + h := handler.NewHexHandlerWithUpstreams(proxy, "http://proxy.test", upstream.URL, upstream.URL) + srv := httptest.NewServer(h.Routes()) + t.Cleanup(srv.Close) + return srv, db +} + +func hexGet(t *testing.T, url string) { + t.Helper() + resp, err := http.Get(url) + if err != nil { + t.Fatalf("GET %s: %v", url, err) + } + body, _ := io.ReadAll(resp.Body) + _ = resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Fatalf("GET %s = %d %s, want 200", url, resp.StatusCode, body) + } +} + +// hexArtifact returns the record of the tarball named filename. +func hexArtifact(t *testing.T, db *database.DB, filename string) database.Artifact { + t.Helper() + var art database.Artifact + if err := db.Get(&art, db.Rebind(`SELECT * FROM artifacts WHERE filename = ?`), filename); err != nil { + t.Fatalf("reading artifact %s: %v", filename, err) + } + return art +} + +func hexBackdate(t *testing.T, db *database.DB, filename string, fetched, accessed time.Time) { + t.Helper() + query := db.Rebind(`UPDATE artifacts SET fetched_at = ?, last_accessed_at = ? WHERE filename = ?`) + if _, err := db.Exec(query, fetched, accessed, filename); err != nil { + t.Fatalf("backdating %s: %v", filename, err) + } +} + +func TestHexRegistersForRetention(t *testing.T) { + if _, ok := retention.Default.Lookup("hex"); !ok { + t.Fatal("hex is not registered for retention") + } + if key, ok := retention.Default.KeyForDBEcosystem("hex"); !ok || key != "hex" { + t.Errorf("KeyForDBEcosystem(hex) = %q, %v, want hex, true", key, ok) + } +} + +// A tarball served from the cache stays while it is used, is evicted once +// its rule runs out, comes back from upstream on the next download, and +// stays after that even with the evicted copy's old access time on the +// record. +func TestHexRetentionEvictsAndRefetches(t *testing.T) { + upstream, downloads := hexRetentionUpstream(t) + srv, db := newHexRetentionProxy(t, upstream) + const tarball = "phoenix-1.7.10.tar" + + hexGet(t, srv.URL+hexPhoenixPath) + hexGet(t, srv.URL+hexPhoenixPath) + if got := downloads.Load(); got != 1 { + t.Fatalf("upstream downloads = %d, want 1", got) + } + if !hexArtifact(t, db, tarball).LastAccessedAt.Valid { + t.Fatal("cache hit did not record an access") + } + + rules := retention.Rules{Ecosystems: map[string]time.Duration{"hex": retentionDay}} + now := time.Now() + sweepRetention(context.Background(), db, discardLogger(), retention.Default, rules, now) + if !hexArtifact(t, db, tarball).StoragePath.Valid { + t.Fatal("tarball used today was evicted") + } + + old := now.Add(-2 * retentionDay) + hexBackdate(t, db, tarball, old, old) + sweepRetention(context.Background(), db, discardLogger(), retention.Default, rules, now) + if hexArtifact(t, db, tarball).StoragePath.Valid { + t.Fatal("tarball unused for two days kept under a one-day rule") + } + + hexGet(t, srv.URL+hexPhoenixPath) + if got := downloads.Load(); got != 2 { + t.Errorf("upstream downloads = %d, want 2 after the eviction", got) + } + art := hexArtifact(t, db, tarball) + hexBackdate(t, db, tarball, art.FetchedAt.Time, now.Add(-30*retentionDay)) + sweepRetention(context.Background(), db, discardLogger(), retention.Default, rules, time.Now()) + if !hexArtifact(t, db, tarball).StoragePath.Valid { + t.Error("refetched tarball evicted again by the next sweep") + } +} + +// A key with a namespace parses as a hex PURL but can never match a stored +// package, so it is refused at validation. A plain key is accepted. +func TestHexRetentionRefusesNamespacedKeys(t *testing.T) { + cfg := config.Default() + cfg.Storage.Retention.Packages = map[string]string{"pkg:hex/phoenix": "0"} + if _, err := cfg.RetentionRules(retention.Default); err != nil { + t.Fatalf("RetentionRules(pkg:hex/phoenix): %v", err) + } + cfg.Storage.Retention.Packages = map[string]string{"pkg:hex/acme/phoenix": "0"} + _, err := cfg.RetentionRules(retention.Default) + if err == nil || !strings.Contains(err.Error(), "does not match how hex packages are stored") { + t.Errorf("RetentionRules(pkg:hex/acme/phoenix) error = %v, want the key refused", err) + } +} + +// A package rule written as a hex PURL reaches the stored package. +func TestHexRetentionPackageRule(t *testing.T) { + upstream, _ := hexRetentionUpstream(t) + srv, db := newHexRetentionProxy(t, upstream) + const phoenixTarball, plugTarball = "phoenix-1.7.10.tar", "plug-1.15.3.tar" + hexGet(t, srv.URL+hexPhoenixPath) + hexGet(t, srv.URL+hexPlugPath) + old := time.Now().Add(-30 * retentionDay) + hexBackdate(t, db, phoenixTarball, old, old) + hexBackdate(t, db, plugTarball, old, old) + + cfg := config.Default() + cfg.Storage.Retention.Ecosystems = map[string]string{"hex": "1d"} + cfg.Storage.Retention.Packages = map[string]string{"pkg:hex/phoenix": "0"} + rules, err := cfg.RetentionRules(retention.Default) + if err != nil { + t.Fatalf("RetentionRules: %v", err) + } + sweepRetention(context.Background(), db, discardLogger(), retention.Default, rules, time.Now()) + + if !hexArtifact(t, db, phoenixTarball).StoragePath.Valid { + t.Error("phoenix evicted despite its never-evict rule") + } + if hexArtifact(t, db, plugTarball).StoragePath.Valid { + t.Error("plug kept past the hex rule") + } +} + +// hexRewriteTransport sends every request to the test upstream, so the +// mirror command's fixed repo.hex.pm URLs reach it. +type hexRewriteTransport struct { + target *url.URL + base http.RoundTripper +} + +func (r hexRewriteTransport) RoundTrip(req *http.Request) (*http.Response, error) { + req = req.Clone(req.Context()) + req.URL.Scheme = r.target.Scheme + req.URL.Host = r.target.Host + req.Host = r.target.Host + return r.base.RoundTrip(req) +} + +// A package preloaded with the mirror command falls under the hex rule. +func TestHexRetentionCoversMirroredPackages(t *testing.T) { + upstream, downloads := hexRetentionUpstream(t) + target, err := url.Parse(upstream.URL) + if err != nil { + t.Fatal(err) + } + db, store := setupEvictionTest(t) + client := &http.Client{Transport: hexRewriteTransport{target: target, base: upstream.Client().Transport}} + fetcher := fetch.NewFetcher(fetch.WithHTTPClient(client), fetch.WithMaxRetries(0)) + t.Cleanup(func() { _ = fetcher.Close() }) + proxy := handler.NewProxy(db, store, fetcher, fetch.NewResolver(), discardLogger()) + + m := mirror.New(proxy, db, store, discardLogger(), 1) + progress, err := m.Run(context.Background(), &mirror.PURLSource{PURLs: []string{"pkg:hex/phoenix@1.7.10"}}) + if err != nil || progress.Failed != 0 || downloads.Load() != 1 { + t.Fatalf("mirror run = %+v, %v, downloads %d; want one successful download", progress, err, downloads.Load()) + } + + const tarball = "phoenix-1.7.10.tar" + if hexArtifact(t, db, tarball).LastAccessedAt.Valid { + t.Fatal("mirrored tarball has an access recorded before anyone downloaded it") + } + query := db.Rebind(`UPDATE artifacts SET fetched_at = ? WHERE filename = ?`) + if _, err := db.Exec(query, time.Now().Add(-30*retentionDay), tarball); err != nil { + t.Fatalf("backdating %s: %v", tarball, err) + } + rules := retention.Rules{Ecosystems: map[string]time.Duration{"hex": retentionDay}} + sweepRetention(context.Background(), db, discardLogger(), retention.Default, rules, time.Now()) + if hexArtifact(t, db, tarball).StoragePath.Valid { + t.Error("never-served mirrored tarball kept past the hex rule") + } +} From bffba28262f139f796b21e26a24b233426a6420a Mon Sep 17 00:00:00 2001 From: Christian Heim Date: Sat, 10 Oct 2026 23:37:32 +0200 Subject: [PATCH 240/633] Enable storage retention for conda The handler stores packages as {channel}/{name}, which purl writes as pkg:conda/conda-forge%2Fnumpy. Package rules name the channel as the namespace (pkg:conda/conda-forge/numpy) and get mapped to that form, with the package name lowercased the way conda treats it. A key without a channel or with an empty part could never match and is refused. The mirror can't fetch conda packages today, so there's no mirror test. Refs #306 --- docs/configuration.md | 2 +- internal/handler/conda.go | 34 ++++ internal/handler/conda_test.go | 32 ++++ internal/server/retention_conda_test.go | 201 ++++++++++++++++++++++++ 4 files changed, 268 insertions(+), 1 deletion(-) create mode 100644 internal/server/retention_conda_test.go diff --git a/docs/configuration.md b/docs/configuration.md index 97fb0f5..888c429 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -92,7 +92,7 @@ A package rule wins over an ecosystem rule, which wins over `default`. Package r | `cargo` | yes | `pkg:cargo/serde`, the crate name as published; case, `-` and `_` have to match, so `pkg:cargo/inflector` doesn't cover `Inflector`. | | `composer` | yes | `pkg:composer/symfony/console`; case doesn't matter. A key without a vendor is refused. | | `conan` | yes | `pkg:conan/zlib`, the recipe name; a rule covers every version, revision and user/channel of it. | -| `conda` | no | - | +| `conda` | yes | `pkg:conda/conda-forge/numpy`, with the channel as the namespace; the package name's case doesn't matter. A key without a channel is refused. | | `cran` | yes | `pkg:cran/ggplot2`, the package name as published, case included (`pkg:cran/Rcpp`). A rule covers source and binary builds alike. | | `deb` | no | - | | `gem` | no | - | diff --git a/internal/handler/conda.go b/internal/handler/conda.go index 07c2aea..8a204b0 100644 --- a/internal/handler/conda.go +++ b/internal/handler/conda.go @@ -5,8 +5,12 @@ import ( "encoding/json" "io" "net/http" + "slices" "strings" "time" + + "github.com/git-pkgs/proxy/internal/retention" + "github.com/git-pkgs/purl" ) const ( @@ -14,6 +18,36 @@ const ( minCondaParts = 3 // name-version-build requires at least 3 hyphen-separated parts ) +// Packages are stored under the name {channel}/{name}, which purl writes as +// pkg:conda/{channel}%2F{name}. Package rules name the channel as the +// namespace, pkg:conda/conda-forge/numpy, and are mapped to that form. +func init() { + retention.Default.Register(retention.Spec{Key: "conda", CanonicalPackage: condaRetentionKey}) +} + +// condaRetentionKey maps a package rule to the stored package PURL. It +// takes the channel from the namespace, or from the name in the stored +// form, and refuses a key without a channel or with an empty segment, +// since no stored package looks like that. A label channel such as +// conda-forge/label/rc keeps all its segments. The package name is +// lowercased the way conda treats it; the channel stays as the URL had it. +func condaRetentionKey(p *purl.PURL) (string, bool) { + if p == nil { + return "", false + } + full := p.Name + if p.Namespace != "" { + full = p.Namespace + "/" + p.Name + } + segments := strings.Split(full, "/") + if len(segments) < 2 || slices.Contains(segments, "") { + return "", false + } + channel := strings.Join(segments[:len(segments)-1], "/") + name := strings.ToLower(segments[len(segments)-1]) + return retention.DefaultCanonical("conda")(purl.New(p.Type, "", channel+"/"+name, "", nil)) +} + // CondaHandler handles Conda/Anaconda registry protocol requests. type CondaHandler struct { proxy *Proxy diff --git a/internal/handler/conda_test.go b/internal/handler/conda_test.go index 1b57039..30ce5fa 100644 --- a/internal/handler/conda_test.go +++ b/internal/handler/conda_test.go @@ -9,6 +9,7 @@ import ( "time" "github.com/git-pkgs/cooldown" + "github.com/git-pkgs/purl" ) func TestCondaParseFilename(t *testing.T) { @@ -303,3 +304,34 @@ func TestCondaHandleRepodataWithoutCooldown(t *testing.T) { t.Fatalf("status = %d, want %d", w.Code, http.StatusOK) } } + +func TestCondaRetentionKey(t *testing.T) { + tests := []struct { + key string + want string + ok bool + }{ + {"pkg:conda/conda-forge/numpy", "pkg:conda/conda-forge%2Fnumpy", true}, + {"pkg:conda/conda-forge%2Fnumpy", "pkg:conda/conda-forge%2Fnumpy", true}, + {"pkg:conda/conda-forge/PyYAML", "pkg:conda/conda-forge%2Fpyyaml", true}, + {"pkg:conda/Conda-Forge/numpy", "pkg:conda/Conda-Forge%2Fnumpy", true}, + {"pkg:conda/conda-forge/label/rc/numpy", "pkg:conda/conda-forge%2Flabel%2Frc%2Fnumpy", true}, + {"pkg:conda/numpy", "", false}, + {"pkg:conda/%2Fnumpy", "", false}, + {"pkg:conda/conda-forge%2F", "", false}, + {"pkg:npm/conda-forge/numpy", "", false}, + } + for _, tt := range tests { + p, err := purl.Parse(tt.key) + if err != nil { + t.Fatalf("Parse(%s): %v", tt.key, err) + } + got, ok := condaRetentionKey(p) + if ok != tt.ok || (ok && got != tt.want) { + t.Errorf("condaRetentionKey(%s) = %q, %v, want %q, %v", tt.key, got, ok, tt.want, tt.ok) + } + } + if _, ok := condaRetentionKey(nil); ok { + t.Error("condaRetentionKey(nil) accepted") + } +} diff --git a/internal/server/retention_conda_test.go b/internal/server/retention_conda_test.go new file mode 100644 index 0000000..230ecbb --- /dev/null +++ b/internal/server/retention_conda_test.go @@ -0,0 +1,201 @@ +package server + +import ( + "context" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/git-pkgs/proxy/internal/config" + "github.com/git-pkgs/proxy/internal/database" + "github.com/git-pkgs/proxy/internal/handler" + "github.com/git-pkgs/proxy/internal/retention" + "github.com/git-pkgs/registries/fetch" +) + +const ( + condaForgeNumpyPath = "/conda-forge/linux-64/numpy-1.24.0-py311h64a7726_0.conda" + condaAnacondaNumpy = "/anaconda/linux-64/numpy-1.24.0-py311h08b1b3b_0.conda" + condaRequestsPath = "/conda-forge/noarch/requests-2.31.0-pyhd8ed1ab_0.conda" +) + +// condaRetentionUpstream serves package files the way conda.anaconda.org +// does, counting downloads. +func condaRetentionUpstream(t *testing.T) (*httptest.Server, *atomic.Int64) { + t.Helper() + var downloads atomic.Int64 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case condaForgeNumpyPath, condaAnacondaNumpy, condaRequestsPath: + downloads.Add(1) + _, _ = io.WriteString(w, "conda package bytes") + default: + http.NotFound(w, r) + } + })) + t.Cleanup(srv.Close) + return srv, &downloads +} + +func newCondaRetentionProxy(t *testing.T, upstream *httptest.Server) (*httptest.Server, *database.DB) { + t.Helper() + db, store := setupEvictionTest(t) + // The upstream is a plain-HTTP loopback server; the default fetcher + // refuses loopback, a stock client reaches it. + fetcher := fetch.NewFetcher(fetch.WithHTTPClient(upstream.Client()), fetch.WithMaxRetries(0)) + t.Cleanup(func() { _ = fetcher.Close() }) + proxy := handler.NewProxy(db, store, fetcher, fetch.NewResolver(), discardLogger()) + proxy.HTTPClient = upstream.Client() + h := handler.NewCondaHandlerWithUpstream(proxy, "http://proxy.test", upstream.URL) + srv := httptest.NewServer(h.Routes()) + t.Cleanup(srv.Close) + return srv, db +} + +func condaGet(t *testing.T, url string) { + t.Helper() + resp, err := http.Get(url) + if err != nil { + t.Fatalf("GET %s: %v", url, err) + } + body, _ := io.ReadAll(resp.Body) + _ = resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Fatalf("GET %s = %d %s, want 200", url, resp.StatusCode, body) + } +} + +// condaArtifact returns the record of the package file named filename. +func condaArtifact(t *testing.T, db *database.DB, filename string) database.Artifact { + t.Helper() + var art database.Artifact + if err := db.Get(&art, db.Rebind(`SELECT * FROM artifacts WHERE filename = ?`), filename); err != nil { + t.Fatalf("reading artifact %s: %v", filename, err) + } + return art +} + +func condaBackdate(t *testing.T, db *database.DB, filename string, fetched, accessed time.Time) { + t.Helper() + query := db.Rebind(`UPDATE artifacts SET fetched_at = ?, last_accessed_at = ? WHERE filename = ?`) + if _, err := db.Exec(query, fetched, accessed, filename); err != nil { + t.Fatalf("backdating %s: %v", filename, err) + } +} + +func TestCondaRegistersForRetention(t *testing.T) { + if _, ok := retention.Default.Lookup("conda"); !ok { + t.Fatal("conda is not registered for retention") + } + if key, ok := retention.Default.KeyForDBEcosystem("conda"); !ok || key != "conda" { + t.Errorf("KeyForDBEcosystem(conda) = %q, %v, want conda, true", key, ok) + } +} + +// A package file served from the cache stays while it is used, is evicted +// once its rule runs out, comes back from upstream on the next download, +// and stays after that even with the evicted copy's old access time on the +// record. +func TestCondaRetentionEvictsAndRefetches(t *testing.T) { + upstream, downloads := condaRetentionUpstream(t) + srv, db := newCondaRetentionProxy(t, upstream) + const file = "requests-2.31.0-pyhd8ed1ab_0.conda" + + condaGet(t, srv.URL+condaRequestsPath) + condaGet(t, srv.URL+condaRequestsPath) + if got := downloads.Load(); got != 1 { + t.Fatalf("upstream downloads = %d, want 1", got) + } + if !condaArtifact(t, db, file).LastAccessedAt.Valid { + t.Fatal("cache hit did not record an access") + } + + rules := retention.Rules{Ecosystems: map[string]time.Duration{"conda": retentionDay}} + now := time.Now() + sweepRetention(context.Background(), db, discardLogger(), retention.Default, rules, now) + if !condaArtifact(t, db, file).StoragePath.Valid { + t.Fatal("package used today was evicted") + } + + old := now.Add(-2 * retentionDay) + condaBackdate(t, db, file, old, old) + sweepRetention(context.Background(), db, discardLogger(), retention.Default, rules, now) + if condaArtifact(t, db, file).StoragePath.Valid { + t.Fatal("package unused for two days kept under a one-day rule") + } + + condaGet(t, srv.URL+condaRequestsPath) + if got := downloads.Load(); got != 2 { + t.Errorf("upstream downloads = %d, want 2 after the eviction", got) + } + art := condaArtifact(t, db, file) + condaBackdate(t, db, file, art.FetchedAt.Time, now.Add(-30*retentionDay)) + sweepRetention(context.Background(), db, discardLogger(), retention.Default, rules, time.Now()) + if !condaArtifact(t, db, file).StoragePath.Valid { + t.Error("refetched package evicted again by the next sweep") + } +} + +// A key without a channel parses as a conda PURL but can never match a +// stored package, so it is refused at validation. A key with the channel +// as the namespace is accepted. +func TestCondaRetentionRefusesKeysWithoutChannel(t *testing.T) { + cfg := config.Default() + cfg.Storage.Retention.Packages = map[string]string{"pkg:conda/conda-forge/numpy": "0"} + if _, err := cfg.RetentionRules(retention.Default); err != nil { + t.Fatalf("RetentionRules(pkg:conda/conda-forge/numpy): %v", err) + } + cfg.Storage.Retention.Packages = map[string]string{"pkg:conda/numpy": "0"} + _, err := cfg.RetentionRules(retention.Default) + if err == nil || !strings.Contains(err.Error(), "does not match how conda packages are stored") { + t.Errorf("RetentionRules(pkg:conda/numpy) error = %v, want the key refused", err) + } +} + +// A package rule with the channel as the namespace reaches the package in +// that channel only; the same package from another channel and other +// packages fall under the conda rule. +func TestCondaRetentionPackageRule(t *testing.T) { + upstream, _ := condaRetentionUpstream(t) + srv, db := newCondaRetentionProxy(t, upstream) + const forgeNumpy, anacondaNumpy, requests = "numpy-1.24.0-py311h64a7726_0.conda", "numpy-1.24.0-py311h08b1b3b_0.conda", "requests-2.31.0-pyhd8ed1ab_0.conda" + condaGet(t, srv.URL+condaForgeNumpyPath) + condaGet(t, srv.URL+condaAnacondaNumpy) + condaGet(t, srv.URL+condaRequestsPath) + + var stored string + art := condaArtifact(t, db, forgeNumpy) + if err := db.Get(&stored, db.Rebind(`SELECT package_purl FROM versions WHERE purl = ?`), art.VersionPURL); err != nil { + t.Fatalf("reading stored package PURL: %v", err) + } + if stored != "pkg:conda/conda-forge%2Fnumpy" { + t.Errorf("stored package PURL = %q, want pkg:conda/conda-forge%%2Fnumpy", stored) + } + + old := time.Now().Add(-30 * retentionDay) + for _, filename := range []string{forgeNumpy, anacondaNumpy, requests} { + condaBackdate(t, db, filename, old, old) + } + cfg := config.Default() + cfg.Storage.Retention.Ecosystems = map[string]string{"conda": "1d"} + cfg.Storage.Retention.Packages = map[string]string{"pkg:conda/conda-forge/numpy": "0"} + rules, err := cfg.RetentionRules(retention.Default) + if err != nil { + t.Fatalf("RetentionRules: %v", err) + } + sweepRetention(context.Background(), db, discardLogger(), retention.Default, rules, time.Now()) + + if !condaArtifact(t, db, forgeNumpy).StoragePath.Valid { + t.Error("conda-forge numpy evicted despite its never-evict rule") + } + if condaArtifact(t, db, anacondaNumpy).StoragePath.Valid { + t.Error("anaconda numpy kept by the rule for conda-forge") + } + if condaArtifact(t, db, requests).StoragePath.Valid { + t.Error("requests kept past the conda rule") + } +} From c3c120401c205ec0b3eaccaeb68920c1c6dd5d78 Mon Sep 17 00:00:00 2001 From: Christian Heim Date: Mon, 12 Oct 2026 19:57:57 +0200 Subject: [PATCH 241/633] Fix hex tarballs whose version contains a hyphen parseTarballFilename split at the last hyphen before a digit, so amqp_client-4.1.5-1.tar was stored as package amqp_client-4.1.5. Hex names can't contain a hyphen, so it now splits at the first one. The docs caveat is gone. Refs #306 --- docs/configuration.md | 2 +- internal/handler/hex.go | 20 +++++++++----------- internal/handler/hex_test.go | 5 +++++ 3 files changed, 15 insertions(+), 12 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 0a67c47..2aa2a2c 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -98,7 +98,7 @@ A package rule wins over an ecosystem rule, which wins over `default`. Package r | `gem` | no | - | | `golang` | yes | `pkg:golang/github.com/stretchr/testify`, the module path as in `go.mod`; case doesn't matter. A major version like `/v2` (or `.v3` on gopkg.in) is a module of its own and needs its own rule. The `!`-escaped form from proxy URLs and the module cache (`github.com/!azure/...`) is refused. | | `helm` | yes | No package rules: charts are cached per repository and digest, not under the chart name. The `helm` rule and `default` cover charts from HTTP repositories; charts pulled over OCI fall under `oci`. | -| `hex` | yes | `pkg:hex/phoenix`, the package name as published. Package rules don't reach versions like `4.1.5-1` or `0.1.0-dev-10` yet, which the proxy files under a wrong package name; the hex rule and `default` still cover them. | +| `hex` | yes | `pkg:hex/phoenix`, the package name as published. | | `julia` | yes | No package rules: a julia PURL needs a `uuid` qualifier, which the cache doesn't store. The `julia` rule and `default` cover package tarballs, artifacts and registry snapshots alike. | | `maven` | yes | `pkg:maven/com.google.guava/guava`, group and artifact as published, case included. Gradle build cache entries are not covered; they use `gradle.build_cache.max_age`. | | `npm` | yes | `pkg:npm/lodash`, `pkg:npm/@babel/core`; the name has to match exactly, case included, so `pkg:npm/jsonstream` doesn't cover `JSONStream`. | diff --git a/internal/handler/hex.go b/internal/handler/hex.go index adae0de..79be510 100644 --- a/internal/handler/hex.go +++ b/internal/handler/hex.go @@ -20,9 +20,7 @@ const ( // Tarballs are stored under pkg:hex/{name} with the name from the tarball // filename, and purl keeps hex names as they are, so package rules use the -// default key mapping. parseTarballFilename splits at the last hyphen before -// a digit, so a version like 4.1.5-1 lands under a wrong package name that -// package rules don't reach. +// default key mapping. func init() { retention.Default.Register(retention.Spec{Key: "hex", CanonicalPackage: retention.DefaultCanonical("hex")}) } @@ -86,16 +84,16 @@ func (h *HexHandler) handleDownload(w http.ResponseWriter, r *http.Request) { // parseTarballFilename extracts name and version from a hex tarball filename. // e.g., "phoenix-1.7.10.tar" -> ("phoenix", "1.7.10") +// e.g., "amqp_client-4.1.5-1.tar" -> ("amqp_client", "4.1.5-1") +// +// Hex package names can't contain a hyphen, so the first one separates the +// name from a version that may carry hyphens of its own. func (h *HexHandler) parseTarballFilename(filename string) (name, version string) { - base := strings.TrimSuffix(filename, ".tar") - - // Find the last hyphen followed by a version number - for i := len(base) - 1; i >= 0; i-- { - if base[i] == '-' && i+1 < len(base) && base[i+1] >= '0' && base[i+1] <= '9' { - return base[:i], base[i+1:] - } + name, version, ok := strings.Cut(strings.TrimSuffix(filename, ".tar"), "-") + if !ok || name == "" || version == "" { + return "", "" } - return "", "" + return name, version } // hexAPIURL is the Hex HTTP API base URL for fetching package metadata with timestamps. diff --git a/internal/handler/hex_test.go b/internal/handler/hex_test.go index b02540a..92120bd 100644 --- a/internal/handler/hex_test.go +++ b/internal/handler/hex_test.go @@ -26,6 +26,11 @@ func TestHexParseTarballFilename(t *testing.T) { {"phoenix-1.7.10.tar", "phoenix", "1.7.10"}, {"ecto-3.11.0.tar", "ecto", "3.11.0"}, {"phoenix_live_view-0.20.1.tar", "phoenix_live_view", "0.20.1"}, + {"amqp_client-4.1.5-1.tar", "amqp_client", "4.1.5-1"}, + {"bandit-0.1.0-dev-10.tar", "bandit", "0.1.0-dev-10"}, + {"ex_doc-0.34.0-rc.1.tar", "ex_doc", "0.34.0-rc.1"}, + {"-1.0.0.tar", "", ""}, + {"phoenix-.tar", "", ""}, {"invalid", "", ""}, } From c58cdaece9c60f51a7b5842461d65f4c2957eb57 Mon Sep 17 00:00:00 2001 From: Christian Heim Date: Mon, 12 Oct 2026 19:59:35 +0200 Subject: [PATCH 242/633] Refuse conda rules with more than a channel and a name Downloads are routed as /{channel}/{arch}/{filename}, so the stored name is always {channel}/{name}. A key like pkg:conda/conda-forge/label/rc/numpy passed validation and matched nothing; it's now refused. Refs #306 --- internal/handler/conda.go | 14 +++++++------- internal/handler/conda_test.go | 3 ++- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/internal/handler/conda.go b/internal/handler/conda.go index 8a204b0..6e4be7e 100644 --- a/internal/handler/conda.go +++ b/internal/handler/conda.go @@ -27,10 +27,11 @@ func init() { // condaRetentionKey maps a package rule to the stored package PURL. It // takes the channel from the namespace, or from the name in the stored -// form, and refuses a key without a channel or with an empty segment, -// since no stored package looks like that. A label channel such as -// conda-forge/label/rc keeps all its segments. The package name is -// lowercased the way conda treats it; the channel stays as the URL had it. +// form. Downloads are routed as /{channel}/{arch}/{filename}, so the channel +// is always one path segment; a key without a channel, with more segments +// or with an empty one is refused, since no stored package looks like that. +// The package name is lowercased the way conda treats it; the channel stays +// as the URL had it. func condaRetentionKey(p *purl.PURL) (string, bool) { if p == nil { return "", false @@ -40,11 +41,10 @@ func condaRetentionKey(p *purl.PURL) (string, bool) { full = p.Namespace + "/" + p.Name } segments := strings.Split(full, "/") - if len(segments) < 2 || slices.Contains(segments, "") { + if len(segments) != 2 || slices.Contains(segments, "") { return "", false } - channel := strings.Join(segments[:len(segments)-1], "/") - name := strings.ToLower(segments[len(segments)-1]) + channel, name := segments[0], strings.ToLower(segments[1]) return retention.DefaultCanonical("conda")(purl.New(p.Type, "", channel+"/"+name, "", nil)) } diff --git a/internal/handler/conda_test.go b/internal/handler/conda_test.go index 30ce5fa..567bb6f 100644 --- a/internal/handler/conda_test.go +++ b/internal/handler/conda_test.go @@ -315,7 +315,8 @@ func TestCondaRetentionKey(t *testing.T) { {"pkg:conda/conda-forge%2Fnumpy", "pkg:conda/conda-forge%2Fnumpy", true}, {"pkg:conda/conda-forge/PyYAML", "pkg:conda/conda-forge%2Fpyyaml", true}, {"pkg:conda/Conda-Forge/numpy", "pkg:conda/Conda-Forge%2Fnumpy", true}, - {"pkg:conda/conda-forge/label/rc/numpy", "pkg:conda/conda-forge%2Flabel%2Frc%2Fnumpy", true}, + {"pkg:conda/conda-forge/label/rc/numpy", "", false}, + {"pkg:conda/conda-forge%2Flabel%2Frc%2Fnumpy", "", false}, {"pkg:conda/numpy", "", false}, {"pkg:conda/%2Fnumpy", "", false}, {"pkg:conda/conda-forge%2F", "", false}, From 854182fa28e59ddebbe08ed56bc502fb5670c5d9 Mon Sep 17 00:00:00 2001 From: Christian Heim Date: Mon, 12 Oct 2026 20:06:59 +0200 Subject: [PATCH 243/633] Concurrent clients could still end up with a database and a signature that don't belong together. Databases are now stored under their digest like signatures, so a cache row and its bytes always agree, also with several replicas on shared storage. Writes for one database are serialized, and the signature recovery shares the refresh with database requests, so a slow fetch can no longer store an older database over a newer one. When the cache write fails, the proxy serves the database the cache still describes. Unchanged databases are no longer rewritten, and a remembered missing signature is asked for again after the metadata TTL. The new tests force these interleavings with storage and upstream hooks. --- internal/handler/arch.go | 156 +++++++++-- internal/handler/arch_concurrency_test.go | 315 ++++++++++++++++++++++ 2 files changed, 445 insertions(+), 26 deletions(-) create mode 100644 internal/handler/arch_concurrency_test.go diff --git a/internal/handler/arch.go b/internal/handler/arch.go index 64fde48..6706dae 100644 --- a/internal/handler/arch.go +++ b/internal/handler/arch.go @@ -3,11 +3,15 @@ package handler import ( "context" "crypto/sha256" + "database/sql" "encoding/hex" "errors" + "fmt" "net/http" "regexp" "strings" + "sync" + "time" "github.com/git-pkgs/proxy/internal/database" "github.com/git-pkgs/proxy/internal/metrics" @@ -32,6 +36,7 @@ type ArchHandler struct { proxy *Proxy proxyURL string repositories map[string]string + locks sync.Map // database cache key -> *sync.Mutex } // NewArchHandler creates an Arch Linux handler. Without repositories it serves @@ -198,14 +203,7 @@ func (h *ArchHandler) handleDatabase(w http.ResponseWriter, r *http.Request, rep return } p.recordMetadataCacheMiss(archEcosystem) - res, _ := p.coalesceMetadata(r.Context(), d.coalesceKey("database"), func(ctx context.Context) metadataResult { - entry, hit := p.cachedMetadataState(ctx, archEcosystem, d.dbKey, nil) - if hit != nil { - return *hit - } - res, _ := h.refresh(ctx, d, entry) - return res - }) + res := h.sharedRefresh(r.Context(), d, true) h.writeMetadata(w, r, d.dbKey, res) } @@ -227,13 +225,14 @@ func (h *ArchHandler) handleDatabaseSignature(w http.ResponseWriter, r *http.Req // Nothing stored for the cached database, e.g. its signature fetch failed // during the last refresh. The client may already hold that database, so // only a signature for it will do; if the refresh replaces it, fail. - entry, _ := p.DB.GetMetadataCache(archEcosystem, d.dbKey) served := h.databaseDigest(d) res, _ := p.coalesceMetadata(r.Context(), d.coalesceKey("signature\x00"+served), func(ctx context.Context) metadataResult { if res, ok := h.storedSignature(ctx, d, served); ok { return res } - _, err := h.refresh(ctx, d, entry) + if res := h.sharedRefresh(ctx, d, false); res.err != nil { + return res + } if served == "" { if res, ok := h.cachedSignature(ctx, d); ok { return res @@ -241,34 +240,46 @@ func (h *ArchHandler) handleDatabaseSignature(w http.ResponseWriter, r *http.Req } else if res, ok := h.storedSignature(ctx, d, served); ok { return res } - if err == nil || h.databaseDigest(d) != served { - err = errArchSignatureUnpaired - } - return metadataResult{err: err} + return metadataResult{err: errArchSignatureUnpaired} }) h.writeMetadata(w, r, d.sigKey, res) } var errArchSignatureUnpaired = errors.New("database signature could not be paired with the database") +// sharedRefresh runs one refresh per database at a time, so a slow fetch +// cannot store an older database over one cached after it started. With +// ifStale, a refresh that finds the database fresh again skips the upstream. +func (h *ArchHandler) sharedRefresh(ctx context.Context, d archDatabase, ifStale bool) metadataResult { + p := h.proxy + res, _ := p.coalesceMetadata(ctx, d.coalesceKey("database"), func(ctx context.Context) metadataResult { + entry, hit := p.cachedMetadataState(ctx, archEcosystem, d.dbKey, nil) + if hit != nil && ifStale { + return *hit + } + res, _ := h.refresh(ctx, d, entry) + return res + }) + return res +} + // archPairAttempts bounds refetches while the mirror replaces the database. const archPairAttempts = 3 // refresh fetches the database, then its signature, and stores the signature -// for the database's content. It returns the database and why no signature -// could be stored, if so. +// for the database's content. It returns the database as cached and why no +// signature could be stored, if so. func (h *ArchHandler) refresh(ctx context.Context, d archDatabase, entry *database.MetadataCacheEntry) (metadataResult, error) { p := h.proxy - res := p.fetchMetadataFromUpstream(ctx, archEcosystem, d.dbKey, d.dbURL, archAccept, "identity", nil, entry) + res, row := h.fetchDatabase(ctx, d, entry) if res.err != nil { return res, res.err } for range archPairAttempts { - current, _ := p.DB.GetMetadataCache(archEcosystem, d.dbKey) - if current == nil || !current.ContentDigest.Valid { + if row == nil || !row.ContentDigest.Valid { return res, errArchSignatureUnpaired } - dbDigest := current.ContentDigest.String + dbDigest := row.ContentDigest.String if _, ok := h.storedSignature(ctx, d, dbDigest); ok { return res, nil } @@ -283,7 +294,7 @@ func (h *ArchHandler) refresh(ctx context.Context, d archDatabase, entry *databa } // The mirror may have replaced the database in the meantime. - again, err := p.fetchUpstreamMetadata(ctx, d.dbURL, current, archAccept, "identity") + again, err := p.fetchUpstreamMetadata(ctx, d.dbURL, row, archAccept, "identity") if err != nil { return res, err } @@ -291,12 +302,92 @@ func (h *ArchHandler) refresh(ctx context.Context, d archDatabase, entry *databa h.storeSignature(ctx, d, dbDigest, sig) return res, nil } - p.cacheMetadataBlob(ctx, archEcosystem, d.dbKey, metadataStoragePath(archEcosystem, d.dbKey), again) - res = metadataResult{body: again.body, contentType: again.contentType, contentEncoding: again.contentEncoding} + if res, row = h.storeDatabase(ctx, d, again); res.err != nil { + return res, res.err + } } return res, errArchSignatureUnpaired } +// fetchDatabase fetches and caches the database, falling back to the cached +// copy when the upstream fails. +func (h *ArchHandler) fetchDatabase(ctx context.Context, d archDatabase, entry *database.MetadataCacheEntry) (metadataResult, *database.MetadataCacheEntry) { + p := h.proxy + meta, err := p.fetchUpstreamMetadata(ctx, d.dbURL, entry, archAccept, "identity") + if errors.Is(err, errStale304) { + meta, err = p.fetchUpstreamMetadata(ctx, d.dbURL, nil, archAccept, "identity") + } + if err == nil { + return h.storeDatabase(ctx, d, meta) + } + if entry == nil { + return metadataResult{err: fmt.Errorf("upstream failed and no cached metadata: %w", err)}, nil + } + p.Logger.Warn("upstream metadata fetch failed, checking cache", "ecosystem", archEcosystem, "key", d.dbKey, "error", err) + return h.readDatabase(ctx, p.currentMetadataEntry(archEcosystem, d.dbKey, entry), err) +} + +// storeDatabase caches meta under its digest and returns the database the +// cache row now describes. That is meta unless the write failed; bytes the +// row does not describe would be paired with the wrong signature. +func (h *ArchHandler) storeDatabase(ctx context.Context, d archDatabase, meta *upstreamMetadata) (metadataResult, *database.MetadataCacheEntry) { + p := h.proxy + path := archBlobPath(d.dbKey, archDigest(meta.body)) + defer h.lock(d.dbKey)() + + previous, _ := p.DB.GetMetadataCache(archEcosystem, d.dbKey) + if previous != nil && previous.StoragePath == path { + // Same bytes: a failed rewrite would delete the blob the row uses. + h.touchDatabase(previous, meta) + } else { + p.cacheMetadataBlob(ctx, archEcosystem, d.dbKey, path, meta) + } + row, _ := p.DB.GetMetadataCache(archEcosystem, d.dbKey) + if row == nil { + // Nothing cached at all: serve it like an uncached proxy would. + return metadataResult{body: meta.body, contentType: meta.contentType, contentEncoding: meta.contentEncoding}, nil + } + if row.StoragePath != path { + return h.readDatabase(ctx, row, errArchDatabaseNotCached) + } + if previous != nil && previous.StoragePath != path { + _ = p.Storage.Delete(ctx, previous.StoragePath) + } + return metadataResult{body: meta.body, contentType: meta.contentType, contentEncoding: meta.contentEncoding}, row +} + +// touchDatabase marks an unchanged cached database as fetched now. +func (h *ArchHandler) touchDatabase(row *database.MetadataCacheEntry, meta *upstreamMetadata) { + updated := *row + updated.ETag = sql.NullString{String: meta.etag, Valid: meta.etag != ""} + updated.LastModified = sql.NullTime{Time: meta.lastModified, Valid: !meta.lastModified.IsZero()} + updated.FetchedAt = sql.NullTime{Time: time.Now(), Valid: true} + if err := h.proxy.DB.UpsertMetadataCache(&updated); err != nil { + h.proxy.Logger.Warn("failed to record cached metadata", "ecosystem", archEcosystem, "key", row.Name, "error", err) + } +} + +var errArchDatabaseNotCached = errors.New("database could not be cached") + +// readDatabase returns the cached database row's bytes, or cause if they are +// unreadable. +func (h *ArchHandler) readDatabase(ctx context.Context, row *database.MetadataCacheEntry, cause error) (metadataResult, *database.MetadataCacheEntry) { + body, contentType, err := h.proxy.readCachedMetadata(ctx, row, nil) + if err != nil { + return metadataResult{err: fmt.Errorf("cached database unusable (%v): %w", err, cause)}, nil + } + return metadataResult{body: body, contentType: contentType, contentEncoding: row.ContentEncoding.String}, row +} + +// lock serializes cache writes for one database and its signature, so a row +// never points at a blob another write has just removed. +func (h *ArchHandler) lock(dbKey string) func() { + m, _ := h.locks.LoadOrStore(dbKey, &sync.Mutex{}) + mu := m.(*sync.Mutex) + mu.Lock() + return mu.Unlock +} + // cachedSignature returns the signature stored for the cached database. func (h *ArchHandler) cachedSignature(ctx context.Context, d archDatabase) (metadataResult, bool) { dbDigest := h.databaseDigest(d) @@ -318,15 +409,20 @@ func (h *ArchHandler) databaseDigest(d archDatabase) string { // storedSignature returns the signature stored for dbDigest. The storage // path names the digest, so a refresh with unchanged bytes keeps it. func (h *ArchHandler) storedSignature(ctx context.Context, d archDatabase, dbDigest string) (metadataResult, bool) { - entry, _ := h.proxy.DB.GetMetadataCache(archEcosystem, d.sigKey) + p := h.proxy + entry, _ := p.DB.GetMetadataCache(archEcosystem, d.sigKey) if entry == nil { return metadataResult{}, false } switch entry.StoragePath { case archMissingSignaturePath(d, dbDigest): + // Ask again after the TTL, in case the 404 was transient. + if p.MetadataTTL <= 0 || !entry.FetchedAt.Valid || time.Since(entry.FetchedAt.Time) >= p.MetadataTTL { + return metadataResult{}, false + } return metadataResult{err: ErrUpstreamNotFound}, true case archSignaturePath(d, dbDigest): - body, contentType, err := h.proxy.readCachedMetadata(ctx, entry, nil) + body, contentType, err := p.readCachedMetadata(ctx, entry, nil) if err != nil { return metadataResult{}, false } @@ -344,6 +440,8 @@ func (h *ArchHandler) storeSignature(ctx context.Context, d archDatabase, dbDige path = archMissingSignaturePath(d, dbDigest) sig = &upstreamMetadata{contentType: archPackageMediaType} } + defer h.lock(d.dbKey)() + previous, _ := p.DB.GetMetadataCache(archEcosystem, d.sigKey) p.cacheMetadataBlob(ctx, archEcosystem, d.sigKey, path, sig) if previous == nil || previous.StoragePath == path { @@ -354,8 +452,14 @@ func (h *ArchHandler) storeSignature(ctx context.Context, d archDatabase, dbDige } } +// archBlobPath names a database or signature blob after the database digest +// it belongs to, so a cache row and its bytes always agree. +func archBlobPath(key, dbDigest string) string { + return "_metadata/" + archEcosystem + "/" + key + "/" + strings.TrimPrefix(dbDigest, "sha256:") +} + func archSignaturePath(d archDatabase, dbDigest string) string { - return "_metadata/" + archEcosystem + "/" + d.sigKey + "/" + strings.TrimPrefix(dbDigest, "sha256:") + return archBlobPath(d.sigKey, dbDigest) } func archMissingSignaturePath(d archDatabase, dbDigest string) string { diff --git a/internal/handler/arch_concurrency_test.go b/internal/handler/arch_concurrency_test.go new file mode 100644 index 0000000..a2f374e --- /dev/null +++ b/internal/handler/arch_concurrency_test.go @@ -0,0 +1,315 @@ +package handler + +import ( + "context" + "errors" + "io" + "net/http" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/git-pkgs/proxy/internal/storage" +) + +// archHookStorage wraps a storage with hooks to force interleavings. +type archHookStorage struct { + storage.Storage + failStore func(path string) error + afterStore func(path string) + beforeDelete func(path string) +} + +func (s *archHookStorage) Store(ctx context.Context, path string, r io.Reader) (int64, string, error) { + if s.failStore != nil { + if err := s.failStore(path); err != nil { + return 0, "", err + } + } + n, hash, err := s.Storage.Store(ctx, path, r) + if s.afterStore != nil { + s.afterStore(path) + } + return n, hash, err +} + +func (s *archHookStorage) Delete(ctx context.Context, path string) error { + if s.beforeDelete != nil { + s.beforeDelete(path) + } + return s.Storage.Delete(ctx, path) +} + +const archTestDB, archTestSig = "/custom/os/x86_64/custom.db", "/custom/os/x86_64/custom.db.sig" + +// pauseOnce returns a hook that stops the first call for a path starting with +// prefix until release +// closes or a short timeout passes, so a test does not hang when locking +// keeps the other writer out. +func pauseOnce(prefix string, paused, release chan struct{}) func(string) { + var fired atomic.Bool + return func(p string) { + if !strings.HasPrefix(p, prefix) || !fired.CompareAndSwap(false, true) { + return + } + close(paused) + select { + case <-release: + case <-time.After(200 * time.Millisecond): + } + } +} + +// waitPaused waits until a pauseOnce hook fired. +func waitPaused(t *testing.T, paused <-chan struct{}) { + t.Helper() + select { + case <-paused: + case <-time.After(5 * time.Second): + t.Fatal("hook never fired") + } +} + +// waitBriefly waits for done, or gives up after a moment when a lock holds the +// call back. +func waitBriefly(done <-chan struct{}) { + select { + case <-done: + case <-time.After(300 * time.Millisecond): + } +} + +// TestArchHandler_ReplicasWritingOutOfOrder covers two proxy replicas sharing +// database and storage, whose database writes finish out of order: the cache +// row must never describe other bytes than its blob, or every client gets the +// newest database with an older signature. +func TestArchHandler_ReplicasWritingOutOfOrder(t *testing.T) { + upstream := newArchUpstream(t, map[string][]byte{archTestDB: []byte("db v1"), archTestSig: []byte("sig v1")}) + upstream.broken[archTestSig] = true + proxy := newArchTestProxy(t, upstream) + replicaA := NewArchHandler(proxy, "http://proxy.example", map[string]string{"custom": upstream.URL}) + replicaB := NewArchHandler(proxy, "http://proxy.example", map[string]string{"custom": upstream.URL}) + d := replicaA.database("custom", upstream.URL, archTestDB[1:]) + + assertArchBody(t, serveArchRequest(replicaA, http.MethodGet, "/custom"+archTestDB, nil), "db v1", []byte("db v1")) + upstream.mu.Lock() + upstream.files[archTestDB], upstream.files[archTestSig] = []byte("db v2"), []byte("sig v2") + upstream.broken[archTestSig] = false + upstream.mu.Unlock() + ageMetadata(t, proxy, replicaA, "custom", upstream.URL, archTestDB[1:]) + + // Replica A's signature recovery stores db v2 and stalls before its row + // update. + paused, release := make(chan struct{}), make(chan struct{}) + proxy.Storage = &archHookStorage{Storage: proxy.Storage, + afterStore: pauseOnce("_metadata/"+archEcosystem+"/"+d.dbKey+"/", paused, release)} + recovered := make(chan struct{}) + go func() { + serveArchRequest(replicaA, http.MethodGet, "/custom"+archTestSig, nil) + close(recovered) + }() + waitPaused(t, paused) + + // Replica B pairs v2, then caches v3 whose signature fetch fails. + assertArchBody(t, serveArchRequest(replicaB, http.MethodGet, "/custom"+archTestDB, nil), "B db v2", []byte("db v2")) + assertArchBody(t, serveArchRequest(replicaB, http.MethodGet, "/custom"+archTestSig, nil), "B sig v2", []byte("sig v2")) + upstream.mu.Lock() + upstream.files[archTestDB], upstream.files[archTestSig] = []byte("db v3"), []byte("sig v3") + upstream.broken[archTestSig] = true + upstream.mu.Unlock() + ageMetadata(t, proxy, replicaB, "custom", upstream.URL, archTestDB[1:]) + assertArchBody(t, serveArchRequest(replicaB, http.MethodGet, "/custom"+archTestDB, nil), "B db v3", []byte("db v3")) + + close(release) + <-recovered + + db := serveArchRequest(replicaB, http.MethodGet, "/custom"+archTestDB, nil) + sig := serveArchRequest(replicaB, http.MethodGet, "/custom"+archTestSig, nil) + if sig.Code == http.StatusOK && strings.Replace(db.Body.String(), "db", "sig", 1) != sig.Body.String() { + t.Errorf("%q served with %q", db.Body.String(), sig.Body.String()) + } +} + +// TestArchHandler_FailedDatabaseWriteKeepsPair covers a database refresh whose +// cache write fails: the proxy must not hand out the new database while the +// cache still pairs the old one with its signature. +func TestArchHandler_FailedDatabaseWriteKeepsPair(t *testing.T) { + upstream := newArchUpstream(t, map[string][]byte{archTestDB: []byte("db v1"), archTestSig: []byte("sig v1")}) + proxy := newArchTestProxy(t, upstream) + h := NewArchHandler(proxy, "http://proxy.example", map[string]string{"custom": upstream.URL}) + d := h.database("custom", upstream.URL, archTestDB[1:]) + var fail atomic.Bool + proxy.Storage = &archHookStorage{Storage: proxy.Storage, failStore: func(path string) error { + if fail.Load() && strings.HasPrefix(path, archBlobPath(d.dbKey, "")) { + return errors.New("disk full") + } + return nil + }} + + assertArchBody(t, serveArchRequest(h, http.MethodGet, "/custom"+archTestDB, nil), "db v1", []byte("db v1")) + assertArchBody(t, serveArchRequest(h, http.MethodGet, "/custom"+archTestSig, nil), "sig v1", []byte("sig v1")) + upstream.set(archTestDB, []byte("db v2")) + upstream.set(archTestSig, []byte("sig v2")) + ageMetadata(t, proxy, h, "custom", upstream.URL, archTestDB[1:]) + fail.Store(true) + + db := serveArchRequest(h, http.MethodGet, "/custom"+archTestDB, nil) + sig := serveArchRequest(h, http.MethodGet, "/custom"+archTestSig, nil) + if db.Code == http.StatusOK && sig.Code == http.StatusOK && strings.Replace(db.Body.String(), "db", "sig", 1) != sig.Body.String() { + t.Errorf("%q served with %q", db.Body.String(), sig.Body.String()) + } +} + +// TestArchHandler_ConcurrentSignatureWritesLeaveOneBlob covers two signature +// writes overlapping: afterwards exactly one signature blob is left and the +// cache row points at it. +func TestArchHandler_ConcurrentSignatureWritesLeaveOneBlob(t *testing.T) { + d1, d2, d3 := archDigest([]byte("1")), archDigest([]byte("2")), archDigest([]byte("3")) + + for _, tt := range []struct { + name string + hook func(h *ArchHandler, d archDatabase, s *archHookStorage, paused, release chan struct{}) + first string // digest stored by the stalled write + other string // digest stored by the write overtaking it + }{ + {"stalled after store", func(_ *ArchHandler, d archDatabase, s *archHookStorage, paused, release chan struct{}) { + s.afterStore = pauseOnce(archSignaturePath(d, d2), paused, release) + }, d2, d3}, + {"stalled before delete", func(_ *ArchHandler, d archDatabase, s *archHookStorage, paused, release chan struct{}) { + s.beforeDelete = pauseOnce(archSignaturePath(d, d1), paused, release) + }, d2, d1}, + } { + t.Run(tt.name, func(t *testing.T) { + upstream := newArchUpstream(t, map[string][]byte{}) + proxy := newArchTestProxy(t, upstream) + mock := proxy.Storage.(*mockStorage) + h := NewArchHandler(proxy, "http://proxy.example", map[string]string{"custom": upstream.URL}) + d := h.database("custom", upstream.URL, archTestDB[1:]) + ctx := context.Background() + h.storeSignature(ctx, d, d1, &upstreamMetadata{body: []byte("sig 1")}) + + hooks := &archHookStorage{Storage: mock} + proxy.Storage = hooks + paused, release := make(chan struct{}), make(chan struct{}) + tt.hook(h, d, hooks, paused, release) + + stalled := make(chan struct{}) + go func() { + h.storeSignature(ctx, d, tt.first, &upstreamMetadata{body: []byte("sig first")}) + close(stalled) + }() + waitPaused(t, paused) + overtaking := make(chan struct{}) + go func() { + h.storeSignature(ctx, d, tt.other, &upstreamMetadata{body: []byte("sig other")}) + close(overtaking) + }() + waitBriefly(overtaking) + close(release) + <-stalled + <-overtaking + + row, _ := proxy.DB.GetMetadataCache(archEcosystem, d.sigKey) + if _, err := mock.Open(ctx, row.StoragePath); err != nil { + t.Errorf("row points at %s, which is gone: %v", row.StoragePath, err) + } + mock.mu.Lock() + defer mock.mu.Unlock() + var blobs []string + for path := range mock.files { + if strings.HasPrefix(path, archBlobPath(d.sigKey, "")) { + blobs = append(blobs, path) + } + } + if len(blobs) != 1 { + t.Errorf("signature blobs = %v, want only the row's", blobs) + } + }) + } +} + +// TestArchHandler_MissingSignatureRecheckedAfterTTL covers a 404 for the +// signature that was only transient: once the TTL is over the proxy asks +// again instead of answering 404 until the next database is published. +func TestArchHandler_MissingSignatureRecheckedAfterTTL(t *testing.T) { + upstream := newArchUpstream(t, map[string][]byte{archTestDB: []byte("db v1")}) + proxy := newArchTestProxy(t, upstream) + h := NewArchHandler(proxy, "http://proxy.example", map[string]string{"custom": upstream.URL}) + + assertArchBody(t, serveArchRequest(h, http.MethodGet, "/custom"+archTestDB, nil), "db", []byte("db v1")) + if w := serveArchRequest(h, http.MethodGet, "/custom"+archTestSig, nil); w.Code != http.StatusNotFound { + t.Fatalf("signature: status = %d, want 404", w.Code) + } + + upstream.set(archTestSig, []byte("sig v1")) + ageMetadata(t, proxy, h, "custom", upstream.URL, archTestDB[1:]) + ageMetadata(t, proxy, h, "custom", upstream.URL, archTestSig[1:]) + + assertArchBody(t, serveArchRequest(h, http.MethodGet, "/custom"+archTestDB, nil), "db refreshed", []byte("db v1")) + assertArchBody(t, serveArchRequest(h, http.MethodGet, "/custom"+archTestSig, nil), "signature after TTL", []byte("sig v1")) +} + +// TestArchHandler_SlowFetchDoesNotOverwriteNewerDatabase covers a signature +// recovery whose database fetch is still in flight while other clients +// refresh to newer databases. Its result must not replace the newer cached +// database, or a client holding that one gets an older signature. +func TestArchHandler_SlowFetchDoesNotOverwriteNewerDatabase(t *testing.T) { + upstream := newArchUpstream(t, map[string][]byte{archTestDB: []byte("db v1"), archTestSig: []byte("sig v1")}) + upstream.broken[archTestSig] = true + proxy := newArchTestProxy(t, upstream) + h := NewArchHandler(proxy, "http://proxy.example", map[string]string{"custom": upstream.URL}) + + assertArchBody(t, serveArchRequest(h, http.MethodGet, "/custom"+archTestDB, nil), "db v1", []byte("db v1")) + upstream.mu.Lock() + upstream.files[archTestDB], upstream.files[archTestSig] = []byte("db v2"), []byte("sig v2") + upstream.broken[archTestSig] = false + upstream.mu.Unlock() + ageMetadata(t, proxy, h, "custom", upstream.URL, archTestDB[1:]) + + // Client A's signature recovery has fetched db v2; the response is slow. + paused, release := make(chan struct{}), make(chan struct{}) + upstream.mu.Lock() + upstream.served = pauseOnce(archTestDB, paused, release) + upstream.mu.Unlock() + recovered := make(chan struct{}) + go func() { + serveArchRequest(h, http.MethodGet, "/custom"+archTestSig, nil) + close(recovered) + }() + waitPaused(t, paused) + + // Client B refreshes meanwhile, then the upstream moves to v3, whose + // signature fails, and client C refreshes to it. + b := make(chan struct{}) + go func() { + serveArchRequest(h, http.MethodGet, "/custom"+archTestDB, nil) + serveArchRequest(h, http.MethodGet, "/custom"+archTestSig, nil) + close(b) + }() + waitBriefly(b) + upstream.mu.Lock() + upstream.files[archTestDB], upstream.files[archTestSig] = []byte("db v3"), []byte("sig v3") + upstream.broken[archTestSig] = true + upstream.mu.Unlock() + ageMetadata(t, proxy, h, "custom", upstream.URL, archTestDB[1:]) + var cDB string + c := make(chan struct{}) + go func() { + cDB = serveArchRequest(h, http.MethodGet, "/custom"+archTestDB, nil).Body.String() + close(c) + }() + waitBriefly(c) + + close(release) + <-recovered + <-b + <-c + + if sig := serveArchRequest(h, http.MethodGet, "/custom"+archTestSig, nil); sig.Code == http.StatusOK && strings.Replace(cDB, "db", "sig", 1) != sig.Body.String() { + t.Errorf("client holding %q got %q", cDB, sig.Body.String()) + } + if got := serveArchRequest(h, http.MethodGet, "/custom"+archTestDB, nil).Body.String(); got < cDB { + t.Errorf("cache went back to %q after serving %q", got, cDB) + } +} From 9d0175b1e2b40abb01dfeffa99f7f7e834ddfdf8 Mon Sep 17 00:00:00 2001 From: Giles Westwood <201443+gilesw@users.noreply.github.com> Date: Mon, 12 Oct 2026 19:22:42 +0100 Subject: [PATCH 244/633] Try configured generic_artifacts before built-in patterns The first matching pattern names the package, so an upstream's own patterns now take precedence over overlapping built-ins and keep the package names, cache entries and rules already keyed on them. --- docs/configuration.md | 3 ++- internal/handler/generic.go | 10 ++++++---- internal/handler/generic_test.go | 9 ++++++--- 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index ce8aa30..7e94a16 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -371,7 +371,8 @@ the upstream is named: | `https://dl.google.com/go` | `go{version}.{os}-{arch}.tar.gz`, `.zip` and their `.sha256` | the upstream name | Other upstreams can declare their own version-pinned paths in -`generic_artifacts`, which add to any built-in ones. Each pattern is a regular expression that must match the +`generic_artifacts`. These are tried before any built-in ones, so a configured +pattern that overlaps a built-in decides the package name. Each pattern is a regular expression that must match the whole path after `/generic/{name}/`, with `version` and `file` named groups (single path segments) and an optional `name` group for the package name, which otherwise is the upstream name. Matching downloads are cached like diff --git a/internal/handler/generic.go b/internal/handler/generic.go index 954e091..bf7f088 100644 --- a/internal/handler/generic.go +++ b/internal/handler/generic.go @@ -30,7 +30,7 @@ var githubReleaseAssetPattern = regexp.MustCompile(`^([^/]+)/([^/]+)/releases/do // builtinGenericArtifacts maps well-known download hosts to patterns for // their version-pinned paths. They apply to any generic upstream configured -// with that base URL, before the upstream's own generic_artifacts. +// with that base URL, after the upstream's own generic_artifacts. var builtinGenericArtifacts = map[string][]string{ // dist/{project dirs}/{version}/[binaries/]{file}, e.g. maven/maven-3 or maven/mvnd. "https://archive.apache.org": { @@ -78,15 +78,17 @@ func NewGenericHandler(proxy *Proxy, repositories map[string]string, artifactPat artifactPatterns: make(map[string][]*regexp.Regexp), } for name, upstreamURL := range repositories { - upstreamURL = strings.TrimSuffix(upstreamURL, "/") - h.repositories[name] = upstreamURL - h.addArtifactPatterns(name, builtinGenericArtifacts[upstreamURL]) + h.repositories[name] = strings.TrimSuffix(upstreamURL, "/") } + // The first matching pattern wins, so configured patterns go first. if len(artifactPatterns) > 0 { for name, patterns := range artifactPatterns[0] { h.addArtifactPatterns(name, patterns) } } + for name, upstreamURL := range h.repositories { + h.addArtifactPatterns(name, builtinGenericArtifacts[upstreamURL]) + } return h } diff --git a/internal/handler/generic_test.go b/internal/handler/generic_test.go index 1292e8c..2165b1a 100644 --- a/internal/handler/generic_test.go +++ b/internal/handler/generic_test.go @@ -112,7 +112,7 @@ func TestGenericHandler_BuiltinArtifactPatterns(t *testing.T) { "golang": "https://dl.google.com/go", "mirror": "https://mirror.example.com", }, map[string][]string{ - "apache": {`extra/(?P[^/]+)/(?P[^/]+)`}, + "apache": {`extra/(?P[^/]+)/(?P[^/]+)`, testMavenPattern}, }) tests := []struct { @@ -121,10 +121,13 @@ func TestGenericHandler_BuiltinArtifactPatterns(t *testing.T) { want genericArtifact ok bool }{ + // The configured testMavenPattern overlaps the built-in and takes precedence. {"apache", "dist/maven/maven-3/3.9.9/binaries/apache-maven-3.9.9-bin.tar.gz", - genericArtifact{name: "maven/maven-3", version: "3.9.9", filename: "apache-maven-3.9.9-bin.tar.gz"}, true}, + genericArtifact{name: "maven", version: "3.9.9", filename: "apache-maven-3.9.9-bin.tar.gz"}, true}, {"apache", "dist/maven/maven-3/3.9.9/binaries/apache-maven-3.9.9-bin.tar.gz.sha512", - genericArtifact{name: "maven/maven-3", version: "3.9.9", filename: "apache-maven-3.9.9-bin.tar.gz.sha512"}, true}, + genericArtifact{name: "maven", version: "3.9.9", filename: "apache-maven-3.9.9-bin.tar.gz.sha512"}, true}, + {"apache", "dist/maven/maven-4/4.0.0/binaries/apache-maven-4.0.0-bin.tar.gz", + genericArtifact{name: "maven/maven-4", version: "4.0.0", filename: "apache-maven-4.0.0-bin.tar.gz"}, true}, {"apache", "dist/maven/mvnd/1.0.6/maven-mvnd-1.0.6-linux-amd64.tar.gz", genericArtifact{name: "maven/mvnd", version: "1.0.6", filename: "maven-mvnd-1.0.6-linux-amd64.tar.gz"}, true}, {"apache", "dist/maven/maven-3/", genericArtifact{}, false}, From 1c6cfae81360a94e70e8067dcbd0bd1bdf286246 Mon Sep 17 00:00:00 2001 From: Christian Heim Date: Sat, 10 Oct 2026 09:57:43 +0200 Subject: [PATCH 245/633] Serve the Go checksum database from its own URL Everything under /go/sumdb/ was passed on to the module upstream. But proxy.golang.org doesn't serve the checksum database, so those requests always came back 404 and every client had to talk to sum.golang.org itself. The Go handler now serves /sumdb/{name}/ for the databases in upstream.go_sumdbs, which defaults to sum.golang.org. Lookups, latest and tiles come from the database through the metadata cache. Complete tiles never change, so once they're cached we don't ask again. Unknown names and paths that aren't part of the protocol get a 404 without going upstream. If the database answers 404 we pass on up to 4 KiB of its text, because the go command prints it and it tells you why a version is missing. For that the not-found error from the metadata fetch now carries the start of the body. Other handlers still just see ErrUpstreamNotFound. --- go.mod | 2 +- internal/config/config.go | 10 + internal/config/config_test.go | 40 +++ internal/handler/download_test.go | 1 - internal/handler/go.go | 13 +- internal/handler/go_sumdb.go | 173 ++++++++++++ internal/handler/go_sumdb_test.go | 425 ++++++++++++++++++++++++++++++ internal/handler/handler.go | 17 +- internal/server/go_sumdbs_test.go | 38 +++ internal/server/server.go | 1 + 10 files changed, 709 insertions(+), 11 deletions(-) create mode 100644 internal/handler/go_sumdb.go create mode 100644 internal/handler/go_sumdb_test.go create mode 100644 internal/server/go_sumdbs_test.go diff --git a/go.mod b/go.mod index af9984d..052604e 100644 --- a/go.mod +++ b/go.mod @@ -29,6 +29,7 @@ require ( github.com/spdx/tools-golang v0.5.7 github.com/swaggo/swag v1.16.6 gocloud.dev v0.46.0 + golang.org/x/mod v0.41.0 golang.org/x/net v0.59.0 golang.org/x/sync v0.23.0 google.golang.org/protobuf v1.36.12 @@ -117,7 +118,6 @@ require ( go.opentelemetry.io/otel/trace v1.45.0 // indirect go.yaml.in/yaml/v3 v3.0.5 // indirect golang.org/x/crypto v0.57.0 // indirect - golang.org/x/mod v0.41.0 // indirect golang.org/x/oauth2 v0.36.0 // indirect golang.org/x/sys v0.48.0 // indirect golang.org/x/text v0.42.0 // indirect diff --git a/internal/config/config.go b/internal/config/config.go index b79a246..46cda82 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -573,6 +573,13 @@ type UpstreamConfig struct { // Default: https://proxy.golang.org Go string `json:"go" yaml:"go"` + // GoSumDBs maps Go checksum database names to their base URLs, served at + // /go/sumdb/{name}/ so the go command reaches the database through + // GOPROXY. Unknown names get 404, which makes the go command contact the + // database directly. + // Default when empty: {"sum.golang.org": "https://sum.golang.org"}. + GoSumDBs map[string]string `json:"go_sumdbs" yaml:"go_sumdbs"` + // Hex is the upstream Hex repository URL. // Default: https://repo.hex.pm Hex string `json:"hex" yaml:"hex"` @@ -773,6 +780,9 @@ func (u *UpstreamConfig) Validate() error { if err := validateNamedUpstreams("upstream.arch", u.Arch); err != nil { return err } + if err := validateNamedUpstreams("upstream.go_sumdbs", u.GoSumDBs); err != nil { + return err + } if err := validateNamedUpstreams("upstream.oci", u.OCI); err != nil { return err } diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 9bee2dc..88150ae 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -1612,6 +1612,46 @@ func TestValidateNamedUpstreams(t *testing.T) { } } +func TestValidateGoSumDBs(t *testing.T) { + tests := []struct { + name string + modify func(*Config) + wantErr bool + }{ + { + name: "valid Go checksum database", + modify: func(cfg *Config) { + cfg.Upstream.GoSumDBs = map[string]string{"sum.golang.org": "https://sum.golang.org"} + }, + }, + { + name: "Go checksum database name contains path separator", + modify: func(cfg *Config) { + cfg.Upstream.GoSumDBs = map[string]string{"sum.golang.org/x": "https://sum.golang.org"} + }, + wantErr: true, + }, + { + name: "Go checksum database URL is not absolute", + modify: func(cfg *Config) { + cfg.Upstream.GoSumDBs = map[string]string{"sum.golang.org": "sum.golang.org"} + }, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cfg := Default() + tt.modify(cfg) + err := cfg.Validate() + if (err != nil) != tt.wantErr { + t.Errorf("Validate() error = %v, wantErr %t", err, tt.wantErr) + } + }) + } +} + func TestValidateCacheArtifactsDisabled(t *testing.T) { tests := []struct { name string diff --git a/internal/handler/download_test.go b/internal/handler/download_test.go index 0df443d..2797d42 100644 --- a/internal/handler/download_test.go +++ b/internal/handler/download_test.go @@ -330,7 +330,6 @@ func TestGoHandler_UpstreamProxy(t *testing.T) { "/golang.org/x/text/@v/v0.14.0.info", "/golang.org/x/text/@v/v0.14.0.mod", "/golang.org/x/text/@latest", - "/sumdb/sum.golang.org/lookup/golang.org/x/text@v0.14.0", } for _, path := range tests { diff --git a/internal/handler/go.go b/internal/handler/go.go index 353edbb..6348a1e 100644 --- a/internal/handler/go.go +++ b/internal/handler/go.go @@ -31,6 +31,7 @@ type GoHandler struct { proxy *Proxy upstreamURL string proxyURL string + sumDBs map[string]string } // NewGoHandler creates a new Go module proxy handler. @@ -39,6 +40,7 @@ func NewGoHandler(proxy *Proxy, proxyURL string) *GoHandler { proxy: proxy, upstreamURL: goUpstream, proxyURL: strings.TrimSuffix(proxyURL, "/"), + sumDBs: map[string]string{defaultGoSumDBName: defaultGoSumDBURL}, } } @@ -65,9 +67,9 @@ func (h *GoHandler) Routes() http.Handler { func (h *GoHandler) handleRequest(w http.ResponseWriter, r *http.Request) { path := strings.TrimPrefix(r.URL.Path, "/") - // Sumdb requests - proxy through - if strings.HasPrefix(path, "sumdb/") { - h.proxyUpstream(w, r) + // Checksum database requests go to the database, not the module upstream + if rest, ok := strings.CutPrefix(path, "sumdb/"); ok { + h.handleSumDB(w, r, rest) return } @@ -141,11 +143,6 @@ func (h *GoHandler) handleDownload(w http.ResponseWriter, r *http.Request, modul ServeArtifactRequest(w, r, result) } -// proxyUpstream forwards a request to proxy.golang.org without caching. -func (h *GoHandler) proxyUpstream(w http.ResponseWriter, r *http.Request) { - h.proxy.ProxyUpstream(w, r, h.upstreamURL+r.URL.Path, nil) -} - // proxyCached forwards a request with metadata caching. func (h *GoHandler) proxyCached(w http.ResponseWriter, r *http.Request, cacheKey string) { h.proxy.ProxyCached(w, r, h.upstreamURL+r.URL.Path, "golang", cacheKey, "*/*") diff --git a/internal/handler/go_sumdb.go b/internal/handler/go_sumdb.go new file mode 100644 index 0000000..32665f6 --- /dev/null +++ b/internal/handler/go_sumdb.go @@ -0,0 +1,173 @@ +package handler + +import ( + "crypto/sha256" + "encoding/hex" + "errors" + "net/http" + "strconv" + "strings" + + "github.com/git-pkgs/proxy/internal/metrics" + "golang.org/x/mod/module" + "golang.org/x/mod/semver" + "golang.org/x/mod/sumdb/tlog" +) + +const ( + // defaultGoSumDBName is the checksum database the go command uses when + // GOSUMDB is not set. + defaultGoSumDBName = "sum.golang.org" + defaultGoSumDBURL = "https://sum.golang.org" + goEcosystem = "golang" +) + +// SetSumDBs sets the checksum databases served at /sumdb/{name}/, mapping +// each database name to its base URL. An empty map keeps the default, +// sum.golang.org. +func (h *GoHandler) SetSumDBs(sumDBs map[string]string) { + if len(sumDBs) == 0 { + h.sumDBs = map[string]string{defaultGoSumDBName: defaultGoSumDBURL} + return + } + h.sumDBs = make(map[string]string, len(sumDBs)) + for name, sumDBURL := range sumDBs { + h.sumDBs[name] = strings.TrimSuffix(sumDBURL, "/") + } +} + +// handleSumDB serves checksum database requests that the go command sends +// through GOPROXY as /sumdb/{name}/{path}. The module proxy upstream does not +// serve them, so they are fetched from the configured database itself. +// Unknown names get 404, which makes the go command contact the database +// directly. +func (h *GoHandler) handleSumDB(w http.ResponseWriter, r *http.Request, rest string) { + name, path, _ := strings.Cut(rest, "/") + sumDBURL, ok := h.sumDBs[name] + if !ok { + http.NotFound(w, r) + return + } + if path == "supported" { + w.WriteHeader(http.StatusOK) + return + } + immutable, valid := parseGoSumDBPath(path) + if !valid { + http.NotFound(w, r) + return + } + + cacheKey := goSumDBCacheKey(name, sumDBURL, path) + if immutable && h.serveCachedSumDBTile(w, r, cacheKey) { + return + } + h.proxySumDB(w, r, sumDBURL+"/"+path, cacheKey) +} + +// parseGoSumDBPath reports whether path is a request the checksum database +// protocol defines and whether its response can never change. Only complete +// tiles are immutable; partial tiles, latest and lookups follow the metadata +// TTL. +func parseGoSumDBPath(path string) (immutable, valid bool) { + if path == "latest" { + return false, true + } + if mod, ok := strings.CutPrefix(path, "lookup/"); ok { + return false, validGoSumDBLookup(mod) + } + tile, err := tlog.ParseTilePath(path) + if err != nil { + return false, false + } + return tile.W == 1< 0: + w.Header().Set(headerContentType, "text/plain; charset=utf-8") + w.Header().Set("X-Content-Type-Options", "nosniff") + w.WriteHeader(http.StatusNotFound) + _, _ = w.Write(notFound.body) + case errors.Is(err, ErrUpstreamNotFound): + http.Error(w, "not found", http.StatusNotFound) + default: + p.Logger.Error("checksum database fetch failed", "url", upstreamURL, "error", err) + http.Error(w, "failed to fetch from upstream", http.StatusBadGateway) + } + return + } + + p.writeMetadataCachedResponseWithEncoding(w, r, goEcosystem, cacheKey, body, contentType, contentEncoding) +} diff --git a/internal/handler/go_sumdb_test.go b/internal/handler/go_sumdb_test.go new file mode 100644 index 0000000..a9ff1df --- /dev/null +++ b/internal/handler/go_sumdb_test.go @@ -0,0 +1,425 @@ +package handler + +import ( + "bytes" + "crypto/rand" + "errors" + "fmt" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "golang.org/x/mod/sumdb" + "golang.org/x/mod/sumdb/note" +) + +const testSumDBName = "sum.example" + +// fakeSumDB is a checksum database upstream serving fixed files. Unknown +// paths get 404 with the body from notFound, or a default text. +type fakeSumDB struct { + server *httptest.Server + files map[string]string + notFound map[string]string + + mu sync.Mutex + hits map[string]int + down atomic.Bool +} + +func newFakeSumDB(t *testing.T, files map[string]string) *fakeSumDB { + t.Helper() + db := &fakeSumDB{files: files, notFound: map[string]string{}, hits: map[string]int{}} + db.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + db.mu.Lock() + db.hits[r.URL.Path]++ + db.mu.Unlock() + if db.down.Load() { + http.Error(w, "unavailable", http.StatusInternalServerError) + return + } + body, ok := db.files[r.URL.Path] + if !ok { + text, custom := db.notFound[r.URL.Path] + if !custom { + text = "not found: " + r.URL.Path + } + w.Header().Set("Content-Type", "text/plain; charset=utf-8") + w.WriteHeader(http.StatusNotFound) + _, _ = io.WriteString(w, text) + return + } + w.Header().Set("Content-Type", "text/plain; charset=utf-8") + _, _ = io.WriteString(w, body) + })) + t.Cleanup(db.server.Close) + return db +} + +func (db *fakeSumDB) requests(path string) int { + db.mu.Lock() + defer db.mu.Unlock() + return db.hits[path] +} + +func (db *fakeSumDB) totalRequests() int { + db.mu.Lock() + defer db.mu.Unlock() + total := 0 + for _, n := range db.hits { + total += n + } + return total +} + +func newSumDBTestHandler(t *testing.T, sumDBURL string, cacheMetadata bool, ttl time.Duration) *GoHandler { + t.Helper() + proxy, _, _, _ := setupTestProxy(t) + proxy.HTTPClient = http.DefaultClient + proxy.CacheMetadata = cacheMetadata + proxy.MetadataTTL = ttl + h := NewGoHandler(proxy, "http://proxy.example") + h.SetSumDBs(map[string]string{testSumDBName: sumDBURL}) + return h +} + +func serveGoRequest(h *GoHandler, target string) *httptest.ResponseRecorder { + w := httptest.NewRecorder() + h.Routes().ServeHTTP(w, httptest.NewRequest(http.MethodGet, target, nil)) + return w +} + +func TestGoSumDB_SupportedProbe(t *testing.T) { + t.Run("default database", func(t *testing.T) { + proxy, _, _, _ := setupTestProxy(t) + h := NewGoHandler(proxy, "http://proxy.example") + if w := serveGoRequest(h, "/sumdb/sum.golang.org/supported"); w.Code != http.StatusOK { + t.Errorf("supported for default sum.golang.org: status = %d, want 200", w.Code) + } + }) + + t.Run("configured databases replace the default", func(t *testing.T) { + db := newFakeSumDB(t, nil) + h := newSumDBTestHandler(t, db.server.URL, true, time.Hour) + + for target, want := range map[string]int{ + "/sumdb/sum.example/supported": http.StatusOK, + "/sumdb/sum.golang.org/supported": http.StatusNotFound, + "/sumdb/other.example/supported": http.StatusNotFound, + } { + if w := serveGoRequest(h, target); w.Code != want { + t.Errorf("%s: status = %d, want %d", target, w.Code, want) + } + } + if n := db.totalRequests(); n != 0 { + t.Errorf("upstream requests = %d, want 0: supported is answered by the proxy", n) + } + }) +} + +func TestGoSumDB_RequestsGoToTheDatabase(t *testing.T) { + var moduleUpstreamRequests atomic.Int32 + moduleUpstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + moduleUpstreamRequests.Add(1) + http.NotFound(w, r) + })) + defer moduleUpstream.Close() + + const lookup = "/lookup/golang.org/x/text@v0.14.0" + db := newFakeSumDB(t, map[string]string{lookup: "lookup record"}) + + proxy, _, _, _ := setupTestProxy(t) + proxy.HTTPClient = http.DefaultClient + h := NewGoHandlerWithUpstream(proxy, "http://proxy.example", moduleUpstream.URL) + h.SetSumDBs(map[string]string{testSumDBName: db.server.URL + "/"}) + + w := serveGoRequest(h, "/sumdb/sum.example"+lookup) + if w.Code != http.StatusOK || w.Body.String() != "lookup record" { + t.Fatalf("status = %d, body = %q; want 200 with the database's response", w.Code, w.Body.String()) + } + if n := db.requests(lookup); n != 1 { + t.Errorf("database requests for %s = %d, want 1", lookup, n) + } + if n := moduleUpstreamRequests.Load(); n != 0 { + t.Errorf("module upstream requests = %d, want 0", n) + } +} + +func TestGoSumDB_LookupCachedWithinTTL(t *testing.T) { + const lookup = "/lookup/golang.org/x/text@v0.14.0" + db := newFakeSumDB(t, map[string]string{lookup: "lookup record"}) + h := newSumDBTestHandler(t, db.server.URL, true, time.Hour) + + for i := range 2 { + w := serveGoRequest(h, "/sumdb/sum.example"+lookup) + if w.Code != http.StatusOK || w.Body.String() != "lookup record" { + t.Fatalf("request %d: status = %d, body = %q", i, w.Code, w.Body.String()) + } + } + if n := db.requests(lookup); n != 1 { + t.Errorf("database requests = %d, want 1: the second lookup is a cache hit", n) + } +} + +func TestGoSumDB_CompleteTilesSkipRevalidation(t *testing.T) { + files := map[string]string{ + "/tile/8/0/000": "complete hash tile", + "/tile/8/data/000": "complete data tile", + "/tile/8/0/001.p/5": "partial hash tile", + "/tile/8/data/001.p/5": "partial data tile", + "/latest": "signed tree", + } + wantRequests := map[string]int{ + "/tile/8/0/000": 1, + "/tile/8/data/000": 1, + "/tile/8/0/001.p/5": 2, + "/tile/8/data/001.p/5": 2, + "/latest": 2, + } + db := newFakeSumDB(t, files) + // Every cached entry is past its TTL by the second request. + h := newSumDBTestHandler(t, db.server.URL, true, time.Nanosecond) + + for range 2 { + for path, want := range files { + w := serveGoRequest(h, "/sumdb/sum.example"+path) + if w.Code != http.StatusOK || w.Body.String() != want { + t.Fatalf("%s: status = %d, body = %q, want %q", path, w.Code, w.Body.String(), want) + } + } + } + for path, want := range wantRequests { + if n := db.requests(path); n != want { + t.Errorf("%s: database requests = %d, want %d", path, n, want) + } + } +} + +func TestGoSumDB_CompleteTilesNotStoredWithoutMetadataCache(t *testing.T) { + const tile = "/tile/8/0/000" + db := newFakeSumDB(t, map[string]string{tile: "complete hash tile"}) + h := newSumDBTestHandler(t, db.server.URL, false, time.Hour) + + for range 2 { + if w := serveGoRequest(h, "/sumdb/sum.example"+tile); w.Code != http.StatusOK { + t.Fatalf("status = %d, want 200", w.Code) + } + } + if n := db.requests(tile); n != 2 { + t.Errorf("database requests = %d, want 2 with cache_metadata off", n) + } +} + +func TestGoSumDB_OfflineFallback(t *testing.T) { + files := map[string]string{ + "/lookup/golang.org/x/text@v0.14.0": "lookup record", + "/latest": "signed tree", + "/tile/8/0/000": "complete hash tile", + "/tile/8/0/001.p/5": "partial hash tile", + } + db := newFakeSumDB(t, files) + h := newSumDBTestHandler(t, db.server.URL, true, time.Nanosecond) + + for path := range files { + if w := serveGoRequest(h, "/sumdb/sum.example"+path); w.Code != http.StatusOK { + t.Fatalf("%s online: status = %d, want 200", path, w.Code) + } + } + + db.down.Store(true) + for path, want := range files { + w := serveGoRequest(h, "/sumdb/sum.example"+path) + if w.Code != http.StatusOK || w.Body.String() != want { + t.Errorf("%s offline: status = %d, body = %q, want 200 with %q", path, w.Code, w.Body.String(), want) + } + } +} + +func TestGoSumDB_NotFoundRelaysUpstreamText(t *testing.T) { + const lookup = "/lookup/github.com/google/uuid@v9.9.9" + const reason = "not found: github.com/google/uuid@v9.9.9: invalid version: unknown revision v9.9.9" + + for _, cacheMetadata := range []bool{true, false} { + t.Run(fmt.Sprintf("cache_metadata=%v", cacheMetadata), func(t *testing.T) { + db := newFakeSumDB(t, nil) + db.notFound[lookup] = reason + h := newSumDBTestHandler(t, db.server.URL, cacheMetadata, time.Hour) + + w := serveGoRequest(h, "/sumdb/sum.example"+lookup) + if w.Code != http.StatusNotFound { + t.Fatalf("status = %d, want 404", w.Code) + } + if got := strings.TrimSpace(w.Body.String()); got != reason { + t.Errorf("body = %q, want the database's text %q", got, reason) + } + }) + } + + t.Run("long text is cut", func(t *testing.T) { + db := newFakeSumDB(t, nil) + db.notFound[lookup] = strings.Repeat("x", upstreamNotFoundBodyLimit+1000) + h := newSumDBTestHandler(t, db.server.URL, true, time.Hour) + + w := serveGoRequest(h, "/sumdb/sum.example"+lookup) + if w.Code != http.StatusNotFound { + t.Fatalf("status = %d, want 404", w.Code) + } + if got := w.Body.Len(); got != upstreamNotFoundBodyLimit { + t.Errorf("body length = %d, want %d", got, upstreamNotFoundBodyLimit) + } + }) +} + +func TestGoSumDB_InvalidPathsNeverReachUpstream(t *testing.T) { + db := newFakeSumDB(t, nil) + h := newSumDBTestHandler(t, db.server.URL, true, time.Hour) + + for _, target := range []string{ + "/sumdb/sum.example", + "/sumdb/sum.example/", + "/sumdb/sum.example/other", + "/sumdb/sum.example/latest/extra", + "/sumdb/sum.example/lookup/", + "/sumdb/sum.example/lookup/golang.org/x/text", + "/sumdb/sum.example/lookup/golang.org/x/text@", + "/sumdb/sum.example/lookup/golang.org/x/text@master", + "/sumdb/sum.example/lookup/github.com/Google/uuid@v1.6.0", + "/sumdb/sum.example/lookup/../../etc/passwd@v1.0.0", + "/sumdb/sum.example/tile/x/0/000", + "/sumdb/sum.example/tile/8/0/000.p/0", + "/sumdb/sum.example/tile/8/0/000.p/256", + "/sumdb/sum.example/tile/8/0/../../../latest", + "/sumdb/unknown.example/latest", + } { + if w := serveGoRequest(h, target); w.Code != http.StatusNotFound { + t.Errorf("%s: status = %d, want 404", target, w.Code) + } + } + if n := db.totalRequests(); n != 0 { + t.Errorf("upstream requests = %d, want 0", n) + } +} + +// sumDBClientOps lets the x/mod checksum database client read through the +// proxy, keeping its configuration and cache in memory. +type sumDBClientOps struct { + t *testing.T + remote string + key string + + mu sync.Mutex + config map[string][]byte + cache map[string][]byte +} + +func newSumDBClientOps(t *testing.T, remote, key string) *sumDBClientOps { + return &sumDBClientOps{t: t, remote: remote, key: key, config: map[string][]byte{}, cache: map[string][]byte{}} +} + +func (o *sumDBClientOps) ReadRemote(path string) ([]byte, error) { + resp, err := http.Get(o.remote + path) + if err != nil { + return nil, err + } + defer func() { _ = resp.Body.Close() }() + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("GET %s: %s: %s", path, resp.Status, body) + } + return body, nil +} + +func (o *sumDBClientOps) ReadConfig(file string) ([]byte, error) { + if file == "key" { + return []byte(o.key), nil + } + o.mu.Lock() + defer o.mu.Unlock() + return o.config[file], nil +} + +func (o *sumDBClientOps) WriteConfig(file string, old, data []byte) error { + o.mu.Lock() + defer o.mu.Unlock() + if !bytes.Equal(o.config[file], old) { + return sumdb.ErrWriteConflict + } + o.config[file] = data + return nil +} + +func (o *sumDBClientOps) ReadCache(file string) ([]byte, error) { + o.mu.Lock() + defer o.mu.Unlock() + data, ok := o.cache[file] + if !ok { + return nil, errors.New("not cached") + } + return data, nil +} + +func (o *sumDBClientOps) WriteCache(file string, data []byte) { + o.mu.Lock() + defer o.mu.Unlock() + o.cache[file] = data +} + +func (o *sumDBClientOps) Log(msg string) { o.t.Log(msg) } + +func (o *sumDBClientOps) SecurityError(msg string) { + o.t.Errorf("checksum database security error: %s", msg) +} + +// TestGoSumDB_ClientRoundTrip runs the go command's checksum database client +// against a real database implementation through the proxy, so signatures, +// record proofs and tile hashes are checked end to end. A second client then +// verifies the same lookup from the proxy's cache with the database down. +func TestGoSumDB_ClientRoundTrip(t *testing.T) { + signerKey, verifierKey, err := note.GenerateKey(rand.Reader, testSumDBName) + if err != nil { + t.Fatal(err) + } + gosum := func(path, version string) ([]byte, error) { + return fmt.Appendf(nil, "%s %s h1:Y29udGVudA==\n%s %s/go.mod h1:Z28ubW9k\n", path, version, path, version), nil + } + database := sumdb.NewServer(sumdb.NewTestServer(signerKey, gosum)) + var down atomic.Bool + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if down.Load() { + http.Error(w, "unavailable", http.StatusInternalServerError) + return + } + database.ServeHTTP(w, r) + })) + defer upstream.Close() + + h := newSumDBTestHandler(t, upstream.URL, true, time.Hour) + proxy := httptest.NewServer(h.Routes()) + defer proxy.Close() + remote := proxy.URL + "/sumdb/" + testSumDBName + + client := sumdb.NewClient(newSumDBClientOps(t, remote, verifierKey)) + for _, m := range [][2]string{{"rsc.io/quote", "v1.5.2"}, {"golang.org/x/text", "v0.3.0"}, {"github.com/Google/UUID", "v1.6.0"}} { + lines, err := client.Lookup(m[0], m[1]) + if err != nil { + t.Fatalf("lookup %s@%s through the proxy: %v", m[0], m[1], err) + } + if want := m[0] + " " + m[1] + " h1:Y29udGVudA=="; len(lines) != 1 || lines[0] != want { + t.Errorf("lookup %s@%s = %q, want [%q]", m[0], m[1], lines, want) + } + } + + down.Store(true) + offline := sumdb.NewClient(newSumDBClientOps(t, remote, verifierKey)) + if _, err := offline.Lookup("rsc.io/quote", "v1.5.2"); err != nil { + t.Fatalf("lookup from the proxy cache with the database down: %v", err) + } +} diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 776fa8c..26e8c2a 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -1101,6 +1101,20 @@ func JSONError(w http.ResponseWriter, status int, message string) { // ErrUpstreamNotFound indicates the upstream returned 404. var ErrUpstreamNotFound = fmt.Errorf("upstream: %w", fetch.ErrNotFound) +// upstreamNotFoundBodyLimit caps the part of an upstream 404 body kept in +// upstreamNotFoundError. +const upstreamNotFoundBodyLimit = 4 << 10 + +// upstreamNotFoundError is ErrUpstreamNotFound with the start of the +// upstream's 404 body, for handlers that relay the reason to clients. +type upstreamNotFoundError struct { + body []byte +} + +func (e *upstreamNotFoundError) Error() string { return ErrUpstreamNotFound.Error() } + +func (e *upstreamNotFoundError) Unwrap() error { return ErrUpstreamNotFound } + // ErrArtifactBlocked indicates a pre-cache security scan blocked the artifact. var ErrArtifactBlocked = errors.New("artifact blocked by security scan") @@ -1461,7 +1475,8 @@ func (p *Proxy) fetchUpstreamMetadata(ctx context.Context, upstreamURL string, e } if resp.StatusCode == http.StatusNotFound { - return nil, ErrUpstreamNotFound + body, _ := io.ReadAll(io.LimitReader(resp.Body, upstreamNotFoundBodyLimit)) + return nil, &upstreamNotFoundError{body: body} } if resp.StatusCode != http.StatusOK { return nil, fmt.Errorf("upstream returned %d", resp.StatusCode) diff --git a/internal/server/go_sumdbs_test.go b/internal/server/go_sumdbs_test.go new file mode 100644 index 0000000..ff9dce2 --- /dev/null +++ b/internal/server/go_sumdbs_test.go @@ -0,0 +1,38 @@ +package server + +import ( + "io" + "log/slog" + "net/http" + "net/http/httptest" + "testing" + + "github.com/git-pkgs/proxy/internal/config" + "github.com/git-pkgs/proxy/internal/handler" + "github.com/go-chi/chi/v5" +) + +func TestGoSumDBsReachGoHandler(t *testing.T) { + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/lookup/golang.org/x/text@v0.14.0" { + http.NotFound(w, r) + return + } + _, _ = io.WriteString(w, "lookup record") + })) + defer upstream.Close() + + cfg := config.Default() + cfg.BaseURL = "https://proxy.example" + cfg.Upstream.GoSumDBs = map[string]string{"sum.example": upstream.URL} + p := &handler.Proxy{HTTPClient: upstream.Client(), Logger: slog.New(slog.NewTextHandler(io.Discard, nil))} + s := &Server{cfg: cfg} + r := chi.NewRouter() + s.mountProtocolHandlers(r, p) + + w := httptest.NewRecorder() + r.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/go/sumdb/sum.example/lookup/golang.org/x/text@v0.14.0", nil)) + if w.Code != http.StatusOK || w.Body.String() != "lookup record" { + t.Errorf("status = %d, body = %q; want the lookup from upstream.go_sumdbs", w.Code, w.Body.String()) + } +} diff --git a/internal/server/server.go b/internal/server/server.go index 8ee1c62..b6c93bc 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -402,6 +402,7 @@ func (s *Server) mountProtocolHandlers(r chi.Router, proxy *handler.Proxy) { ) gemHandler := handler.NewGemHandlerWithUpstream(proxy, s.cfg.BaseURL, s.cfg.Upstream.Gem) goHandler := handler.NewGoHandlerWithUpstream(proxy, s.cfg.BaseURL, s.cfg.Upstream.Go) + goHandler.SetSumDBs(s.cfg.Upstream.GoSumDBs) hexHandler := handler.NewHexHandlerWithUpstreams( proxy, s.cfg.BaseURL, From 9eb88e499ffe6214f7a43cc207df966d62496212 Mon Sep 17 00:00:00 2001 From: Christian Heim Date: Sat, 10 Oct 2026 09:57:43 +0200 Subject: [PATCH 246/633] Document upstream.go_sumdbs The README now mentions that checksum lookups go through GOPROXY as well, and configuration.md has a section on the new option. --- README.md | 7 +++++++ docs/configuration.md | 28 ++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+) diff --git a/README.md b/README.md index b85dcd3..17e6fde 100644 --- a/README.md +++ b/README.md @@ -202,6 +202,13 @@ export GOPROXY=http://localhost:8080/go,direct Or in your shell profile for persistence. +Checksum lookups go through the same URL: the proxy serves `sum.golang.org` at +`/go/sumdb/sum.golang.org/`, and the `go` command uses it without further +setup, so clients need no route of their own to `sum.golang.org`. The proxy +itself needs to reach `https://sum.golang.org`. Other checksum databases are +listed under `upstream.go_sumdbs`, see +[Go checksum database](docs/configuration.md#go-checksum-database). + ### Homebrew Point Homebrew's JSON API and artifact domain at the proxy: diff --git a/docs/configuration.md b/docs/configuration.md index 89e9141..0c3ceff 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -495,6 +495,34 @@ When `upstream.arch` is empty, a single repository named `archlinux` pointing at that default. Arch Linux ARM has no default: pick a regional HTTPS mirror, not the `mirror.archlinuxarm.org` redirector. +### Go checksum database + +```yaml +upstream: + + # Named Go checksum databases, served at /go/sumdb/{name}/. + go_sumdbs: + sum.golang.org: "https://sum.golang.org" +``` + +The `go` command asks `GOPROXY` for the checksum database before contacting it +directly. The proxy answers for every name listed here and fetches lookups and +tiles from the database's URL, not from `upstream.go`. Unknown names get 404, +so the `go` command falls back to contacting that database itself. When +`upstream.go_sumdbs` is empty, `sum.golang.org` is served; configuring any +entry replaces that default. + +With `cache_metadata` enabled, lookups, partial tiles and the latest signed +tree follow `metadata_ttl` and are served from cache when the database is +unreachable. Complete tiles never change and are not revalidated once cached. +Without `cache_metadata` the requests are forwarded and nothing is stored. A +404 from the database keeps its text (up to 4 KiB), which the `go` command +prints to explain why a version is missing. + +If `upstream.go` points at a module proxy that serves the checksum database +itself, such as Athens, point the entry there instead, for example +`sum.golang.org: "https://athens.example/sumdb/sum.golang.org"`. + ### Debian archives ```yaml From e6aadb297cfd378550d73eb87d81aaa70ecf3805 Mon Sep 17 00:00:00 2001 From: Christian Heim Date: Sat, 10 Oct 2026 10:18:40 +0200 Subject: [PATCH 247/633] Only claim checksum database support when the database is reachable Once a proxy answers 200 on supported, the go command sends all checksum requests there and never goes to the database directly. A proxy that cannot reach sum.golang.org would therefore break every build that needs a lookup. supported now fetches latest first, through the cache, and answers 404 if that fails and nothing is cached, so clients fall back like before. A failed check is remembered for a minute so an unreachable database doesn't make each go command wait for an upstream timeout. --- README.md | 5 ++- docs/configuration.md | 6 +++ internal/handler/go.go | 1 + internal/handler/go_sumdb.go | 64 +++++++++++++++++++++++++++- internal/handler/go_sumdb_test.go | 70 ++++++++++++++++++++++++------- 5 files changed, 128 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 17e6fde..30e09a0 100644 --- a/README.md +++ b/README.md @@ -204,8 +204,9 @@ Or in your shell profile for persistence. Checksum lookups go through the same URL: the proxy serves `sum.golang.org` at `/go/sumdb/sum.golang.org/`, and the `go` command uses it without further -setup, so clients need no route of their own to `sum.golang.org`. The proxy -itself needs to reach `https://sum.golang.org`. Other checksum databases are +setup, so clients need no route of their own to `sum.golang.org`. If the +proxy cannot reach `https://sum.golang.org`, the `go` command contacts it +directly as before. Other checksum databases are listed under `upstream.go_sumdbs`, see [Go checksum database](docs/configuration.md#go-checksum-database). diff --git a/docs/configuration.md b/docs/configuration.md index 0c3ceff..acee848 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -512,6 +512,12 @@ so the `go` command falls back to contacting that database itself. When `upstream.go_sumdbs` is empty, `sum.golang.org` is served; configuring any entry replaces that default. +Once the proxy says it serves a database, the `go` command no longer contacts +that database directly. The proxy therefore only says so when it can fetch the +database's latest signed tree, or has it cached. If it cannot, it answers 404 +as for an unknown name, and clients go to the database themselves. After a +failed check the proxy waits a minute before it tries the database again. + With `cache_metadata` enabled, lookups, partial tiles and the latest signed tree follow `metadata_ttl` and are served from cache when the database is unreachable. Complete tiles never change and are not revalidated once cached. diff --git a/internal/handler/go.go b/internal/handler/go.go index 6348a1e..e5c2f28 100644 --- a/internal/handler/go.go +++ b/internal/handler/go.go @@ -32,6 +32,7 @@ type GoHandler struct { upstreamURL string proxyURL string sumDBs map[string]string + sumDBProbes goSumDBProbes } // NewGoHandler creates a new Go module proxy handler. diff --git a/internal/handler/go_sumdb.go b/internal/handler/go_sumdb.go index 32665f6..ff3c711 100644 --- a/internal/handler/go_sumdb.go +++ b/internal/handler/go_sumdb.go @@ -7,6 +7,8 @@ import ( "net/http" "strconv" "strings" + "sync" + "time" "github.com/git-pkgs/proxy/internal/metrics" "golang.org/x/mod/module" @@ -20,8 +22,40 @@ const ( defaultGoSumDBName = "sum.golang.org" defaultGoSumDBURL = "https://sum.golang.org" goEcosystem = "golang" + + // goSumDBProbeBackoff is how long supported keeps answering 404 after the + // proxy failed to reach a checksum database, before it tries again. + goSumDBProbeBackoff = time.Minute ) +// goSumDBProbes remembers when the proxy last failed to reach each checksum +// database, so a database that is unreachable does not make every go command +// wait for an upstream timeout before it falls back. +type goSumDBProbes struct { + mu sync.Mutex + failed map[string]time.Time +} + +func (p *goSumDBProbes) recentlyFailed(name string) bool { + p.mu.Lock() + defer p.mu.Unlock() + failedAt, ok := p.failed[name] + return ok && time.Since(failedAt) < goSumDBProbeBackoff +} + +func (p *goSumDBProbes) record(name string, reachable bool) { + p.mu.Lock() + defer p.mu.Unlock() + if reachable { + delete(p.failed, name) + return + } + if p.failed == nil { + p.failed = make(map[string]time.Time) + } + p.failed[name] = time.Now() +} + // SetSumDBs sets the checksum databases served at /sumdb/{name}/, mapping // each database name to its base URL. An empty map keeps the default, // sum.golang.org. @@ -49,7 +83,11 @@ func (h *GoHandler) handleSumDB(w http.ResponseWriter, r *http.Request, rest str return } if path == "supported" { - w.WriteHeader(http.StatusOK) + if h.sumDBReachable(r, name, sumDBURL) { + w.WriteHeader(http.StatusOK) + } else { + http.NotFound(w, r) + } return } immutable, valid := parseGoSumDBPath(path) @@ -65,6 +103,30 @@ func (h *GoHandler) handleSumDB(w http.ResponseWriter, r *http.Request, rest str h.proxySumDB(w, r, sumDBURL+"/"+path, cacheKey) } +// sumDBReachable reports whether the proxy can serve a checksum database: it +// can fetch the latest signed tree, or has it cached. Once supported answers +// 200 the go command uses only the proxy for that database and never falls +// back to contacting it directly, so a database the proxy cannot reach gets +// 404 instead. The probe goes through the same cache entry as a latest +// request, so it costs at most one upstream request per metadata TTL. +func (h *GoHandler) sumDBReachable(r *http.Request, name, sumDBURL string) bool { + if h.sumDBProbes.recentlyFailed(name) { + return false + } + cacheKey := goSumDBCacheKey(name, sumDBURL, "latest") + _, _, _, err := h.proxy.fetchOrCacheMetadata(r.Context(), goEcosystem, cacheKey, sumDBURL+"/latest", "identity", nil, "*/*") + if err != nil && r.Context().Err() != nil { + // The client went away; that says nothing about the database. + return false + } + h.sumDBProbes.record(name, err == nil) + if err != nil { + h.proxy.Logger.Info("checksum database unreachable, sending go clients to it directly", + "name", name, "url", sumDBURL, "error", err) + } + return err == nil +} + // parseGoSumDBPath reports whether path is a request the checksum database // protocol defines and whether its response can never change. Only complete // tiles are immutable; partial tiles, latest and lookups follow the metadata diff --git a/internal/handler/go_sumdb_test.go b/internal/handler/go_sumdb_test.go index a9ff1df..e74c595 100644 --- a/internal/handler/go_sumdb_test.go +++ b/internal/handler/go_sumdb_test.go @@ -94,19 +94,22 @@ func serveGoRequest(h *GoHandler, target string) *httptest.ResponseRecorder { return w } +func TestGoSumDB_DefaultsToSumGolangOrg(t *testing.T) { + h := NewGoHandler(nil, "http://proxy.example") + if got := h.sumDBs[defaultGoSumDBName]; got != defaultGoSumDBURL { + t.Errorf("default checksum database = %q, want %q", got, defaultGoSumDBURL) + } + h.SetSumDBs(nil) + if got := h.sumDBs[defaultGoSumDBName]; got != defaultGoSumDBURL || len(h.sumDBs) != 1 { + t.Errorf("SetSumDBs(nil) = %v, want only the default", h.sumDBs) + } +} + func TestGoSumDB_SupportedProbe(t *testing.T) { - t.Run("default database", func(t *testing.T) { - proxy, _, _, _ := setupTestProxy(t) - h := NewGoHandler(proxy, "http://proxy.example") - if w := serveGoRequest(h, "/sumdb/sum.golang.org/supported"); w.Code != http.StatusOK { - t.Errorf("supported for default sum.golang.org: status = %d, want 200", w.Code) - } - }) - - t.Run("configured databases replace the default", func(t *testing.T) { - db := newFakeSumDB(t, nil) - h := newSumDBTestHandler(t, db.server.URL, true, time.Hour) + db := newFakeSumDB(t, map[string]string{"/latest": "signed tree"}) + h := newSumDBTestHandler(t, db.server.URL, true, time.Hour) + for range 2 { for target, want := range map[string]int{ "/sumdb/sum.example/supported": http.StatusOK, "/sumdb/sum.golang.org/supported": http.StatusNotFound, @@ -116,10 +119,47 @@ func TestGoSumDB_SupportedProbe(t *testing.T) { t.Errorf("%s: status = %d, want %d", target, w.Code, want) } } - if n := db.totalRequests(); n != 0 { - t.Errorf("upstream requests = %d, want 0: supported is answered by the proxy", n) - } - }) + } + if n := db.totalRequests(); n != 1 { + t.Errorf("upstream requests = %d, want 1: the probe fetches latest once, then uses the cache", n) + } +} + +// TestGoSumDB_SupportedFallsBackWhenUnreachable covers the go command's only +// way back to the database: once supported answers 200 it never contacts the +// database directly, so a database the proxy cannot reach must get 404. +func TestGoSumDB_SupportedFallsBackWhenUnreachable(t *testing.T) { + for _, cacheMetadata := range []bool{true, false} { + t.Run(fmt.Sprintf("cache_metadata=%v", cacheMetadata), func(t *testing.T) { + db := newFakeSumDB(t, map[string]string{"/latest": "signed tree"}) + db.down.Store(true) + h := newSumDBTestHandler(t, db.server.URL, cacheMetadata, time.Hour) + + for i := range 2 { + if w := serveGoRequest(h, "/sumdb/sum.example/supported"); w.Code != http.StatusNotFound { + t.Fatalf("request %d: status = %d, want 404", i, w.Code) + } + } + if n := db.requests("/latest"); n != 1 { + t.Errorf("probes = %d, want 1: a failed probe is not repeated within %s", n, goSumDBProbeBackoff) + } + }) + } +} + +// TestGoSumDB_SupportedWithCachedTreeWhileUnreachable keeps the offline case: +// a proxy that has served the database before can still serve it from cache. +func TestGoSumDB_SupportedWithCachedTreeWhileUnreachable(t *testing.T) { + db := newFakeSumDB(t, map[string]string{"/latest": "signed tree"}) + h := newSumDBTestHandler(t, db.server.URL, true, time.Nanosecond) + + if w := serveGoRequest(h, "/sumdb/sum.example/latest"); w.Code != http.StatusOK { + t.Fatalf("latest: status = %d, want 200", w.Code) + } + db.down.Store(true) + if w := serveGoRequest(h, "/sumdb/sum.example/supported"); w.Code != http.StatusOK { + t.Errorf("supported with a cached tree: status = %d, want 200", w.Code) + } } func TestGoSumDB_RequestsGoToTheDatabase(t *testing.T) { From 9a93ce05b4af6c35c7d3fdc27cc974387683746f Mon Sep 17 00:00:00 2001 From: Christian Heim Date: Sat, 10 Oct 2026 10:19:39 +0200 Subject: [PATCH 248/633] Check tile sizes before caching them Complete tiles are never fetched again once cached, so a bad response would stick around until someone deletes it by hand. That could be an error page from a proxy in between that comes back with 200. A hash tile is always W*32 bytes, so anything else is refused now and not stored, and the same check runs when a cached tile is served. Data tiles have no fixed size and only must not be empty. A cached tile that fails the check is fetched again without asking the database whether it changed. Past the TTL the cache would ask, and a database that sends an ETag would answer 304 and hand back the damaged copy every time. The fake database in the tests now guards its maps with the mutex, since the tests change them between requests. --- docs/configuration.md | 3 +- internal/handler/go_sumdb.go | 74 +++++++++++++++++++++++------ internal/handler/go_sumdb_test.go | 77 +++++++++++++++++++++++++------ 3 files changed, 126 insertions(+), 28 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index acee848..38f79a4 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -520,7 +520,8 @@ failed check the proxy waits a minute before it tries the database again. With `cache_metadata` enabled, lookups, partial tiles and the latest signed tree follow `metadata_ttl` and are served from cache when the database is -unreachable. Complete tiles never change and are not revalidated once cached. +unreachable. Complete tiles never change and are not revalidated once cached; +a tile of the wrong size is refused and not stored. Without `cache_metadata` the requests are forwarded and nothing is stored. A 404 from the database keeps its text (up to 4 KiB), which the `go` command prints to explain why a version is missing. diff --git a/internal/handler/go_sumdb.go b/internal/handler/go_sumdb.go index ff3c711..ac7f404 100644 --- a/internal/handler/go_sumdb.go +++ b/internal/handler/go_sumdb.go @@ -1,9 +1,11 @@ package handler import ( + "context" "crypto/sha256" "encoding/hex" "errors" + "fmt" "net/http" "strconv" "strings" @@ -97,10 +99,15 @@ func (h *GoHandler) handleSumDB(w http.ResponseWriter, r *http.Request, rest str } cacheKey := goSumDBCacheKey(name, sumDBURL, path) - if immutable && h.serveCachedSumDBTile(w, r, cacheKey) { + check := goSumDBTileCheck(path) + served, damaged := false, false + if immutable { + served, damaged = h.serveCachedSumDBTile(w, r, cacheKey, check) + } + if served { return } - h.proxySumDB(w, r, sumDBURL+"/"+path, cacheKey) + h.proxySumDB(w, r, sumDBURL+"/"+path, cacheKey, check, damaged) } // sumDBReachable reports whether the proxy can serve a checksum database: it @@ -145,6 +152,28 @@ func parseGoSumDBPath(path string) (immutable, valid bool) { return tile.W == 1<= 0 && len(body) != tile.W*tlog.HashSize { + return fmt.Errorf("checksum database tile %s has %d bytes, want %d", path, len(body), tile.W*tlog.HashSize) + } + if len(body) == 0 { + return fmt.Errorf("checksum database tile %s is empty", path) + } + return nil + } +} + // validGoSumDBLookup checks an escaped module@version the way the checksum // database does. Requiring a semantic version also keeps characters such as // '#' and '%' out of the upstream URL. @@ -169,20 +198,24 @@ func goSumDBCacheKey(name, sumDBURL, path string) string { } // serveCachedSumDBTile serves a complete tile from the metadata cache -// regardless of its age, since its content never changes. It reports false -// when the tile is not cached. -func (h *GoHandler) serveCachedSumDBTile(w http.ResponseWriter, r *http.Request, cacheKey string) bool { +// regardless of its age, since its content never changes. It reports +// served false when the tile is not cached, and damaged as well when the +// cached copy fails check. +func (h *GoHandler) serveCachedSumDBTile(w http.ResponseWriter, r *http.Request, cacheKey string, check func([]byte) error) (served, damaged bool) { p := h.proxy if !p.CacheMetadata || p.DB == nil { - return false + return false, false } entry, err := p.DB.GetMetadataCache(goEcosystem, cacheKey) if err != nil || entry == nil { - return false + return false, false } body, contentType, err := p.readCachedMetadata(r.Context(), entry, nil) if err != nil { - return false + return false, false + } + if check != nil && check(body) != nil { + return false, true } metrics.RecordCacheHit(goEcosystem) @@ -190,7 +223,7 @@ func (h *GoHandler) serveCachedSumDBTile(w http.ResponseWriter, r *http.Request, w.Header().Set(headerETag, entry.ETag.String) if ifNoneMatchHits(r.Header.Get("If-None-Match"), entry.ETag.String) { w.WriteHeader(http.StatusNotModified) - return true + return true, false } } w.Header().Set(headerContentType, contentType) @@ -200,20 +233,22 @@ func (h *GoHandler) serveCachedSumDBTile(w http.ResponseWriter, r *http.Request, } w.WriteHeader(http.StatusOK) _, _ = w.Write(body) - return true + return true, false } // proxySumDB fetches a checksum database response through the metadata // cache. Unlike ProxyCached it relays the start of an upstream 404 body: the -// go command prints it, and it says why a module version is missing. -func (h *GoHandler) proxySumDB(w http.ResponseWriter, r *http.Request, upstreamURL, cacheKey string) { +// go command prints it, and it says why a module version is missing. check, +// when set, must accept a response before it is cached or served. damaged +// says the cached copy failed check. +func (h *GoHandler) proxySumDB(w http.ResponseWriter, r *http.Request, upstreamURL, cacheKey string, check func([]byte) error, damaged bool) { p := h.proxy if !p.CacheMetadata { p.proxyMetadataStream(w, r, upstreamURL, "identity", "*/*") return } - body, contentType, contentEncoding, err := p.fetchOrCacheMetadata(r.Context(), goEcosystem, cacheKey, upstreamURL, "identity", nil, "*/*") + body, contentType, contentEncoding, err := h.fetchSumDB(r.Context(), upstreamURL, cacheKey, check, damaged) if err != nil { var notFound *upstreamNotFoundError switch { @@ -233,3 +268,16 @@ func (h *GoHandler) proxySumDB(w http.ResponseWriter, r *http.Request, upstreamU p.writeMetadataCachedResponseWithEncoding(w, r, goEcosystem, cacheKey, body, contentType, contentEncoding) } + +// fetchSumDB fetches a checksum database response through the metadata +// cache. A cached copy that failed check is fetched again without +// revalidating it: past the TTL, a database that sends an ETag would answer +// 304 and hand back the damaged copy for good. +func (h *GoHandler) fetchSumDB(ctx context.Context, upstreamURL, cacheKey string, check func([]byte) error, damaged bool) ([]byte, string, string, error) { + p := h.proxy + if damaged { + res := p.fetchMetadataFromUpstream(ctx, goEcosystem, cacheKey, upstreamURL, "*/*", "identity", check, nil) + return res.body, res.contentType, res.contentEncoding, res.err + } + return p.fetchOrCacheMetadata(ctx, goEcosystem, cacheKey, upstreamURL, "identity", check, "*/*") +} diff --git a/internal/handler/go_sumdb_test.go b/internal/handler/go_sumdb_test.go index e74c595..744a45f 100644 --- a/internal/handler/go_sumdb_test.go +++ b/internal/handler/go_sumdb_test.go @@ -16,36 +16,45 @@ import ( "golang.org/x/mod/sumdb" "golang.org/x/mod/sumdb/note" + "golang.org/x/mod/sumdb/tlog" ) const testSumDBName = "sum.example" +// hashTile returns a hash tile body of the given width. +func hashTile(width int, fill string) string { + return strings.Repeat(fill, width*tlog.HashSize) +} + // fakeSumDB is a checksum database upstream serving fixed files. Unknown // paths get 404 with the body from notFound, or a default text. type fakeSumDB struct { - server *httptest.Server + server *httptest.Server + + mu sync.Mutex files map[string]string notFound map[string]string - - mu sync.Mutex - hits map[string]int - down atomic.Bool + hits map[string]int + down atomic.Bool } func newFakeSumDB(t *testing.T, files map[string]string) *fakeSumDB { t.Helper() + if files == nil { + files = map[string]string{} + } db := &fakeSumDB{files: files, notFound: map[string]string{}, hits: map[string]int{}} db.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { db.mu.Lock() db.hits[r.URL.Path]++ + body, ok := db.files[r.URL.Path] + text, custom := db.notFound[r.URL.Path] db.mu.Unlock() if db.down.Load() { http.Error(w, "unavailable", http.StatusInternalServerError) return } - body, ok := db.files[r.URL.Path] if !ok { - text, custom := db.notFound[r.URL.Path] if !custom { text = "not found: " + r.URL.Path } @@ -61,6 +70,18 @@ func newFakeSumDB(t *testing.T, files map[string]string) *fakeSumDB { return db } +func (db *fakeSumDB) setFile(path, body string) { + db.mu.Lock() + defer db.mu.Unlock() + db.files[path] = body +} + +func (db *fakeSumDB) setNotFound(path, text string) { + db.mu.Lock() + defer db.mu.Unlock() + db.notFound[path] = text +} + func (db *fakeSumDB) requests(path string) int { db.mu.Lock() defer db.mu.Unlock() @@ -208,9 +229,9 @@ func TestGoSumDB_LookupCachedWithinTTL(t *testing.T) { func TestGoSumDB_CompleteTilesSkipRevalidation(t *testing.T) { files := map[string]string{ - "/tile/8/0/000": "complete hash tile", + "/tile/8/0/000": hashTile(256, "a"), "/tile/8/data/000": "complete data tile", - "/tile/8/0/001.p/5": "partial hash tile", + "/tile/8/0/001.p/5": hashTile(5, "b"), "/tile/8/data/001.p/5": "partial data tile", "/latest": "signed tree", } @@ -240,9 +261,37 @@ func TestGoSumDB_CompleteTilesSkipRevalidation(t *testing.T) { } } +// TestGoSumDB_BadTilesAreNotCached guards complete tiles, which are never +// revalidated once cached: a response of the wrong size must not be stored. +func TestGoSumDB_BadTilesAreNotCached(t *testing.T) { + const tile = "/tile/8/0/000" + db := newFakeSumDB(t, map[string]string{ + tile: "please log in", + "/tile/8/data/000.p/3": "", + }) + h := newSumDBTestHandler(t, db.server.URL, true, time.Hour) + + if w := serveGoRequest(h, "/sumdb/sum.example"+tile); w.Code != http.StatusBadGateway { + t.Fatalf("wrong-size tile: status = %d, want 502", w.Code) + } + if w := serveGoRequest(h, "/sumdb/sum.example/tile/8/data/000.p/3"); w.Code != http.StatusBadGateway { + t.Errorf("empty data tile: status = %d, want 502", w.Code) + } + + want := hashTile(256, "a") + db.setFile(tile, want) + w := serveGoRequest(h, "/sumdb/sum.example"+tile) + if w.Code != http.StatusOK || w.Body.String() != want { + t.Fatalf("after upstream recovered: status = %d, body length %d, want the %d-byte tile", w.Code, w.Body.Len(), len(want)) + } + if n := db.requests(tile); n != 2 { + t.Errorf("upstream requests = %d, want 2: the bad response was not cached", n) + } +} + func TestGoSumDB_CompleteTilesNotStoredWithoutMetadataCache(t *testing.T) { const tile = "/tile/8/0/000" - db := newFakeSumDB(t, map[string]string{tile: "complete hash tile"}) + db := newFakeSumDB(t, map[string]string{tile: hashTile(256, "a")}) h := newSumDBTestHandler(t, db.server.URL, false, time.Hour) for range 2 { @@ -259,8 +308,8 @@ func TestGoSumDB_OfflineFallback(t *testing.T) { files := map[string]string{ "/lookup/golang.org/x/text@v0.14.0": "lookup record", "/latest": "signed tree", - "/tile/8/0/000": "complete hash tile", - "/tile/8/0/001.p/5": "partial hash tile", + "/tile/8/0/000": hashTile(256, "a"), + "/tile/8/0/001.p/5": hashTile(5, "b"), } db := newFakeSumDB(t, files) h := newSumDBTestHandler(t, db.server.URL, true, time.Nanosecond) @@ -287,7 +336,7 @@ func TestGoSumDB_NotFoundRelaysUpstreamText(t *testing.T) { for _, cacheMetadata := range []bool{true, false} { t.Run(fmt.Sprintf("cache_metadata=%v", cacheMetadata), func(t *testing.T) { db := newFakeSumDB(t, nil) - db.notFound[lookup] = reason + db.setNotFound(lookup, reason) h := newSumDBTestHandler(t, db.server.URL, cacheMetadata, time.Hour) w := serveGoRequest(h, "/sumdb/sum.example"+lookup) @@ -302,7 +351,7 @@ func TestGoSumDB_NotFoundRelaysUpstreamText(t *testing.T) { t.Run("long text is cut", func(t *testing.T) { db := newFakeSumDB(t, nil) - db.notFound[lookup] = strings.Repeat("x", upstreamNotFoundBodyLimit+1000) + db.setNotFound(lookup, strings.Repeat("x", upstreamNotFoundBodyLimit+1000)) h := newSumDBTestHandler(t, db.server.URL, true, time.Hour) w := serveGoRequest(h, "/sumdb/sum.example"+lookup) From f47dd2f91c5d198b2db8c470448e08efdc9c4b4b Mon Sep 17 00:00:00 2001 From: Christian Heim Date: Sat, 10 Oct 2026 10:19:46 +0200 Subject: [PATCH 249/633] Say that the 4 KiB limit on 404 text only applies with the metadata cache --- docs/configuration.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 38f79a4..02d39b8 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -522,9 +522,11 @@ With `cache_metadata` enabled, lookups, partial tiles and the latest signed tree follow `metadata_ttl` and are served from cache when the database is unreachable. Complete tiles never change and are not revalidated once cached; a tile of the wrong size is refused and not stored. -Without `cache_metadata` the requests are forwarded and nothing is stored. A -404 from the database keeps its text (up to 4 KiB), which the `go` command -prints to explain why a version is missing. +Without `cache_metadata` the requests are forwarded and nothing is stored. + +A 404 from the database keeps its text, which the `go` command prints to +explain why a version is missing. With `cache_metadata` enabled the text is +cut at 4 KiB; without it the response is passed on unchanged. If `upstream.go` points at a module proxy that serves the checksum database itself, such as Athens, point the entry there instead, for example From c1a597d2d874c16a32fdb805ccb1106e47251d4f Mon Sep 17 00:00:00 2001 From: Christian Heim Date: Sat, 10 Oct 2026 11:31:01 +0200 Subject: [PATCH 250/633] Don't let the supported check hang, and don't take a block page for an answer Two problems with the check behind supported. The fetch it waits on is shared with other callers and only bounded by the upstream timeout, which defaults to 30 seconds. A firewall that silently drops packets to sum.golang.org would stall the first go command every minute for that long, and the go command itself has no timeout on this request. Now we wait five seconds at most; the fetch carries on in the background, records how it went and fills the cache for the next check. A timeout only counts as a failure if the fetch hasn't succeeded in the meantime. The fetch reports no error when it falls back to the cached tree after the database failed, so the metadata result now says when it did, and the check treats that as the database not answering. A fresh cached tree (within metadata_ttl) still lets a timed-out check say yes, since the fetch only falls back once upstream has failed, which a silent database delays past our timeout. An older tree does not count: once the proxy has lost the database, clients go there directly, as they would without the proxy. And the check believed any 200. Egress proxies tend to answer blocked hosts with a 200 HTML page, which we would have cached as the signed tree and taken as proof that the database is reachable. latest now has to start with "go.sum database tree", or it is refused like a bad tile. Lookups had the same gap. A block page cached as a lookup would also replace the good copy that is served during an outage, so a lookup now has to be a record followed by a signed tree, the way the go command reads it. --- docs/configuration.md | 8 +- internal/handler/go_sumdb.go | 181 +++++++++++++++++++++++++++++------ internal/handler/handler.go | 5 +- 3 files changed, 159 insertions(+), 35 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 02d39b8..be7df9e 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -514,9 +514,11 @@ entry replaces that default. Once the proxy says it serves a database, the `go` command no longer contacts that database directly. The proxy therefore only says so when it can fetch the -database's latest signed tree, or has it cached. If it cannot, it answers 404 -as for an unknown name, and clients go to the database themselves. After a -failed check the proxy waits a minute before it tries the database again. +database's latest signed tree, or has a fresh one cached (fetched within +`metadata_ttl`). If it cannot, it answers 404 as for an unknown name, and +clients go to the database themselves, as they would without the proxy. The +check waits five seconds at most, and after a failed check the proxy waits a +minute before it tries the database again. With `cache_metadata` enabled, lookups, partial tiles and the latest signed tree follow `metadata_ttl` and are served from cache when the database is diff --git a/internal/handler/go_sumdb.go b/internal/handler/go_sumdb.go index ac7f404..0e0bcbf 100644 --- a/internal/handler/go_sumdb.go +++ b/internal/handler/go_sumdb.go @@ -12,6 +12,7 @@ import ( "sync" "time" + "github.com/git-pkgs/proxy/internal/database" "github.com/git-pkgs/proxy/internal/metrics" "golang.org/x/mod/module" "golang.org/x/mod/semver" @@ -28,14 +29,29 @@ const ( // goSumDBProbeBackoff is how long supported keeps answering 404 after the // proxy failed to reach a checksum database, before it tries again. goSumDBProbeBackoff = time.Minute + // goSumDBProbeTimeout is how long supported waits for the database. The + // go command's own request has no timeout, so a database that drops + // packets must not hold every build for the full upstream timeout. + goSumDBProbeTimeout = 5 * time.Second ) // goSumDBProbes remembers when the proxy last failed to reach each checksum // database, so a database that is unreachable does not make every go command // wait for an upstream timeout before it falls back. type goSumDBProbes struct { - mu sync.Mutex - failed map[string]time.Time + // timeout overrides goSumDBProbeTimeout when set (tests). + timeout time.Duration + + mu sync.Mutex + failed map[string]time.Time + succeeded map[string]time.Time +} + +func (p *goSumDBProbes) probeTimeout() time.Duration { + if p.timeout > 0 { + return p.timeout + } + return goSumDBProbeTimeout } func (p *goSumDBProbes) recentlyFailed(name string) bool { @@ -50,6 +66,24 @@ func (p *goSumDBProbes) record(name string, reachable bool) { defer p.mu.Unlock() if reachable { delete(p.failed, name) + if p.succeeded == nil { + p.succeeded = make(map[string]time.Time) + } + p.succeeded[name] = time.Now() + return + } + if p.failed == nil { + p.failed = make(map[string]time.Time) + } + p.failed[name] = time.Now() +} + +// recordTimeout records a failure for a probe started at start, unless the +// fetch it gave up waiting for has succeeded in the meantime. +func (p *goSumDBProbes) recordTimeout(name string, start time.Time) { + p.mu.Lock() + defer p.mu.Unlock() + if !p.succeeded[name].Before(start) { return } if p.failed == nil { @@ -99,7 +133,7 @@ func (h *GoHandler) handleSumDB(w http.ResponseWriter, r *http.Request, rest str } cacheKey := goSumDBCacheKey(name, sumDBURL, path) - check := goSumDBTileCheck(path) + check := goSumDBResponseCheck(path) served, damaged := false, false if immutable { served, damaged = h.serveCachedSumDBTile(w, r, cacheKey, check) @@ -115,23 +149,91 @@ func (h *GoHandler) handleSumDB(w http.ResponseWriter, r *http.Request, rest str // 200 the go command uses only the proxy for that database and never falls // back to contacting it directly, so a database the proxy cannot reach gets // 404 instead. The probe goes through the same cache entry as a latest -// request, so it costs at most one upstream request per metadata TTL. +// request: with cache_metadata it costs at most one upstream request per +// metadata TTL, without it each probe fetches latest. func (h *GoHandler) sumDBReachable(r *http.Request, name, sumDBURL string) bool { - if h.sumDBProbes.recentlyFailed(name) { - return false - } cacheKey := goSumDBCacheKey(name, sumDBURL, "latest") - _, _, _, err := h.proxy.fetchOrCacheMetadata(r.Context(), goEcosystem, cacheKey, sumDBURL+"/latest", "identity", nil, "*/*") - if err != nil && r.Context().Err() != nil { + treeCheck := goSumDBResponseCheck("latest") + if h.sumDBProbes.recentlyFailed(name) { + return h.cachedTreeServes(r.Context(), cacheKey, treeCheck) + } + // The fetch is shared with concurrent callers and runs detached from this + // request, so it is bounded by the upstream timeout, not by ours. Wait + // for it only up to the probe timeout; it finishes in the background, + // records its own outcome and fills the cache for the next probe. + start := time.Now() + done := make(chan error, 1) + go func() { + done <- h.probeSumDB(context.WithoutCancel(r.Context()), name, cacheKey, sumDBURL, treeCheck) + }() + var err error + select { + case err = <-done: + case <-time.After(h.sumDBProbes.probeTimeout()): + err = fmt.Errorf("no answer within %s", h.sumDBProbes.probeTimeout()) + h.sumDBProbes.recordTimeout(name, start) + case <-r.Context().Done(): // The client went away; that says nothing about the database. return false } - h.sumDBProbes.record(name, err == nil) - if err != nil { - h.proxy.Logger.Info("checksum database unreachable, sending go clients to it directly", - "name", name, "url", sumDBURL, "error", err) + if err == nil { + return true } - return err == nil + // The stale-cache fallback inside the fetch only runs once upstream has + // failed, which a silent database delays past the probe timeout. A tree + // cached earlier may still serve lookups, so look for it here. + if h.cachedTreeServes(r.Context(), cacheKey, treeCheck) { + return true + } + h.proxy.Logger.Info("checksum database unreachable, sending go clients to it directly", + "name", name, "url", sumDBURL, "error", err) + return false +} + +// probeSumDB fetches the database's latest tree and records whether the +// database answered. A tree served from the cache within its TTL counts as +// an answer; one served because upstream failed does not. +func (h *GoHandler) probeSumDB(ctx context.Context, name, cacheKey, sumDBURL string, check func([]byte) error) error { + res := h.proxy.lookupMetadata(ctx, goEcosystem, cacheKey, sumDBURL+"/latest", "*/*", "identity", check) + err := res.err + if err == nil && res.stale { + err = errors.New("checksum database did not answer, tree served from cache") + } + h.sumDBProbes.record(name, err == nil) + return err +} + +// cachedTreeServes reports whether a cached tree lets the proxy answer for +// the database although the database did not: only while the tree is fresh, +// so clients fall back to the database once the proxy has lost it. +func (h *GoHandler) cachedTreeServes(ctx context.Context, cacheKey string, check func([]byte) error) bool { + entry, _, _, ok := h.readCachedSumDB(ctx, cacheKey, check) + return ok && h.treeFresh(entry) +} + +// treeFresh reports whether a cached tree is within the metadata TTL. +func (h *GoHandler) treeFresh(entry *database.MetadataCacheEntry) bool { + ttl := h.proxy.MetadataTTL + return ttl > 0 && entry.FetchedAt.Valid && time.Since(entry.FetchedAt.Time) < ttl +} + +// readCachedSumDB reads a checksum database response from the metadata +// cache regardless of its age. It reports false when nothing usable is +// cached. +func (h *GoHandler) readCachedSumDB(ctx context.Context, cacheKey string, check func([]byte) error) (entry *database.MetadataCacheEntry, body []byte, contentType string, ok bool) { + p := h.proxy + if !p.CacheMetadata || p.DB == nil { + return nil, nil, "", false + } + entry, err := p.DB.GetMetadataCache(goEcosystem, cacheKey) + if err != nil || entry == nil { + return nil, nil, "", false + } + body, contentType, err = p.readCachedMetadata(ctx, entry, check) + if err != nil { + return nil, nil, "", false + } + return entry, body, contentType, true } // parseGoSumDBPath reports whether path is a request the checksum database @@ -152,13 +254,38 @@ func parseGoSumDBPath(path string) (immutable, valid bool) { return tile.W == 1< Date: Sat, 10 Oct 2026 11:31:01 +0200 Subject: [PATCH 251/633] Test the probe timeout, the block pages and a damaged cached tile Also the cases around a silent or slow database: a fresh cached tree still gets a yes on supported, a stale one does not, a late answer from the database clears the remembered failure, a fallback to the cached tree is not mistaken for an answer, and an answer still counts when the cache write fails afterwards. The damaged tile test guards the size check on the cached copy of a complete tile: it shortens the stored blob and expects the tile to be fetched again, also past the TTL from a database that sends an ETag. A block page in place of a lookup must not replace the cached record, not even one that starts like a record. The lookup fixtures now look like real records, since lookups are checked too. --- internal/handler/go_sumdb_test.go | 309 ++++++++++++++++++++++++++++-- 1 file changed, 296 insertions(+), 13 deletions(-) diff --git a/internal/handler/go_sumdb_test.go b/internal/handler/go_sumdb_test.go index 744a45f..a849ad8 100644 --- a/internal/handler/go_sumdb_test.go +++ b/internal/handler/go_sumdb_test.go @@ -19,7 +19,13 @@ import ( "golang.org/x/mod/sumdb/tlog" ) -const testSumDBName = "sum.example" +const ( + testSumDBName = "sum.example" + // testSumDBTree looks like a signed tree as far as the proxy checks. + testSumDBTree = "go.sum database tree\n1\nhash\n\n— sum.example signature\n" + // testSumDBRecord looks like a lookup answer as far as the proxy checks. + testSumDBRecord = "1\ngolang.org/x/text v0.14.0 h1:hash=\ngolang.org/x/text v0.14.0/go.mod h1:hash=\n\n" + testSumDBTree +) // hashTile returns a hash tile body of the given width. func hashTile(width int, fill string) string { @@ -127,7 +133,7 @@ func TestGoSumDB_DefaultsToSumGolangOrg(t *testing.T) { } func TestGoSumDB_SupportedProbe(t *testing.T) { - db := newFakeSumDB(t, map[string]string{"/latest": "signed tree"}) + db := newFakeSumDB(t, map[string]string{"/latest": testSumDBTree}) h := newSumDBTestHandler(t, db.server.URL, true, time.Hour) for range 2 { @@ -144,6 +150,9 @@ func TestGoSumDB_SupportedProbe(t *testing.T) { if n := db.totalRequests(); n != 1 { t.Errorf("upstream requests = %d, want 1: the probe fetches latest once, then uses the cache", n) } + if h.sumDBProbes.recentlyFailed(testSumDBName) { + t.Error("a probe served from the fresh cache was recorded as a failure") + } } // TestGoSumDB_SupportedFallsBackWhenUnreachable covers the go command's only @@ -152,7 +161,7 @@ func TestGoSumDB_SupportedProbe(t *testing.T) { func TestGoSumDB_SupportedFallsBackWhenUnreachable(t *testing.T) { for _, cacheMetadata := range []bool{true, false} { t.Run(fmt.Sprintf("cache_metadata=%v", cacheMetadata), func(t *testing.T) { - db := newFakeSumDB(t, map[string]string{"/latest": "signed tree"}) + db := newFakeSumDB(t, map[string]string{"/latest": testSumDBTree}) db.down.Store(true) h := newSumDBTestHandler(t, db.server.URL, cacheMetadata, time.Hour) @@ -171,15 +180,242 @@ func TestGoSumDB_SupportedFallsBackWhenUnreachable(t *testing.T) { // TestGoSumDB_SupportedWithCachedTreeWhileUnreachable keeps the offline case: // a proxy that has served the database before can still serve it from cache. func TestGoSumDB_SupportedWithCachedTreeWhileUnreachable(t *testing.T) { - db := newFakeSumDB(t, map[string]string{"/latest": "signed tree"}) + t.Run("fresh tree", func(t *testing.T) { + db := newFakeSumDB(t, map[string]string{"/latest": testSumDBTree}) + h := newSumDBTestHandler(t, db.server.URL, true, time.Hour) + if w := serveGoRequest(h, "/sumdb/sum.example/latest"); w.Code != http.StatusOK { + t.Fatalf("latest: status = %d, want 200", w.Code) + } + db.down.Store(true) + if w := serveGoRequest(h, "/sumdb/sum.example/supported"); w.Code != http.StatusOK { + t.Errorf("supported with a fresh cached tree: status = %d, want 200", w.Code) + } + }) + + // A stale tree does not count: clients that can reach the database + // themselves should go there. + t.Run("stale tree", func(t *testing.T) { + db := newFakeSumDB(t, map[string]string{"/latest": testSumDBTree}) + h := newSumDBTestHandler(t, db.server.URL, true, time.Nanosecond) + if w := serveGoRequest(h, "/sumdb/sum.example/latest"); w.Code != http.StatusOK { + t.Fatalf("latest: status = %d, want 200", w.Code) + } + db.down.Store(true) + for i := range 2 { // second probe hits the remembered failure + if w := serveGoRequest(h, "/sumdb/sum.example/supported"); w.Code != http.StatusNotFound { + t.Errorf("probe %d: status = %d, want 404", i, w.Code) + } + } + if n := db.requests("/latest"); n != 2 { + t.Errorf("upstream requests = %d, want 2: the failure is remembered", n) + } + }) +} + +// TestGoSumDB_SupportedGivesUpOnSilentDatabase: a database that accepts the +// connection but never answers (a firewall dropping packets) must not hold +// the go command, which waits for supported without a timeout of its own. +func TestGoSumDB_SupportedGivesUpOnSilentDatabase(t *testing.T) { + release := make(chan struct{}) + db := newFakeSumDB(t, nil) + t.Cleanup(func() { close(release) }) // runs before the server closes + db.server.Config.Handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + <-release + }) + h := newSumDBTestHandler(t, db.server.URL, true, time.Hour) + h.sumDBProbes.timeout = 50 * time.Millisecond + + answered := make(chan *httptest.ResponseRecorder, 1) + go func() { answered <- serveGoRequest(h, "/sumdb/sum.example/supported") }() + select { + case w := <-answered: + if w.Code != http.StatusNotFound { + t.Fatalf("status = %d, want 404", w.Code) + } + case <-time.After(time.Second): + t.Fatal("supported still waiting for the database after a second") + } +} + +// TestGoSumDB_SupportedWithStaleTreeWhileSilent: a silent database holds +// the fetch past the probe timeout. The stale tree does not count, and the +// second probe must answer at once from the remembered failure. +func TestGoSumDB_SupportedWithStaleTreeWhileSilent(t *testing.T) { + release := make(chan struct{}) + db := newFakeSumDB(t, map[string]string{"/latest": testSumDBTree}) + t.Cleanup(func() { close(release) }) h := newSumDBTestHandler(t, db.server.URL, true, time.Nanosecond) + h.sumDBProbes.timeout = 50 * time.Millisecond if w := serveGoRequest(h, "/sumdb/sum.example/latest"); w.Code != http.StatusOK { t.Fatalf("latest: status = %d, want 200", w.Code) } - db.down.Store(true) + db.server.Config.Handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + <-release + }) + + for i := range 2 { // second probe hits the remembered failure + answered := make(chan *httptest.ResponseRecorder, 1) + go func() { answered <- serveGoRequest(h, "/sumdb/sum.example/supported") }() + select { + case w := <-answered: + if w.Code != http.StatusNotFound { + t.Fatalf("probe %d: status = %d, want 404", i, w.Code) + } + case <-time.After(time.Second): + t.Fatalf("probe %d: supported still waiting after a second", i) + } + } +} + +// TestGoSumDB_SlowProbeSuccessClearsFailure: a database that answers after +// the probe timeout gets a 404 the first time, but once its answer is in +// the cache, supported says yes without waiting for the next backoff. +func TestGoSumDB_SlowProbeSuccessClearsFailure(t *testing.T) { + db := newFakeSumDB(t, map[string]string{"/latest": testSumDBTree}) + inner := db.server.Config.Handler + db.server.Config.Handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + time.Sleep(100 * time.Millisecond) + inner.ServeHTTP(w, r) + }) + h := newSumDBTestHandler(t, db.server.URL, true, time.Hour) + h.sumDBProbes.timeout = 10 * time.Millisecond + + if w := serveGoRequest(h, "/sumdb/sum.example/supported"); w.Code != http.StatusNotFound { + t.Fatalf("first probe: status = %d, want 404", w.Code) + } + deadline := time.Now().Add(5 * time.Second) + for h.sumDBProbes.recentlyFailed(testSumDBName) { + if time.Now().After(deadline) { + t.Fatal("the late fetch never cleared the failure") + } + time.Sleep(10 * time.Millisecond) + } if w := serveGoRequest(h, "/sumdb/sum.example/supported"); w.Code != http.StatusOK { - t.Errorf("supported with a cached tree: status = %d, want 200", w.Code) + t.Errorf("after the late answer: status = %d, want 200", w.Code) + } + if n := db.requests("/latest"); n != 1 { + t.Errorf("upstream requests = %d, want 1: the second probe is a cache hit", n) + } +} + +// TestGoSumDB_StaleFallbackIsNotAnAnswer: the fetch reports no error when +// it falls back to the cached tree after the database failed. That must +// not count as the database answering, or the failure would be forgotten +// and every probe would wait for the timeout again. +func TestGoSumDB_StaleFallbackIsNotAnAnswer(t *testing.T) { + db := newFakeSumDB(t, map[string]string{"/latest": testSumDBTree}) + h := newSumDBTestHandler(t, db.server.URL, true, time.Nanosecond) + h.sumDBProbes.timeout = 10 * time.Millisecond + + if w := serveGoRequest(h, "/sumdb/sum.example/latest"); w.Code != http.StatusOK { + t.Fatalf("latest: status = %d, want 200", w.Code) + } + inner := db.server.Config.Handler + db.server.Config.Handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + time.Sleep(100 * time.Millisecond) + inner.ServeHTTP(w, r) + }) + db.down.Store(true) + + if w := serveGoRequest(h, "/sumdb/sum.example/supported"); w.Code != http.StatusNotFound { + t.Fatalf("status = %d, want 404", w.Code) + } + time.Sleep(300 * time.Millisecond) // the fetch has failed and fallen back by now + if !h.sumDBProbes.recentlyFailed(testSumDBName) { + t.Error("the stale fallback cleared the remembered failure") + } +} + +// TestGoSumDB_AnswerCountsWhenCacheWriteFails: an answer from the database +// is an answer even when it could not be cached afterwards, whether or not +// a stale tree was cached before. +func TestGoSumDB_AnswerCountsWhenCacheWriteFails(t *testing.T) { + for _, staleTree := range []bool{false, true} { + t.Run(fmt.Sprintf("stale tree cached=%v", staleTree), func(t *testing.T) { + db := newFakeSumDB(t, map[string]string{"/latest": testSumDBTree}) + h := newSumDBTestHandler(t, db.server.URL, true, time.Nanosecond) + if staleTree { + if w := serveGoRequest(h, "/sumdb/sum.example/latest"); w.Code != http.StatusOK { + t.Fatalf("latest: status = %d, want 200", w.Code) + } + } + h.proxy.Storage.(*mockStorage).storeErr = errors.New("disk full") + + if w := serveGoRequest(h, "/sumdb/sum.example/supported"); w.Code != http.StatusOK { + t.Errorf("status = %d, want 200: the database answered", w.Code) + } + if h.sumDBProbes.recentlyFailed(testSumDBName) { + t.Error("an answered probe was recorded as a failure") + } + }) + } +} + +func TestGoSumDBProbes_TimeoutDoesNotOverwriteLaterSuccess(t *testing.T) { + var probes goSumDBProbes + start := time.Now() + probes.record(testSumDBName, true) + probes.recordTimeout(testSumDBName, start) + if probes.recentlyFailed(testSumDBName) { + t.Error("a success recorded after the probe started was overwritten by its timeout") + } + probes.recordTimeout(testSumDBName, time.Now().Add(time.Second)) + if !probes.recentlyFailed(testSumDBName) { + t.Error("a timeout with no later success was not recorded") + } +} + +// TestGoSumDB_SupportedRefusesBlockPage: egress proxies answer blocked hosts +// with a 200 page. That is not a reachable database, and must not be cached +// as its signed tree either. +func TestGoSumDB_SupportedRefusesBlockPage(t *testing.T) { + db := newFakeSumDB(t, map[string]string{"/latest": "access to this site is blocked"}) + h := newSumDBTestHandler(t, db.server.URL, true, time.Hour) + + if w := serveGoRequest(h, "/sumdb/sum.example/supported"); w.Code != http.StatusNotFound { + t.Errorf("supported: status = %d, want 404", w.Code) + } + if w := serveGoRequest(h, "/sumdb/sum.example/latest"); w.Code != http.StatusBadGateway { + t.Errorf("latest: status = %d, want 502 for a block page", w.Code) + } + + // The page was not cached: once the database answers, its tree is served. + db.setFile("/latest", testSumDBTree) + w := serveGoRequest(h, "/sumdb/sum.example/latest") + if w.Code != http.StatusOK || w.Body.String() != testSumDBTree { + t.Errorf("latest after recovery: status = %d, body = %q, want the signed tree", w.Code, w.Body.String()) + } +} + +// TestGoSumDB_LookupRefusesBlockPage: a block page in place of a lookup must +// not be cached, where it would replace the copy served during an outage. +// That includes a page that happens to start like a record. +func TestGoSumDB_LookupRefusesBlockPage(t *testing.T) { + const lookup = "/lookup/golang.org/x/text@v0.14.0" + pages := map[string]string{ + "html": "access to this site is blocked", + "record without tree": "1\ngolang.org/x/text v0.14.0 h1:hash=\n\naccess to this site is blocked", + } + for name, page := range pages { + t.Run(name, func(t *testing.T) { + db := newFakeSumDB(t, map[string]string{lookup: testSumDBRecord}) + h := newSumDBTestHandler(t, db.server.URL, true, time.Nanosecond) + + if w := serveGoRequest(h, "/sumdb/sum.example"+lookup); w.Code != http.StatusOK { + t.Fatalf("lookup: status = %d, want 200", w.Code) + } + db.setFile(lookup, page) + if w := serveGoRequest(h, "/sumdb/sum.example"+lookup); w.Code != http.StatusOK || w.Body.String() != testSumDBRecord { + t.Errorf("behind a block page: status = %d, body = %q, want the cached record", w.Code, w.Body.String()) + } + + // The page did not replace the record: it is still there in an outage. + db.down.Store(true) + if w := serveGoRequest(h, "/sumdb/sum.example"+lookup); w.Code != http.StatusOK || w.Body.String() != testSumDBRecord { + t.Errorf("in an outage: status = %d, body = %q, want the cached record", w.Code, w.Body.String()) + } + }) } } @@ -192,7 +428,7 @@ func TestGoSumDB_RequestsGoToTheDatabase(t *testing.T) { defer moduleUpstream.Close() const lookup = "/lookup/golang.org/x/text@v0.14.0" - db := newFakeSumDB(t, map[string]string{lookup: "lookup record"}) + db := newFakeSumDB(t, map[string]string{lookup: testSumDBRecord}) proxy, _, _, _ := setupTestProxy(t) proxy.HTTPClient = http.DefaultClient @@ -200,7 +436,7 @@ func TestGoSumDB_RequestsGoToTheDatabase(t *testing.T) { h.SetSumDBs(map[string]string{testSumDBName: db.server.URL + "/"}) w := serveGoRequest(h, "/sumdb/sum.example"+lookup) - if w.Code != http.StatusOK || w.Body.String() != "lookup record" { + if w.Code != http.StatusOK || w.Body.String() != testSumDBRecord { t.Fatalf("status = %d, body = %q; want 200 with the database's response", w.Code, w.Body.String()) } if n := db.requests(lookup); n != 1 { @@ -213,12 +449,12 @@ func TestGoSumDB_RequestsGoToTheDatabase(t *testing.T) { func TestGoSumDB_LookupCachedWithinTTL(t *testing.T) { const lookup = "/lookup/golang.org/x/text@v0.14.0" - db := newFakeSumDB(t, map[string]string{lookup: "lookup record"}) + db := newFakeSumDB(t, map[string]string{lookup: testSumDBRecord}) h := newSumDBTestHandler(t, db.server.URL, true, time.Hour) for i := range 2 { w := serveGoRequest(h, "/sumdb/sum.example"+lookup) - if w.Code != http.StatusOK || w.Body.String() != "lookup record" { + if w.Code != http.StatusOK || w.Body.String() != testSumDBRecord { t.Fatalf("request %d: status = %d, body = %q", i, w.Code, w.Body.String()) } } @@ -233,7 +469,7 @@ func TestGoSumDB_CompleteTilesSkipRevalidation(t *testing.T) { "/tile/8/data/000": "complete data tile", "/tile/8/0/001.p/5": hashTile(5, "b"), "/tile/8/data/001.p/5": "partial data tile", - "/latest": "signed tree", + "/latest": testSumDBTree, } wantRequests := map[string]int{ "/tile/8/0/000": 1, @@ -289,6 +525,53 @@ func TestGoSumDB_BadTilesAreNotCached(t *testing.T) { } } +// TestGoSumDB_DamagedCachedTileIsFetchedAgain guards the check on the cached +// copy: a complete tile is never revalidated, so a damaged blob would +// otherwise be served for good. The database sends an ETag, so past the TTL +// a request that asks whether the tile changed would get 304 and the damaged +// copy back. +func TestGoSumDB_DamagedCachedTileIsFetchedAgain(t *testing.T) { + const tile = "/tile/8/0/000" + want := hashTile(256, "a") + for _, ttl := range []time.Duration{time.Hour, time.Nanosecond} { + t.Run(fmt.Sprintf("ttl=%s", ttl), func(t *testing.T) { + db := newFakeSumDB(t, map[string]string{tile: want}) + inner := db.server.Config.Handler + db.server.Config.Handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("ETag", `"tile"`) + if r.Header.Get("If-None-Match") == `"tile"` { + w.WriteHeader(http.StatusNotModified) + return + } + inner.ServeHTTP(w, r) + }) + h := newSumDBTestHandler(t, db.server.URL, true, ttl) + + if w := serveGoRequest(h, "/sumdb/sum.example"+tile); w.Code != http.StatusOK { + t.Fatalf("status = %d, want 200", w.Code) + } + + store := h.proxy.Storage.(*mockStorage) + blob := metadataStoragePath(goEcosystem, goSumDBCacheKey(testSumDBName, db.server.URL, "tile/8/0/000")) + store.mu.Lock() + if _, ok := store.files[blob]; !ok { + store.mu.Unlock() + t.Fatalf("tile not stored at %s", blob) + } + store.files[blob] = []byte(want[:100]) + store.mu.Unlock() + + w := serveGoRequest(h, "/sumdb/sum.example"+tile) + if w.Code != http.StatusOK || w.Body.String() != want { + t.Fatalf("after damage: status = %d, body length %d, want the %d-byte tile", w.Code, w.Body.Len(), len(want)) + } + if n := db.requests(tile); n != 2 { + t.Errorf("upstream requests = %d, want 2: the damaged copy was fetched again", n) + } + }) + } +} + func TestGoSumDB_CompleteTilesNotStoredWithoutMetadataCache(t *testing.T) { const tile = "/tile/8/0/000" db := newFakeSumDB(t, map[string]string{tile: hashTile(256, "a")}) @@ -306,8 +589,8 @@ func TestGoSumDB_CompleteTilesNotStoredWithoutMetadataCache(t *testing.T) { func TestGoSumDB_OfflineFallback(t *testing.T) { files := map[string]string{ - "/lookup/golang.org/x/text@v0.14.0": "lookup record", - "/latest": "signed tree", + "/lookup/golang.org/x/text@v0.14.0": testSumDBRecord, + "/latest": testSumDBTree, "/tile/8/0/000": hashTile(256, "a"), "/tile/8/0/001.p/5": hashTile(5, "b"), } From c7f8e2fc6469444c45393ef76366d8341e1a710a Mon Sep 17 00:00:00 2001 From: Christian Heim Date: Sat, 10 Oct 2026 11:31:01 +0200 Subject: [PATCH 252/633] README: say when the go command still goes to the database itself The sentence promised a direct fallback whenever sum.golang.org is unreachable. That's only true once the cached tree has gone stale; a fresh one keeps the proxy answering. configuration.md has the details. --- README.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 30e09a0..2c8b2e2 100644 --- a/README.md +++ b/README.md @@ -205,9 +205,10 @@ Or in your shell profile for persistence. Checksum lookups go through the same URL: the proxy serves `sum.golang.org` at `/go/sumdb/sum.golang.org/`, and the `go` command uses it without further setup, so clients need no route of their own to `sum.golang.org`. If the -proxy cannot reach `https://sum.golang.org`, the `go` command contacts it -directly as before. Other checksum databases are -listed under `upstream.go_sumdbs`, see +proxy loses `https://sum.golang.org`, it keeps answering until its cached +copy of the database's signed tree is older than `metadata_ttl`; after that +the `go` command contacts the database directly as before. Other checksum +databases are listed under `upstream.go_sumdbs`, see [Go checksum database](docs/configuration.md#go-checksum-database). ### Homebrew From 9532c6a94e0e58245e24db47e3e63e45f413477f Mon Sep 17 00:00:00 2001 From: Christian Heim Date: Sat, 10 Oct 2026 17:55:04 +0200 Subject: [PATCH 253/633] Add upstream.go_sumdbs_offline for networks where only the proxy has egress On a build network where clients cannot reach sum.golang.org at all, the proxy should keep answering for the database during an outage, since a 404 on supported would only send the go command somewhere it cannot go. With the switch set, a cached tree of any age is enough; without it, the tree has to be fresh, as before. That does not make every cached module verify. The go command checks a lookup against tiles of the tree that came with it, and sum.golang.org attaches a different tree to almost every lookup, so it can ask for a tile nobody fetched through the proxy before. configuration.md says so. While the database is known to be unreachable, offline mode also serves cached lookups and tiles straight away. Otherwise each of them would sit through the upstream timeout before the fetch falls back to the cache, and a build would take minutes for as long as the outage lasts. A lookup or tile that falls back to the cache renews what the proxy knows about the database, unless a probe has reached it since the request started. The go command asks supported only once per process, so a long build would otherwise be back to waiting a minute into the outage. recordTimeout is now recordFailure, since lookups use it too. The option needs cache_metadata, since without it there is no cached tree to fall back on, and the config refuses the combination. --- README.md | 5 +- docs/configuration.md | 17 +++ internal/config/config.go | 23 +++- internal/config/config_test.go | 15 +++ internal/handler/go.go | 11 +- internal/handler/go_sumdb.go | 55 ++++++--- internal/handler/go_sumdb_test.go | 187 +++++++++++++++++++++++++----- internal/server/server.go | 2 +- 8 files changed, 259 insertions(+), 56 deletions(-) diff --git a/README.md b/README.md index 2c8b2e2..3aecba4 100644 --- a/README.md +++ b/README.md @@ -207,8 +207,9 @@ Checksum lookups go through the same URL: the proxy serves `sum.golang.org` at setup, so clients need no route of their own to `sum.golang.org`. If the proxy loses `https://sum.golang.org`, it keeps answering until its cached copy of the database's signed tree is older than `metadata_ttl`; after that -the `go` command contacts the database directly as before. Other checksum -databases are listed under `upstream.go_sumdbs`, see +the `go` command contacts the database directly as before. Set +`upstream.go_sumdbs_offline` on networks where clients cannot do that. Other +checksum databases are listed under `upstream.go_sumdbs`, see [Go checksum database](docs/configuration.md#go-checksum-database). ### Homebrew diff --git a/docs/configuration.md b/docs/configuration.md index be7df9e..e0e9047 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -233,6 +233,7 @@ Each upstream used by a built-in package route can be set in YAML or JSON under | `upstream.allow_loopback` | `PROXY_UPSTREAM_ALLOW_LOOPBACK` | `false` | | `upstream.npm` | `PROXY_UPSTREAM_NPM` | `https://registry.npmjs.org` | | `upstream.npm_full_metadata` | `PROXY_UPSTREAM_NPM_FULL_METADATA` | `false` | +| `upstream.go_sumdbs_offline` | `PROXY_UPSTREAM_GO_SUMDBS_OFFLINE` | `false` | | `upstream.cargo` | `PROXY_UPSTREAM_CARGO` | `https://index.crates.io` | | `upstream.cargo_download` | `PROXY_UPSTREAM_CARGO_DOWNLOAD` | `https://static.crates.io/crates` | | `upstream.gem` | `PROXY_UPSTREAM_GEM` | `https://rubygems.org` | @@ -503,6 +504,11 @@ upstream: # Named Go checksum databases, served at /go/sumdb/{name}/. go_sumdbs: sum.golang.org: "https://sum.golang.org" + + # Keep answering for a database from its cached tree, however old, when + # the database cannot be reached. For networks where clients have no + # route to the database of their own. Requires cache_metadata. + go_sumdbs_offline: false ``` The `go` command asks `GOPROXY` for the checksum database before contacting it @@ -520,6 +526,17 @@ clients go to the database themselves, as they would without the proxy. The check waits five seconds at most, and after a failed check the proxy waits a minute before it tries the database again. +Set `upstream.go_sumdbs_offline: true` when clients have no route to the +database of their own. The proxy then keeps answering for it from the cached +tree for as long as the outage lasts and serves cached lookups and tiles. +That does not mean every cached module still verifies: the `go` command +checks each lookup against tiles of the signed tree that came with it, the +database attaches a different tree to almost every lookup, and a tile no +client fetched through the proxy before cannot be served during the outage. +Lookups for modules that were never cached fail as well. Builds whose +`go.sum` already lists every module do not ask the database at all and are +not affected. This requires `cache_metadata`. + With `cache_metadata` enabled, lookups, partial tiles and the latest signed tree follow `metadata_ttl` and are served from cache when the database is unreachable. Complete tiles never change and are not revalidated once cached; diff --git a/internal/config/config.go b/internal/config/config.go index 46cda82..080209d 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -580,6 +580,14 @@ type UpstreamConfig struct { // Default when empty: {"sum.golang.org": "https://sum.golang.org"}. GoSumDBs map[string]string `json:"go_sumdbs" yaml:"go_sumdbs"` + // GoSumDBsOffline keeps answering for a checksum database from its + // cached signed tree, however old, when the database cannot be reached. + // Set this when clients have no route to the database of their own; + // otherwise the proxy stops answering once the cached tree is older than + // metadata_ttl, and the go command contacts the database directly. + // Requires cache_metadata. Default: false. + GoSumDBsOffline bool `json:"go_sumdbs_offline" yaml:"go_sumdbs_offline"` + // Hex is the upstream Hex repository URL. // Default: https://repo.hex.pm Hex string `json:"hex" yaml:"hex"` @@ -823,6 +831,18 @@ func (u *UpstreamConfig) validateGenericArtifacts() error { return nil } +// validateUpstream checks the upstream block, including the one setting in +// it that depends on a top-level option. +func (c *Config) validateUpstream() error { + if err := c.Upstream.Validate(); err != nil { + return err + } + if c.Upstream.GoSumDBsOffline && !c.CacheMetadata { + return fmt.Errorf("upstream.go_sumdbs_offline requires cache_metadata") + } + return nil +} + // CompileGenericArtifactPattern compiles an upstream.generic_artifacts pattern // so that it only matches a whole path, and checks it has the `version` and // `file` groups. @@ -1150,6 +1170,7 @@ func (c *Config) LoadFromEnv() { setEnvBool(&c.Upstream.AllowLoopback, "PROXY_UPSTREAM_ALLOW_LOOPBACK") setEnvString(&c.Upstream.NPM, "PROXY_UPSTREAM_NPM") setEnvBool(&c.Upstream.NPMFullMetadata, "PROXY_UPSTREAM_NPM_FULL_METADATA") + setEnvBool(&c.Upstream.GoSumDBsOffline, "PROXY_UPSTREAM_GO_SUMDBS_OFFLINE") setEnvString(&c.Upstream.Cargo, "PROXY_UPSTREAM_CARGO") setEnvString(&c.Upstream.CargoDownload, "PROXY_UPSTREAM_CARGO_DOWNLOAD") setEnvString(&c.Upstream.Gem, "PROXY_UPSTREAM_GEM") @@ -1325,7 +1346,7 @@ func (c *Config) validateComponents() error { return err } - if err := c.Upstream.Validate(); err != nil { + if err := c.validateUpstream(); err != nil { return err } diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 88150ae..d7b5326 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -1638,6 +1638,21 @@ func TestValidateGoSumDBs(t *testing.T) { }, wantErr: true, }, + { + name: "Go checksum database offline mode requires metadata caching", + modify: func(cfg *Config) { + cfg.CacheMetadata = false + cfg.Upstream.GoSumDBsOffline = true + }, + wantErr: true, + }, + { + name: "Go checksum database offline mode with metadata caching", + modify: func(cfg *Config) { + cfg.CacheMetadata = true + cfg.Upstream.GoSumDBsOffline = true + }, + }, } for _, tt := range tests { diff --git a/internal/handler/go.go b/internal/handler/go.go index e5c2f28..a27c37b 100644 --- a/internal/handler/go.go +++ b/internal/handler/go.go @@ -28,11 +28,12 @@ func init() { // GoHandler handles Go module proxy protocol requests. type GoHandler struct { - proxy *Proxy - upstreamURL string - proxyURL string - sumDBs map[string]string - sumDBProbes goSumDBProbes + proxy *Proxy + upstreamURL string + proxyURL string + sumDBs map[string]string + sumDBOffline bool + sumDBProbes goSumDBProbes } // NewGoHandler creates a new Go module proxy handler. diff --git a/internal/handler/go_sumdb.go b/internal/handler/go_sumdb.go index 0e0bcbf..4f53a0a 100644 --- a/internal/handler/go_sumdb.go +++ b/internal/handler/go_sumdb.go @@ -78,9 +78,10 @@ func (p *goSumDBProbes) record(name string, reachable bool) { p.failed[name] = time.Now() } -// recordTimeout records a failure for a probe started at start, unless the -// fetch it gave up waiting for has succeeded in the meantime. -func (p *goSumDBProbes) recordTimeout(name string, start time.Time) { +// recordFailure records that the database did not answer a request started +// at start, unless a probe has reached it since: a probe that timed out, or +// a lookup that fell back to the cache, must not overwrite a newer success. +func (p *goSumDBProbes) recordFailure(name string, start time.Time) { p.mu.Lock() defer p.mu.Unlock() if !p.succeeded[name].Before(start) { @@ -94,8 +95,11 @@ func (p *goSumDBProbes) recordTimeout(name string, start time.Time) { // SetSumDBs sets the checksum databases served at /sumdb/{name}/, mapping // each database name to its base URL. An empty map keeps the default, -// sum.golang.org. -func (h *GoHandler) SetSumDBs(sumDBs map[string]string) { +// sum.golang.org. offline makes a cached tree of any age enough to keep +// answering for a database the proxy cannot reach, for networks where +// clients have no route to the database of their own. +func (h *GoHandler) SetSumDBs(sumDBs map[string]string, offline bool) { + h.sumDBOffline = offline if len(sumDBs) == 0 { h.sumDBs = map[string]string{defaultGoSumDBName: defaultGoSumDBURL} return @@ -141,7 +145,7 @@ func (h *GoHandler) handleSumDB(w http.ResponseWriter, r *http.Request, rest str if served { return } - h.proxySumDB(w, r, sumDBURL+"/"+path, cacheKey, check, damaged) + h.proxySumDB(w, r, name, sumDBURL+"/"+path, cacheKey, check, damaged) } // sumDBReachable reports whether the proxy can serve a checksum database: it @@ -171,7 +175,7 @@ func (h *GoHandler) sumDBReachable(r *http.Request, name, sumDBURL string) bool case err = <-done: case <-time.After(h.sumDBProbes.probeTimeout()): err = fmt.Errorf("no answer within %s", h.sumDBProbes.probeTimeout()) - h.sumDBProbes.recordTimeout(name, start) + h.sumDBProbes.recordFailure(name, start) case <-r.Context().Done(): // The client went away; that says nothing about the database. return false @@ -204,11 +208,11 @@ func (h *GoHandler) probeSumDB(ctx context.Context, name, cacheKey, sumDBURL str } // cachedTreeServes reports whether a cached tree lets the proxy answer for -// the database although the database did not: only while the tree is fresh, -// so clients fall back to the database once the proxy has lost it. +// the database although the database did not: always with go_sumdbs_offline, +// otherwise only while the tree is fresh. func (h *GoHandler) cachedTreeServes(ctx context.Context, cacheKey string, check func([]byte) error) bool { entry, _, _, ok := h.readCachedSumDB(ctx, cacheKey, check) - return ok && h.treeFresh(entry) + return ok && (h.sumDBOffline || h.treeFresh(entry)) } // treeFresh reports whether a cached tree is within the metadata TTL. @@ -360,15 +364,31 @@ func (h *GoHandler) serveCachedSumDBTile(w http.ResponseWriter, r *http.Request, // go command prints it, and it says why a module version is missing. check, // when set, must accept a response before it is cached or served. damaged // says the cached copy failed check. -func (h *GoHandler) proxySumDB(w http.ResponseWriter, r *http.Request, upstreamURL, cacheKey string, check func([]byte) error, damaged bool) { +func (h *GoHandler) proxySumDB(w http.ResponseWriter, r *http.Request, name, upstreamURL, cacheKey string, check func([]byte) error, damaged bool) { p := h.proxy if !p.CacheMetadata { p.proxyMetadataStream(w, r, upstreamURL, "identity", "*/*") return } + // While the database is known to be unreachable, offline mode serves + // the cached copy at once rather than letting every request wait for + // the upstream timeout before the fetch falls back to it. + if h.sumDBOffline && h.sumDBProbes.recentlyFailed(name) { + if entry, body, contentType, ok := h.readCachedSumDB(r.Context(), cacheKey, check); ok { + p.writeMetadataCachedResponseWithEncoding(w, r, goEcosystem, cacheKey, body, contentType, entry.ContentEncoding.String) + return + } + } - body, contentType, contentEncoding, err := h.fetchSumDB(r.Context(), upstreamURL, cacheKey, check, damaged) - if err != nil { + start := time.Now() + res := h.fetchSumDB(r.Context(), upstreamURL, cacheKey, check, damaged) + if res.stale { + // The database did not answer. Remember it here too: the go command + // asks supported only once per process, so a long build would lose + // the shortcut above a minute into an outage. + h.sumDBProbes.recordFailure(name, start) + } + if err := res.err; err != nil { var notFound *upstreamNotFoundError switch { case errors.As(err, ¬Found) && len(notFound.body) > 0: @@ -385,18 +405,17 @@ func (h *GoHandler) proxySumDB(w http.ResponseWriter, r *http.Request, upstreamU return } - p.writeMetadataCachedResponseWithEncoding(w, r, goEcosystem, cacheKey, body, contentType, contentEncoding) + p.writeMetadataCachedResponseWithEncoding(w, r, goEcosystem, cacheKey, res.body, res.contentType, res.contentEncoding) } // fetchSumDB fetches a checksum database response through the metadata // cache. A cached copy that failed check is fetched again without // revalidating it: past the TTL, a database that sends an ETag would answer // 304 and hand back the damaged copy for good. -func (h *GoHandler) fetchSumDB(ctx context.Context, upstreamURL, cacheKey string, check func([]byte) error, damaged bool) ([]byte, string, string, error) { +func (h *GoHandler) fetchSumDB(ctx context.Context, upstreamURL, cacheKey string, check func([]byte) error, damaged bool) metadataResult { p := h.proxy if damaged { - res := p.fetchMetadataFromUpstream(ctx, goEcosystem, cacheKey, upstreamURL, "*/*", "identity", check, nil) - return res.body, res.contentType, res.contentEncoding, res.err + return p.fetchMetadataFromUpstream(ctx, goEcosystem, cacheKey, upstreamURL, "*/*", "identity", check, nil) } - return p.fetchOrCacheMetadata(ctx, goEcosystem, cacheKey, upstreamURL, "identity", check, "*/*") + return p.lookupMetadata(ctx, goEcosystem, cacheKey, upstreamURL, "*/*", "identity", check) } diff --git a/internal/handler/go_sumdb_test.go b/internal/handler/go_sumdb_test.go index a849ad8..8bb687c 100644 --- a/internal/handler/go_sumdb_test.go +++ b/internal/handler/go_sumdb_test.go @@ -111,7 +111,7 @@ func newSumDBTestHandler(t *testing.T, sumDBURL string, cacheMetadata bool, ttl proxy.CacheMetadata = cacheMetadata proxy.MetadataTTL = ttl h := NewGoHandler(proxy, "http://proxy.example") - h.SetSumDBs(map[string]string{testSumDBName: sumDBURL}) + h.SetSumDBs(map[string]string{testSumDBName: sumDBURL}, false) return h } @@ -126,7 +126,7 @@ func TestGoSumDB_DefaultsToSumGolangOrg(t *testing.T) { if got := h.sumDBs[defaultGoSumDBName]; got != defaultGoSumDBURL { t.Errorf("default checksum database = %q, want %q", got, defaultGoSumDBURL) } - h.SetSumDBs(nil) + h.SetSumDBs(nil, false) if got := h.sumDBs[defaultGoSumDBName]; got != defaultGoSumDBURL || len(h.sumDBs) != 1 { t.Errorf("SetSumDBs(nil) = %v, want only the default", h.sumDBs) } @@ -192,24 +192,31 @@ func TestGoSumDB_SupportedWithCachedTreeWhileUnreachable(t *testing.T) { } }) - // A stale tree does not count: clients that can reach the database - // themselves should go there. - t.Run("stale tree", func(t *testing.T) { - db := newFakeSumDB(t, map[string]string{"/latest": testSumDBTree}) - h := newSumDBTestHandler(t, db.server.URL, true, time.Nanosecond) - if w := serveGoRequest(h, "/sumdb/sum.example/latest"); w.Code != http.StatusOK { - t.Fatalf("latest: status = %d, want 200", w.Code) - } - db.down.Store(true) - for i := range 2 { // second probe hits the remembered failure - if w := serveGoRequest(h, "/sumdb/sum.example/supported"); w.Code != http.StatusNotFound { - t.Errorf("probe %d: status = %d, want 404", i, w.Code) + // A stale tree only counts with go_sumdbs_offline. Without it, clients + // that can reach the database themselves should go there. + for _, offline := range []bool{true, false} { + t.Run(fmt.Sprintf("stale tree offline=%v", offline), func(t *testing.T) { + db := newFakeSumDB(t, map[string]string{"/latest": testSumDBTree}) + h := newSumDBTestHandler(t, db.server.URL, true, time.Nanosecond) + h.sumDBOffline = offline + if w := serveGoRequest(h, "/sumdb/sum.example/latest"); w.Code != http.StatusOK { + t.Fatalf("latest: status = %d, want 200", w.Code) } - } - if n := db.requests("/latest"); n != 2 { - t.Errorf("upstream requests = %d, want 2: the failure is remembered", n) - } - }) + db.down.Store(true) + want := http.StatusNotFound + if offline { + want = http.StatusOK + } + for i := range 2 { // second probe hits the remembered failure + if w := serveGoRequest(h, "/sumdb/sum.example/supported"); w.Code != want { + t.Errorf("probe %d: status = %d, want %d", i, w.Code, want) + } + } + if n := db.requests("/latest"); n != 2 { + t.Errorf("upstream requests = %d, want 2: the failure is remembered", n) + } + }) + } } // TestGoSumDB_SupportedGivesUpOnSilentDatabase: a database that accepts the @@ -237,14 +244,16 @@ func TestGoSumDB_SupportedGivesUpOnSilentDatabase(t *testing.T) { } } -// TestGoSumDB_SupportedWithStaleTreeWhileSilent: a silent database holds -// the fetch past the probe timeout. The stale tree does not count, and the -// second probe must answer at once from the remembered failure. +// TestGoSumDB_SupportedWithStaleTreeWhileSilent: the fetch only falls back +// to a stale cached tree once upstream has failed, which a silent database +// delays past the probe timeout. The proxy can still serve from cache, so +// supported must say so. func TestGoSumDB_SupportedWithStaleTreeWhileSilent(t *testing.T) { release := make(chan struct{}) db := newFakeSumDB(t, map[string]string{"/latest": testSumDBTree}) t.Cleanup(func() { close(release) }) h := newSumDBTestHandler(t, db.server.URL, true, time.Nanosecond) + h.sumDBOffline = true h.sumDBProbes.timeout = 50 * time.Millisecond if w := serveGoRequest(h, "/sumdb/sum.example/latest"); w.Code != http.StatusOK { @@ -259,8 +268,8 @@ func TestGoSumDB_SupportedWithStaleTreeWhileSilent(t *testing.T) { go func() { answered <- serveGoRequest(h, "/sumdb/sum.example/supported") }() select { case w := <-answered: - if w.Code != http.StatusNotFound { - t.Fatalf("probe %d: status = %d, want 404", i, w.Code) + if w.Code != http.StatusOK { + t.Fatalf("probe %d: status = %d, want 200 from the cached tree", i, w.Code) } case <-time.After(time.Second): t.Fatalf("probe %d: supported still waiting after a second", i) @@ -306,6 +315,7 @@ func TestGoSumDB_SlowProbeSuccessClearsFailure(t *testing.T) { func TestGoSumDB_StaleFallbackIsNotAnAnswer(t *testing.T) { db := newFakeSumDB(t, map[string]string{"/latest": testSumDBTree}) h := newSumDBTestHandler(t, db.server.URL, true, time.Nanosecond) + h.sumDBOffline = true h.sumDBProbes.timeout = 10 * time.Millisecond if w := serveGoRequest(h, "/sumdb/sum.example/latest"); w.Code != http.StatusOK { @@ -318,8 +328,8 @@ func TestGoSumDB_StaleFallbackIsNotAnAnswer(t *testing.T) { }) db.down.Store(true) - if w := serveGoRequest(h, "/sumdb/sum.example/supported"); w.Code != http.StatusNotFound { - t.Fatalf("status = %d, want 404", w.Code) + if w := serveGoRequest(h, "/sumdb/sum.example/supported"); w.Code != http.StatusOK { + t.Fatalf("status = %d, want 200 from the cached tree", w.Code) } time.Sleep(300 * time.Millisecond) // the fetch has failed and fallen back by now if !h.sumDBProbes.recentlyFailed(testSumDBName) { @@ -352,15 +362,134 @@ func TestGoSumDB_AnswerCountsWhenCacheWriteFails(t *testing.T) { } } +// TestGoSumDB_OfflineServesCacheWithoutWaiting: in offline mode, once the +// database is known to be unreachable, stale lookups must come from the +// cache at once instead of waiting for the upstream timeout each time. +func TestGoSumDB_OfflineServesCacheWithoutWaiting(t *testing.T) { + const lookup = "/lookup/golang.org/x/text@v0.14.0" + release := make(chan struct{}) + db := newFakeSumDB(t, map[string]string{"/latest": testSumDBTree, lookup: testSumDBRecord}) + t.Cleanup(func() { close(release) }) + h := newSumDBTestHandler(t, db.server.URL, true, time.Nanosecond) + h.sumDBOffline = true + h.sumDBProbes.timeout = 50 * time.Millisecond + + for _, path := range []string{"/latest", lookup} { + if w := serveGoRequest(h, "/sumdb/sum.example"+path); w.Code != http.StatusOK { + t.Fatalf("%s: status = %d, want 200", path, w.Code) + } + } + db.server.Config.Handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + <-release + }) + + for _, path := range []string{"/supported", "/latest", lookup} { + answered := make(chan *httptest.ResponseRecorder, 1) + go func() { answered <- serveGoRequest(h, "/sumdb/sum.example"+path) }() + select { + case w := <-answered: + if w.Code != http.StatusOK { + t.Errorf("%s: status = %d, want 200 from the cache", path, w.Code) + } + case <-time.After(time.Second): + t.Fatalf("%s: still waiting for the database after a second", path) + } + } +} + +// TestGoSumDB_StaleLookupRemembersFailure: the go command asks supported +// only once per process. A lookup that falls back to the cache must renew +// the remembered failure itself, or offline mode would wait for the upstream +// timeout on every lookup a minute into a long build. +func TestGoSumDB_StaleLookupRemembersFailure(t *testing.T) { + const lookup = "/lookup/golang.org/x/text@v0.14.0" + db := newFakeSumDB(t, map[string]string{lookup: testSumDBRecord}) + h := newSumDBTestHandler(t, db.server.URL, true, time.Nanosecond) + h.sumDBOffline = true + + if w := serveGoRequest(h, "/sumdb/sum.example"+lookup); w.Code != http.StatusOK { + t.Fatalf("online: status = %d, want 200", w.Code) + } + if h.sumDBProbes.recentlyFailed(testSumDBName) { + t.Fatal("an answered lookup was recorded as a failure") + } + db.down.Store(true) + if w := serveGoRequest(h, "/sumdb/sum.example"+lookup); w.Code != http.StatusOK || w.Body.String() != testSumDBRecord { + t.Fatalf("outage: status = %d, body = %q, want the cached record", w.Code, w.Body.String()) + } + if !h.sumDBProbes.recentlyFailed(testSumDBName) { + t.Error("a lookup served from the cache after the database failed did not remember the failure") + } +} + +// TestGoSumDB_LateStaleLookupKeepsNewerSuccess: a lookup sent while the +// database was failing, which falls back to the cache only after a probe has +// reached the database again, must not mark it unreachable. +func TestGoSumDB_LateStaleLookupKeepsNewerSuccess(t *testing.T) { + const lookup = "/lookup/golang.org/x/text@v0.14.0" + db := newFakeSumDB(t, map[string]string{"/latest": testSumDBTree, lookup: testSumDBRecord}) + h := newSumDBTestHandler(t, db.server.URL, true, time.Nanosecond) + if w := serveGoRequest(h, "/sumdb/sum.example"+lookup); w.Code != http.StatusOK { + t.Fatalf("lookup: status = %d, want 200", w.Code) + } + + var sentOnce sync.Once + sent := make(chan struct{}) + release := make(chan struct{}) + inner := db.server.Config.Handler + db.server.Config.Handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == lookup { + sentOnce.Do(func() { close(sent) }) + <-release + http.Error(w, "unavailable", http.StatusServiceUnavailable) + return + } + inner.ServeHTTP(w, r) + }) + done := make(chan *httptest.ResponseRecorder, 1) + go func() { done <- serveGoRequest(h, "/sumdb/sum.example"+lookup) }() + <-sent + if w := serveGoRequest(h, "/sumdb/sum.example/supported"); w.Code != http.StatusOK { + t.Fatalf("supported: status = %d, want 200", w.Code) + } + close(release) + if w := <-done; w.Code != http.StatusOK || w.Body.String() != testSumDBRecord { + t.Fatalf("lookup: status = %d, body = %q, want the cached record", w.Code, w.Body.String()) + } + if h.sumDBProbes.recentlyFailed(testSumDBName) { + t.Error("a lookup that fell back after the probe succeeded marked the database unreachable") + } +} + +// TestGoSumDB_ShortcutNeedsOfflineMode: without go_sumdbs_offline, a +// remembered failure does not keep cached lookups from asking the database, +// which may well be back. +func TestGoSumDB_ShortcutNeedsOfflineMode(t *testing.T) { + const lookup = "/lookup/golang.org/x/text@v0.14.0" + db := newFakeSumDB(t, map[string]string{lookup: testSumDBRecord}) + h := newSumDBTestHandler(t, db.server.URL, true, time.Nanosecond) + if w := serveGoRequest(h, "/sumdb/sum.example"+lookup); w.Code != http.StatusOK { + t.Fatalf("lookup: status = %d, want 200", w.Code) + } + + h.sumDBProbes.record(testSumDBName, false) + if w := serveGoRequest(h, "/sumdb/sum.example"+lookup); w.Code != http.StatusOK { + t.Fatalf("lookup with a remembered failure: status = %d, want 200", w.Code) + } + if n := db.requests(lookup); n != 2 { + t.Errorf("database requests = %d, want 2: the cached lookup was served without asking", n) + } +} + func TestGoSumDBProbes_TimeoutDoesNotOverwriteLaterSuccess(t *testing.T) { var probes goSumDBProbes start := time.Now() probes.record(testSumDBName, true) - probes.recordTimeout(testSumDBName, start) + probes.recordFailure(testSumDBName, start) if probes.recentlyFailed(testSumDBName) { t.Error("a success recorded after the probe started was overwritten by its timeout") } - probes.recordTimeout(testSumDBName, time.Now().Add(time.Second)) + probes.recordFailure(testSumDBName, time.Now().Add(time.Second)) if !probes.recentlyFailed(testSumDBName) { t.Error("a timeout with no later success was not recorded") } @@ -433,7 +562,7 @@ func TestGoSumDB_RequestsGoToTheDatabase(t *testing.T) { proxy, _, _, _ := setupTestProxy(t) proxy.HTTPClient = http.DefaultClient h := NewGoHandlerWithUpstream(proxy, "http://proxy.example", moduleUpstream.URL) - h.SetSumDBs(map[string]string{testSumDBName: db.server.URL + "/"}) + h.SetSumDBs(map[string]string{testSumDBName: db.server.URL + "/"}, false) w := serveGoRequest(h, "/sumdb/sum.example"+lookup) if w.Code != http.StatusOK || w.Body.String() != testSumDBRecord { diff --git a/internal/server/server.go b/internal/server/server.go index b6c93bc..3ebc3e2 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -402,7 +402,7 @@ func (s *Server) mountProtocolHandlers(r chi.Router, proxy *handler.Proxy) { ) gemHandler := handler.NewGemHandlerWithUpstream(proxy, s.cfg.BaseURL, s.cfg.Upstream.Gem) goHandler := handler.NewGoHandlerWithUpstream(proxy, s.cfg.BaseURL, s.cfg.Upstream.Go) - goHandler.SetSumDBs(s.cfg.Upstream.GoSumDBs) + goHandler.SetSumDBs(s.cfg.Upstream.GoSumDBs, s.cfg.Upstream.GoSumDBsOffline) hexHandler := handler.NewHexHandlerWithUpstreams( proxy, s.cfg.BaseURL, From 943009c6692461e58e524d56728ad62302e46bb8 Mon Sep 17 00:00:00 2001 From: Christian Heim Date: Mon, 12 Oct 2026 22:35:20 +0200 Subject: [PATCH 254/633] Check that a data tile holds W records before caching it. Complete tiles are never fetched again, so a block page cached as one stayed for good. --- internal/handler/go_sumdb.go | 22 ++++++++++++++---- internal/handler/go_sumdb_test.go | 38 +++++++++++++++++++++++++++++-- 2 files changed, 54 insertions(+), 6 deletions(-) diff --git a/internal/handler/go_sumdb.go b/internal/handler/go_sumdb.go index 4f53a0a..32f2a9c 100644 --- a/internal/handler/go_sumdb.go +++ b/internal/handler/go_sumdb.go @@ -270,8 +270,8 @@ const goSumDBTreePrefix = "go.sum database tree\n" // followed by a signed tree, which is how the go command reads it; a block // page cached in its place would also replace the copy kept for an outage. // A hash tile holds exactly W hashes, and complete tiles are never -// revalidated, so a bad one would otherwise stay for good. Data tiles have no -// fixed size and only must not be empty. +// revalidated, so a bad one would otherwise stay for good. A data tile holds +// exactly W records. func goSumDBResponseCheck(path string) func([]byte) error { if path == "latest" { return func(body []byte) error { @@ -298,13 +298,27 @@ func goSumDBResponseCheck(path string) func([]byte) error { if tile.L >= 0 && len(body) != tile.W*tlog.HashSize { return fmt.Errorf("checksum database tile %s has %d bytes, want %d", path, len(body), tile.W*tlog.HashSize) } - if len(body) == 0 { - return fmt.Errorf("checksum database tile %s is empty", path) + if tile.L < 0 { + if n := goSumDBDataTileRecords(body); n != tile.W { + return fmt.Errorf("checksum database tile %s has %d records, want %d", path, n, tile.W) + } } return nil } } +// goSumDBDataTileRecords counts the records in a data tile. Each record ends +// with a newline and a blank line separates it from the next; whether one +// follows the last record differs between servers. Anything that does not end +// with a newline counts as no records. +func goSumDBDataTileRecords(body []byte) int { + text, ok := strings.CutSuffix(string(body), "\n") + if !ok || text == "" { + return 0 + } + return strings.Count(text, "\n\n") + 1 +} + // validGoSumDBLookup checks an escaped module@version the way the checksum // database does. Requiring a semantic version also keeps characters such as // '#' and '%' out of the upstream URL. diff --git a/internal/handler/go_sumdb_test.go b/internal/handler/go_sumdb_test.go index 8bb687c..98ab7f7 100644 --- a/internal/handler/go_sumdb_test.go +++ b/internal/handler/go_sumdb_test.go @@ -32,6 +32,17 @@ func hashTile(width int, fill string) string { return strings.Repeat(fill, width*tlog.HashSize) } +// dataTile returns a data tile body holding width records, laid out the way +// sum.golang.org serves it: a blank line between records, none after the +// last. +func dataTile(width int) string { + records := make([]string, width) + for i := range records { + records[i] = fmt.Sprintf("example.com/m%d v1.0.0 h1:hash=\nexample.com/m%d v1.0.0/go.mod h1:hash=\n", i, i) + } + return strings.Join(records, "\n") +} + // fakeSumDB is a checksum database upstream serving fixed files. Unknown // paths get 404 with the body from notFound, or a default text. type fakeSumDB struct { @@ -481,6 +492,25 @@ func TestGoSumDB_ShortcutNeedsOfflineMode(t *testing.T) { } } +func TestGoSumDBDataTileRecords(t *testing.T) { + tests := []struct { + name string + body string + want int + }{ + {"sum.golang.org layout", dataTile(3), 3}, + {"blank line after the last record", dataTile(3) + "\n", 3}, + {"one record", dataTile(1), 1}, + {"empty", "", 0}, + {"no final newline", "please log in", 0}, + } + for _, tt := range tests { + if got := goSumDBDataTileRecords([]byte(tt.body)); got != tt.want { + t.Errorf("%s: records = %d, want %d", tt.name, got, tt.want) + } + } +} + func TestGoSumDBProbes_TimeoutDoesNotOverwriteLaterSuccess(t *testing.T) { var probes goSumDBProbes start := time.Now() @@ -595,9 +625,9 @@ func TestGoSumDB_LookupCachedWithinTTL(t *testing.T) { func TestGoSumDB_CompleteTilesSkipRevalidation(t *testing.T) { files := map[string]string{ "/tile/8/0/000": hashTile(256, "a"), - "/tile/8/data/000": "complete data tile", + "/tile/8/data/000": dataTile(256), "/tile/8/0/001.p/5": hashTile(5, "b"), - "/tile/8/data/001.p/5": "partial data tile", + "/tile/8/data/001.p/5": dataTile(5), "/latest": testSumDBTree, } wantRequests := map[string]int{ @@ -633,6 +663,7 @@ func TestGoSumDB_BadTilesAreNotCached(t *testing.T) { db := newFakeSumDB(t, map[string]string{ tile: "please log in", "/tile/8/data/000.p/3": "", + "/tile/8/data/000": "please log in\n", }) h := newSumDBTestHandler(t, db.server.URL, true, time.Hour) @@ -642,6 +673,9 @@ func TestGoSumDB_BadTilesAreNotCached(t *testing.T) { if w := serveGoRequest(h, "/sumdb/sum.example/tile/8/data/000.p/3"); w.Code != http.StatusBadGateway { t.Errorf("empty data tile: status = %d, want 502", w.Code) } + if w := serveGoRequest(h, "/sumdb/sum.example/tile/8/data/000"); w.Code != http.StatusBadGateway { + t.Errorf("block page as a data tile: status = %d, want 502", w.Code) + } want := hashTile(256, "a") db.setFile(tile, want) From 5332f1f5e0ffb564c6128932e25486b30b67ea22 Mon Sep 17 00:00:00 2001 From: Christian Heim Date: Mon, 12 Oct 2026 22:35:23 +0200 Subject: [PATCH 255/633] Don't count a tree fresh in the cache as the database answering. The probe never asked it, and during an outage that kept lookups from remembering the failure, so offline mode waited for the upstream timeout on every request until the tree expired. --- internal/handler/go_sumdb.go | 9 +++++++-- internal/handler/go_sumdb_test.go | 21 +++++++++++++++++++++ 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/internal/handler/go_sumdb.go b/internal/handler/go_sumdb.go index 32f2a9c..c74b0c8 100644 --- a/internal/handler/go_sumdb.go +++ b/internal/handler/go_sumdb.go @@ -195,9 +195,14 @@ func (h *GoHandler) sumDBReachable(r *http.Request, name, sumDBURL string) bool } // probeSumDB fetches the database's latest tree and records whether the -// database answered. A tree served from the cache within its TTL counts as -// an answer; one served because upstream failed does not. +// database answered. A tree still fresh in the cache is enough for supported +// but records nothing, since the database was not asked: counting it as an +// answer would keep lookups that fail during an outage from being +// remembered. A tree served because upstream failed is not an answer. func (h *GoHandler) probeSumDB(ctx context.Context, name, cacheKey, sumDBURL string, check func([]byte) error) error { + if entry, _, _, ok := h.readCachedSumDB(ctx, cacheKey, check); ok && h.treeFresh(entry) { + return nil + } res := h.proxy.lookupMetadata(ctx, goEcosystem, cacheKey, sumDBURL+"/latest", "*/*", "identity", check) err := res.err if err == nil && res.stale { diff --git a/internal/handler/go_sumdb_test.go b/internal/handler/go_sumdb_test.go index 98ab7f7..fa23690 100644 --- a/internal/handler/go_sumdb_test.go +++ b/internal/handler/go_sumdb_test.go @@ -511,6 +511,27 @@ func TestGoSumDBDataTileRecords(t *testing.T) { } } +// TestGoSumDB_FreshTreeIsNotAnAnswer: a supported check answered from a +// fresh cached tree did not reach the database, so it must not cancel a +// failure that a lookup started earlier reports afterwards. +func TestGoSumDB_FreshTreeIsNotAnAnswer(t *testing.T) { + db := newFakeSumDB(t, map[string]string{"/latest": testSumDBTree}) + h := newSumDBTestHandler(t, db.server.URL, true, time.Hour) + if w := serveGoRequest(h, "/sumdb/sum.example/supported"); w.Code != http.StatusOK { + t.Fatalf("supported: status = %d, want 200", w.Code) + } + + start := time.Now() + db.down.Store(true) + if w := serveGoRequest(h, "/sumdb/sum.example/supported"); w.Code != http.StatusOK { + t.Fatalf("supported with a fresh cached tree: status = %d, want 200", w.Code) + } + h.sumDBProbes.recordFailure(testSumDBName, start) + if !h.sumDBProbes.recentlyFailed(testSumDBName) { + t.Error("a supported check answered from the cache counted as reaching the database") + } +} + func TestGoSumDBProbes_TimeoutDoesNotOverwriteLaterSuccess(t *testing.T) { var probes goSumDBProbes start := time.Now() From c2ab81031e9752c382ce21d2d230af32c2dd3633 Mon Sep 17 00:00:00 2001 From: Christian Heim Date: Mon, 12 Oct 2026 22:35:23 +0200 Subject: [PATCH 256/633] Move upstream.go_sumdbs_offline next to upstream.go in the configuration table. --- docs/configuration.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/configuration.md b/docs/configuration.md index e0e9047..42a80c0 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -233,11 +233,11 @@ Each upstream used by a built-in package route can be set in YAML or JSON under | `upstream.allow_loopback` | `PROXY_UPSTREAM_ALLOW_LOOPBACK` | `false` | | `upstream.npm` | `PROXY_UPSTREAM_NPM` | `https://registry.npmjs.org` | | `upstream.npm_full_metadata` | `PROXY_UPSTREAM_NPM_FULL_METADATA` | `false` | -| `upstream.go_sumdbs_offline` | `PROXY_UPSTREAM_GO_SUMDBS_OFFLINE` | `false` | | `upstream.cargo` | `PROXY_UPSTREAM_CARGO` | `https://index.crates.io` | | `upstream.cargo_download` | `PROXY_UPSTREAM_CARGO_DOWNLOAD` | `https://static.crates.io/crates` | | `upstream.gem` | `PROXY_UPSTREAM_GEM` | `https://rubygems.org` | | `upstream.go` | `PROXY_UPSTREAM_GO` | `https://proxy.golang.org` | +| `upstream.go_sumdbs_offline` | `PROXY_UPSTREAM_GO_SUMDBS_OFFLINE` | `false` | | `upstream.hex` | `PROXY_UPSTREAM_HEX` | `https://repo.hex.pm` | | `upstream.hex_api` | `PROXY_UPSTREAM_HEX_API` | `https://hex.pm` | | `upstream.pub` | `PROXY_UPSTREAM_PUB` | `https://pub.dev` | From e3cdf73a895969cebc520493c00e97804e2c9256 Mon Sep 17 00:00:00 2001 From: Christian Heim Date: Mon, 12 Oct 2026 23:08:08 +0200 Subject: [PATCH 257/633] Wait for the clock to move on in the fresh-tree test, since on Windows the first probe's success and the start of the lookup can get the same time. --- internal/handler/go_sumdb_test.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/internal/handler/go_sumdb_test.go b/internal/handler/go_sumdb_test.go index fa23690..1cc3cfe 100644 --- a/internal/handler/go_sumdb_test.go +++ b/internal/handler/go_sumdb_test.go @@ -521,7 +521,15 @@ func TestGoSumDB_FreshTreeIsNotAnAnswer(t *testing.T) { t.Fatalf("supported: status = %d, want 200", w.Code) } + // Coarse clocks (Windows) can give the first probe's success and start + // the same time; start must come after it. + h.sumDBProbes.mu.Lock() + succeeded := h.sumDBProbes.succeeded[testSumDBName] + h.sumDBProbes.mu.Unlock() start := time.Now() + for !start.After(succeeded) { + start = time.Now() + } db.down.Store(true) if w := serveGoRequest(h, "/sumdb/sum.example/supported"); w.Code != http.StatusOK { t.Fatalf("supported with a fresh cached tree: status = %d, want 200", w.Code) From 42df87e035054657feef6cbb77bcd95d48f92fa9 Mon Sep 17 00:00:00 2001 From: Christian Heim Date: Tue, 13 Oct 2026 16:23:12 +0200 Subject: [PATCH 258/633] Forward data tiles without storing them. The go command never fetches them, and kept for good they would add up to the whole log; the record count check from before is no longer needed. --- docs/configuration.md | 10 +++-- internal/handler/go_sumdb.go | 29 ++++-------- internal/handler/go_sumdb_test.go | 74 +++++++++++++++---------------- 3 files changed, 51 insertions(+), 62 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 42a80c0..2ff1ab5 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -537,10 +537,12 @@ Lookups for modules that were never cached fail as well. Builds whose `go.sum` already lists every module do not ask the database at all and are not affected. This requires `cache_metadata`. -With `cache_metadata` enabled, lookups, partial tiles and the latest signed -tree follow `metadata_ttl` and are served from cache when the database is -unreachable. Complete tiles never change and are not revalidated once cached; -a tile of the wrong size is refused and not stored. +With `cache_metadata` enabled, lookups, partial hash tiles and the latest +signed tree follow `metadata_ttl` and are served from cache when the database +is unreachable. Complete hash tiles never change and are not revalidated once +cached; a tile of the wrong size is refused and not stored. Data tiles are +always forwarded and never stored: the `go` command does not fetch them, only +mirrors and auditors do. Without `cache_metadata` the requests are forwarded and nothing is stored. A 404 from the database keeps its text, which the `go` command prints to diff --git a/internal/handler/go_sumdb.go b/internal/handler/go_sumdb.go index c74b0c8..020a126 100644 --- a/internal/handler/go_sumdb.go +++ b/internal/handler/go_sumdb.go @@ -135,6 +135,12 @@ func (h *GoHandler) handleSumDB(w http.ResponseWriter, r *http.Request, rest str http.NotFound(w, r) return } + // Only mirrors and auditors fetch data tiles, never the go command, and + // together they hold the whole log, so they are not stored. + if tile, err := tlog.ParseTilePath(path); err == nil && tile.L < 0 { + h.proxy.proxyMetadataStream(w, r, sumDBURL+"/"+path, "identity", "*/*") + return + } cacheKey := goSumDBCacheKey(name, sumDBURL, path) check := goSumDBResponseCheck(path) @@ -275,8 +281,8 @@ const goSumDBTreePrefix = "go.sum database tree\n" // followed by a signed tree, which is how the go command reads it; a block // page cached in its place would also replace the copy kept for an outage. // A hash tile holds exactly W hashes, and complete tiles are never -// revalidated, so a bad one would otherwise stay for good. A data tile holds -// exactly W records. +// revalidated, so a bad one would otherwise stay for good. Data tiles are +// never cached and not checked. func goSumDBResponseCheck(path string) func([]byte) error { if path == "latest" { return func(body []byte) error { @@ -300,30 +306,13 @@ func goSumDBResponseCheck(path string) func([]byte) error { return nil } return func(body []byte) error { - if tile.L >= 0 && len(body) != tile.W*tlog.HashSize { + if len(body) != tile.W*tlog.HashSize { return fmt.Errorf("checksum database tile %s has %d bytes, want %d", path, len(body), tile.W*tlog.HashSize) } - if tile.L < 0 { - if n := goSumDBDataTileRecords(body); n != tile.W { - return fmt.Errorf("checksum database tile %s has %d records, want %d", path, n, tile.W) - } - } return nil } } -// goSumDBDataTileRecords counts the records in a data tile. Each record ends -// with a newline and a blank line separates it from the next; whether one -// follows the last record differs between servers. Anything that does not end -// with a newline counts as no records. -func goSumDBDataTileRecords(body []byte) int { - text, ok := strings.CutSuffix(string(body), "\n") - if !ok || text == "" { - return 0 - } - return strings.Count(text, "\n\n") + 1 -} - // validGoSumDBLookup checks an escaped module@version the way the checksum // database does. Requiring a semantic version also keeps characters such as // '#' and '%' out of the upstream URL. diff --git a/internal/handler/go_sumdb_test.go b/internal/handler/go_sumdb_test.go index 1cc3cfe..6a0c47b 100644 --- a/internal/handler/go_sumdb_test.go +++ b/internal/handler/go_sumdb_test.go @@ -492,25 +492,6 @@ func TestGoSumDB_ShortcutNeedsOfflineMode(t *testing.T) { } } -func TestGoSumDBDataTileRecords(t *testing.T) { - tests := []struct { - name string - body string - want int - }{ - {"sum.golang.org layout", dataTile(3), 3}, - {"blank line after the last record", dataTile(3) + "\n", 3}, - {"one record", dataTile(1), 1}, - {"empty", "", 0}, - {"no final newline", "please log in", 0}, - } - for _, tt := range tests { - if got := goSumDBDataTileRecords([]byte(tt.body)); got != tt.want { - t.Errorf("%s: records = %d, want %d", tt.name, got, tt.want) - } - } -} - // TestGoSumDB_FreshTreeIsNotAnAnswer: a supported check answered from a // fresh cached tree did not reach the database, so it must not cancel a // failure that a lookup started earlier reports afterwards. @@ -653,18 +634,14 @@ func TestGoSumDB_LookupCachedWithinTTL(t *testing.T) { func TestGoSumDB_CompleteTilesSkipRevalidation(t *testing.T) { files := map[string]string{ - "/tile/8/0/000": hashTile(256, "a"), - "/tile/8/data/000": dataTile(256), - "/tile/8/0/001.p/5": hashTile(5, "b"), - "/tile/8/data/001.p/5": dataTile(5), - "/latest": testSumDBTree, + "/tile/8/0/000": hashTile(256, "a"), + "/tile/8/0/001.p/5": hashTile(5, "b"), + "/latest": testSumDBTree, } wantRequests := map[string]int{ - "/tile/8/0/000": 1, - "/tile/8/data/000": 1, - "/tile/8/0/001.p/5": 2, - "/tile/8/data/001.p/5": 2, - "/latest": 2, + "/tile/8/0/000": 1, + "/tile/8/0/001.p/5": 2, + "/latest": 2, } db := newFakeSumDB(t, files) // Every cached entry is past its TTL by the second request. @@ -685,26 +662,47 @@ func TestGoSumDB_CompleteTilesSkipRevalidation(t *testing.T) { } } +// TestGoSumDB_DataTilesAreNotStored: the go command never fetches data +// tiles, and stored they would add up to the whole log. +func TestGoSumDB_DataTilesAreNotStored(t *testing.T) { + files := map[string]string{ + "/tile/8/data/000": dataTile(256), + "/tile/8/data/001.p/5": dataTile(5), + } + db := newFakeSumDB(t, files) + h := newSumDBTestHandler(t, db.server.URL, true, time.Hour) + + for range 2 { + for path, want := range files { + w := serveGoRequest(h, "/sumdb/sum.example"+path) + if w.Code != http.StatusOK || w.Body.String() != want { + t.Fatalf("%s: status = %d, body length %d, want the %d-byte tile", path, w.Code, w.Body.Len(), len(want)) + } + } + } + for path := range files { + if n := db.requests(path); n != 2 { + t.Errorf("%s: database requests = %d, want 2", path, n) + } + key := goSumDBCacheKey(testSumDBName, db.server.URL, strings.TrimPrefix(path, "/")) + if entry, _ := h.proxy.DB.GetMetadataCache(goEcosystem, key); entry != nil { + t.Errorf("%s: stored in the metadata cache", path) + } + } +} + // TestGoSumDB_BadTilesAreNotCached guards complete tiles, which are never // revalidated once cached: a response of the wrong size must not be stored. func TestGoSumDB_BadTilesAreNotCached(t *testing.T) { const tile = "/tile/8/0/000" db := newFakeSumDB(t, map[string]string{ - tile: "please log in", - "/tile/8/data/000.p/3": "", - "/tile/8/data/000": "please log in\n", + tile: "please log in", }) h := newSumDBTestHandler(t, db.server.URL, true, time.Hour) if w := serveGoRequest(h, "/sumdb/sum.example"+tile); w.Code != http.StatusBadGateway { t.Fatalf("wrong-size tile: status = %d, want 502", w.Code) } - if w := serveGoRequest(h, "/sumdb/sum.example/tile/8/data/000.p/3"); w.Code != http.StatusBadGateway { - t.Errorf("empty data tile: status = %d, want 502", w.Code) - } - if w := serveGoRequest(h, "/sumdb/sum.example/tile/8/data/000"); w.Code != http.StatusBadGateway { - t.Errorf("block page as a data tile: status = %d, want 502", w.Code) - } want := hashTile(256, "a") db.setFile(tile, want) From b0570bdf9d5d88afc33f7854a1967c47916f3006 Mon Sep 17 00:00:00 2001 From: Christian Heim Date: Tue, 13 Oct 2026 17:02:32 +0200 Subject: [PATCH 259/633] Mirroring a tag now checks it with a HEAD first and reuses the cached manifest when the digest is unchanged, falls back to the cached tag when the upstream body is cut off, and keeps a tag that a client pull moved while the walk ran. The platform filter is checked across the whole image, so a nested index without the wanted platform no longer fails it, and a 304 for a per-Accept fallback entry only refreshes that client's own keys instead of overwriting the shared entry. --- internal/handler/container_manifest.go | 31 +++-- .../handler/container_manifest_key_test.go | 53 +++++++- internal/handler/container_mirror.go | 110 +++++++++++----- internal/handler/container_mirror_test.go | 123 +++++++++++++++++- 4 files changed, 276 insertions(+), 41 deletions(-) diff --git a/internal/handler/container_manifest.go b/internal/handler/container_manifest.go index 9d8921b..cb390bd 100644 --- a/internal/handler/container_manifest.go +++ b/internal/handler/container_manifest.go @@ -47,7 +47,7 @@ type cachedContainerManifest struct { func (h *ContainerHandler) serveManifest(w http.ResponseWriter, r *http.Request, registryURL, name, reference string) { accept := containerManifestAccept(r) cacheAccept := normalizeContainerManifestAccept(accept) - cached := h.loadContainerManifestForAccept(r.Context(), registryURL, name, reference, accept, cacheAccept) + cached, fromFallback := h.loadContainerManifestForAccept(r.Context(), registryURL, name, reference, accept, cacheAccept) immutable := manifestDigestReferencePattern.MatchString(reference) if cached != nil && (immutable || h.containerManifestFresh(cached)) { @@ -75,7 +75,13 @@ func (h *ContainerHandler) serveManifest(w http.ResponseWriter, r *http.Request, if resp.StatusCode == http.StatusNotModified && cached != nil { cached.fetchedAt = time.Now() - h.storeContainerManifestForAccept(r.Context(), registryURL, name, reference, accept, cacheAccept, cached) + if fromFallback { + // The shared entry holds another representation for other + // clients; only this client's own entries are refreshed. + h.storeContainerManifestLegacyEntries(r.Context(), registryURL, name, reference, accept, cacheAccept, cached) + } else { + h.storeContainerManifestForAccept(r.Context(), registryURL, name, reference, accept, cacheAccept, cached) + } writeContainerManifest(w, r, cached, false) return } @@ -184,15 +190,18 @@ func (h *ContainerHandler) containerManifestLegacyCacheKeys(registryURL, name, r return keys } -func (h *ContainerHandler) loadContainerManifestForAccept(ctx context.Context, registryURL, name, reference, accept, cacheAccept string) *cachedContainerManifest { +// loadContainerManifestForAccept returns the cached manifest for the client and +// whether it came from a per-Accept fallback key while the shared key holds a +// representation the client does not accept. +func (h *ContainerHandler) loadContainerManifestForAccept(ctx context.Context, registryURL, name, reference, accept, cacheAccept string) (*cachedContainerManifest, bool) { cacheKey := h.containerManifestSharedCacheKey(registryURL, name, reference, cacheAccept) cached, err := h.loadContainerManifest(ctx, cacheKey) if err != nil { h.proxy.Logger.Warn("failed to read cached container manifest", "error", err) - return nil + return nil, false } if cached != nil && containerManifestCacheCompatible(accept, cached) { - return cached + return cached, false } // The shared entry may hold a representation this client does not accept, // for example an OCI index for a client that only accepts Docker manifest @@ -203,7 +212,7 @@ func (h *ContainerHandler) loadContainerManifestForAccept(ctx context.Context, r cached, err = h.loadContainerManifest(ctx, legacyCacheKey) if err != nil { h.proxy.Logger.Warn("failed to read legacy cached container manifest", "error", err) - return nil + return nil, false } if cached == nil || !containerManifestCacheCompatible(accept, cached) { continue @@ -213,9 +222,9 @@ func (h *ContainerHandler) loadContainerManifestForAccept(ctx context.Context, r h.proxy.Logger.Warn("failed to migrate cached container manifest", "error", err) } } - return cached + return cached, sharedEntryExists } - return nil + return nil, false } func (h *ContainerHandler) storeContainerManifestForAccept(ctx context.Context, registryURL, name, reference, accept, cacheAccept string, manifest *cachedContainerManifest) { @@ -224,6 +233,12 @@ func (h *ContainerHandler) storeContainerManifestForAccept(ctx context.Context, h.proxy.Logger.Warn("failed to cache container manifest", "error", err) } + h.storeContainerManifestLegacyEntries(ctx, registryURL, name, reference, accept, cacheAccept, manifest) +} + +// storeContainerManifestLegacyEntries writes only the per-Accept keys. +func (h *ContainerHandler) storeContainerManifestLegacyEntries(ctx context.Context, registryURL, name, reference, accept, cacheAccept string, manifest *cachedContainerManifest) { + cacheKey := h.containerManifestSharedCacheKey(registryURL, name, reference, cacheAccept) for _, legacyCacheKey := range h.containerManifestLegacyCacheKeys(registryURL, name, reference, accept, cacheAccept, cacheKey) { if err := h.storeContainerManifest(ctx, legacyCacheKey, manifest); err != nil { h.proxy.Logger.Warn("failed to cache legacy container manifest", "error", err) diff --git a/internal/handler/container_manifest_key_test.go b/internal/handler/container_manifest_key_test.go index 62ceac8..75ad3d5 100644 --- a/internal/handler/container_manifest_key_test.go +++ b/internal/handler/container_manifest_key_test.go @@ -4,6 +4,7 @@ import ( "io" "net/http" "net/http/httptest" + "strings" "sync/atomic" "testing" ) @@ -188,7 +189,55 @@ func TestContainerHandler_IncompatibleSharedEntryFallsBackToAcceptEntry(t *testi if response.Body.String() != list { t.Errorf("offline Docker-only body = %q, want the manifest list", response.Body.String()) } - if response := getManifest(t, h, "/library/app/manifests/latest", ociImageIndexMediaType); response.Body.String() != index { - t.Errorf("offline OCI body = %q, want the index still in the shared entry", response.Body.String()) + if response.Header().Get("Warning") == "" { + t.Error("offline Docker-only response has no stale warning, so it did not come from the stale path") + } + // Serving the fallback entry must not copy it over the shared entry, + // which still holds the index for clients that accept both. + if response := getManifest(t, h, "/library/app/manifests/latest", dockerManifestAccept); response.Body.String() != index { + t.Errorf("offline body for a client accepting both = %q, want the index still in the shared entry", response.Body.String()) + } +} + +func TestContainerHandler_NotModifiedFallbackKeepsSharedEntry(t *testing.T) { + list := `{"schemaVersion":2,"mediaType":"application/vnd.docker.distribution.manifest.list.v2+json","manifests":[]}` + index := `{"schemaVersion":2,"mediaType":"application/vnd.oci.image.index.v1+json","manifests":[]}` + var offline atomic.Bool + // The registry answers per Accept header, with ETags, like registries + // that convert OCI indexes for clients without OCI support. + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if offline.Load() { + http.Error(w, "offline", http.StatusServiceUnavailable) + return + } + body, contentType := list, dockerManifestListMediaType + if strings.Contains(r.Header.Get("Accept"), ociImageIndexMediaType) { + body, contentType = index, ociImageIndexMediaType + } + etag := `"` + sha256Digest([]byte(body)) + `"` + if r.Header.Get("If-None-Match") == etag { + w.WriteHeader(http.StatusNotModified) + return + } + w.Header().Set("Content-Type", contentType) + w.Header().Set("ETag", etag) + _, _ = io.WriteString(w, body) + })) + t.Cleanup(upstream.Close) + proxy, _, _, _ := setupTestProxy(t) + proxy.HTTPClient = upstream.Client() + h := &ContainerHandler{proxy: proxy, registryURL: upstream.URL, proxyURL: "http://localhost:8080"} + + const dockerOnly = "application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.docker.distribution.manifest.v2+json" + getManifest(t, h, "/library/app/manifests/latest", dockerOnly) + getManifest(t, h, "/library/app/manifests/latest", ociImageIndexMediaType) + // Revalidates the Docker-only fallback entry and gets 304. + if response := getManifest(t, h, "/library/app/manifests/latest", dockerOnly); response.Body.String() != list { + t.Fatalf("Docker-only revalidation body = %q, want the list", response.Body.String()) + } + offline.Store(true) + + if response := getManifest(t, h, "/library/app/manifests/latest", dockerManifestAccept); response.Body.String() != index { + t.Errorf("body for a client accepting both = %q, want the index; the 304 refresh overwrote the shared entry", response.Body.String()) } } diff --git a/internal/handler/container_mirror.go b/internal/handler/container_mirror.go index aa0c764..cb98a97 100644 --- a/internal/handler/container_mirror.go +++ b/internal/handler/container_mirror.go @@ -127,6 +127,9 @@ type imageMirror struct { platforms []Platform seen map[string]bool result ImageResult + // platformMatches counts index entries that matched the platform filter + // during the current walk. + platformMatches int } // MirrorImage fetches an image the way a client pull does: the manifest or @@ -214,9 +217,9 @@ var errTagUpstreamUnavailable = errors.New("upstream unavailable") func (m *imageMirror) mirrorTag(ctx context.Context, tag string) (string, error) { accept := containerManifestAccept(&http.Request{Header: http.Header{}}) cacheAccept := normalizeContainerManifestAccept(accept) - before := m.h.loadContainerManifestForAccept(ctx, m.registryURL, m.upstreamName, tag, accept, cacheAccept) + before, _ := m.h.loadContainerManifestForAccept(ctx, m.registryURL, m.upstreamName, tag, accept, cacheAccept) - manifest, err := m.fetchTagManifest(ctx, tag, accept) + manifest, err := m.resolveTag(ctx, tag, accept, before) if err != nil { if before == nil || !errors.Is(err, errTagUpstreamUnavailable) { return "", fmt.Errorf("manifest %s: %w", tag, err) @@ -231,41 +234,53 @@ func (m *imageMirror) mirrorTag(ctx context.Context, tag string) (string, error) digest := manifest.contentDigest if before == nil || before.contentDigest != digest { m.result.Fetched++ + m.h.storeContainerManifestForAccept(ctx, m.registryURL, m.upstreamName, digest, accept, cacheAccept, manifest) } - m.h.storeContainerManifestForAccept(ctx, m.registryURL, m.upstreamName, digest, accept, cacheAccept, manifest) if _, err := m.mirrorManifest(ctx, digest, 0); err != nil { return "", err } + + // A client pull may have moved the tag while the walk ran. Its entry is + // newer than what this mirror resolved, so it is kept. + current, _ := m.h.loadContainerManifestForAccept(ctx, m.registryURL, m.upstreamName, tag, accept, cacheAccept) + if current != nil && current.contentDigest != digest && (before == nil || current.contentDigest != before.contentDigest) { + m.result.Warnings = append(m.result.Warnings, fmt.Sprintf("tag %s moved to %s while it was mirrored; kept that entry", tag, current.contentDigest)) + return digest, nil + } m.h.storeContainerManifestForAccept(ctx, m.registryURL, m.upstreamName, tag, accept, cacheAccept, manifest) return digest, nil } -// fetchTagManifest fetches and verifies the manifest a tag points to without -// caching it. -func (m *imageMirror) fetchTagManifest(ctx context.Context, tag, accept string) (*cachedContainerManifest, error) { - upstreamURL := fmt.Sprintf("%s/v2/%s/manifests/%s", m.registryURL, m.upstreamName, tag) - request, err := http.NewRequestWithContext(ctx, http.MethodGet, upstreamURL, nil) +// resolveTag returns the manifest a tag points to upstream without caching +// it. A HEAD request comes first: if the tag still points to the cached +// digest, the cached manifest is reused with a new fetch time, which spares a +// GET (registries such as Docker Hub do not count HEAD requests against pull +// limits). +func (m *imageMirror) resolveTag(ctx context.Context, tag, accept string, before *cachedContainerManifest) (*cachedContainerManifest, error) { + if before != nil { + response, err := m.requestTag(ctx, http.MethodHead, tag, accept) + if err != nil { + return nil, err + } + _ = response.Body.Close() + if response.Header.Get("Docker-Content-Digest") == before.contentDigest { + unchanged := *before + unchanged.fetchedAt = time.Now() + return &unchanged, nil + } + } + + response, err := m.requestTag(ctx, http.MethodGet, tag, accept) if err != nil { return nil, err } - request.Header.Set("Accept", accept) - response, err := m.h.proxy.HTTPClient.Do(request) + defer func() { _ = response.Body.Close() }() + body, err := m.h.proxy.ReadMetadata(response.Body) if err != nil { if ctx.Err() != nil { return nil, ctx.Err() } - return nil, fmt.Errorf("%w: %w", errTagUpstreamUnavailable, err) - } - defer func() { _ = response.Body.Close() }() - if response.StatusCode != http.StatusOK { - if shouldServeStaleManifest(response.StatusCode) { - return nil, fmt.Errorf("%w: status %d", errTagUpstreamUnavailable, response.StatusCode) - } - return nil, fmt.Errorf("upstream status %d", response.StatusCode) - } - body, err := m.h.proxy.ReadMetadata(response.Body) - if err != nil { - return nil, fmt.Errorf("reading manifest: %w", err) + return nil, fmt.Errorf("%w: reading manifest: %w", errTagUpstreamUnavailable, err) } digest := sha256Digest(body) if contentDigest := response.Header.Get("Docker-Content-Digest"); strings.HasPrefix(contentDigest, "sha256:") && contentDigest != digest { @@ -282,12 +297,38 @@ func (m *imageMirror) fetchTagManifest(ctx context.Context, tag, accept string) }, nil } +// requestTag sends a manifest request for a tag upstream. Only 200 responses +// are returned; network errors, 429 and 5xx wrap errTagUpstreamUnavailable. +func (m *imageMirror) requestTag(ctx context.Context, method, tag, accept string) (*http.Response, error) { + upstreamURL := fmt.Sprintf("%s/v2/%s/manifests/%s", m.registryURL, m.upstreamName, tag) + request, err := http.NewRequestWithContext(ctx, method, upstreamURL, nil) + if err != nil { + return nil, err + } + request.Header.Set("Accept", accept) + response, err := m.h.proxy.HTTPClient.Do(request) + if err != nil { + if ctx.Err() != nil { + return nil, ctx.Err() + } + return nil, fmt.Errorf("%w: %w", errTagUpstreamUnavailable, err) + } + if response.StatusCode != http.StatusOK { + _ = response.Body.Close() + if shouldServeStaleManifest(response.StatusCode) { + return nil, fmt.Errorf("%w: status %d", errTagUpstreamUnavailable, response.StatusCode) + } + return nil, fmt.Errorf("upstream status %d", response.StatusCode) + } + return response, nil +} + // mirrorManifest stores the manifest for reference through the handler's own // manifest route, then follows what it references. It returns the manifest // digest. func (m *imageMirror) mirrorManifest(ctx context.Context, reference string, depth int) (string, error) { cacheAccept := normalizeContainerManifestAccept(containerManifestAccept(&http.Request{Header: http.Header{}})) - before := m.h.loadContainerManifestForAccept(ctx, m.registryURL, m.upstreamName, reference, cacheAccept, cacheAccept) + before, _ := m.h.loadContainerManifestForAccept(ctx, m.registryURL, m.upstreamName, reference, cacheAccept, cacheAccept) target := "/" + m.ref.Name + "/manifests/" + reference if m.ref.Registry != "" { @@ -324,10 +365,21 @@ func (m *imageMirror) mirrorManifest(ctx context.Context, reference string, dept if manifest.SchemaVersion == 1 { return "", fmt.Errorf("manifest %s: schema 1 manifests are not supported", reference) } - if manifest.Manifests != nil { - return digest, m.mirrorIndex(ctx, reference, &manifest, depth) + if manifest.Manifests == nil { + return digest, m.mirrorBlobs(ctx, &manifest) } - return digest, m.mirrorBlobs(ctx, &manifest) + if depth == 0 { + m.platformMatches = 0 + } + if err := m.mirrorIndex(ctx, reference, &manifest, depth); err != nil { + return "", err + } + // A nested index may lack the wanted platforms as long as the image as a + // whole has them. + if depth == 0 && len(m.platforms) > 0 && m.platformMatches == 0 { + return "", fmt.Errorf("index %s: no manifest for the requested platforms", reference) + } + return digest, nil } // mirrorIndex follows the index entries for the wanted platforms. Docker @@ -350,11 +402,11 @@ func (m *imageMirror) mirrorIndex(ctx context.Context, reference string, index * if m.wantsPlatform(entry.Platform) { selected = append(selected, entry) selectedDigests[entry.Digest] = true + if entry.Platform != nil { + m.platformMatches++ + } } } - if len(selected) == 0 && len(m.platforms) > 0 { - return fmt.Errorf("index %s: no manifest for the requested platforms", reference) - } for _, entry := range attestations { if len(m.platforms) == 0 || selectedDigests[entry.Annotations[dockerReferenceDigestAnnotation]] { selected = append(selected, entry) diff --git a/internal/handler/container_mirror_test.go b/internal/handler/container_mirror_test.go index ebd32b6..332851a 100644 --- a/internal/handler/container_mirror_test.go +++ b/internal/handler/container_mirror_test.go @@ -7,6 +7,7 @@ import ( "net/http" "net/http/httptest" "net/url" + "strconv" "strings" "sync/atomic" "testing" @@ -22,6 +23,12 @@ type fakeImageRegistry struct { manifests map[string]fakeManifest blobs map[string]string blobGets atomic.Int32 + // tagGets counts GET requests for manifests by tag. + tagGets atomic.Int32 + // truncateManifests makes manifest GETs end after a few bytes. + truncateManifests atomic.Bool + // onBlob, if set, runs before a blob is served. + onBlob func(path string) } type fakeImage struct { @@ -112,10 +119,21 @@ func newFakeImageRegistry(t *testing.T) (*fakeImageRegistry, fakeImage) { if doc, ok := r.manifests[req.URL.Path]; ok { w.Header().Set("Content-Type", doc.contentType) w.Header().Set("Docker-Content-Digest", sha256Digest([]byte(doc.body))) + if req.Method == http.MethodGet && !strings.Contains(req.URL.Path, "/manifests/sha256:") { + r.tagGets.Add(1) + } + if req.Method == http.MethodGet && r.truncateManifests.Load() { + w.Header().Set("Content-Length", strconv.Itoa(len(doc.body))) + _, _ = io.WriteString(w, doc.body[:4]) + return + } _, _ = io.WriteString(w, doc.body) return } if blob, ok := r.blobs[req.URL.Path]; ok { + if r.onBlob != nil { + r.onBlob(req.URL.Path) + } r.blobGets.Add(1) _, _ = io.WriteString(w, blob) return @@ -318,8 +336,9 @@ func TestMirrorImageKeepsPreviousTagWhenMovedTagIsIncomplete(t *testing.T) { registry.manifests["/v2/team/app/manifests/single"] = brokenDoc registry.manifests["/v2/team/app/manifests/"+sha256Digest(broken)] = brokenDoc - if _, err := h.MirrorImage(context.Background(), ref, nil); err == nil { - t.Fatal("mirroring the incomplete image did not fail") + missing := sha256Digest([]byte("missing layer")) + if _, err := h.MirrorImage(context.Background(), ref, nil); err == nil || !strings.Contains(err.Error(), missing) { + t.Fatalf("mirroring the incomplete image: err = %v, want a failure naming layer %s", err, missing) } registry.offline.Store(true) @@ -331,3 +350,103 @@ func TestMirrorImageKeepsPreviousTagWhenMovedTagIsIncomplete(t *testing.T) { t.Errorf("offline tag points to %s, want the previously mirrored image", sha256Digest(response.Body.Bytes())) } } + +func TestMirrorImagePlatformFilterAcrossNestedIndex(t *testing.T) { + registry, image := newFakeImageRegistry(t) + proxy := newMirrorTestProxy(t, registry) + h := NewContainerHandlerWithRegistry(proxy, "http://proxy.example", registry.server.URL) + + nested := `{"schemaVersion":2,"mediaType":"application/vnd.oci.image.index.v1+json","manifests":[` + + `{"mediaType":"application/vnd.oci.image.manifest.v1+json","digest":"` + image.arm64Digest + `","platform":{"os":"linux","architecture":"arm64"}}]}` + nestedDigest := sha256Digest([]byte(nested)) + top := `{"schemaVersion":2,"mediaType":"application/vnd.oci.image.index.v1+json","manifests":[` + + `{"mediaType":"application/vnd.oci.image.manifest.v1+json","digest":"` + image.amd64Digest + `","platform":{"os":"linux","architecture":"amd64"}},` + + `{"mediaType":"application/vnd.oci.image.index.v1+json","digest":"` + nestedDigest + `"}]}` + for _, reference := range []string{"nested", nestedDigest} { + registry.manifests["/v2/team/app/manifests/"+reference] = fakeManifest{contentType: ociImageIndexMediaType, body: nested} + } + for _, reference := range []string{"top", sha256Digest([]byte(top))} { + registry.manifests["/v2/team/app/manifests/"+reference] = fakeManifest{contentType: ociImageIndexMediaType, body: top} + } + + amd64, _ := ParsePlatform("linux/amd64") + if _, err := h.MirrorImage(context.Background(), ImageRef{Name: "team/app", References: []string{"top"}}, []Platform{amd64}); err != nil { + t.Fatalf("MirrorImage: %v (a nested index without amd64 must not fail the image)", err) + } + riscv, _ := ParsePlatform("linux/riscv64") + if _, err := h.MirrorImage(context.Background(), ImageRef{Name: "team/app", References: []string{"top"}}, []Platform{riscv}); err == nil { + t.Error("a filter matching nothing in the whole image did not fail") + } +} + +func TestMirrorImageRevalidatesUnchangedTagWithHead(t *testing.T) { + registry, _ := newFakeImageRegistry(t) + proxy := newMirrorTestProxy(t, registry) + h := NewContainerHandlerWithRegistry(proxy, "http://proxy.example", registry.server.URL) + ref := ImageRef{Name: "team/app", References: []string{"multi"}} + + if _, err := h.MirrorImage(context.Background(), ref, nil); err != nil { + t.Fatalf("first MirrorImage: %v", err) + } + gets := registry.tagGets.Load() + if _, err := h.MirrorImage(context.Background(), ref, nil); err != nil { + t.Fatalf("second MirrorImage: %v", err) + } + if extra := registry.tagGets.Load() - gets; extra != 0 { + t.Errorf("second run sent %d GETs for the unchanged tag, want a HEAD only", extra) + } +} + +func TestMirrorImageFallsBackWhenTagBodyIsCutOff(t *testing.T) { + registry, image := newFakeImageRegistry(t) + proxy := newMirrorTestProxy(t, registry) + h := NewContainerHandlerWithRegistry(proxy, "http://proxy.example", registry.server.URL) + ref := ImageRef{Name: "team/app", References: []string{"single"}} + + if _, err := h.MirrorImage(context.Background(), ref, nil); err != nil { + t.Fatalf("first MirrorImage: %v", err) + } + // Point the tag elsewhere so the HEAD check sends a GET, then cut that + // GET's body short. + registry.manifests["/v2/team/app/manifests/single"] = registry.manifests["/v2/team/app/manifests/"+image.amd64Digest] + registry.truncateManifests.Store(true) + + result, err := h.MirrorImage(context.Background(), ref, nil) + if err != nil { + t.Fatalf("MirrorImage with a cut-off body: %v, want the cached tag to be used", err) + } + if !strings.Contains(strings.Join(result.Warnings, "\n"), "stale cache") { + t.Errorf("warnings = %v, want the stale-cache warning", result.Warnings) + } +} + +func TestMirrorImageKeepsTagMovedByPullDuringWalk(t *testing.T) { + registry, image := newFakeImageRegistry(t) + proxy := newMirrorTestProxy(t, registry) + h := NewContainerHandlerWithRegistry(proxy, "http://proxy.example", registry.server.URL) + + // While the mirror fetches the first layer of "single", the tag moves + // upstream and a client pull caches the new target. + moved := registry.manifests["/v2/team/app/manifests/"+image.amd64Digest] + var once atomic.Bool + registry.onBlob = func(string) { + if once.CompareAndSwap(false, true) { + registry.manifests["/v2/team/app/manifests/single"] = moved + if response := pull(t, h, "/team/app/manifests/single", dockerManifestAccept); response.Code != http.StatusOK { + t.Errorf("client pull during the walk: status %d", response.Code) + } + } + } + result, err := h.MirrorImage(context.Background(), ImageRef{Name: "team/app", References: []string{"single"}}, nil) + if err != nil { + t.Fatalf("MirrorImage: %v", err) + } + if !strings.Contains(strings.Join(result.Warnings, "\n"), "moved") { + t.Errorf("warnings = %v, want the moved-tag warning", result.Warnings) + } + registry.onBlob = nil + registry.offline.Store(true) + if response := pull(t, h, "/team/app/manifests/single", dockerManifestAccept); response.Body.String() != moved.body { + t.Errorf("offline tag points to %s, want the newer target %s", sha256Digest(response.Body.Bytes()), image.amd64Digest) + } +} From 329780050f908438fcdd0ed19c96c5c24bab5502 Mon Sep 17 00:00:00 2001 From: Christian Heim Date: Sun, 11 Oct 2026 01:12:41 +0200 Subject: [PATCH 260/633] Enable storage retention for deb Packages are stored as pkg:deb/{name}, without a vendor and whichever configured archive they came from. Package rules may name a vendor the way PURLs usually do (pkg:deb/debian/curl, pkg:deb/ubuntu/curl); it's dropped, so a rule covers the package from every archive. The docs row warns that Debian and Ubuntu archives only keep the current version per suite. The mirror can't fetch deb packages, so there's no mirror test. Refs #306 --- docs/configuration.md | 2 +- internal/handler/debian.go | 22 +++ internal/handler/debian_test.go | 30 ++++ internal/server/retention_deb_test.go | 196 ++++++++++++++++++++++++++ 4 files changed, 249 insertions(+), 1 deletion(-) create mode 100644 internal/server/retention_deb_test.go diff --git a/docs/configuration.md b/docs/configuration.md index 89e9141..57d66de 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -95,7 +95,7 @@ A package rule wins over an ecosystem rule, which wins over `default`. Package r | `conan` | yes | `pkg:conan/zlib`, the recipe name; a rule covers every version, revision and user/channel of it. | | `conda` | yes | `pkg:conda/conda-forge/numpy`, with the channel as the namespace; the package name's case doesn't matter. A key without a channel is refused. | | `cran` | yes | `pkg:cran/ggplot2`, the package name as published, case included (`pkg:cran/Rcpp`). A rule covers source and binary builds alike. | -| `deb` | no | - | +| `deb` | yes | `pkg:deb/curl`; case doesn't matter. A vendor namespace such as `pkg:deb/debian/curl` is accepted and dropped, so a rule covers the package from every configured archive. Debian and Ubuntu archives only keep the current version per suite, so an evicted older one can't be fetched again. | | `gem` | yes | `pkg:gem/rails`, the gem name as published, case included. | | `golang` | yes | `pkg:golang/github.com/stretchr/testify`, the module path as in `go.mod`; case doesn't matter. A major version like `/v2` (or `.v3` on gopkg.in) is a module of its own and needs its own rule. The `!`-escaped form from proxy URLs and the module cache (`github.com/!azure/...`) is refused. | | `helm` | yes | No package rules: charts are cached per repository and digest, not under the chart name. The `helm` rule and `default` cover charts from HTTP repositories; charts pulled over OCI fall under `oci`. | diff --git a/internal/handler/debian.go b/internal/handler/debian.go index 7b6635e..dccb823 100644 --- a/internal/handler/debian.go +++ b/internal/handler/debian.go @@ -8,6 +8,9 @@ import ( "regexp" "sort" "strings" + + "github.com/git-pkgs/proxy/internal/retention" + "github.com/git-pkgs/purl" ) const ( @@ -15,6 +18,25 @@ const ( debMatchCount = 4 // full match + name + version + arch ) +// Packages are stored as pkg:deb/{name}, without a vendor and whichever +// configured archive they came from. Package rules may name a vendor the way +// PURLs usually do, pkg:deb/debian/curl or pkg:deb/ubuntu/curl; it is dropped, +// so the rule covers the package from every archive. +func init() { + retention.Default.Register(retention.Spec{Key: "deb", CanonicalPackage: debRetentionKey}) +} + +// debRetentionKey maps a package rule to the stored package PURL. A name with +// a slash, or a namespace with more than one segment, can never match a stored +// package and is refused. +func debRetentionKey(p *purl.PURL) (string, bool) { + if p == nil || p.Type != "deb" || strings.Contains(p.Namespace, "/") || strings.Contains(p.Name, "/") { + return "", false + } + canonical := purl.MakePURLString("deb", p.Name, "") + return canonical, canonical != "" +} + // DebianHandler handles APT/Debian repository protocol requests. // It proxies requests to upstream Debian/Ubuntu repositories and caches .deb packages. // diff --git a/internal/handler/debian_test.go b/internal/handler/debian_test.go index acfd165..b94919f 100644 --- a/internal/handler/debian_test.go +++ b/internal/handler/debian_test.go @@ -8,6 +8,7 @@ import ( "testing" "time" + "github.com/git-pkgs/purl" "github.com/git-pkgs/registries/fetch" ) @@ -258,3 +259,32 @@ func serveDebianRequest(h *DebianHandler, target string) *httptest.ResponseRecor h.Routes().ServeHTTP(w, httptest.NewRequest(http.MethodGet, target, nil)) return w } + +func TestDebRetentionKey(t *testing.T) { + tests := []struct { + key string + want string + ok bool + }{ + {"pkg:deb/curl", "pkg:deb/curl", true}, + {"pkg:deb/debian/curl", "pkg:deb/curl", true}, + {"pkg:deb/ubuntu/Curl", "pkg:deb/curl", true}, + {"pkg:deb/debian/libstdc++6", "pkg:deb/libstdc%2B%2B6", true}, + {"pkg:deb/debian/main/curl", "", false}, + {"pkg:deb/debian/pool%2Fcurl", "", false}, + {"pkg:apk/debian/curl", "", false}, + } + for _, tt := range tests { + p, err := purl.Parse(tt.key) + if err != nil { + t.Fatalf("Parse(%s): %v", tt.key, err) + } + got, ok := debRetentionKey(p) + if ok != tt.ok || (ok && got != tt.want) { + t.Errorf("debRetentionKey(%s) = %q, %v, want %q, %v", tt.key, got, ok, tt.want, tt.ok) + } + } + if _, ok := debRetentionKey(nil); ok { + t.Error("debRetentionKey(nil) accepted") + } +} diff --git a/internal/server/retention_deb_test.go b/internal/server/retention_deb_test.go new file mode 100644 index 0000000..8081224 --- /dev/null +++ b/internal/server/retention_deb_test.go @@ -0,0 +1,196 @@ +package server + +import ( + "context" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/git-pkgs/proxy/internal/config" + "github.com/git-pkgs/proxy/internal/database" + "github.com/git-pkgs/proxy/internal/handler" + "github.com/git-pkgs/proxy/internal/retention" + "github.com/git-pkgs/registries/fetch" +) + +const ( + debCurlPath = "/pool/main/c/curl/curl_8.14.1-2_amd64.deb" + debBusyboxPath = "/pool/main/b/busybox/busybox_1.37.0-6_amd64.deb" + debCurlFile = "curl_8.14.1-2_amd64.deb" + debBusyboxFile = "busybox_1.37.0-6_amd64.deb" +) + +// debRetentionUpstream serves package files the way a Debian archive does, +// counting downloads. +func debRetentionUpstream(t *testing.T) (*httptest.Server, *atomic.Int64) { + t.Helper() + var downloads atomic.Int64 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case debCurlPath, debBusyboxPath: + downloads.Add(1) + _, _ = io.WriteString(w, "deb bytes") + default: + http.NotFound(w, r) + } + })) + t.Cleanup(srv.Close) + return srv, &downloads +} + +// newDebRetentionProxy serves the upstream as the main archive and again as +// a named "security" archive. +func newDebRetentionProxy(t *testing.T, upstream *httptest.Server) (*httptest.Server, *database.DB) { + t.Helper() + db, store := setupEvictionTest(t) + // The upstream is a plain-HTTP loopback server; the default fetcher + // refuses loopback, a stock client reaches it. + fetcher := fetch.NewFetcher(fetch.WithHTTPClient(upstream.Client()), fetch.WithMaxRetries(0)) + t.Cleanup(func() { _ = fetcher.Close() }) + proxy := handler.NewProxy(db, store, fetcher, fetch.NewResolver(), discardLogger()) + proxy.HTTPClient = upstream.Client() + h := handler.NewDebianHandler(proxy, "http://proxy.test", upstream.URL, map[string]string{"security": upstream.URL}) + srv := httptest.NewServer(h.Routes()) + t.Cleanup(srv.Close) + return srv, db +} + +func debGet(t *testing.T, url string) { + t.Helper() + resp, err := http.Get(url) + if err != nil { + t.Fatalf("GET %s: %v", url, err) + } + body, _ := io.ReadAll(resp.Body) + _ = resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Fatalf("GET %s = %d %s, want 200", url, resp.StatusCode, body) + } +} + +// debArtifact returns the record of the file cached under filename, which a +// named archive prefixes with its name. +func debArtifact(t *testing.T, db *database.DB, filename string) database.Artifact { + t.Helper() + var art database.Artifact + if err := db.Get(&art, db.Rebind(`SELECT * FROM artifacts WHERE filename = ?`), filename); err != nil { + t.Fatalf("reading artifact %s: %v", filename, err) + } + return art +} + +func debBackdate(t *testing.T, db *database.DB, filename string, fetched, accessed time.Time) { + t.Helper() + query := db.Rebind(`UPDATE artifacts SET fetched_at = ?, last_accessed_at = ? WHERE filename = ?`) + if _, err := db.Exec(query, fetched, accessed, filename); err != nil { + t.Fatalf("backdating %s: %v", filename, err) + } +} + +func TestDebRegistersForRetention(t *testing.T) { + if _, ok := retention.Default.Lookup("deb"); !ok { + t.Fatal("deb is not registered for retention") + } + if key, ok := retention.Default.KeyForDBEcosystem("deb"); !ok || key != "deb" { + t.Errorf("KeyForDBEcosystem(deb) = %q, %v, want deb, true", key, ok) + } +} + +// A package served from the cache stays while it is used, is evicted once +// its rule runs out, comes back from upstream on the next download, and +// stays after that even with the evicted copy's old access time on the +// record. +func TestDebRetentionEvictsAndRefetches(t *testing.T) { + upstream, downloads := debRetentionUpstream(t) + srv, db := newDebRetentionProxy(t, upstream) + + debGet(t, srv.URL+debCurlPath) + debGet(t, srv.URL+debCurlPath) + if got := downloads.Load(); got != 1 { + t.Fatalf("upstream downloads = %d, want 1", got) + } + if !debArtifact(t, db, debCurlFile).LastAccessedAt.Valid { + t.Fatal("cache hit did not record an access") + } + + rules := retention.Rules{Ecosystems: map[string]time.Duration{"deb": retentionDay}} + now := time.Now() + sweepRetention(context.Background(), db, discardLogger(), retention.Default, rules, now) + if !debArtifact(t, db, debCurlFile).StoragePath.Valid { + t.Fatal("package used today was evicted") + } + + old := now.Add(-2 * retentionDay) + debBackdate(t, db, debCurlFile, old, old) + sweepRetention(context.Background(), db, discardLogger(), retention.Default, rules, now) + if debArtifact(t, db, debCurlFile).StoragePath.Valid { + t.Fatal("package unused for two days kept under a one-day rule") + } + + debGet(t, srv.URL+debCurlPath) + if got := downloads.Load(); got != 2 { + t.Errorf("upstream downloads = %d, want 2 after the eviction", got) + } + art := debArtifact(t, db, debCurlFile) + debBackdate(t, db, debCurlFile, art.FetchedAt.Time, now.Add(-30*retentionDay)) + sweepRetention(context.Background(), db, discardLogger(), retention.Default, rules, time.Now()) + if !debArtifact(t, db, debCurlFile).StoragePath.Valid { + t.Error("refetched package evicted again by the next sweep") + } +} + +// Keys that parse as deb PURLs but can never match a stored package are +// refused at validation. Keys with and without a vendor are accepted. +func TestDebRetentionRefusesUnmatchableKeys(t *testing.T) { + cfg := config.Default() + cfg.Storage.Retention.Packages = map[string]string{"pkg:deb/curl": "0", "pkg:deb/ubuntu/busybox": "0"} + if _, err := cfg.RetentionRules(retention.Default); err != nil { + t.Fatalf("RetentionRules: %v", err) + } + cfg.Storage.Retention.Packages = map[string]string{"pkg:deb/debian/main/curl": "0"} + _, err := cfg.RetentionRules(retention.Default) + if err == nil || !strings.Contains(err.Error(), "does not match how deb packages are stored") { + t.Errorf("RetentionRules(pkg:deb/debian/main/curl) error = %v, want the key refused", err) + } +} + +// A package rule reaches the package in the main archive and in a named +// one, with or without a vendor. +func TestDebRetentionPackageRules(t *testing.T) { + upstream, _ := debRetentionUpstream(t) + srv, db := newDebRetentionProxy(t, upstream) + files := []struct { + path, filename string + kept bool + }{ + {debCurlPath, debCurlFile, true}, + {"/security" + debCurlPath, "security/" + debCurlFile, true}, + {debBusyboxPath, debBusyboxFile, false}, + } + + for _, key := range []string{"pkg:deb/curl", "pkg:deb/debian/curl"} { + old := time.Now().Add(-30 * retentionDay) + for _, f := range files { + debGet(t, srv.URL+f.path) + debBackdate(t, db, f.filename, old, old) + } + cfg := config.Default() + cfg.Storage.Retention.Ecosystems = map[string]string{"deb": "1d"} + cfg.Storage.Retention.Packages = map[string]string{key: "0"} + rules, err := cfg.RetentionRules(retention.Default) + if err != nil { + t.Fatalf("RetentionRules(%s): %v", key, err) + } + sweepRetention(context.Background(), db, discardLogger(), retention.Default, rules, time.Now()) + + for _, f := range files { + if got := debArtifact(t, db, f.filename).StoragePath.Valid; got != f.kept { + t.Errorf("with %s: %s cached = %v, want %v", key, f.filename, got, f.kept) + } + } + } +} From d7a6a891e389266bb9740d0d143daa348cef003d Mon Sep 17 00:00:00 2001 From: simon chrzanowski Date: Wed, 14 Oct 2026 08:36:21 +0200 Subject: [PATCH 261/633] Abort blob writes when the download fails gocloud commits whatever a writer received when it is closed, unless its context was cancelled first. Store closed the writer after a failed copy, so every download that broke off midway left a partial object under its fetch path. No database row points at it, so it was never served, but nothing deleted it either and the orphans kept taking disk space. - Cancel the writer's context before closing it when the copy fails - Add a test storing from a reader that errors midway --- internal/storage/blob.go | 8 +++++++- internal/storage/blob_test.go | 38 +++++++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 1 deletion(-) diff --git a/internal/storage/blob.go b/internal/storage/blob.go index 4cadf8a..7a34d7d 100644 --- a/internal/storage/blob.go +++ b/internal/storage/blob.go @@ -207,14 +207,20 @@ func (b *Blob) Store(ctx context.Context, path string, r io.Reader) (int64, stri h := sha256.New() tee := io.TeeReader(r, h) + // gocloud commits whatever was written when the writer closes, unless + // its context is cancelled first, so a failed copy cancels it to abort. + writeCtx, abort := context.WithCancel(ctx) + defer abort() + opts := &blob.WriterOptions{} - w, err := b.bucket.NewWriter(ctx, path, opts) + w, err := b.bucket.NewWriter(writeCtx, path, opts) if err != nil { return 0, "", fmt.Errorf("creating writer: %w", err) } size, err := io.Copy(w, tee) if err != nil { + abort() _ = w.Close() return 0, "", fmt.Errorf("writing content: %w", err) } diff --git a/internal/storage/blob_test.go b/internal/storage/blob_test.go index efc1b76..5533174 100644 --- a/internal/storage/blob_test.go +++ b/internal/storage/blob_test.go @@ -660,3 +660,41 @@ func TestStoreSucceedsWhenSidecarCannotBeRemoved(t *testing.T) { t.Errorf("Store failed because cleanup could not complete: %v", err) } } + +type failingReader struct { + data []byte + err error +} + +func (r *failingReader) Read(p []byte) (int, error) { + if len(r.data) == 0 { + return 0, r.err + } + n := copy(p, r.data) + r.data = r.data[n:] + return n, nil +} + +// A download that breaks off midway must not leave its first half behind as +// an object, or every failed fetch adds an orphan nothing ever deletes. +func TestBlobStoreFailedCopyLeavesNoObject(t *testing.T) { + b := createTestBlob(t) + ctx := context.Background() + readErr := errors.New("connection reset") + + _, _, err := b.Store(ctx, "npm/lodash/4.17.21/lodash.tgz", &failingReader{ + data: []byte(strings.Repeat("partial", 1024)), + err: readErr, + }) + if !errors.Is(err, readErr) { + t.Fatalf("Store error = %v, want %v", err, readErr) + } + + exists, err := b.Exists(ctx, "npm/lodash/4.17.21/lodash.tgz") + if err != nil { + t.Fatalf("Exists failed: %v", err) + } + if exists { + t.Error("failed Store left a partial object behind") + } +} From 03c8e680cc8fc002fc18f045af764f0edfa5de47 Mon Sep 17 00:00:00 2001 From: simon chrzanowski Date: Wed, 14 Oct 2026 08:37:39 +0200 Subject: [PATCH 262/633] Withhold Composer archives still in cooldown Cooldown only filtered the metadata. A composer.lock names the proxy's /files URL directly, so `composer install` fetched versions still inside the cooldown window without ever seeing the filtered metadata. npm already checks the publish time on tarball downloads. - Return the version's publish time with its dist URL, following the minified format's inheritance - Answer 404 for a version in cooldown before fetching its archive - Read the download lookup's metadata through ReadMetadata, so it has the same size limit as other metadata reads --- internal/handler/composer.go | 85 +++++++++++-------- .../handler/composer_rewrite_bench_test.go | 6 +- internal/handler/composer_rewrite_test.go | 8 +- internal/handler/composer_test.go | 76 ++++++++++++++++- 4 files changed, 128 insertions(+), 47 deletions(-) diff --git a/internal/handler/composer.go b/internal/handler/composer.go index 53d0772..5b9ab96 100644 --- a/internal/handler/composer.go +++ b/internal/handler/composer.go @@ -6,7 +6,6 @@ import ( "encoding/json" "errors" "fmt" - "io" "net/http" "net/url" "path" @@ -576,21 +575,21 @@ func (h *ComposerHandler) handleDownload(w http.ResponseWriter, r *http.Request) "package", packageName, "version", version, "metadata_urls", metaURLs) - var downloadURL, distType string + var dist composerDist for _, metaURL := range metaURLs { - url, typ, err := h.findDownloadURLFromMetadata(r.Context(), metaURL, packageName, version) + found, err := h.findDownloadURLFromMetadata(r.Context(), metaURL, packageName, version) if err != nil { h.proxy.Logger.Error("failed to fetch metadata", "error", err, "url", metaURL) http.Error(w, "failed to fetch metadata", http.StatusBadGateway) return } - if url != "" { - downloadURL, distType = url, typ + if found.url != "" { + dist = found break } } - if downloadURL == "" { + if dist.url == "" { h.proxy.Logger.Debug("version not found in any metadata source", "package", packageName, "version", version, "tried_urls", metaURLs) @@ -598,6 +597,13 @@ func (h *ComposerHandler) handleDownload(w http.ResponseWriter, r *http.Request) return } + // Lock files name archive URLs directly, so a version filtered out of the + // metadata still has to be withheld here. + if h.shouldFilterVersion(canonicalPackagePURL("composer", packageName), packageName, version, dist.published) { + http.Error(w, "version not found", http.StatusNotFound) + return + } + // The archive is cached under the file name from the path, so fetch the // one that name stands for. Compare it as sent, the way rewriteDist handed // it out, query included. @@ -605,8 +611,8 @@ func (h *ComposerHandler) handleDownload(w http.ResponseWriter, r *http.Request) if r.URL.RawQuery != "" { requested += "?" + r.URL.RawQuery } - listedURL := downloadURL - downloadURL, ok := upstreamDistURL(listedURL, distType, requested, isDevVersion(version)) + listedURL := dist.url + downloadURL, ok := upstreamDistURL(listedURL, dist.distType, requested, isDevVersion(version)) if !ok { h.proxy.Logger.Info("composer file not in upstream metadata", "package", packageName, "version", version, @@ -696,23 +702,29 @@ func (h *ComposerHandler) metadataURLsForVersion(vendor, pkg, version string) [] return []string{stable, dev} } +// composerDist is what a version's metadata says about its archive: the dist +// URL and type, and the raw publish time the cooldown is checked against. +type composerDist struct { + url, distType string + published []byte +} + // findDownloadURLFromMetadata fetches a metadata document and returns the dist -// URL and type for the given version, or an empty URL if the version is not -// present. An error is returned only on transport failure; a missing document -// (non-200) or a missing version both yield an empty URL so the caller can -// fall back. -func (h *ComposerHandler) findDownloadURLFromMetadata(ctx context.Context, metaURL, packageName, version string) (string, string, error) { +// of the given version, with an empty URL if the version is not present. An +// error is returned only on transport failure; a missing document (non-200) +// or a missing version both yield an empty URL so the caller can fall back. +func (h *ComposerHandler) findDownloadURLFromMetadata(ctx context.Context, metaURL, packageName, version string) (composerDist, error) { h.proxy.Logger.Debug("fetching upstream metadata for download lookup", "url", metaURL, "package", packageName, "version", version) req, err := http.NewRequestWithContext(ctx, http.MethodGet, metaURL, nil) if err != nil { - return "", "", err + return composerDist{}, err } resp, err := h.proxy.HTTPClient.Do(req) if err != nil { - return "", "", err + return composerDist{}, err } defer func() { _ = resp.Body.Close() }() @@ -720,42 +732,42 @@ func (h *ComposerHandler) findDownloadURLFromMetadata(ctx context.Context, metaU "url", metaURL, "status", resp.StatusCode) if resp.StatusCode != http.StatusOK { - return "", "", nil + return composerDist{}, nil } - body, err := io.ReadAll(resp.Body) + body, err := h.proxy.ReadMetadata(resp.Body) if err != nil { - return "", "", err + return composerDist{}, err } - url, distType, err := composerDistURL(body, packageName, version) + dist, err := composerDistURL(body, packageName, version) if err != nil { - return "", "", err + return composerDist{}, err } h.proxy.Logger.Debug("download URL lookup result", "url", metaURL, "package", packageName, "version", version, - "download_url", url) - return url, distType, nil + "download_url", dist.url) + return dist, nil } -// composerDistURL returns the upstream dist URL and type of version from -// Composer metadata, or "" when the package has no such version or it has no -// dist URL. It walks the version list in place, expanding only as much of the -// minified format as it needs, since it runs on each archive the proxy has not -// cached yet. -func composerDistURL(body []byte, packageName, version string) (string, string, error) { +// composerDistURL returns the upstream dist of version from Composer +// metadata, with an empty URL when the package has no such version or it has +// no dist URL. It walks the version list in place, expanding only as much of +// the minified format as it needs, since it runs on each archive the proxy +// has not cached yet. +func composerDistURL(body []byte, packageName, version string) (composerDist, error) { format, _, err := lookupJSONString(body, "minified") if err != nil { - return "", "", err + return composerDist{}, err } versions, err := lookupJSON(body, "packages", packageName) if err != nil || len(versions) == 0 || versions[0] != '[' { - return "", "", err + return composerDist{}, err } minified := format == composerMinified fields := newComposerFields() - var url, distType string + var dist composerDist err = forEachJSONElement(versions, func(entry []byte) error { if minified && jsonStringIs(entry, composerDevReset) { fields.reset() @@ -773,10 +785,11 @@ func composerDistURL(body []byte, packageName, version string) (string, string, if !jsonStringIs(fields.get("version"), version) { return nil } - dist := fields.get("dist") - url, _, _ = lookupJSONString(dist, "url") - if url != "" { - distType, _, _ = lookupJSONString(dist, "type") + distRaw := fields.get("dist") + dist.url, _, _ = lookupJSONString(distRaw, "url") + if dist.url != "" { + dist.distType, _, _ = lookupJSONString(distRaw, "type") + dist.published = fields.get("time") return errStopScan } return nil @@ -784,7 +797,7 @@ func composerDistURL(body []byte, packageName, version string) (string, string, if errors.Is(err, errStopScan) { err = nil } - return url, distType, err + return dist, err } // proxyUpstream forwards a request to packagist.org without caching. diff --git a/internal/handler/composer_rewrite_bench_test.go b/internal/handler/composer_rewrite_bench_test.go index 9f0ab7b..1fc8c5a 100644 --- a/internal/handler/composer_rewrite_bench_test.go +++ b/internal/handler/composer_rewrite_bench_test.go @@ -73,9 +73,9 @@ func BenchmarkComposerFindDownloadURL(b *testing.B) { b.SetBytes(int64(len(body))) b.ReportAllocs() for b.Loop() { - url, _, err := h.findDownloadURLFromMetadata(context.Background(), srv.URL, "big/sdk", "3.2500.0") - if err != nil || url == "" { - b.Fatalf("download URL not found: %q, %v", url, err) + dist, err := h.findDownloadURLFromMetadata(context.Background(), srv.URL, "big/sdk", "3.2500.0") + if err != nil || dist.url == "" { + b.Fatalf("download URL not found: %q, %v", dist.url, err) } } } diff --git a/internal/handler/composer_rewrite_test.go b/internal/handler/composer_rewrite_test.go index c484e1a..1eb6f7f 100644 --- a/internal/handler/composer_rewrite_test.go +++ b/internal/handler/composer_rewrite_test.go @@ -240,19 +240,19 @@ func TestComposerDistURL(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, _, err := composerDistURL([]byte(tt.body), "v/p", tt.version) + got, err := composerDistURL([]byte(tt.body), "v/p", tt.version) if err != nil { t.Fatal(err) } - if got != tt.want { - t.Errorf("composerDistURL(%s) = %q, want %q", tt.version, got, tt.want) + if got.url != tt.want { + t.Errorf("composerDistURL(%s) = %q, want %q", tt.version, got.url, tt.want) } }) } } func TestComposerDistURLMalformed(t *testing.T) { - if _, _, err := composerDistURL([]byte(`{"packages":{"v/p":[{"version":`), "v/p", "1.0.0"); err == nil { + if _, err := composerDistURL([]byte(`{"packages":{"v/p":[{"version":`), "v/p", "1.0.0"); err == nil { t.Error("expected an error for truncated JSON") } } diff --git a/internal/handler/composer_test.go b/internal/handler/composer_test.go index be672cc..e6a9962 100644 --- a/internal/handler/composer_test.go +++ b/internal/handler/composer_test.go @@ -494,13 +494,13 @@ func TestComposerDownloadDevVersionUsesDevMetadata(t *testing.T) { // OLD behavior: fetching only the regular file fails to resolve the dev // version, which is what produced the 404 before the fix. stableURL := srv.URL + "/p2/phpmd/phpmd.json" - got, _, err := h.findDownloadURLFromMetadata(ctx, stableURL, pkg, version) + found, err := h.findDownloadURLFromMetadata(ctx, stableURL, pkg, version) if err != nil { t.Fatalf("unexpected error fetching regular metadata: %v", err) } - if got != "" { + if found.url != "" { t.Fatalf("regular metadata unexpectedly contained dev version %q (got %q); "+ - "the test no longer reproduces the original bug", version, got) + "the test no longer reproduces the original bug", version, found.url) } // FIXED behavior: the handler consults the ~dev file (it is first in the @@ -512,10 +512,11 @@ func TestComposerDownloadDevVersionUsesDevMetadata(t *testing.T) { var resolved string for _, u := range urls { - resolved, _, err = h.findDownloadURLFromMetadata(ctx, u, pkg, version) + found, err := h.findDownloadURLFromMetadata(ctx, u, pkg, version) if err != nil { t.Fatalf("unexpected error fetching metadata %q: %v", u, err) } + resolved = found.url if resolved != "" { break } @@ -1007,3 +1008,70 @@ func TestComposerRetentionKey(t *testing.T) { t.Error("composerRetentionKey(nil) accepted") } } + +// TestComposerDownloadCooldown checks that an archive still in cooldown is +// withheld when a lock file requests it directly, without the metadata that +// would have filtered it out. In minified metadata a version inherits the +// time of the entry before it. +func TestComposerDownloadCooldown(t *testing.T) { + now := time.Now() + old := now.Add(-10 * 24 * time.Hour).Format(time.RFC3339) + recent := now.Add(-1 * time.Hour).Format(time.RFC3339) + + var archiveRequests atomic.Int32 + var srv *httptest.Server + srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + dist := func(version string) string { + return fmt.Sprintf(`"dist":{"url":%q,"type":"zip"}`, srv.URL+"/archives/"+version+".zip") + } + switch r.URL.Path { + case "/p2/v/p.json": + _, _ = fmt.Fprintf(w, `{"minified":"composer/2.0","packages":{"v/p":[`+ + `{"version":"2.0.0","time":%q,%s},`+ + `{"version":"1.9.0",%s},`+ + `{"version":"1.0.0","time":%q,%s}]}}`, + recent, dist("2.0.0"), dist("1.9.0"), old, dist("1.0.0")) + case "/p2/v/p~dev.json": + http.NotFound(w, r) + default: + archiveRequests.Add(1) + _, _ = io.WriteString(w, "archive "+r.URL.Path) + } + })) + t.Cleanup(srv.Close) + + proxy, _, _, _ := setupTestProxy(t) + proxy.Cooldown = &cooldown.Config{Default: "3d"} + fetcher := fetch.NewFetcher(fetch.WithHTTPClient(srv.Client()), fetch.WithMaxRetries(0)) + proxy.Fetcher = fetcher + t.Cleanup(func() { _ = fetcher.Close() }) + h := NewComposerHandlerWithUpstreams(proxy, "http://proxy", srv.URL, srv.URL).Routes() + + tests := []struct { + version string + wantStatus int + }{ + {"2.0.0", http.StatusNotFound}, + {"1.9.0", http.StatusNotFound}, + {"1.0.0", http.StatusOK}, + } + for _, tt := range tests { + t.Run(tt.version, func(t *testing.T) { + archiveRequests.Store(0) + req := httptest.NewRequest(http.MethodGet, "/files/v/p/"+tt.version+"/"+tt.version+".zip", nil) + w := httptest.NewRecorder() + h.ServeHTTP(w, req) + + if w.Code != tt.wantStatus { + t.Fatalf("status = %d, want %d (body %q)", w.Code, tt.wantStatus, w.Body.String()) + } + wantRequests := int32(0) + if tt.wantStatus == http.StatusOK { + wantRequests = 1 + } + if got := archiveRequests.Load(); got != wantRequests { + t.Errorf("upstream archive requests = %d, want %d", got, wantRequests) + } + }) + } +} From f5e7599749ce47b2a65d172497c94851509b32dd Mon Sep 17 00:00:00 2001 From: simon chrzanowski Date: Wed, 14 Oct 2026 08:37:40 +0200 Subject: [PATCH 263/633] Stop interpolating archive paths into inline handlers The source browser built its file tree with onclick handlers such as loadFile('${escapeHTML(file.path)}'). The browser decodes HTML entities in an attribute before running the handler, so the ' that escapeHTML produces turns back into a quote and ends the string. A cached package with a crafted file name ran script on the proxy origin for anyone who browsed it. - Carry the path in data-path and dispatch clicks from one delegated listener on the file tree - Render the binary download button as a link instead of an onclick that assigns window.location - Fail the template tests when an inline handler interpolates data --- .../server/templates/pages/browse_source.html | 27 ++++++++++++---- internal/server/templates_handlers_test.go | 32 +++++++++++++++++++ 2 files changed, 53 insertions(+), 6 deletions(-) create mode 100644 internal/server/templates_handlers_test.go diff --git a/internal/server/templates/pages/browse_source.html b/internal/server/templates/pages/browse_source.html index a949111..0d5332a 100644 --- a/internal/server/templates/pages/browse_source.html +++ b/internal/server/templates/pages/browse_source.html @@ -104,7 +104,7 @@ function renderFileTree(files, basePath) { const parentPath = basePath.split('/').slice(0, -2).join('/'); html += `
+ data-action="open-dir" data-path="${escapeHTML(parentPath)}"> ..
`; @@ -119,13 +119,13 @@ function renderFileTree(files, basePath) { if (file.is_dir) { html += ` -
+
${iconHTML}${escapeHTML(file.name)}
`; } else { html += ` -
+
${iconHTML}${escapeHTML(file.name)} ${formatSize(file.size)}
@@ -179,10 +179,10 @@ async function loadFile(path) { container.innerHTML = `

Binary file (${formatSize(content.length)})

- +
`; } @@ -207,6 +207,21 @@ function formatSize(bytes) { return (bytes / (1024 * 1024)).toFixed(1) + ' MB'; } +// Entries carry their path in data-path rather than in inline handlers: the +// browser decodes entities in a handler attribute before running it, so an +// escaped quote in a file name would end the handler's string. +document.getElementById('file-tree').addEventListener('click', (event) => { + const entry = event.target.closest('[data-action]'); + if (!entry) return; + const path = entry.dataset.path; + if (entry.dataset.action === 'open-dir') { + loadFileTree(path); + currentPath = path; + } else if (entry.dataset.action === 'open-file') { + loadFile(path); + } +}); + // Load root directory on page load loadFileTree(); diff --git a/internal/server/templates_handlers_test.go b/internal/server/templates_handlers_test.go new file mode 100644 index 0000000..1e3f1f2 --- /dev/null +++ b/internal/server/templates_handlers_test.go @@ -0,0 +1,32 @@ +package server + +import ( + "io/fs" + "regexp" + "testing" +) + +// inlineHandlerWithData matches an inline event handler attribute whose value +// interpolates a JavaScript template expression. The browser decodes HTML +// entities in the attribute before running the handler, so escapeHTML cannot +// keep a quote in the interpolated value from ending the handler's string. +var inlineHandlerWithData = regexp.MustCompile(`\son[a-z]+\s*=\s*"[^"]*\$\{`) + +func TestTemplatesInterpolateNoDataIntoInlineHandlers(t *testing.T) { + err := fs.WalkDir(templatesFS, "templates", func(path string, d fs.DirEntry, err error) error { + if err != nil || d.IsDir() { + return err + } + source, err := fs.ReadFile(templatesFS, path) + if err != nil { + return err + } + for _, match := range inlineHandlerWithData.FindAll(source, -1) { + t.Errorf("%s: inline handler interpolates data: %s", path, match) + } + return nil + }) + if err != nil { + t.Fatal(err) + } +} From 832c800ea5dda86a194faac30185ba13cab3c314 Mon Sep 17 00:00:00 2001 From: simon chrzanowski Date: Wed, 14 Oct 2026 08:37:43 +0200 Subject: [PATCH 264/633] Serve diff2html from the proxy instead of jsDelivr The compare page loaded diff2html from cdn.jsdelivr.net without a version or integrity hash, so whatever jsDelivr served as the latest release ran with the proxy origin's privileges, and the page broke where the proxy cannot reach the internet. - Vendor diff2html 3.4.56 (MIT) under static/vendor like tailwind and lucide - Load its script and stylesheet from /ui/static/vendor - Fail a test when any template loads a script or stylesheet from another origin --- internal/server/server_test.go | 2 ++ .../server/static/vendor/diff2html-ui.min.js | 2 ++ .../server/static/vendor/diff2html.min.css | 2 ++ .../templates/pages/compare_versions.html | 4 +-- internal/server/templates_test.go | 27 +++++++++++++++++++ 5 files changed, 35 insertions(+), 2 deletions(-) create mode 100644 internal/server/static/vendor/diff2html-ui.min.js create mode 100644 internal/server/static/vendor/diff2html.min.css diff --git a/internal/server/server_test.go b/internal/server/server_test.go index 06b43fe..5109d23 100644 --- a/internal/server/server_test.go +++ b/internal/server/server_test.go @@ -761,6 +761,8 @@ func TestStaticFiles(t *testing.T) { }{ {"/ui/static/vendor/tailwind.js", []string{"text/javascript", "application/javascript"}}, {"/ui/static/vendor/lucide.min.js", []string{"text/javascript", "application/javascript"}}, + {"/ui/static/vendor/diff2html-ui.min.js", []string{"text/javascript", "application/javascript"}}, + {"/ui/static/vendor/diff2html.min.css", []string{"text/css"}}, {"/ui/static/style.css", []string{"text/css"}}, } diff --git a/internal/server/static/vendor/diff2html-ui.min.js b/internal/server/static/vendor/diff2html-ui.min.js new file mode 100644 index 0000000..e098ea6 --- /dev/null +++ b/internal/server/static/vendor/diff2html-ui.min.js @@ -0,0 +1,2 @@ +/*! diff2html v3.4.56 | MIT License | Copyright 2014-2016 Rodrigo Fernandes | https://github.com/rtfpessoa/diff2html */ +!function(e,t){if("object"==typeof exports&&"object"==typeof module)module.exports=t();else if("function"==typeof define&&define.amd)define([],t);else{var n=t();for(var a in n)("object"==typeof exports?exports:e)[a]=n[a]}}(this,(()=>{return e={78(e){e.exports=function(e){const t="[a-z][a-zA-Z0-9_]*",n={className:"string",begin:"\\$.{1}"},a={className:"symbol",begin:"#"+e.UNDERSCORE_IDENT_RE};return{name:"Smalltalk",aliases:["st"],keywords:["self","super","nil","true","false","thisContext"],contains:[e.COMMENT('"','"'),e.APOS_STRING_MODE,{className:"type",begin:"\\b[A-Z][A-Za-z0-9_]*",relevance:0},{begin:t+":",relevance:0},e.C_NUMBER_MODE,a,n,{begin:"\\|[ ]*"+t+"([ ]+"+t+")*[ ]*\\|",returnBegin:!0,end:/\|/,illegal:/\S/,contains:[{begin:"(\\|[ ]*)?"+t}]},{begin:"#\\(",end:"\\)",contains:[e.APOS_STRING_MODE,n,e.C_NUMBER_MODE,a]}]}}},92(e,t,n){"use strict";Object.defineProperty(t,"__esModule",{value:!0}),t.Diff2HtmlUI=t.defaultDiff2HtmlUIConfig=void 0;const a=n(4206),r=n(6166);t.defaultDiff2HtmlUIConfig=Object.assign(Object.assign({},r.defaultDiff2HtmlConfig),{synchronisedScroll:!0,highlight:!0,fileListToggle:!0,fileListStartVisible:!1,highlightLanguages:new Map,smartSelection:!0,fileContentToggle:!0,stickyFileHeaders:!0}),t.Diff2HtmlUI=class{constructor(e,n,a={},i){this.hljs=null,this.currentSelectionColumnId=-1,this.config=Object.assign(Object.assign({},t.defaultDiff2HtmlUIConfig),a),this.diffHtml=void 0!==n?(0,r.html)(n,this.config):e.innerHTML,this.targetElement=e,void 0!==i&&(this.hljs=i)}draw(){this.targetElement.innerHTML=this.diffHtml,this.config.synchronisedScroll&&this.synchronisedScroll(),this.config.highlight&&this.highlightCode(),this.config.fileListToggle&&this.fileListToggle(this.config.fileListStartVisible),this.config.fileContentToggle&&this.fileContentToggle(),this.config.stickyFileHeaders&&this.stickyFileHeaders()}synchronisedScroll(){this.targetElement.querySelectorAll(".d2h-file-wrapper").forEach((e=>{const[t,n]=Array().slice.call(e.querySelectorAll(".d2h-file-side-diff"));if(void 0===t||void 0===n)return;const a=e=>{null!==e&&null!==e.target&&(e.target===t?(n.scrollTop=t.scrollTop,n.scrollLeft=t.scrollLeft):(t.scrollTop=n.scrollTop,t.scrollLeft=n.scrollLeft))};t.addEventListener("scroll",a),n.addEventListener("scroll",a)}))}fileListToggle(e){const t=this.targetElement.querySelector(".d2h-show"),n=this.targetElement.querySelector(".d2h-hide"),a=this.targetElement.querySelector(".d2h-file-list");if(null===t||null===n||null===a)return;const r=()=>{t.style.display="none",n.style.display="inline",a.style.display="block"},i=()=>{t.style.display="inline",n.style.display="none",a.style.display="none"};t.addEventListener("click",(()=>r())),n.addEventListener("click",(()=>i()));const o=this.getHashTag();"files-summary-show"===o?r():"files-summary-hide"===o?i():e?r():i()}fileContentToggle(){this.targetElement.querySelectorAll(".d2h-file-collapse").forEach((e=>{e.style.display="flex";const t=t=>{var n;const a=null===(n=e.closest(".d2h-file-wrapper"))||void 0===n?void 0:n.querySelector(t);null!=a&&(e.classList.toggle("d2h-selected"),a.classList.toggle("d2h-d-none"))};e.addEventListener("click",(n=>(n=>{e!==n.target&&(t(".d2h-file-diff"),t(".d2h-files-diff"))})(n)))}))}highlightCode(){const e=this.hljs;if(null===e)throw new Error("Missing a `highlight.js` implementation. Please provide one when instantiating Diff2HtmlUI.");this.targetElement.querySelectorAll(".d2h-file-wrapper").forEach((t=>{const n=t.getAttribute("data-lang");this.config.highlightLanguages instanceof Map||(this.config.highlightLanguages=new Map(Object.entries(this.config.highlightLanguages)));let r=n&&this.config.highlightLanguages.has(n)?this.config.highlightLanguages.get(n):n?(0,a.getLanguage)(n):"plaintext";void 0===e.getLanguage(r)&&(r="plaintext"),t.querySelectorAll(".d2h-code-line-ctn").forEach((t=>{const n=t.textContent,i=t.parentNode;if(null===n||null===i||!this.isElement(i))return;const o=(0,a.closeTags)(e.highlight(n,{language:r,ignoreIllegals:!0})),s=(0,a.nodeStream)(t);if(s.length){const e=document.createElementNS("http://www.w3.org/1999/xhtml","div");e.innerHTML=o.value,o.value=(0,a.mergeStreams)(s,(0,a.nodeStream)(e),n)}t.classList.add("hljs"),o.language&&t.classList.add(o.language),t.innerHTML=o.value}))}))}stickyFileHeaders(){this.targetElement.querySelectorAll(".d2h-file-header").forEach((e=>{e.classList.add("d2h-sticky-header")}))}smartSelection(){console.warn("Smart selection is now enabled by default with CSS. No need to call this method anymore.")}getHashTag(){const e=document.URL,t=e.indexOf("#");let n=null;return-1!==t&&(n=e.substr(t+1)),n}isElement(e){return null!==e&&void 0!==(null==e?void 0:e.classList)}}},114(e){e.exports=function(e){const t=e.regex,n=t.concat(/[\p{L}_]/u,t.optional(/[\p{L}0-9_.-]*:/u),/[\p{L}0-9_.-]*/u),a={className:"symbol",begin:/&[a-z]+;|&#[0-9]+;|&#x[a-f0-9]+;/},r={begin:/\s/,contains:[{className:"keyword",begin:/#?[a-z_][a-z1-9_-]+/,illegal:/\n/}]},i=e.inherit(r,{begin:/\(/,end:/\)/}),o=e.inherit(e.APOS_STRING_MODE,{className:"string"}),s=e.inherit(e.QUOTE_STRING_MODE,{className:"string"}),l={endsWithParent:!0,illegal:/`]+/}]}]}]};return{name:"HTML, XML",aliases:["html","xhtml","rss","atom","xjb","xsd","xsl","plist","wsf","svg"],case_insensitive:!0,unicodeRegex:!0,contains:[{className:"meta",begin://,relevance:10,contains:[r,s,o,i,{begin:/\[/,end:/\]/,contains:[{className:"meta",begin://,contains:[r,i,s,o]}]}]},e.COMMENT(//,{relevance:10}),{begin://,relevance:10},a,{className:"meta",end:/\?>/,variants:[{begin:/<\?xml/,relevance:10,contains:[s]},{begin:/<\?[a-z][a-z0-9]+/}]},{className:"tag",begin:/)/,end:/>/,keywords:{name:"style"},contains:[l],starts:{end:/<\/style>/,returnEnd:!0,subLanguage:["css","xml"]}},{className:"tag",begin:/)/,end:/>/,keywords:{name:"script"},contains:[l],starts:{end:/<\/script>/,returnEnd:!0,subLanguage:["javascript","handlebars","xml"]}},{className:"tag",begin:/<>|<\/>/},{className:"tag",begin:t.concat(//,/>/,/\s/)))),end:/\/?>/,contains:[{className:"name",begin:n,relevance:0,starts:l}]},{className:"tag",begin:t.concat(/<\//,t.lookahead(t.concat(n,/>/))),contains:[{className:"name",begin:n,relevance:0},{begin:/>/,relevance:0,endsParent:!0}]}]}}},140(e){e.exports=function(e){const t=e.regex,n={className:"variable.constant",begin:t.concat(/\$/,t.either("ADMINTOOLS","APPDATA","CDBURN_AREA","CMDLINE","COMMONFILES32","COMMONFILES64","COMMONFILES","COOKIES","DESKTOP","DOCUMENTS","EXEDIR","EXEFILE","EXEPATH","FAVORITES","FONTS","HISTORY","HWNDPARENT","INSTDIR","INTERNET_CACHE","LANGUAGE","LOCALAPPDATA","MUSIC","NETHOOD","OUTDIR","PICTURES","PLUGINSDIR","PRINTHOOD","PROFILE","PROGRAMFILES32","PROGRAMFILES64","PROGRAMFILES","QUICKLAUNCH","RECENT","RESOURCES_LOCALIZED","RESOURCES","SENDTO","SMPROGRAMS","SMSTARTUP","STARTMENU","SYSDIR","TEMP","TEMPLATES","VIDEOS","WINDIR"))},a={className:"variable",begin:/\$+\{[\!\w.:-]+\}/},r={className:"variable",begin:/\$+\w[\w\.]*/,illegal:/\(\)\{\}/},i={className:"variable",begin:/\$+\([\w^.:!-]+\)/},o={className:"params",begin:t.either("ARCHIVE","FILE_ATTRIBUTE_ARCHIVE","FILE_ATTRIBUTE_NORMAL","FILE_ATTRIBUTE_OFFLINE","FILE_ATTRIBUTE_READONLY","FILE_ATTRIBUTE_SYSTEM","FILE_ATTRIBUTE_TEMPORARY","HKCR","HKCU","HKDD","HKEY_CLASSES_ROOT","HKEY_CURRENT_CONFIG","HKEY_CURRENT_USER","HKEY_DYN_DATA","HKEY_LOCAL_MACHINE","HKEY_PERFORMANCE_DATA","HKEY_USERS","HKLM","HKPD","HKU","IDABORT","IDCANCEL","IDIGNORE","IDNO","IDOK","IDRETRY","IDYES","MB_ABORTRETRYIGNORE","MB_DEFBUTTON1","MB_DEFBUTTON2","MB_DEFBUTTON3","MB_DEFBUTTON4","MB_ICONEXCLAMATION","MB_ICONINFORMATION","MB_ICONQUESTION","MB_ICONSTOP","MB_OK","MB_OKCANCEL","MB_RETRYCANCEL","MB_RIGHT","MB_RTLREADING","MB_SETFOREGROUND","MB_TOPMOST","MB_USERICON","MB_YESNO","NORMAL","OFFLINE","READONLY","SHCTX","SHELL_CONTEXT","SYSTEM|TEMPORARY")},s={className:"keyword",begin:t.concat(/!/,t.either("addincludedir","addplugindir","appendfile","assert","cd","define","delfile","echo","else","endif","error","execute","finalize","getdllversion","gettlbversion","if","ifdef","ifmacrodef","ifmacrondef","ifndef","include","insertmacro","macro","macroend","makensis","packhdr","searchparse","searchreplace","system","tempfile","undef","uninstfinalize","verbose","warning"))},l={className:"string",variants:[{begin:'"',end:'"'},{begin:"'",end:"'"},{begin:"`",end:"`"}],illegal:/\n/,contains:[{className:"char.escape",begin:/\$(\\[nrt]|\$)/},n,a,r,i]},c={match:[/Function/,/\s+/,t.concat(/(\.)?/,e.IDENT_RE)],scope:{1:"keyword",3:"title.function"}};return{name:"NSIS",case_insensitive:!0,keywords:{keyword:["Abort","AddBrandingImage","AddSize","AllowRootDirInstall","AllowSkipFiles","AutoCloseWindow","BGFont","BGGradient","BrandingText","BringToFront","Call","CallInstDLL","Caption","ChangeUI","CheckBitmap","ClearErrors","CompletedText","ComponentText","CopyFiles","CRCCheck","CreateDirectory","CreateFont","CreateShortCut","Delete","DeleteINISec","DeleteINIStr","DeleteRegKey","DeleteRegValue","DetailPrint","DetailsButtonText","DirText","DirVar","DirVerify","EnableWindow","EnumRegKey","EnumRegValue","Exch","Exec","ExecShell","ExecShellWait","ExecWait","ExpandEnvStrings","File","FileBufSize","FileClose","FileErrorText","FileOpen","FileRead","FileReadByte","FileReadUTF16LE","FileReadWord","FileWriteUTF16LE","FileSeek","FileWrite","FileWriteByte","FileWriteWord","FindClose","FindFirst","FindNext","FindWindow","FlushINI","GetCurInstType","GetCurrentAddress","GetDlgItem","GetDLLVersion","GetDLLVersionLocal","GetErrorLevel","GetFileTime","GetFileTimeLocal","GetFullPathName","GetFunctionAddress","GetInstDirError","GetKnownFolderPath","GetLabelAddress","GetTempFileName","GetWinVer","Goto","HideWindow","Icon","IfAbort","IfErrors","IfFileExists","IfRebootFlag","IfRtlLanguage","IfShellVarContextAll","IfSilent","InitPluginsDir","InstallButtonText","InstallColors","InstallDir","InstallDirRegKey","InstProgressFlags","InstType","InstTypeGetText","InstTypeSetText","Int64Cmp","Int64CmpU","Int64Fmt","IntCmp","IntCmpU","IntFmt","IntOp","IntPtrCmp","IntPtrCmpU","IntPtrOp","IsWindow","LangString","LicenseBkColor","LicenseData","LicenseForceSelection","LicenseLangString","LicenseText","LoadAndSetImage","LoadLanguageFile","LockWindow","LogSet","LogText","ManifestDPIAware","ManifestLongPathAware","ManifestMaxVersionTested","ManifestSupportedOS","MessageBox","MiscButtonText","Name|0","Nop","OutFile","Page","PageCallbacks","PEAddResource","PEDllCharacteristics","PERemoveResource","PESubsysVer","Pop","Push","Quit","ReadEnvStr","ReadINIStr","ReadRegDWORD","ReadRegStr","Reboot","RegDLL","Rename","RequestExecutionLevel","ReserveFile","Return","RMDir","SearchPath","SectionGetFlags","SectionGetInstTypes","SectionGetSize","SectionGetText","SectionIn","SectionSetFlags","SectionSetInstTypes","SectionSetSize","SectionSetText","SendMessage","SetAutoClose","SetBrandingImage","SetCompress","SetCompressor","SetCompressorDictSize","SetCtlColors","SetCurInstType","SetDatablockOptimize","SetDateSave","SetDetailsPrint","SetDetailsView","SetErrorLevel","SetErrors","SetFileAttributes","SetFont","SetOutPath","SetOverwrite","SetRebootFlag","SetRegView","SetShellVarContext","SetSilent","ShowInstDetails","ShowUninstDetails","ShowWindow","SilentInstall","SilentUnInstall","Sleep","SpaceTexts","StrCmp","StrCmpS","StrCpy","StrLen","SubCaption","Unicode","UninstallButtonText","UninstallCaption","UninstallIcon","UninstallSubCaption","UninstallText","UninstPage","UnRegDLL","Var","VIAddVersionKey","VIFileVersion","VIProductVersion","WindowIcon","WriteINIStr","WriteRegBin","WriteRegDWORD","WriteRegExpandStr","WriteRegMultiStr","WriteRegNone","WriteRegStr","WriteUninstaller","XPStyle"],literal:["admin","all","auto","both","bottom","bzip2","colored","components","current","custom","directory","false","force","hide","highest","ifdiff","ifnewer","instfiles","lastused","leave","left","license","listonly","lzma","nevershow","none","normal","notset","off","on","open","print","right","show","silent","silentlog","smooth","textonly","top","true","try","un.components","un.custom","un.directory","un.instfiles","un.license","uninstConfirm","user","Win10","Win7","Win8","WinVista","zlib"]},contains:[e.HASH_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE,e.COMMENT(";","$",{relevance:0}),{match:[/Var/,/\s+/,/(?:\/GLOBAL\s+)?/,/[A-Za-z][\w.]*/],scope:{1:"keyword",3:"params",4:"variable"}},c,{beginKeywords:"Function PageEx Section SectionGroup FunctionEnd SectionEnd"},l,s,a,r,i,o,{className:"title.function",begin:/\w+::\w+/},e.NUMBER_MODE]}}},150(e){e.exports=function(e){return{name:"Tagger Script",contains:[{className:"comment",begin:/\$noop\(/,end:/\)/,contains:[{begin:/\\[()]/},{begin:/\(/,end:/\)/,contains:[{begin:/\\[()]/},"self"]}],relevance:10},{className:"keyword",begin:/\$[_a-zA-Z0-9]+(?=\()/},{className:"variable",begin:/%[_a-zA-Z0-9:]+%/},{className:"symbol",begin:/\\[\\nt$%,()]/},{className:"symbol",begin:/\\u[a-fA-F0-9]{4}/}]}}},151(e,t,n){"use strict";var a,r=this&&this.__createBinding||(Object.create?function(e,t,n,a){void 0===a&&(a=n);var r=Object.getOwnPropertyDescriptor(t,n);r&&!("get"in r?!t.__esModule:r.writable||r.configurable)||(r={enumerable:!0,get:function(){return t[n]}}),Object.defineProperty(e,a,r)}:function(e,t,n,a){void 0===a&&(a=n),e[a]=t[n]}),i=this&&this.__setModuleDefault||(Object.create?function(e,t){Object.defineProperty(e,"default",{enumerable:!0,value:t})}:function(e,t){e.default=t}),o=this&&this.__importStar||(a=function(e){return a=Object.getOwnPropertyNames||function(e){var t=[];for(var n in e)Object.prototype.hasOwnProperty.call(e,n)&&(t[t.length]=n);return t},a(e)},function(e){if(e&&e.__esModule)return e;var t={};if(null!=e)for(var n=a(e),o=0;o{let t;return t=e.blocks.length?this.generateFileHtml(e):this.generateEmptyDiff(),this.makeFileDiffHtml(e,t)})).join("\n");return this.hoganUtils.render(d,"wrapper",{colorScheme:l.colorSchemeToCss(this.config.colorScheme),content:t})}makeFileDiffHtml(e,t){if(this.config.renderNothingWhenEmpty&&Array.isArray(e.blocks)&&0===e.blocks.length)return"";const n=this.hoganUtils.template("side-by-side","file-diff"),a=this.hoganUtils.template(d,"file-path"),r=this.hoganUtils.template("icon","file"),i=this.hoganUtils.template("tag",l.getFileIcon(e));return n.render({file:e,fileHtmlId:l.getHtmlId(e),diffs:t,filePath:a.render({fileDiffName:l.filenameDiff(e)},{fileIcon:r,fileTag:i})})}generateEmptyDiff(){return{right:"",left:this.hoganUtils.render(d,"empty-diff",{contentClass:"d2h-code-side-line",CSSLineClass:l.CSSLineClass})}}generateFileHtml(e){const t=s.newMatcherFn(s.newDistanceFn((t=>l.deconstructLine(t.content,e.isCombined).content)));return e.blocks.map((n=>{const a={left:this.makeHeaderHtml(n.header,e),right:this.makeHeaderHtml("")};return this.applyLineGroupping(n).forEach((([n,r,i])=>{if(r.length&&i.length&&!n.length)this.applyRematchMatching(r,i,t).map((([t,n])=>{const{left:r,right:i}=this.processChangedLines(e.isCombined,t,n);a.left+=r,a.right+=i}));else if(n.length)n.forEach((t=>{const{prefix:n,content:r}=l.deconstructLine(t.content,e.isCombined),{left:i,right:o}=this.generateLineHtml({type:l.CSSLineClass.CONTEXT,prefix:n,content:r,number:t.oldNumber},{type:l.CSSLineClass.CONTEXT,prefix:n,content:r,number:t.newNumber});a.left+=i,a.right+=o}));else if(r.length||i.length){const{left:t,right:n}=this.processChangedLines(e.isCombined,r,i);a.left+=t,a.right+=n}else console.error("Unknown state reached while processing groups of lines",n,r,i)})),a})).reduce(((e,t)=>({left:e.left+t.left,right:e.right+t.right})),{left:"",right:""})}applyLineGroupping(e){const t=[];let n=[],a=[];for(let r=0;r0)&&(t.push([[],n,a]),n=[],a=[]),i.type===c.LineType.CONTEXT?t.push([[i],[],[]]):i.type===c.LineType.INSERT&&0===n.length?t.push([[],[],[i]]):i.type===c.LineType.INSERT&&n.length>0?a.push(i):i.type===c.LineType.DELETE&&n.push(i)}return(n.length||a.length)&&(t.push([[],n,a]),n=[],a=[]),t}applyRematchMatching(e,t,n){const a=e.length*t.length,r=(0,_.max)(e.concat(t).map((e=>e.content.length)));return a",subLanguage:"vbscript"}]}}},222(e){e.exports=function(e){const t=e.regex,n=["absolute_url","asset|0","asset_version","attribute","block","constant","controller|0","country_timezones","csrf_token","cycle","date","dump","expression","form|0","form_end","form_errors","form_help","form_label","form_rest","form_row","form_start","form_widget","html_classes","include","is_granted","logout_path","logout_url","max","min","parent","path|0","random","range","relative_path","render","render_esi","source","template_from_string","url|0"];let a=["apply","autoescape","block","cache","deprecated","do","embed","extends","filter","flush","for","form_theme","from","if","import","include","macro","sandbox","set","stopwatch","trans","trans_default_domain","transchoice","use","verbatim","with"];a=a.concat(a.map((e=>`end${e}`)));const r={scope:"string",variants:[{begin:/'/,end:/'/},{begin:/"/,end:/"/}]},i={scope:"number",match:/\d+/},o={begin:/\(/,end:/\)/,excludeBegin:!0,excludeEnd:!0,contains:[r,i]},s={beginKeywords:n.join(" "),keywords:{name:n},relevance:0,contains:[o]},l={match:/\|(?=[A-Za-z_]+:?)/,beginScope:"punctuation",relevance:0,contains:[{match:/[A-Za-z_]+:?/,keywords:["abs","abbr_class","abbr_method","batch","capitalize","column","convert_encoding","country_name","currency_name","currency_symbol","data_uri","date","date_modify","default","escape","file_excerpt","file_link","file_relative","filter","first","format","format_args","format_args_as_text","format_currency","format_date","format_datetime","format_file","format_file_from_text","format_number","format_time","html_to_markdown","humanize","inky_to_html","inline_css","join","json_encode","keys","language_name","last","length","locale_name","lower","map","markdown","markdown_to_html","merge","nl2br","number_format","raw","reduce","replace","reverse","round","slice","slug","sort","spaceless","split","striptags","timezone_name","title","trans","transchoice","trim","u|0","upper","url_encode","yaml_dump","yaml_encode"]}]},c=(e,{relevance:n})=>({beginScope:{1:"template-tag",3:"name"},relevance:n||2,endScope:"template-tag",begin:[/\{%/,/\s*/,t.either(...e)],end:/%\}/,keywords:"in",contains:[l,s,r,i]}),_=c(a,{relevance:2}),d=c([/[a-z_]+/],{relevance:1});return{name:"Twig",aliases:["craftcms"],case_insensitive:!0,subLanguage:"xml",contains:[e.COMMENT(/\{#/,/#\}/),_,d,{className:"template-variable",begin:/\{\{/,end:/\}\}/,contains:["self",l,s,r,i]}]}}},253(e){function t(e,t={}){return t.variants=e,t}e.exports=function(e){const n=e.regex,a="[A-Za-z0-9_$]+",r=t([e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE,e.COMMENT("/\\*\\*","\\*/",{relevance:0,contains:[{begin:/\w+@/,relevance:0},{className:"doctag",begin:"@[A-Za-z]+"}]})]),i={className:"regexp",begin:/~?\/[^\/\n]+\//,contains:[e.BACKSLASH_ESCAPE]},o=t([e.BINARY_NUMBER_MODE,e.C_NUMBER_MODE]),s=t([{begin:/"""/,end:/"""/},{begin:/'''/,end:/'''/},{begin:"\\$/",end:"/\\$",relevance:10},e.APOS_STRING_MODE,e.QUOTE_STRING_MODE],{className:"string"}),l={match:[/(class|interface|trait|enum|record|extends|implements)/,/\s+/,e.UNDERSCORE_IDENT_RE],scope:{1:"keyword",3:"title.class"}};return{name:"Groovy",keywords:{"variable.language":"this super",literal:"true false null",type:["byte","short","char","int","long","boolean","float","double","void"],keyword:["def","as","in","assert","trait","abstract","static","volatile","transient","public","private","protected","synchronized","final","class","interface","enum","if","else","for","while","switch","case","break","default","continue","throw","throws","try","catch","finally","implements","extends","new","import","package","return","instanceof","var"]},contains:[e.SHEBANG({binary:"groovy",relevance:10}),r,s,i,o,l,{className:"meta",begin:"@[A-Za-z]+",relevance:0},{className:"attr",begin:a+"[ \t]*:",relevance:0},{begin:/\?/,end:/:/,relevance:0,contains:[r,s,i,o,"self"]},{className:"symbol",begin:"^[ \t]*"+n.lookahead(a+":"),excludeBegin:!0,end:a+":",relevance:0}],illegal:/#|<\//}}},283(e){e.exports=function(e){return{aliases:["pycon"],contains:[{className:"meta.prompt",starts:{end:/ |$/,starts:{end:"$",subLanguage:"python"}},variants:[{begin:/^>>>(?=[ ]|$)/},{begin:/^\.\.\.(?=[ ]|$)/}]}]}}},315(e){e.exports=function(e){const t=e.regex,n=e.COMMENT("--","$"),a=["abs","acos","array_agg","asin","atan","avg","cast","ceil","ceiling","coalesce","corr","cos","cosh","count","covar_pop","covar_samp","cume_dist","dense_rank","deref","element","exp","extract","first_value","floor","json_array","json_arrayagg","json_exists","json_object","json_objectagg","json_query","json_table","json_table_primitive","json_value","lag","last_value","lead","listagg","ln","log","log10","lower","max","min","mod","nth_value","ntile","nullif","percent_rank","percentile_cont","percentile_disc","position","position_regex","power","rank","regr_avgx","regr_avgy","regr_count","regr_intercept","regr_r2","regr_slope","regr_sxx","regr_sxy","regr_syy","row_number","sin","sinh","sqrt","stddev_pop","stddev_samp","substring","substring_regex","sum","tan","tanh","translate","translate_regex","treat","trim","trim_array","unnest","upper","value_of","var_pop","var_samp","width_bucket"],r=a,i=["abs","acos","all","allocate","alter","and","any","are","array","array_agg","array_max_cardinality","as","asensitive","asin","asymmetric","at","atan","atomic","authorization","avg","begin","begin_frame","begin_partition","between","bigint","binary","blob","boolean","both","by","call","called","cardinality","cascaded","case","cast","ceil","ceiling","char","char_length","character","character_length","check","classifier","clob","close","coalesce","collate","collect","column","commit","condition","connect","constraint","contains","convert","copy","corr","corresponding","cos","cosh","count","covar_pop","covar_samp","create","cross","cube","cume_dist","current","current_catalog","current_date","current_default_transform_group","current_path","current_role","current_row","current_schema","current_time","current_timestamp","current_path","current_role","current_transform_group_for_type","current_user","cursor","cycle","date","day","deallocate","dec","decimal","decfloat","declare","default","define","delete","dense_rank","deref","describe","deterministic","disconnect","distinct","double","drop","dynamic","each","element","else","empty","end","end_frame","end_partition","end-exec","equals","escape","every","except","exec","execute","exists","exp","external","extract","false","fetch","filter","first_value","float","floor","for","foreign","frame_row","free","from","full","function","fusion","get","global","grant","group","grouping","groups","having","hold","hour","identity","in","indicator","initial","inner","inout","insensitive","insert","int","integer","intersect","intersection","interval","into","is","join","json_array","json_arrayagg","json_exists","json_object","json_objectagg","json_query","json_table","json_table_primitive","json_value","lag","language","large","last_value","lateral","lead","leading","left","like","like_regex","listagg","ln","local","localtime","localtimestamp","log","log10","lower","match","match_number","match_recognize","matches","max","member","merge","method","min","minute","mod","modifies","module","month","multiset","national","natural","nchar","nclob","new","no","none","normalize","not","nth_value","ntile","null","nullif","numeric","octet_length","occurrences_regex","of","offset","old","omit","on","one","only","open","or","order","out","outer","over","overlaps","overlay","parameter","partition","pattern","per","percent","percent_rank","percentile_cont","percentile_disc","period","portion","position","position_regex","power","precedes","precision","prepare","primary","procedure","ptf","range","rank","reads","real","recursive","ref","references","referencing","regr_avgx","regr_avgy","regr_count","regr_intercept","regr_r2","regr_slope","regr_sxx","regr_sxy","regr_syy","release","result","return","returns","revoke","right","rollback","rollup","row","row_number","rows","running","savepoint","scope","scroll","search","second","seek","select","sensitive","session_user","set","show","similar","sin","sinh","skip","smallint","some","specific","specifictype","sql","sqlexception","sqlstate","sqlwarning","sqrt","start","static","stddev_pop","stddev_samp","submultiset","subset","substring","substring_regex","succeeds","sum","symmetric","system","system_time","system_user","table","tablesample","tan","tanh","then","time","timestamp","timezone_hour","timezone_minute","to","trailing","translate","translate_regex","translation","treat","trigger","trim","trim_array","true","truncate","uescape","union","unique","unknown","unnest","update","upper","user","using","value","values","value_of","var_pop","var_samp","varbinary","varchar","varying","versioning","when","whenever","where","width_bucket","window","with","within","without","year","add","asc","collation","desc","final","first","last","view"].filter((e=>!a.includes(e))),o={match:t.concat(/\b/,t.either(...r),/\s*\(/),relevance:0,keywords:{built_in:r}};function s(e){return t.concat(/\b/,t.either(...e.map((e=>e.replace(/\s+/,"\\s+")))),/\b/)}const l={scope:"keyword",match:s(["create table","insert into","primary key","foreign key","not null","alter table","add constraint","grouping sets","on overflow","character set","respect nulls","ignore nulls","nulls first","nulls last","depth first","breadth first"]),relevance:0};return{name:"SQL",case_insensitive:!0,illegal:/[{}]|<\//,keywords:{$pattern:/\b[\w\.]+/,keyword:function(e,{exceptions:t,when:n}={}){const a=n;return t=t||[],e.map((e=>e.match(/\|\d+$/)||t.includes(e)?e:a(e)?`${e}|0`:e))}(i,{when:e=>e.length<3}),literal:["true","false","unknown"],type:["bigint","binary","blob","boolean","char","character","clob","date","dec","decfloat","decimal","float","int","integer","interval","nchar","nclob","national","numeric","real","row","smallint","time","timestamp","varchar","varying","varbinary"],built_in:["current_catalog","current_date","current_default_transform_group","current_path","current_role","current_schema","current_transform_group_for_type","current_user","session_user","system_time","system_user","current_time","localtime","current_timestamp","localtimestamp"]},contains:[{scope:"type",match:s(["double precision","large object","with timezone","without timezone"])},l,o,{scope:"variable",match:/@[a-z0-9][a-z0-9_]*/},{scope:"string",variants:[{begin:/'/,end:/'/,contains:[{match:/''/}]}]},{begin:/"/,end:/"/,contains:[{match:/""/}]},e.C_NUMBER_MODE,e.C_BLOCK_COMMENT_MODE,n,{scope:"operator",match:/[-+*/=%^~]|&&?|\|\|?|!=?|<(?:=>?|<|>)?|>[>=]?/,relevance:0}]}}},320(e){e.exports=function(e){const t=e.regex,n={keyword:["assert","else","if","in","inherit","let","or","rec","then","with"],literal:["true","false","null"],built_in:["abort","baseNameOf","builtins","derivation","derivationStrict","dirOf","fetchGit","fetchMercurial","fetchTarball","fetchTree","fromTOML","import","isNull","map","placeholder","removeAttrs","scopedImport","throw","toString"]},a={scope:"built_in",match:t.either(...["abort","add","addDrvOutputDependencies","addErrorContext","all","any","appendContext","attrNames","attrValues","baseNameOf","bitAnd","bitOr","bitXor","break","builtins","catAttrs","ceil","compareVersions","concatLists","concatMap","concatStringsSep","convertHash","currentSystem","currentTime","deepSeq","derivation","derivationStrict","dirOf","div","elem","elemAt","false","fetchGit","fetchMercurial","fetchTarball","fetchTree","fetchurl","filter","filterSource","findFile","flakeRefToString","floor","foldl'","fromJSON","fromTOML","functionArgs","genList","genericClosure","getAttr","getContext","getEnv","getFlake","groupBy","hasAttr","hasContext","hashFile","hashString","head","import","intersectAttrs","isAttrs","isBool","isFloat","isFunction","isInt","isList","isNull","isPath","isString","langVersion","length","lessThan","listToAttrs","map","mapAttrs","match","mul","nixPath","nixVersion","null","parseDrvName","parseFlakeRef","partition","path","pathExists","placeholder","readDir","readFile","readFileType","removeAttrs","replaceStrings","scopedImport","seq","sort","split","splitVersion","storeDir","storePath","stringLength","sub","substring","tail","throw","toFile","toJSON","toPath","toString","toXML","trace","traceVerbose","true","tryEval","typeOf","unsafeDiscardOutputDependency","unsafeDiscardStringContext","unsafeGetAttrPos","warn","zipAttrsWith"].map((e=>`builtins\\.${e}`))),relevance:10},r="[A-Za-z_][A-Za-z0-9_'-]*",i={scope:"symbol",match:new RegExp(`<${r}(/${r})*>`)},o="[A-Za-z0-9_\\+\\.-]+",s={scope:"symbol",match:new RegExp(`(\\.\\.|\\.|~)?/(${o})?(/${o})*(?=[\\s;])`)},l=t.either("==","=","\\+\\+","\\+","<=","<\\|","<",">=",">","->","//","/","!=","!","\\|\\|","\\|>","\\?","\\*","&&"),c={scope:"operator",match:t.concat(l,/(?!-)/),relevance:0},_={scope:"number",match:new RegExp(`${e.NUMBER_RE}(?!-)`),relevance:0},d={variants:[{scope:"operator",beforeMatch:/\s/,begin:/-(?!>)/},{begin:[new RegExp(`${e.NUMBER_RE}`),/-/,/(?!>)/],beginScope:{1:"number",2:"operator"}},{begin:[l,/-/,/(?!>)/],beginScope:{1:"operator",2:"operator"}}],relevance:0},p={beforeMatch:/(^|\{|;)\s*/,begin:new RegExp(`${r}(\\.${r})*\\s*=(?!=)`),returnBegin:!0,relevance:0,contains:[{scope:"attr",match:new RegExp(`${r}(\\.${r})*(?=\\s*=)`),relevance:.2}]},u={scope:"subst",begin:/\$\{/,end:/\}/,keywords:n},m={scope:"char.escape",match:/\\(?!\$)./},g={scope:"string",variants:[{begin:"''",end:"''",contains:[{scope:"char.escape",match:/''\$/},u,{scope:"char.escape",match:/'''/},m]},{begin:'"',end:'"',contains:[{scope:"char.escape",match:/\\\$/},u,m]}]},E={scope:"params",match:new RegExp(`${r}\\s*:(?=\\s)`)},S=[_,e.HASH_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE,e.COMMENT(/\/\*\*(?!\/)/,/\*\//,{subLanguage:"markdown",relevance:0}),a,g,i,s,E,p,d,c];return u.contains=S,{name:"Nix",aliases:["nixos"],keywords:n,contains:S.concat([{scope:"meta.prompt",match:/^nix-repl>(?=\s)/,relevance:10},{scope:"meta",beforeMatch:/\s+/,begin:/:([a-z]+|\?)/}])}}},379(e){e.exports=function(e){const t="([0-9]_*)+",n="([0-9a-fA-F]_*)+",a="([!#$%&*+.\\/<=>?@\\\\^~-]|(?!([(),;\\[\\]`|{}]|[_:\"']))(\\p{S}|\\p{P}))",r={variants:[e.COMMENT("--+","$"),e.COMMENT(/\{-/,/-\}/,{contains:["self"]})]},i={className:"meta",begin:/\{-#/,end:/#-\}/},o={className:"meta",begin:"^#",end:"$"},s={className:"type",begin:"\\b[A-Z][\\w']*",relevance:0},l={begin:"\\(",end:"\\)",illegal:'"',contains:[i,o,{className:"type",begin:"\\b[A-Z][\\w]*(\\((\\.\\.|,|\\w+)\\))?"},e.inherit(e.TITLE_MODE,{begin:"[_a-z][\\w']*"}),r]},c={className:"number",relevance:0,variants:[{match:`\\b(${t})(\\.(${t}))?([eE][+-]?(${t}))?\\b`},{match:`\\b0[xX]_*(${n})(\\.(${n}))?([pP][+-]?(${t}))?\\b`},{match:"\\b0[oO](([0-7]_*)+)\\b"},{match:"\\b0[bB](([01]_*)+)\\b"}]};return{name:"Haskell",aliases:["hs"],keywords:"let in if then else case of where do module import hiding qualified type data newtype deriving class instance as default infix infixl infixr foreign export ccall stdcall cplusplus jvm dotnet safe unsafe family forall mdo proc rec",unicodeRegex:!0,contains:[{beginKeywords:"module",end:"where",keywords:"module where",contains:[l,r],illegal:"\\W\\.|;"},{begin:"\\bimport\\b",end:"$",keywords:"import qualified as hiding",contains:[l,r],illegal:"\\W\\.|;"},{className:"class",begin:"^(\\s*)?(class|instance)\\b",end:"where",keywords:"class family instance where",contains:[s,l,r]},{className:"class",begin:"\\b(data|(new)?type)\\b",end:"$",keywords:"data family type newtype deriving",contains:[i,s,l,{begin:/\{/,end:/\}/,contains:l.contains},r]},{beginKeywords:"default",end:"$",contains:[s,l,r]},{beginKeywords:"infix infixl infixr",end:"$",contains:[e.C_NUMBER_MODE,r]},{begin:"\\bforeign\\b",end:"$",keywords:"foreign import export ccall stdcall cplusplus jvm dotnet safe unsafe",contains:[s,e.QUOTE_STRING_MODE,r]},{className:"meta",begin:"#!\\/usr\\/bin\\/env runhaskell",end:"$"},i,o,{scope:"string",begin:/'(?=\\?.')/,end:/'/,contains:[{scope:"char.escape",match:/\\./}]},e.QUOTE_STRING_MODE,c,s,e.inherit(e.TITLE_MODE,{begin:"^[_a-z][\\w']*"}),{begin:`(?!-)${a}--+|--+(?!-)${a}`},r,{begin:"->|<-"}]}}},395(e){e.exports=function(e){const t=[e.C_NUMBER_MODE,{className:"string",begin:"'|\"",end:"'|\"",contains:[e.BACKSLASH_ESCAPE,{begin:"''"}]}];return{name:"Scilab",aliases:["sci"],keywords:{$pattern:/%?\w+/,keyword:"abort break case clear catch continue do elseif else endfunction end for function global if pause return resume select try then while",literal:"%f %F %t %T %pi %eps %inf %nan %e %i %z %s",built_in:"abs and acos asin atan ceil cd chdir clearglobal cosh cos cumprod deff disp error exec execstr exists exp eye gettext floor fprintf fread fsolve imag isdef isempty isinfisnan isvector lasterror length load linspace list listfiles log10 log2 log max min msprintf mclose mopen ones or pathconvert poly printf prod pwd rand real round sinh sin size gsort sprintf sqrt strcat strcmps tring sum system tanh tan type typename warning zeros matrix"},illegal:'("|#|/\\*|\\s+/\\w+)',contains:[{className:"function",beginKeywords:"function",end:"$",contains:[e.UNDERSCORE_TITLE_MODE,{className:"params",begin:"\\(",end:"\\)"}]},{begin:"[a-zA-Z_][a-zA-Z_0-9]*[\\.']+",relevance:0},{begin:"\\[",end:"\\][\\.']*",relevance:0,contains:t},e.COMMENT("//","$")].concat(t)}}},507(e){e.exports=function(e){const t=["assembly","module","package","import","alias","class","interface","object","given","value","assign","void","function","new","of","extends","satisfies","abstracts","in","out","return","break","continue","throw","assert","dynamic","if","else","switch","case","for","while","try","catch","finally","then","let","this","outer","super","is","exists","nonempty"],n={className:"subst",excludeBegin:!0,excludeEnd:!0,begin:/``/,end:/``/,keywords:t,relevance:10},a=[{className:"string",begin:'"""',end:'"""',relevance:10},{className:"string",begin:'"',end:'"',contains:[n]},{className:"string",begin:"'",end:"'"},{className:"number",begin:"#[0-9a-fA-F_]+|\\$[01_]+|[0-9_]+(?:\\.[0-9_](?:[eE][+-]?\\d+)?)?[kMGTPmunpf]?",relevance:0}];return n.contains=a,{name:"Ceylon",keywords:{keyword:t.concat(["shared","abstract","formal","default","actual","variable","late","native","deprecated","final","sealed","annotation","suppressWarnings","small"]),meta:["doc","by","license","see","throws","tagged"]},illegal:"\\$[^01]|#[^0-9a-fA-F]",contains:[e.C_LINE_COMMENT_MODE,e.COMMENT("/\\*","\\*/",{contains:["self"]}),{className:"meta",begin:'@[a-z]\\w*(?::"[^"]*")?'}].concat(a)}}},602(e){e.exports=function(e){const t={begin:/<\/?[A-Za-z_]/,end:">",subLanguage:"xml",relevance:0},n={variants:[{begin:/\[.+?\]\[.*?\]/,relevance:0},{begin:/\[.+?\]\(((data|javascript|mailto):|(?:http|ftp)s?:\/\/).*?\)/,relevance:2},{begin:e.regex.concat(/\[.+?\]\(/,/[A-Za-z][A-Za-z0-9+.-]*/,/:\/\/.*?\)/),relevance:2},{begin:/\[.+?\]\([./?&#].*?\)/,relevance:1},{begin:/\[.*?\]\(.*?\)/,relevance:0}],returnBegin:!0,contains:[{match:/\[(?=\])/},{className:"string",relevance:0,begin:"\\[",end:"\\]",excludeBegin:!0,returnEnd:!0},{className:"link",relevance:0,begin:"\\]\\(",end:"\\)",excludeBegin:!0,excludeEnd:!0},{className:"symbol",relevance:0,begin:"\\]\\[",end:"\\]",excludeBegin:!0,excludeEnd:!0}]},a={className:"strong",contains:[],variants:[{begin:/_{2}(?!\s)/,end:/_{2}/},{begin:/\*{2}(?!\s)/,end:/\*{2}/}]},r={className:"emphasis",contains:[],variants:[{begin:/\*(?![*\s])/,end:/\*/},{begin:/_(?![_\s])/,end:/_/,relevance:0}]},i=e.inherit(a,{contains:[]}),o=e.inherit(r,{contains:[]});a.contains.push(o),r.contains.push(i);let s=[t,n];return[a,r,i,o].forEach((e=>{e.contains=e.contains.concat(s)})),s=s.concat(a,r),{name:"Markdown",aliases:["md","mkdown","mkd"],contains:[{className:"section",variants:[{begin:"^#{1,6}",end:"$",contains:s},{begin:"(?=^.+?\\n[=-]{2,}$)",contains:[{begin:"^[=-]*$"},{begin:"^",end:"\\n",contains:s}]}]},t,{className:"bullet",begin:"^[ \t]*([*+-]|(\\d+\\.))(?=\\s+)",end:"\\s+",excludeEnd:!0},a,r,{className:"quote",begin:"^>\\s+",contains:s,end:"$"},{className:"code",variants:[{begin:"(`{3,})[^`](.|\\n)*?\\1`*[ ]*"},{begin:"(~{3,})[^~](.|\\n)*?\\1~*[ ]*"},{begin:"```",end:"```+[ ]*$"},{begin:"~~~",end:"~~~+[ ]*$"},{begin:"`.+?`"},{begin:"(?=^( {4}|\\t))",contains:[{begin:"^( {4}|\\t)",end:"(\\n)$"}],relevance:0}]},{begin:"^[-\\*]{3,}",end:"$"},n,{begin:/^\[[^\n]+\]:/,returnBegin:!0,contains:[{className:"symbol",begin:/\[/,end:/\]/,excludeBegin:!0,excludeEnd:!0},{className:"link",begin:/:\s*/,end:/$/,excludeBegin:!0}]},{scope:"literal",match:/&([a-zA-Z0-9]+|#[0-9]{1,7}|#[Xx][0-9a-fA-F]{1,6});/}]}}},621(e){e.exports=function(e){const t=["true","false","null"],n={scope:"literal",beginKeywords:t.join(" ")};return{name:"JSON",aliases:["jsonc"],keywords:{literal:t},contains:[{className:"attr",begin:/"(\\.|[^\\"\r\n])*"(?=\s*:)/,relevance:1.01},{match:/[{}[\],:]/,className:"punctuation",relevance:0},e.QUOTE_STRING_MODE,n,e.C_NUMBER_MODE,e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE],illegal:"\\S"}}},624(e){e.exports=function(e){const t=e.regex,n=e.inherit(e.QUOTE_STRING_MODE,{illegal:null}),a={className:"params",begin:/\(/,end:/\)/,contains:["self",e.C_NUMBER_MODE,n]},r=e.COMMENT(/--/,/$/),i=[r,e.COMMENT(/\(\*/,/\*\)/,{contains:["self",r]}),e.HASH_COMMENT_MODE];return{name:"AppleScript",aliases:["osascript"],keywords:{keyword:"about above after against and around as at back before beginning behind below beneath beside between but by considering contain contains continue copy div does eighth else end equal equals error every exit fifth first for fourth from front get given global if ignoring in into is it its last local me middle mod my ninth not of on onto or over prop property put ref reference repeat returning script second set seventh since sixth some tell tenth that the|0 then third through thru timeout times to transaction try until where while whose with without",literal:"AppleScript false linefeed return pi quote result space tab true",built_in:"alias application boolean class constant date file integer list number real record string text activate beep count delay launch log offset read round run say summarize write character characters contents day frontmost id item length month name|0 paragraph paragraphs rest reverse running time version weekday word words year"},contains:[n,e.C_NUMBER_MODE,{className:"built_in",begin:t.concat(/\b/,t.either(/clipboard info/,/the clipboard/,/info for/,/list (disks|folder)/,/mount volume/,/path to/,/(close|open for) access/,/(get|set) eof/,/current date/,/do shell script/,/get volume settings/,/random number/,/set volume/,/system attribute/,/system info/,/time to GMT/,/(load|run|store) script/,/scripting components/,/ASCII (character|number)/,/localized string/,/choose (application|color|file|file name|folder|from list|remote application|URL)/,/display (alert|dialog)/),/\b/)},{className:"built_in",begin:/^\s*return\b/},{className:"literal",begin:/\b(text item delimiters|current application|missing value)\b/},{className:"keyword",begin:t.concat(/\b/,t.either(/apart from/,/aside from/,/instead of/,/out of/,/greater than/,/isn't|(doesn't|does not) (equal|come before|come after|contain)/,/(greater|less) than( or equal)?/,/(starts?|ends|begins?) with/,/contained by/,/comes (before|after)/,/a (ref|reference)/,/POSIX (file|path)/,/(date|time) string/,/quoted form/),/\b/)},{beginKeywords:"on",illegal:/[${=;\n]/,contains:[e.UNDERSCORE_TITLE_MODE,a]},...i],illegal:/\/\/|->|=>|\[\[/}}},629(e){e.exports=function(e){return{name:"AVR Assembly",case_insensitive:!0,keywords:{$pattern:"\\.?"+e.IDENT_RE,keyword:"adc add adiw and andi asr bclr bld brbc brbs brcc brcs break breq brge brhc brhs brid brie brlo brlt brmi brne brpl brsh brtc brts brvc brvs bset bst call cbi cbr clc clh cli cln clr cls clt clv clz com cp cpc cpi cpse dec eicall eijmp elpm eor fmul fmuls fmulsu icall ijmp in inc jmp ld ldd ldi lds lpm lsl lsr mov movw mul muls mulsu neg nop or ori out pop push rcall ret reti rjmp rol ror sbc sbr sbrc sbrs sec seh sbi sbci sbic sbis sbiw sei sen ser ses set sev sez sleep spm st std sts sub subi swap tst wdr",built_in:"r0 r1 r2 r3 r4 r5 r6 r7 r8 r9 r10 r11 r12 r13 r14 r15 r16 r17 r18 r19 r20 r21 r22 r23 r24 r25 r26 r27 r28 r29 r30 r31 x|0 xh xl y|0 yh yl z|0 zh zl ucsr1c udr1 ucsr1a ucsr1b ubrr1l ubrr1h ucsr0c ubrr0h tccr3c tccr3a tccr3b tcnt3h tcnt3l ocr3ah ocr3al ocr3bh ocr3bl ocr3ch ocr3cl icr3h icr3l etimsk etifr tccr1c ocr1ch ocr1cl twcr twdr twar twsr twbr osccal xmcra xmcrb eicra spmcsr spmcr portg ddrg ping portf ddrf sreg sph spl xdiv rampz eicrb eimsk gimsk gicr eifr gifr timsk tifr mcucr mcucsr tccr0 tcnt0 ocr0 assr tccr1a tccr1b tcnt1h tcnt1l ocr1ah ocr1al ocr1bh ocr1bl icr1h icr1l tccr2 tcnt2 ocr2 ocdr wdtcr sfior eearh eearl eedr eecr porta ddra pina portb ddrb pinb portc ddrc pinc portd ddrd pind spdr spsr spcr udr0 ucsr0a ucsr0b ubrr0l acsr admux adcsr adch adcl porte ddre pine pinf",meta:".byte .cseg .db .def .device .dseg .dw .endmacro .equ .eseg .exit .include .list .listmac .macro .nolist .org .set"},contains:[e.C_BLOCK_COMMENT_MODE,e.COMMENT(";","$",{relevance:0}),e.C_NUMBER_MODE,e.BINARY_NUMBER_MODE,{className:"number",begin:"\\b(\\$[a-zA-Z0-9]+|0o[0-7]+)"},e.QUOTE_STRING_MODE,{className:"string",begin:"'",end:"[^\\\\]'",illegal:"[^\\\\][^']"},{className:"symbol",begin:"^[A-Za-z0-9_.$]+:"},{className:"meta",begin:"#",end:"$"},{className:"subst",begin:"@[0-9]+"}]}}},722(e){e.exports=function(e){const t=e.regex,n=e.COMMENT("//","$",{contains:[{begin:/\\\n/}]}),a="decltype\\(auto\\)",r="[a-zA-Z_]\\w*::",i="("+a+"|"+t.optional(r)+"[a-zA-Z_]\\w*"+t.optional("<[^<>]+>")+")",o={className:"type",variants:[{begin:"\\b[a-z\\d_]*_t\\b"},{match:/\batomic_[a-z]{3,6}\b/}]},s={className:"string",variants:[{begin:'(u8?|U|L)?"',end:'"',illegal:"\\n",contains:[e.BACKSLASH_ESCAPE]},{begin:"(u8?|U|L)?'(\\\\(x[0-9A-Fa-f]{2}|u[0-9A-Fa-f]{4,8}|[0-7]{3}|\\S)|.)",end:"'",illegal:"."},e.END_SAME_AS_BEGIN({begin:/(?:u8?|U|L)?R"([^()\\ ]{0,16})\(/,end:/\)([^()\\ ]{0,16})"/})]},l={className:"number",variants:[{match:/\b(0b[01']+)/},{match:/(-?)\b([\d']+(\.[\d']*)?|\.[\d']+)((ll|LL|l|L)(u|U)?|(u|U)(ll|LL|l|L)?|f|F|b|B)/},{match:/(-?)\b(0[xX][a-fA-F0-9]+(?:'[a-fA-F0-9]+)*(?:\.[a-fA-F0-9]*(?:'[a-fA-F0-9]*)*)?(?:[pP][-+]?[0-9]+)?(l|L)?(u|U)?)/},{match:/(-?)\b\d+(?:'\d+)*(?:\.\d*(?:'\d*)*)?(?:[eE][-+]?\d+)?/}],relevance:0},c={className:"meta",begin:/#\s*[a-z]+\b/,end:/$/,keywords:{keyword:"if else elif endif define undef warning error line pragma _Pragma ifdef ifndef elifdef elifndef include"},contains:[{begin:/\\\n/,relevance:0},e.inherit(s,{className:"string"}),{className:"string",begin:/<.*?>/},n,e.C_BLOCK_COMMENT_MODE]},_={className:"title",begin:t.optional(r)+e.IDENT_RE,relevance:0},d=t.optional(r)+e.IDENT_RE+"\\s*\\(",p={keyword:["asm","auto","break","case","continue","default","do","else","enum","extern","for","fortran","goto","if","inline","register","restrict","return","sizeof","typeof","typeof_unqual","struct","switch","typedef","union","volatile","while","_Alignas","_Alignof","_Atomic","_Generic","_Noreturn","_Static_assert","_Thread_local","alignas","alignof","noreturn","static_assert","thread_local","_Pragma"],type:["float","double","signed","unsigned","int","short","long","char","void","_Bool","_BitInt","_Complex","_Imaginary","_Decimal32","_Decimal64","_Decimal96","_Decimal128","_Decimal64x","_Decimal128x","_Float16","_Float32","_Float64","_Float128","_Float32x","_Float64x","_Float128x","const","static","constexpr","complex","bool","imaginary"],literal:"true false NULL",built_in:"std string wstring cin cout cerr clog stdin stdout stderr stringstream istringstream ostringstream auto_ptr deque list queue stack vector map set pair bitset multiset multimap unordered_set unordered_map unordered_multiset unordered_multimap priority_queue make_pair array shared_ptr abort terminate abs acos asin atan2 atan calloc ceil cosh cos exit exp fabs floor fmod fprintf fputs free frexp fscanf future isalnum isalpha iscntrl isdigit isgraph islower isprint ispunct isspace isupper isxdigit tolower toupper labs ldexp log10 log malloc realloc memchr memcmp memcpy memset modf pow printf putchar puts scanf sinh sin snprintf sprintf sqrt sscanf strcat strchr strcmp strcpy strcspn strlen strncat strncmp strncpy strpbrk strrchr strspn strstr tanh tan vfprintf vprintf vsprintf endl initializer_list unique_ptr"},u=[c,o,n,e.C_BLOCK_COMMENT_MODE,l,s],m={variants:[{begin:/=/,end:/;/},{begin:/\(/,end:/\)/},{beginKeywords:"new throw return else",end:/;/}],keywords:p,contains:u.concat([{begin:/\(/,end:/\)/,keywords:p,contains:u.concat(["self"]),relevance:0}]),relevance:0},g={begin:"("+i+"[\\*&\\s]+)+"+d,returnBegin:!0,end:/[{;=]/,excludeEnd:!0,keywords:p,illegal:/[^\w\s\*&:<>.]/,contains:[{begin:a,keywords:p,relevance:0},{begin:d,returnBegin:!0,contains:[e.inherit(_,{className:"title.function"})],relevance:0},{relevance:0,match:/,/},{className:"params",begin:/\(/,end:/\)/,keywords:p,relevance:0,contains:[n,e.C_BLOCK_COMMENT_MODE,s,l,o,{begin:/\(/,end:/\)/,keywords:p,relevance:0,contains:["self",n,e.C_BLOCK_COMMENT_MODE,s,l,o]}]},o,n,e.C_BLOCK_COMMENT_MODE,c]};return{name:"C",aliases:["h"],keywords:p,disableAutodetect:!0,illegal:"=]/,contains:[{beginKeywords:"final class struct"},e.TITLE_MODE]}]),exports:{preprocessor:c,strings:s,keywords:p}}}},943(e){e.exports=function(e){const t=/[a-zA-Z@][a-zA-Z0-9_]*/,n={$pattern:t,keyword:["@interface","@class","@protocol","@implementation"]};return{name:"Objective-C",aliases:["mm","objc","obj-c","obj-c++","objective-c++"],keywords:{"variable.language":["this","super"],$pattern:t,keyword:["while","export","sizeof","typedef","const","struct","for","union","volatile","static","mutable","if","do","return","goto","enum","else","break","extern","asm","case","default","register","explicit","typename","switch","continue","inline","readonly","assign","readwrite","self","@synchronized","id","typeof","nonatomic","IBOutlet","IBAction","strong","weak","copy","in","out","inout","bycopy","byref","oneway","__strong","__weak","__block","__autoreleasing","@private","@protected","@public","@try","@property","@end","@throw","@catch","@finally","@autoreleasepool","@synthesize","@dynamic","@selector","@optional","@required","@encode","@package","@import","@defs","@compatibility_alias","__bridge","__bridge_transfer","__bridge_retained","__bridge_retain","__covariant","__contravariant","__kindof","_Nonnull","_Nullable","_Null_unspecified","__FUNCTION__","__PRETTY_FUNCTION__","__attribute__","getter","setter","retain","unsafe_unretained","nonnull","nullable","null_unspecified","null_resettable","class","instancetype","NS_DESIGNATED_INITIALIZER","NS_UNAVAILABLE","NS_REQUIRES_SUPER","NS_RETURNS_INNER_POINTER","NS_INLINE","NS_AVAILABLE","NS_DEPRECATED","NS_ENUM","NS_OPTIONS","NS_SWIFT_UNAVAILABLE","NS_ASSUME_NONNULL_BEGIN","NS_ASSUME_NONNULL_END","NS_REFINED_FOR_SWIFT","NS_SWIFT_NAME","NS_SWIFT_NOTHROW","NS_DURING","NS_HANDLER","NS_ENDHANDLER","NS_VALUERETURN","NS_VOIDRETURN"],literal:["false","true","FALSE","TRUE","nil","YES","NO","NULL"],built_in:["dispatch_once_t","dispatch_queue_t","dispatch_sync","dispatch_async","dispatch_once"],type:["int","float","char","unsigned","signed","short","long","double","wchar_t","unichar","void","bool","BOOL","id|0","_Bool"]},illegal:"/,end:/$/,illegal:"\\n"},e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE]},{className:"class",begin:"("+n.keyword.join("|")+")\\b",end:/(\{|$)/,excludeEnd:!0,keywords:n,contains:[e.UNDERSCORE_TITLE_MODE]},{begin:"\\."+e.UNDERSCORE_IDENT_RE,relevance:0}]}}},944(e){e.exports=function(e){return{name:"Mizar",keywords:"environ vocabularies notations constructors definitions registrations theorems schemes requirements begin end definition registration cluster existence pred func defpred deffunc theorem proof let take assume then thus hence ex for st holds consider reconsider such that and in provided of as from be being by means equals implies iff redefine define now not or attr is mode suppose per cases set thesis contradiction scheme reserve struct correctness compatibility coherence symmetry assymetry reflexivity irreflexivity connectedness uniqueness commutativity idempotence involutiveness projectivity",contains:[e.COMMENT("::","$")]}}},946(e){e.exports=function(e){const t=e.regex,n=/[dualxmsipngr]{0,12}/,a={$pattern:/[\w.]+/,keyword:["abs","accept","alarm","and","atan2","bind","binmode","bless","break","caller","chdir","chmod","chomp","chop","chown","chr","chroot","class","close","closedir","connect","continue","cos","crypt","dbmclose","dbmopen","defined","delete","die","do","dump","each","else","elsif","endgrent","endhostent","endnetent","endprotoent","endpwent","endservent","eof","eval","exec","exists","exit","exp","fcntl","field","fileno","flock","for","foreach","fork","format","formline","getc","getgrent","getgrgid","getgrnam","gethostbyaddr","gethostbyname","gethostent","getlogin","getnetbyaddr","getnetbyname","getnetent","getpeername","getpgrp","getpriority","getprotobyname","getprotobynumber","getprotoent","getpwent","getpwnam","getpwuid","getservbyname","getservbyport","getservent","getsockname","getsockopt","given","glob","gmtime","goto","grep","gt","hex","if","index","int","ioctl","join","keys","kill","last","lc","lcfirst","length","link","listen","local","localtime","log","lstat","lt","ma","map","method","mkdir","msgctl","msgget","msgrcv","msgsnd","my","ne","next","no","not","oct","open","opendir","or","ord","our","pack","package","pipe","pop","pos","print","printf","prototype","push","q|0","qq","quotemeta","qw","qx","rand","read","readdir","readline","readlink","readpipe","recv","redo","ref","rename","require","reset","return","reverse","rewinddir","rindex","rmdir","say","scalar","seek","seekdir","select","semctl","semget","semop","send","setgrent","sethostent","setnetent","setpgrp","setpriority","setprotoent","setpwent","setservent","setsockopt","shift","shmctl","shmget","shmread","shmwrite","shutdown","sin","sleep","socket","socketpair","sort","splice","split","sprintf","sqrt","srand","stat","state","study","sub","substr","symlink","syscall","sysopen","sysread","sysseek","system","syswrite","tell","telldir","tie","tied","time","times","tr","truncate","uc","ucfirst","umask","undef","unless","unlink","unpack","unshift","untie","until","use","utime","values","vec","wait","waitpid","wantarray","warn","when","while","write","x|0","xor","y|0"].join(" ")},r={className:"subst",begin:"[$@]\\{",end:"\\}",keywords:a},i={begin:/->\{/,end:/\}/},o={scope:"attr",match:/\s+:\s*\w+(\s*\(.*?\))?/},s={scope:"variable",variants:[{begin:/\$\d/},{begin:t.concat(/[$%@](?!")(\^\w\b|#\w+(::\w+)*|\{\w+\}|\w+(::\w*)*)/,"(?![A-Za-z])(?![@$%])")},{begin:/[$%@](?!")[^\s\w{=]|\$=/,relevance:0}],contains:[o]},l={className:"number",variants:[{match:/0?\.[0-9][0-9_]+\b/},{match:/\bv?(0|[1-9][0-9_]*(\.[0-9_]+)?|[1-9][0-9_]*)\b/},{match:/\b0[0-7][0-7_]*\b/},{match:/\b0x[0-9a-fA-F][0-9a-fA-F_]*\b/},{match:/\b0b[0-1][0-1_]*\b/}],relevance:0},c=[e.BACKSLASH_ESCAPE,r,s],_=[/!/,/\//,/\|/,/\?/,/'/,/"/,/#/],d=(e,a,r="\\1")=>{const i="\\1"===r?r:t.concat(r,a);return t.concat(t.concat("(?:",e,")"),a,/(?:\\.|[^\\\/])*?/,i,/(?:\\.|[^\\\/])*?/,r,n)},p=(e,a,r)=>t.concat(t.concat("(?:",e,")"),a,/(?:\\.|[^\\\/])*?/,r,n),u=[s,e.HASH_COMMENT_MODE,e.COMMENT(/^=\w/,/=cut/,{endsWithParent:!0}),i,{className:"string",contains:c,variants:[{begin:"q[qwxr]?\\s*\\(",end:"\\)",relevance:5},{begin:"q[qwxr]?\\s*\\[",end:"\\]",relevance:5},{begin:"q[qwxr]?\\s*\\{",end:"\\}",relevance:5},{begin:"q[qwxr]?\\s*\\|",end:"\\|",relevance:5},{begin:"q[qwxr]?\\s*<",end:">",relevance:5},{begin:"qw\\s+q",end:"q",relevance:5},{begin:"'",end:"'",contains:[e.BACKSLASH_ESCAPE]},{begin:'"',end:'"'},{begin:"`",end:"`",contains:[e.BACKSLASH_ESCAPE]},{begin:/\{\w+\}/,relevance:0},{begin:"-?\\w+\\s*=>",relevance:0}]},l,{begin:"(\\/\\/|"+e.RE_STARTERS_RE+"|\\b(split|return|print|reverse|grep)\\b)\\s*",keywords:"split return print reverse grep",relevance:0,contains:[e.HASH_COMMENT_MODE,{className:"regexp",variants:[{begin:d("s|tr|y",t.either(..._,{capture:!0}))},{begin:d("s|tr|y","\\(","\\)")},{begin:d("s|tr|y","\\[","\\]")},{begin:d("s|tr|y","\\{","\\}")}],relevance:2},{className:"regexp",variants:[{begin:/(m|qr)\/\//,relevance:0},{begin:p("(?:m|qr)?",/\//,/\//)},{begin:p("m|qr",t.either(..._,{capture:!0}),/\1/)},{begin:p("m|qr",/\(/,/\)/)},{begin:p("m|qr",/\[/,/\]/)},{begin:p("m|qr",/\{/,/\}/)}]}]},{className:"function",beginKeywords:"sub method",end:"(\\s*\\(.*?\\))?[;{]",excludeEnd:!0,relevance:5,contains:[e.TITLE_MODE,o]},{className:"class",beginKeywords:"class",end:"[;{]",excludeEnd:!0,relevance:5,contains:[e.TITLE_MODE,o,l]},{begin:"-\\w\\b",relevance:0},{begin:"^__DATA__$",end:"^__END__$",subLanguage:"mojolicious",contains:[{begin:"^@@.*",end:"$",className:"comment"}]}];return r.contains=u,i.contains=u,{name:"Perl",aliases:["pl","pm"],keywords:a,contains:u}}},969(e){e.exports=function(e){const t={className:"built_in",begin:"\\b(void|bool|int8|int16|int32|int64|int|uint8|uint16|uint32|uint64|uint|string|ref|array|double|float|auto|dictionary)"},n={className:"symbol",begin:"[a-zA-Z0-9_]+@"},a={className:"keyword",begin:"<",end:">",contains:[t,n]};return t.contains=[a],n.contains=[a],{name:"AngelScript",aliases:["asc"],keywords:["for","in|0","break","continue","while","do|0","return","if","else","case","switch","namespace","is","cast","or","and","xor","not","get|0","in","inout|10","out","override","set|0","private","public","const","default|0","final","shared","external","mixin|10","enum","typedef","funcdef","this","super","import","from","interface","abstract|0","try","catch","protected","explicit","property"],illegal:"(^using\\s+[A-Za-z0-9_\\.]+;$|\\bfunction\\s*[^\\(])",contains:[{className:"string",begin:"'",end:"'",illegal:"\\n",contains:[e.BACKSLASH_ESCAPE],relevance:0},{className:"string",begin:'"""',end:'"""'},{className:"string",begin:'"',end:'"',illegal:"\\n",contains:[e.BACKSLASH_ESCAPE],relevance:0},e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE,{className:"string",begin:"^\\s*\\[",end:"\\]"},{beginKeywords:"interface namespace",end:/\{/,illegal:"[;.\\-]",contains:[{className:"symbol",begin:"[a-zA-Z0-9_]+"}]},{beginKeywords:"class",end:/\{/,illegal:"[;.\\-]",contains:[{className:"symbol",begin:"[a-zA-Z0-9_]+",contains:[{begin:"[:,]\\s*",contains:[{className:"symbol",begin:"[a-zA-Z0-9_]+"}]}]}]},t,n,{className:"literal",begin:"\\b(null|true|false)"},{className:"number",relevance:0,begin:"(-?)(\\b0[xXbBoOdD][a-fA-F0-9]+|(\\b\\d+(\\.\\d*)?f?|\\.\\d+f?)([eE][-+]?\\d+f?)?)"}]}}},987(e){e.exports=function(e){const t={className:"number",begin:"[1-9][0-9]*",relevance:0},n={className:"symbol",begin:":[^\\]]+"};return{name:"TP",keywords:{keyword:["ABORT","ACC","ADJUST","AND","AP_LD","BREAK","CALL","CNT","COL","CONDITION","CONFIG","DA","DB","DIV","DETECT","ELSE","END","ENDFOR","ERR_NUM","ERROR_PROG","FINE","FOR","GP","GUARD","INC","IF","JMP","LINEAR_MAX_SPEED","LOCK","MOD","MONITOR","OFFSET","Offset","OR","OVERRIDE","PAUSE","PREG","PTH","RT_LD","RUN","SELECT","SKIP","Skip","TA","TB","TO","TOOL_OFFSET","Tool_Offset","UF","UT","UFRAME_NUM","UTOOL_NUM","UNLOCK","WAIT","X","Y","Z","W","P","R","STRLEN","SUBSTR","FINDSTR","VOFFSET","PROG","ATTR","MN","POS"],literal:["ON","OFF","max_speed","LPOS","JPOS","ENABLE","DISABLE","START","STOP","RESET"]},contains:[{className:"built_in",begin:"(AR|P|PAYLOAD|PR|R|SR|RSR|LBL|VR|UALM|MESSAGE|UTOOL|UFRAME|TIMER|TIMER_OVERFLOW|JOINT_MAX_SPEED|RESUME_PROG|DIAG_REC)\\[",end:"\\]",contains:["self",t,n]},{className:"built_in",begin:"(AI|AO|DI|DO|F|RI|RO|UI|UO|GI|GO|SI|SO)\\[",end:"\\]",contains:["self",t,e.QUOTE_STRING_MODE,n]},{className:"keyword",begin:"/(PROG|ATTR|MN|POS|END)\\b"},{className:"keyword",begin:"(CALL|RUN|POINT_LOGIC|LBL)\\b"},{className:"keyword",begin:"\\b(ACC|CNT|Skip|Offset|PSPD|RT_LD|AP_LD|Tool_Offset)"},{className:"number",begin:"\\d+(sec|msec|mm/sec|cm/min|inch/min|deg/sec|mm|in|cm)?\\b",relevance:0},e.COMMENT("//","[;$]"),e.COMMENT("!","[;$]"),e.COMMENT("--eg:","$"),e.QUOTE_STRING_MODE,{className:"string",begin:"'",end:"'"},e.C_NUMBER_MODE,{className:"variable",begin:"\\$[A-Za-z0-9_]+"}]}}},1050(e){e.exports=function(e){return{name:"Clean",aliases:["icl","dcl"],keywords:{keyword:["if","let","in","with","where","case","of","class","instance","otherwise","implementation","definition","system","module","from","import","qualified","as","special","code","inline","foreign","export","ccall","stdcall","generic","derive","infix","infixl","infixr"],built_in:"Int Real Char Bool",literal:"True False"},contains:[e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE,e.APOS_STRING_MODE,e.QUOTE_STRING_MODE,e.C_NUMBER_MODE,{begin:"->|<-[|:]?|#!?|>>=|\\{\\||\\|\\}|:==|=:|<>"}]}}},1117(e){e.exports=function(e){const t=e.regex,n=/[\p{XID_Start}_]\p{XID_Continue}*/u,a=["and","as","assert","async","await","break","case","class","continue","def","del","elif","else","except","finally","for","from","global","if","import","in","is","lambda","match","nonlocal|10","not","or","pass","raise","return","try","while","with","yield"],r={$pattern:/[A-Za-z]\w+|__\w+__/,keyword:a,built_in:["__import__","abs","all","any","ascii","bin","bool","breakpoint","bytearray","bytes","callable","chr","classmethod","compile","complex","delattr","dict","dir","divmod","enumerate","eval","exec","filter","float","format","frozenset","getattr","globals","hasattr","hash","help","hex","id","input","int","isinstance","issubclass","iter","len","list","locals","map","max","memoryview","min","next","object","oct","open","ord","pow","print","property","range","repr","reversed","round","set","setattr","slice","sorted","staticmethod","str","sum","super","tuple","type","vars","zip"],literal:["__debug__","Ellipsis","False","None","NotImplemented","True"],type:["Any","Callable","Coroutine","Dict","List","Literal","Generic","Optional","Sequence","Set","Tuple","Type","Union"]},i={className:"meta",begin:/^(>>>|\.\.\.) /},o={className:"subst",begin:/\{/,end:/\}/,keywords:r,illegal:/#/},s={begin:/\{\{/,relevance:0},l={className:"string",contains:[e.BACKSLASH_ESCAPE],variants:[{begin:/([uU]|[bB]|[rR]|[bB][rR]|[rR][bB])?'''/,end:/'''/,contains:[e.BACKSLASH_ESCAPE,i],relevance:10},{begin:/([uU]|[bB]|[rR]|[bB][rR]|[rR][bB])?"""/,end:/"""/,contains:[e.BACKSLASH_ESCAPE,i],relevance:10},{begin:/([fF][rR]|[rR][fF]|[fF])'''/,end:/'''/,contains:[e.BACKSLASH_ESCAPE,i,s,o]},{begin:/([fF][rR]|[rR][fF]|[fF])"""/,end:/"""/,contains:[e.BACKSLASH_ESCAPE,i,s,o]},{begin:/([uU]|[rR])'/,end:/'/,relevance:10},{begin:/([uU]|[rR])"/,end:/"/,relevance:10},{begin:/([bB]|[bB][rR]|[rR][bB])'/,end:/'/},{begin:/([bB]|[bB][rR]|[rR][bB])"/,end:/"/},{begin:/([fF][rR]|[rR][fF]|[fF])'/,end:/'/,contains:[e.BACKSLASH_ESCAPE,s,o]},{begin:/([fF][rR]|[rR][fF]|[fF])"/,end:/"/,contains:[e.BACKSLASH_ESCAPE,s,o]},e.APOS_STRING_MODE,e.QUOTE_STRING_MODE]},c="[0-9](_?[0-9])*",_=`(\\b(${c}))?\\.(${c})|\\b(${c})\\.`,d=`\\b|${a.join("|")}`,p={className:"number",relevance:0,variants:[{begin:`(\\b(${c})|(${_}))[eE][+-]?(${c})[jJ]?(?=${d})`},{begin:`(${_})[jJ]?`},{begin:`\\b([1-9](_?[0-9])*|0+(_?0)*)[lLjJ]?(?=${d})`},{begin:`\\b0[bB](_?[01])+[lL]?(?=${d})`},{begin:`\\b0[oO](_?[0-7])+[lL]?(?=${d})`},{begin:`\\b0[xX](_?[0-9a-fA-F])+[lL]?(?=${d})`},{begin:`\\b(${c})[jJ](?=${d})`}]},u={className:"comment",begin:t.lookahead(/# type:/),end:/$/,keywords:r,contains:[{begin:/# type:/},{begin:/#/,end:/\b\B/,endsWithParent:!0}]},m={className:"params",variants:[{className:"",begin:/\(\s*\)/,skip:!0},{begin:/\(/,end:/\)/,excludeBegin:!0,excludeEnd:!0,keywords:r,contains:["self",i,p,l,e.HASH_COMMENT_MODE]}]};return o.contains=[l,p,i],{name:"Python",aliases:["py","gyp","ipython"],unicodeRegex:!0,keywords:r,illegal:/(<\/|\?)|=>/,contains:[i,p,{scope:"variable.language",match:/\bself\b/},{beginKeywords:"if",relevance:0},{match:/\bor\b/,scope:"keyword"},l,u,e.HASH_COMMENT_MODE,{match:[/\bdef/,/\s+/,n],scope:{1:"keyword",3:"title.function"},contains:[m]},{variants:[{match:[/\bclass/,/\s+/,n,/\s*/,/\(\s*/,n,/\s*\)/]},{match:[/\bclass/,/\s+/,n]}],scope:{1:"keyword",3:"title.class",6:"title.class.inherited"}},{className:"meta",begin:/^[\t ]*@/,end:/(?=#)|$/,contains:[p,m,l]}]}}},1189(e){e.exports=function(e){const t=e.regex,n={variants:[e.COMMENT("!","$",{relevance:0}),e.COMMENT("^C[ ]","$",{relevance:0}),e.COMMENT("^C$","$",{relevance:0})]},a=/(_[a-z_\d]+)?/,r=/([de][+-]?\d+)?/,i={className:"number",variants:[{begin:t.concat(/\b\d+/,/\.(\d*)/,r,a)},{begin:t.concat(/\b\d+/,r,a)},{begin:t.concat(/\.\d+/,r,a)}],relevance:0},o={className:"function",beginKeywords:"subroutine function program",illegal:"[${=\\n]",contains:[e.UNDERSCORE_TITLE_MODE,{className:"params",begin:"\\(",end:"\\)"}]};return{name:"Fortran",case_insensitive:!0,aliases:["f90","f95"],keywords:{$pattern:/\b[a-z][a-z0-9_]+\b|\.[a-z][a-z0-9_]+\./,keyword:["kind","do","concurrent","local","shared","while","private","call","intrinsic","where","elsewhere","type","endtype","endmodule","endselect","endinterface","end","enddo","endif","if","forall","endforall","only","contains","default","return","stop","then","block","endblock","endassociate","public","subroutine|10","function","program",".and.",".or.",".not.",".le.",".eq.",".ge.",".gt.",".lt.","goto","save","else","use","module","select","case","access","blank","direct","exist","file","fmt","form","formatted","iostat","name","named","nextrec","number","opened","rec","recl","sequential","status","unformatted","unit","continue","format","pause","cycle","exit","c_null_char","c_alert","c_backspace","c_form_feed","flush","wait","decimal","round","iomsg","synchronous","nopass","non_overridable","pass","protected","volatile","abstract","extends","import","non_intrinsic","value","deferred","generic","final","enumerator","class","associate","bind","enum","c_int","c_short","c_long","c_long_long","c_signed_char","c_size_t","c_int8_t","c_int16_t","c_int32_t","c_int64_t","c_int_least8_t","c_int_least16_t","c_int_least32_t","c_int_least64_t","c_int_fast8_t","c_int_fast16_t","c_int_fast32_t","c_int_fast64_t","c_intmax_t","C_intptr_t","c_float","c_double","c_long_double","c_float_complex","c_double_complex","c_long_double_complex","c_bool","c_char","c_null_ptr","c_null_funptr","c_new_line","c_carriage_return","c_horizontal_tab","c_vertical_tab","iso_c_binding","c_loc","c_funloc","c_associated","c_f_pointer","c_ptr","c_funptr","iso_fortran_env","character_storage_size","error_unit","file_storage_size","input_unit","iostat_end","iostat_eor","numeric_storage_size","output_unit","c_f_procpointer","ieee_arithmetic","ieee_support_underflow_control","ieee_get_underflow_mode","ieee_set_underflow_mode","newunit","contiguous","recursive","pad","position","action","delim","readwrite","eor","advance","nml","interface","procedure","namelist","include","sequence","elemental","pure","impure","integer","real","character","complex","logical","codimension","dimension","allocatable|10","parameter","external","implicit|10","none","double","precision","assign","intent","optional","pointer","target","in","out","common","equivalence","data"],literal:[".False.",".True."],built_in:["alog","alog10","amax0","amax1","amin0","amin1","amod","cabs","ccos","cexp","clog","csin","csqrt","dabs","dacos","dasin","datan","datan2","dcos","dcosh","ddim","dexp","dint","dlog","dlog10","dmax1","dmin1","dmod","dnint","dsign","dsin","dsinh","dsqrt","dtan","dtanh","float","iabs","idim","idint","idnint","ifix","isign","max0","max1","min0","min1","sngl","algama","cdabs","cdcos","cdexp","cdlog","cdsin","cdsqrt","cqabs","cqcos","cqexp","cqlog","cqsin","cqsqrt","dcmplx","dconjg","derf","derfc","dfloat","dgamma","dimag","dlgama","iqint","qabs","qacos","qasin","qatan","qatan2","qcmplx","qconjg","qcos","qcosh","qdim","qerf","qerfc","qexp","qgamma","qimag","qlgama","qlog","qlog10","qmax1","qmin1","qmod","qnint","qsign","qsin","qsinh","qsqrt","qtan","qtanh","abs","acos","aimag","aint","anint","asin","atan","atan2","char","cmplx","conjg","cos","cosh","exp","ichar","index","int","log","log10","max","min","nint","sign","sin","sinh","sqrt","tan","tanh","print","write","dim","lge","lgt","lle","llt","mod","nullify","allocate","deallocate","adjustl","adjustr","all","allocated","any","associated","bit_size","btest","ceiling","count","cshift","date_and_time","digits","dot_product","eoshift","epsilon","exponent","floor","fraction","huge","iand","ibclr","ibits","ibset","ieor","ior","ishft","ishftc","lbound","len_trim","matmul","maxexponent","maxloc","maxval","merge","minexponent","minloc","minval","modulo","mvbits","nearest","pack","present","product","radix","random_number","random_seed","range","repeat","reshape","rrspacing","scale","scan","selected_int_kind","selected_real_kind","set_exponent","shape","size","spacing","spread","sum","system_clock","tiny","transpose","trim","ubound","unpack","verify","achar","iachar","transfer","dble","entry","dprod","cpu_time","command_argument_count","get_command","get_command_argument","get_environment_variable","is_iostat_end","ieee_arithmetic","ieee_support_underflow_control","ieee_get_underflow_mode","ieee_set_underflow_mode","is_iostat_eor","move_alloc","new_line","selected_char_kind","same_type_as","extends_type_of","acosh","asinh","atanh","bessel_j0","bessel_j1","bessel_jn","bessel_y0","bessel_y1","bessel_yn","erf","erfc","erfc_scaled","gamma","log_gamma","hypot","norm2","atomic_define","atomic_ref","execute_command_line","leadz","trailz","storage_size","merge_bits","bge","bgt","ble","blt","dshiftl","dshiftr","findloc","iall","iany","iparity","image_index","lcobound","ucobound","maskl","maskr","num_images","parity","popcnt","poppar","shifta","shiftl","shiftr","this_image","sync","change","team","co_broadcast","co_max","co_min","co_sum","co_reduce"]},illegal:/\/\*/,contains:[{className:"string",relevance:0,variants:[e.APOS_STRING_MODE,e.QUOTE_STRING_MODE]},o,{begin:/^C\s*=(?!=)/,relevance:0},n,i]}}},1195(e){e.exports=function(e){return{name:"GML",case_insensitive:!1,keywords:{keyword:["#endregion","#macro","#region","and","begin","break","case","constructor","continue","default","delete","div","do","else","end","enum","exit","for","function","globalvar","if","mod","new","not","or","repeat","return","static","switch","then","until","var","while","with","xor"],built_in:["abs","alarm_get","alarm_set","angle_difference","animcurve_channel_evaluate","animcurve_channel_new","animcurve_create","animcurve_destroy","animcurve_exists","animcurve_get","animcurve_get_channel","animcurve_get_channel_index","animcurve_point_new","ansi_char","application_get_position","application_surface_draw_enable","application_surface_enable","application_surface_is_enabled","arccos","arcsin","arctan","arctan2","array_all","array_any","array_concat","array_contains","array_contains_ext","array_copy","array_copy_while","array_create","array_create_ext","array_delete","array_equals","array_filter","array_filter_ext","array_find_index","array_first","array_foreach","array_get","array_get_index","array_insert","array_intersection","array_last","array_length","array_map","array_map_ext","array_pop","array_push","array_reduce","array_resize","array_reverse","array_reverse_ext","array_set","array_shuffle","array_shuffle_ext","array_sort","array_union","array_unique","array_unique_ext","asset_add_tags","asset_clear_tags","asset_get_ids","asset_get_index","asset_get_tags","asset_get_type","asset_has_any_tag","asset_has_tags","asset_remove_tags","audio_bus_clear_emitters","audio_bus_create","audio_bus_get_emitters","audio_channel_num","audio_create_buffer_sound","audio_create_play_queue","audio_create_stream","audio_create_sync_group","audio_debug","audio_destroy_stream","audio_destroy_sync_group","audio_effect_create","audio_emitter_bus","audio_emitter_create","audio_emitter_exists","audio_emitter_falloff","audio_emitter_free","audio_emitter_gain","audio_emitter_get_bus","audio_emitter_get_gain","audio_emitter_get_listener_mask","audio_emitter_get_pitch","audio_emitter_get_vx","audio_emitter_get_vy","audio_emitter_get_vz","audio_emitter_get_x","audio_emitter_get_y","audio_emitter_get_z","audio_emitter_pitch","audio_emitter_position","audio_emitter_set_listener_mask","audio_emitter_velocity","audio_exists","audio_falloff_set_model","audio_free_buffer_sound","audio_free_play_queue","audio_get_listener_count","audio_get_listener_info","audio_get_listener_mask","audio_get_master_gain","audio_get_name","audio_get_recorder_count","audio_get_recorder_info","audio_get_type","audio_group_get_assets","audio_group_get_gain","audio_group_is_loaded","audio_group_load","audio_group_load_progress","audio_group_name","audio_group_set_gain","audio_group_stop_all","audio_group_unload","audio_is_paused","audio_is_playing","audio_listener_get_data","audio_listener_orientation","audio_listener_position","audio_listener_set_orientation","audio_listener_set_position","audio_listener_set_velocity","audio_listener_velocity","audio_master_gain","audio_pause_all","audio_pause_sound","audio_pause_sync_group","audio_play_in_sync_group","audio_play_sound","audio_play_sound_at","audio_play_sound_ext","audio_play_sound_on","audio_queue_sound","audio_resume_all","audio_resume_sound","audio_resume_sync_group","audio_set_listener_mask","audio_set_master_gain","audio_sound_gain","audio_sound_get_audio_group","audio_sound_get_gain","audio_sound_get_listener_mask","audio_sound_get_loop","audio_sound_get_loop_end","audio_sound_get_loop_start","audio_sound_get_pitch","audio_sound_get_track_position","audio_sound_is_playable","audio_sound_length","audio_sound_loop","audio_sound_loop_end","audio_sound_loop_start","audio_sound_pitch","audio_sound_set_listener_mask","audio_sound_set_track_position","audio_start_recording","audio_start_sync_group","audio_stop_all","audio_stop_recording","audio_stop_sound","audio_stop_sync_group","audio_sync_group_debug","audio_sync_group_get_track_pos","audio_sync_group_is_paused","audio_sync_group_is_playing","audio_system_is_available","audio_system_is_initialised","base64_decode","base64_encode","bool","browser_input_capture","buffer_async_group_begin","buffer_async_group_end","buffer_async_group_option","buffer_base64_decode","buffer_base64_decode_ext","buffer_base64_encode","buffer_compress","buffer_copy","buffer_copy_from_vertex_buffer","buffer_copy_stride","buffer_crc32","buffer_create","buffer_create_from_vertex_buffer","buffer_create_from_vertex_buffer_ext","buffer_decompress","buffer_delete","buffer_exists","buffer_fill","buffer_get_address","buffer_get_alignment","buffer_get_size","buffer_get_surface","buffer_get_type","buffer_load","buffer_load_async","buffer_load_ext","buffer_load_partial","buffer_md5","buffer_peek","buffer_poke","buffer_read","buffer_resize","buffer_save","buffer_save_async","buffer_save_ext","buffer_seek","buffer_set_surface","buffer_set_used_size","buffer_sha1","buffer_sizeof","buffer_tell","buffer_write","call_cancel","call_later","camera_apply","camera_copy_transforms","camera_create","camera_create_view","camera_destroy","camera_get_active","camera_get_begin_script","camera_get_default","camera_get_end_script","camera_get_proj_mat","camera_get_update_script","camera_get_view_angle","camera_get_view_border_x","camera_get_view_border_y","camera_get_view_height","camera_get_view_mat","camera_get_view_speed_x","camera_get_view_speed_y","camera_get_view_target","camera_get_view_width","camera_get_view_x","camera_get_view_y","camera_set_begin_script","camera_set_default","camera_set_end_script","camera_set_proj_mat","camera_set_update_script","camera_set_view_angle","camera_set_view_border","camera_set_view_mat","camera_set_view_pos","camera_set_view_size","camera_set_view_speed","camera_set_view_target","ceil","choose","chr","clamp","clickable_add","clickable_add_ext","clickable_change","clickable_change_ext","clickable_delete","clickable_exists","clickable_set_style","clipboard_get_text","clipboard_has_text","clipboard_set_text","cloud_file_save","cloud_string_save","cloud_synchronise","code_is_compiled","collision_circle","collision_circle_list","collision_ellipse","collision_ellipse_list","collision_line","collision_line_list","collision_point","collision_point_list","collision_rectangle","collision_rectangle_list","color_get_blue","color_get_green","color_get_hue","color_get_red","color_get_saturation","color_get_value","colour_get_blue","colour_get_green","colour_get_hue","colour_get_red","colour_get_saturation","colour_get_value","cos","darccos","darcsin","darctan","darctan2","date_compare_date","date_compare_datetime","date_compare_time","date_create_datetime","date_current_datetime","date_date_of","date_date_string","date_datetime_string","date_day_span","date_days_in_month","date_days_in_year","date_get_day","date_get_day_of_year","date_get_hour","date_get_hour_of_year","date_get_minute","date_get_minute_of_year","date_get_month","date_get_second","date_get_second_of_year","date_get_timezone","date_get_week","date_get_weekday","date_get_year","date_hour_span","date_inc_day","date_inc_hour","date_inc_minute","date_inc_month","date_inc_second","date_inc_week","date_inc_year","date_is_today","date_leap_year","date_minute_span","date_month_span","date_second_span","date_set_timezone","date_time_of","date_time_string","date_valid_datetime","date_week_span","date_year_span","db_to_lin","dbg_add_font_glyphs","dbg_button","dbg_checkbox","dbg_color","dbg_colour","dbg_drop_down","dbg_same_line","dbg_section","dbg_section_delete","dbg_section_exists","dbg_slider","dbg_slider_int","dbg_sprite","dbg_text","dbg_text_input","dbg_view","dbg_view_delete","dbg_view_exists","dbg_watch","dcos","debug_event","debug_get_callstack","degtorad","device_get_tilt_x","device_get_tilt_y","device_get_tilt_z","device_is_keypad_open","device_mouse_check_button","device_mouse_check_button_pressed","device_mouse_check_button_released","device_mouse_dbclick_enable","device_mouse_raw_x","device_mouse_raw_y","device_mouse_x","device_mouse_x_to_gui","device_mouse_y","device_mouse_y_to_gui","directory_create","directory_destroy","directory_exists","display_get_dpi_x","display_get_dpi_y","display_get_frequency","display_get_gui_height","display_get_gui_width","display_get_height","display_get_orientation","display_get_sleep_margin","display_get_timing_method","display_get_width","display_mouse_get_x","display_mouse_get_y","display_mouse_set","display_reset","display_set_gui_maximise","display_set_gui_maximize","display_set_gui_size","display_set_sleep_margin","display_set_timing_method","display_set_ui_visibility","distance_to_object","distance_to_point","dot_product","dot_product_3d","dot_product_3d_normalised","dot_product_3d_normalized","dot_product_normalised","dot_product_normalized","draw_arrow","draw_button","draw_circle","draw_circle_color","draw_circle_colour","draw_clear","draw_clear_alpha","draw_ellipse","draw_ellipse_color","draw_ellipse_colour","draw_enable_drawevent","draw_enable_skeleton_blendmodes","draw_enable_swf_aa","draw_flush","draw_get_alpha","draw_get_color","draw_get_colour","draw_get_enable_skeleton_blendmodes","draw_get_font","draw_get_halign","draw_get_lighting","draw_get_swf_aa_level","draw_get_valign","draw_getpixel","draw_getpixel_ext","draw_healthbar","draw_highscore","draw_light_define_ambient","draw_light_define_direction","draw_light_define_point","draw_light_enable","draw_light_get","draw_light_get_ambient","draw_line","draw_line_color","draw_line_colour","draw_line_width","draw_line_width_color","draw_line_width_colour","draw_path","draw_point","draw_point_color","draw_point_colour","draw_primitive_begin","draw_primitive_begin_texture","draw_primitive_end","draw_rectangle","draw_rectangle_color","draw_rectangle_colour","draw_roundrect","draw_roundrect_color","draw_roundrect_color_ext","draw_roundrect_colour","draw_roundrect_colour_ext","draw_roundrect_ext","draw_self","draw_set_alpha","draw_set_circle_precision","draw_set_color","draw_set_colour","draw_set_font","draw_set_halign","draw_set_lighting","draw_set_swf_aa_level","draw_set_valign","draw_skeleton","draw_skeleton_collision","draw_skeleton_instance","draw_skeleton_time","draw_sprite","draw_sprite_ext","draw_sprite_general","draw_sprite_part","draw_sprite_part_ext","draw_sprite_pos","draw_sprite_stretched","draw_sprite_stretched_ext","draw_sprite_tiled","draw_sprite_tiled_ext","draw_surface","draw_surface_ext","draw_surface_general","draw_surface_part","draw_surface_part_ext","draw_surface_stretched","draw_surface_stretched_ext","draw_surface_tiled","draw_surface_tiled_ext","draw_text","draw_text_color","draw_text_colour","draw_text_ext","draw_text_ext_color","draw_text_ext_colour","draw_text_ext_transformed","draw_text_ext_transformed_color","draw_text_ext_transformed_colour","draw_text_transformed","draw_text_transformed_color","draw_text_transformed_colour","draw_texture_flush","draw_tile","draw_tilemap","draw_triangle","draw_triangle_color","draw_triangle_colour","draw_vertex","draw_vertex_color","draw_vertex_colour","draw_vertex_texture","draw_vertex_texture_color","draw_vertex_texture_colour","ds_exists","ds_grid_add","ds_grid_add_disk","ds_grid_add_grid_region","ds_grid_add_region","ds_grid_clear","ds_grid_copy","ds_grid_create","ds_grid_destroy","ds_grid_get","ds_grid_get_disk_max","ds_grid_get_disk_mean","ds_grid_get_disk_min","ds_grid_get_disk_sum","ds_grid_get_max","ds_grid_get_mean","ds_grid_get_min","ds_grid_get_sum","ds_grid_height","ds_grid_multiply","ds_grid_multiply_disk","ds_grid_multiply_grid_region","ds_grid_multiply_region","ds_grid_read","ds_grid_resize","ds_grid_set","ds_grid_set_disk","ds_grid_set_grid_region","ds_grid_set_region","ds_grid_shuffle","ds_grid_sort","ds_grid_to_mp_grid","ds_grid_value_disk_exists","ds_grid_value_disk_x","ds_grid_value_disk_y","ds_grid_value_exists","ds_grid_value_x","ds_grid_value_y","ds_grid_width","ds_grid_write","ds_list_add","ds_list_clear","ds_list_copy","ds_list_create","ds_list_delete","ds_list_destroy","ds_list_empty","ds_list_find_index","ds_list_find_value","ds_list_insert","ds_list_is_list","ds_list_is_map","ds_list_mark_as_list","ds_list_mark_as_map","ds_list_read","ds_list_replace","ds_list_set","ds_list_shuffle","ds_list_size","ds_list_sort","ds_list_write","ds_map_add","ds_map_add_list","ds_map_add_map","ds_map_clear","ds_map_copy","ds_map_create","ds_map_delete","ds_map_destroy","ds_map_empty","ds_map_exists","ds_map_find_first","ds_map_find_last","ds_map_find_next","ds_map_find_previous","ds_map_find_value","ds_map_is_list","ds_map_is_map","ds_map_keys_to_array","ds_map_read","ds_map_replace","ds_map_replace_list","ds_map_replace_map","ds_map_secure_load","ds_map_secure_load_buffer","ds_map_secure_save","ds_map_secure_save_buffer","ds_map_set","ds_map_size","ds_map_values_to_array","ds_map_write","ds_priority_add","ds_priority_change_priority","ds_priority_clear","ds_priority_copy","ds_priority_create","ds_priority_delete_max","ds_priority_delete_min","ds_priority_delete_value","ds_priority_destroy","ds_priority_empty","ds_priority_find_max","ds_priority_find_min","ds_priority_find_priority","ds_priority_read","ds_priority_size","ds_priority_write","ds_queue_clear","ds_queue_copy","ds_queue_create","ds_queue_dequeue","ds_queue_destroy","ds_queue_empty","ds_queue_enqueue","ds_queue_head","ds_queue_read","ds_queue_size","ds_queue_tail","ds_queue_write","ds_set_precision","ds_stack_clear","ds_stack_copy","ds_stack_create","ds_stack_destroy","ds_stack_empty","ds_stack_pop","ds_stack_push","ds_stack_read","ds_stack_size","ds_stack_top","ds_stack_write","dsin","dtan","effect_clear","effect_create_above","effect_create_below","effect_create_depth","effect_create_layer","environment_get_variable","event_inherited","event_perform","event_perform_async","event_perform_object","event_user","exception_unhandled_handler","exp","extension_exists","extension_get_option_count","extension_get_option_names","extension_get_option_value","extension_get_options","extension_get_version","external_call","external_define","external_free","file_attributes","file_bin_close","file_bin_open","file_bin_position","file_bin_read_byte","file_bin_rewrite","file_bin_seek","file_bin_size","file_bin_write_byte","file_copy","file_delete","file_exists","file_find_close","file_find_first","file_find_next","file_rename","file_text_close","file_text_eof","file_text_eoln","file_text_open_append","file_text_open_from_string","file_text_open_read","file_text_open_write","file_text_read_real","file_text_read_string","file_text_readln","file_text_write_real","file_text_write_string","file_text_writeln","filename_change_ext","filename_dir","filename_drive","filename_ext","filename_name","filename_path","floor","font_add","font_add_enable_aa","font_add_get_enable_aa","font_add_sprite","font_add_sprite_ext","font_cache_glyph","font_delete","font_enable_effects","font_enable_sdf","font_exists","font_get_bold","font_get_first","font_get_fontname","font_get_info","font_get_italic","font_get_last","font_get_name","font_get_sdf_enabled","font_get_sdf_spread","font_get_size","font_get_texture","font_get_uvs","font_replace_sprite","font_replace_sprite_ext","font_sdf_spread","font_set_cache_size","frac","fx_create","fx_get_name","fx_get_parameter","fx_get_parameter_names","fx_get_parameters","fx_get_single_layer","fx_set_parameter","fx_set_parameters","fx_set_single_layer","game_change","game_end","game_get_speed","game_load","game_load_buffer","game_restart","game_save","game_save_buffer","game_set_speed","gamepad_axis_count","gamepad_axis_value","gamepad_button_check","gamepad_button_check_pressed","gamepad_button_check_released","gamepad_button_count","gamepad_button_value","gamepad_get_axis_deadzone","gamepad_get_button_threshold","gamepad_get_description","gamepad_get_device_count","gamepad_get_guid","gamepad_get_mapping","gamepad_get_option","gamepad_hat_count","gamepad_hat_value","gamepad_is_connected","gamepad_is_supported","gamepad_remove_mapping","gamepad_set_axis_deadzone","gamepad_set_button_threshold","gamepad_set_color","gamepad_set_colour","gamepad_set_option","gamepad_set_vibration","gamepad_test_mapping","gc_collect","gc_enable","gc_get_stats","gc_get_target_frame_time","gc_is_enabled","gc_target_frame_time","gesture_double_tap_distance","gesture_double_tap_time","gesture_drag_distance","gesture_drag_time","gesture_flick_speed","gesture_get_double_tap_distance","gesture_get_double_tap_time","gesture_get_drag_distance","gesture_get_drag_time","gesture_get_flick_speed","gesture_get_pinch_angle_away","gesture_get_pinch_angle_towards","gesture_get_pinch_distance","gesture_get_rotate_angle","gesture_get_rotate_time","gesture_get_tap_count","gesture_pinch_angle_away","gesture_pinch_angle_towards","gesture_pinch_distance","gesture_rotate_angle","gesture_rotate_time","gesture_tap_count","get_integer","get_integer_async","get_login_async","get_open_filename","get_open_filename_ext","get_save_filename","get_save_filename_ext","get_string","get_string_async","get_timer","gif_add_surface","gif_open","gif_save","gif_save_buffer","gml_pragma","gml_release_mode","gpu_get_alphatestenable","gpu_get_alphatestref","gpu_get_blendenable","gpu_get_blendmode","gpu_get_blendmode_dest","gpu_get_blendmode_destalpha","gpu_get_blendmode_ext","gpu_get_blendmode_ext_sepalpha","gpu_get_blendmode_src","gpu_get_blendmode_srcalpha","gpu_get_colorwriteenable","gpu_get_colourwriteenable","gpu_get_cullmode","gpu_get_depth","gpu_get_fog","gpu_get_state","gpu_get_tex_filter","gpu_get_tex_filter_ext","gpu_get_tex_max_aniso","gpu_get_tex_max_aniso_ext","gpu_get_tex_max_mip","gpu_get_tex_max_mip_ext","gpu_get_tex_min_mip","gpu_get_tex_min_mip_ext","gpu_get_tex_mip_bias","gpu_get_tex_mip_bias_ext","gpu_get_tex_mip_enable","gpu_get_tex_mip_enable_ext","gpu_get_tex_mip_filter","gpu_get_tex_mip_filter_ext","gpu_get_tex_repeat","gpu_get_tex_repeat_ext","gpu_get_texfilter","gpu_get_texfilter_ext","gpu_get_texrepeat","gpu_get_texrepeat_ext","gpu_get_zfunc","gpu_get_ztestenable","gpu_get_zwriteenable","gpu_pop_state","gpu_push_state","gpu_set_alphatestenable","gpu_set_alphatestref","gpu_set_blendenable","gpu_set_blendmode","gpu_set_blendmode_ext","gpu_set_blendmode_ext_sepalpha","gpu_set_colorwriteenable","gpu_set_colourwriteenable","gpu_set_cullmode","gpu_set_depth","gpu_set_fog","gpu_set_state","gpu_set_tex_filter","gpu_set_tex_filter_ext","gpu_set_tex_max_aniso","gpu_set_tex_max_aniso_ext","gpu_set_tex_max_mip","gpu_set_tex_max_mip_ext","gpu_set_tex_min_mip","gpu_set_tex_min_mip_ext","gpu_set_tex_mip_bias","gpu_set_tex_mip_bias_ext","gpu_set_tex_mip_enable","gpu_set_tex_mip_enable_ext","gpu_set_tex_mip_filter","gpu_set_tex_mip_filter_ext","gpu_set_tex_repeat","gpu_set_tex_repeat_ext","gpu_set_texfilter","gpu_set_texfilter_ext","gpu_set_texrepeat","gpu_set_texrepeat_ext","gpu_set_zfunc","gpu_set_ztestenable","gpu_set_zwriteenable","handle_parse","highscore_add","highscore_clear","highscore_name","highscore_value","http_get","http_get_file","http_get_request_crossorigin","http_post_string","http_request","http_set_request_crossorigin","iap_acquire","iap_activate","iap_consume","iap_enumerate_products","iap_product_details","iap_purchase_details","iap_restore_all","iap_status","ini_close","ini_key_delete","ini_key_exists","ini_open","ini_open_from_string","ini_read_real","ini_read_string","ini_section_delete","ini_section_exists","ini_write_real","ini_write_string","instance_activate_all","instance_activate_layer","instance_activate_object","instance_activate_region","instance_change","instance_copy","instance_create_depth","instance_create_layer","instance_deactivate_all","instance_deactivate_layer","instance_deactivate_object","instance_deactivate_region","instance_destroy","instance_exists","instance_find","instance_furthest","instance_id_get","instance_nearest","instance_number","instance_place","instance_place_list","instance_position","instance_position_list","instanceof","int64","io_clear","irandom","irandom_range","is_array","is_bool","is_callable","is_debug_overlay_open","is_handle","is_infinity","is_instanceof","is_int32","is_int64","is_keyboard_used_debug_overlay","is_method","is_mouse_over_debug_overlay","is_nan","is_numeric","is_ptr","is_real","is_string","is_struct","is_undefined","json_decode","json_encode","json_parse","json_stringify","keyboard_check","keyboard_check_direct","keyboard_check_pressed","keyboard_check_released","keyboard_clear","keyboard_get_map","keyboard_get_numlock","keyboard_key_press","keyboard_key_release","keyboard_set_map","keyboard_set_numlock","keyboard_unset_map","keyboard_virtual_height","keyboard_virtual_hide","keyboard_virtual_show","keyboard_virtual_status","layer_add_instance","layer_background_alpha","layer_background_blend","layer_background_change","layer_background_create","layer_background_destroy","layer_background_exists","layer_background_get_alpha","layer_background_get_blend","layer_background_get_htiled","layer_background_get_id","layer_background_get_index","layer_background_get_speed","layer_background_get_sprite","layer_background_get_stretch","layer_background_get_visible","layer_background_get_vtiled","layer_background_get_xscale","layer_background_get_yscale","layer_background_htiled","layer_background_index","layer_background_speed","layer_background_sprite","layer_background_stretch","layer_background_visible","layer_background_vtiled","layer_background_xscale","layer_background_yscale","layer_clear_fx","layer_create","layer_depth","layer_destroy","layer_destroy_instances","layer_element_move","layer_enable_fx","layer_exists","layer_force_draw_depth","layer_fx_is_enabled","layer_get_all","layer_get_all_elements","layer_get_depth","layer_get_element_layer","layer_get_element_type","layer_get_forced_depth","layer_get_fx","layer_get_hspeed","layer_get_id","layer_get_id_at_depth","layer_get_name","layer_get_script_begin","layer_get_script_end","layer_get_shader","layer_get_target_room","layer_get_visible","layer_get_vspeed","layer_get_x","layer_get_y","layer_has_instance","layer_hspeed","layer_instance_get_instance","layer_is_draw_depth_forced","layer_reset_target_room","layer_script_begin","layer_script_end","layer_sequence_angle","layer_sequence_create","layer_sequence_destroy","layer_sequence_exists","layer_sequence_get_angle","layer_sequence_get_headdir","layer_sequence_get_headpos","layer_sequence_get_instance","layer_sequence_get_length","layer_sequence_get_sequence","layer_sequence_get_speedscale","layer_sequence_get_x","layer_sequence_get_xscale","layer_sequence_get_y","layer_sequence_get_yscale","layer_sequence_headdir","layer_sequence_headpos","layer_sequence_is_finished","layer_sequence_is_paused","layer_sequence_pause","layer_sequence_play","layer_sequence_speedscale","layer_sequence_x","layer_sequence_xscale","layer_sequence_y","layer_sequence_yscale","layer_set_fx","layer_set_target_room","layer_set_visible","layer_shader","layer_sprite_alpha","layer_sprite_angle","layer_sprite_blend","layer_sprite_change","layer_sprite_create","layer_sprite_destroy","layer_sprite_exists","layer_sprite_get_alpha","layer_sprite_get_angle","layer_sprite_get_blend","layer_sprite_get_id","layer_sprite_get_index","layer_sprite_get_speed","layer_sprite_get_sprite","layer_sprite_get_x","layer_sprite_get_xscale","layer_sprite_get_y","layer_sprite_get_yscale","layer_sprite_index","layer_sprite_speed","layer_sprite_x","layer_sprite_xscale","layer_sprite_y","layer_sprite_yscale","layer_tile_alpha","layer_tile_blend","layer_tile_change","layer_tile_create","layer_tile_destroy","layer_tile_exists","layer_tile_get_alpha","layer_tile_get_blend","layer_tile_get_region","layer_tile_get_sprite","layer_tile_get_visible","layer_tile_get_x","layer_tile_get_xscale","layer_tile_get_y","layer_tile_get_yscale","layer_tile_region","layer_tile_visible","layer_tile_x","layer_tile_xscale","layer_tile_y","layer_tile_yscale","layer_tilemap_create","layer_tilemap_destroy","layer_tilemap_exists","layer_tilemap_get_id","layer_vspeed","layer_x","layer_y","lengthdir_x","lengthdir_y","lerp","lin_to_db","ln","load_csv","log10","log2","logn","make_color_hsv","make_color_rgb","make_colour_hsv","make_colour_rgb","math_get_epsilon","math_set_epsilon","matrix_build","matrix_build_identity","matrix_build_lookat","matrix_build_projection_ortho","matrix_build_projection_perspective","matrix_build_projection_perspective_fov","matrix_get","matrix_multiply","matrix_set","matrix_stack_clear","matrix_stack_is_empty","matrix_stack_pop","matrix_stack_push","matrix_stack_set","matrix_stack_top","matrix_transform_vertex","max","md5_file","md5_string_unicode","md5_string_utf8","mean","median","merge_color","merge_colour","method","method_call","method_get_index","method_get_self","min","motion_add","motion_set","mouse_check_button","mouse_check_button_pressed","mouse_check_button_released","mouse_clear","mouse_wheel_down","mouse_wheel_up","move_and_collide","move_bounce_all","move_bounce_solid","move_contact_all","move_contact_solid","move_outside_all","move_outside_solid","move_random","move_snap","move_towards_point","move_wrap","mp_grid_add_cell","mp_grid_add_instances","mp_grid_add_rectangle","mp_grid_clear_all","mp_grid_clear_cell","mp_grid_clear_rectangle","mp_grid_create","mp_grid_destroy","mp_grid_draw","mp_grid_get_cell","mp_grid_path","mp_grid_to_ds_grid","mp_linear_path","mp_linear_path_object","mp_linear_step","mp_linear_step_object","mp_potential_path","mp_potential_path_object","mp_potential_settings","mp_potential_step","mp_potential_step_object","nameof","network_connect","network_connect_async","network_connect_raw","network_connect_raw_async","network_create_server","network_create_server_raw","network_create_socket","network_create_socket_ext","network_destroy","network_resolve","network_send_broadcast","network_send_packet","network_send_raw","network_send_udp","network_send_udp_raw","network_set_config","network_set_timeout","object_exists","object_get_mask","object_get_name","object_get_parent","object_get_persistent","object_get_physics","object_get_solid","object_get_sprite","object_get_visible","object_is_ancestor","object_set_mask","object_set_persistent","object_set_solid","object_set_sprite","object_set_visible","ord","os_check_permission","os_get_config","os_get_info","os_get_language","os_get_region","os_is_network_connected","os_is_paused","os_lock_orientation","os_powersave_enable","os_request_permission","os_set_orientation_lock","parameter_count","parameter_string","part_emitter_burst","part_emitter_clear","part_emitter_create","part_emitter_delay","part_emitter_destroy","part_emitter_destroy_all","part_emitter_enable","part_emitter_exists","part_emitter_interval","part_emitter_region","part_emitter_relative","part_emitter_stream","part_particles_burst","part_particles_clear","part_particles_count","part_particles_create","part_particles_create_color","part_particles_create_colour","part_system_angle","part_system_automatic_draw","part_system_automatic_update","part_system_clear","part_system_color","part_system_colour","part_system_create","part_system_create_layer","part_system_depth","part_system_destroy","part_system_draw_order","part_system_drawit","part_system_exists","part_system_get_info","part_system_get_layer","part_system_global_space","part_system_layer","part_system_position","part_system_update","part_type_alpha1","part_type_alpha2","part_type_alpha3","part_type_blend","part_type_clear","part_type_color1","part_type_color2","part_type_color3","part_type_color_hsv","part_type_color_mix","part_type_color_rgb","part_type_colour1","part_type_colour2","part_type_colour3","part_type_colour_hsv","part_type_colour_mix","part_type_colour_rgb","part_type_create","part_type_death","part_type_destroy","part_type_direction","part_type_exists","part_type_gravity","part_type_life","part_type_orientation","part_type_scale","part_type_shape","part_type_size","part_type_size_x","part_type_size_y","part_type_speed","part_type_sprite","part_type_step","part_type_subimage","particle_exists","particle_get_info","path_add","path_add_point","path_append","path_assign","path_change_point","path_clear_points","path_delete","path_delete_point","path_duplicate","path_end","path_exists","path_flip","path_get_closed","path_get_kind","path_get_length","path_get_name","path_get_number","path_get_point_speed","path_get_point_x","path_get_point_y","path_get_precision","path_get_speed","path_get_x","path_get_y","path_insert_point","path_mirror","path_rescale","path_reverse","path_rotate","path_set_closed","path_set_kind","path_set_precision","path_shift","path_start","physics_apply_angular_impulse","physics_apply_force","physics_apply_impulse","physics_apply_local_force","physics_apply_local_impulse","physics_apply_torque","physics_draw_debug","physics_fixture_add_point","physics_fixture_bind","physics_fixture_bind_ext","physics_fixture_create","physics_fixture_delete","physics_fixture_set_angular_damping","physics_fixture_set_awake","physics_fixture_set_box_shape","physics_fixture_set_chain_shape","physics_fixture_set_circle_shape","physics_fixture_set_collision_group","physics_fixture_set_density","physics_fixture_set_edge_shape","physics_fixture_set_friction","physics_fixture_set_kinematic","physics_fixture_set_linear_damping","physics_fixture_set_polygon_shape","physics_fixture_set_restitution","physics_fixture_set_sensor","physics_get_density","physics_get_friction","physics_get_restitution","physics_joint_delete","physics_joint_distance_create","physics_joint_enable_motor","physics_joint_friction_create","physics_joint_gear_create","physics_joint_get_value","physics_joint_prismatic_create","physics_joint_pulley_create","physics_joint_revolute_create","physics_joint_rope_create","physics_joint_set_value","physics_joint_weld_create","physics_joint_wheel_create","physics_mass_properties","physics_particle_count","physics_particle_create","physics_particle_delete","physics_particle_delete_region_box","physics_particle_delete_region_circle","physics_particle_delete_region_poly","physics_particle_draw","physics_particle_draw_ext","physics_particle_get_damping","physics_particle_get_data","physics_particle_get_data_particle","physics_particle_get_density","physics_particle_get_gravity_scale","physics_particle_get_group_flags","physics_particle_get_max_count","physics_particle_get_radius","physics_particle_group_add_point","physics_particle_group_begin","physics_particle_group_box","physics_particle_group_circle","physics_particle_group_count","physics_particle_group_delete","physics_particle_group_end","physics_particle_group_get_ang_vel","physics_particle_group_get_angle","physics_particle_group_get_centre_x","physics_particle_group_get_centre_y","physics_particle_group_get_data","physics_particle_group_get_inertia","physics_particle_group_get_mass","physics_particle_group_get_vel_x","physics_particle_group_get_vel_y","physics_particle_group_get_x","physics_particle_group_get_y","physics_particle_group_join","physics_particle_group_polygon","physics_particle_set_category_flags","physics_particle_set_damping","physics_particle_set_density","physics_particle_set_flags","physics_particle_set_gravity_scale","physics_particle_set_group_flags","physics_particle_set_max_count","physics_particle_set_radius","physics_pause_enable","physics_remove_fixture","physics_set_density","physics_set_friction","physics_set_restitution","physics_test_overlap","physics_world_create","physics_world_draw_debug","physics_world_gravity","physics_world_update_iterations","physics_world_update_speed","place_empty","place_free","place_meeting","place_snapped","point_direction","point_distance","point_distance_3d","point_in_circle","point_in_rectangle","point_in_triangle","position_change","position_destroy","position_empty","position_meeting","power","ptr","radtodeg","random","random_get_seed","random_range","random_set_seed","randomise","randomize","real","rectangle_in_circle","rectangle_in_rectangle","rectangle_in_triangle","ref_create","rollback_chat","rollback_create_game","rollback_define_extra_network_latency","rollback_define_input","rollback_define_input_frame_delay","rollback_define_mock_input","rollback_define_player","rollback_display_events","rollback_get_info","rollback_get_input","rollback_get_player_prefs","rollback_join_game","rollback_leave_game","rollback_set_player_prefs","rollback_start_game","rollback_sync_on_frame","rollback_use_late_join","rollback_use_manual_start","rollback_use_player_prefs","rollback_use_random_input","room_add","room_assign","room_duplicate","room_exists","room_get_camera","room_get_info","room_get_name","room_get_viewport","room_goto","room_goto_next","room_goto_previous","room_instance_add","room_instance_clear","room_next","room_previous","room_restart","room_set_camera","room_set_height","room_set_persistent","room_set_view_enabled","room_set_viewport","room_set_width","round","scheduler_resolution_get","scheduler_resolution_set","screen_save","screen_save_part","script_execute","script_execute_ext","script_exists","script_get_name","sequence_create","sequence_destroy","sequence_exists","sequence_get","sequence_get_objects","sequence_instance_override_object","sequence_keyframe_new","sequence_keyframedata_new","sequence_track_new","sha1_file","sha1_string_unicode","sha1_string_utf8","shader_current","shader_enable_corner_id","shader_get_name","shader_get_sampler_index","shader_get_uniform","shader_is_compiled","shader_reset","shader_set","shader_set_uniform_f","shader_set_uniform_f_array","shader_set_uniform_f_buffer","shader_set_uniform_i","shader_set_uniform_i_array","shader_set_uniform_matrix","shader_set_uniform_matrix_array","shaders_are_supported","shop_leave_rating","show_debug_message","show_debug_message_ext","show_debug_overlay","show_error","show_message","show_message_async","show_question","show_question_async","sign","sin","skeleton_animation_clear","skeleton_animation_get","skeleton_animation_get_duration","skeleton_animation_get_event_frames","skeleton_animation_get_ext","skeleton_animation_get_frame","skeleton_animation_get_frames","skeleton_animation_get_position","skeleton_animation_is_finished","skeleton_animation_is_looping","skeleton_animation_list","skeleton_animation_mix","skeleton_animation_set","skeleton_animation_set_ext","skeleton_animation_set_frame","skeleton_animation_set_position","skeleton_attachment_create","skeleton_attachment_create_color","skeleton_attachment_create_colour","skeleton_attachment_destroy","skeleton_attachment_exists","skeleton_attachment_get","skeleton_attachment_replace","skeleton_attachment_replace_color","skeleton_attachment_replace_colour","skeleton_attachment_set","skeleton_bone_data_get","skeleton_bone_data_set","skeleton_bone_list","skeleton_bone_state_get","skeleton_bone_state_set","skeleton_collision_draw_set","skeleton_find_slot","skeleton_get_bounds","skeleton_get_minmax","skeleton_get_num_bounds","skeleton_skin_create","skeleton_skin_get","skeleton_skin_list","skeleton_skin_set","skeleton_slot_alpha_get","skeleton_slot_color_get","skeleton_slot_color_set","skeleton_slot_colour_get","skeleton_slot_colour_set","skeleton_slot_data","skeleton_slot_data_instance","skeleton_slot_list","sprite_add","sprite_add_ext","sprite_add_from_surface","sprite_assign","sprite_collision_mask","sprite_create_from_surface","sprite_delete","sprite_duplicate","sprite_exists","sprite_flush","sprite_flush_multi","sprite_get_bbox_bottom","sprite_get_bbox_left","sprite_get_bbox_mode","sprite_get_bbox_right","sprite_get_bbox_top","sprite_get_height","sprite_get_info","sprite_get_name","sprite_get_nineslice","sprite_get_number","sprite_get_speed","sprite_get_speed_type","sprite_get_texture","sprite_get_tpe","sprite_get_uvs","sprite_get_width","sprite_get_xoffset","sprite_get_yoffset","sprite_merge","sprite_nineslice_create","sprite_prefetch","sprite_prefetch_multi","sprite_replace","sprite_save","sprite_save_strip","sprite_set_alpha_from_sprite","sprite_set_bbox","sprite_set_bbox_mode","sprite_set_cache_size","sprite_set_cache_size_ext","sprite_set_nineslice","sprite_set_offset","sprite_set_speed","sqr","sqrt","static_get","static_set","string","string_byte_at","string_byte_length","string_char_at","string_concat","string_concat_ext","string_copy","string_count","string_delete","string_digits","string_ends_with","string_ext","string_foreach","string_format","string_hash_to_newline","string_height","string_height_ext","string_insert","string_join","string_join_ext","string_last_pos","string_last_pos_ext","string_length","string_letters","string_lettersdigits","string_lower","string_ord_at","string_pos","string_pos_ext","string_repeat","string_replace","string_replace_all","string_set_byte_at","string_split","string_split_ext","string_starts_with","string_trim","string_trim_end","string_trim_start","string_upper","string_width","string_width_ext","struct_exists","struct_foreach","struct_get","struct_get_from_hash","struct_get_names","struct_names_count","struct_remove","struct_set","struct_set_from_hash","surface_copy","surface_copy_part","surface_create","surface_create_ext","surface_depth_disable","surface_exists","surface_format_is_supported","surface_free","surface_get_depth_disable","surface_get_format","surface_get_height","surface_get_target","surface_get_target_ext","surface_get_texture","surface_get_width","surface_getpixel","surface_getpixel_ext","surface_reset_target","surface_resize","surface_save","surface_save_part","surface_set_target","surface_set_target_ext","tag_get_asset_ids","tag_get_assets","tan","texture_debug_messages","texture_flush","texture_get_height","texture_get_texel_height","texture_get_texel_width","texture_get_uvs","texture_get_width","texture_global_scale","texture_is_ready","texture_prefetch","texture_set_stage","texturegroup_get_fonts","texturegroup_get_names","texturegroup_get_sprites","texturegroup_get_status","texturegroup_get_textures","texturegroup_get_tilesets","texturegroup_load","texturegroup_set_mode","texturegroup_unload","tile_get_empty","tile_get_flip","tile_get_index","tile_get_mirror","tile_get_rotate","tile_set_empty","tile_set_flip","tile_set_index","tile_set_mirror","tile_set_rotate","tilemap_clear","tilemap_get","tilemap_get_at_pixel","tilemap_get_cell_x_at_pixel","tilemap_get_cell_y_at_pixel","tilemap_get_frame","tilemap_get_global_mask","tilemap_get_height","tilemap_get_mask","tilemap_get_tile_height","tilemap_get_tile_width","tilemap_get_tileset","tilemap_get_width","tilemap_get_x","tilemap_get_y","tilemap_set","tilemap_set_at_pixel","tilemap_set_global_mask","tilemap_set_height","tilemap_set_mask","tilemap_set_width","tilemap_tileset","tilemap_x","tilemap_y","tileset_get_info","tileset_get_name","tileset_get_texture","tileset_get_uvs","time_bpm_to_seconds","time_seconds_to_bpm","time_source_create","time_source_destroy","time_source_exists","time_source_get_children","time_source_get_parent","time_source_get_period","time_source_get_reps_completed","time_source_get_reps_remaining","time_source_get_state","time_source_get_time_remaining","time_source_get_units","time_source_pause","time_source_reconfigure","time_source_reset","time_source_resume","time_source_start","time_source_stop","timeline_add","timeline_clear","timeline_delete","timeline_exists","timeline_get_name","timeline_max_moment","timeline_moment_add_script","timeline_moment_clear","timeline_size","typeof","url_get_domain","url_open","url_open_ext","url_open_full","uwp_device_touchscreen_available","uwp_livetile_badge_clear","uwp_livetile_badge_notification","uwp_livetile_notification_begin","uwp_livetile_notification_end","uwp_livetile_notification_expiry","uwp_livetile_notification_image_add","uwp_livetile_notification_secondary_begin","uwp_livetile_notification_tag","uwp_livetile_notification_template_add","uwp_livetile_notification_text_add","uwp_livetile_queue_enable","uwp_livetile_tile_clear","uwp_secondarytile_badge_clear","uwp_secondarytile_badge_notification","uwp_secondarytile_delete","uwp_secondarytile_pin","uwp_secondarytile_tile_clear","variable_clone","variable_get_hash","variable_global_exists","variable_global_get","variable_global_set","variable_instance_exists","variable_instance_get","variable_instance_get_names","variable_instance_names_count","variable_instance_set","variable_struct_exists","variable_struct_get","variable_struct_get_names","variable_struct_names_count","variable_struct_remove","variable_struct_set","vertex_argb","vertex_begin","vertex_color","vertex_colour","vertex_create_buffer","vertex_create_buffer_ext","vertex_create_buffer_from_buffer","vertex_create_buffer_from_buffer_ext","vertex_delete_buffer","vertex_end","vertex_float1","vertex_float2","vertex_float3","vertex_float4","vertex_format_add_color","vertex_format_add_colour","vertex_format_add_custom","vertex_format_add_normal","vertex_format_add_position","vertex_format_add_position_3d","vertex_format_add_texcoord","vertex_format_begin","vertex_format_delete","vertex_format_end","vertex_format_get_info","vertex_freeze","vertex_get_buffer_size","vertex_get_number","vertex_normal","vertex_position","vertex_position_3d","vertex_submit","vertex_submit_ext","vertex_texcoord","vertex_ubyte4","vertex_update_buffer_from_buffer","vertex_update_buffer_from_vertex","video_close","video_draw","video_enable_loop","video_get_duration","video_get_format","video_get_position","video_get_status","video_get_volume","video_is_looping","video_open","video_pause","video_resume","video_seek_to","video_set_volume","view_get_camera","view_get_hport","view_get_surface_id","view_get_visible","view_get_wport","view_get_xport","view_get_yport","view_set_camera","view_set_hport","view_set_surface_id","view_set_visible","view_set_wport","view_set_xport","view_set_yport","virtual_key_add","virtual_key_delete","virtual_key_hide","virtual_key_show","wallpaper_set_config","wallpaper_set_subscriptions","weak_ref_alive","weak_ref_any_alive","weak_ref_create","window_center","window_device","window_enable_borderless_fullscreen","window_get_borderless_fullscreen","window_get_caption","window_get_color","window_get_colour","window_get_cursor","window_get_fullscreen","window_get_height","window_get_showborder","window_get_visible_rects","window_get_width","window_get_x","window_get_y","window_handle","window_has_focus","window_mouse_get_delta_x","window_mouse_get_delta_y","window_mouse_get_locked","window_mouse_get_x","window_mouse_get_y","window_mouse_set","window_mouse_set_locked","window_set_caption","window_set_color","window_set_colour","window_set_cursor","window_set_fullscreen","window_set_max_height","window_set_max_width","window_set_min_height","window_set_min_width","window_set_position","window_set_rectangle","window_set_showborder","window_set_size","window_view_mouse_get_x","window_view_mouse_get_y","window_views_mouse_get_x","window_views_mouse_get_y","winphone_tile_background_color","winphone_tile_background_colour","zip_add_file","zip_create","zip_save","zip_unzip","zip_unzip_async"],symbol:["AudioEffect","AudioEffectType","AudioLFOType","GM_build_date","GM_build_type","GM_is_sandboxed","GM_project_filename","GM_runtime_version","GM_version","NaN","_GMFILE_","_GMFUNCTION_","_GMLINE_","alignmentH","alignmentV","all","animcurvetype_bezier","animcurvetype_catmullrom","animcurvetype_linear","asset_animationcurve","asset_font","asset_object","asset_path","asset_room","asset_script","asset_sequence","asset_shader","asset_sound","asset_sprite","asset_tiles","asset_timeline","asset_unknown","audio_3D","audio_bus_main","audio_falloff_exponent_distance","audio_falloff_exponent_distance_clamped","audio_falloff_exponent_distance_scaled","audio_falloff_inverse_distance","audio_falloff_inverse_distance_clamped","audio_falloff_inverse_distance_scaled","audio_falloff_linear_distance","audio_falloff_linear_distance_clamped","audio_falloff_none","audio_mono","audio_stereo","bboxkind_diamond","bboxkind_ellipse","bboxkind_precise","bboxkind_rectangular","bboxmode_automatic","bboxmode_fullimage","bboxmode_manual","bm_add","bm_dest_alpha","bm_dest_color","bm_dest_colour","bm_inv_dest_alpha","bm_inv_dest_color","bm_inv_dest_colour","bm_inv_src_alpha","bm_inv_src_color","bm_inv_src_colour","bm_max","bm_normal","bm_one","bm_src_alpha","bm_src_alpha_sat","bm_src_color","bm_src_colour","bm_subtract","bm_zero","browser_chrome","browser_edge","browser_firefox","browser_ie","browser_ie_mobile","browser_not_a_browser","browser_opera","browser_safari","browser_safari_mobile","browser_tizen","browser_unknown","browser_windows_store","buffer_bool","buffer_f16","buffer_f32","buffer_f64","buffer_fast","buffer_fixed","buffer_grow","buffer_s16","buffer_s32","buffer_s8","buffer_seek_end","buffer_seek_relative","buffer_seek_start","buffer_string","buffer_text","buffer_u16","buffer_u32","buffer_u64","buffer_u8","buffer_vbuffer","buffer_wrap","c_aqua","c_black","c_blue","c_dkgray","c_dkgrey","c_fuchsia","c_gray","c_green","c_grey","c_lime","c_ltgray","c_ltgrey","c_maroon","c_navy","c_olive","c_orange","c_purple","c_red","c_silver","c_teal","c_white","c_yellow","cache_directory","characterSpacing","cmpfunc_always","cmpfunc_equal","cmpfunc_greater","cmpfunc_greaterequal","cmpfunc_less","cmpfunc_lessequal","cmpfunc_never","cmpfunc_notequal","coreColor","coreColour","cr_appstart","cr_arrow","cr_beam","cr_cross","cr_default","cr_drag","cr_handpoint","cr_hourglass","cr_none","cr_size_all","cr_size_nesw","cr_size_ns","cr_size_nwse","cr_size_we","cr_uparrow","cull_clockwise","cull_counterclockwise","cull_noculling","device_emulator","device_ios_ipad","device_ios_ipad_retina","device_ios_iphone","device_ios_iphone5","device_ios_iphone6","device_ios_iphone6plus","device_ios_iphone_retina","device_ios_unknown","device_tablet","display_landscape","display_landscape_flipped","display_portrait","display_portrait_flipped","dll_cdecl","dll_stdcall","dropShadowEnabled","dropShadowEnabled","ds_type_grid","ds_type_list","ds_type_map","ds_type_priority","ds_type_queue","ds_type_stack","ef_cloud","ef_ellipse","ef_explosion","ef_firework","ef_flare","ef_rain","ef_ring","ef_smoke","ef_smokeup","ef_snow","ef_spark","ef_star","effectsEnabled","effectsEnabled","ev_alarm","ev_animation_end","ev_animation_event","ev_animation_update","ev_async_audio_playback","ev_async_audio_playback_ended","ev_async_audio_recording","ev_async_dialog","ev_async_push_notification","ev_async_save_load","ev_async_save_load","ev_async_social","ev_async_system_event","ev_async_web","ev_async_web_cloud","ev_async_web_iap","ev_async_web_image_load","ev_async_web_networking","ev_async_web_steam","ev_audio_playback","ev_audio_playback_ended","ev_audio_recording","ev_boundary","ev_boundary_view0","ev_boundary_view1","ev_boundary_view2","ev_boundary_view3","ev_boundary_view4","ev_boundary_view5","ev_boundary_view6","ev_boundary_view7","ev_broadcast_message","ev_cleanup","ev_collision","ev_create","ev_destroy","ev_dialog_async","ev_draw","ev_draw_begin","ev_draw_end","ev_draw_normal","ev_draw_post","ev_draw_pre","ev_end_of_path","ev_game_end","ev_game_start","ev_gesture","ev_gesture_double_tap","ev_gesture_drag_end","ev_gesture_drag_start","ev_gesture_dragging","ev_gesture_flick","ev_gesture_pinch_end","ev_gesture_pinch_in","ev_gesture_pinch_out","ev_gesture_pinch_start","ev_gesture_rotate_end","ev_gesture_rotate_start","ev_gesture_rotating","ev_gesture_tap","ev_global_gesture_double_tap","ev_global_gesture_drag_end","ev_global_gesture_drag_start","ev_global_gesture_dragging","ev_global_gesture_flick","ev_global_gesture_pinch_end","ev_global_gesture_pinch_in","ev_global_gesture_pinch_out","ev_global_gesture_pinch_start","ev_global_gesture_rotate_end","ev_global_gesture_rotate_start","ev_global_gesture_rotating","ev_global_gesture_tap","ev_global_left_button","ev_global_left_press","ev_global_left_release","ev_global_middle_button","ev_global_middle_press","ev_global_middle_release","ev_global_right_button","ev_global_right_press","ev_global_right_release","ev_gui","ev_gui_begin","ev_gui_end","ev_joystick1_button1","ev_joystick1_button2","ev_joystick1_button3","ev_joystick1_button4","ev_joystick1_button5","ev_joystick1_button6","ev_joystick1_button7","ev_joystick1_button8","ev_joystick1_down","ev_joystick1_left","ev_joystick1_right","ev_joystick1_up","ev_joystick2_button1","ev_joystick2_button2","ev_joystick2_button3","ev_joystick2_button4","ev_joystick2_button5","ev_joystick2_button6","ev_joystick2_button7","ev_joystick2_button8","ev_joystick2_down","ev_joystick2_left","ev_joystick2_right","ev_joystick2_up","ev_keyboard","ev_keypress","ev_keyrelease","ev_left_button","ev_left_press","ev_left_release","ev_middle_button","ev_middle_press","ev_middle_release","ev_mouse","ev_mouse_enter","ev_mouse_leave","ev_mouse_wheel_down","ev_mouse_wheel_up","ev_no_button","ev_no_more_health","ev_no_more_lives","ev_other","ev_outside","ev_outside_view0","ev_outside_view1","ev_outside_view2","ev_outside_view3","ev_outside_view4","ev_outside_view5","ev_outside_view6","ev_outside_view7","ev_pre_create","ev_push_notification","ev_right_button","ev_right_press","ev_right_release","ev_room_end","ev_room_start","ev_social","ev_step","ev_step_begin","ev_step_end","ev_step_normal","ev_system_event","ev_trigger","ev_user0","ev_user1","ev_user10","ev_user11","ev_user12","ev_user13","ev_user14","ev_user15","ev_user2","ev_user3","ev_user4","ev_user5","ev_user6","ev_user7","ev_user8","ev_user9","ev_web_async","ev_web_cloud","ev_web_iap","ev_web_image_load","ev_web_networking","ev_web_sound_load","ev_web_steam","fa_archive","fa_bottom","fa_center","fa_directory","fa_hidden","fa_left","fa_middle","fa_none","fa_readonly","fa_right","fa_sysfile","fa_top","fa_volumeid","false","frameSizeX","frameSizeY","gamespeed_fps","gamespeed_microseconds","global","glowColor","glowColour","glowEnabled","glowEnabled","glowEnd","glowStart","gp_axis_acceleration_x","gp_axis_acceleration_y","gp_axis_acceleration_z","gp_axis_angular_velocity_x","gp_axis_angular_velocity_y","gp_axis_angular_velocity_z","gp_axis_orientation_w","gp_axis_orientation_x","gp_axis_orientation_y","gp_axis_orientation_z","gp_axislh","gp_axislv","gp_axisrh","gp_axisrv","gp_face1","gp_face2","gp_face3","gp_face4","gp_padd","gp_padl","gp_padr","gp_padu","gp_select","gp_shoulderl","gp_shoulderlb","gp_shoulderr","gp_shoulderrb","gp_start","gp_stickl","gp_stickr","iap_available","iap_canceled","iap_ev_consume","iap_ev_product","iap_ev_purchase","iap_ev_restore","iap_ev_storeload","iap_failed","iap_purchased","iap_refunded","iap_status_available","iap_status_loading","iap_status_processing","iap_status_restoring","iap_status_unavailable","iap_status_uninitialised","iap_storeload_failed","iap_storeload_ok","iap_unavailable","infinity","kbv_autocapitalize_characters","kbv_autocapitalize_none","kbv_autocapitalize_sentences","kbv_autocapitalize_words","kbv_returnkey_continue","kbv_returnkey_default","kbv_returnkey_done","kbv_returnkey_emergency","kbv_returnkey_go","kbv_returnkey_google","kbv_returnkey_join","kbv_returnkey_next","kbv_returnkey_route","kbv_returnkey_search","kbv_returnkey_send","kbv_returnkey_yahoo","kbv_type_ascii","kbv_type_default","kbv_type_email","kbv_type_numbers","kbv_type_phone","kbv_type_phone_name","kbv_type_url","layerelementtype_background","layerelementtype_instance","layerelementtype_oldtilemap","layerelementtype_particlesystem","layerelementtype_sequence","layerelementtype_sprite","layerelementtype_tile","layerelementtype_tilemap","layerelementtype_undefined","leaderboard_type_number","leaderboard_type_time_mins_secs","lighttype_dir","lighttype_point","lineSpacing","m_axisx","m_axisx_gui","m_axisy","m_axisy_gui","m_scroll_down","m_scroll_up","matrix_projection","matrix_view","matrix_world","mb_any","mb_left","mb_middle","mb_none","mb_right","mb_side1","mb_side2","mip_markedonly","mip_off","mip_on","network_config_avoid_time_wait","network_config_connect_timeout","network_config_disable_multicast","network_config_disable_reliable_udp","network_config_enable_multicast","network_config_enable_reliable_udp","network_config_use_non_blocking_socket","network_config_websocket_protocol","network_connect_active","network_connect_blocking","network_connect_nonblocking","network_connect_none","network_connect_passive","network_send_binary","network_send_text","network_socket_bluetooth","network_socket_tcp","network_socket_udp","network_socket_ws","network_socket_wss","network_type_connect","network_type_data","network_type_disconnect","network_type_down","network_type_non_blocking_connect","network_type_up","network_type_up_failed","nineslice_blank","nineslice_bottom","nineslice_center","nineslice_centre","nineslice_hide","nineslice_left","nineslice_mirror","nineslice_repeat","nineslice_right","nineslice_stretch","nineslice_top","noone","of_challenge_lose","of_challenge_tie","of_challenge_win","os_android","os_gdk","os_gxgames","os_ios","os_linux","os_macosx","os_operagx","os_permission_denied","os_permission_denied_dont_request","os_permission_granted","os_ps3","os_ps4","os_ps5","os_psvita","os_switch","os_tvos","os_unknown","os_uwp","os_win8native","os_windows","os_winphone","os_xboxone","os_xboxseriesxs","other","outlineColor","outlineColour","outlineDist","outlineEnabled","outlineEnabled","paragraphSpacing","path_action_continue","path_action_restart","path_action_reverse","path_action_stop","phy_debug_render_aabb","phy_debug_render_collision_pairs","phy_debug_render_coms","phy_debug_render_core_shapes","phy_debug_render_joints","phy_debug_render_obb","phy_debug_render_shapes","phy_joint_anchor_1_x","phy_joint_anchor_1_y","phy_joint_anchor_2_x","phy_joint_anchor_2_y","phy_joint_angle","phy_joint_angle_limits","phy_joint_damping_ratio","phy_joint_frequency","phy_joint_length_1","phy_joint_length_2","phy_joint_lower_angle_limit","phy_joint_max_force","phy_joint_max_length","phy_joint_max_motor_force","phy_joint_max_motor_torque","phy_joint_max_torque","phy_joint_motor_force","phy_joint_motor_speed","phy_joint_motor_torque","phy_joint_reaction_force_x","phy_joint_reaction_force_y","phy_joint_reaction_torque","phy_joint_speed","phy_joint_translation","phy_joint_upper_angle_limit","phy_particle_data_flag_category","phy_particle_data_flag_color","phy_particle_data_flag_colour","phy_particle_data_flag_position","phy_particle_data_flag_typeflags","phy_particle_data_flag_velocity","phy_particle_flag_colormixing","phy_particle_flag_colourmixing","phy_particle_flag_elastic","phy_particle_flag_powder","phy_particle_flag_spring","phy_particle_flag_tensile","phy_particle_flag_viscous","phy_particle_flag_wall","phy_particle_flag_water","phy_particle_flag_zombie","phy_particle_group_flag_rigid","phy_particle_group_flag_solid","pi","pointer_invalid","pointer_null","pr_linelist","pr_linestrip","pr_pointlist","pr_trianglefan","pr_trianglelist","pr_trianglestrip","ps_distr_gaussian","ps_distr_invgaussian","ps_distr_linear","ps_mode_burst","ps_mode_stream","ps_shape_diamond","ps_shape_ellipse","ps_shape_line","ps_shape_rectangle","pt_shape_circle","pt_shape_cloud","pt_shape_disk","pt_shape_explosion","pt_shape_flare","pt_shape_line","pt_shape_pixel","pt_shape_ring","pt_shape_smoke","pt_shape_snow","pt_shape_spark","pt_shape_sphere","pt_shape_square","pt_shape_star","rollback_chat_message","rollback_connect_error","rollback_connect_info","rollback_connected_to_peer","rollback_connection_rejected","rollback_disconnected_from_peer","rollback_end_game","rollback_game_full","rollback_game_info","rollback_game_interrupted","rollback_game_resumed","rollback_high_latency","rollback_player_prefs","rollback_protocol_rejected","rollback_synchronized_with_peer","rollback_synchronizing_with_peer","self","seqaudiokey_loop","seqaudiokey_oneshot","seqdir_left","seqdir_right","seqinterpolation_assign","seqinterpolation_lerp","seqplay_loop","seqplay_oneshot","seqplay_pingpong","seqtextkey_bottom","seqtextkey_center","seqtextkey_justify","seqtextkey_left","seqtextkey_middle","seqtextkey_right","seqtextkey_top","seqtracktype_audio","seqtracktype_bool","seqtracktype_clipmask","seqtracktype_clipmask_mask","seqtracktype_clipmask_subject","seqtracktype_color","seqtracktype_colour","seqtracktype_empty","seqtracktype_graphic","seqtracktype_group","seqtracktype_instance","seqtracktype_message","seqtracktype_moment","seqtracktype_particlesystem","seqtracktype_real","seqtracktype_sequence","seqtracktype_spriteframes","seqtracktype_string","seqtracktype_text","shadowColor","shadowColour","shadowOffsetX","shadowOffsetY","shadowSoftness","sprite_add_ext_error_cancelled","sprite_add_ext_error_decompressfailed","sprite_add_ext_error_loadfailed","sprite_add_ext_error_setupfailed","sprite_add_ext_error_spritenotfound","sprite_add_ext_error_unknown","spritespeed_framespergameframe","spritespeed_framespersecond","surface_r16float","surface_r32float","surface_r8unorm","surface_rg8unorm","surface_rgba16float","surface_rgba32float","surface_rgba4unorm","surface_rgba8unorm","texturegroup_status_fetched","texturegroup_status_loaded","texturegroup_status_loading","texturegroup_status_unloaded","tf_anisotropic","tf_linear","tf_point","thickness","tile_flip","tile_index_mask","tile_mirror","tile_rotate","time_source_expire_after","time_source_expire_nearest","time_source_game","time_source_global","time_source_state_active","time_source_state_initial","time_source_state_paused","time_source_state_stopped","time_source_units_frames","time_source_units_seconds","timezone_local","timezone_utc","tm_countvsyncs","tm_sleep","tm_systemtiming","true","ty_real","ty_string","undefined","vertex_type_color","vertex_type_colour","vertex_type_float1","vertex_type_float2","vertex_type_float3","vertex_type_float4","vertex_type_ubyte4","vertex_usage_binormal","vertex_usage_blendindices","vertex_usage_blendweight","vertex_usage_color","vertex_usage_colour","vertex_usage_depth","vertex_usage_fog","vertex_usage_normal","vertex_usage_position","vertex_usage_psize","vertex_usage_sample","vertex_usage_tangent","vertex_usage_texcoord","video_format_rgba","video_format_yuv","video_status_closed","video_status_paused","video_status_playing","video_status_preparing","vk_add","vk_alt","vk_anykey","vk_backspace","vk_control","vk_decimal","vk_delete","vk_divide","vk_down","vk_end","vk_enter","vk_escape","vk_f1","vk_f10","vk_f11","vk_f12","vk_f2","vk_f3","vk_f4","vk_f5","vk_f6","vk_f7","vk_f8","vk_f9","vk_home","vk_insert","vk_lalt","vk_lcontrol","vk_left","vk_lshift","vk_multiply","vk_nokey","vk_numpad0","vk_numpad1","vk_numpad2","vk_numpad3","vk_numpad4","vk_numpad5","vk_numpad6","vk_numpad7","vk_numpad8","vk_numpad9","vk_pagedown","vk_pageup","vk_pause","vk_printscreen","vk_ralt","vk_rcontrol","vk_return","vk_right","vk_rshift","vk_shift","vk_space","vk_subtract","vk_tab","vk_up","wallpaper_config","wallpaper_subscription_data","wrap"],"variable.language":["alarm","application_surface","argument","argument0","argument1","argument2","argument3","argument4","argument5","argument6","argument7","argument8","argument9","argument10","argument11","argument12","argument13","argument14","argument15","argument_count","async_load","background_color","background_colour","background_showcolor","background_showcolour","bbox_bottom","bbox_left","bbox_right","bbox_top","browser_height","browser_width","colour?ColourTrack","current_day","current_hour","current_minute","current_month","current_second","current_time","current_weekday","current_year","cursor_sprite","debug_mode","delta_time","depth","direction","display_aa","drawn_by_sequence","event_action","event_data","event_number","event_object","event_type","font_texture_page_size","fps","fps_real","friction","game_display_name","game_id","game_project_name","game_save_id","gravity","gravity_direction","health","hspeed","iap_data","id","image_alpha","image_angle","image_blend","image_index","image_number","image_speed","image_xscale","image_yscale","in_collision_tree","in_sequence","instance_count","instance_id","keyboard_key","keyboard_lastchar","keyboard_lastkey","keyboard_string","layer","lives","longMessage","managed","mask_index","message","mouse_button","mouse_lastbutton","mouse_x","mouse_y","object_index","os_browser","os_device","os_type","os_version","path_endaction","path_index","path_orientation","path_position","path_positionprevious","path_scale","path_speed","persistent","phy_active","phy_angular_damping","phy_angular_velocity","phy_bullet","phy_col_normal_x","phy_col_normal_y","phy_collision_points","phy_collision_x","phy_collision_y","phy_com_x","phy_com_y","phy_dynamic","phy_fixed_rotation","phy_inertia","phy_kinematic","phy_linear_damping","phy_linear_velocity_x","phy_linear_velocity_y","phy_mass","phy_position_x","phy_position_xprevious","phy_position_y","phy_position_yprevious","phy_rotation","phy_sleeping","phy_speed","phy_speed_x","phy_speed_y","player_avatar_sprite","player_avatar_url","player_id","player_local","player_type","player_user_id","program_directory","rollback_api_server","rollback_confirmed_frame","rollback_current_frame","rollback_event_id","rollback_event_param","rollback_game_running","room","room_first","room_height","room_last","room_persistent","room_speed","room_width","score","script","sequence_instance","solid","speed","sprite_height","sprite_index","sprite_width","sprite_xoffset","sprite_yoffset","stacktrace","temp_directory","timeline_index","timeline_loop","timeline_position","timeline_running","timeline_speed","view_camera","view_current","view_enabled","view_hport","view_surface_id","view_visible","view_wport","view_xport","view_yport","visible","vspeed","webgl_enabled","working_directory","x","xprevious","xstart","y","yprevious","ystart"]},contains:[e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE,e.APOS_STRING_MODE,e.QUOTE_STRING_MODE,e.C_NUMBER_MODE]}}},1200(e){e.exports=function(e){const t=e.regex,n=["lcase","month","vartype","instrrev","ubound","setlocale","getobject","rgb","getref","string","weekdayname","rnd","dateadd","monthname","now","day","minute","isarray","cbool","round","formatcurrency","conversions","csng","timevalue","second","year","space","abs","clng","timeserial","fixs","len","asc","isempty","maths","dateserial","atn","timer","isobject","filter","weekday","datevalue","ccur","isdate","instr","datediff","formatdatetime","replace","isnull","right","sgn","array","snumeric","log","cdbl","hex","chr","lbound","msgbox","ucase","getlocale","cos","cdate","cbyte","rtrim","join","hour","oct","typename","trim","strcomp","int","createobject","loadpicture","tan","formatnumber","mid","split","cint","sin","datepart","ltrim","sqr","time","derived","eval","date","formatpercent","exp","inputbox","left","ascw","chrw","regexp","cstr","err"];return{name:"VBScript",aliases:["vbs"],case_insensitive:!0,keywords:{keyword:["call","class","const","dim","do","loop","erase","execute","executeglobal","exit","for","each","next","function","if","then","else","on","error","option","explicit","new","private","property","let","get","public","randomize","redim","rem","select","case","set","stop","sub","while","wend","with","end","to","elseif","is","or","xor","and","not","class_initialize","class_terminate","default","preserve","in","me","byval","byref","step","resume","goto"],built_in:["server","response","request","scriptengine","scriptenginebuildversion","scriptengineminorversion","scriptenginemajorversion"],literal:["true","false","null","nothing","empty"]},illegal:"//",contains:[{begin:t.concat(t.either(...n),"\\s*\\("),relevance:0,keywords:{built_in:n}},e.inherit(e.QUOTE_STRING_MODE,{contains:[{begin:'""'}]}),e.COMMENT(/'/,/$/,{relevance:0}),e.C_NUMBER_MODE]}}},1272(e){e.exports=function(e){const t=e.regex,n=/[a-zA-Z_][a-zA-Z0-9_]*/,a={className:"number",variants:[e.BINARY_NUMBER_MODE,e.C_NUMBER_MODE]};return{name:"Tcl",aliases:["tk"],keywords:["after","append","apply","array","auto_execok","auto_import","auto_load","auto_mkindex","auto_mkindex_old","auto_qualify","auto_reset","bgerror","binary","break","catch","cd","chan","clock","close","concat","continue","dde","dict","encoding","eof","error","eval","exec","exit","expr","fblocked","fconfigure","fcopy","file","fileevent","filename","flush","for","foreach","format","gets","glob","global","history","http","if","incr","info","interp","join","lappend|10","lassign|10","lindex|10","linsert|10","list","llength|10","load","lrange|10","lrepeat|10","lreplace|10","lreverse|10","lsearch|10","lset|10","lsort|10","mathfunc","mathop","memory","msgcat","namespace","open","package","parray","pid","pkg::create","pkg_mkIndex","platform","platform::shell","proc","puts","pwd","read","refchan","regexp","registry","regsub|10","rename","return","safe","scan","seek","set","socket","source","split","string","subst","switch","tcl_endOfWord","tcl_findLibrary","tcl_startOfNextWord","tcl_startOfPreviousWord","tcl_wordBreakAfter","tcl_wordBreakBefore","tcltest","tclvars","tell","time","tm","trace","unknown","unload","unset","update","uplevel","upvar","variable","vwait","while"],contains:[e.COMMENT(";[ \\t]*#","$"),e.COMMENT("^[ \\t]*#","$"),{beginKeywords:"proc",end:"[\\{]",excludeEnd:!0,contains:[{className:"title",begin:"[ \\t\\n\\r]+(::)?[a-zA-Z_]((::)?[a-zA-Z0-9_])*",end:"[ \\t\\n\\r]",endsWithParent:!0,excludeEnd:!0}]},{className:"variable",variants:[{begin:t.concat(/\$/,t.optional(/::/),n,"(::",n,")*")},{begin:"\\$\\{(::)?[a-zA-Z_]((::)?[a-zA-Z0-9_])*",end:"\\}",contains:[a]}]},{className:"string",contains:[e.BACKSLASH_ESCAPE],variants:[e.inherit(e.QUOTE_STRING_MODE,{illegal:null})]},a]}}},1432(e){e.exports=function(e){return{name:"Gradle",case_insensitive:!0,keywords:["task","project","allprojects","subprojects","artifacts","buildscript","configurations","dependencies","repositories","sourceSets","description","delete","from","into","include","exclude","source","classpath","destinationDir","includes","options","sourceCompatibility","targetCompatibility","group","flatDir","doLast","doFirst","flatten","todir","fromdir","ant","def","abstract","break","case","catch","continue","default","do","else","extends","final","finally","for","if","implements","instanceof","native","new","private","protected","public","return","static","switch","synchronized","throw","throws","transient","try","volatile","while","strictfp","package","import","false","null","super","this","true","antlrtask","checkstyle","codenarc","copy","boolean","byte","char","class","double","float","int","interface","long","short","void","compile","runTime","file","fileTree","abs","any","append","asList","asWritable","call","collect","compareTo","count","div","dump","each","eachByte","eachFile","eachLine","every","find","findAll","flatten","getAt","getErr","getIn","getOut","getText","grep","immutable","inject","inspect","intersect","invokeMethods","isCase","join","leftShift","minus","multiply","newInputStream","newOutputStream","newPrintWriter","newReader","newWriter","next","plus","pop","power","previous","print","println","push","putAt","read","readBytes","readLines","reverse","reverseEach","round","size","sort","splitEachLine","step","subMap","times","toInteger","toList","tokenize","upto","waitForOrKill","withPrintWriter","withReader","withStream","withWriter","withWriterAppend","write","writeLine"],contains:[e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE,e.APOS_STRING_MODE,e.QUOTE_STRING_MODE,e.NUMBER_MODE,e.REGEXP_MODE]}}},1496(e){function t(e){return e?"string"==typeof e?e:e.source:null}function n(e){return a("(?=",e,")")}function a(...e){return e.map((e=>t(e))).join("")}function r(...e){const n=function(e){const t=e[e.length-1];return"object"==typeof t&&t.constructor===Object?(e.splice(e.length-1,1),t):{}}(e);return"("+(n.capture?"":"?:")+e.map((e=>t(e))).join("|")+")"}const i=e=>a(/\b/,e,/\w$/.test(e)?/\b/:/\B/),o=["Protocol","Type"].map(i),s=["init","self"].map(i),l=["Any","Self"],c=["actor","any","associatedtype","async","await",/as\?/,/as!/,"as","borrowing","break","case","catch","class","consume","consuming","continue","convenience","copy","default","defer","deinit","didSet","distributed","do","dynamic","each","else","enum","extension","fallthrough",/fileprivate\(set\)/,"fileprivate","final","for","func","get","guard","if","import","indirect","infix",/init\?/,/init!/,"inout",/internal\(set\)/,"internal","in","is","isolated","nonisolated","lazy","let","macro","mutating","nonmutating",/open\(set\)/,"open","operator","optional","override","package","postfix","precedencegroup","prefix",/private\(set\)/,"private","protocol",/public\(set\)/,"public","repeat","required","rethrows","return","set","some","static","struct","subscript","super","switch","throws","throw",/try\?/,/try!/,"try","typealias",/unowned\(safe\)/,/unowned\(unsafe\)/,"unowned","var","weak","where","while","willSet"],_=["false","nil","true"],d=["assignment","associativity","higherThan","left","lowerThan","none","right"],p=["#colorLiteral","#column","#dsohandle","#else","#elseif","#endif","#error","#file","#fileID","#fileLiteral","#filePath","#function","#if","#imageLiteral","#keyPath","#line","#selector","#sourceLocation","#warning"],u=["abs","all","any","assert","assertionFailure","debugPrint","dump","fatalError","getVaList","isKnownUniquelyReferenced","max","min","numericCast","pointwiseMax","pointwiseMin","precondition","preconditionFailure","print","readLine","repeatElement","sequence","stride","swap","swift_unboxFromSwiftValueWithType","transcode","type","unsafeBitCast","unsafeDowncast","withExtendedLifetime","withUnsafeMutablePointer","withUnsafePointer","withVaList","withoutActuallyEscaping","zip"],m=r(/[/=\-+!*%<>&|^~?]/,/[\u00A1-\u00A7]/,/[\u00A9\u00AB]/,/[\u00AC\u00AE]/,/[\u00B0\u00B1]/,/[\u00B6\u00BB\u00BF\u00D7\u00F7]/,/[\u2016-\u2017]/,/[\u2020-\u2027]/,/[\u2030-\u203E]/,/[\u2041-\u2053]/,/[\u2055-\u205E]/,/[\u2190-\u23FF]/,/[\u2500-\u2775]/,/[\u2794-\u2BFF]/,/[\u2E00-\u2E7F]/,/[\u3001-\u3003]/,/[\u3008-\u3020]/,/[\u3030]/),g=r(m,/[\u0300-\u036F]/,/[\u1DC0-\u1DFF]/,/[\u20D0-\u20FF]/,/[\uFE00-\uFE0F]/,/[\uFE20-\uFE2F]/),E=a(m,g,"*"),S=r(/[a-zA-Z_]/,/[\u00A8\u00AA\u00AD\u00AF\u00B2-\u00B5\u00B7-\u00BA]/,/[\u00BC-\u00BE\u00C0-\u00D6\u00D8-\u00F6\u00F8-\u00FF]/,/[\u0100-\u02FF\u0370-\u167F\u1681-\u180D\u180F-\u1DBF]/,/[\u1E00-\u1FFF]/,/[\u200B-\u200D\u202A-\u202E\u203F-\u2040\u2054\u2060-\u206F]/,/[\u2070-\u20CF\u2100-\u218F\u2460-\u24FF\u2776-\u2793]/,/[\u2C00-\u2DFF\u2E80-\u2FFF]/,/[\u3004-\u3007\u3021-\u302F\u3031-\u303F\u3040-\uD7FF]/,/[\uF900-\uFD3D\uFD40-\uFDCF\uFDF0-\uFE1F\uFE30-\uFE44]/,/[\uFE47-\uFEFE\uFF00-\uFFFD]/),b=r(S,/\d/,/[\u0300-\u036F\u1DC0-\u1DFF\u20D0-\u20FF\uFE20-\uFE2F]/),f=a(S,b,"*"),T=a(/[A-Z]/,b,"*"),h=["attached","autoclosure",a(/convention\(/,r("swift","block","c"),/\)/),"discardableResult","dynamicCallable","dynamicMemberLookup","escaping","freestanding","frozen","GKInspectable","IBAction","IBDesignable","IBInspectable","IBOutlet","IBSegueAction","inlinable","main","nonobjc","NSApplicationMain","NSCopying","NSManaged",a(/objc\(/,f,/\)/),"objc","objcMembers","propertyWrapper","requires_stored_property_inits","resultBuilder","Sendable","testable","UIApplicationMain","unchecked","unknown","usableFromInline","warn_unqualified_access"],C=["iOS","iOSApplicationExtension","macOS","macOSApplicationExtension","macCatalyst","macCatalystApplicationExtension","watchOS","watchOSApplicationExtension","tvOS","tvOSApplicationExtension","swift"];e.exports=function(e){const t={match:/\s+/,relevance:0},m=e.COMMENT("/\\*","\\*/",{contains:["self"]}),S=[e.C_LINE_COMMENT_MODE,m],O={match:[/\./,r(...o,...s)],className:{2:"keyword"}},N={match:a(/\./,r(...c)),relevance:0},R=c.filter((e=>"string"==typeof e)).concat(["_|0"]),v={variants:[{className:"keyword",match:r(...c.filter((e=>"string"!=typeof e)).concat(l).map(i),...s)}]},y={$pattern:r(/\b\w+/,/#\w+/),keyword:R.concat(p),literal:_},A=[O,N,v],I=[{match:a(/\./,r(...u)),relevance:0},{className:"built_in",match:a(/\b/,r(...u),/(?=\()/)}],D={match:/->/,relevance:0},x=[D,{className:"operator",relevance:0,variants:[{match:E},{match:`\\.(\\.|${g})+`}]}],w="([0-9]_*)+",M="([0-9a-fA-F]_*)+",L={className:"number",relevance:0,variants:[{match:`\\b(${w})(\\.(${w}))?([eE][+-]?(${w}))?\\b`},{match:`\\b0x(${M})(\\.(${M}))?([pP][+-]?(${w}))?\\b`},{match:/\b0o([0-7]_*)+\b/},{match:/\b0b([01]_*)+\b/}]},P=(e="")=>({className:"subst",variants:[{match:a(/\\/,e,/[0\\tnr"']/)},{match:a(/\\/,e,/u\{[0-9a-fA-F]{1,8}\}/)}]}),k=(e="")=>({className:"subst",match:a(/\\/,e,/[\t ]*(?:[\r\n]|\r\n)/)}),F=(e="")=>({className:"subst",label:"interpol",begin:a(/\\/,e,/\(/),end:/\)/}),U=(e="")=>({begin:a(e,/"""/),end:a(/"""/,e),contains:[P(e),k(e),F(e)]}),B=(e="")=>({begin:a(e,/"/),end:a(/"/,e),contains:[P(e),F(e)]}),G={className:"string",variants:[U(),U("#"),U("##"),U("###"),B(),B("#"),B("##"),B("###")]},Y=[e.BACKSLASH_ESCAPE,{begin:/\[/,end:/\]/,relevance:0,contains:[e.BACKSLASH_ESCAPE]}],H={begin:/\/[^\s](?=[^/\n]*\/)/,end:/\//,contains:Y},V=e=>{const t=a(e,/\//),n=a(/\//,e);return{begin:t,end:n,contains:[...Y,{scope:"comment",begin:`#(?!.*${n})`,end:/$/}]}},q={scope:"regexp",variants:[V("###"),V("##"),V("#"),H]},z={match:a(/`/,f,/`/)},j=[z,{className:"variable",match:/\$\d+/},{className:"variable",match:`\\$${b}+`}],$=[{match:/(@|#(un)?)available/,scope:"keyword",starts:{contains:[{begin:/\(/,end:/\)/,keywords:C,contains:[...x,L,G]}]}},{scope:"keyword",match:a(/@/,r(...h),n(r(/\(/,/\s+/)))},{scope:"meta",match:a(/@/,f)}],W={match:n(/\b[A-Z]/),relevance:0,contains:[{className:"type",match:a(/(AV|CA|CF|CG|CI|CL|CM|CN|CT|MK|MP|MTK|MTL|NS|SCN|SK|UI|WK|XC)/,b,"+")},{className:"type",match:T,relevance:0},{match:/[?!]+/,relevance:0},{match:/\.\.\./,relevance:0},{match:a(/\s+&\s+/,n(T)),relevance:0}]},Q={begin://,keywords:y,contains:[...S,...A,...$,D,W]};W.contains.push(Q);const K={begin:/\(/,end:/\)/,relevance:0,keywords:y,contains:["self",{match:a(f,/\s*:/),keywords:"_|0",relevance:0},...S,q,...A,...I,...x,L,G,...j,...$,W]},X={begin://,keywords:"repeat each",contains:[...S,W]},Z={begin:/\(/,end:/\)/,keywords:y,contains:[{begin:r(n(a(f,/\s*:/)),n(a(f,/\s+/,f,/\s*:/))),end:/:/,relevance:0,contains:[{className:"keyword",match:/\b_\b/},{className:"params",match:f}]},...S,...A,...x,L,G,...$,W,K],endsParent:!0,illegal:/["']/},J={match:[/(func|macro)/,/\s+/,r(z.match,f,E)],className:{1:"keyword",3:"title.function"},contains:[X,Z,t],illegal:[/\[/,/%/]},ee={match:[/\b(?:subscript|init[?!]?)/,/\s*(?=[<(])/],className:{1:"keyword"},contains:[X,Z,t],illegal:/\[|%/},te={match:[/operator/,/\s+/,E],className:{1:"keyword",3:"title"}},ne={begin:[/precedencegroup/,/\s+/,T],className:{1:"keyword",3:"title"},contains:[W],keywords:[...d,..._],end:/}/},ae={begin:[/(struct|protocol|class|extension|enum|actor)/,/\s+/,f,/\s*/],beginScope:{1:"keyword",3:"title.class"},keywords:y,contains:[X,...A,{begin:/:/,end:/\{/,keywords:y,contains:[{scope:"title.class.inherited",match:T},...A],relevance:0}]};for(const e of G.variants){const t=e.contains.find((e=>"interpol"===e.label));t.keywords=y;const n=[...A,...I,...x,L,G,...j];t.contains=[...n,{begin:/\(/,end:/\)/,contains:["self",...n]}]}return{name:"Swift",keywords:y,contains:[...S,J,ee,{match:[/class\b/,/\s+/,/func\b/,/\s+/,/\b[A-Za-z_][A-Za-z0-9_]*\b/],scope:{1:"keyword",3:"keyword",5:"title.function"}},{match:[/class\b/,/\s+/,/var\b/],scope:{1:"keyword",3:"keyword"}},ae,te,ne,{beginKeywords:"import",end:/$/,contains:[...S],relevance:0},q,...A,...I,...x,L,G,...j,...$,W,K]}}},1500(e){e.exports=function(e){return{name:"DNS Zone",aliases:["bind","zone"],keywords:["IN","A","AAAA","AFSDB","APL","CAA","CDNSKEY","CDS","CERT","CNAME","DHCID","DLV","DNAME","DNSKEY","DS","HIP","IPSECKEY","KEY","KX","LOC","MX","NAPTR","NS","NSEC","NSEC3","NSEC3PARAM","PTR","RRSIG","RP","SIG","SOA","SRV","SSHFP","TA","TKEY","TLSA","TSIG","TXT"],contains:[e.COMMENT(";","$",{relevance:0}),{className:"meta",begin:/^\$(TTL|GENERATE|INCLUDE|ORIGIN)\b/},{className:"number",begin:"((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)(\\.(25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)(\\.(25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)(\\.(25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)(\\.(25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)(\\.(25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)(\\.(25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)(\\.(25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)){3}))|:)))\\b"},{className:"number",begin:"((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9]).){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\\b"},e.inherit(e.NUMBER_MODE,{begin:/\b\d+[dhwm]?/})]}}},1533(e){e.exports=function(e){const t=e.regex,n={className:"number",relevance:0,variants:[{begin:/([+-]+)?[\d]+_[\d_]+/},{begin:e.NUMBER_RE}]},a=e.COMMENT();a.variants=[{begin:/;/,end:/$/},{begin:/#/,end:/$/}];const r={className:"variable",variants:[{begin:/\$[\w\d"][\w\d_]*/},{begin:/\$\{(.*?)\}/}]},i={className:"literal",begin:/\bon|off|true|false|yes|no\b/},o={className:"string",contains:[e.BACKSLASH_ESCAPE],variants:[{begin:"'''",end:"'''",relevance:10},{begin:'"""',end:'"""',relevance:10},{begin:'"',end:'"'},{begin:"'",end:"'"}]},s={begin:/\[/,end:/\]/,contains:[a,i,r,o,n,"self"],relevance:0},l=t.either(/[A-Za-z0-9_-]+/,/"(\\"|[^"])*"/,/'[^']*'/);return{name:"TOML, also INI",aliases:["toml"],case_insensitive:!0,illegal:/\S/,contains:[a,{className:"section",begin:/\[+/,end:/\]+/},{begin:t.concat(l,"(\\s*\\.\\s*",l,")*",t.lookahead(/\s*=\s*[^#\s]/)),className:"attr",starts:{end:/$/,contains:[a,s,i,r,o,n]}}]}}},1611(e){const t=["a","abbr","address","article","aside","audio","b","blockquote","body","button","canvas","caption","cite","code","dd","del","details","dfn","div","dl","dt","em","fieldset","figcaption","figure","footer","form","h1","h2","h3","h4","h5","h6","header","hgroup","html","i","iframe","img","input","ins","kbd","label","legend","li","main","mark","menu","nav","object","ol","optgroup","option","p","picture","q","quote","samp","section","select","source","span","strong","summary","sup","table","tbody","td","textarea","tfoot","th","thead","time","tr","ul","var","video","defs","g","marker","mask","pattern","svg","switch","symbol","feBlend","feColorMatrix","feComponentTransfer","feComposite","feConvolveMatrix","feDiffuseLighting","feDisplacementMap","feFlood","feGaussianBlur","feImage","feMerge","feMorphology","feOffset","feSpecularLighting","feTile","feTurbulence","linearGradient","radialGradient","stop","circle","ellipse","image","line","path","polygon","polyline","rect","text","use","textPath","tspan","foreignObject","clipPath"],n=["any-hover","any-pointer","aspect-ratio","color","color-gamut","color-index","device-aspect-ratio","device-height","device-width","display-mode","forced-colors","grid","height","hover","inverted-colors","monochrome","orientation","overflow-block","overflow-inline","pointer","prefers-color-scheme","prefers-contrast","prefers-reduced-motion","prefers-reduced-transparency","resolution","scan","scripting","update","width","min-width","max-width","min-height","max-height"].sort().reverse(),a=["active","any-link","blank","checked","current","default","defined","dir","disabled","drop","empty","enabled","first","first-child","first-of-type","fullscreen","future","focus","focus-visible","focus-within","has","host","host-context","hover","indeterminate","in-range","invalid","is","lang","last-child","last-of-type","left","link","local-link","not","nth-child","nth-col","nth-last-child","nth-last-col","nth-last-of-type","nth-of-type","only-child","only-of-type","optional","out-of-range","past","placeholder-shown","read-only","read-write","required","right","root","scope","target","target-within","user-invalid","valid","visited","where"].sort().reverse(),r=["after","backdrop","before","cue","cue-region","first-letter","first-line","grammar-error","marker","part","placeholder","selection","slotted","spelling-error"].sort().reverse(),i=["accent-color","align-content","align-items","align-self","alignment-baseline","all","anchor-name","animation","animation-composition","animation-delay","animation-direction","animation-duration","animation-fill-mode","animation-iteration-count","animation-name","animation-play-state","animation-range","animation-range-end","animation-range-start","animation-timeline","animation-timing-function","appearance","aspect-ratio","backdrop-filter","backface-visibility","background","background-attachment","background-blend-mode","background-clip","background-color","background-image","background-origin","background-position","background-position-x","background-position-y","background-repeat","background-size","baseline-shift","block-size","border","border-block","border-block-color","border-block-end","border-block-end-color","border-block-end-style","border-block-end-width","border-block-start","border-block-start-color","border-block-start-style","border-block-start-width","border-block-style","border-block-width","border-bottom","border-bottom-color","border-bottom-left-radius","border-bottom-right-radius","border-bottom-style","border-bottom-width","border-collapse","border-color","border-end-end-radius","border-end-start-radius","border-image","border-image-outset","border-image-repeat","border-image-slice","border-image-source","border-image-width","border-inline","border-inline-color","border-inline-end","border-inline-end-color","border-inline-end-style","border-inline-end-width","border-inline-start","border-inline-start-color","border-inline-start-style","border-inline-start-width","border-inline-style","border-inline-width","border-left","border-left-color","border-left-style","border-left-width","border-radius","border-right","border-right-color","border-right-style","border-right-width","border-spacing","border-start-end-radius","border-start-start-radius","border-style","border-top","border-top-color","border-top-left-radius","border-top-right-radius","border-top-style","border-top-width","border-width","bottom","box-align","box-decoration-break","box-direction","box-flex","box-flex-group","box-lines","box-ordinal-group","box-orient","box-pack","box-shadow","box-sizing","break-after","break-before","break-inside","caption-side","caret-color","clear","clip","clip-path","clip-rule","color","color-interpolation","color-interpolation-filters","color-profile","color-rendering","color-scheme","column-count","column-fill","column-gap","column-rule","column-rule-color","column-rule-style","column-rule-width","column-span","column-width","columns","contain","contain-intrinsic-block-size","contain-intrinsic-height","contain-intrinsic-inline-size","contain-intrinsic-size","contain-intrinsic-width","container","container-name","container-type","content","content-visibility","counter-increment","counter-reset","counter-set","cue","cue-after","cue-before","cursor","cx","cy","direction","display","dominant-baseline","empty-cells","enable-background","field-sizing","fill","fill-opacity","fill-rule","filter","flex","flex-basis","flex-direction","flex-flow","flex-grow","flex-shrink","flex-wrap","float","flood-color","flood-opacity","flow","font","font-display","font-family","font-feature-settings","font-kerning","font-language-override","font-optical-sizing","font-palette","font-size","font-size-adjust","font-smooth","font-smoothing","font-stretch","font-style","font-synthesis","font-synthesis-position","font-synthesis-small-caps","font-synthesis-style","font-synthesis-weight","font-variant","font-variant-alternates","font-variant-caps","font-variant-east-asian","font-variant-emoji","font-variant-ligatures","font-variant-numeric","font-variant-position","font-variation-settings","font-weight","forced-color-adjust","gap","glyph-orientation-horizontal","glyph-orientation-vertical","grid","grid-area","grid-auto-columns","grid-auto-flow","grid-auto-rows","grid-column","grid-column-end","grid-column-start","grid-gap","grid-row","grid-row-end","grid-row-start","grid-template","grid-template-areas","grid-template-columns","grid-template-rows","hanging-punctuation","height","hyphenate-character","hyphenate-limit-chars","hyphens","icon","image-orientation","image-rendering","image-resolution","ime-mode","initial-letter","initial-letter-align","inline-size","inset","inset-area","inset-block","inset-block-end","inset-block-start","inset-inline","inset-inline-end","inset-inline-start","isolation","justify-content","justify-items","justify-self","kerning","left","letter-spacing","lighting-color","line-break","line-height","line-height-step","list-style","list-style-image","list-style-position","list-style-type","margin","margin-block","margin-block-end","margin-block-start","margin-bottom","margin-inline","margin-inline-end","margin-inline-start","margin-left","margin-right","margin-top","margin-trim","marker","marker-end","marker-mid","marker-start","marks","mask","mask-border","mask-border-mode","mask-border-outset","mask-border-repeat","mask-border-slice","mask-border-source","mask-border-width","mask-clip","mask-composite","mask-image","mask-mode","mask-origin","mask-position","mask-repeat","mask-size","mask-type","masonry-auto-flow","math-depth","math-shift","math-style","max-block-size","max-height","max-inline-size","max-width","min-block-size","min-height","min-inline-size","min-width","mix-blend-mode","nav-down","nav-index","nav-left","nav-right","nav-up","none","normal","object-fit","object-position","offset","offset-anchor","offset-distance","offset-path","offset-position","offset-rotate","opacity","order","orphans","outline","outline-color","outline-offset","outline-style","outline-width","overflow","overflow-anchor","overflow-block","overflow-clip-margin","overflow-inline","overflow-wrap","overflow-x","overflow-y","overlay","overscroll-behavior","overscroll-behavior-block","overscroll-behavior-inline","overscroll-behavior-x","overscroll-behavior-y","padding","padding-block","padding-block-end","padding-block-start","padding-bottom","padding-inline","padding-inline-end","padding-inline-start","padding-left","padding-right","padding-top","page","page-break-after","page-break-before","page-break-inside","paint-order","pause","pause-after","pause-before","perspective","perspective-origin","place-content","place-items","place-self","pointer-events","position","position-anchor","position-visibility","print-color-adjust","quotes","r","resize","rest","rest-after","rest-before","right","rotate","row-gap","ruby-align","ruby-position","scale","scroll-behavior","scroll-margin","scroll-margin-block","scroll-margin-block-end","scroll-margin-block-start","scroll-margin-bottom","scroll-margin-inline","scroll-margin-inline-end","scroll-margin-inline-start","scroll-margin-left","scroll-margin-right","scroll-margin-top","scroll-padding","scroll-padding-block","scroll-padding-block-end","scroll-padding-block-start","scroll-padding-bottom","scroll-padding-inline","scroll-padding-inline-end","scroll-padding-inline-start","scroll-padding-left","scroll-padding-right","scroll-padding-top","scroll-snap-align","scroll-snap-stop","scroll-snap-type","scroll-timeline","scroll-timeline-axis","scroll-timeline-name","scrollbar-color","scrollbar-gutter","scrollbar-width","shape-image-threshold","shape-margin","shape-outside","shape-rendering","speak","speak-as","src","stop-color","stop-opacity","stroke","stroke-dasharray","stroke-dashoffset","stroke-linecap","stroke-linejoin","stroke-miterlimit","stroke-opacity","stroke-width","tab-size","table-layout","text-align","text-align-all","text-align-last","text-anchor","text-combine-upright","text-decoration","text-decoration-color","text-decoration-line","text-decoration-skip","text-decoration-skip-ink","text-decoration-style","text-decoration-thickness","text-emphasis","text-emphasis-color","text-emphasis-position","text-emphasis-style","text-indent","text-justify","text-orientation","text-overflow","text-rendering","text-shadow","text-size-adjust","text-transform","text-underline-offset","text-underline-position","text-wrap","text-wrap-mode","text-wrap-style","timeline-scope","top","touch-action","transform","transform-box","transform-origin","transform-style","transition","transition-behavior","transition-delay","transition-duration","transition-property","transition-timing-function","translate","unicode-bidi","user-modify","user-select","vector-effect","vertical-align","view-timeline","view-timeline-axis","view-timeline-inset","view-timeline-name","view-transition-name","visibility","voice-balance","voice-duration","voice-family","voice-pitch","voice-range","voice-rate","voice-stress","voice-volume","white-space","white-space-collapse","widows","width","will-change","word-break","word-spacing","word-wrap","writing-mode","x","y","z-index","zoom"].sort().reverse();e.exports=function(e){const o=(e=>({IMPORTANT:{scope:"meta",begin:"!important"},BLOCK_COMMENT:e.C_BLOCK_COMMENT_MODE,HEXCOLOR:{scope:"number",begin:/#(([0-9a-fA-F]{3,4})|(([0-9a-fA-F]{2}){3,4}))\b/},FUNCTION_DISPATCH:{className:"built_in",begin:/[\w-]+(?=\()/},ATTRIBUTE_SELECTOR_MODE:{scope:"selector-attr",begin:/\[/,end:/\]/,illegal:"$",contains:[e.APOS_STRING_MODE,e.QUOTE_STRING_MODE]},CSS_NUMBER_MODE:{scope:"number",begin:e.NUMBER_RE+"(%|em|ex|ch|rem|vw|vh|vmin|vmax|cm|mm|in|pt|pc|px|deg|grad|rad|turn|s|ms|Hz|kHz|dpi|dpcm|dppx)?",relevance:0},CSS_VARIABLE:{className:"attr",begin:/--[A-Za-z_][A-Za-z0-9_-]*/}}))(e),s=r,l=a,c="@[a-z-]+",_={className:"variable",begin:"(\\$[a-zA-Z-][a-zA-Z0-9_-]*)\\b",relevance:0};return{name:"SCSS",case_insensitive:!0,illegal:"[=/|']",contains:[e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE,o.CSS_NUMBER_MODE,{className:"selector-id",begin:"#[A-Za-z0-9_-]+",relevance:0},{className:"selector-class",begin:"\\.[A-Za-z0-9_-]+",relevance:0},o.ATTRIBUTE_SELECTOR_MODE,{className:"selector-tag",begin:"\\b("+t.join("|")+")\\b",relevance:0},{className:"selector-pseudo",begin:":("+l.join("|")+")"},{className:"selector-pseudo",begin:":(:)?("+s.join("|")+")"},_,{begin:/\(/,end:/\)/,contains:[o.CSS_NUMBER_MODE]},o.CSS_VARIABLE,{className:"attribute",begin:"\\b("+i.join("|")+")\\b"},{begin:"\\b(whitespace|wait|w-resize|visible|vertical-text|vertical-ideographic|uppercase|upper-roman|upper-alpha|underline|transparent|top|thin|thick|text|text-top|text-bottom|tb-rl|table-header-group|table-footer-group|sw-resize|super|strict|static|square|solid|small-caps|separate|se-resize|scroll|s-resize|rtl|row-resize|ridge|right|repeat|repeat-y|repeat-x|relative|progress|pointer|overline|outside|outset|oblique|nowrap|not-allowed|normal|none|nw-resize|no-repeat|no-drop|newspaper|ne-resize|n-resize|move|middle|medium|ltr|lr-tb|lowercase|lower-roman|lower-alpha|loose|list-item|line|line-through|line-edge|lighter|left|keep-all|justify|italic|inter-word|inter-ideograph|inside|inset|inline|inline-block|inherit|inactive|ideograph-space|ideograph-parenthesis|ideograph-numeric|ideograph-alpha|horizontal|hidden|help|hand|groove|fixed|ellipsis|e-resize|double|dotted|distribute|distribute-space|distribute-letter|distribute-all-lines|disc|disabled|default|decimal|dashed|crosshair|collapse|col-resize|circle|char|center|capitalize|break-word|break-all|bottom|both|bolder|bold|block|bidi-override|below|baseline|auto|always|all-scroll|absolute|table|table-cell)\\b"},{begin:/:/,end:/[;}{]/,relevance:0,contains:[o.BLOCK_COMMENT,_,o.HEXCOLOR,o.CSS_NUMBER_MODE,e.QUOTE_STRING_MODE,e.APOS_STRING_MODE,o.IMPORTANT,o.FUNCTION_DISPATCH]},{begin:"@(page|font-face)",keywords:{$pattern:c,keyword:"@page @font-face"}},{begin:"@",end:"[{;]",returnBegin:!0,keywords:{$pattern:/[a-z-]+/,keyword:"and or not only",attribute:n.join(" ")},contains:[{begin:c,className:"keyword"},{begin:/[a-z-]+(?=:)/,className:"attribute"},_,e.QUOTE_STRING_MODE,e.APOS_STRING_MODE,o.HEXCOLOR,o.CSS_NUMBER_MODE]},o.FUNCTION_DISPATCH]}}},1613(e,t){"use strict";var n,a;Object.defineProperty(t,"__esModule",{value:!0}),t.ColorSchemeType=t.DiffStyleType=t.LineMatchingType=t.OutputFormatType=t.LineType=void 0,function(e){e.INSERT="insert",e.DELETE="delete",e.CONTEXT="context"}(n||(t.LineType=n={})),t.OutputFormatType={LINE_BY_LINE:"line-by-line",SIDE_BY_SIDE:"side-by-side"},t.LineMatchingType={LINES:"lines",WORDS:"words",NONE:"none"},t.DiffStyleType={WORD:"word",CHAR:"char"},function(e){e.AUTO="auto",e.DARK="dark",e.LIGHT="light"}(a||(t.ColorSchemeType=a={}))},1726(e){e.exports=function(e){return{name:"PHP template",subLanguage:"xml",contains:[{begin:/<\?(php|=)?/,end:/\?>/,subLanguage:"php",contains:[{begin:"/\\*",end:"\\*/",skip:!0},{begin:'b"',end:'"',skip:!0},{begin:"b'",end:"'",skip:!0},e.inherit(e.APOS_STRING_MODE,{illegal:null,className:null,contains:null,skip:!0}),e.inherit(e.QUOTE_STRING_MODE,{illegal:null,className:null,contains:null,skip:!0})]}]}}},1747(e){e.exports=function(e){return{name:"Nim",keywords:{keyword:["addr","and","as","asm","bind","block","break","case","cast","concept","const","continue","converter","defer","discard","distinct","div","do","elif","else","end","enum","except","export","finally","for","from","func","generic","guarded","if","import","in","include","interface","is","isnot","iterator","let","macro","method","mixin","mod","nil","not","notin","object","of","or","out","proc","ptr","raise","ref","return","shared","shl","shr","static","template","try","tuple","type","using","var","when","while","with","without","xor","yield"],literal:["true","false"],type:["int","int8","int16","int32","int64","uint","uint8","uint16","uint32","uint64","float","float32","float64","bool","char","string","cstring","pointer","expr","stmt","void","auto","any","range","array","openarray","varargs","seq","set","clong","culong","cchar","cschar","cshort","cint","csize","clonglong","cfloat","cdouble","clongdouble","cuchar","cushort","cuint","culonglong","cstringarray","semistatic"],built_in:["stdin","stdout","stderr","result"]},contains:[{className:"meta",begin:/\{\./,end:/\.\}/,relevance:10},{className:"string",begin:/[a-zA-Z]\w*"/,end:/"/,contains:[{begin:/""/}]},{className:"string",begin:/([a-zA-Z]\w*)?"""/,end:/"""/},e.QUOTE_STRING_MODE,{className:"type",begin:/\b[A-Z]\w+\b/,relevance:0},{className:"number",relevance:0,variants:[{begin:/\b(0[xX][0-9a-fA-F][_0-9a-fA-F]*)('?[iIuU](8|16|32|64))?/},{begin:/\b(0o[0-7][_0-7]*)('?[iIuUfF](8|16|32|64))?/},{begin:/\b(0(b|B)[01][_01]*)('?[iIuUfF](8|16|32|64))?/},{begin:/\b(\d[_\d]*)('?[iIuUfF](8|16|32|64))?/}]},e.HASH_COMMENT_MODE]}}},1756(e){e.exports=function(e){const t="group clone ms master location colocation order fencing_topology rsc_ticket acl_target acl_group user role tag xml";return{name:"crmsh",aliases:["crm","pcmk"],case_insensitive:!0,keywords:{keyword:"params meta operations op rule attributes utilization read write deny defined not_defined in_range date spec in ref reference attribute type xpath version and or lt gt tag lte gte eq ne \\ number string",literal:"Master Started Slave Stopped start promote demote stop monitor true false"},contains:[e.HASH_COMMENT_MODE,{beginKeywords:"node",starts:{end:"\\s*([\\w_-]+:)?",starts:{className:"title",end:"\\s*[\\$\\w_][\\w_-]*"}}},{beginKeywords:"primitive rsc_template",starts:{className:"title",end:"\\s*[\\$\\w_][\\w_-]*",starts:{end:"\\s*@?[\\w_][\\w_\\.:-]*"}}},{begin:"\\b("+t.split(" ").join("|")+")\\s+",keywords:t,starts:{className:"title",end:"[\\$\\w_][\\w_-]*"}},{beginKeywords:"property rsc_defaults op_defaults",starts:{className:"title",end:"\\s*([\\w_-]+:)?"}},e.QUOTE_STRING_MODE,{className:"meta",begin:"(ocf|systemd|service|lsb):[\\w_:-]+",relevance:0},{className:"number",begin:"\\b\\d+(\\.\\d+)?(ms|s|h|m)?",relevance:0},{className:"literal",begin:"[-]?(infinity|inf)",relevance:0},{className:"attr",begin:/([A-Za-z$_#][\w_-]+)=/,relevance:0},{className:"tag",begin:"",relevance:0}]}}},1791(e){e.exports=function(e){return{name:"OCaml",aliases:["ml"],keywords:{$pattern:"[a-z_]\\w*!?",keyword:"and as assert asr begin class constraint do done downto else end exception external for fun function functor if in include inherit! inherit initializer land lazy let lor lsl lsr lxor match method!|10 method mod module mutable new object of open! open or private rec sig struct then to try type val! val virtual when while with parser value",built_in:"array bool bytes char exn|5 float int int32 int64 list lazy_t|5 nativeint|5 string unit in_channel out_channel ref",literal:"true false"},illegal:/\/\/|>>/,contains:[{className:"literal",begin:"\\[(\\|\\|)?\\]|\\(\\)",relevance:0},e.COMMENT("\\(\\*","\\*\\)",{contains:["self"]}),{className:"symbol",begin:"'[A-Za-z_](?!')[\\w']*"},{className:"type",begin:"`[A-Z][\\w']*"},{className:"type",begin:"\\b[A-Z][\\w']*",relevance:0},{begin:"[a-z_]\\w*'[\\w']*",relevance:0},e.inherit(e.APOS_STRING_MODE,{className:"string",relevance:0}),e.inherit(e.QUOTE_STRING_MODE,{illegal:null}),{className:"number",begin:"\\b(0[xX][a-fA-F0-9_]+[Lln]?|0[oO][0-7_]+[Lln]?|0[bB][01_]+[Lln]?|[0-9][0-9_]*([Lln]|(\\.[0-9_]*)?([eE][-+]?[0-9_]+)?)?)",relevance:0},{begin:/->/}]}}},1799(e){e.exports=function(e){const t="[A-Za-z_][0-9A-Za-z_]*",n={keyword:["break","case","catch","continue","debugger","do","else","export","for","function","if","import","in","new","of","return","switch","try","var","void","while"],literal:["BackSlash","DoubleQuote","ForwardSlash","Infinity","NaN","NewLine","PI","SingleQuote","Tab","TextFormatting","false","null","true","undefined"],built_in:["Abs","Acos","All","Angle","Any","Area","AreaGeodetic","Array","Asin","Atan","Atan2","Attachments","Average","Back","Bearing","Boolean","Buffer","BufferGeodetic","Ceil","Centroid","ChangeTimeZone","Clip","Concatenate","Console","Constrain","Contains","ConvertDirection","ConvexHull","Cos","Count","Crosses","Cut","Date|0","DateAdd","DateDiff","DateOnly","Day","Decode","DefaultValue","Densify","DensifyGeodetic","Dictionary","Difference","Disjoint","Distance","DistanceGeodetic","DistanceToCoordinate","Distinct","Domain","DomainCode","DomainName","EnvelopeIntersects","Equals","Erase","Exp","Expects","Extent","Feature","FeatureInFilter","FeatureSet","FeatureSetByAssociation","FeatureSetById","FeatureSetByName","FeatureSetByPortalItem","FeatureSetByRelationshipClass","FeatureSetByRelationshipName","Filter","FilterBySubtypeCode","Find","First|0","Floor","FromCharCode","FromCodePoint","FromJSON","Front","GdbVersion","Generalize","Geometry","GetEnvironment","GetFeatureSet","GetFeatureSetInfo","GetUser","GroupBy","Guid","HasKey","HasValue","Hash","Hour","IIf","ISOMonth","ISOWeek","ISOWeekday","ISOYear","Includes","IndexOf","Insert","Intersection","Intersects","IsEmpty","IsNan","IsSelfIntersecting","IsSimple","KnowledgeGraphByPortalItem","Left|0","Length","Length3D","LengthGeodetic","Log","Lower","Map","Max","Mean","MeasureToCoordinate","Mid","Millisecond","Min","Minute","Month","MultiPartToSinglePart","Multipoint","NearestCoordinate","NearestVertex","NextSequenceValue","None","Now","Number","Offset","OrderBy","Overlaps","Point","PointToCoordinate","Polygon","Polyline","Pop","Portal","Pow","Proper","Push","QueryGraph","Random","Reduce","Relate","Replace","Resize","Reverse","Right|0","RingIsClockwise","Rotate","Round","Schema","Second","SetGeometry","Simplify","Sin","Slice","Sort","Splice","Split","Sqrt","StandardizeFilename","StandardizeGuid","Stdev","SubtypeCode","SubtypeName","Subtypes","Sum","SymmetricDifference","Tan","Text","Time","TimeZone","TimeZoneOffset","Timestamp","ToCharCode","ToCodePoint","ToHex","ToLocal","ToUTC","Today","Top|0","Touches","TrackAccelerationAt","TrackAccelerationWindow","TrackCurrentAcceleration","TrackCurrentDistance","TrackCurrentSpeed","TrackCurrentTime","TrackDistanceAt","TrackDistanceWindow","TrackDuration","TrackFieldWindow","TrackGeometryWindow","TrackIndex","TrackSpeedAt","TrackSpeedWindow","TrackStartTime","TrackWindow","Trim","TypeOf","Union","Upper","UrlEncode","Variance","Week","Weekday","When|0","Within","Year|0"]},a={className:"symbol",begin:"\\$"+e.regex.either("aggregatedFeatures","analytic","config","datapoint","datastore","editcontext","feature","featureSet","feedfeature","fencefeature","fencenotificationtype","graph","join","layer","locationupdate","map","measure","measure","originalFeature","record","reference","rowindex","sourcedatastore","sourcefeature","sourcelayer","target","targetdatastore","targetfeature","targetlayer","userInput","value","variables","view")},r={className:"number",variants:[{begin:"\\b(0[bB][01]+)"},{begin:"\\b(0[oO][0-7]+)"},{begin:e.C_NUMBER_RE}],relevance:0},i={className:"subst",begin:"\\$\\{",end:"\\}",keywords:n,contains:[]},o={className:"string",begin:"`",end:"`",contains:[e.BACKSLASH_ESCAPE,i]};i.contains=[e.APOS_STRING_MODE,e.QUOTE_STRING_MODE,o,r,e.REGEXP_MODE];const s=i.contains.concat([e.C_BLOCK_COMMENT_MODE,e.C_LINE_COMMENT_MODE]);return{name:"ArcGIS Arcade",case_insensitive:!0,keywords:n,contains:[e.APOS_STRING_MODE,e.QUOTE_STRING_MODE,o,e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE,a,r,{begin:/[{,]\s*/,relevance:0,contains:[{begin:t+"\\s*:",returnBegin:!0,relevance:0,contains:[{className:"attr",begin:t,relevance:0}]}]},{begin:"("+e.RE_STARTERS_RE+"|\\b(return)\\b)\\s*",keywords:"return",contains:[e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE,e.REGEXP_MODE,{className:"function",begin:"(\\(.*?\\)|"+t+")\\s*=>",returnBegin:!0,end:"\\s*=>",contains:[{className:"params",variants:[{begin:t},{begin:/\(\s*\)/},{begin:/\(/,end:/\)/,excludeBegin:!0,excludeEnd:!0,keywords:n,contains:s}]}]}],relevance:0},{beginKeywords:"function",end:/\{/,excludeEnd:!0,contains:[e.inherit(e.TITLE_MODE,{className:"title.function",begin:t}),{className:"params",begin:/\(/,end:/\)/,excludeBegin:!0,excludeEnd:!0,contains:s}],illegal:/\[|%/},{begin:/\$[(.]/}],illegal:/#(?!!)/}}},1980(e){e.exports=function(e){const t={className:"literal",begin:/[+-]+/,relevance:0};return{name:"Brainfuck",aliases:["bf"],contains:[e.COMMENT(/[^\[\]\.,\+\-<> \r\n]/,/[\[\]\.,\+\-<> \r\n]/,{contains:[{match:/[ ]+[^\[\]\.,\+\-<> \r\n]/,relevance:0}],returnEnd:!0,relevance:0}),{className:"title",begin:"[\\[\\]]",relevance:0},{className:"string",begin:"[\\.,]",relevance:0},{begin:/(?=\+\+|--)/,contains:[t]},t]}}},2008(e){e.exports=function(e){return{name:"CMake",aliases:["cmake.in"],case_insensitive:!0,keywords:{keyword:"break cmake_host_system_information cmake_minimum_required cmake_parse_arguments cmake_policy configure_file continue elseif else endforeach endfunction endif endmacro endwhile execute_process file find_file find_library find_package find_path find_program foreach function get_cmake_property get_directory_property get_filename_component get_property if include include_guard list macro mark_as_advanced math message option return separate_arguments set_directory_properties set_property set site_name string unset variable_watch while add_compile_definitions add_compile_options add_custom_command add_custom_target add_definitions add_dependencies add_executable add_library add_link_options add_subdirectory add_test aux_source_directory build_command create_test_sourcelist define_property enable_language enable_testing export fltk_wrap_ui get_source_file_property get_target_property get_test_property include_directories include_external_msproject include_regular_expression install link_directories link_libraries load_cache project qt_wrap_cpp qt_wrap_ui remove_definitions set_source_files_properties set_target_properties set_tests_properties source_group target_compile_definitions target_compile_features target_compile_options target_include_directories target_link_directories target_link_libraries target_link_options target_sources try_compile try_run ctest_build ctest_configure ctest_coverage ctest_empty_binary_directory ctest_memcheck ctest_read_custom_files ctest_run_script ctest_sleep ctest_start ctest_submit ctest_test ctest_update ctest_upload build_name exec_program export_library_dependencies install_files install_programs install_targets load_command make_directory output_required_files remove subdir_depends subdirs use_mangled_mesa utility_source variable_requires write_file qt5_use_modules qt5_use_package qt5_wrap_cpp on off true false and or not command policy target test exists is_newer_than is_directory is_symlink is_absolute matches less greater equal less_equal greater_equal strless strgreater strequal strless_equal strgreater_equal version_less version_greater version_equal version_less_equal version_greater_equal in_list defined"},contains:[{className:"variable",begin:/\$\{/,end:/\}/},e.COMMENT(/#\[\[/,/]]/),e.HASH_COMMENT_MODE,e.QUOTE_STRING_MODE,e.NUMBER_MODE]}}},2021(e){e.exports=function(e){const t={variants:[e.COMMENT(";","$",{relevance:0}),e.COMMENT("#cs","#ce"),e.COMMENT("#comments-start","#comments-end")]},n={begin:"\\$[A-z0-9_]+"},a={className:"string",variants:[{begin:/"/,end:/"/,contains:[{begin:/""/,relevance:0}]},{begin:/'/,end:/'/,contains:[{begin:/''/,relevance:0}]}]},r={variants:[e.BINARY_NUMBER_MODE,e.C_NUMBER_MODE]};return{name:"AutoIt",case_insensitive:!0,illegal:/\/\*/,keywords:{keyword:"ByRef Case Const ContinueCase ContinueLoop Dim Do Else ElseIf EndFunc EndIf EndSelect EndSwitch EndWith Enum Exit ExitLoop For Func Global If In Local Next ReDim Return Select Static Step Switch Then To Until Volatile WEnd While With",built_in:"Abs ACos AdlibRegister AdlibUnRegister Asc AscW ASin Assign ATan AutoItSetOption AutoItWinGetTitle AutoItWinSetTitle Beep Binary BinaryLen BinaryMid BinaryToString BitAND BitNOT BitOR BitRotate BitShift BitXOR BlockInput Break Call CDTray Ceiling Chr ChrW ClipGet ClipPut ConsoleRead ConsoleWrite ConsoleWriteError ControlClick ControlCommand ControlDisable ControlEnable ControlFocus ControlGetFocus ControlGetHandle ControlGetPos ControlGetText ControlHide ControlListView ControlMove ControlSend ControlSetText ControlShow ControlTreeView Cos Dec DirCopy DirCreate DirGetSize DirMove DirRemove DllCall DllCallAddress DllCallbackFree DllCallbackGetPtr DllCallbackRegister DllClose DllOpen DllStructCreate DllStructGetData DllStructGetPtr DllStructGetSize DllStructSetData DriveGetDrive DriveGetFileSystem DriveGetLabel DriveGetSerial DriveGetType DriveMapAdd DriveMapDel DriveMapGet DriveSetLabel DriveSpaceFree DriveSpaceTotal DriveStatus EnvGet EnvSet EnvUpdate Eval Execute Exp FileChangeDir FileClose FileCopy FileCreateNTFSLink FileCreateShortcut FileDelete FileExists FileFindFirstFile FileFindNextFile FileFlush FileGetAttrib FileGetEncoding FileGetLongName FileGetPos FileGetShortcut FileGetShortName FileGetSize FileGetTime FileGetVersion FileInstall FileMove FileOpen FileOpenDialog FileRead FileReadLine FileReadToArray FileRecycle FileRecycleEmpty FileSaveDialog FileSelectFolder FileSetAttrib FileSetEnd FileSetPos FileSetTime FileWrite FileWriteLine Floor FtpSetProxy FuncName GUICreate GUICtrlCreateAvi GUICtrlCreateButton GUICtrlCreateCheckbox GUICtrlCreateCombo GUICtrlCreateContextMenu GUICtrlCreateDate GUICtrlCreateDummy GUICtrlCreateEdit GUICtrlCreateGraphic GUICtrlCreateGroup GUICtrlCreateIcon GUICtrlCreateInput GUICtrlCreateLabel GUICtrlCreateList GUICtrlCreateListView GUICtrlCreateListViewItem GUICtrlCreateMenu GUICtrlCreateMenuItem GUICtrlCreateMonthCal GUICtrlCreateObj GUICtrlCreatePic GUICtrlCreateProgress GUICtrlCreateRadio GUICtrlCreateSlider GUICtrlCreateTab GUICtrlCreateTabItem GUICtrlCreateTreeView GUICtrlCreateTreeViewItem GUICtrlCreateUpdown GUICtrlDelete GUICtrlGetHandle GUICtrlGetState GUICtrlRead GUICtrlRecvMsg GUICtrlRegisterListViewSort GUICtrlSendMsg GUICtrlSendToDummy GUICtrlSetBkColor GUICtrlSetColor GUICtrlSetCursor GUICtrlSetData GUICtrlSetDefBkColor GUICtrlSetDefColor GUICtrlSetFont GUICtrlSetGraphic GUICtrlSetImage GUICtrlSetLimit GUICtrlSetOnEvent GUICtrlSetPos GUICtrlSetResizing GUICtrlSetState GUICtrlSetStyle GUICtrlSetTip GUIDelete GUIGetCursorInfo GUIGetMsg GUIGetStyle GUIRegisterMsg GUISetAccelerators GUISetBkColor GUISetCoord GUISetCursor GUISetFont GUISetHelp GUISetIcon GUISetOnEvent GUISetState GUISetStyle GUIStartGroup GUISwitch Hex HotKeySet HttpSetProxy HttpSetUserAgent HWnd InetClose InetGet InetGetInfo InetGetSize InetRead IniDelete IniRead IniReadSection IniReadSectionNames IniRenameSection IniWrite IniWriteSection InputBox Int IsAdmin IsArray IsBinary IsBool IsDeclared IsDllStruct IsFloat IsFunc IsHWnd IsInt IsKeyword IsNumber IsObj IsPtr IsString Log MemGetStats Mod MouseClick MouseClickDrag MouseDown MouseGetCursor MouseGetPos MouseMove MouseUp MouseWheel MsgBox Number ObjCreate ObjCreateInterface ObjEvent ObjGet ObjName OnAutoItExitRegister OnAutoItExitUnRegister Ping PixelChecksum PixelGetColor PixelSearch ProcessClose ProcessExists ProcessGetStats ProcessList ProcessSetPriority ProcessWait ProcessWaitClose ProgressOff ProgressOn ProgressSet Ptr Random RegDelete RegEnumKey RegEnumVal RegRead RegWrite Round Run RunAs RunAsWait RunWait Send SendKeepActive SetError SetExtended ShellExecute ShellExecuteWait Shutdown Sin Sleep SoundPlay SoundSetWaveVolume SplashImageOn SplashOff SplashTextOn Sqrt SRandom StatusbarGetText StderrRead StdinWrite StdioClose StdoutRead String StringAddCR StringCompare StringFormat StringFromASCIIArray StringInStr StringIsAlNum StringIsAlpha StringIsASCII StringIsDigit StringIsFloat StringIsInt StringIsLower StringIsSpace StringIsUpper StringIsXDigit StringLeft StringLen StringLower StringMid StringRegExp StringRegExpReplace StringReplace StringReverse StringRight StringSplit StringStripCR StringStripWS StringToASCIIArray StringToBinary StringTrimLeft StringTrimRight StringUpper Tan TCPAccept TCPCloseSocket TCPConnect TCPListen TCPNameToIP TCPRecv TCPSend TCPShutdown, UDPShutdown TCPStartup, UDPStartup TimerDiff TimerInit ToolTip TrayCreateItem TrayCreateMenu TrayGetMsg TrayItemDelete TrayItemGetHandle TrayItemGetState TrayItemGetText TrayItemSetOnEvent TrayItemSetState TrayItemSetText TraySetClick TraySetIcon TraySetOnEvent TraySetPauseIcon TraySetState TraySetToolTip TrayTip UBound UDPBind UDPCloseSocket UDPOpen UDPRecv UDPSend VarGetType WinActivate WinActive WinClose WinExists WinFlash WinGetCaretPos WinGetClassList WinGetClientSize WinGetHandle WinGetPos WinGetProcess WinGetState WinGetText WinGetTitle WinKill WinList WinMenuSelectItem WinMinimizeAll WinMinimizeAllUndo WinMove WinSetOnTop WinSetState WinSetTitle WinSetTrans WinWait WinWaitActive WinWaitClose WinWaitNotActive",literal:"True False And Null Not Or Default"},contains:[t,n,a,r,{className:"meta",begin:"#",end:"$",keywords:{keyword:["EndRegion","forcedef","forceref","ignorefunc","include","include-once","NoTrayIcon","OnAutoItStartRegister","pragma","Region","RequireAdmin","Tidy_Off","Tidy_On","Tidy_Parameters"]},contains:[{begin:/\\\n/,relevance:0},{beginKeywords:"include",keywords:{keyword:"include"},end:"$",contains:[a,{className:"string",variants:[{begin:"<",end:">"},{begin:/"/,end:/"/,contains:[{begin:/""/,relevance:0}]},{begin:/'/,end:/'/,contains:[{begin:/''/,relevance:0}]}]}]},a,t]},{className:"symbol",begin:"@[A-z0-9_]+"},{beginKeywords:"Func",end:"$",illegal:"\\$|\\[|%",contains:[e.inherit(e.UNDERSCORE_TITLE_MODE,{className:"title.function"}),{className:"params",begin:"\\(",end:"\\)",contains:[n,a,r]}]}]}}},2188(e,t){"use strict";Object.defineProperty(t,"__esModule",{value:!0});var n=function(){function e(){}return e.prototype.diff=function(e,t,n){var a;void 0===n&&(n={}),"function"==typeof n?(a=n,n={}):"callback"in n&&(a=n.callback);var r=this.castInput(e,n),i=this.castInput(t,n),o=this.removeEmpty(this.tokenize(r,n)),s=this.removeEmpty(this.tokenize(i,n));return this.diffWithOptionsObj(o,s,n,a)},e.prototype.diffWithOptionsObj=function(e,t,n,a){var r,i=this,o=function(e){return e=i.postProcess(e,n),a?void setTimeout((function(){a(e)}),0):e},s=t.length,l=e.length,c=1,_=s+l;null!=n.maxEditLength&&(_=Math.min(_,n.maxEditLength));var d=null!==(r=n.timeout)&&void 0!==r?r:1/0,p=Date.now()+d,u=[{oldPos:-1,lastComponent:void 0}],m=this.extractCommon(u[0],t,e,0,n);if(u[0].oldPos+1>=l&&m+1>=s)return o(this.buildValues(u[0].lastComponent,t,e));var g=-1/0,E=1/0,S=function(){for(var a=Math.max(g,-c);a<=Math.min(E,c);a+=2){var r=void 0,_=u[a-1],d=u[a+1];_&&(u[a-1]=void 0);var p=!1;if(d){var S=d.oldPos-a;p=d&&0<=S&&S=l&&m+1>=s)return o(i.buildValues(r.lastComponent,t,e))||!0;u[a]=r,r.oldPos+1>=l&&(E=Math.min(E,a-1)),m+1>=s&&(g=Math.max(g,a+1))}else u[a]=void 0}c++};if(a)!function e(){setTimeout((function(){if(c>_||Date.now()>p)return a(void 0);S()||e()}),0)}();else for(;c<=_&&Date.now()<=p;){var b=S();if(b)return b}},e.prototype.addToPath=function(e,t,n,a,r){var i=e.lastComponent;return i&&!r.oneChangePerToken&&i.added===t&&i.removed===n?{oldPos:e.oldPos+a,lastComponent:{count:i.count+1,added:t,removed:n,previousComponent:i.previousComponent}}:{oldPos:e.oldPos+a,lastComponent:{count:1,added:t,removed:n,previousComponent:i}}},e.prototype.extractCommon=function(e,t,n,a,r){for(var i=t.length,o=n.length,s=e.oldPos,l=s-a,c=0;l+1e.length?a:e})),c.value=this.join(_)}else c.value=this.join(t.slice(s,s+c.count));s+=c.count,c.added||(l+=c.count)}}return r},e}();t.default=n},2274(e){e.exports=function(e){const t={className:"subst",variants:[{begin:"\\$[A-Za-z0-9_]+"}]},n={className:"subst",variants:[{begin:/\$\{/,end:/\}/}],keywords:"true false null this is new super"},a={className:"number",relevance:0,variants:[{match:/\b[0-9][0-9_]*(\.[0-9][0-9_]*)?([eE][+-]?[0-9][0-9_]*)?\b/},{match:/\b0[xX][0-9A-Fa-f][0-9A-Fa-f_]*\b/}]},r={className:"string",variants:[{begin:"r'''",end:"'''"},{begin:'r"""',end:'"""'},{begin:"r'",end:"'",illegal:"\\n"},{begin:'r"',end:'"',illegal:"\\n"},{begin:"'''",end:"'''",contains:[e.BACKSLASH_ESCAPE,t,n]},{begin:'"""',end:'"""',contains:[e.BACKSLASH_ESCAPE,t,n]},{begin:"'",end:"'",illegal:"\\n",contains:[e.BACKSLASH_ESCAPE,t,n]},{begin:'"',end:'"',illegal:"\\n",contains:[e.BACKSLASH_ESCAPE,t,n]}]};n.contains=[a,r];const i=["Comparable","DateTime","Duration","Function","Iterable","Iterator","List","Map","Match","Object","Pattern","RegExp","Set","Stopwatch","String","StringBuffer","StringSink","Symbol","Type","Uri","bool","double","int","num","Element","ElementList"],o=i.map((e=>`${e}?`));return{name:"Dart",keywords:{keyword:["abstract","as","assert","async","await","base","break","case","catch","class","const","continue","covariant","default","deferred","do","dynamic","else","enum","export","extends","extension","external","factory","false","final","finally","for","Function","get","hide","if","implements","import","in","interface","is","late","library","mixin","new","null","on","operator","part","required","rethrow","return","sealed","set","show","static","super","switch","sync","this","throw","true","try","typedef","var","void","when","while","with","yield"],built_in:i.concat(o).concat(["Never","Null","dynamic","print","document","querySelector","querySelectorAll","window"]),$pattern:/[A-Za-z][A-Za-z0-9_]*\??/},contains:[r,e.COMMENT(/\/\*\*(?!\/)/,/\*\//,{subLanguage:"markdown",relevance:0}),e.COMMENT(/\/{3,} ?/,/$/,{contains:[{subLanguage:"markdown",begin:".",end:"$",relevance:0}]}),e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE,{className:"class",beginKeywords:"class interface",end:/\{/,excludeEnd:!0,contains:[{beginKeywords:"extends implements"},e.UNDERSCORE_TITLE_MODE]},a,{className:"meta",begin:"@[A-Za-z]+"},{begin:"=>"}]}}},2302(e,t,n){"use strict";var a,r=this&&this.__createBinding||(Object.create?function(e,t,n,a){void 0===a&&(a=n);var r=Object.getOwnPropertyDescriptor(t,n);r&&!("get"in r?!t.__esModule:r.writable||r.configurable)||(r={enumerable:!0,get:function(){return t[n]}}),Object.defineProperty(e,a,r)}:function(e,t,n,a){void 0===a&&(a=n),e[a]=t[n]}),i=this&&this.__setModuleDefault||(Object.create?function(e,t){Object.defineProperty(e,"default",{enumerable:!0,value:t})}:function(e,t){e.default=t}),o=this&&this.__importStar||(a=function(e){return a=Object.getOwnPropertyNames||function(e){var t=[];for(var n in e)Object.prototype.hasOwnProperty.call(e,n)&&(t[t.length]=n);return t},a(e)},function(e){if(e&&e.__esModule)return e;var t={};if(null!=e)for(var n=a(e),o=0;oo||d.content.length>o)return{oldLine:{prefix:_.prefix,content:E(_.content)},newLine:{prefix:d.prefix,content:E(d.content)}};const m="char"===c?s.diffChars(_.content,d.content):s.diffWordsWithSpace(_.content,d.content),b=[];if("word"===c&&"words"===i){const e=m.filter((e=>e.removed)),t=m.filter((e=>e.added));u(t,e).forEach((e=>{1===e[0].length&&1===e[1].length&&p(e[0][0],e[1][0]){const n=t.added?"ins":t.removed?"del":null,a=b.indexOf(t)>-1?' class="d2h-change"':"",r=E(t.value);return null!==n?`${e}<${n}${a}>${r}`:`${e}${r}`}),"");return{oldLine:{prefix:_.prefix,content:(T=f,T.replace(/(]*>((.|\n)*?)<\/ins>)/g,""))},newLine:{prefix:d.prefix,content:g(f)}};var T};const s=o(n(6801)),l=n(6185),c=o(n(4598)),_=n(1613);t.CSSLineClass={INSERTS:"d2h-ins",DELETES:"d2h-del",CONTEXT:"d2h-cntx",INFO:"d2h-info",INSERT_CHANGES:"d2h-ins d2h-change",DELETE_CHANGES:"d2h-del d2h-change"},t.defaultRenderConfig={matching:_.LineMatchingType.NONE,matchWordsThreshold:.25,maxLineLengthHighlight:1e4,diffStyle:_.DiffStyleType.WORD,colorScheme:_.ColorSchemeType.LIGHT};const d="/",p=c.newDistanceFn((e=>e.value)),u=c.newMatcherFn(p);function m(e){return-1!==e.indexOf("dev/null")}function g(e){return e.replace(/(]*>((.|\n)*?)<\/del>)/g,"")}function E(e){return e.slice(0).replace(/&/g,"&").replace(//g,">").replace(/"/g,""").replace(/'/g,"'").replace(/\//g,"/")}function S(e,t,n=!0){const a=function(e){return e?2:1}(t);return{prefix:e.substring(0,a),content:n?E(e.substring(a)):e.substring(a)}}function b(e){const t=(0,l.unifyPath)(e.oldName),n=(0,l.unifyPath)(e.newName);if(t===n||m(t)||m(n))return m(n)?t:n;{const e=[],a=[],r=t.split(d),i=n.split(d);let o=0,s=r.length-1,l=i.length-1;for(;oo&&l>o&&r[s]===i[l];)a.unshift(i[l]),s-=1,l-=1;const c=e.join(d),_=a.join(d),p=r.slice(o,s+1).join(d),u=i.slice(o,l+1).join(d);return c.length&&_.length?c+d+"{"+p+" → "+u+"}"+d+_:c.length?c+d+"{"+p+" → "+u+"}":_.length?"{"+p+" → "+u+"}"+d+_:t+" → "+n}}},2480(e){e.exports=function(e){const t={className:"number",relevance:0,begin:e.C_NUMBER_RE};return{name:"LSL (Linden Scripting Language)",illegal:":",contains:[{className:"string",begin:'"',end:'"',contains:[{className:"subst",begin:/\\[tn"\\]/}]},{className:"comment",variants:[e.COMMENT("//","$"),e.COMMENT("/\\*","\\*/")],relevance:0},t,{className:"section",variants:[{begin:"\\b(state|default)\\b"},{begin:"\\b(state_(entry|exit)|touch(_(start|end))?|(land_)?collision(_(start|end))?|timer|listen|(no_)?sensor|control|(not_)?at_(rot_)?target|money|email|experience_permissions(_denied)?|run_time_permissions|changed|attach|dataserver|moving_(start|end)|link_message|(on|object)_rez|remote_data|http_re(sponse|quest)|path_update|transaction_result)\\b"}]},{className:"built_in",begin:"\\b(ll(AgentInExperience|(Create|DataSize|Delete|KeyCount|Keys|Read|Update)KeyValue|GetExperience(Details|ErrorMessage)|ReturnObjectsBy(ID|Owner)|Json(2List|[GS]etValue|ValueType)|Sin|Cos|Tan|Atan2|Sqrt|Pow|Abs|Fabs|Frand|Floor|Ceil|Round|Vec(Mag|Norm|Dist)|Rot(Between|2(Euler|Fwd|Left|Up))|(Euler|Axes)2Rot|Whisper|(Region|Owner)?Say|Shout|Listen(Control|Remove)?|Sensor(Repeat|Remove)?|Detected(Name|Key|Owner|Type|Pos|Vel|Grab|Rot|Group|LinkNumber)|Die|Ground|Wind|([GS]et)(AnimationOverride|MemoryLimit|PrimMediaParams|ParcelMusicURL|Object(Desc|Name)|PhysicsMaterial|Status|Scale|Color|Alpha|Texture|Pos|Rot|Force|Torque)|ResetAnimationOverride|(Scale|Offset|Rotate)Texture|(Rot)?Target(Remove)?|(Stop)?MoveToTarget|Apply(Rotational)?Impulse|Set(KeyframedMotion|ContentType|RegionPos|(Angular)?Velocity|Buoyancy|HoverHeight|ForceAndTorque|TimerEvent|ScriptState|Damage|TextureAnim|Sound(Queueing|Radius)|Vehicle(Type|(Float|Vector|Rotation)Param)|(Touch|Sit)?Text|Camera(Eye|At)Offset|PrimitiveParams|ClickAction|Link(Alpha|Color|PrimitiveParams(Fast)?|Texture(Anim)?|Camera|Media)|RemoteScriptAccessPin|PayPrice|LocalRot)|ScaleByFactor|Get((Max|Min)ScaleFactor|ClosestNavPoint|StaticPath|SimStats|Env|PrimitiveParams|Link(PrimitiveParams|Number(OfSides)?|Key|Name|Media)|HTTPHeader|FreeURLs|Object(Details|PermMask|PrimCount)|Parcel(MaxPrims|Details|Prim(Count|Owners))|Attached(List)?|(SPMax|Free|Used)Memory|Region(Name|TimeDilation|FPS|Corner|AgentCount)|Root(Position|Rotation)|UnixTime|(Parcel|Region)Flags|(Wall|GMT)clock|SimulatorHostname|BoundingBox|GeometricCenter|Creator|NumberOf(Prims|NotecardLines|Sides)|Animation(List)?|(Camera|Local)(Pos|Rot)|Vel|Accel|Omega|Time(stamp|OfDay)|(Object|CenterOf)?Mass|MassMKS|Energy|Owner|(Owner)?Key|SunDirection|Texture(Offset|Scale|Rot)|Inventory(Number|Name|Key|Type|Creator|PermMask)|Permissions(Key)?|StartParameter|List(Length|EntryType)|Date|Agent(Size|Info|Language|List)|LandOwnerAt|NotecardLine|Script(Name|State))|(Get|Reset|GetAndReset)Time|PlaySound(Slave)?|LoopSound(Master|Slave)?|(Trigger|Stop|Preload)Sound|((Get|Delete)Sub|Insert)String|To(Upper|Lower)|Give(InventoryList|Money)|RezObject|(Stop)?LookAt|Sleep|CollisionFilter|(Take|Release)Controls|DetachFromAvatar|AttachToAvatar(Temp)?|InstantMessage|(GetNext)?Email|StopHover|MinEventDelay|RotLookAt|String(Length|Trim)|(Start|Stop)Animation|TargetOmega|Request(Experience)?Permissions|(Create|Break)Link|BreakAllLinks|(Give|Remove)Inventory|Water|PassTouches|Request(Agent|Inventory)Data|TeleportAgent(Home|GlobalCoords)?|ModifyLand|CollisionSound|ResetScript|MessageLinked|PushObject|PassCollisions|AxisAngle2Rot|Rot2(Axis|Angle)|A(cos|sin)|AngleBetween|AllowInventoryDrop|SubStringIndex|List2(CSV|Integer|Json|Float|String|Key|Vector|Rot|List(Strided)?)|DeleteSubList|List(Statistics|Sort|Randomize|(Insert|Find|Replace)List)|EdgeOfWorld|AdjustSoundVolume|Key2Name|TriggerSoundLimited|EjectFromLand|(CSV|ParseString)2List|OverMyLand|SameGroup|UnSit|Ground(Slope|Normal|Contour)|GroundRepel|(Set|Remove)VehicleFlags|SitOnLink|(AvatarOn)?(Link)?SitTarget|Script(Danger|Profiler)|Dialog|VolumeDetect|ResetOtherScript|RemoteLoadScriptPin|(Open|Close)RemoteDataChannel|SendRemoteData|RemoteDataReply|(Integer|String)ToBase64|XorBase64|Log(10)?|Base64To(String|Integer)|ParseStringKeepNulls|RezAtRoot|RequestSimulatorData|ForceMouselook|(Load|Release|(E|Une)scape)URL|ParcelMedia(CommandList|Query)|ModPow|MapDestination|(RemoveFrom|AddTo|Reset)Land(Pass|Ban)List|(Set|Clear)CameraParams|HTTP(Request|Response)|TextBox|DetectedTouch(UV|Face|Pos|(N|Bin)ormal|ST)|(MD5|SHA1|DumpList2)String|Request(Secure)?URL|Clear(Prim|Link)Media|(Link)?ParticleSystem|(Get|Request)(Username|DisplayName)|RegionSayTo|CastRay|GenerateKey|TransferLindenDollars|ManageEstateAccess|(Create|Delete)Character|ExecCharacterCmd|Evade|FleeFrom|NavigateTo|PatrolPoints|Pursue|UpdateCharacter|WanderWithin))\\b"},{className:"literal",variants:[{begin:"\\b(PI|TWO_PI|PI_BY_TWO|DEG_TO_RAD|RAD_TO_DEG|SQRT2)\\b"},{begin:"\\b(XP_ERROR_(EXPERIENCES_DISABLED|EXPERIENCE_(DISABLED|SUSPENDED)|INVALID_(EXPERIENCE|PARAMETERS)|KEY_NOT_FOUND|MATURITY_EXCEEDED|NONE|NOT_(FOUND|PERMITTED(_LAND)?)|NO_EXPERIENCE|QUOTA_EXCEEDED|RETRY_UPDATE|STORAGE_EXCEPTION|STORE_DISABLED|THROTTLED|UNKNOWN_ERROR)|JSON_APPEND|STATUS_(PHYSICS|ROTATE_[XYZ]|PHANTOM|SANDBOX|BLOCK_GRAB(_OBJECT)?|(DIE|RETURN)_AT_EDGE|CAST_SHADOWS|OK|MALFORMED_PARAMS|TYPE_MISMATCH|BOUNDS_ERROR|NOT_(FOUND|SUPPORTED)|INTERNAL_ERROR|WHITELIST_FAILED)|AGENT(_(BY_(LEGACY_|USER)NAME|FLYING|ATTACHMENTS|SCRIPTED|MOUSELOOK|SITTING|ON_OBJECT|AWAY|WALKING|IN_AIR|TYPING|CROUCHING|BUSY|ALWAYS_RUN|AUTOPILOT|LIST_(PARCEL(_OWNER)?|REGION)))?|CAMERA_(PITCH|DISTANCE|BEHINDNESS_(ANGLE|LAG)|(FOCUS|POSITION)(_(THRESHOLD|LOCKED|LAG))?|FOCUS_OFFSET|ACTIVE)|ANIM_ON|LOOP|REVERSE|PING_PONG|SMOOTH|ROTATE|SCALE|ALL_SIDES|LINK_(ROOT|SET|ALL_(OTHERS|CHILDREN)|THIS)|ACTIVE|PASS(IVE|_(ALWAYS|IF_NOT_HANDLED|NEVER))|SCRIPTED|CONTROL_(FWD|BACK|(ROT_)?(LEFT|RIGHT)|UP|DOWN|(ML_)?LBUTTON)|PERMISSION_(RETURN_OBJECTS|DEBIT|OVERRIDE_ANIMATIONS|SILENT_ESTATE_MANAGEMENT|TAKE_CONTROLS|TRIGGER_ANIMATION|ATTACH|CHANGE_LINKS|(CONTROL|TRACK)_CAMERA|TELEPORT)|INVENTORY_(TEXTURE|SOUND|OBJECT|SCRIPT|LANDMARK|CLOTHING|NOTECARD|BODYPART|ANIMATION|GESTURE|ALL|NONE)|CHANGED_(INVENTORY|COLOR|SHAPE|SCALE|TEXTURE|LINK|ALLOWED_DROP|OWNER|REGION(_START)?|TELEPORT|MEDIA)|OBJECT_(CLICK_ACTION|HOVER_HEIGHT|LAST_OWNER_ID|(PHYSICS|SERVER|STREAMING)_COST|UNKNOWN_DETAIL|CHARACTER_TIME|PHANTOM|PHYSICS|TEMP_(ATTACHED|ON_REZ)|NAME|DESC|POS|PRIM_(COUNT|EQUIVALENCE)|RETURN_(PARCEL(_OWNER)?|REGION)|REZZER_KEY|ROO?T|VELOCITY|OMEGA|OWNER|GROUP(_TAG)?|CREATOR|ATTACHED_(POINT|SLOTS_AVAILABLE)|RENDER_WEIGHT|(BODY_SHAPE|PATHFINDING)_TYPE|(RUNNING|TOTAL)_SCRIPT_COUNT|TOTAL_INVENTORY_COUNT|SCRIPT_(MEMORY|TIME))|TYPE_(INTEGER|FLOAT|STRING|KEY|VECTOR|ROTATION|INVALID)|(DEBUG|PUBLIC)_CHANNEL|ATTACH_(AVATAR_CENTER|CHEST|HEAD|BACK|PELVIS|MOUTH|CHIN|NECK|NOSE|BELLY|[LR](SHOULDER|HAND|FOOT|EAR|EYE|[UL](ARM|LEG)|HIP)|(LEFT|RIGHT)_PEC|HUD_(CENTER_[12]|TOP_(RIGHT|CENTER|LEFT)|BOTTOM(_(RIGHT|LEFT))?)|[LR]HAND_RING1|TAIL_(BASE|TIP)|[LR]WING|FACE_(JAW|[LR]EAR|[LR]EYE|TOUNGE)|GROIN|HIND_[LR]FOOT)|LAND_(LEVEL|RAISE|LOWER|SMOOTH|NOISE|REVERT)|DATA_(ONLINE|NAME|BORN|SIM_(POS|STATUS|RATING)|PAYINFO)|PAYMENT_INFO_(ON_FILE|USED)|REMOTE_DATA_(CHANNEL|REQUEST|REPLY)|PSYS_(PART_(BF_(ZERO|ONE(_MINUS_(DEST_COLOR|SOURCE_(ALPHA|COLOR)))?|DEST_COLOR|SOURCE_(ALPHA|COLOR))|BLEND_FUNC_(DEST|SOURCE)|FLAGS|(START|END)_(COLOR|ALPHA|SCALE|GLOW)|MAX_AGE|(RIBBON|WIND|INTERP_(COLOR|SCALE)|BOUNCE|FOLLOW_(SRC|VELOCITY)|TARGET_(POS|LINEAR)|EMISSIVE)_MASK)|SRC_(MAX_AGE|PATTERN|ANGLE_(BEGIN|END)|BURST_(RATE|PART_COUNT|RADIUS|SPEED_(MIN|MAX))|ACCEL|TEXTURE|TARGET_KEY|OMEGA|PATTERN_(DROP|EXPLODE|ANGLE(_CONE(_EMPTY)?)?)))|VEHICLE_(REFERENCE_FRAME|TYPE_(NONE|SLED|CAR|BOAT|AIRPLANE|BALLOON)|(LINEAR|ANGULAR)_(FRICTION_TIMESCALE|MOTOR_DIRECTION)|LINEAR_MOTOR_OFFSET|HOVER_(HEIGHT|EFFICIENCY|TIMESCALE)|BUOYANCY|(LINEAR|ANGULAR)_(DEFLECTION_(EFFICIENCY|TIMESCALE)|MOTOR_(DECAY_)?TIMESCALE)|VERTICAL_ATTRACTION_(EFFICIENCY|TIMESCALE)|BANKING_(EFFICIENCY|MIX|TIMESCALE)|FLAG_(NO_DEFLECTION_UP|LIMIT_(ROLL_ONLY|MOTOR_UP)|HOVER_((WATER|TERRAIN|UP)_ONLY|GLOBAL_HEIGHT)|MOUSELOOK_(STEER|BANK)|CAMERA_DECOUPLED))|PRIM_(ALLOW_UNSIT|ALPHA_MODE(_(BLEND|EMISSIVE|MASK|NONE))?|NORMAL|SPECULAR|TYPE(_(BOX|CYLINDER|PRISM|SPHERE|TORUS|TUBE|RING|SCULPT))?|HOLE_(DEFAULT|CIRCLE|SQUARE|TRIANGLE)|MATERIAL(_(STONE|METAL|GLASS|WOOD|FLESH|PLASTIC|RUBBER))?|SHINY_(NONE|LOW|MEDIUM|HIGH)|BUMP_(NONE|BRIGHT|DARK|WOOD|BARK|BRICKS|CHECKER|CONCRETE|TILE|STONE|DISKS|GRAVEL|BLOBS|SIDING|LARGETILE|STUCCO|SUCTION|WEAVE)|TEXGEN_(DEFAULT|PLANAR)|SCRIPTED_SIT_ONLY|SCULPT_(TYPE_(SPHERE|TORUS|PLANE|CYLINDER|MASK)|FLAG_(MIRROR|INVERT))|PHYSICS(_(SHAPE_(CONVEX|NONE|PRIM|TYPE)))?|(POS|ROT)_LOCAL|SLICE|TEXT|FLEXIBLE|POINT_LIGHT|TEMP_ON_REZ|PHANTOM|POSITION|SIT_TARGET|SIZE|ROTATION|TEXTURE|NAME|OMEGA|DESC|LINK_TARGET|COLOR|BUMP_SHINY|FULLBRIGHT|TEXGEN|GLOW|MEDIA_(ALT_IMAGE_ENABLE|CONTROLS|(CURRENT|HOME)_URL|AUTO_(LOOP|PLAY|SCALE|ZOOM)|FIRST_CLICK_INTERACT|(WIDTH|HEIGHT)_PIXELS|WHITELIST(_ENABLE)?|PERMS_(INTERACT|CONTROL)|PARAM_MAX|CONTROLS_(STANDARD|MINI)|PERM_(NONE|OWNER|GROUP|ANYONE)|MAX_(URL_LENGTH|WHITELIST_(SIZE|COUNT)|(WIDTH|HEIGHT)_PIXELS)))|MASK_(BASE|OWNER|GROUP|EVERYONE|NEXT)|PERM_(TRANSFER|MODIFY|COPY|MOVE|ALL)|PARCEL_(MEDIA_COMMAND_(STOP|PAUSE|PLAY|LOOP|TEXTURE|URL|TIME|AGENT|UNLOAD|AUTO_ALIGN|TYPE|SIZE|DESC|LOOP_SET)|FLAG_(ALLOW_(FLY|(GROUP_)?SCRIPTS|LANDMARK|TERRAFORM|DAMAGE|CREATE_(GROUP_)?OBJECTS)|USE_(ACCESS_(GROUP|LIST)|BAN_LIST|LAND_PASS_LIST)|LOCAL_SOUND_ONLY|RESTRICT_PUSHOBJECT|ALLOW_(GROUP|ALL)_OBJECT_ENTRY)|COUNT_(TOTAL|OWNER|GROUP|OTHER|SELECTED|TEMP)|DETAILS_(NAME|DESC|OWNER|GROUP|AREA|ID|SEE_AVATARS))|LIST_STAT_(MAX|MIN|MEAN|MEDIAN|STD_DEV|SUM(_SQUARES)?|NUM_COUNT|GEOMETRIC_MEAN|RANGE)|PAY_(HIDE|DEFAULT)|REGION_FLAG_(ALLOW_DAMAGE|FIXED_SUN|BLOCK_TERRAFORM|SANDBOX|DISABLE_(COLLISIONS|PHYSICS)|BLOCK_FLY|ALLOW_DIRECT_TELEPORT|RESTRICT_PUSHOBJECT)|HTTP_(METHOD|MIMETYPE|BODY_(MAXLENGTH|TRUNCATED)|CUSTOM_HEADER|PRAGMA_NO_CACHE|VERBOSE_THROTTLE|VERIFY_CERT)|SIT_(INVALID_(AGENT|LINK_OBJECT)|NO(T_EXPERIENCE|_(ACCESS|EXPERIENCE_PERMISSION|SIT_TARGET)))|STRING_(TRIM(_(HEAD|TAIL))?)|CLICK_ACTION_(NONE|TOUCH|SIT|BUY|PAY|OPEN(_MEDIA)?|PLAY|ZOOM)|TOUCH_INVALID_FACE|PROFILE_(NONE|SCRIPT_MEMORY)|RC_(DATA_FLAGS|DETECT_PHANTOM|GET_(LINK_NUM|NORMAL|ROOT_KEY)|MAX_HITS|REJECT_(TYPES|AGENTS|(NON)?PHYSICAL|LAND))|RCERR_(CAST_TIME_EXCEEDED|SIM_PERF_LOW|UNKNOWN)|ESTATE_ACCESS_(ALLOWED_(AGENT|GROUP)_(ADD|REMOVE)|BANNED_AGENT_(ADD|REMOVE))|DENSITY|FRICTION|RESTITUTION|GRAVITY_MULTIPLIER|KFM_(COMMAND|CMD_(PLAY|STOP|PAUSE)|MODE|FORWARD|LOOP|PING_PONG|REVERSE|DATA|ROTATION|TRANSLATION)|ERR_(GENERIC|PARCEL_PERMISSIONS|MALFORMED_PARAMS|RUNTIME_PERMISSIONS|THROTTLED)|CHARACTER_(CMD_((SMOOTH_)?STOP|JUMP)|DESIRED_(TURN_)?SPEED|RADIUS|STAY_WITHIN_PARCEL|LENGTH|ORIENTATION|ACCOUNT_FOR_SKIPPED_FRAMES|AVOIDANCE_MODE|TYPE(_([ABCD]|NONE))?|MAX_(DECEL|TURN_RADIUS|(ACCEL|SPEED)))|PURSUIT_(OFFSET|FUZZ_FACTOR|GOAL_TOLERANCE|INTERCEPT)|REQUIRE_LINE_OF_SIGHT|FORCE_DIRECT_PATH|VERTICAL|HORIZONTAL|AVOID_(CHARACTERS|DYNAMIC_OBSTACLES|NONE)|PU_(EVADE_(HIDDEN|SPOTTED)|FAILURE_(DYNAMIC_PATHFINDING_DISABLED|INVALID_(GOAL|START)|NO_(NAVMESH|VALID_DESTINATION)|OTHER|TARGET_GONE|(PARCEL_)?UNREACHABLE)|(GOAL|SLOWDOWN_DISTANCE)_REACHED)|TRAVERSAL_TYPE(_(FAST|NONE|SLOW))?|CONTENT_TYPE_(ATOM|FORM|HTML|JSON|LLSD|RSS|TEXT|XHTML|XML)|GCNP_(RADIUS|STATIC)|(PATROL|WANDER)_PAUSE_AT_WAYPOINTS|OPT_(AVATAR|CHARACTER|EXCLUSION_VOLUME|LEGACY_LINKSET|MATERIAL_VOLUME|OTHER|STATIC_OBSTACLE|WALKABLE)|SIM_STAT_PCT_CHARS_STEPPED)\\b"},{begin:"\\b(FALSE|TRUE)\\b"},{begin:"\\b(ZERO_ROTATION)\\b"},{begin:"\\b(EOF|JSON_(ARRAY|DELETE|FALSE|INVALID|NULL|NUMBER|OBJECT|STRING|TRUE)|NULL_KEY|TEXTURE_(BLANK|DEFAULT|MEDIA|PLYWOOD|TRANSPARENT)|URL_REQUEST_(GRANTED|DENIED))\\b"},{begin:"\\b(ZERO_VECTOR|TOUCH_INVALID_(TEXCOORD|VECTOR))\\b"}]},{className:"type",begin:"\\b(integer|float|string|key|vector|quaternion|rotation|list)\\b"}]}}},2491(e){e.exports=function(e){const t="[a-zA-Z_\\-+\\*\\/<=>&#][a-zA-Z0-9_\\-+*\\/<=>&#!]*",n="\\|[^]*?\\|",a="(-|\\+)?\\d+(\\.\\d+|\\/\\d+)?((d|e|f|l|s|D|E|F|L|S)(\\+|-)?\\d+)?",r={className:"literal",begin:"\\b(t{1}|nil)\\b"},i={className:"number",variants:[{begin:a,relevance:0},{begin:"#(b|B)[0-1]+(/[0-1]+)?"},{begin:"#(o|O)[0-7]+(/[0-7]+)?"},{begin:"#(x|X)[0-9a-fA-F]+(/[0-9a-fA-F]+)?"},{begin:"#(c|C)\\("+a+" +"+a,end:"\\)"}]},o=e.inherit(e.QUOTE_STRING_MODE,{illegal:null}),s=e.COMMENT(";","$",{relevance:0}),l={begin:"\\*",end:"\\*"},c={className:"symbol",begin:"[:&]"+t},_={begin:t,relevance:0},d={begin:n},p={contains:[i,o,l,c,{begin:"\\(",end:"\\)",contains:["self",r,o,i,_]},_],variants:[{begin:"['`]\\(",end:"\\)"},{begin:"\\(quote ",end:"\\)",keywords:{name:"quote"}},{begin:"'"+n}]},u={variants:[{begin:"'"+t},{begin:"#'"+t+"(::"+t+")*"}]},m={begin:"\\(\\s*",end:"\\)"},g={endsWithParent:!0,relevance:0};return m.contains=[{className:"name",variants:[{begin:t,relevance:0},{begin:n}]},g],g.contains=[p,u,m,r,i,o,s,l,c,d,_],{name:"Lisp",illegal:/\S/,contains:[i,e.SHEBANG(),r,o,s,p,u,m,_]}}},2546(e){e.exports=function(e){return{name:"Stata",aliases:["do","ado"],case_insensitive:!0,keywords:"if else in foreach for forv forva forval forvalu forvalue forvalues by bys bysort xi quietly qui capture about ac ac_7 acprplot acprplot_7 adjust ado adopath adoupdate alpha ameans an ano anov anova anova_estat anova_terms anovadef aorder ap app appe appen append arch arch_dr arch_estat arch_p archlm areg areg_p args arima arima_dr arima_estat arima_p as asmprobit asmprobit_estat asmprobit_lf asmprobit_mfx__dlg asmprobit_p ass asse asser assert avplot avplot_7 avplots avplots_7 bcskew0 bgodfrey bias binreg bip0_lf biplot bipp_lf bipr_lf bipr_p biprobit bitest bitesti bitowt blogit bmemsize boot bootsamp bootstrap bootstrap_8 boxco_l boxco_p boxcox boxcox_6 boxcox_p bprobit br break brier bro brow brows browse brr brrstat bs bs_7 bsampl_w bsample bsample_7 bsqreg bstat bstat_7 bstat_8 bstrap bstrap_7 bubble bubbleplot ca ca_estat ca_p cabiplot camat canon canon_8 canon_8_p canon_estat canon_p cap caprojection capt captu captur capture cat cc cchart cchart_7 cci cd censobs_table centile cf char chdir checkdlgfiles checkestimationsample checkhlpfiles checksum chelp ci cii cl class classutil clear cli clis clist clo clog clog_lf clog_p clogi clogi_sw clogit clogit_lf clogit_p clogitp clogl_sw cloglog clonevar clslistarray cluster cluster_measures cluster_stop cluster_tree cluster_tree_8 clustermat cmdlog cnr cnre cnreg cnreg_p cnreg_sw cnsreg codebook collaps4 collapse colormult_nb colormult_nw compare compress conf confi confir confirm conren cons const constr constra constrai constrain constraint continue contract copy copyright copysource cor corc corr corr2data corr_anti corr_kmo corr_smc corre correl correla correlat correlate corrgram cou coun count cox cox_p cox_sw coxbase coxhaz coxvar cprplot cprplot_7 crc cret cretu cretur creturn cross cs cscript cscript_log csi ct ct_is ctset ctst_5 ctst_st cttost cumsp cumsp_7 cumul cusum cusum_7 cutil d|0 datasig datasign datasigna datasignat datasignatu datasignatur datasignature datetof db dbeta de dec deco decod decode deff des desc descr descri describ describe destring dfbeta dfgls dfuller di di_g dir dirstats dis discard disp disp_res disp_s displ displa display distinct do doe doed doedi doedit dotplot dotplot_7 dprobit drawnorm drop ds ds_util dstdize duplicates durbina dwstat dydx e|0 ed edi edit egen eivreg emdef en enc enco encod encode eq erase ereg ereg_lf ereg_p ereg_sw ereghet ereghet_glf ereghet_glf_sh ereghet_gp ereghet_ilf ereghet_ilf_sh ereghet_ip eret eretu eretur ereturn err erro error esize est est_cfexist est_cfname est_clickable est_expand est_hold est_table est_unhold est_unholdok estat estat_default estat_summ estat_vce_only esti estimates etodow etof etomdy ex exi exit expand expandcl fac fact facto factor factor_estat factor_p factor_pca_rotated factor_rotate factormat fcast fcast_compute fcast_graph fdades fdadesc fdadescr fdadescri fdadescrib fdadescribe fdasav fdasave fdause fh_st file open file read file close file filefilter fillin find_hlp_file findfile findit findit_7 fit fl fli flis flist for5_0 forest forestplot form forma format fpredict frac_154 frac_adj frac_chk frac_cox frac_ddp frac_dis frac_dv frac_in frac_mun frac_pp frac_pq frac_pv frac_wgt frac_xo fracgen fracplot fracplot_7 fracpoly fracpred fron_ex fron_hn fron_p fron_tn fron_tn2 frontier ftodate ftoe ftomdy ftowdate funnel funnelplot g|0 gamhet_glf gamhet_gp gamhet_ilf gamhet_ip gamma gamma_d2 gamma_p gamma_sw gammahet gdi_hexagon gdi_spokes ge gen gene gener genera generat generate genrank genstd genvmean gettoken gl gladder gladder_7 glim_l01 glim_l02 glim_l03 glim_l04 glim_l05 glim_l06 glim_l07 glim_l08 glim_l09 glim_l10 glim_l11 glim_l12 glim_lf glim_mu glim_nw1 glim_nw2 glim_nw3 glim_p glim_v1 glim_v2 glim_v3 glim_v4 glim_v5 glim_v6 glim_v7 glm glm_6 glm_p glm_sw glmpred glo glob globa global glogit glogit_8 glogit_p gmeans gnbre_lf gnbreg gnbreg_5 gnbreg_p gomp_lf gompe_sw gomper_p gompertz gompertzhet gomphet_glf gomphet_glf_sh gomphet_gp gomphet_ilf gomphet_ilf_sh gomphet_ip gphdot gphpen gphprint gprefs gprobi_p gprobit gprobit_8 gr gr7 gr_copy gr_current gr_db gr_describe gr_dir gr_draw gr_draw_replay gr_drop gr_edit gr_editviewopts gr_example gr_example2 gr_export gr_print gr_qscheme gr_query gr_read gr_rename gr_replay gr_save gr_set gr_setscheme gr_table gr_undo gr_use graph graph7 grebar greigen greigen_7 greigen_8 grmeanby grmeanby_7 gs_fileinfo gs_filetype gs_graphinfo gs_stat gsort gwood h|0 hadimvo hareg hausman haver he heck_d2 heckma_p heckman heckp_lf heckpr_p heckprob hel help hereg hetpr_lf hetpr_p hetprob hettest hexdump hilite hist hist_7 histogram hlogit hlu hmeans hotel hotelling hprobit hreg hsearch icd9 icd9_ff icd9p iis impute imtest inbase include inf infi infil infile infix inp inpu input ins insheet insp inspe inspec inspect integ inten intreg intreg_7 intreg_p intrg2_ll intrg_ll intrg_ll2 ipolate iqreg ir irf irf_create irfm iri is_svy is_svysum isid istdize ivprob_1_lf ivprob_lf ivprobit ivprobit_p ivreg ivreg_footnote ivtob_1_lf ivtob_lf ivtobit ivtobit_p jackknife jacknife jknife jknife_6 jknife_8 jkstat joinby kalarma1 kap kap_3 kapmeier kappa kapwgt kdensity kdensity_7 keep ksm ksmirnov ktau kwallis l|0 la lab labbe labbeplot labe label labelbook ladder levels levelsof leverage lfit lfit_p li lincom line linktest lis list lloghet_glf lloghet_glf_sh lloghet_gp lloghet_ilf lloghet_ilf_sh lloghet_ip llogi_sw llogis_p llogist llogistic llogistichet lnorm_lf lnorm_sw lnorma_p lnormal lnormalhet lnormhet_glf lnormhet_glf_sh lnormhet_gp lnormhet_ilf lnormhet_ilf_sh lnormhet_ip lnskew0 loadingplot loc loca local log logi logis_lf logistic logistic_p logit logit_estat logit_p loglogs logrank loneway lookfor lookup lowess lowess_7 lpredict lrecomp lroc lroc_7 lrtest ls lsens lsens_7 lsens_x lstat ltable ltable_7 ltriang lv lvr2plot lvr2plot_7 m|0 ma mac macr macro makecns man manova manova_estat manova_p manovatest mantel mark markin markout marksample mat mat_capp mat_order mat_put_rr mat_rapp mata mata_clear mata_describe mata_drop mata_matdescribe mata_matsave mata_matuse mata_memory mata_mlib mata_mosave mata_rename mata_which matalabel matcproc matlist matname matr matri matrix matrix_input__dlg matstrik mcc mcci md0_ md1_ md1debug_ md2_ md2debug_ mds mds_estat mds_p mdsconfig mdslong mdsmat mdsshepard mdytoe mdytof me_derd mean means median memory memsize menl meqparse mer merg merge meta mfp mfx mhelp mhodds minbound mixed_ll mixed_ll_reparm mkassert mkdir mkmat mkspline ml ml_5 ml_adjs ml_bhhhs ml_c_d ml_check ml_clear ml_cnt ml_debug ml_defd ml_e0 ml_e0_bfgs ml_e0_cycle ml_e0_dfp ml_e0i ml_e1 ml_e1_bfgs ml_e1_bhhh ml_e1_cycle ml_e1_dfp ml_e2 ml_e2_cycle ml_ebfg0 ml_ebfr0 ml_ebfr1 ml_ebh0q ml_ebhh0 ml_ebhr0 ml_ebr0i ml_ecr0i ml_edfp0 ml_edfr0 ml_edfr1 ml_edr0i ml_eds ml_eer0i ml_egr0i ml_elf ml_elf_bfgs ml_elf_bhhh ml_elf_cycle ml_elf_dfp ml_elfi ml_elfs ml_enr0i ml_enrr0 ml_erdu0 ml_erdu0_bfgs ml_erdu0_bhhh ml_erdu0_bhhhq ml_erdu0_cycle ml_erdu0_dfp ml_erdu0_nrbfgs ml_exde ml_footnote ml_geqnr ml_grad0 ml_graph ml_hbhhh ml_hd0 ml_hold ml_init ml_inv ml_log ml_max ml_mlout ml_mlout_8 ml_model ml_nb0 ml_opt ml_p ml_plot ml_query ml_rdgrd ml_repor ml_s_e ml_score ml_searc ml_technique ml_unhold mleval mlf_ mlmatbysum mlmatsum mlog mlogi mlogit mlogit_footnote mlogit_p mlopts mlsum mlvecsum mnl0_ mor more mov move mprobit mprobit_lf mprobit_p mrdu0_ mrdu1_ mvdecode mvencode mvreg mvreg_estat n|0 nbreg nbreg_al nbreg_lf nbreg_p nbreg_sw nestreg net newey newey_7 newey_p news nl nl_7 nl_9 nl_9_p nl_p nl_p_7 nlcom nlcom_p nlexp2 nlexp2_7 nlexp2a nlexp2a_7 nlexp3 nlexp3_7 nlgom3 nlgom3_7 nlgom4 nlgom4_7 nlinit nllog3 nllog3_7 nllog4 nllog4_7 nlog_rd nlogit nlogit_p nlogitgen nlogittree nlpred no nobreak noi nois noisi noisil noisily note notes notes_dlg nptrend numlabel numlist odbc old_ver olo olog ologi ologi_sw ologit ologit_p ologitp on one onew onewa oneway op_colnm op_comp op_diff op_inv op_str opr opro oprob oprob_sw oprobi oprobi_p oprobit oprobitp opts_exclusive order orthog orthpoly ou out outf outfi outfil outfile outs outsh outshe outshee outsheet ovtest pac pac_7 palette parse parse_dissim pause pca pca_8 pca_display pca_estat pca_p pca_rotate pcamat pchart pchart_7 pchi pchi_7 pcorr pctile pentium pergram pergram_7 permute permute_8 personal peto_st pkcollapse pkcross pkequiv pkexamine pkexamine_7 pkshape pksumm pksumm_7 pl plo plot plugin pnorm pnorm_7 poisgof poiss_lf poiss_sw poisso_p poisson poisson_estat post postclose postfile postutil pperron pr prais prais_e prais_e2 prais_p predict predictnl preserve print pro prob probi probit probit_estat probit_p proc_time procoverlay procrustes procrustes_estat procrustes_p profiler prog progr progra program prop proportion prtest prtesti pwcorr pwd q\\s qby qbys qchi qchi_7 qladder qladder_7 qnorm qnorm_7 qqplot qqplot_7 qreg qreg_c qreg_p qreg_sw qu quadchk quantile quantile_7 que quer query range ranksum ratio rchart rchart_7 rcof recast reclink recode reg reg3 reg3_p regdw regr regre regre_p2 regres regres_p regress regress_estat regriv_p remap ren rena renam rename renpfix repeat replace report reshape restore ret retu retur return rm rmdir robvar roccomp roccomp_7 roccomp_8 rocf_lf rocfit rocfit_8 rocgold rocplot rocplot_7 roctab roctab_7 rolling rologit rologit_p rot rota rotat rotate rotatemat rreg rreg_p ru run runtest rvfplot rvfplot_7 rvpplot rvpplot_7 sa safesum sample sampsi sav save savedresults saveold sc sca scal scala scalar scatter scm_mine sco scob_lf scob_p scobi_sw scobit scor score scoreplot scoreplot_help scree screeplot screeplot_help sdtest sdtesti se search separate seperate serrbar serrbar_7 serset set set_defaults sfrancia sh she shel shell shewhart shewhart_7 signestimationsample signrank signtest simul simul_7 simulate simulate_8 sktest sleep slogit slogit_d2 slogit_p smooth snapspan so sor sort spearman spikeplot spikeplot_7 spikeplt spline_x split sqreg sqreg_p sret sretu sretur sreturn ssc st st_ct st_hc st_hcd st_hcd_sh st_is st_issys st_note st_promo st_set st_show st_smpl st_subid stack statsby statsby_8 stbase stci stci_7 stcox stcox_estat stcox_fr stcox_fr_ll stcox_p stcox_sw stcoxkm stcoxkm_7 stcstat stcurv stcurve stcurve_7 stdes stem stepwise stereg stfill stgen stir stjoin stmc stmh stphplot stphplot_7 stphtest stphtest_7 stptime strate strate_7 streg streg_sw streset sts sts_7 stset stsplit stsum sttocc sttoct stvary stweib su suest suest_8 sum summ summa summar summari summariz summarize sunflower sureg survcurv survsum svar svar_p svmat svy svy_disp svy_dreg svy_est svy_est_7 svy_estat svy_get svy_gnbreg_p svy_head svy_header svy_heckman_p svy_heckprob_p svy_intreg_p svy_ivreg_p svy_logistic_p svy_logit_p svy_mlogit_p svy_nbreg_p svy_ologit_p svy_oprobit_p svy_poisson_p svy_probit_p svy_regress_p svy_sub svy_sub_7 svy_x svy_x_7 svy_x_p svydes svydes_8 svygen svygnbreg svyheckman svyheckprob svyintreg svyintreg_7 svyintrg svyivreg svylc svylog_p svylogit svymarkout svymarkout_8 svymean svymlog svymlogit svynbreg svyolog svyologit svyoprob svyoprobit svyopts svypois svypois_7 svypoisson svyprobit svyprobt svyprop svyprop_7 svyratio svyreg svyreg_p svyregress svyset svyset_7 svyset_8 svytab svytab_7 svytest svytotal sw sw_8 swcnreg swcox swereg swilk swlogis swlogit swologit swoprbt swpois swprobit swqreg swtobit swweib symmetry symmi symplot symplot_7 syntax sysdescribe sysdir sysuse szroeter ta tab tab1 tab2 tab_or tabd tabdi tabdis tabdisp tabi table tabodds tabodds_7 tabstat tabu tabul tabula tabulat tabulate te tempfile tempname tempvar tes test testnl testparm teststd tetrachoric time_it timer tis tob tobi tobit tobit_p tobit_sw token tokeni tokeniz tokenize tostring total translate translator transmap treat_ll treatr_p treatreg trim trimfill trnb_cons trnb_mean trpoiss_d2 trunc_ll truncr_p truncreg tsappend tset tsfill tsline tsline_ex tsreport tsrevar tsrline tsset tssmooth tsunab ttest ttesti tut_chk tut_wait tutorial tw tware_st two twoway twoway__fpfit_serset twoway__function_gen twoway__histogram_gen twoway__ipoint_serset twoway__ipoints_serset twoway__kdensity_gen twoway__lfit_serset twoway__normgen_gen twoway__pci_serset twoway__qfit_serset twoway__scatteri_serset twoway__sunflower_gen twoway_ksm_serset ty typ type typeof u|0 unab unabbrev unabcmd update us use uselabel var var_mkcompanion var_p varbasic varfcast vargranger varirf varirf_add varirf_cgraph varirf_create varirf_ctable varirf_describe varirf_dir varirf_drop varirf_erase varirf_graph varirf_ograph varirf_rename varirf_set varirf_table varlist varlmar varnorm varsoc varstable varstable_w varstable_w2 varwle vce vec vec_fevd vec_mkphi vec_p vec_p_w vecirf_create veclmar veclmar_w vecnorm vecnorm_w vecrank vecstable verinst vers versi versio version view viewsource vif vwls wdatetof webdescribe webseek webuse weib1_lf weib2_lf weib_lf weib_lf0 weibhet_glf weibhet_glf_sh weibhet_glfa weibhet_glfa_sh weibhet_gp weibhet_ilf weibhet_ilf_sh weibhet_ilfa weibhet_ilfa_sh weibhet_ip weibu_sw weibul_p weibull weibull_c weibull_s weibullhet wh whelp whi which whil while wilc_st wilcoxon win wind windo window winexec wntestb wntestb_7 wntestq xchart xchart_7 xcorr xcorr_7 xi xi_6 xmlsav xmlsave xmluse xpose xsh xshe xshel xshell xt_iis xt_tis xtab_p xtabond xtbin_p xtclog xtcloglog xtcloglog_8 xtcloglog_d2 xtcloglog_pa_p xtcloglog_re_p xtcnt_p xtcorr xtdata xtdes xtfront_p xtfrontier xtgee xtgee_elink xtgee_estat xtgee_makeivar xtgee_p xtgee_plink xtgls xtgls_p xthaus xthausman xtht_p xthtaylor xtile xtint_p xtintreg xtintreg_8 xtintreg_d2 xtintreg_p xtivp_1 xtivp_2 xtivreg xtline xtline_ex xtlogit xtlogit_8 xtlogit_d2 xtlogit_fe_p xtlogit_pa_p xtlogit_re_p xtmixed xtmixed_estat xtmixed_p xtnb_fe xtnb_lf xtnbreg xtnbreg_pa_p xtnbreg_refe_p xtpcse xtpcse_p xtpois xtpoisson xtpoisson_d2 xtpoisson_pa_p xtpoisson_refe_p xtpred xtprobit xtprobit_8 xtprobit_d2 xtprobit_re_p xtps_fe xtps_lf xtps_ren xtps_ren_8 xtrar_p xtrc xtrc_p xtrchh xtrefe_p xtreg xtreg_be xtreg_fe xtreg_ml xtreg_pa_p xtreg_re xtregar xtrere_p xtset xtsf_ll xtsf_llti xtsum xttab xttest0 xttobit xttobit_8 xttobit_p xttrans yx yxview__barlike_draw yxview_area_draw yxview_bar_draw yxview_dot_draw yxview_dropline_draw yxview_function_draw yxview_iarrow_draw yxview_ilabels_draw yxview_normal_draw yxview_pcarrow_draw yxview_pcbarrow_draw yxview_pccapsym_draw yxview_pcscatter_draw yxview_pcspike_draw yxview_rarea_draw yxview_rbar_draw yxview_rbarm_draw yxview_rcap_draw yxview_rcapsym_draw yxview_rconnected_draw yxview_rline_draw yxview_rscatter_draw yxview_rspike_draw yxview_spike_draw yxview_sunflower_draw zap_s zinb zinb_llf zinb_plf zip zip_llf zip_p zip_plf zt_ct_5 zt_hc_5 zt_hcd_5 zt_is_5 zt_iss_5 zt_sho_5 zt_smp_5 ztbase_5 ztcox_5 ztdes_5 ztereg_5 ztfill_5 ztgen_5 ztir_5 ztjoin_5 ztnb ztnb_p ztp ztp_p zts_5 ztset_5 ztspli_5 ztsum_5 zttoct_5 ztvary_5 ztweib_5",contains:[{className:"symbol",begin:/`[a-zA-Z0-9_]+'/},{className:"variable",begin:/\$\{?[a-zA-Z0-9_]+\}?/,relevance:0},{className:"string",variants:[{begin:'`"[^\r\n]*?"\''},{begin:'"[^\r\n"]*"'}]},{className:"built_in",variants:[{begin:"\\b(abs|acos|asin|atan|atan2|atanh|ceil|cloglog|comb|cos|digamma|exp|floor|invcloglog|invlogit|ln|lnfact|lnfactorial|lngamma|log|log10|max|min|mod|reldif|round|sign|sin|sqrt|sum|tan|tanh|trigamma|trunc|betaden|Binomial|binorm|binormal|chi2|chi2tail|dgammapda|dgammapdada|dgammapdadx|dgammapdx|dgammapdxdx|F|Fden|Ftail|gammaden|gammap|ibeta|invbinomial|invchi2|invchi2tail|invF|invFtail|invgammap|invibeta|invnchi2|invnFtail|invnibeta|invnorm|invnormal|invttail|nbetaden|nchi2|nFden|nFtail|nibeta|norm|normal|normalden|normd|npnchi2|tden|ttail|uniform|abbrev|char|index|indexnot|length|lower|ltrim|match|plural|proper|real|regexm|regexr|regexs|reverse|rtrim|string|strlen|strlower|strltrim|strmatch|strofreal|strpos|strproper|strreverse|strrtrim|strtrim|strupper|subinstr|subinword|substr|trim|upper|word|wordcount|_caller|autocode|byteorder|chop|clip|cond|e|epsdouble|epsfloat|group|inlist|inrange|irecode|matrix|maxbyte|maxdouble|maxfloat|maxint|maxlong|mi|minbyte|mindouble|minfloat|minint|minlong|missing|r|recode|replay|return|s|scalar|d|date|day|dow|doy|halfyear|mdy|month|quarter|week|year|d|daily|dofd|dofh|dofm|dofq|dofw|dofy|h|halfyearly|hofd|m|mofd|monthly|q|qofd|quarterly|tin|twithin|w|weekly|wofd|y|yearly|yh|ym|yofd|yq|yw|cholesky|colnumb|colsof|corr|det|diag|diag0cnt|el|get|hadamard|I|inv|invsym|issym|issymmetric|J|matmissing|matuniform|mreldif|nullmat|rownumb|rowsof|sweep|syminv|trace|vec|vecdiag)(?=\\()"}]},e.COMMENT("^[ \t]*\\*.*$",!1),e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE]}}},2570(e){e.exports=function(e){const t={className:"string",variants:[e.inherit(e.QUOTE_STRING_MODE,{begin:'((u8?|U)|L)?"'}),{begin:'(u8?|U)?R"',end:'"',contains:[e.BACKSLASH_ESCAPE]},{begin:"'\\\\?.",end:"'",illegal:"."}]},n={className:"number",variants:[{begin:"\\b(\\d+(\\.\\d*)?|\\.\\d+)(u|U|l|L|ul|UL|f|F)"},{begin:e.C_NUMBER_RE}],relevance:0},a={className:"meta",begin:"#",end:"$",keywords:{keyword:"if else elif endif define undef ifdef ifndef"},contains:[{begin:/\\\n/,relevance:0},{beginKeywords:"include",end:"$",keywords:{keyword:"include"},contains:[e.inherit(t,{className:"string"}),{className:"string",begin:"<",end:">",illegal:"\\n"}]},t,e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE]},r={className:"variable",begin:/&[a-z\d_]*\b/};return{name:"Device Tree",contains:[{className:"title.class",begin:/^\/(?=\s*\{)/,relevance:10},r,{className:"keyword",begin:"/[a-z][a-z\\d-]*/"},{className:"symbol",begin:"^\\s*[a-zA-Z_][a-zA-Z\\d_]*:"},{className:"title.class",begin:/[a-zA-Z_][a-zA-Z\d_@-]*(?=\s\{)/,relevance:.2},{relevance:0,match:[/[a-z][a-z-,]+/,/\s*/,/=/],scope:{1:"attr",3:"operator"}},{match:/[a-z][a-z-,]+(?=;)/,relevance:0,scope:"attr"},{className:"params",relevance:0,begin:"<",end:">",contains:[n,r]},e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE,n,t,a,{scope:"punctuation",relevance:0,match:/\};|[;{}]/},{begin:e.IDENT_RE+"::",keywords:""}]}}},2702(e){e.exports=function(e){return{name:"Awk",keywords:{keyword:"BEGIN END if else while do for in break continue delete next nextfile function func exit|10"},contains:[{className:"variable",variants:[{begin:/\$[\w\d#@][\w\d_]*/},{begin:/\$\{(.*?)\}/}]},{className:"string",contains:[e.BACKSLASH_ESCAPE],variants:[{begin:/(u|b)?r?'''/,end:/'''/,relevance:10},{begin:/(u|b)?r?"""/,end:/"""/,relevance:10},{begin:/(u|r|ur)'/,end:/'/,relevance:10},{begin:/(u|r|ur)"/,end:/"/,relevance:10},{begin:/(b|br)'/,end:/'/},{begin:/(b|br)"/,end:/"/},e.APOS_STRING_MODE,e.QUOTE_STRING_MODE]},e.REGEXP_MODE,e.HASH_COMMENT_MODE,e.NUMBER_MODE]}}},2738(e,t){"use strict";Object.defineProperty(t,"__esModule",{value:!0}),t.default=function(e,t,n){var a=!0,r=!1,i=!1,o=1;return function s(){if(a&&!i){if(r?o++:a=!1,e+o<=n)return e+o;i=!0}if(!r)return i||(a=!0),t<=e-o?e-o++:(r=!0,s())}}},2826(e,t){"use strict";Object.defineProperty(t,"__esModule",{value:!0}),t.generateOptions=function(e,t){if("function"==typeof e)t.callback=e;else if(e)for(var n in e)Object.prototype.hasOwnProperty.call(e,n)&&(t[n]=e[n]);return t}},2838(e){var t="[0-9](_*[0-9])*",n=`\\.(${t})`,a="[0-9a-fA-F](_*[0-9a-fA-F])*",r={className:"number",variants:[{begin:`(\\b(${t})((${n})|\\.)?|(${n}))[eE][+-]?(${t})[fFdD]?\\b`},{begin:`\\b(${t})((${n})[fFdD]?\\b|\\.([fFdD]\\b)?)`},{begin:`(${n})[fFdD]?\\b`},{begin:`\\b(${t})[fFdD]\\b`},{begin:`\\b0[xX]((${a})\\.?|(${a})?\\.(${a}))[pP][+-]?(${t})[fFdD]?\\b`},{begin:"\\b(0|[1-9](_*[0-9])*)[lL]?\\b"},{begin:`\\b0[xX](${a})[lL]?\\b`},{begin:"\\b0(_*[0-7])*[lL]?\\b"},{begin:"\\b0[bB][01](_*[01])*[lL]?\\b"}],relevance:0};e.exports=function(e){const t={keyword:"abstract as val var vararg get set class object open private protected public noinline crossinline dynamic final enum if else do while for when throw try catch finally import package is in fun override companion reified inline lateinit init interface annotation data sealed internal infix operator out by constructor super tailrec where const inner suspend typealias external expect actual",built_in:"Byte Short Char Int Long Boolean Float Double Void Unit Nothing",literal:"true false null"},n={className:"symbol",begin:e.UNDERSCORE_IDENT_RE+"@"},a={className:"subst",begin:/\$\{/,end:/\}/,contains:[e.C_NUMBER_MODE]},i={className:"variable",begin:"\\$"+e.UNDERSCORE_IDENT_RE},o={className:"string",variants:[{begin:'"""',end:'"""(?=[^"])',contains:[i,a]},{begin:"'",end:"'",illegal:/\n/,contains:[e.BACKSLASH_ESCAPE]},{begin:'"',end:'"',illegal:/\n/,contains:[e.BACKSLASH_ESCAPE,i,a]}]};a.contains.push(o);const s={className:"meta",begin:"@(?:file|property|field|get|set|receiver|param|setparam|delegate)\\s*:(?:\\s*"+e.UNDERSCORE_IDENT_RE+")?"},l={className:"meta",begin:"@"+e.UNDERSCORE_IDENT_RE,contains:[{begin:/\(/,end:/\)/,contains:[e.inherit(o,{className:"string"}),"self"]}]},c=r,_=e.COMMENT("/\\*","\\*/",{contains:[e.C_BLOCK_COMMENT_MODE]}),d={variants:[{className:"type",begin:e.UNDERSCORE_IDENT_RE},{begin:/\(/,end:/\)/,contains:[]}]},p=d;return p.variants[1].contains=[d],d.variants[1].contains=[p],{name:"Kotlin",aliases:["kt","kts"],keywords:t,contains:[e.COMMENT("/\\*\\*","\\*/",{relevance:0,contains:[{className:"doctag",begin:"@[A-Za-z]+"}]}),e.C_LINE_COMMENT_MODE,_,{className:"keyword",begin:/\b(break|continue|return|this)\b/,starts:{contains:[{className:"symbol",begin:/@\w+/}]}},n,s,l,{className:"function",beginKeywords:"fun",end:"[(]|$",returnBegin:!0,excludeEnd:!0,keywords:t,relevance:5,contains:[{begin:e.UNDERSCORE_IDENT_RE+"\\s*\\(",returnBegin:!0,relevance:0,contains:[e.UNDERSCORE_TITLE_MODE]},{className:"type",begin://,keywords:"reified",relevance:0},{className:"params",begin:/\(/,end:/\)/,endsParent:!0,keywords:t,relevance:0,contains:[{begin:/:/,end:/[=,\/]/,endsWithParent:!0,contains:[d,e.C_LINE_COMMENT_MODE,_],relevance:0},e.C_LINE_COMMENT_MODE,_,s,l,o,e.C_NUMBER_MODE]},_]},{begin:[/class|interface|trait/,/\s+/,e.UNDERSCORE_IDENT_RE],beginScope:{3:"title.class"},keywords:"class interface trait",end:/[:\{(]|$/,excludeEnd:!0,illegal:"extends implements",contains:[{beginKeywords:"public protected internal private constructor"},e.UNDERSCORE_TITLE_MODE,{className:"type",begin://,excludeBegin:!0,excludeEnd:!0,relevance:0},{className:"type",begin:/[,:]\s*/,end:/[<\(,){\s]|$/,excludeBegin:!0,returnEnd:!0},s,l]},o,{className:"meta",begin:"^#!/usr/bin/env",end:"$",illegal:"\n"},c]}}},2868(e,t){"use strict";Object.defineProperty(t,"__esModule",{value:!0}),t.convertChangesToDMP=function(e){for(var t,n,a=[],r=0;r|->)',contains:[n,a,e.C_LINE_COMMENT_MODE,r,i,e.NUMBER_MODE,o,{scope:"punctuation",match:/;/,relevance:0}]}}},3042(e,t,n){"use strict";var a,r=this&&this.__extends||(a=function(e,t){return a=Object.setPrototypeOf||{__proto__:[]}instanceof Array&&function(e,t){e.__proto__=t}||function(e,t){for(var n in t)Object.prototype.hasOwnProperty.call(t,n)&&(e[n]=t[n])},a(e,t)},function(e,t){if("function"!=typeof t&&null!==t)throw new TypeError("Class extends value "+String(t)+" is not a constructor or null");function n(){this.constructor=e}a(e,t),e.prototype=null===t?Object.create(t):(n.prototype=t.prototype,new n)});Object.defineProperty(t,"__esModule",{value:!0}),t.cssDiff=void 0,t.diffCss=function(e,n,a){return t.cssDiff.diff(e,n,a)};var i=function(e){function t(){return null!==e&&e.apply(this,arguments)||this}return r(t,e),t.prototype.tokenize=function(e){return e.split(/([{}:;,]|\s+)/)},t}(n(2188).default);t.cssDiff=new i},3051(e){e.exports=function(e){return{name:"XQuery",aliases:["xpath","xq","xqm"],case_insensitive:!1,illegal:/(proc)|(abstract)|(extends)|(until)|(#)/,keywords:{$pattern:/[a-zA-Z$][a-zA-Z0-9_:-]*/,keyword:["module","schema","namespace","boundary-space","preserve","no-preserve","strip","default","collation","base-uri","ordering","context","decimal-format","decimal-separator","copy-namespaces","empty-sequence","except","exponent-separator","external","grouping-separator","inherit","no-inherit","lax","minus-sign","per-mille","percent","schema-attribute","schema-element","strict","unordered","zero-digit","declare","import","option","function","validate","variable","for","at","in","let","where","order","group","by","return","if","then","else","tumbling","sliding","window","start","when","only","end","previous","next","stable","ascending","descending","allowing","empty","greatest","least","some","every","satisfies","switch","case","typeswitch","try","catch","and","or","to","union","intersect","instance","of","treat","as","castable","cast","map","array","delete","insert","into","replace","value","rename","copy","modify","update"],type:["item","document-node","node","attribute","document","element","comment","namespace","namespace-node","processing-instruction","text","construction","xs:anyAtomicType","xs:untypedAtomic","xs:duration","xs:time","xs:decimal","xs:float","xs:double","xs:gYearMonth","xs:gYear","xs:gMonthDay","xs:gMonth","xs:gDay","xs:boolean","xs:base64Binary","xs:hexBinary","xs:anyURI","xs:QName","xs:NOTATION","xs:dateTime","xs:dateTimeStamp","xs:date","xs:string","xs:normalizedString","xs:token","xs:language","xs:NMTOKEN","xs:Name","xs:NCName","xs:ID","xs:IDREF","xs:ENTITY","xs:integer","xs:nonPositiveInteger","xs:negativeInteger","xs:long","xs:int","xs:short","xs:byte","xs:nonNegativeInteger","xs:unisignedLong","xs:unsignedInt","xs:unsignedShort","xs:unsignedByte","xs:positiveInteger","xs:yearMonthDuration","xs:dayTimeDuration"],literal:["eq","ne","lt","le","gt","ge","is","self::","child::","descendant::","descendant-or-self::","attribute::","following::","following-sibling::","parent::","ancestor::","ancestor-or-self::","preceding::","preceding-sibling::","NaN"]},contains:[{className:"variable",begin:/[$][\w\-:]+/},{className:"built_in",variants:[{begin:/\barray:/,end:/(?:append|filter|flatten|fold-(?:left|right)|for-each(?:-pair)?|get|head|insert-before|join|put|remove|reverse|size|sort|subarray|tail)\b/},{begin:/\bmap:/,end:/(?:contains|entry|find|for-each|get|keys|merge|put|remove|size)\b/},{begin:/\bmath:/,end:/(?:a(?:cos|sin|tan[2]?)|cos|exp(?:10)?|log(?:10)?|pi|pow|sin|sqrt|tan)\b/},{begin:/\bop:/,end:/\(/,excludeEnd:!0},{begin:/\bfn:/,end:/\(/,excludeEnd:!0},{begin:/[^/,end:/(\/[\w._:-]+>)/,subLanguage:"xml",contains:[{begin:/\{/,end:/\}/,subLanguage:"xquery"},"self"]}]}}},3111(e){e.exports=function(e){const t=e.regex,n=/(?![A-Za-z0-9])(?![$])/,a=t.concat(/[a-zA-Z_\x7f-\xff][a-zA-Z0-9_\x7f-\xff]*/,n),r=t.concat(/(\\?[A-Z][a-z0-9_\x7f-\xff]+|\\?[A-Z]+(?=[A-Z][a-z0-9_\x7f-\xff])){1,}/,n),i=t.concat(/[A-Z]+/,n),o={scope:"variable",match:"\\$+"+a},s={scope:"subst",variants:[{begin:/\$\w+/},{begin:/\{\$/,end:/\}/}]},l=e.inherit(e.APOS_STRING_MODE,{illegal:null}),c="[ \t\n]",_={scope:"string",variants:[e.inherit(e.QUOTE_STRING_MODE,{illegal:null,contains:e.QUOTE_STRING_MODE.contains.concat(s)}),l,{begin:/<<<[ \t]*(?:(\w+)|"(\w+)")\n/,end:/[ \t]*(\w+)\b/,contains:e.QUOTE_STRING_MODE.contains.concat(s),"on:begin":(e,t)=>{t.data._beginMatch=e[1]||e[2]},"on:end":(e,t)=>{t.data._beginMatch!==e[1]&&t.ignoreMatch()}},e.END_SAME_AS_BEGIN({begin:/<<<[ \t]*'(\w+)'\n/,end:/[ \t]*(\w+)\b/})]},d={scope:"number",variants:[{begin:"\\b0[bB][01]+(?:_[01]+)*\\b"},{begin:"\\b0[oO][0-7]+(?:_[0-7]+)*\\b"},{begin:"\\b0[xX][\\da-fA-F]+(?:_[\\da-fA-F]+)*\\b"},{begin:"(?:\\b\\d+(?:_\\d+)*(\\.(?:\\d+(?:_\\d+)*))?|\\B\\.\\d+)(?:[eE][+-]?\\d+)?"}],relevance:0},p=["false","null","true"],u=["__CLASS__","__DIR__","__FILE__","__FUNCTION__","__COMPILER_HALT_OFFSET__","__LINE__","__METHOD__","__NAMESPACE__","__TRAIT__","die","echo","exit","include","include_once","print","require","require_once","array","abstract","and","as","binary","bool","boolean","break","callable","case","catch","class","clone","const","continue","declare","default","do","double","else","elseif","empty","enddeclare","endfor","endforeach","endif","endswitch","endwhile","enum","eval","extends","final","finally","float","for","foreach","from","global","goto","if","implements","instanceof","insteadof","int","integer","interface","isset","iterable","list","match|0","mixed","new","never","object","or","private","protected","public","readonly","real","return","string","switch","throw","trait","try","unset","use","var","void","while","xor","yield"],m=["Error|0","AppendIterator","ArgumentCountError","ArithmeticError","ArrayIterator","ArrayObject","AssertionError","BadFunctionCallException","BadMethodCallException","CachingIterator","CallbackFilterIterator","CompileError","Countable","DirectoryIterator","DivisionByZeroError","DomainException","EmptyIterator","ErrorException","Exception","FilesystemIterator","FilterIterator","GlobIterator","InfiniteIterator","InvalidArgumentException","IteratorIterator","LengthException","LimitIterator","LogicException","MultipleIterator","NoRewindIterator","OutOfBoundsException","OutOfRangeException","OuterIterator","OverflowException","ParentIterator","ParseError","RangeException","RecursiveArrayIterator","RecursiveCachingIterator","RecursiveCallbackFilterIterator","RecursiveDirectoryIterator","RecursiveFilterIterator","RecursiveIterator","RecursiveIteratorIterator","RecursiveRegexIterator","RecursiveTreeIterator","RegexIterator","RuntimeException","SeekableIterator","SplDoublyLinkedList","SplFileInfo","SplFileObject","SplFixedArray","SplHeap","SplMaxHeap","SplMinHeap","SplObjectStorage","SplObserver","SplPriorityQueue","SplQueue","SplStack","SplSubject","SplTempFileObject","TypeError","UnderflowException","UnexpectedValueException","UnhandledMatchError","ArrayAccess","BackedEnum","Closure","Fiber","Generator","Iterator","IteratorAggregate","Serializable","Stringable","Throwable","Traversable","UnitEnum","WeakReference","WeakMap","Directory","__PHP_Incomplete_Class","parent","php_user_filter","self","static","stdClass"],g={keyword:u,literal:(e=>{const t=[];return e.forEach((e=>{t.push(e),e.toLowerCase()===e?t.push(e.toUpperCase()):t.push(e.toLowerCase())})),t})(p),built_in:m},E=e=>e.map((e=>e.replace(/\|\d+$/,""))),S={variants:[{match:[/new/,t.concat(c,"+"),t.concat("(?!",E(m).join("\\b|"),"\\b)"),r],scope:{1:"keyword",4:"title.class"}}]},b=t.concat(a,"\\b(?!\\()"),f={variants:[{match:[t.concat(/::/,t.lookahead(/(?!class\b)/)),b],scope:{2:"variable.constant"}},{match:[/::/,/class/],scope:{2:"variable.language"}},{match:[r,t.concat(/::/,t.lookahead(/(?!class\b)/)),b],scope:{1:"title.class",3:"variable.constant"}},{match:[r,t.concat("::",t.lookahead(/(?!class\b)/))],scope:{1:"title.class"}},{match:[r,/::/,/class/],scope:{1:"title.class",3:"variable.language"}}]},T={scope:"attr",match:t.concat(a,t.lookahead(":"),t.lookahead(/(?!::)/))},h={relevance:0,begin:/\(/,end:/\)/,keywords:g,contains:[T,o,f,e.C_BLOCK_COMMENT_MODE,_,d,S]},C={relevance:0,match:[/\b/,t.concat("(?!fn\\b|function\\b|",E(u).join("\\b|"),"|",E(m).join("\\b|"),"\\b)"),a,t.concat(c,"*"),t.lookahead(/(?=\()/)],scope:{3:"title.function.invoke"},contains:[h]};h.contains.push(C);const O=[T,f,e.C_BLOCK_COMMENT_MODE,_,d,S],N={begin:t.concat(/#\[\s*\\?/,t.either(r,i)),beginScope:"meta",end:/]/,endScope:"meta",keywords:{literal:p,keyword:["new","array"]},contains:[{begin:/\[/,end:/]/,keywords:{literal:p,keyword:["new","array"]},contains:["self",...O]},...O,{scope:"meta",variants:[{match:r},{match:i}]}]};return{case_insensitive:!1,keywords:g,contains:[N,e.HASH_COMMENT_MODE,e.COMMENT("//","$"),e.COMMENT("/\\*","\\*/",{contains:[{scope:"doctag",match:"@[A-Za-z]+"}]}),{match:/__halt_compiler\(\);/,keywords:"__halt_compiler",starts:{scope:"comment",end:e.MATCH_NOTHING_RE,contains:[{match:/\?>/,scope:"meta",endsParent:!0}]}},{scope:"meta",variants:[{begin:/<\?php/,relevance:10},{begin:/<\?=/},{begin:/<\?/,relevance:.1},{begin:/\?>/}]},{scope:"variable.language",match:/\$this\b/},o,C,f,{match:[/const/,/\s/,a],scope:{1:"keyword",3:"variable.constant"}},S,{scope:"function",relevance:0,beginKeywords:"fn function",end:/[;{]/,excludeEnd:!0,illegal:"[$%\\[]",contains:[{beginKeywords:"use"},e.UNDERSCORE_TITLE_MODE,{begin:"=>",endsParent:!0},{scope:"params",begin:"\\(",end:"\\)",excludeBegin:!0,excludeEnd:!0,keywords:g,contains:["self",N,o,f,e.C_BLOCK_COMMENT_MODE,_,d]}]},{scope:"class",variants:[{beginKeywords:"enum",illegal:/[($"]/},{beginKeywords:"class interface trait",illegal:/[:($"]/}],relevance:0,end:/\{/,excludeEnd:!0,contains:[{beginKeywords:"extends implements"},e.UNDERSCORE_TITLE_MODE]},{beginKeywords:"namespace",relevance:0,end:";",illegal:/[.']/,contains:[e.inherit(e.UNDERSCORE_TITLE_MODE,{scope:"title.class"})]},{beginKeywords:"use",relevance:0,end:";",contains:[{match:/\b(as|const|function)\b/,scope:"keyword"},e.UNDERSCORE_TITLE_MODE]},_,d]}}},3187(e){e.exports=function(e){const t=e.regex,n={$pattern:/[\w.\/]+/,built_in:["action","bindattr","collection","component","concat","debugger","each","each-in","get","hash","if","in","input","link-to","loc","log","lookup","mut","outlet","partial","query-params","render","template","textarea","unbound","unless","view","with","yield"]},a=/\[\]|\[[^\]]+\]/,r=/[^\s!"#%&'()*+,.\/;<=>@\[\\\]^`{|}~]+/,i=t.either(/""|"[^"]+"/,/''|'[^']+'/,a,r),o=t.concat(t.optional(/\.|\.\/|\//),i,t.anyNumberOfTimes(t.concat(/(\.|\/)/,i))),s=t.concat("(",a,"|",r,")(?==)"),l={begin:o},c=e.inherit(l,{keywords:{$pattern:/[\w.\/]+/,literal:["true","false","undefined","null"]}}),_={begin:/\(/,end:/\)/},d={className:"attr",begin:s,relevance:0,starts:{begin:/=/,end:/=/,starts:{contains:[e.NUMBER_MODE,e.QUOTE_STRING_MODE,e.APOS_STRING_MODE,c,_]}}},p={contains:[e.NUMBER_MODE,e.QUOTE_STRING_MODE,e.APOS_STRING_MODE,{begin:/as\s+\|/,keywords:{keyword:"as"},end:/\|/,contains:[{begin:/\w+/}]},d,c,_],returnEnd:!0},u=e.inherit(l,{className:"name",keywords:n,starts:e.inherit(p,{end:/\)/})});_.contains=[u];const m=e.inherit(l,{keywords:n,className:"name",starts:e.inherit(p,{end:/\}\}/})}),g=e.inherit(l,{keywords:n,className:"name"}),E=e.inherit(l,{className:"name",keywords:n,starts:e.inherit(p,{end:/\}\}/})});return{name:"Handlebars",aliases:["hbs","html.hbs","html.handlebars","htmlbars"],case_insensitive:!0,subLanguage:"xml",contains:[{begin:/\\\{\{/,skip:!0},{begin:/\\\\(?=\{\{)/,skip:!0},e.COMMENT(/\{\{!--/,/--\}\}/),e.COMMENT(/\{\{!/,/\}\}/),{className:"template-tag",begin:/\{\{\{\{(?!\/)/,end:/\}\}\}\}/,contains:[m],starts:{end:/\{\{\{\{\//,returnEnd:!0,subLanguage:"xml"}},{className:"template-tag",begin:/\{\{\{\{\//,end:/\}\}\}\}/,contains:[g]},{className:"template-tag",begin:/\{\{#/,end:/\}\}/,contains:[m]},{className:"template-tag",begin:/\{\{(?=else\}\})/,end:/\}\}/,keywords:"else"},{className:"template-tag",begin:/\{\{(?=else if)/,end:/\}\}/,keywords:"else if"},{className:"template-tag",begin:/\{\{\//,end:/\}\}/,contains:[g]},{className:"template-variable",begin:/\{\{\{/,end:/\}\}\}/,contains:[E]},{className:"template-variable",begin:/\{\{/,end:/\}\}/,contains:[E]}]}}},3223(e,t,n){var a=n(8416);a.registerLanguage("1c",n(5975)),a.registerLanguage("abnf",n(5360)),a.registerLanguage("accesslog",n(5031)),a.registerLanguage("actionscript",n(9090)),a.registerLanguage("ada",n(3307)),a.registerLanguage("angelscript",n(969)),a.registerLanguage("apache",n(7869)),a.registerLanguage("applescript",n(624)),a.registerLanguage("arcade",n(1799)),a.registerLanguage("arduino",n(4943)),a.registerLanguage("armasm",n(7826)),a.registerLanguage("xml",n(114)),a.registerLanguage("asciidoc",n(4196)),a.registerLanguage("aspectj",n(9993)),a.registerLanguage("autohotkey",n(4438)),a.registerLanguage("autoit",n(2021)),a.registerLanguage("avrasm",n(629)),a.registerLanguage("awk",n(2702)),a.registerLanguage("axapta",n(3940)),a.registerLanguage("bash",n(8641)),a.registerLanguage("basic",n(6545)),a.registerLanguage("bnf",n(6797)),a.registerLanguage("brainfuck",n(1980)),a.registerLanguage("c",n(722)),a.registerLanguage("cal",n(6571)),a.registerLanguage("capnproto",n(9639)),a.registerLanguage("ceylon",n(507)),a.registerLanguage("clean",n(1050)),a.registerLanguage("clojure",n(7579)),a.registerLanguage("clojure-repl",n(9601)),a.registerLanguage("cmake",n(2008)),a.registerLanguage("coffeescript",n(8364)),a.registerLanguage("coq",n(9722)),a.registerLanguage("cos",n(9296)),a.registerLanguage("cpp",n(6570)),a.registerLanguage("crmsh",n(1756)),a.registerLanguage("crystal",n(8661)),a.registerLanguage("csharp",n(7120)),a.registerLanguage("csp",n(6813)),a.registerLanguage("css",n(8612)),a.registerLanguage("d",n(6683)),a.registerLanguage("markdown",n(602)),a.registerLanguage("dart",n(2274)),a.registerLanguage("delphi",n(8443)),a.registerLanguage("diff",n(8596)),a.registerLanguage("django",n(3496)),a.registerLanguage("dns",n(1500)),a.registerLanguage("dockerfile",n(9197)),a.registerLanguage("dos",n(8857)),a.registerLanguage("dsconfig",n(8284)),a.registerLanguage("dts",n(2570)),a.registerLanguage("dust",n(9723)),a.registerLanguage("ebnf",n(8340)),a.registerLanguage("elixir",n(9632)),a.registerLanguage("elm",n(8155)),a.registerLanguage("ruby",n(5015)),a.registerLanguage("erb",n(4356)),a.registerLanguage("erlang-repl",n(7572)),a.registerLanguage("erlang",n(8188)),a.registerLanguage("excel",n(7288)),a.registerLanguage("fix",n(8008)),a.registerLanguage("flix",n(9744)),a.registerLanguage("fortran",n(1189)),a.registerLanguage("fsharp",n(9301)),a.registerLanguage("gams",n(7495)),a.registerLanguage("gauss",n(3770)),a.registerLanguage("gcode",n(8491)),a.registerLanguage("gherkin",n(3297)),a.registerLanguage("glsl",n(3779)),a.registerLanguage("gml",n(1195)),a.registerLanguage("go",n(9777)),a.registerLanguage("golo",n(7240)),a.registerLanguage("gradle",n(1432)),a.registerLanguage("graphql",n(7474)),a.registerLanguage("groovy",n(253)),a.registerLanguage("haml",n(4517)),a.registerLanguage("handlebars",n(3187)),a.registerLanguage("haskell",n(379)),a.registerLanguage("haxe",n(4139)),a.registerLanguage("hsp",n(4008)),a.registerLanguage("http",n(3383)),a.registerLanguage("hy",n(4688)),a.registerLanguage("inform7",n(6969)),a.registerLanguage("ini",n(1533)),a.registerLanguage("irpf90",n(4499)),a.registerLanguage("isbl",n(5665)),a.registerLanguage("java",n(4895)),a.registerLanguage("javascript",n(6035)),a.registerLanguage("jboss-cli",n(5623)),a.registerLanguage("json",n(621)),a.registerLanguage("julia",n(8874)),a.registerLanguage("julia-repl",n(8514)),a.registerLanguage("kotlin",n(2838)),a.registerLanguage("lasso",n(6089)),a.registerLanguage("latex",n(3483)),a.registerLanguage("ldif",n(7172)),a.registerLanguage("leaf",n(6703)),a.registerLanguage("less",n(8330)),a.registerLanguage("lisp",n(2491)),a.registerLanguage("livecodeserver",n(6859)),a.registerLanguage("livescript",n(5914)),a.registerLanguage("llvm",n(6398)),a.registerLanguage("lsl",n(2480)),a.registerLanguage("lua",n(3873)),a.registerLanguage("makefile",n(7667)),a.registerLanguage("mathematica",n(6455)),a.registerLanguage("matlab",n(8722)),a.registerLanguage("maxima",n(8216)),a.registerLanguage("mel",n(9953)),a.registerLanguage("mercury",n(5582)),a.registerLanguage("mipsasm",n(8269)),a.registerLanguage("mizar",n(944)),a.registerLanguage("perl",n(946)),a.registerLanguage("mojolicious",n(6078)),a.registerLanguage("monkey",n(7850)),a.registerLanguage("moonscript",n(9241)),a.registerLanguage("n1ql",n(9827)),a.registerLanguage("nestedtext",n(4683)),a.registerLanguage("nginx",n(7259)),a.registerLanguage("nim",n(1747)),a.registerLanguage("nix",n(320)),a.registerLanguage("node-repl",n(6673)),a.registerLanguage("nsis",n(140)),a.registerLanguage("objectivec",n(943)),a.registerLanguage("ocaml",n(1791)),a.registerLanguage("openscad",n(6672)),a.registerLanguage("oxygene",n(3028)),a.registerLanguage("parser3",n(9163)),a.registerLanguage("pf",n(8837)),a.registerLanguage("pgsql",n(6876)),a.registerLanguage("php",n(3111)),a.registerLanguage("php-template",n(1726)),a.registerLanguage("plaintext",n(9040)),a.registerLanguage("pony",n(6903)),a.registerLanguage("powershell",n(3982)),a.registerLanguage("processing",n(9200)),a.registerLanguage("profile",n(5206)),a.registerLanguage("prolog",n(6168)),a.registerLanguage("properties",n(7646)),a.registerLanguage("protobuf",n(7300)),a.registerLanguage("puppet",n(8545)),a.registerLanguage("purebasic",n(7995)),a.registerLanguage("python",n(1117)),a.registerLanguage("python-repl",n(283)),a.registerLanguage("q",n(9384)),a.registerLanguage("qml",n(9797)),a.registerLanguage("r",n(8129)),a.registerLanguage("reasonml",n(6446)),a.registerLanguage("rib",n(6238)),a.registerLanguage("roboconf",n(2925)),a.registerLanguage("routeros",n(5980)),a.registerLanguage("rsl",n(6650)),a.registerLanguage("ruleslanguage",n(5690)),a.registerLanguage("rust",n(5409)),a.registerLanguage("sas",n(7754)),a.registerLanguage("scala",n(3449)),a.registerLanguage("scheme",n(6244)),a.registerLanguage("scilab",n(395)),a.registerLanguage("scss",n(1611)),a.registerLanguage("shell",n(8813)),a.registerLanguage("smali",n(4119)),a.registerLanguage("smalltalk",n(78)),a.registerLanguage("sml",n(6959)),a.registerLanguage("sqf",n(6325)),a.registerLanguage("sql",n(315)),a.registerLanguage("stan",n(7891)),a.registerLanguage("stata",n(2546)),a.registerLanguage("step21",n(8128)),a.registerLanguage("stylus",n(5733)),a.registerLanguage("subunit",n(6475)),a.registerLanguage("swift",n(1496)),a.registerLanguage("taggerscript",n(150)),a.registerLanguage("yaml",n(5588)),a.registerLanguage("tap",n(8942)),a.registerLanguage("tcl",n(1272)),a.registerLanguage("thrift",n(9888)),a.registerLanguage("tp",n(987)),a.registerLanguage("twig",n(222)),a.registerLanguage("typescript",n(8640)),a.registerLanguage("vala",n(9461)),a.registerLanguage("vbnet",n(8928)),a.registerLanguage("vbscript",n(1200)),a.registerLanguage("vbscript-html",n(164)),a.registerLanguage("verilog",n(8249)),a.registerLanguage("vhdl",n(8721)),a.registerLanguage("vim",n(8907)),a.registerLanguage("wasm",n(9351)),a.registerLanguage("wren",n(5069)),a.registerLanguage("x86asm",n(7514)),a.registerLanguage("xl",n(8675)),a.registerLanguage("xquery",n(3051)),a.registerLanguage("zephir",n(8745)),a.HighlightJS=a,a.default=a,e.exports=a},3297(e){e.exports=function(e){return{name:"Gherkin",aliases:["feature"],keywords:"Feature Background Ability Business Need Scenario Scenarios Scenario Outline Scenario Template Examples Given And Then But When",contains:[{className:"symbol",begin:"\\*",relevance:0},{className:"meta",begin:"@[^@\\s]+"},{begin:"\\|",end:"\\|\\w*$",contains:[{className:"string",begin:"[^|]+"}]},{className:"variable",begin:"<",end:">"},e.HASH_COMMENT_MODE,{className:"string",begin:'"""',end:'"""'},e.QUOTE_STRING_MODE]}}},3307(e){e.exports=function(e){const t="\\d(_|\\d)*",n="[eE][-+]?"+t,a="\\b("+t+"#\\w+(\\.\\w+)?#("+n+")?|"+t+"(\\."+t+")?("+n+")?)",r="[A-Za-z](_?[A-Za-z0-9.])*",i="[]\\{\\}%#'\"",o=e.COMMENT("--","$"),s={begin:"\\s+:\\s+",end:"\\s*(:=|;|\\)|=>|$)",illegal:i,contains:[{beginKeywords:"loop for declare others",endsParent:!0},{className:"keyword",beginKeywords:"not null constant access function procedure in out aliased exception"},{className:"type",begin:r,endsParent:!0,relevance:0}]};return{name:"Ada",case_insensitive:!0,keywords:{keyword:["abort","else","new","return","abs","elsif","not","reverse","abstract","end","accept","entry","select","access","exception","of","separate","aliased","exit","or","some","all","others","subtype","and","for","out","synchronized","array","function","overriding","at","tagged","generic","package","task","begin","goto","pragma","terminate","body","private","then","if","procedure","type","case","in","protected","constant","interface","is","raise","use","declare","range","delay","limited","record","when","delta","loop","rem","while","digits","renames","with","do","mod","requeue","xor"],literal:["True","False"]},contains:[o,{className:"string",begin:/"/,end:/"/,contains:[{begin:/""/,relevance:0}]},{className:"string",begin:/'.'/},{className:"number",begin:a,relevance:0},{className:"symbol",begin:"'"+r},{className:"title",begin:"(\\bwith\\s+)?(\\bprivate\\s+)?\\bpackage\\s+(\\bbody\\s+)?",end:"(is|$)",keywords:"package body",excludeBegin:!0,excludeEnd:!0,illegal:i},{begin:"(\\b(with|overriding)\\s+)?\\b(function|procedure)\\s+",end:"(\\bis|\\bwith|\\brenames|\\)\\s*;)",keywords:"overriding function procedure with is renames return",returnBegin:!0,contains:[o,{className:"title",begin:"(\\bwith\\s+)?\\b(function|procedure)\\s+",end:"(\\(|\\s+|$)",excludeBegin:!0,excludeEnd:!0,illegal:i},s,{className:"type",begin:"\\breturn\\s+",end:"(\\s+|;|$)",keywords:"return",excludeBegin:!0,excludeEnd:!0,endsParent:!0,illegal:i}]},{className:"type",begin:"\\b(sub)?type\\s+",end:"\\s+",keywords:"type",excludeBegin:!0,illegal:i},s]}}},3383(e){e.exports=function(e){const t="HTTP/([32]|1\\.[01])",n={className:"attribute",begin:e.regex.concat("^",/[A-Za-z][A-Za-z0-9-]*/,"(?=\\:\\s)"),starts:{contains:[{className:"punctuation",begin:/: /,relevance:0,starts:{end:"$",relevance:0}}]}},a=[n,{begin:"\\n\\n",starts:{subLanguage:[],endsWithParent:!0}}];return{name:"HTTP",aliases:["https"],illegal:/\S/,contains:[{begin:"^(?="+t+" \\d{3})",end:/$/,contains:[{className:"meta",begin:t},{className:"number",begin:"\\b\\d{3}\\b"}],starts:{end:/\b\B/,illegal:/\S/,contains:a}},{begin:"(?=^[A-Z]+ (.*?) "+t+"$)",end:/$/,contains:[{className:"string",begin:" ",end:" ",excludeBegin:!0,excludeEnd:!0},{className:"meta",begin:t},{className:"keyword",begin:"[A-Z]+"}],starts:{end:/\b\B/,illegal:/\S/,contains:a}},e.inherit(n,{relevance:0})]}}},3406(e,t){"use strict";var n=this&&this.__assign||function(){return n=Object.assign||function(e){for(var t,n=1,a=arguments.length;n",/\s+/,/using/,/\s+/,/\S+/],beginScope:{1:"comment",3:"keyword",5:"type"},end:/$/,contains:[{className:"string",begin:/\S+/}]},e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE,a,r,s,o,e.C_NUMBER_MODE,{begin:[/^\s*/,"extension",/\s+(?=[[(])/],beginScope:{2:"keyword"}},{begin:[/^\s*/,/end/,/\s+/,/(extension\b)?/],beginScope:{2:"keyword",4:"keyword"}},{match:/\.inline\b/},{begin:/\binline(?=\s)/,keywords:"inline"},{begin:[/\(\s*/,/using/,/\s+(?!\))/],beginScope:{2:"keyword"}},{className:"meta",begin:"@[A-Za-z]+"}]}}},3483(e){e.exports=function(e){const t=[{begin:/\^{6}[0-9a-f]{6}/},{begin:/\^{5}[0-9a-f]{5}/},{begin:/\^{4}[0-9a-f]{4}/},{begin:/\^{3}[0-9a-f]{3}/},{begin:/\^{2}[0-9a-f]{2}/},{begin:/\^{2}[\u0000-\u007f]/}],n=[{className:"keyword",begin:/\\/,relevance:0,contains:[{endsParent:!0,begin:e.regex.either(...["(?:NeedsTeXFormat|RequirePackage|GetIdInfo)","Provides(?:Expl)?(?:Package|Class|File)","(?:DeclareOption|ProcessOptions)","(?:documentclass|usepackage|input|include)","makeat(?:letter|other)","ExplSyntax(?:On|Off)","(?:new|renew|provide)?command","(?:re)newenvironment","(?:New|Renew|Provide|Declare)(?:Expandable)?DocumentCommand","(?:New|Renew|Provide|Declare)DocumentEnvironment","(?:(?:e|g|x)?def|let)","(?:begin|end)","(?:part|chapter|(?:sub){0,2}section|(?:sub)?paragraph)","caption","(?:label|(?:eq|page|name)?ref|(?:paren|foot|super)?cite)","(?:alpha|beta|[Gg]amma|[Dd]elta|(?:var)?epsilon|zeta|eta|[Tt]heta|vartheta)","(?:iota|(?:var)?kappa|[Ll]ambda|mu|nu|[Xx]i|[Pp]i|varpi|(?:var)rho)","(?:[Ss]igma|varsigma|tau|[Uu]psilon|[Pp]hi|varphi|chi|[Pp]si|[Oo]mega)","(?:frac|sum|prod|lim|infty|times|sqrt|leq|geq|left|right|middle|[bB]igg?)","(?:[lr]angle|q?quad|[lcvdi]?dots|d?dot|hat|tilde|bar)"].map((e=>e+"(?![a-zA-Z@:_])")))},{endsParent:!0,begin:new RegExp(["(?:__)?[a-zA-Z]{2,}_[a-zA-Z](?:_?[a-zA-Z])+:[a-zA-Z]*","[lgc]__?[a-zA-Z](?:_?[a-zA-Z])*_[a-zA-Z]{2,}","[qs]__?[a-zA-Z](?:_?[a-zA-Z])+","use(?:_i)?:[a-zA-Z]*","(?:else|fi|or):","(?:if|cs|exp):w","(?:hbox|vbox):n","::[a-zA-Z]_unbraced","::[a-zA-Z:]"].map((e=>e+"(?![a-zA-Z:_])")).join("|"))},{endsParent:!0,variants:t},{endsParent:!0,relevance:0,variants:[{begin:/[a-zA-Z@]+/},{begin:/[^a-zA-Z@]?/}]}]},{className:"params",relevance:0,begin:/#+\d?/},{variants:t},{className:"built_in",relevance:0,begin:/[$&^_]/},{className:"meta",begin:/% ?!(T[eE]X|tex|BIB|bib)/,end:"$",relevance:10},e.COMMENT("%","$",{relevance:0})],a={begin:/\{/,end:/\}/,relevance:0,contains:["self",...n]},r=e.inherit(a,{relevance:0,endsParent:!0,contains:[a,...n]}),i={begin:/\[/,end:/\]/,endsParent:!0,relevance:0,contains:[a,...n]},o={begin:/\s+/,relevance:0},s=[r],l=[i],c=function(e,t){return{contains:[o],starts:{relevance:0,contains:e,starts:t}}},_=function(e,t){return{begin:"\\\\"+e+"(?![a-zA-Z@:_])",keywords:{$pattern:/\\[a-zA-Z]+/,keyword:"\\"+e},relevance:0,contains:[o],starts:t}},d=function(t,n){return e.inherit({begin:"\\\\begin(?=[ \t]*(\\r?\\n[ \t]*)?\\{"+t+"\\})",keywords:{$pattern:/\\[a-zA-Z]+/,keyword:"\\begin"},relevance:0},c(s,n))},p=(t="string")=>e.END_SAME_AS_BEGIN({className:t,begin:/(.|\r?\n)/,end:/(.|\r?\n)/,excludeBegin:!0,excludeEnd:!0,endsParent:!0}),u=function(e){return{className:"string",end:"(?=\\\\end\\{"+e+"\\})"}},m=(e="string")=>({relevance:0,begin:/\{/,starts:{endsParent:!0,contains:[{className:e,end:/(?=\})/,endsParent:!0,contains:[{begin:/\{/,end:/\}/,relevance:0,contains:["self"]}]}]}});return{name:"LaTeX",aliases:["tex"],contains:[...["verb","lstinline"].map((e=>_(e,{contains:[p()]}))),_("mint",c(s,{contains:[p()]})),_("mintinline",c(s,{contains:[m(),p()]})),_("url",{contains:[m("link"),m("link")]}),_("hyperref",{contains:[m("link")]}),_("href",c(l,{contains:[m("link")]})),...[].concat(...["","\\*"].map((e=>[d("verbatim"+e,u("verbatim"+e)),d("filecontents"+e,c(s,u("filecontents"+e))),...["","B","L"].map((t=>d(t+"Verbatim"+e,c(l,u(t+"Verbatim"+e)))))]))),d("minted",c(l,c(s,u("minted")))),...n]}}},3496(e){e.exports=function(e){const t={begin:/\|[A-Za-z]+:?/,keywords:{name:"truncatewords removetags linebreaksbr yesno get_digit timesince random striptags filesizeformat escape linebreaks length_is ljust rjust cut urlize fix_ampersands title floatformat capfirst pprint divisibleby add make_list unordered_list urlencode timeuntil urlizetrunc wordcount stringformat linenumbers slice date dictsort dictsortreversed default_if_none pluralize lower join center default truncatewords_html upper length phone2numeric wordwrap time addslashes slugify first escapejs force_escape iriencode last safe safeseq truncatechars localize unlocalize localtime utc timezone"},contains:[e.QUOTE_STRING_MODE,e.APOS_STRING_MODE]};return{name:"Django",aliases:["jinja"],case_insensitive:!0,subLanguage:"xml",contains:[e.COMMENT(/\{%\s*comment\s*%\}/,/\{%\s*endcomment\s*%\}/),e.COMMENT(/\{#/,/#\}/),{className:"template-tag",begin:/\{%/,end:/%\}/,contains:[{className:"name",begin:/\w+/,keywords:{name:"comment endcomment load templatetag ifchanged endifchanged if endif firstof for endfor ifnotequal endifnotequal widthratio extends include spaceless endspaceless regroup ifequal endifequal ssi now with cycle url filter endfilter debug block endblock else autoescape endautoescape csrf_token empty elif endwith static trans blocktrans endblocktrans get_static_prefix get_media_prefix plural get_current_language language get_available_languages get_current_language_bidi get_language_info get_language_info_list localize endlocalize localtime endlocaltime timezone endtimezone get_current_timezone verbatim"},starts:{endsWithParent:!0,keywords:"in by as",contains:[t],relevance:0}}]},{className:"template-variable",begin:/\{\{/,end:/\}\}/,contains:[t]}]}}},3622(e,t,n){"use strict";var a,r=this&&this.__extends||(a=function(e,t){return a=Object.setPrototypeOf||{__proto__:[]}instanceof Array&&function(e,t){e.__proto__=t}||function(e,t){for(var n in t)Object.prototype.hasOwnProperty.call(t,n)&&(e[n]=t[n])},a(e,t)},function(e,t){if("function"!=typeof t&&null!==t)throw new TypeError("Class extends value "+String(t)+" is not a constructor or null");function n(){this.constructor=e}a(e,t),e.prototype=null===t?Object.create(t):(n.prototype=t.prototype,new n)});Object.defineProperty(t,"__esModule",{value:!0}),t.characterDiff=void 0,t.diffChars=function(e,n,a){return t.characterDiff.diff(e,n,a)};var i=function(e){function t(){return null!==e&&e.apply(this,arguments)||this}return r(t,e),t}(n(2188).default);t.characterDiff=new i},3770(e){e.exports=function(e){const t={keyword:"bool break call callexe checkinterrupt clear clearg closeall cls comlog compile continue create debug declare delete disable dlibrary dllcall do dos ed edit else elseif enable end endfor endif endp endo errorlog errorlogat expr external fn for format goto gosub graph if keyword let lib library line load loadarray loadexe loadf loadk loadm loadp loads loadx local locate loopnextindex lprint lpwidth lshow matrix msym ndpclex new open output outwidth plot plotsym pop prcsn print printdos proc push retp return rndcon rndmod rndmult rndseed run save saveall screen scroll setarray show sparse stop string struct system trace trap threadfor threadendfor threadbegin threadjoin threadstat threadend until use while winprint ne ge le gt lt and xor or not eq eqv",built_in:"abs acf aconcat aeye amax amean AmericanBinomCall AmericanBinomCall_Greeks AmericanBinomCall_ImpVol AmericanBinomPut AmericanBinomPut_Greeks AmericanBinomPut_ImpVol AmericanBSCall AmericanBSCall_Greeks AmericanBSCall_ImpVol AmericanBSPut AmericanBSPut_Greeks AmericanBSPut_ImpVol amin amult annotationGetDefaults annotationSetBkd annotationSetFont annotationSetLineColor annotationSetLineStyle annotationSetLineThickness annualTradingDays arccos arcsin areshape arrayalloc arrayindex arrayinit arraytomat asciiload asclabel astd astds asum atan atan2 atranspose axmargin balance band bandchol bandcholsol bandltsol bandrv bandsolpd bar base10 begwind besselj bessely beta box boxcox cdfBeta cdfBetaInv cdfBinomial cdfBinomialInv cdfBvn cdfBvn2 cdfBvn2e cdfCauchy cdfCauchyInv cdfChic cdfChii cdfChinc cdfChincInv cdfExp cdfExpInv cdfFc cdfFnc cdfFncInv cdfGam cdfGenPareto cdfHyperGeo cdfLaplace cdfLaplaceInv cdfLogistic cdfLogisticInv cdfmControlCreate cdfMvn cdfMvn2e cdfMvnce cdfMvne cdfMvt2e cdfMvtce cdfMvte cdfN cdfN2 cdfNc cdfNegBinomial cdfNegBinomialInv cdfNi cdfPoisson cdfPoissonInv cdfRayleigh cdfRayleighInv cdfTc cdfTci cdfTnc cdfTvn cdfWeibull cdfWeibullInv cdir ceil ChangeDir chdir chiBarSquare chol choldn cholsol cholup chrs close code cols colsf combinate combinated complex con cond conj cons ConScore contour conv convertsatostr convertstrtosa corrm corrms corrvc corrx corrxs cos cosh counts countwts crossprd crout croutp csrcol csrlin csvReadM csvReadSA cumprodc cumsumc curve cvtos datacreate datacreatecomplex datalist dataload dataloop dataopen datasave date datestr datestring datestrymd dayinyr dayofweek dbAddDatabase dbClose dbCommit dbCreateQuery dbExecQuery dbGetConnectOptions dbGetDatabaseName dbGetDriverName dbGetDrivers dbGetHostName dbGetLastErrorNum dbGetLastErrorText dbGetNumericalPrecPolicy dbGetPassword dbGetPort dbGetTableHeaders dbGetTables dbGetUserName dbHasFeature dbIsDriverAvailable dbIsOpen dbIsOpenError dbOpen dbQueryBindValue dbQueryClear dbQueryCols dbQueryExecPrepared dbQueryFetchAllM dbQueryFetchAllSA dbQueryFetchOneM dbQueryFetchOneSA dbQueryFinish dbQueryGetBoundValue dbQueryGetBoundValues dbQueryGetField dbQueryGetLastErrorNum dbQueryGetLastErrorText dbQueryGetLastInsertID dbQueryGetLastQuery dbQueryGetPosition dbQueryIsActive dbQueryIsForwardOnly dbQueryIsNull dbQueryIsSelect dbQueryIsValid dbQueryPrepare dbQueryRows dbQuerySeek dbQuerySeekFirst dbQuerySeekLast dbQuerySeekNext dbQuerySeekPrevious dbQuerySetForwardOnly dbRemoveDatabase dbRollback dbSetConnectOptions dbSetDatabaseName dbSetHostName dbSetNumericalPrecPolicy dbSetPort dbSetUserName dbTransaction DeleteFile delif delrows denseToSp denseToSpRE denToZero design det detl dfft dffti diag diagrv digamma doswin DOSWinCloseall DOSWinOpen dotfeq dotfeqmt dotfge dotfgemt dotfgt dotfgtmt dotfle dotflemt dotflt dotfltmt dotfne dotfnemt draw drop dsCreate dstat dstatmt dstatmtControlCreate dtdate dtday dttime dttodtv dttostr dttoutc dtvnormal dtvtodt dtvtoutc dummy dummybr dummydn eig eigh eighv eigv elapsedTradingDays endwind envget eof eqSolve eqSolvemt eqSolvemtControlCreate eqSolvemtOutCreate eqSolveset erf erfc erfccplx erfcplx error etdays ethsec etstr EuropeanBinomCall EuropeanBinomCall_Greeks EuropeanBinomCall_ImpVol EuropeanBinomPut EuropeanBinomPut_Greeks EuropeanBinomPut_ImpVol EuropeanBSCall EuropeanBSCall_Greeks EuropeanBSCall_ImpVol EuropeanBSPut EuropeanBSPut_Greeks EuropeanBSPut_ImpVol exctsmpl exec execbg exp extern eye fcheckerr fclearerr feq feqmt fflush fft ffti fftm fftmi fftn fge fgemt fgets fgetsa fgetsat fgetst fgt fgtmt fileinfo filesa fle flemt floor flt fltmt fmod fne fnemt fonts fopen formatcv formatnv fputs fputst fseek fstrerror ftell ftocv ftos ftostrC gamma gammacplx gammaii gausset gdaAppend gdaCreate gdaDStat gdaDStatMat gdaGetIndex gdaGetName gdaGetNames gdaGetOrders gdaGetType gdaGetTypes gdaGetVarInfo gdaIsCplx gdaLoad gdaPack gdaRead gdaReadByIndex gdaReadSome gdaReadSparse gdaReadStruct gdaReportVarInfo gdaSave gdaUpdate gdaUpdateAndPack gdaVars gdaWrite gdaWrite32 gdaWriteSome getarray getdims getf getGAUSShome getmatrix getmatrix4D getname getnamef getNextTradingDay getNextWeekDay getnr getorders getpath getPreviousTradingDay getPreviousWeekDay getRow getscalar3D getscalar4D getTrRow getwind glm gradcplx gradMT gradMTm gradMTT gradMTTm gradp graphprt graphset hasimag header headermt hess hessMT hessMTg hessMTgw hessMTm hessMTmw hessMTT hessMTTg hessMTTgw hessMTTm hessMTw hessp hist histf histp hsec imag indcv indexcat indices indices2 indicesf indicesfn indnv indsav integrate1d integrateControlCreate intgrat2 intgrat3 inthp1 inthp2 inthp3 inthp4 inthpControlCreate intquad1 intquad2 intquad3 intrleav intrleavsa intrsect intsimp inv invpd invswp iscplx iscplxf isden isinfnanmiss ismiss key keyav keyw lag lag1 lagn lapEighb lapEighi lapEighvb lapEighvi lapgEig lapgEigh lapgEighv lapgEigv lapgSchur lapgSvdcst lapgSvds lapgSvdst lapSvdcusv lapSvds lapSvdusv ldlp ldlsol linSolve listwise ln lncdfbvn lncdfbvn2 lncdfmvn lncdfn lncdfn2 lncdfnc lnfact lngammacplx lnpdfmvn lnpdfmvt lnpdfn lnpdft loadd loadstruct loadwind loess loessmt loessmtControlCreate log loglog logx logy lower lowmat lowmat1 ltrisol lu lusol machEpsilon make makevars makewind margin matalloc matinit mattoarray maxbytes maxc maxindc maxv maxvec mbesselei mbesselei0 mbesselei1 mbesseli mbesseli0 mbesseli1 meanc median mergeby mergevar minc minindc minv miss missex missrv moment momentd movingave movingaveExpwgt movingaveWgt nextindex nextn nextnevn nextwind ntos null null1 numCombinations ols olsmt olsmtControlCreate olsqr olsqr2 olsqrmt ones optn optnevn orth outtyp pacf packedToSp packr parse pause pdfCauchy pdfChi pdfExp pdfGenPareto pdfHyperGeo pdfLaplace pdfLogistic pdfn pdfPoisson pdfRayleigh pdfWeibull pi pinv pinvmt plotAddArrow plotAddBar plotAddBox plotAddHist plotAddHistF plotAddHistP plotAddPolar plotAddScatter plotAddShape plotAddTextbox plotAddTS plotAddXY plotArea plotBar plotBox plotClearLayout plotContour plotCustomLayout plotGetDefaults plotHist plotHistF plotHistP plotLayout plotLogLog plotLogX plotLogY plotOpenWindow plotPolar plotSave plotScatter plotSetAxesPen plotSetBar plotSetBarFill plotSetBarStacked plotSetBkdColor plotSetFill plotSetGrid plotSetLegend plotSetLineColor plotSetLineStyle plotSetLineSymbol plotSetLineThickness plotSetNewWindow plotSetTitle plotSetWhichYAxis plotSetXAxisShow plotSetXLabel plotSetXRange plotSetXTicInterval plotSetXTicLabel plotSetYAxisShow plotSetYLabel plotSetYRange plotSetZAxisShow plotSetZLabel plotSurface plotTS plotXY polar polychar polyeval polygamma polyint polymake polymat polymroot polymult polyroot pqgwin previousindex princomp printfm printfmt prodc psi putarray putf putvals pvCreate pvGetIndex pvGetParNames pvGetParVector pvLength pvList pvPack pvPacki pvPackm pvPackmi pvPacks pvPacksi pvPacksm pvPacksmi pvPutParVector pvTest pvUnpack QNewton QNewtonmt QNewtonmtControlCreate QNewtonmtOutCreate QNewtonSet QProg QProgmt QProgmtInCreate qqr qqre qqrep qr qre qrep qrsol qrtsol qtyr qtyre qtyrep quantile quantiled qyr qyre qyrep qz rank rankindx readr real reclassify reclassifyCuts recode recserar recsercp recserrc rerun rescale reshape rets rev rfft rffti rfftip rfftn rfftnp rfftp rndBernoulli rndBeta rndBinomial rndCauchy rndChiSquare rndCon rndCreateState rndExp rndGamma rndGeo rndGumbel rndHyperGeo rndi rndKMbeta rndKMgam rndKMi rndKMn rndKMnb rndKMp rndKMu rndKMvm rndLaplace rndLCbeta rndLCgam rndLCi rndLCn rndLCnb rndLCp rndLCu rndLCvm rndLogNorm rndMTu rndMVn rndMVt rndn rndnb rndNegBinomial rndp rndPoisson rndRayleigh rndStateSkip rndu rndvm rndWeibull rndWishart rotater round rows rowsf rref sampleData satostrC saved saveStruct savewind scale scale3d scalerr scalinfnanmiss scalmiss schtoc schur searchsourcepath seekr select selif seqa seqm setdif setdifsa setvars setvwrmode setwind shell shiftr sin singleindex sinh sleep solpd sortc sortcc sortd sorthc sorthcc sortind sortindc sortmc sortr sortrc spBiconjGradSol spChol spConjGradSol spCreate spDenseSubmat spDiagRvMat spEigv spEye spLDL spline spLU spNumNZE spOnes spreadSheetReadM spreadSheetReadSA spreadSheetWrite spScale spSubmat spToDense spTrTDense spTScalar spZeros sqpSolve sqpSolveMT sqpSolveMTControlCreate sqpSolveMTlagrangeCreate sqpSolveMToutCreate sqpSolveSet sqrt statements stdc stdsc stocv stof strcombine strindx strlen strput strrindx strsect strsplit strsplitPad strtodt strtof strtofcplx strtriml strtrimr strtrunc strtruncl strtruncpad strtruncr submat subscat substute subvec sumc sumr surface svd svd1 svd2 svdcusv svds svdusv sysstate tab tan tanh tempname time timedt timestr timeutc title tkf2eps tkf2ps tocart todaydt toeplitz token topolar trapchk trigamma trimr trunc type typecv typef union unionsa uniqindx uniqindxsa unique uniquesa upmat upmat1 upper utctodt utctodtv utrisol vals varCovMS varCovXS varget vargetl varmall varmares varput varputl vartypef vcm vcms vcx vcxs vec vech vecr vector vget view viewxyz vlist vnamecv volume vput vread vtypecv wait waitc walkindex where window writer xlabel xlsGetSheetCount xlsGetSheetSize xlsGetSheetTypes xlsMakeRange xlsReadM xlsReadSA xlsWrite xlsWriteM xlsWriteSA xpnd xtics xy xyz ylabel ytics zeros zeta zlabel ztics cdfEmpirical dot h5create h5open h5read h5readAttribute h5write h5writeAttribute ldl plotAddErrorBar plotAddSurface plotCDFEmpirical plotSetColormap plotSetContourLabels plotSetLegendFont plotSetTextInterpreter plotSetXTicCount plotSetYTicCount plotSetZLevels powerm strjoin sylvester strtrim",literal:"DB_AFTER_LAST_ROW DB_ALL_TABLES DB_BATCH_OPERATIONS DB_BEFORE_FIRST_ROW DB_BLOB DB_EVENT_NOTIFICATIONS DB_FINISH_QUERY DB_HIGH_PRECISION DB_LAST_INSERT_ID DB_LOW_PRECISION_DOUBLE DB_LOW_PRECISION_INT32 DB_LOW_PRECISION_INT64 DB_LOW_PRECISION_NUMBERS DB_MULTIPLE_RESULT_SETS DB_NAMED_PLACEHOLDERS DB_POSITIONAL_PLACEHOLDERS DB_PREPARED_QUERIES DB_QUERY_SIZE DB_SIMPLE_LOCKING DB_SYSTEM_TABLES DB_TABLES DB_TRANSACTIONS DB_UNICODE DB_VIEWS __STDIN __STDOUT __STDERR __FILE_DIR"},n=e.COMMENT("@","@"),a={className:"meta",begin:"#",end:"$",keywords:{keyword:"define definecs|10 undef ifdef ifndef iflight ifdllcall ifmac ifos2win ifunix else endif lineson linesoff srcfile srcline"},contains:[{begin:/\\\n/,relevance:0},{beginKeywords:"include",end:"$",keywords:{keyword:"include"},contains:[{className:"string",begin:'"',end:'"',illegal:"\\n"}]},e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE,n]},r={begin:/\bstruct\s+/,end:/\s/,keywords:"struct",contains:[{className:"type",begin:e.UNDERSCORE_IDENT_RE,relevance:0}]},i=[{className:"params",begin:/\(/,end:/\)/,excludeBegin:!0,excludeEnd:!0,endsWithParent:!0,relevance:0,contains:[{className:"literal",begin:/\.\.\./},e.C_NUMBER_MODE,e.C_BLOCK_COMMENT_MODE,n,r]}],o={className:"title",begin:e.UNDERSCORE_IDENT_RE,relevance:0},s=function(t,a,r){const s=e.inherit({className:"function",beginKeywords:t,end:a,excludeEnd:!0,contains:[].concat(i)},{});return s.contains.push(o),s.contains.push(e.C_NUMBER_MODE),s.contains.push(e.C_BLOCK_COMMENT_MODE),s.contains.push(n),s},l={className:"built_in",begin:"\\b("+t.built_in.split(" ").join("|")+")\\b"},c={className:"string",begin:'"',end:'"',contains:[e.BACKSLASH_ESCAPE],relevance:0},_={begin:e.UNDERSCORE_IDENT_RE+"\\s*\\(",returnBegin:!0,keywords:t,relevance:0,contains:[{beginKeywords:t.keyword},l,{className:"built_in",begin:e.UNDERSCORE_IDENT_RE,relevance:0}]},d={begin:/\(/,end:/\)/,relevance:0,keywords:{built_in:t.built_in,literal:t.literal},contains:[e.C_NUMBER_MODE,e.C_BLOCK_COMMENT_MODE,n,l,_,c,"self"]};return _.contains.push(d),{name:"GAUSS",aliases:["gss"],case_insensitive:!0,keywords:t,illegal:/(\{[%#]|[%#]\}| <- )/,contains:[e.C_NUMBER_MODE,e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE,n,c,a,{className:"keyword",begin:/\bexternal (matrix|string|array|sparse matrix|struct|proc|keyword|fn)/},s("proc keyword",";"),s("fn","="),{beginKeywords:"for threadfor",end:/;/,relevance:0,contains:[e.C_BLOCK_COMMENT_MODE,n,d]},{variants:[{begin:e.UNDERSCORE_IDENT_RE+"\\."+e.UNDERSCORE_IDENT_RE},{begin:e.UNDERSCORE_IDENT_RE+"\\s*="}],relevance:0},_,r]}}},3779(e){e.exports=function(e){return{name:"GLSL",keywords:{keyword:"break continue discard do else for if return while switch case default attribute binding buffer ccw centroid centroid varying coherent column_major const cw depth_any depth_greater depth_less depth_unchanged early_fragment_tests equal_spacing flat fractional_even_spacing fractional_odd_spacing highp in index inout invariant invocations isolines layout line_strip lines lines_adjacency local_size_x local_size_y local_size_z location lowp max_vertices mediump noperspective offset origin_upper_left out packed patch pixel_center_integer point_mode points precise precision quads r11f_g11f_b10f r16 r16_snorm r16f r16i r16ui r32f r32i r32ui r8 r8_snorm r8i r8ui readonly restrict rg16 rg16_snorm rg16f rg16i rg16ui rg32f rg32i rg32ui rg8 rg8_snorm rg8i rg8ui rgb10_a2 rgb10_a2ui rgba16 rgba16_snorm rgba16f rgba16i rgba16ui rgba32f rgba32i rgba32ui rgba8 rgba8_snorm rgba8i rgba8ui row_major sample shared smooth std140 std430 stream triangle_strip triangles triangles_adjacency uniform varying vertices volatile writeonly",type:"atomic_uint bool bvec2 bvec3 bvec4 dmat2 dmat2x2 dmat2x3 dmat2x4 dmat3 dmat3x2 dmat3x3 dmat3x4 dmat4 dmat4x2 dmat4x3 dmat4x4 double dvec2 dvec3 dvec4 float iimage1D iimage1DArray iimage2D iimage2DArray iimage2DMS iimage2DMSArray iimage2DRect iimage3D iimageBuffer iimageCube iimageCubeArray image1D image1DArray image2D image2DArray image2DMS image2DMSArray image2DRect image3D imageBuffer imageCube imageCubeArray int isampler1D isampler1DArray isampler2D isampler2DArray isampler2DMS isampler2DMSArray isampler2DRect isampler3D isamplerBuffer isamplerCube isamplerCubeArray ivec2 ivec3 ivec4 mat2 mat2x2 mat2x3 mat2x4 mat3 mat3x2 mat3x3 mat3x4 mat4 mat4x2 mat4x3 mat4x4 sampler1D sampler1DArray sampler1DArrayShadow sampler1DShadow sampler2D sampler2DArray sampler2DArrayShadow sampler2DMS sampler2DMSArray sampler2DRect sampler2DRectShadow sampler2DShadow sampler3D samplerBuffer samplerCube samplerCubeArray samplerCubeArrayShadow samplerCubeShadow image1D uimage1DArray uimage2D uimage2DArray uimage2DMS uimage2DMSArray uimage2DRect uimage3D uimageBuffer uimageCube uimageCubeArray uint usampler1D usampler1DArray usampler2D usampler2DArray usampler2DMS usampler2DMSArray usampler2DRect usampler3D samplerBuffer usamplerCube usamplerCubeArray uvec2 uvec3 uvec4 vec2 vec3 vec4 void",built_in:"gl_MaxAtomicCounterBindings gl_MaxAtomicCounterBufferSize gl_MaxClipDistances gl_MaxClipPlanes gl_MaxCombinedAtomicCounterBuffers gl_MaxCombinedAtomicCounters gl_MaxCombinedImageUniforms gl_MaxCombinedImageUnitsAndFragmentOutputs gl_MaxCombinedTextureImageUnits gl_MaxComputeAtomicCounterBuffers gl_MaxComputeAtomicCounters gl_MaxComputeImageUniforms gl_MaxComputeTextureImageUnits gl_MaxComputeUniformComponents gl_MaxComputeWorkGroupCount gl_MaxComputeWorkGroupSize gl_MaxDrawBuffers gl_MaxFragmentAtomicCounterBuffers gl_MaxFragmentAtomicCounters gl_MaxFragmentImageUniforms gl_MaxFragmentInputComponents gl_MaxFragmentInputVectors gl_MaxFragmentUniformComponents gl_MaxFragmentUniformVectors gl_MaxGeometryAtomicCounterBuffers gl_MaxGeometryAtomicCounters gl_MaxGeometryImageUniforms gl_MaxGeometryInputComponents gl_MaxGeometryOutputComponents gl_MaxGeometryOutputVertices gl_MaxGeometryTextureImageUnits gl_MaxGeometryTotalOutputComponents gl_MaxGeometryUniformComponents gl_MaxGeometryVaryingComponents gl_MaxImageSamples gl_MaxImageUnits gl_MaxLights gl_MaxPatchVertices gl_MaxProgramTexelOffset gl_MaxTessControlAtomicCounterBuffers gl_MaxTessControlAtomicCounters gl_MaxTessControlImageUniforms gl_MaxTessControlInputComponents gl_MaxTessControlOutputComponents gl_MaxTessControlTextureImageUnits gl_MaxTessControlTotalOutputComponents gl_MaxTessControlUniformComponents gl_MaxTessEvaluationAtomicCounterBuffers gl_MaxTessEvaluationAtomicCounters gl_MaxTessEvaluationImageUniforms gl_MaxTessEvaluationInputComponents gl_MaxTessEvaluationOutputComponents gl_MaxTessEvaluationTextureImageUnits gl_MaxTessEvaluationUniformComponents gl_MaxTessGenLevel gl_MaxTessPatchComponents gl_MaxTextureCoords gl_MaxTextureImageUnits gl_MaxTextureUnits gl_MaxVaryingComponents gl_MaxVaryingFloats gl_MaxVaryingVectors gl_MaxVertexAtomicCounterBuffers gl_MaxVertexAtomicCounters gl_MaxVertexAttribs gl_MaxVertexImageUniforms gl_MaxVertexOutputComponents gl_MaxVertexOutputVectors gl_MaxVertexTextureImageUnits gl_MaxVertexUniformComponents gl_MaxVertexUniformVectors gl_MaxViewports gl_MinProgramTexelOffset gl_BackColor gl_BackLightModelProduct gl_BackLightProduct gl_BackMaterial gl_BackSecondaryColor gl_ClipDistance gl_ClipPlane gl_ClipVertex gl_Color gl_DepthRange gl_EyePlaneQ gl_EyePlaneR gl_EyePlaneS gl_EyePlaneT gl_Fog gl_FogCoord gl_FogFragCoord gl_FragColor gl_FragCoord gl_FragData gl_FragDepth gl_FrontColor gl_FrontFacing gl_FrontLightModelProduct gl_FrontLightProduct gl_FrontMaterial gl_FrontSecondaryColor gl_GlobalInvocationID gl_InstanceID gl_InvocationID gl_Layer gl_LightModel gl_LightSource gl_LocalInvocationID gl_LocalInvocationIndex gl_ModelViewMatrix gl_ModelViewMatrixInverse gl_ModelViewMatrixInverseTranspose gl_ModelViewMatrixTranspose gl_ModelViewProjectionMatrix gl_ModelViewProjectionMatrixInverse gl_ModelViewProjectionMatrixInverseTranspose gl_ModelViewProjectionMatrixTranspose gl_MultiTexCoord0 gl_MultiTexCoord1 gl_MultiTexCoord2 gl_MultiTexCoord3 gl_MultiTexCoord4 gl_MultiTexCoord5 gl_MultiTexCoord6 gl_MultiTexCoord7 gl_Normal gl_NormalMatrix gl_NormalScale gl_NumSamples gl_NumWorkGroups gl_ObjectPlaneQ gl_ObjectPlaneR gl_ObjectPlaneS gl_ObjectPlaneT gl_PatchVerticesIn gl_Point gl_PointCoord gl_PointSize gl_Position gl_PrimitiveID gl_PrimitiveIDIn gl_ProjectionMatrix gl_ProjectionMatrixInverse gl_ProjectionMatrixInverseTranspose gl_ProjectionMatrixTranspose gl_SampleID gl_SampleMask gl_SampleMaskIn gl_SamplePosition gl_SecondaryColor gl_TessCoord gl_TessLevelInner gl_TessLevelOuter gl_TexCoord gl_TextureEnvColor gl_TextureMatrix gl_TextureMatrixInverse gl_TextureMatrixInverseTranspose gl_TextureMatrixTranspose gl_Vertex gl_VertexID gl_ViewportIndex gl_WorkGroupID gl_WorkGroupSize gl_in gl_out EmitStreamVertex EmitVertex EndPrimitive EndStreamPrimitive abs acos acosh all any asin asinh atan atanh atomicAdd atomicAnd atomicCompSwap atomicCounter atomicCounterDecrement atomicCounterIncrement atomicExchange atomicMax atomicMin atomicOr atomicXor barrier bitCount bitfieldExtract bitfieldInsert bitfieldReverse ceil clamp cos cosh cross dFdx dFdy degrees determinant distance dot equal exp exp2 faceforward findLSB findMSB floatBitsToInt floatBitsToUint floor fma fract frexp ftransform fwidth greaterThan greaterThanEqual groupMemoryBarrier imageAtomicAdd imageAtomicAnd imageAtomicCompSwap imageAtomicExchange imageAtomicMax imageAtomicMin imageAtomicOr imageAtomicXor imageLoad imageSize imageStore imulExtended intBitsToFloat interpolateAtCentroid interpolateAtOffset interpolateAtSample inverse inversesqrt isinf isnan ldexp length lessThan lessThanEqual log log2 matrixCompMult max memoryBarrier memoryBarrierAtomicCounter memoryBarrierBuffer memoryBarrierImage memoryBarrierShared min mix mod modf noise1 noise2 noise3 noise4 normalize not notEqual outerProduct packDouble2x32 packHalf2x16 packSnorm2x16 packSnorm4x8 packUnorm2x16 packUnorm4x8 pow radians reflect refract round roundEven shadow1D shadow1DLod shadow1DProj shadow1DProjLod shadow2D shadow2DLod shadow2DProj shadow2DProjLod sign sin sinh smoothstep sqrt step tan tanh texelFetch texelFetchOffset texture texture1D texture1DLod texture1DProj texture1DProjLod texture2D texture2DLod texture2DProj texture2DProjLod texture3D texture3DLod texture3DProj texture3DProjLod textureCube textureCubeLod textureGather textureGatherOffset textureGatherOffsets textureGrad textureGradOffset textureLod textureLodOffset textureOffset textureProj textureProjGrad textureProjGradOffset textureProjLod textureProjLodOffset textureProjOffset textureQueryLevels textureQueryLod textureSize transpose trunc uaddCarry uintBitsToFloat umulExtended unpackDouble2x32 unpackHalf2x16 unpackSnorm2x16 unpackSnorm4x8 unpackUnorm2x16 unpackUnorm4x8 usubBorrow",literal:"true false"},illegal:'"',contains:[e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE,e.C_NUMBER_MODE,{className:"meta",begin:"#",end:"$"}]}}},3873(e){e.exports=function(e){const t="\\[=*\\[",n="\\]=*\\]",a={begin:t,end:n,contains:["self"]},r=[e.COMMENT("--(?!"+t+")","$"),e.COMMENT("--"+t,n,{contains:[a],relevance:10})];return{name:"Lua",aliases:["pluto"],keywords:{$pattern:e.UNDERSCORE_IDENT_RE,literal:"true false nil",keyword:"and break do else elseif end for goto if in local not or repeat return then until while",built_in:"_G _ENV _VERSION __index __newindex __mode __call __metatable __tostring __len __gc __add __sub __mul __div __mod __pow __concat __unm __eq __lt __le assert collectgarbage dofile error getfenv getmetatable ipairs load loadfile loadstring module next pairs pcall print rawequal rawget rawset require select setfenv setmetatable tonumber tostring type unpack xpcall arg self coroutine resume yield status wrap create running debug getupvalue debug sethook getmetatable gethook setmetatable setlocal traceback setfenv getinfo setupvalue getlocal getregistry getfenv io lines write close flush open output type read stderr stdin input stdout popen tmpfile math log max acos huge ldexp pi cos tanh pow deg tan cosh sinh random randomseed frexp ceil floor rad abs sqrt modf asin min mod fmod log10 atan2 exp sin atan os exit setlocale date getenv difftime remove time clock tmpname rename execute package preload loadlib loaded loaders cpath config path seeall string sub upper len gfind rep find match char dump gmatch reverse byte format gsub lower table setn insert getn foreachi maxn foreach concat sort remove"},contains:r.concat([{className:"function",beginKeywords:"function",end:"\\)",contains:[e.inherit(e.TITLE_MODE,{begin:"([_a-zA-Z]\\w*\\.)*([_a-zA-Z]\\w*:)?[_a-zA-Z]\\w*"}),{className:"params",begin:"\\(",endsWithParent:!0,contains:r}].concat(r)},e.C_NUMBER_MODE,e.APOS_STRING_MODE,e.QUOTE_STRING_MODE,{className:"string",begin:t,end:n,contains:[a],relevance:5}])}}},3895(e,t,n){"use strict";var a,r=this&&this.__createBinding||(Object.create?function(e,t,n,a){void 0===a&&(a=n);var r=Object.getOwnPropertyDescriptor(t,n);r&&!("get"in r?!t.__esModule:r.writable||r.configurable)||(r={enumerable:!0,get:function(){return t[n]}}),Object.defineProperty(e,a,r)}:function(e,t,n,a){void 0===a&&(a=n),e[a]=t[n]}),i=this&&this.__setModuleDefault||(Object.create?function(e,t){Object.defineProperty(e,"default",{enumerable:!0,value:t})}:function(e,t){e.default=t}),o=this&&this.__importStar||(a=function(e){return a=Object.getOwnPropertyNames||function(e){var t=[];for(var n in e)Object.prototype.hasOwnProperty.call(e,n)&&(t[t.length]=n);return t},a(e)},function(e){if(e&&e.__esModule)return e;var t={};if(null!=e)for(var n=a(e),o=0;o{let t;return t=e.blocks.length?this.generateFileHtml(e):this.generateEmptyDiff(),this.makeFileDiffHtml(e,t)})).join("\n");return this.hoganUtils.render(d,"wrapper",{colorScheme:l.colorSchemeToCss(this.config.colorScheme),content:t})}makeFileDiffHtml(e,t){if(this.config.renderNothingWhenEmpty&&Array.isArray(e.blocks)&&0===e.blocks.length)return"";const n=this.hoganUtils.template(p,"file-diff"),a=this.hoganUtils.template(d,"file-path"),r=this.hoganUtils.template("icon","file"),i=this.hoganUtils.template("tag",l.getFileIcon(e));return n.render({file:e,fileHtmlId:l.getHtmlId(e),diffs:t,filePath:a.render({fileDiffName:l.filenameDiff(e)},{fileIcon:r,fileTag:i})})}generateEmptyDiff(){return this.hoganUtils.render(d,"empty-diff",{contentClass:"d2h-code-line",CSSLineClass:l.CSSLineClass})}generateFileHtml(e){const t=s.newMatcherFn(s.newDistanceFn((t=>l.deconstructLine(t.content,e.isCombined).content)));return e.blocks.map((n=>{let a=this.hoganUtils.render(d,"block-header",{CSSLineClass:l.CSSLineClass,blockHeader:e.isTooBig?n.header:l.escapeForHtml(n.header),lineClass:"d2h-code-linenumber",contentClass:"d2h-code-line"});return this.applyLineGroupping(n).forEach((([n,r,i])=>{if(r.length&&i.length&&!n.length)this.applyRematchMatching(r,i,t).map((([t,n])=>{const{left:r,right:i}=this.processChangedLines(e,e.isCombined,t,n);a+=r,a+=i}));else if(n.length)n.forEach((t=>{const{prefix:n,content:r}=l.deconstructLine(t.content,e.isCombined);a+=this.generateSingleLineHtml(e,{type:l.CSSLineClass.CONTEXT,prefix:n,content:r,oldNumber:t.oldNumber,newNumber:t.newNumber})}));else if(r.length||i.length){const{left:t,right:n}=this.processChangedLines(e,e.isCombined,r,i);a+=t,a+=n}else console.error("Unknown state reached while processing groups of lines",n,r,i)})),a})).join("\n")}applyLineGroupping(e){const t=[];let n=[],a=[];for(let r=0;r0)&&(t.push([[],n,a]),n=[],a=[]),i.type===c.LineType.CONTEXT?t.push([[i],[],[]]):i.type===c.LineType.INSERT&&0===n.length?t.push([[],[],[i]]):i.type===c.LineType.INSERT&&n.length>0?a.push(i):i.type===c.LineType.DELETE&&n.push(i)}return(n.length||a.length)&&(t.push([[],n,a]),n=[],a=[]),t}applyRematchMatching(e,t,n){const a=e.length*t.length,r=(0,_.max)(e.concat(t).map((e=>e.content.length)));return a{if(!e||e.startsWith("*"))return;let P;const k=L[_-1],B=L[_+1],G=L[_+2];if(e.startsWith("diff --git")||e.startsWith("diff --combined")){if(F(),(P=/^diff --git "?([a-ciow]\/.+)"? "?([a-ciow]\/.+)"?/.exec(e))&&(p=l(P[1],void 0,t.dstPrefix),u=l(P[2],void 0,t.srcPrefix)),null===r)throw new Error("Where is my file !!!");return void(r.isGitDiff=!0)}if(e.startsWith("Binary files")&&!(null==r?void 0:r.isGitDiff)){if(F(),(P=/^Binary files "?([a-ciow]\/.+)"? and "?([a-ciow]\/.+)"? differ/.exec(e))&&(p=l(P[1],void 0,t.dstPrefix),u=l(P[2],void 0,t.srcPrefix)),null===r)throw new Error("Where is my file !!!");return void(r.isBinary=!0)}if((!r||!r.isGitDiff&&r&&e.startsWith(m)&&B.startsWith(g)&&G.startsWith(E))&&F(),null==r?void 0:r.isTooBig)return;if(r&&("number"==typeof t.diffMaxChanges&&r.addedLines+r.deletedLines>t.diffMaxChanges||"number"==typeof t.diffMaxLineLength&&e.length>t.diffMaxLineLength))return r.isTooBig=!0,r.addedLines=0,r.deletedLines=0,r.blocks=[],s=null,void U("function"==typeof t.diffTooBigMessage?t.diffTooBigMessage(n.length):"Diff too big to be displayed");if(e.startsWith(m)&&B.startsWith(g)||e.startsWith(g)&&k.startsWith(m)){if(r&&!r.oldName&&e.startsWith("--- ")&&(P=function(e,t){return l(e,"---",t)}(e,t.srcPrefix)))return r.oldName=P,void(r.language=i(r.oldName,r.language));if(r&&!r.newName&&e.startsWith("+++ ")&&(P=function(e,t){return l(e,"+++",t)}(e,t.dstPrefix)))return r.newName=P,void(r.language=i(r.newName,r.language))}if(r&&(e.startsWith(E)||r.isGitDiff&&r.oldName&&r.newName&&!s))return void U(e);if(s&&(e.startsWith("+")||e.startsWith("-")||e.startsWith(" ")))return void function(e){if(null===r||null===s||null===c||null===d)return;const t={content:e},n=r.isCombined?["+ "," +","++"]:["+"],i=r.isCombined?["- "," -","--"]:["-"];o(e,n)?(r.addedLines++,t.type=a.LineType.INSERT,t.oldNumber=void 0,t.newNumber=d++):o(e,i)?(r.deletedLines++,t.type=a.LineType.DELETE,t.oldNumber=c++,t.newNumber=void 0):(t.type=a.LineType.CONTEXT,t.oldNumber=c++,t.newNumber=d++),s.lines.push(t)}(e);const Y=!function(e,t){let n=t;for(;n1?n[n.length-1]:t}function o(e,t){return t.reduce(((t,n)=>t||e.startsWith(n)),!1)}const s=["a/","b/","i/","w/","c/","o/"];function l(e,t,n){const a=void 0!==n?[...s,n]:s,i=t?new RegExp(`^${(0,r.escapeForRegExp)(t)} "?(.+?)"?$`):new RegExp('^"?(.+?)"?$'),[,o=""]=i.exec(e)||[],l=a.find((e=>0===o.indexOf(e)));return(l?o.slice(l.length):o).replace(/\s+\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}(?:\.\d+)? [+-]\d{4}.*$/,"")}},3982(e){e.exports=function(e){const t={$pattern:/-?[A-z\.\-]+\b/,keyword:"if else foreach return do while until elseif begin for trap data dynamicparam end break throw param continue finally in switch exit filter try process catch hidden static parameter",built_in:"ac asnp cat cd CFS chdir clc clear clhy cli clp cls clv cnsn compare copy cp cpi cpp curl cvpa dbp del diff dir dnsn ebp echo|0 epal epcsv epsn erase etsn exsn fc fhx fl ft fw gal gbp gc gcb gci gcm gcs gdr gerr ghy gi gin gjb gl gm gmo gp gps gpv group gsn gsnp gsv gtz gu gv gwmi h history icm iex ihy ii ipal ipcsv ipmo ipsn irm ise iwmi iwr kill lp ls man md measure mi mount move mp mv nal ndr ni nmo npssc nsn nv ogv oh popd ps pushd pwd r rbp rcjb rcsn rd rdr ren ri rjb rm rmdir rmo rni rnp rp rsn rsnp rujb rv rvpa rwmi sajb sal saps sasv sbp sc scb select set shcm si sl sleep sls sort sp spjb spps spsv start stz sujb sv swmi tee trcm type wget where wjb write"},n={begin:"`[\\s\\S]",relevance:0},a={className:"variable",variants:[{begin:/\$\B/},{className:"keyword",begin:/\$this/},{begin:/\$[\w\d][\w\d_:]*/}]},r={className:"string",variants:[{begin:/"/,end:/"/},{begin:/@"/,end:/^"@/}],contains:[n,a,{className:"variable",begin:/\$[A-z]/,end:/[^A-z]/}]},i={className:"string",variants:[{begin:/'/,end:/'/},{begin:/@'/,end:/^'@/}]},o=e.inherit(e.COMMENT(null,null),{variants:[{begin:/#/,end:/$/},{begin:/<#/,end:/#>/}],contains:[{className:"doctag",variants:[{begin:/\.(synopsis|description|example|inputs|outputs|notes|link|component|role|functionality)/},{begin:/\.(parameter|forwardhelptargetname|forwardhelpcategory|remotehelprunspace|externalhelp)\s+\S+/}]}]}),s={className:"built_in",variants:[{begin:"(".concat("Add|Clear|Close|Copy|Enter|Exit|Find|Format|Get|Hide|Join|Lock|Move|New|Open|Optimize|Pop|Push|Redo|Remove|Rename|Reset|Resize|Search|Select|Set|Show|Skip|Split|Step|Switch|Undo|Unlock|Watch|Backup|Checkpoint|Compare|Compress|Convert|ConvertFrom|ConvertTo|Dismount|Edit|Expand|Export|Group|Import|Initialize|Limit|Merge|Mount|Out|Publish|Restore|Save|Sync|Unpublish|Update|Approve|Assert|Build|Complete|Confirm|Deny|Deploy|Disable|Enable|Install|Invoke|Register|Request|Restart|Resume|Start|Stop|Submit|Suspend|Uninstall|Unregister|Wait|Debug|Measure|Ping|Repair|Resolve|Test|Trace|Connect|Disconnect|Read|Receive|Send|Write|Block|Grant|Protect|Revoke|Unblock|Unprotect|Use|ForEach|Sort|Tee|Where",")+(-)[\\w\\d]+")}]},l={className:"class",beginKeywords:"class enum",end:/\s*[{]/,excludeEnd:!0,relevance:0,contains:[e.TITLE_MODE]},c={className:"function",begin:/function\s+/,end:/\s*\{|$/,excludeEnd:!0,returnBegin:!0,relevance:0,contains:[{begin:"function",relevance:0,className:"keyword"},{className:"title",begin:/\w[\w\d]*((-)[\w\d]+)*/,relevance:0},{begin:/\(/,end:/\)/,className:"params",relevance:0,contains:[a]}]},_={begin:/using\s/,end:/$/,returnBegin:!0,contains:[r,i,{className:"keyword",begin:/(using|assembly|command|module|namespace|type)/}]},d={variants:[{className:"operator",begin:"(".concat("-and|-as|-band|-bnot|-bor|-bxor|-casesensitive|-ccontains|-ceq|-cge|-cgt|-cle|-clike|-clt|-cmatch|-cne|-cnotcontains|-cnotlike|-cnotmatch|-contains|-creplace|-csplit|-eq|-exact|-f|-file|-ge|-gt|-icontains|-ieq|-ige|-igt|-ile|-ilike|-ilt|-imatch|-in|-ine|-inotcontains|-inotlike|-inotmatch|-ireplace|-is|-isnot|-isplit|-join|-le|-like|-lt|-match|-ne|-not|-notcontains|-notin|-notlike|-notmatch|-or|-regex|-replace|-shl|-shr|-split|-wildcard|-xor",")\\b")},{className:"literal",begin:/(-){1,2}[\w\d-]+/,relevance:0}]},p={className:"function",begin:/\[.*\]\s*[\w]+[ ]??\(/,end:/$/,returnBegin:!0,relevance:0,contains:[{className:"keyword",begin:"(".concat(t.keyword.toString().replace(/\s/g,"|"),")\\b"),endsParent:!0,relevance:0},e.inherit(e.TITLE_MODE,{endsParent:!0})]},u=[p,o,n,e.NUMBER_MODE,r,i,s,a,{className:"literal",begin:/\$(null|true|false)\b/},{className:"selector-tag",begin:/@\B/,relevance:0}],m={begin:/\[/,end:/\]/,excludeBegin:!0,excludeEnd:!0,relevance:0,contains:[].concat("self",u,{begin:"("+["string","char","byte","int","long","bool","decimal","single","double","DateTime","xml","array","hashtable","void"].join("|")+")",className:"built_in",relevance:0},{className:"type",begin:/[\.\w\d]+/,relevance:0})};return p.contains.unshift(m),{name:"PowerShell",aliases:["pwsh","ps","ps1"],case_insensitive:!0,keywords:t,contains:u.concat(l,c,_,d,m)}}},4008(e){e.exports=function(e){return{name:"HSP",case_insensitive:!0,keywords:{$pattern:/[\w._]+/,keyword:"goto gosub return break repeat loop continue wait await dim sdim foreach dimtype dup dupptr end stop newmod delmod mref run exgoto on mcall assert logmes newlab resume yield onexit onerror onkey onclick oncmd exist delete mkdir chdir dirlist bload bsave bcopy memfile if else poke wpoke lpoke getstr chdpm memexpand memcpy memset notesel noteadd notedel noteload notesave randomize noteunsel noteget split strrep setease button chgdisp exec dialog mmload mmplay mmstop mci pset pget syscolor mes print title pos circle cls font sysfont objsize picload color palcolor palette redraw width gsel gcopy gzoom gmode bmpsave hsvcolor getkey listbox chkbox combox input mesbox buffer screen bgscr mouse objsel groll line clrobj boxf objprm objmode stick grect grotate gsquare gradf objimage objskip objenable celload celdiv celput newcom querycom delcom cnvstow comres axobj winobj sendmsg comevent comevarg sarrayconv callfunc cnvwtos comevdisp libptr system hspstat hspver stat cnt err strsize looplev sublev iparam wparam lparam refstr refdval int rnd strlen length length2 length3 length4 vartype gettime peek wpeek lpeek varptr varuse noteinfo instr abs limit getease str strmid strf getpath strtrim sin cos tan atan sqrt double absf expf logf limitf powf geteasef mousex mousey mousew hwnd hinstance hdc ginfo objinfo dirinfo sysinfo thismod __hspver__ __hsp30__ __date__ __time__ __line__ __file__ _debug __hspdef__ and or xor not screen_normal screen_palette screen_hide screen_fixedsize screen_tool screen_frame gmode_gdi gmode_mem gmode_rgb0 gmode_alpha gmode_rgb0alpha gmode_add gmode_sub gmode_pixela ginfo_mx ginfo_my ginfo_act ginfo_sel ginfo_wx1 ginfo_wy1 ginfo_wx2 ginfo_wy2 ginfo_vx ginfo_vy ginfo_sizex ginfo_sizey ginfo_winx ginfo_winy ginfo_mesx ginfo_mesy ginfo_r ginfo_g ginfo_b ginfo_paluse ginfo_dispx ginfo_dispy ginfo_cx ginfo_cy ginfo_intid ginfo_newid ginfo_sx ginfo_sy objinfo_mode objinfo_bmscr objinfo_hwnd notemax notesize dir_cur dir_exe dir_win dir_sys dir_cmdline dir_desktop dir_mydoc dir_tv font_normal font_bold font_italic font_underline font_strikeout font_antialias objmode_normal objmode_guifont objmode_usefont gsquare_grad msgothic msmincho do until while wend for next _break _continue switch case default swbreak swend ddim ldim alloc m_pi rad2deg deg2rad ease_linear ease_quad_in ease_quad_out ease_quad_inout ease_cubic_in ease_cubic_out ease_cubic_inout ease_quartic_in ease_quartic_out ease_quartic_inout ease_bounce_in ease_bounce_out ease_bounce_inout ease_shake_in ease_shake_out ease_shake_inout ease_loop"},contains:[e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE,e.QUOTE_STRING_MODE,e.APOS_STRING_MODE,{className:"string",begin:/\{"/,end:/"\}/,contains:[e.BACKSLASH_ESCAPE]},e.COMMENT(";","$",{relevance:0}),{className:"meta",begin:"#",end:"$",keywords:{keyword:"addion cfunc cmd cmpopt comfunc const defcfunc deffunc define else endif enum epack func global if ifdef ifndef include modcfunc modfunc modinit modterm module pack packopt regcmd runtime undef usecom uselib"},contains:[e.inherit(e.QUOTE_STRING_MODE,{className:"string"}),e.NUMBER_MODE,e.C_NUMBER_MODE,e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE]},{className:"symbol",begin:"^\\*(\\w+|@)"},e.NUMBER_MODE,e.C_NUMBER_MODE]}}},4119(e){e.exports=function(e){const t=["add","and","cmp","cmpg","cmpl","const","div","double","float","goto","if","int","long","move","mul","neg","new","nop","not","or","rem","return","shl","shr","sput","sub","throw","ushr","xor"];return{name:"Smali",contains:[{className:"string",begin:'"',end:'"',relevance:0},e.COMMENT("#","$",{relevance:0}),{className:"keyword",variants:[{begin:"\\s*\\.end\\s[a-zA-Z0-9]*"},{begin:"^[ ]*\\.[a-zA-Z]*",relevance:0},{begin:"\\s:[a-zA-Z_0-9]*",relevance:0},{begin:"\\s("+["transient","constructor","abstract","final","synthetic","public","private","protected","static","bridge","system"].join("|")+")"}]},{className:"built_in",variants:[{begin:"\\s("+t.join("|")+")\\s"},{begin:"\\s("+t.join("|")+")((-|/)[a-zA-Z0-9]+)+\\s",relevance:10},{begin:"\\s("+["aget","aput","array","check","execute","fill","filled","goto/16","goto/32","iget","instance","invoke","iput","monitor","packed","sget","sparse"].join("|")+")((-|/)[a-zA-Z0-9]+)*\\s",relevance:10}]},{className:"class",begin:"L[^(;:\n]*;",relevance:0},{begin:"[vp][0-9]+"}]}}},4139(e){e.exports=function(e){return{name:"Haxe",aliases:["hx"],keywords:{keyword:"abstract break case cast catch continue default do dynamic else enum extern final for function here if import in inline is macro never new override package private get set public return static super switch this throw trace try typedef untyped using var while Int Float String Bool Dynamic Void Array ",built_in:"trace this",literal:"true false null _"},contains:[{className:"string",begin:"'",end:"'",contains:[e.BACKSLASH_ESCAPE,{className:"subst",begin:/\$\{/,end:/\}/},{className:"subst",begin:/\$/,end:/\W\}/}]},e.QUOTE_STRING_MODE,e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE,{className:"number",begin:/(-?)(\b0[xX][a-fA-F0-9_]+|(\b\d+(\.[\d_]*)?|\.[\d_]+)(([eE][-+]?\d+)|i32|u32|i64|f64)?)/,relevance:0},{className:"variable",begin:"\\$[a-zA-Z_$][a-zA-Z0-9_$]*"},{className:"meta",begin:/@:?/,end:/\(|$/,excludeEnd:!0},{className:"meta",begin:"#",end:"$",keywords:{keyword:"if else elseif end error"}},{className:"type",begin:/:[ \t]*/,end:/[^A-Za-z0-9_ \t\->]/,excludeBegin:!0,excludeEnd:!0,relevance:0},{className:"type",begin:/:[ \t]*/,end:/\W/,excludeBegin:!0,excludeEnd:!0},{className:"type",beginKeywords:"new",end:/\W/,excludeBegin:!0,excludeEnd:!0},{className:"title.class",beginKeywords:"enum",end:/\{/,contains:[e.TITLE_MODE]},{className:"title.class",begin:"\\babstract\\b(?=\\s*"+e.IDENT_RE+"\\s*\\()",end:/[\{$]/,contains:[{className:"type",begin:/\(/,end:/\)/,excludeBegin:!0,excludeEnd:!0},{className:"type",begin:/from +/,end:/\W/,excludeBegin:!0,excludeEnd:!0},{className:"type",begin:/to +/,end:/\W/,excludeBegin:!0,excludeEnd:!0},e.TITLE_MODE],keywords:{keyword:"abstract from to"}},{className:"title.class",begin:/\b(class|interface) +/,end:/[\{$]/,excludeEnd:!0,keywords:"class interface",contains:[{className:"keyword",begin:/\b(extends|implements) +/,keywords:"extends implements",contains:[{className:"type",begin:e.IDENT_RE,relevance:0}]},e.TITLE_MODE]},{className:"title.function",beginKeywords:"function",end:/\(/,excludeEnd:!0,illegal:/\S/,contains:[e.TITLE_MODE]}],illegal:/<\//}}},4178(e,t,n){"use strict";var a,r=this&&this.__createBinding||(Object.create?function(e,t,n,a){void 0===a&&(a=n);var r=Object.getOwnPropertyDescriptor(t,n);r&&!("get"in r?!t.__esModule:r.writable||r.configurable)||(r={enumerable:!0,get:function(){return t[n]}}),Object.defineProperty(e,a,r)}:function(e,t,n,a){void 0===a&&(a=n),e[a]=t[n]}),i=this&&this.__setModuleDefault||(Object.create?function(e,t){Object.defineProperty(e,"default",{enumerable:!0,value:t})}:function(e,t){e.default=t}),o=this&&this.__importStar||(a=function(e){return a=Object.getOwnPropertyNames||function(e){var t=[];for(var n in e)Object.prototype.hasOwnProperty.call(e,n)&&(t[t.length]=n);return t},a(e)},function(e){if(e&&e.__esModule)return e;var t={};if(null!=e)for(var n=a(e),o=0;o{const a=s.compile(n,{asString:!1});return Object.assign(Object.assign({},e),{[t]:a})}),{});this.preCompiledTemplates=Object.assign(Object.assign(Object.assign({},l.defaultTemplates),e),n)}static compile(e){return s.compile(e,{asString:!1})}render(e,t,n,a,r){const i=this.templateKey(e,t);try{return this.preCompiledTemplates[i].render(n,a,r)}catch(e){throw new Error(`Could not find template to render '${i}'`)}}template(e,t){return this.preCompiledTemplates[this.templateKey(e,t)]}templateKey(e,t){return`${e}-${t}`}}},4196(e){e.exports=function(e){const t=e.regex,n=[{className:"strong",begin:/\*{2}([^\n]+?)\*{2}/},{className:"strong",begin:t.concat(/\*\*/,/((\*(?!\*)|\\[^\n]|[^*\n\\])+\n)+/,/(\*(?!\*)|\\[^\n]|[^*\n\\])*/,/\*\*/),relevance:0},{className:"strong",begin:/\B\*(\S|\S[^\n]*?\S)\*(?!\w)/},{className:"strong",begin:/\*[^\s]([^\n]+\n)+([^\n]+)\*/}],a=[{className:"emphasis",begin:/_{2}([^\n]+?)_{2}/},{className:"emphasis",begin:t.concat(/__/,/((_(?!_)|\\[^\n]|[^_\n\\])+\n)+/,/(_(?!_)|\\[^\n]|[^_\n\\])*/,/__/),relevance:0},{className:"emphasis",begin:/\b_(\S|\S[^\n]*?\S)_(?!\w)/},{className:"emphasis",begin:/_[^\s]([^\n]+\n)+([^\n]+)_/},{className:"emphasis",begin:"\\B'(?!['\\s])",end:"(\\n{2}|')",contains:[{begin:"\\\\'\\w",relevance:0}],relevance:0}];return{name:"AsciiDoc",aliases:["adoc"],contains:[e.COMMENT("^/{4,}\\n","\\n/{4,}$",{relevance:10}),e.COMMENT("^//","$",{relevance:0}),{className:"title",begin:"^\\.\\w.*$"},{begin:"^[=\\*]{4,}\\n",end:"\\n^[=\\*]{4,}$",relevance:10},{className:"section",relevance:10,variants:[{begin:"^(={1,6})[ \t].+?([ \t]\\1)?$"},{begin:"^[^\\[\\]\\n]+?\\n[=\\-~\\^\\+]{2,}$"}]},{className:"meta",begin:"^:.+?:",end:"\\s",excludeEnd:!0,relevance:10},{className:"meta",begin:"^\\[.+?\\]$",relevance:0},{className:"quote",begin:"^_{4,}\\n",end:"\\n_{4,}$",relevance:10},{className:"code",begin:"^[\\-\\.]{4,}\\n",end:"\\n[\\-\\.]{4,}$",relevance:10},{begin:"^\\+{4,}\\n",end:"\\n\\+{4,}$",contains:[{begin:"<",end:">",subLanguage:"xml",relevance:0}],relevance:10},{className:"bullet",begin:"^(\\*+|-+|\\.+|[^\\n]+?::)\\s+"},{className:"symbol",begin:"^(NOTE|TIP|IMPORTANT|WARNING|CAUTION):\\s+",relevance:10},{begin:/\\[*_`]/},{begin:/\\\\\*{2}[^\n]*?\*{2}/},{begin:/\\\\_{2}[^\n]*_{2}/},{begin:/\\\\`{2}[^\n]*`{2}/},{begin:/[:;}][*_`](?![*_`])/},...n,...a,{className:"string",variants:[{begin:"``.+?''"},{begin:"`.+?'"}]},{className:"code",begin:/`{2}/,end:/(\n{2}|`{2})/},{className:"code",begin:"(`.+?`|\\+.+?\\+)",relevance:0},{className:"code",begin:"^[ \\t]",end:"$",relevance:0},{begin:"^'{3,}[ \\t]*$",relevance:10},{begin:"(link:)?(http|https|ftp|file|irc|image:?):\\S+?\\[[^[]*?\\]",returnBegin:!0,contains:[{begin:"(link|image:?):",relevance:0},{className:"link",begin:"\\w",end:"[^\\[]+",relevance:0},{className:"string",begin:"\\[",end:"\\]",excludeBegin:!0,excludeEnd:!0,relevance:0}],relevance:10}]}}},4206(e,t){"use strict";function n(e){return e.replace(/&/gm,"&").replace(//gm,">")}function a(e){return e.nodeName.toLowerCase()}Object.defineProperty(t,"__esModule",{value:!0}),t.nodeStream=function(e){const t=[],n=(e,r)=>{for(let i=e.firstChild;i;i=i.nextSibling)3===i.nodeType&&null!==i.nodeValue?r+=i.nodeValue.length:1===i.nodeType&&(t.push({event:"start",offset:r,node:i}),r=n(i,r),a(i).match(/br|hr|img|input/)||t.push({event:"stop",offset:r,node:i}));return r};return n(e,0),t},t.mergeStreams=function(e,t,r){let i=0,o="";const s=[];function l(){return e.length&&t.length?e[0].offset!==t[0].offset?e[0].offset`${e.nodeName}="${n(e.value).replace(/"/g,""")}"`)).join(" ")}>`}function _(e){o+=""}function d(e){("start"===e.event?c:_)(e.node)}for(;e.length||t.length;){let t=l();if(o+=n(r.substring(i,t[0].offset)),i=t[0].offset,t===e){s.reverse().forEach(_);do{d(t.splice(0,1)[0]),t=l()}while(t===e&&t.length&&t[0].offset===i);s.reverse().forEach(c)}else"start"===t[0].event?s.push(t[0].node):s.pop(),d(t.splice(0,1)[0])}return o+n(r.substr(i))},t.closeTags=function(e){const t=new Array;return e.value=e.value.split("\n").map((e=>{const n=t.map((e=>``)).join(""),a=e.matchAll(/(|<\/span>)/g);return Array.from(a).forEach((e=>{""===e[0]?t.shift():t.unshift(e[2])})),n+e+"".repeat(t.length)})).join("\n"),e},t.getLanguage=function(e){var t;return null!==(t=r[e])&&void 0!==t?t:"plaintext"};const r={"1c":"1c",abnf:"abnf",accesslog:"accesslog",as:"actionscript",adb:"ada",ada:"ada",ads:"ada",angelscript:"angelscript",apache:"apache",applescript:"applescript",scpt:"applescript",arcade:"arcade",cpp:"cpp",hpp:"cpp",arduino:"arduino",ino:"arduino",armasm:"armasm",arm:"armasm",xml:"xml",html:"xml",xhtml:"xml",rss:"xml",atom:"xml",xjb:"xml",xsd:"xml",xsl:"xml",plist:"xml",svg:"xml",asciidoc:"asciidoc",adoc:"asciidoc",asc:"asciidoc",aspectj:"aspectj",ahk:"autohotkey",ahkl:"autohotkey",au3:"autoit",avrasm:"avrasm",awk:"awk",axapta:"axapta","x++":"axapta",bash:"bash",sh:"bash",zsh:"bash",b:"basic",bnf:"bnf",bf:"brainfuck",c:"c",h:"c",cats:"c",idc:"c",cal:"cal",capnproto:"capnproto",capnp:"capnproto",ceylon:"ceylon",clean:"clean",clj:"clojure",boot:"clojure",cl2:"clojure",cljc:"clojure",cljs:"clojure","cljs.hl":"clojure",cljscm:"clojure",cljx:"clojure",hic:"clojure","clojure-repl":"clojure-repl",cmake:"cmake","cmake.in":"cmake",coffee:"coffeescript",_coffee:"coffeescript",cake:"coffeescript",cjsx:"coffeescript",iced:"coffeescript",cson:"coffeescript",coq:"coq",cos:"cos",cls:"cos",crmsh:"crmsh",crm:"crmsh",pcmk:"crmsh",cr:"crystal",cs:"csharp",csx:"csharp",csp:"csp",css:"css",d:"d",di:"d",md:"markdown",markdown:"markdown",mdown:"markdown",mdwn:"markdown",mkd:"markdown",mkdn:"markdown",mkdown:"markdown",ronn:"markdown",workbook:"markdown",dart:"dart",dpr:"delphi",dfm:"delphi",pas:"delphi",pascal:"delphi",diff:"diff",patch:"diff",django:"django",jinja:"django",dns:"dns",zone:"dns",bind:"dns",dockerfile:"dockerfile",docker:"dockerfile",dos:"dos",bat:"dos",cmd:"dos",dsconfig:"dsconfig",dts:"dts",dust:"dust",dst:"dust",ebnf:"ebnf",ex:"elixir",exs:"elixir",elm:"elm",rb:"ruby",builder:"ruby",eye:"ruby",gemspec:"ruby",god:"ruby",jbuilder:"ruby",mspec:"ruby",pluginspec:"ruby",podspec:"ruby",rabl:"ruby",rake:"ruby",rbuild:"ruby",rbw:"ruby",rbx:"ruby",ru:"ruby",ruby:"ruby",spec:"ruby",thor:"ruby",watchr:"ruby",erb:"erb","erlang-repl":"erlang-repl",erl:"erlang","app.src":"erlang",escript:"erlang",hrl:"erlang",xrl:"erlang",yrl:"erlang",excel:"excel",xls:"excel",xlsx:"excel",fix:"fix",flix:"flix",f90:"fortran",f:"fortran",f03:"fortran",f08:"fortran",f77:"fortran",f95:"fortran",for:"fortran",fpp:"fortran",fs:"fsharp",fsx:"fsharp",gams:"gams",gms:"gams",gauss:"gauss",gss:"gauss",gcode:"gcode",nc:"gcode",gherkin:"gherkin",glsl:"glsl",fp:"glsl",frag:"glsl",frg:"glsl",fsh:"glsl",fshader:"glsl",geo:"glsl",geom:"glsl",glslv:"glsl",gshader:"glsl",shader:"glsl",tesc:"glsl",tese:"glsl",vert:"glsl",vrx:"glsl",vsh:"glsl",vshader:"glsl",gml:"gml",go:"go",bal:"go",golo:"golo",gololang:"golo",gradle:"gradle",groovy:"groovy",grt:"groovy",gtpl:"groovy",gvy:"groovy",haml:"haml","haml.deface":"haml",handlebars:"handlebars",hbs:"handlebars","html.hbs":"handlebars","html.handlebars":"handlebars",hs:"haskell",hsc:"haskell",idr:"haskell",purs:"haskell",hx:"haxe",hxsl:"haxe",hsp:"hsp",htmlbars:"htmlbars",http:"http",https:"http",hy:"hy",inform7:"inform7",i7:"inform7",ini:"ini",toml:"ini",cfg:"ini",prefs:"ini",irpf90:"irpf90",isbl:"isbl",java:"java",jsp:"java",js:"javascript",jsx:"javascript",_js:"javascript",bones:"javascript",es:"javascript",es6:"javascript",gs:"javascript",jake:"javascript",jsb:"javascript",jscad:"javascript",jsfl:"javascript",jsm:"javascript",jss:"javascript",mjs:"javascript",njs:"javascript",pac:"javascript",sjs:"javascript",ssjs:"javascript",xsjs:"javascript",xsjslib:"javascript",cfc:"javascript","jboss-cli":"jboss-cli",json:"json",avsc:"json",geojson:"json",gltf:"json","JSON-tmLanguage":"json",jsonl:"json",tfstate:"json","tfstate.backup":"json",topojson:"json",webapp:"json",webmanifest:"json",jl:"julia","julia-repl":"julia-repl",kt:"kotlin",ktm:"kotlin",kts:"kotlin",lasso:"lasso",lassoscript:"lasso",tex:"latex",ldif:"ldif",leaf:"leaf",less:"less",lisp:"lisp",factor:"lisp",livecodeserver:"livecodeserver",ls:"livescript",_ls:"livescript",llvm:"llvm",lsl:"lsl",lua:"lua",nse:"lua",p8:"lua",pd_lua:"lua",rbxs:"lua",wlua:"lua",mak:"makefile",make:"makefile",mk:"makefile",mkfile:"makefile",mathematica:"mathematica",mma:"mathematica",wl:"mathematica",matlab:"matlab",maxima:"maxima",mel:"mel",mercury:"mercury",mipsasm:"mipsasm",miz:"mizar",voc:"mizar",al:"perl",cgi:"perl",fcgi:"perl",perl:"perl",ph:"perl",plx:"perl",pl:"perl",pm:"perl",psgi:"perl",t:"perl",mojolicious:"mojolicious",monkey:"monkey",monkey2:"monkey",moonscript:"moonscript",moon:"moonscript",n1ql:"n1ql",nginxconf:"nginx",nim:"nim",nimrod:"nim",nix:"nix",nsi:"nsis",nsh:"nsis",m:"objectivec",objc:"objectivec",mm:"objectivec","obj-c":"objectivec","obj-c++":"objectivec","objective-c++":"objectivec",fun:"ocaml",sig:"ocaml",ml:"ocaml",mli:"ocaml",eliom:"ocaml",eliomi:"ocaml",ml4:"ocaml",mll:"ocaml",mly:"ocaml",openscad:"openscad",oxygene:"oxygene",parser3:"parser3",pf:"pf","pf.conf":"pf",pgsql:"pgsql",postgres:"pgsql",postgresql:"pgsql",php:"php",aw:"php",ctp:"php",inc:"php",php3:"php",php4:"php",php5:"php",phps:"php",phpt:"php","php-template":"php-template",plaintext:"plaintext",txt:"plaintext",text:"plaintext",pony:"pony",ps:"powershell",ps1:"powershell",psd1:"powershell",psm1:"powershell",pde:"processing",profile:"profile",pro:"prolog",prolog:"prolog",yap:"prolog",properties:"properties",proto:"protobuf",puppet:"puppet",pp:"puppet",purebasic:"purebasic",py:"python",bzl:"python",gyp:"python",gypi:"python",lmi:"python",py3:"python",pyde:"python",pyi:"python",pyp:"python",pyt:"python",pyw:"python",rpy:"python",tac:"python",wsgi:"python",xpy:"python","python-repl":"python-repl",pycon:"python-repl",q:"q",k:"q",kdb:"q",qml:"qml",r:"r",rd:"r",rsx:"r",reasonml:"reasonml",re:"reasonml",rib:"rib",roboconf:"roboconf",graph:"roboconf",instances:"roboconf",routeros:"routeros",rsl:"rsl",ruleslanguage:"ruleslanguage",rs:"rust","rs.in":"rust",sas:"sas",scala:"scala",kojo:"scala",sbt:"scala",sc:"scala",scm:"scheme",sch:"scheme",sld:"scheme",sls:"scheme",sps:"scheme",ss:"scheme",rkt:"scheme",scilab:"scilab",scss:"scss",shell:"shell",smali:"smali",st:"smalltalk",sml:"sml",sqf:"sqf",sql:"sql",cql:"sql",ddl:"sql",mysql:"sql",prc:"sql",tab:"sql",udf:"sql",viw:"sql",stan:"stan",stanfuncs:"stan",stata:"stata",step21:"step21",step:"step21",stp:"step21",styl:"stylus",subunit:"subunit",swift:"swift",taggerscript:"taggerscript",yml:"yaml",mir:"yaml",reek:"yaml",rviz:"yaml","sublime-syntax":"yaml",syntax:"yaml",yaml:"yaml","yaml-tmlanguage":"yaml","yml.mysql":"yaml",tap:"tap",tcl:"tcl",adp:"tcl",tm:"tcl",thrift:"thrift",tp:"tp",twig:"twig",craftcms:"twig",ts:"typescript",tsx:"typescript",vala:"vala",vbnet:"vbnet",vb:"vbnet",vbscript:"vbscript",vbs:"vbscript","vbscript-html":"vbscript-html",v:"verilog",veo:"verilog",vhdl:"vhdl",vhd:"vhdl",vhf:"vhdl",vhi:"vhdl",vho:"vhdl",vhs:"vhdl",vht:"vhdl",vhw:"vhdl",vim:"vim",x86asm:"x86asm",xl:"xl",xquery:"xquery",xpath:"xquery",xq:"xquery",zephir:"zephir",zep:"zephir"}},4356(e){e.exports=function(e){return{name:"ERB",subLanguage:"xml",contains:[e.COMMENT("<%#","%>"),{begin:"<%[%=-]?",end:"[%-]?%>",subLanguage:"ruby",excludeBegin:!0,excludeEnd:!0}]}}},4438(e){e.exports=function(e){const t={begin:"`[\\s\\S]"};return{name:"AutoHotkey",case_insensitive:!0,aliases:["ahk"],keywords:{keyword:"Break Continue Critical Exit ExitApp Gosub Goto New OnExit Pause return SetBatchLines SetTimer Suspend Thread Throw Until ahk_id ahk_class ahk_pid ahk_exe ahk_group",literal:"true false NOT AND OR",built_in:"ComSpec Clipboard ClipboardAll ErrorLevel"},contains:[t,e.inherit(e.QUOTE_STRING_MODE,{contains:[t]}),e.COMMENT(";","$",{relevance:0}),e.C_BLOCK_COMMENT_MODE,{className:"number",begin:e.NUMBER_RE,relevance:0},{className:"variable",begin:"%[a-zA-Z0-9#_$@]+%"},{className:"built_in",begin:"^\\s*\\w+\\s*(,|%)"},{className:"title",variants:[{begin:'^[^\\n";]+::(?!=)'},{begin:'^[^\\n";]+:(?!=)',relevance:0}]},{className:"meta",begin:"^\\s*#\\w+",end:"$",relevance:0},{className:"built_in",begin:"A_[a-zA-Z0-9]+"},{begin:",\\s*,"}]}}},4484(e,t,n){"use strict";var a,r=this&&this.__extends||(a=function(e,t){return a=Object.setPrototypeOf||{__proto__:[]}instanceof Array&&function(e,t){e.__proto__=t}||function(e,t){for(var n in t)Object.prototype.hasOwnProperty.call(t,n)&&(e[n]=t[n])},a(e,t)},function(e,t){if("function"!=typeof t&&null!==t)throw new TypeError("Class extends value "+String(t)+" is not a constructor or null");function n(){this.constructor=e}a(e,t),e.prototype=null===t?Object.create(t):(n.prototype=t.prototype,new n)});Object.defineProperty(t,"__esModule",{value:!0}),t.sentenceDiff=void 0,t.diffSentences=function(e,n,a){return t.sentenceDiff.diff(e,n,a)};var i=function(e){function t(){return null!==e&&e.apply(this,arguments)||this}return r(t,e),t.prototype.tokenize=function(e){for(var t,n,a=[],r=0,i=0;i'),a.b("\n"+n),a.b(' '),a.b("\n"+n),a.b(a.rp("'),a.b(a.v(a.f("fileName",e,t,0))),a.b(""),a.b("\n"+n),a.b(' '),a.b("\n"+n),a.b(' '),a.b(a.v(a.f("addedLines",e,t,0))),a.b(""),a.b("\n"+n),a.b(' '),a.b(a.v(a.f("deletedLines",e,t,0))),a.b(""),a.b("\n"+n),a.b(" "),a.b("\n"+n),a.b(" "),a.b("\n"+n),a.b(""),a.fl()},partials:{"'),a.b("\n"+n),a.b('
'),a.b("\n"+n),a.b(' Files changed ('),a.b(a.v(a.f("filesNumber",e,t,0))),a.b(")"),a.b("\n"+n),a.b(' hide'),a.b("\n"+n),a.b(' show'),a.b("\n"+n),a.b("
"),a.b("\n"+n),a.b('
    '),a.b("\n"+n),a.b(" "),a.b(a.t(a.f("files",e,t,0))),a.b("\n"+n),a.b("
"),a.b("\n"+n),a.b("
"),a.fl()},partials:{},subs:{}}),t.defaultTemplates["generic-block-header"]=new s.Template({code:function(e,t,n){var a=this;return a.b(n=n||""),a.b("
"),a.b("\n"+n),a.b(' '),a.b("\n"+n),a.b(' "),a.b("\n"+n),a.b(""),a.fl()},partials:{},subs:{}}),t.defaultTemplates["generic-empty-diff"]=new s.Template({code:function(e,t,n){var a=this;return a.b(n=n||""),a.b(""),a.b("\n"+n),a.b(' "),a.b("\n"+n),a.b(""),a.fl()},partials:{},subs:{}}),t.defaultTemplates["generic-file-path"]=new s.Template({code:function(e,t,n){var a=this;return a.b(n=n||""),a.b(''),a.b("\n"+n),a.b(a.rp("'),a.b(a.v(a.f("fileDiffName",e,t,0))),a.b(""),a.b("\n"+n),a.b(a.rp(""),a.b("\n"+n),a.b('"),a.fl()},partials:{""),a.b("\n"+n),a.b(' "),a.b("\n"+n),a.b(' "),a.b("\n"+n),a.b(""),a.fl()},partials:{},subs:{}}),t.defaultTemplates["generic-wrapper"]=new s.Template({code:function(e,t,n){var a=this;return a.b(n=n||""),a.b('
'),a.b("\n"+n),a.b(" "),a.b(a.t(a.f("content",e,t,0))),a.b("\n"+n),a.b("
"),a.fl()},partials:{},subs:{}}),t.defaultTemplates["icon-file-added"]=new s.Template({code:function(e,t,n){var a=this;return a.b(n=n||""),a.b('"),a.fl()},partials:{},subs:{}}),t.defaultTemplates["icon-file-changed"]=new s.Template({code:function(e,t,n){var a=this;return a.b(n=n||""),a.b('"),a.fl()},partials:{},subs:{}}),t.defaultTemplates["icon-file-deleted"]=new s.Template({code:function(e,t,n){var a=this;return a.b(n=n||""),a.b('"),a.fl()},partials:{},subs:{}}),t.defaultTemplates["icon-file-renamed"]=new s.Template({code:function(e,t,n){var a=this;return a.b(n=n||""),a.b('"),a.fl()},partials:{},subs:{}}),t.defaultTemplates["icon-file"]=new s.Template({code:function(e,t,n){var a=this;return a.b(n=n||""),a.b('"),a.fl()},partials:{},subs:{}}),t.defaultTemplates["line-by-line-file-diff"]=new s.Template({code:function(e,t,n){var a=this;return a.b(n=n||""),a.b('
'),a.b("\n"+n),a.b('
'),a.b("\n"+n),a.b(" "),a.b(a.t(a.f("filePath",e,t,0))),a.b("\n"+n),a.b("
"),a.b("\n"+n),a.b('
'),a.b("\n"+n),a.b('
'),a.b("\n"+n),a.b('
'),a.b("\n"+n),a.b('
'),a.s(a.f("blockHeader",e,t,1),e,t,0,156,173,"{{ }}")&&(a.rs(e,t,(function(e,t,n){n.b(n.t(n.f("blockHeader",e,t,0)))})),e.pop()),a.s(a.f("blockHeader",e,t,1),e,t,1,0,0,"")||a.b(" "),a.b("
"),a.b("\n"+n),a.b("
'),a.b("\n"+n),a.b('
'),a.b("\n"+n),a.b(" File without changes"),a.b("\n"+n),a.b("
"),a.b("\n"+n),a.b("
'),a.b("\n"+n),a.b(" "),a.b(a.t(a.f("lineNumber",e,t,0))),a.b("\n"+n),a.b(" '),a.b("\n"+n),a.b('
'),a.b("\n"+n),a.s(a.f("prefix",e,t,1),e,t,0,162,238,"{{ }}")&&(a.rs(e,t,(function(e,t,a){a.b(' '),a.b(a.t(a.f("prefix",e,t,0))),a.b(""),a.b("\n"+n)})),e.pop()),a.s(a.f("prefix",e,t,1),e,t,1,0,0,"")||(a.b('  '),a.b("\n"+n)),a.s(a.f("content",e,t,1),e,t,0,371,445,"{{ }}")&&(a.rs(e,t,(function(e,t,a){a.b(' '),a.b(a.t(a.f("content",e,t,0))),a.b(""),a.b("\n"+n)})),e.pop()),a.s(a.f("content",e,t,1),e,t,1,0,0,"")||(a.b('
'),a.b("\n"+n)),a.b("
"),a.b("\n"+n),a.b("
'),a.b("\n"+n),a.b(' '),a.b("\n"+n),a.b(" "),a.b(a.t(a.f("diffs",e,t,0))),a.b("\n"+n),a.b(" "),a.b("\n"+n),a.b("
"),a.b("\n"+n),a.b("
"),a.b("\n"+n),a.b("
"),a.b("\n"+n),a.b("
"),a.fl()},partials:{},subs:{}}),t.defaultTemplates["line-by-line-numbers"]=new s.Template({code:function(e,t,n){var a=this;return a.b(n=n||""),a.b('
'),a.b(a.v(a.f("oldNumber",e,t,0))),a.b("
"),a.b("\n"+n),a.b('
'),a.b(a.v(a.f("newNumber",e,t,0))),a.b("
"),a.fl()},partials:{},subs:{}}),t.defaultTemplates["side-by-side-file-diff"]=new s.Template({code:function(e,t,n){var a=this;return a.b(n=n||""),a.b('
'),a.b("\n"+n),a.b('
'),a.b("\n"+n),a.b(" "),a.b(a.t(a.f("filePath",e,t,0))),a.b("\n"+n),a.b("
"),a.b("\n"+n),a.b('
'),a.b("\n"+n),a.b('
'),a.b("\n"+n),a.b('
'),a.b("\n"+n),a.b(' '),a.b("\n"+n),a.b(' '),a.b("\n"+n),a.b(" "),a.b(a.t(a.d("diffs.left",e,t,0))),a.b("\n"+n),a.b(" "),a.b("\n"+n),a.b("
"),a.b("\n"+n),a.b("
"),a.b("\n"+n),a.b("
"),a.b("\n"+n),a.b('
'),a.b("\n"+n),a.b('
'),a.b("\n"+n),a.b(' '),a.b("\n"+n),a.b(' '),a.b("\n"+n),a.b(" "),a.b(a.t(a.d("diffs.right",e,t,0))),a.b("\n"+n),a.b(" "),a.b("\n"+n),a.b("
"),a.b("\n"+n),a.b("
"),a.b("\n"+n),a.b("
"),a.b("\n"+n),a.b("
"),a.b("\n"+n),a.b("
"),a.fl()},partials:{},subs:{}}),t.defaultTemplates["tag-file-added"]=new s.Template({code:function(e,t,n){var a=this;return a.b(n=n||""),a.b('ADDED'),a.fl()},partials:{},subs:{}}),t.defaultTemplates["tag-file-changed"]=new s.Template({code:function(e,t,n){var a=this;return a.b(n=n||""),a.b('CHANGED'),a.fl()},partials:{},subs:{}}),t.defaultTemplates["tag-file-deleted"]=new s.Template({code:function(e,t,n){var a=this;return a.b(n=n||""),a.b('DELETED'),a.fl()},partials:{},subs:{}}),t.defaultTemplates["tag-file-renamed"]=new s.Template({code:function(e,t,n){var a=this;return a.b(n=n||""),a.b('RENAMED'),a.fl()},partials:{},subs:{}})},4499(e){e.exports=function(e){const t=e.regex,n=/(_[a-z_\d]+)?/,a=/([de][+-]?\d+)?/,r={className:"number",variants:[{begin:t.concat(/\b\d+/,/\.(\d*)/,a,n)},{begin:t.concat(/\b\d+/,a,n)},{begin:t.concat(/\.\d+/,a,n)}],relevance:0};return{name:"IRPF90",case_insensitive:!0,keywords:{literal:".False. .True.",keyword:"kind do while private call intrinsic where elsewhere type endtype endmodule endselect endinterface end enddo endif if forall endforall only contains default return stop then public subroutine|10 function program .and. .or. .not. .le. .eq. .ge. .gt. .lt. goto save else use module select case access blank direct exist file fmt form formatted iostat name named nextrec number opened rec recl sequential status unformatted unit continue format pause cycle exit c_null_char c_alert c_backspace c_form_feed flush wait decimal round iomsg synchronous nopass non_overridable pass protected volatile abstract extends import non_intrinsic value deferred generic final enumerator class associate bind enum c_int c_short c_long c_long_long c_signed_char c_size_t c_int8_t c_int16_t c_int32_t c_int64_t c_int_least8_t c_int_least16_t c_int_least32_t c_int_least64_t c_int_fast8_t c_int_fast16_t c_int_fast32_t c_int_fast64_t c_intmax_t C_intptr_t c_float c_double c_long_double c_float_complex c_double_complex c_long_double_complex c_bool c_char c_null_ptr c_null_funptr c_new_line c_carriage_return c_horizontal_tab c_vertical_tab iso_c_binding c_loc c_funloc c_associated c_f_pointer c_ptr c_funptr iso_fortran_env character_storage_size error_unit file_storage_size input_unit iostat_end iostat_eor numeric_storage_size output_unit c_f_procpointer ieee_arithmetic ieee_support_underflow_control ieee_get_underflow_mode ieee_set_underflow_mode newunit contiguous recursive pad position action delim readwrite eor advance nml interface procedure namelist include sequence elemental pure integer real character complex logical dimension allocatable|10 parameter external implicit|10 none double precision assign intent optional pointer target in out common equivalence data begin_provider &begin_provider end_provider begin_shell end_shell begin_template end_template subst assert touch soft_touch provide no_dep free irp_if irp_else irp_endif irp_write irp_read",built_in:"alog alog10 amax0 amax1 amin0 amin1 amod cabs ccos cexp clog csin csqrt dabs dacos dasin datan datan2 dcos dcosh ddim dexp dint dlog dlog10 dmax1 dmin1 dmod dnint dsign dsin dsinh dsqrt dtan dtanh float iabs idim idint idnint ifix isign max0 max1 min0 min1 sngl algama cdabs cdcos cdexp cdlog cdsin cdsqrt cqabs cqcos cqexp cqlog cqsin cqsqrt dcmplx dconjg derf derfc dfloat dgamma dimag dlgama iqint qabs qacos qasin qatan qatan2 qcmplx qconjg qcos qcosh qdim qerf qerfc qexp qgamma qimag qlgama qlog qlog10 qmax1 qmin1 qmod qnint qsign qsin qsinh qsqrt qtan qtanh abs acos aimag aint anint asin atan atan2 char cmplx conjg cos cosh exp ichar index int log log10 max min nint sign sin sinh sqrt tan tanh print write dim lge lgt lle llt mod nullify allocate deallocate adjustl adjustr all allocated any associated bit_size btest ceiling count cshift date_and_time digits dot_product eoshift epsilon exponent floor fraction huge iand ibclr ibits ibset ieor ior ishft ishftc lbound len_trim matmul maxexponent maxloc maxval merge minexponent minloc minval modulo mvbits nearest pack present product radix random_number random_seed range repeat reshape rrspacing scale scan selected_int_kind selected_real_kind set_exponent shape size spacing spread sum system_clock tiny transpose trim ubound unpack verify achar iachar transfer dble entry dprod cpu_time command_argument_count get_command get_command_argument get_environment_variable is_iostat_end ieee_arithmetic ieee_support_underflow_control ieee_get_underflow_mode ieee_set_underflow_mode is_iostat_eor move_alloc new_line selected_char_kind same_type_as extends_type_of acosh asinh atanh bessel_j0 bessel_j1 bessel_jn bessel_y0 bessel_y1 bessel_yn erf erfc erfc_scaled gamma log_gamma hypot norm2 atomic_define atomic_ref execute_command_line leadz trailz storage_size merge_bits bge bgt ble blt dshiftl dshiftr findloc iall iany iparity image_index lcobound ucobound maskl maskr num_images parity popcnt poppar shifta shiftl shiftr this_image IRP_ALIGN irp_here"},illegal:/\/\*/,contains:[e.inherit(e.APOS_STRING_MODE,{className:"string",relevance:0}),e.inherit(e.QUOTE_STRING_MODE,{className:"string",relevance:0}),{className:"function",beginKeywords:"subroutine function program",illegal:"[${=\\n]",contains:[e.UNDERSCORE_TITLE_MODE,{className:"params",begin:"\\(",end:"\\)"}]},e.COMMENT("!","$",{relevance:0}),e.COMMENT("begin_doc","end_doc",{relevance:10}),r]}}},4517(e){e.exports=function(e){return{name:"HAML",case_insensitive:!0,contains:[{className:"meta",begin:"^!!!( (5|1\\.1|Strict|Frameset|Basic|Mobile|RDFa|XML\\b.*))?$",relevance:10},e.COMMENT("^\\s*(!=#|=#|-#|/).*$",null,{relevance:0}),{begin:"^\\s*(-|=|!=)(?!#)",end:/$/,subLanguage:"ruby",excludeBegin:!0,excludeEnd:!0},{className:"tag",begin:"^\\s*%",contains:[{className:"selector-tag",begin:"\\w+"},{className:"selector-id",begin:"#[\\w-]+"},{className:"selector-class",begin:"\\.[\\w-]+"},{begin:/\{\s*/,end:/\s*\}/,contains:[{begin:":\\w+\\s*=>",end:",\\s+",returnBegin:!0,endsWithParent:!0,contains:[{className:"attr",begin:":\\w+"},e.APOS_STRING_MODE,e.QUOTE_STRING_MODE,{begin:"\\w+",relevance:0}]}]},{begin:"\\(\\s*",end:"\\s*\\)",excludeEnd:!0,contains:[{begin:"\\w+\\s*=",end:"\\s+",returnBegin:!0,endsWithParent:!0,contains:[{className:"attr",begin:"\\w+",relevance:0},e.APOS_STRING_MODE,e.QUOTE_STRING_MODE,{begin:"\\w+",relevance:0}]}]}]},{begin:"^\\s*[=~]\\s*"},{begin:/#\{/,end:/\}/,subLanguage:"ruby",excludeBegin:!0,excludeEnd:!0}]}}},4598(e,t){"use strict";function n(e,t){if(0===e.length)return t.length;if(0===t.length)return e.length;const n=[];let a,r;for(a=0;a<=t.length;a++)n[a]=[a];for(r=0;r<=e.length;r++)n[0][r]=r;for(a=1;a<=t.length;a++)for(r=1;r<=e.length;r++)t.charAt(a-1)===e.charAt(r-1)?n[a][r]=n[a-1][r-1]:n[a][r]=Math.min(n[a-1][r-1]+1,Math.min(n[a][r-1]+1,n[a-1][r]+1));return n[t.length][e.length]}Object.defineProperty(t,"__esModule",{value:!0}),t.levenshtein=n,t.newDistanceFn=function(e){return(t,a)=>{const r=e(t).trim(),i=e(a).trim();return n(r,i)/(r.length+i.length)}},t.newMatcherFn=function(e){return function t(n,a,r=0,i=new Map){const o=function(t,n,a=new Map){let r,i=1/0;for(let o=0;o0||o.indexB>0)&&(b=g.concat(b)),(n.length>d||a.length>p)&&(b=b.concat(S)),b}}},4683(e){e.exports=function(e){return{name:"Nested Text",aliases:["nt"],contains:[e.inherit(e.HASH_COMMENT_MODE,{begin:/^\s*(?=#)/,excludeBegin:!0}),{variants:[{match:[/^\s*/,/-/,/[ ]/,/.*$/]},{match:[/^\s*/,/-$/]}],className:{2:"bullet",4:"string"}},{match:[/^\s*/,/>/,/[ ]/,/.*$/],className:{2:"punctuation",4:"string"}},{match:[/^\s*(?=\S)/,/[^:]+/,/:\s*/,/$/],className:{2:"attribute",3:"punctuation"}},{match:[/^\s*(?=\S)/,/[^:]*[^: ]/,/[ ]*:/,/[ ]/,/.*$/],className:{2:"attribute",3:"punctuation",5:"string"}}]}}},4688(e){e.exports=function(e){const t="a-zA-Z_\\-!.?+*=<>&#'",n="["+t+"]["+t+"0-9/;:]*",a={$pattern:n,built_in:"!= % %= & &= * ** **= *= *map + += , --build-class-- --import-- -= . / // //= /= < << <<= <= = > >= >> >>= @ @= ^ ^= abs accumulate all and any ap-compose ap-dotimes ap-each ap-each-while ap-filter ap-first ap-if ap-last ap-map ap-map-when ap-pipe ap-reduce ap-reject apply as-> ascii assert assoc bin break butlast callable calling-module-name car case cdr chain chr coll? combinations compile compress cond cons cons? continue count curry cut cycle dec def default-method defclass defmacro defmacro-alias defmacro/g! defmain defmethod defmulti defn defn-alias defnc defnr defreader defseq del delattr delete-route dict-comp dir disassemble dispatch-reader-macro distinct divmod do doto drop drop-last drop-while empty? end-sequence eval eval-and-compile eval-when-compile even? every? except exec filter first flatten float? fn fnc fnr for for* format fraction genexpr gensym get getattr global globals group-by hasattr hash hex id identity if if* if-not if-python2 import in inc input instance? integer integer-char? integer? interleave interpose is is-coll is-cons is-empty is-even is-every is-float is-instance is-integer is-integer-char is-iterable is-iterator is-keyword is-neg is-none is-not is-numeric is-odd is-pos is-string is-symbol is-zero isinstance islice issubclass iter iterable? iterate iterator? keyword keyword? lambda last len let lif lif-not list* list-comp locals loop macro-error macroexpand macroexpand-1 macroexpand-all map max merge-with method-decorator min multi-decorator multicombinations name neg? next none? nonlocal not not-in not? nth numeric? oct odd? open or ord partition permutations pos? post-route postwalk pow prewalk print product profile/calls profile/cpu put-route quasiquote quote raise range read read-str recursive-replace reduce remove repeat repeatedly repr require rest round route route-with-methods rwm second seq set-comp setattr setv some sorted string string? sum switch symbol? take take-nth take-while tee try unless unquote unquote-splicing vars walk when while with with* with-decorator with-gensyms xi xor yield yield-from zero? zip zip-longest | |= ~"},r={begin:n,relevance:0},i={className:"number",begin:"[-+]?\\d+(\\.\\d+)?",relevance:0},o=e.inherit(e.QUOTE_STRING_MODE,{illegal:null}),s=e.COMMENT(";","$",{relevance:0}),l={className:"literal",begin:/\b([Tt]rue|[Ff]alse|nil|None)\b/},c={begin:"[\\[\\{]",end:"[\\]\\}]",relevance:0},_={className:"comment",begin:"\\^"+n},d=e.COMMENT("\\^\\{","\\}"),p={className:"symbol",begin:"[:]{1,2}"+n},u={begin:"\\(",end:"\\)"},m={endsWithParent:!0,relevance:0},g={className:"name",relevance:0,keywords:a,begin:n,starts:m},E=[u,o,_,d,s,p,c,i,l,r];return u.contains=[e.COMMENT("comment",""),g,m],m.contains=E,c.contains=E,{name:"Hy",aliases:["hylang"],illegal:/\S/,contains:[e.SHEBANG(),u,o,_,d,s,p,c,i,l]}}},4895(e){var t="[0-9](_*[0-9])*",n=`\\.(${t})`,a="[0-9a-fA-F](_*[0-9a-fA-F])*",r={className:"number",variants:[{begin:`(\\b(${t})((${n})|\\.)?|(${n}))[eE][+-]?(${t})[fFdD]?\\b`},{begin:`\\b(${t})((${n})[fFdD]?\\b|\\.([fFdD]\\b)?)`},{begin:`(${n})[fFdD]?\\b`},{begin:`\\b(${t})[fFdD]\\b`},{begin:`\\b0[xX]((${a})\\.?|(${a})?\\.(${a}))[pP][+-]?(${t})[fFdD]?\\b`},{begin:"\\b(0|[1-9](_*[0-9])*)[lL]?\\b"},{begin:`\\b0[xX](${a})[lL]?\\b`},{begin:"\\b0(_*[0-7])*[lL]?\\b"},{begin:"\\b0[bB][01](_*[01])*[lL]?\\b"}],relevance:0};function i(e,t,n){return-1===n?"":e.replace(t,(a=>i(e,t,n-1)))}e.exports=function(e){const t=e.regex,n="[À-ʸa-zA-Z_$][À-ʸa-zA-Z_$0-9]*",a=n+i("(?:<"+n+"~~~(?:\\s*,\\s*"+n+"~~~)*>)?",/~~~/g,2),o={keyword:["synchronized","abstract","private","var","static","if","const ","for","while","strictfp","finally","protected","import","native","final","void","enum","else","break","transient","catch","instanceof","volatile","case","assert","package","default","public","try","switch","continue","throws","protected","public","private","module","requires","exports","do","sealed","yield","permits","goto","when"],literal:["false","true","null"],type:["char","boolean","long","float","int","byte","short","double"],built_in:["super","this"]},s={className:"meta",begin:"@"+n,contains:[{begin:/\(/,end:/\)/,contains:["self"]}]},l={className:"params",begin:/\(/,end:/\)/,keywords:o,relevance:0,contains:[e.C_BLOCK_COMMENT_MODE],endsParent:!0};return{name:"Java",aliases:["jsp"],keywords:o,illegal:/<\/|#/,contains:[e.COMMENT("/\\*\\*","\\*/",{relevance:0,contains:[{begin:/\w+@/,relevance:0},{className:"doctag",begin:"@[A-Za-z]+"}]}),{begin:/import java\.[a-z]+\./,keywords:"import",relevance:2},e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE,{begin:/"""/,end:/"""/,className:"string",contains:[e.BACKSLASH_ESCAPE]},e.APOS_STRING_MODE,e.QUOTE_STRING_MODE,{match:[/\b(?:class|interface|enum|extends|implements|new)/,/\s+/,n],className:{1:"keyword",3:"title.class"}},{match:/non-sealed/,scope:"keyword"},{begin:[t.concat(/(?!else)/,n),/\s+/,n,/\s+/,/=(?!=)/],className:{1:"type",3:"variable",5:"operator"}},{begin:[/record/,/\s+/,n],className:{1:"keyword",3:"title.class"},contains:[l,e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE]},{beginKeywords:"new throw return else",relevance:0},{begin:["(?:"+a+"\\s+)",e.UNDERSCORE_IDENT_RE,/\s*(?=\()/],className:{2:"title.function"},keywords:o,contains:[{className:"params",begin:/\(/,end:/\)/,keywords:o,relevance:0,contains:[s,e.APOS_STRING_MODE,e.QUOTE_STRING_MODE,r,e.C_BLOCK_COMMENT_MODE]},e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE]},r,s]}}},4943(e){e.exports=function(e){const t={type:["boolean","byte","word","String"],built_in:["KeyboardController","MouseController","SoftwareSerial","EthernetServer","EthernetClient","LiquidCrystal","RobotControl","GSMVoiceCall","EthernetUDP","EsploraTFT","HttpClient","RobotMotor","WiFiClient","GSMScanner","FileSystem","Scheduler","GSMServer","YunClient","YunServer","IPAddress","GSMClient","GSMModem","Keyboard","Ethernet","Console","GSMBand","Esplora","Stepper","Process","WiFiUDP","GSM_SMS","Mailbox","USBHost","Firmata","PImage","Client","Server","GSMPIN","FileIO","Bridge","Serial","EEPROM","Stream","Mouse","Audio","Servo","File","Task","GPRS","WiFi","Wire","TFT","GSM","SPI","SD"],_hints:["setup","loop","runShellCommandAsynchronously","analogWriteResolution","retrieveCallingNumber","printFirmwareVersion","analogReadResolution","sendDigitalPortPair","noListenOnLocalhost","readJoystickButton","setFirmwareVersion","readJoystickSwitch","scrollDisplayRight","getVoiceCallStatus","scrollDisplayLeft","writeMicroseconds","delayMicroseconds","beginTransmission","getSignalStrength","runAsynchronously","getAsynchronously","listenOnLocalhost","getCurrentCarrier","readAccelerometer","messageAvailable","sendDigitalPorts","lineFollowConfig","countryNameWrite","runShellCommand","readStringUntil","rewindDirectory","readTemperature","setClockDivider","readLightSensor","endTransmission","analogReference","detachInterrupt","countryNameRead","attachInterrupt","encryptionType","readBytesUntil","robotNameWrite","readMicrophone","robotNameRead","cityNameWrite","userNameWrite","readJoystickY","readJoystickX","mouseReleased","openNextFile","scanNetworks","noInterrupts","digitalWrite","beginSpeaker","mousePressed","isActionDone","mouseDragged","displayLogos","noAutoscroll","addParameter","remoteNumber","getModifiers","keyboardRead","userNameRead","waitContinue","processInput","parseCommand","printVersion","readNetworks","writeMessage","blinkVersion","cityNameRead","readMessage","setDataMode","parsePacket","isListening","setBitOrder","beginPacket","isDirectory","motorsWrite","drawCompass","digitalRead","clearScreen","serialEvent","rightToLeft","setTextSize","leftToRight","requestFrom","keyReleased","compassRead","analogWrite","interrupts","WiFiServer","disconnect","playMelody","parseFloat","autoscroll","getPINUsed","setPINUsed","setTimeout","sendAnalog","readSlider","analogRead","beginWrite","createChar","motorsStop","keyPressed","tempoWrite","readButton","subnetMask","debugPrint","macAddress","writeGreen","randomSeed","attachGPRS","readString","sendString","remotePort","releaseAll","mouseMoved","background","getXChange","getYChange","answerCall","getResult","voiceCall","endPacket","constrain","getSocket","writeJSON","getButton","available","connected","findUntil","readBytes","exitValue","readGreen","writeBlue","startLoop","IPAddress","isPressed","sendSysex","pauseMode","gatewayIP","setCursor","getOemKey","tuneWrite","noDisplay","loadImage","switchPIN","onRequest","onReceive","changePIN","playFile","noBuffer","parseInt","overflow","checkPIN","knobRead","beginTFT","bitClear","updateIR","bitWrite","position","writeRGB","highByte","writeRed","setSpeed","readBlue","noStroke","remoteIP","transfer","shutdown","hangCall","beginSMS","endWrite","attached","maintain","noCursor","checkReg","checkPUK","shiftOut","isValid","shiftIn","pulseIn","connect","println","localIP","pinMode","getIMEI","display","noBlink","process","getBand","running","beginSD","drawBMP","lowByte","setBand","release","bitRead","prepare","pointTo","readRed","setMode","noFill","remove","listen","stroke","detach","attach","noTone","exists","buffer","height","bitSet","circle","config","cursor","random","IRread","setDNS","endSMS","getKey","micros","millis","begin","print","write","ready","flush","width","isPIN","blink","clear","press","mkdir","rmdir","close","point","yield","image","BSSID","click","delay","read","text","move","peek","beep","rect","line","open","seek","fill","size","turn","stop","home","find","step","tone","sqrt","RSSI","SSID","end","bit","tan","cos","sin","pow","map","abs","max","min","get","run","put"],literal:["DIGITAL_MESSAGE","FIRMATA_STRING","ANALOG_MESSAGE","REPORT_DIGITAL","REPORT_ANALOG","INPUT_PULLUP","SET_PIN_MODE","INTERNAL2V56","SYSTEM_RESET","LED_BUILTIN","INTERNAL1V1","SYSEX_START","INTERNAL","EXTERNAL","DEFAULT","OUTPUT","INPUT","HIGH","LOW"]},n=function(e){const t=e.regex,n=e.COMMENT("//","$",{contains:[{begin:/\\\n/}]}),a="decltype\\(auto\\)",r="[a-zA-Z_]\\w*::",i="(?!struct)("+a+"|"+t.optional(r)+"[a-zA-Z_]\\w*"+t.optional("<[^<>]+>")+")",o={className:"type",begin:"\\b[a-z\\d_]*_t\\b"},s={className:"string",variants:[{begin:'(u8?|U|L)?"',end:'"',illegal:"\\n",contains:[e.BACKSLASH_ESCAPE]},{begin:"(u8?|U|L)?'(\\\\(x[0-9A-Fa-f]{2}|u[0-9A-Fa-f]{4,8}|[0-7]{3}|\\S)|.)",end:"'",illegal:"."},e.END_SAME_AS_BEGIN({begin:/(?:u8?|U|L)?R"([^()\\ ]{0,16})\(/,end:/\)([^()\\ ]{0,16})"/})]},l={className:"number",variants:[{begin:"[+-]?(?:(?:[0-9](?:'?[0-9])*\\.(?:[0-9](?:'?[0-9])*)?|\\.[0-9](?:'?[0-9])*)(?:[Ee][+-]?[0-9](?:'?[0-9])*)?|[0-9](?:'?[0-9])*[Ee][+-]?[0-9](?:'?[0-9])*|0[Xx](?:[0-9A-Fa-f](?:'?[0-9A-Fa-f])*(?:\\.(?:[0-9A-Fa-f](?:'?[0-9A-Fa-f])*)?)?|\\.[0-9A-Fa-f](?:'?[0-9A-Fa-f])*)[Pp][+-]?[0-9](?:'?[0-9])*)(?:[Ff](?:16|32|64|128)?|(BF|bf)16|[Ll]|)"},{begin:"[+-]?\\b(?:0[Bb][01](?:'?[01])*|0[Xx][0-9A-Fa-f](?:'?[0-9A-Fa-f])*|0(?:'?[0-7])*|[1-9](?:'?[0-9])*)(?:[Uu](?:LL?|ll?)|[Uu][Zz]?|(?:LL?|ll?)[Uu]?|[Zz][Uu]|)"}],relevance:0},c={className:"meta",begin:/#\s*[a-z]+\b/,end:/$/,keywords:{keyword:"if else elif endif define undef warning error line pragma _Pragma ifdef ifndef include"},contains:[{begin:/\\\n/,relevance:0},e.inherit(s,{className:"string"}),{className:"string",begin:/<.*?>/},n,e.C_BLOCK_COMMENT_MODE]},_={className:"title",begin:t.optional(r)+e.IDENT_RE,relevance:0},d=t.optional(r)+e.IDENT_RE+"\\s*\\(",p={type:["bool","char","char16_t","char32_t","char8_t","double","float","int","long","short","void","wchar_t","unsigned","signed","const","static"],keyword:["alignas","alignof","and","and_eq","asm","atomic_cancel","atomic_commit","atomic_noexcept","auto","bitand","bitor","break","case","catch","class","co_await","co_return","co_yield","compl","concept","const_cast|10","consteval","constexpr","constinit","continue","decltype","default","delete","do","dynamic_cast|10","else","enum","explicit","export","extern","false","final","for","friend","goto","if","import","inline","module","mutable","namespace","new","noexcept","not","not_eq","nullptr","operator","or","or_eq","override","private","protected","public","reflexpr","register","reinterpret_cast|10","requires","return","sizeof","static_assert","static_cast|10","struct","switch","synchronized","template","this","thread_local","throw","transaction_safe","transaction_safe_dynamic","true","try","typedef","typeid","typename","union","using","virtual","volatile","while","xor","xor_eq"],literal:["NULL","false","nullopt","nullptr","true"],built_in:["_Pragma"],_type_hints:["any","auto_ptr","barrier","binary_semaphore","bitset","complex","condition_variable","condition_variable_any","counting_semaphore","deque","false_type","flat_map","flat_set","future","imaginary","initializer_list","istringstream","jthread","latch","lock_guard","multimap","multiset","mutex","optional","ostringstream","packaged_task","pair","promise","priority_queue","queue","recursive_mutex","recursive_timed_mutex","scoped_lock","set","shared_future","shared_lock","shared_mutex","shared_timed_mutex","shared_ptr","stack","string_view","stringstream","timed_mutex","thread","true_type","tuple","unique_lock","unique_ptr","unordered_map","unordered_multimap","unordered_multiset","unordered_set","variant","vector","weak_ptr","wstring","wstring_view"]},u={className:"function.dispatch",relevance:0,keywords:{_hint:["abort","abs","acos","apply","as_const","asin","atan","atan2","calloc","ceil","cerr","cin","clog","cos","cosh","cout","declval","endl","exchange","exit","exp","fabs","floor","fmod","forward","fprintf","fputs","free","frexp","fscanf","future","invoke","isalnum","isalpha","iscntrl","isdigit","isgraph","islower","isprint","ispunct","isspace","isupper","isxdigit","labs","launder","ldexp","log","log10","make_pair","make_shared","make_shared_for_overwrite","make_tuple","make_unique","malloc","memchr","memcmp","memcpy","memset","modf","move","pow","printf","putchar","puts","realloc","scanf","sin","sinh","snprintf","sprintf","sqrt","sscanf","std","stderr","stdin","stdout","strcat","strchr","strcmp","strcpy","strcspn","strlen","strncat","strncmp","strncpy","strpbrk","strrchr","strspn","strstr","swap","tan","tanh","terminate","to_underlying","tolower","toupper","vfprintf","visit","vprintf","vsprintf"]},begin:t.concat(/\b/,/(?!decltype)/,/(?!if)/,/(?!for)/,/(?!switch)/,/(?!while)/,e.IDENT_RE,t.lookahead(/(<[^<>]+>|)\s*\(/))},m=[u,c,o,n,e.C_BLOCK_COMMENT_MODE,l,s],g={variants:[{begin:/=/,end:/;/},{begin:/\(/,end:/\)/},{beginKeywords:"new throw return else",end:/;/}],keywords:p,contains:m.concat([{begin:/\(/,end:/\)/,keywords:p,contains:m.concat(["self"]),relevance:0}]),relevance:0},E={className:"function",begin:"("+i+"[\\*&\\s]+)+"+d,returnBegin:!0,end:/[{;=]/,excludeEnd:!0,keywords:p,illegal:/[^\w\s\*&:<>.]/,contains:[{begin:a,keywords:p,relevance:0},{begin:d,returnBegin:!0,contains:[_],relevance:0},{begin:/::/,relevance:0},{begin:/:/,endsWithParent:!0,contains:[s,l]},{relevance:0,match:/,/},{className:"params",begin:/\(/,end:/\)/,keywords:p,relevance:0,contains:[n,e.C_BLOCK_COMMENT_MODE,s,l,o,{begin:/\(/,end:/\)/,keywords:p,relevance:0,contains:["self",n,e.C_BLOCK_COMMENT_MODE,s,l,o]}]},o,n,e.C_BLOCK_COMMENT_MODE,c]};return{name:"C++",aliases:["cc","c++","h++","hpp","hh","hxx","cxx"],keywords:p,illegal:"",keywords:p,contains:["self",o]},{begin:e.IDENT_RE+"::",keywords:p},{match:[/\b(?:enum(?:\s+(?:class|struct))?|class|struct|union)/,/\s+/,/\w+/],className:{1:"keyword",3:"title.class"}}])}}(e),a=n.keywords;return a.type=[...a.type,...t.type],a.literal=[...a.literal,...t.literal],a.built_in=[...a.built_in,...t.built_in],a._hints=t._hints,n.name="Arduino",n.aliases=["ino"],n.supersetOf="cpp",n}},4968(e,t,n){"use strict";var a,r=this&&this.__extends||(a=function(e,t){return a=Object.setPrototypeOf||{__proto__:[]}instanceof Array&&function(e,t){e.__proto__=t}||function(e,t){for(var n in t)Object.prototype.hasOwnProperty.call(t,n)&&(e[n]=t[n])},a(e,t)},function(e,t){if("function"!=typeof t&&null!==t)throw new TypeError("Class extends value "+String(t)+" is not a constructor or null");function n(){this.constructor=e}a(e,t),e.prototype=null===t?Object.create(t):(n.prototype=t.prototype,new n)});Object.defineProperty(t,"__esModule",{value:!0}),t.arrayDiff=void 0,t.diffArrays=function(e,n,a){return t.arrayDiff.diff(e,n,a)};var i=function(e){function t(){return null!==e&&e.apply(this,arguments)||this}return r(t,e),t.prototype.tokenize=function(e){return e.slice()},t.prototype.join=function(e){return e},t.prototype.removeEmpty=function(e){return e},t}(n(2188).default);t.arrayDiff=new i},5015(e){e.exports=function(e){const t=e.regex,n="([a-zA-Z_]\\w*[!?=]?|[-+~]@|<<|>>|=~|===?|<=>|[<>]=?|\\*\\*|[-/+%^&*~`|]|\\[\\]=?)",a=t.either(/\b([A-Z]+[a-z0-9]+)+/,/\b([A-Z]+[a-z0-9]+)+[A-Z]+/),r=t.concat(a,/(::\w+)*/),i={"variable.constant":["__FILE__","__LINE__","__ENCODING__"],"variable.language":["self","super"],keyword:["alias","and","begin","BEGIN","break","case","class","defined","do","else","elsif","end","END","ensure","for","if","in","module","next","not","or","redo","require","rescue","retry","return","then","undef","unless","until","when","while","yield","include","extend","prepend","public","private","protected","raise","throw"],built_in:["proc","lambda","attr_accessor","attr_reader","attr_writer","define_method","private_constant","module_function"],literal:["true","false","nil"]},o={className:"doctag",begin:"@[A-Za-z]+"},s={begin:"#<",end:">"},l=[e.COMMENT("#","$",{contains:[o]}),e.COMMENT("^=begin","^=end",{contains:[o],relevance:10}),e.COMMENT("^__END__",e.MATCH_NOTHING_RE)],c={className:"subst",begin:/#\{/,end:/\}/,keywords:i},_={className:"string",contains:[e.BACKSLASH_ESCAPE,c],variants:[{begin:/'/,end:/'/},{begin:/"/,end:/"/},{begin:/`/,end:/`/},{begin:/%[qQwWx]?\(/,end:/\)/},{begin:/%[qQwWx]?\[/,end:/\]/},{begin:/%[qQwWx]?\{/,end:/\}/},{begin:/%[qQwWx]?/},{begin:/%[qQwWx]?\//,end:/\//},{begin:/%[qQwWx]?%/,end:/%/},{begin:/%[qQwWx]?-/,end:/-/},{begin:/%[qQwWx]?\|/,end:/\|/},{begin:/\B\?(\\\d{1,3})/},{begin:/\B\?(\\x[A-Fa-f0-9]{1,2})/},{begin:/\B\?(\\u\{?[A-Fa-f0-9]{1,6}\}?)/},{begin:/\B\?(\\M-\\C-|\\M-\\c|\\c\\M-|\\M-|\\C-\\M-)[\x20-\x7e]/},{begin:/\B\?\\(c|C-)[\x20-\x7e]/},{begin:/\B\?\\?\S/},{begin:t.concat(/<<[-~]?'?/,t.lookahead(/(\w+)(?=\W)[^\n]*\n(?:[^\n]*\n)*?\s*\1\b/)),contains:[e.END_SAME_AS_BEGIN({begin:/(\w+)/,end:/(\w+)/,contains:[e.BACKSLASH_ESCAPE,c]})]}]},d="[0-9](_?[0-9])*",p={className:"number",relevance:0,variants:[{begin:`\\b([1-9](_?[0-9])*|0)(\\.(${d}))?([eE][+-]?(${d})|r)?i?\\b`},{begin:"\\b0[dD][0-9](_?[0-9])*r?i?\\b"},{begin:"\\b0[bB][0-1](_?[0-1])*r?i?\\b"},{begin:"\\b0[oO][0-7](_?[0-7])*r?i?\\b"},{begin:"\\b0[xX][0-9a-fA-F](_?[0-9a-fA-F])*r?i?\\b"},{begin:"\\b0(_?[0-7])+r?i?\\b"}]},u={variants:[{match:/\(\)/},{className:"params",begin:/\(/,end:/(?=\))/,excludeBegin:!0,endsParent:!0,keywords:i}]},m=[_,{variants:[{match:[/class\s+/,r,/\s+<\s+/,r]},{match:[/\b(class|module)\s+/,r]}],scope:{2:"title.class",4:"title.class.inherited"},keywords:i},{match:[/(include|extend)\s+/,r],scope:{2:"title.class"},keywords:i},{relevance:0,match:[r,/\.new[. (]/],scope:{1:"title.class"}},{relevance:0,match:/\b[A-Z][A-Z_0-9]+\b/,className:"variable.constant"},{relevance:0,match:a,scope:"title.class"},{match:[/def/,/\s+/,n],scope:{1:"keyword",3:"title.function"},contains:[u]},{begin:e.IDENT_RE+"::"},{className:"symbol",begin:e.UNDERSCORE_IDENT_RE+"(!|\\?)?:",relevance:0},{className:"symbol",begin:":(?!\\s)",contains:[_,{begin:n}],relevance:0},p,{className:"variable",begin:"(\\$\\W)|((\\$|@@?)(\\w+))(?=[^@$?])(?![A-Za-z])(?![@$?'])"},{className:"params",begin:/\|(?!=)/,end:/\|/,excludeBegin:!0,excludeEnd:!0,relevance:0,keywords:i},{begin:"("+e.RE_STARTERS_RE+"|unless)\\s*",keywords:"unless",contains:[{className:"regexp",contains:[e.BACKSLASH_ESCAPE,c],illegal:/\n/,variants:[{begin:"/",end:"/[a-z]*"},{begin:/%r\{/,end:/\}[a-z]*/},{begin:"%r\\(",end:"\\)[a-z]*"},{begin:"%r!",end:"![a-z]*"},{begin:"%r\\[",end:"\\][a-z]*"}]}].concat(s,l),relevance:0}].concat(s,l);c.contains=m,u.contains=m;const g=[{begin:/^\s*=>/,starts:{end:"$",contains:m}},{className:"meta.prompt",begin:"^([>?]>|[\\w#]+\\(\\w+\\):\\d+:\\d+[>*]|(\\w+-)?\\d+\\.\\d+\\.\\d+(p\\d+)?[^\\d][^>]+>)(?=[ ])",starts:{end:"$",keywords:i,contains:m}}];return l.unshift(s),{name:"Ruby",aliases:["rb","gemspec","podspec","thor","irb"],keywords:i,illegal:/\/\*/,contains:[e.SHEBANG({binary:"ruby"})].concat(g).concat(l).concat(m)}}},5031(e){e.exports=function(e){const t=e.regex,n=["GET","POST","HEAD","PUT","DELETE","CONNECT","OPTIONS","PATCH","TRACE"];return{name:"Apache Access Log",contains:[{className:"number",begin:/^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}(:\d{1,5})?\b/,relevance:5},{className:"number",begin:/\b\d+\b/,relevance:0},{className:"string",begin:t.concat(/"/,t.either(...n)),end:/"/,keywords:n,illegal:/\n/,relevance:5,contains:[{begin:/HTTP\/[12]\.\d'/,relevance:5}]},{className:"string",begin:/\[\d[^\]\n]{8,}\]/,illegal:/\n/,relevance:1},{className:"string",begin:/\[/,end:/\]/,illegal:/\n/,relevance:0},{className:"string",begin:/"Mozilla\/\d\.\d \(/,end:/"/,illegal:/\n/,relevance:3},{className:"string",begin:/"/,end:/"/,illegal:/\n/,relevance:0}]}}},5069(e){e.exports=function(e){const t=e.regex,n=/[a-zA-Z]\w*/,a=["as","break","class","construct","continue","else","for","foreign","if","import","in","is","return","static","var","while"],r=["true","false","null"],i=["this","super"],o=["-","~",/\*/,"%",/\.\.\./,/\.\./,/\+/,"<<",">>",">=","<=","<",">",/\^/,/!=/,/!/,/\bis\b/,"==","&&","&",/\|\|/,/\|/,/\?:/,"="],s={relevance:0,match:t.concat(/\b(?!(if|while|for|else|super)\b)/,n,/(?=\s*[({])/),className:"title.function"},l={match:t.concat(t.either(t.concat(/\b(?!(if|while|for|else|super)\b)/,n),t.either(...o)),/(?=\s*\([^)]+\)\s*\{)/),className:"title.function",starts:{contains:[{begin:/\(/,end:/\)/,contains:[{relevance:0,scope:"params",match:n}]}]}},c={variants:[{match:[/class\s+/,n,/\s+is\s+/,n]},{match:[/class\s+/,n]}],scope:{2:"title.class",4:"title.class.inherited"},keywords:a},_={relevance:0,match:t.either(...o),className:"operator"},d={className:"property",begin:t.concat(/\./,t.lookahead(n)),end:n,excludeBegin:!0,relevance:0},p={relevance:0,match:t.concat(/\b_/,n),scope:"variable"},u={relevance:0,match:/\b[A-Z]+[a-z]+([A-Z]+[a-z]+)*/,scope:"title.class",keywords:{_:["Bool","Class","Fiber","Fn","List","Map","Null","Num","Object","Range","Sequence","String","System"]}},m=e.C_NUMBER_MODE,g={match:[n,/\s*/,/=/,/\s*/,/\(/,n,/\)\s*\{/],scope:{1:"title.function",3:"operator",6:"params"}},E=e.COMMENT(/\/\*\*/,/\*\//,{contains:[{match:/@[a-z]+/,scope:"doctag"},"self"]}),S={scope:"subst",begin:/%\(/,end:/\)/,contains:[m,u,s,p,_]},b={scope:"string",begin:/"/,end:/"/,contains:[S,{scope:"char.escape",variants:[{match:/\\\\|\\["0%abefnrtv]/},{match:/\\x[0-9A-F]{2}/},{match:/\\u[0-9A-F]{4}/},{match:/\\U[0-9A-F]{8}/}]}]};S.contains.push(b);const f=[...a,...i,...r],T={relevance:0,match:t.concat("\\b(?!",f.join("|"),"\\b)",/[a-zA-Z_]\w*(?:[?!]|\b)/),className:"variable"};return{name:"Wren",keywords:{keyword:a,"variable.language":i,literal:r},contains:[{scope:"comment",variants:[{begin:[/#!?/,/[A-Za-z_]+(?=\()/],beginScope:{},keywords:{literal:r},contains:[],end:/\)/},{begin:[/#!?/,/[A-Za-z_]+/],beginScope:{},end:/$/}]},m,b,{className:"string",begin:/"""/,end:/"""/},E,e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE,u,c,g,l,s,_,p,d,T]}}},5111(e,t,n){var a=n(7453);a.Template=n(7766).Template,a.template=a.Template,e.exports=a},5206(e){e.exports=function(e){return{name:"Python profiler",contains:[e.C_NUMBER_MODE,{begin:"[a-zA-Z_][\\da-zA-Z_]+\\.[\\da-zA-Z_]{1,3}",end:":",excludeEnd:!0},{begin:"(ncalls|tottime|cumtime)",end:"$",keywords:"ncalls tottime|10 cumtime|10 filename",relevance:10},{begin:"function calls",end:"$",contains:[e.C_NUMBER_MODE],relevance:10},e.APOS_STRING_MODE,e.QUOTE_STRING_MODE,{className:"string",begin:"\\(",end:"\\)$",excludeBegin:!0,excludeEnd:!0,relevance:0}]}}},5360(e){e.exports=function(e){const t=e.regex,n=e.COMMENT(/;/,/$/);return{name:"Augmented Backus-Naur Form",illegal:/[!@#$^&',?+~`|:]/,keywords:["ALPHA","BIT","CHAR","CR","CRLF","CTL","DIGIT","DQUOTE","HEXDIG","HTAB","LF","LWSP","OCTET","SP","VCHAR","WSP"],contains:[{scope:"operator",match:/=\/?/},{scope:"attribute",match:t.concat(/^[a-zA-Z][a-zA-Z0-9-]*/,/(?=\s*=)/)},n,{scope:"symbol",match:/%b[0-1]+(-[0-1]+|(\.[0-1]+)+)?/},{scope:"symbol",match:/%d[0-9]+(-[0-9]+|(\.[0-9]+)+)?/},{scope:"symbol",match:/%x[0-9A-F]+(-[0-9A-F]+|(\.[0-9A-F]+)+)?/},{scope:"symbol",match:/%[si](?=".*")/},e.QUOTE_STRING_MODE,e.NUMBER_MODE]}}},5409(e){e.exports=function(e){const t=e.regex,n=/(r#)?/,a=t.concat(n,e.UNDERSCORE_IDENT_RE),r=t.concat(n,e.IDENT_RE),i={className:"title.function.invoke",relevance:0,begin:t.concat(/\b/,/(?!let|for|while|if|else|match\b)/,r,t.lookahead(/\s*\(/))},o="([ui](8|16|32|64|128|size)|f(32|64))?",s=["drop ","Copy","Send","Sized","Sync","Drop","Fn","FnMut","FnOnce","ToOwned","Clone","Debug","PartialEq","PartialOrd","Eq","Ord","AsRef","AsMut","Into","From","Default","Iterator","Extend","IntoIterator","DoubleEndedIterator","ExactSizeIterator","SliceConcatExt","ToString","assert!","assert_eq!","bitflags!","bytes!","cfg!","col!","concat!","concat_idents!","debug_assert!","debug_assert_eq!","env!","eprintln!","panic!","file!","format!","format_args!","include_bytes!","include_str!","line!","local_data_key!","module_path!","option_env!","print!","println!","select!","stringify!","try!","unimplemented!","unreachable!","vec!","write!","writeln!","macro_rules!","assert_ne!","debug_assert_ne!"],l=["i8","i16","i32","i64","i128","isize","u8","u16","u32","u64","u128","usize","f32","f64","str","char","bool","Box","Option","Result","String","Vec"];return{name:"Rust",aliases:["rs"],keywords:{$pattern:e.IDENT_RE+"!?",type:l,keyword:["abstract","as","async","await","become","box","break","const","continue","crate","do","dyn","else","enum","extern","false","final","fn","for","if","impl","in","let","loop","macro","match","mod","move","mut","override","priv","pub","ref","return","self","Self","static","struct","super","trait","true","try","type","typeof","union","unsafe","unsized","use","virtual","where","while","yield"],literal:["true","false","Some","None","Ok","Err"],built_in:s},illegal:""},i]}}},5501(e,t,n){"use strict";var a,r=this&&this.__createBinding||(Object.create?function(e,t,n,a){void 0===a&&(a=n);var r=Object.getOwnPropertyDescriptor(t,n);r&&!("get"in r?!t.__esModule:r.writable||r.configurable)||(r={enumerable:!0,get:function(){return t[n]}}),Object.defineProperty(e,a,r)}:function(e,t,n,a){void 0===a&&(a=n),e[a]=t[n]}),i=this&&this.__setModuleDefault||(Object.create?function(e,t){Object.defineProperty(e,"default",{enumerable:!0,value:t})}:function(e,t){e.default=t}),o=this&&this.__importStar||(a=function(e){return a=Object.getOwnPropertyNames||function(e){var t=[];for(var n in e)Object.prototype.hasOwnProperty.call(e,n)&&(t[t.length]=n);return t},a(e)},function(e){if(e&&e.__esModule)return e;var t={};if(null!=e)for(var n=a(e),o=0;othis.hoganUtils.render(l,"line",{fileHtmlId:s.getHtmlId(e),oldName:e.oldName,newName:e.newName,fileName:s.filenameDiff(e),deletedLines:"-"+e.deletedLines,addedLines:"+"+e.addedLines},{fileIcon:this.hoganUtils.template("icon",s.getFileIcon(e))}))).join("\n");return this.hoganUtils.render(l,"wrapper",{colorScheme:s.colorSchemeToCss(this.config.colorScheme),filesNumber:e.length,files:t})}}},5582(e){e.exports=function(e){const t=e.COMMENT("%","$"),n=e.inherit(e.APOS_STRING_MODE,{relevance:0}),a=e.inherit(e.QUOTE_STRING_MODE,{relevance:0});return a.contains=a.contains.slice(),a.contains.push({className:"subst",begin:"\\\\[abfnrtv]\\|\\\\x[0-9a-fA-F]*\\\\\\|%[-+# *.0-9]*[dioxXucsfeEgGp]",relevance:0}),{name:"Mercury",aliases:["m","moo"],keywords:{keyword:"module use_module import_module include_module end_module initialise mutable initialize finalize finalise interface implementation pred mode func type inst solver any_pred any_func is semidet det nondet multi erroneous failure cc_nondet cc_multi typeclass instance where pragma promise external trace atomic or_else require_complete_switch require_det require_semidet require_multi require_nondet require_cc_multi require_cc_nondet require_erroneous require_failure",meta:"inline no_inline type_spec source_file fact_table obsolete memo loop_check minimal_model terminates does_not_terminate check_termination promise_equivalent_clauses foreign_proc foreign_decl foreign_code foreign_type foreign_import_module foreign_export_enum foreign_export foreign_enum may_call_mercury will_not_call_mercury thread_safe not_thread_safe maybe_thread_safe promise_pure promise_semipure tabled_for_io local untrailed trailed attach_to_io_state can_pass_as_mercury_type stable will_not_throw_exception may_modify_trail will_not_modify_trail may_duplicate may_not_duplicate affects_liveness does_not_affect_liveness doesnt_affect_liveness no_sharing unknown_sharing sharing",built_in:"some all not if then else true fail false try catch catch_any semidet_true semidet_false semidet_fail impure_true impure semipure"},contains:[{className:"built_in",variants:[{begin:"<=>"},{begin:"<=",relevance:0},{begin:"=>",relevance:0},{begin:"/\\\\"},{begin:"\\\\/"}]},{className:"built_in",variants:[{begin:":-\\|--\x3e"},{begin:"=",relevance:0}]},t,e.C_BLOCK_COMMENT_MODE,{className:"number",begin:"0'.\\|0[box][0-9a-fA-F]*"},e.NUMBER_MODE,n,a,{begin:/:-/},{begin:/\.$/}]}}},5588(e){e.exports=function(e){const t="true false yes no null",n="[\\w#;/?:@&=+$,.~*'()[\\]]+",a={className:"string",relevance:0,variants:[{begin:/"/,end:/"/},{begin:/\S+/}],contains:[e.BACKSLASH_ESCAPE,{className:"template-variable",variants:[{begin:/\{\{/,end:/\}\}/},{begin:/%\{/,end:/\}/}]}]},r=e.inherit(a,{variants:[{begin:/'/,end:/'/,contains:[{begin:/''/,relevance:0}]},{begin:/"/,end:/"/},{begin:/[^\s,{}[\]]+/}]}),i={end:",",endsWithParent:!0,excludeEnd:!0,keywords:t,relevance:0},o={begin:/\{/,end:/\}/,contains:[i],illegal:"\\n",relevance:0},s={begin:"\\[",end:"\\]",contains:[i],illegal:"\\n",relevance:0},l=[{className:"attr",variants:[{begin:/[\w*@][\w*@ :()\./-]*:(?=[ \t]|$)/},{begin:/"[\w*@][\w*@ :()\./-]*":(?=[ \t]|$)/},{begin:/'[\w*@][\w*@ :()\./-]*':(?=[ \t]|$)/}]},{className:"meta",begin:"^---\\s*$",relevance:10},{className:"string",begin:"[\\|>]([1-9]?[+-])?[ ]*\\n( +)[^ ][^\\n]*\\n(\\2[^\\n]+\\n?)*"},{begin:"<%[%=-]?",end:"[%-]?%>",subLanguage:"ruby",excludeBegin:!0,excludeEnd:!0,relevance:0},{className:"type",begin:"!\\w+!"+n},{className:"type",begin:"!<"+n+">"},{className:"type",begin:"!"+n},{className:"type",begin:"!!"+n},{className:"meta",begin:"&"+e.UNDERSCORE_IDENT_RE+"$"},{className:"meta",begin:"\\*"+e.UNDERSCORE_IDENT_RE+"$"},{className:"bullet",begin:"-(?=[ ]|$)",relevance:0},e.HASH_COMMENT_MODE,{beginKeywords:t,keywords:{literal:t}},{className:"number",begin:"\\b[0-9]{4}(-[0-9][0-9]){0,2}([Tt \\t][0-9][0-9]?(:[0-9][0-9]){2})?(\\.[0-9]*)?([ \\t])*(Z|[-+][0-9][0-9]?(:[0-9][0-9])?)?\\b"},{className:"number",begin:e.C_NUMBER_RE+"\\b",relevance:0},o,s,{className:"string",relevance:0,begin:/'/,end:/'/,contains:[{match:/''/,scope:"char.escape",relevance:0}]},a],c=[...l];return c.pop(),c.push(r),i.contains=c,{name:"YAML",case_insensitive:!0,aliases:["yml"],contains:l}}},5604(e,t,n){"use strict";var a=this&&this.__importDefault||function(e){return e&&e.__esModule?e:{default:e}};Object.defineProperty(t,"__esModule",{value:!0}),t.defaultDiff2HtmlUIConfig=t.Diff2HtmlUI=void 0;const r=a(n(3223)),i=n(92);Object.defineProperty(t,"defaultDiff2HtmlUIConfig",{enumerable:!0,get:function(){return i.defaultDiff2HtmlUIConfig}});class o extends i.Diff2HtmlUI{constructor(e,t,n={}){super(e,t,n,r.default)}}t.Diff2HtmlUI=o},5623(e){e.exports=function(e){return{name:"JBoss CLI",aliases:["wildfly-cli"],keywords:{$pattern:"[a-z-]+",keyword:"alias batch cd clear command connect connection-factory connection-info data-source deploy deployment-info deployment-overlay echo echo-dmr help history if jdbc-driver-info jms-queue|20 jms-topic|20 ls patch pwd quit read-attribute read-operation reload rollout-plan run-batch set shutdown try unalias undeploy unset version xa-data-source",literal:"true false"},contains:[e.HASH_COMMENT_MODE,e.QUOTE_STRING_MODE,{className:"params",begin:/--[\w\-=\/]+/},{className:"function",begin:/:[\w\-.]+/,relevance:0},{className:"string",begin:/\B([\/.])[\w\-.\/=]+/},{className:"params",begin:/\(/,end:/\)/,contains:[{begin:/[\w-]+ *=/,returnBegin:!0,relevance:0,contains:[{className:"attr",begin:/[\w-]+/}]}],relevance:0}]}}},5665(e){e.exports=function(e){const t="[A-Za-zА-Яа-яёЁ_!][A-Za-zА-Яа-яёЁ_0-9]*",n={className:"number",begin:e.NUMBER_RE,relevance:0},a={className:"string",variants:[{begin:'"',end:'"'},{begin:"'",end:"'"}]},r={className:"doctag",begin:"\\b(?:TODO|DONE|BEGIN|END|STUB|CHG|FIXME|NOTE|BUG|XXX)\\b",relevance:0},i={variants:[{className:"comment",begin:"//",end:"$",relevance:0,contains:[e.PHRASAL_WORDS_MODE,r]},{className:"comment",begin:"/\\*",end:"\\*/",relevance:0,contains:[e.PHRASAL_WORDS_MODE,r]}]},o={$pattern:t,keyword:"and и else иначе endexcept endfinally endforeach конецвсе endif конецесли endwhile конецпока except exitfor finally foreach все if если in в not не or или try while пока ",built_in:"SYSRES_CONST_ACCES_RIGHT_TYPE_EDIT SYSRES_CONST_ACCES_RIGHT_TYPE_FULL SYSRES_CONST_ACCES_RIGHT_TYPE_VIEW SYSRES_CONST_ACCESS_MODE_REQUISITE_CODE SYSRES_CONST_ACCESS_NO_ACCESS_VIEW SYSRES_CONST_ACCESS_NO_ACCESS_VIEW_CODE SYSRES_CONST_ACCESS_RIGHTS_ADD_REQUISITE_CODE SYSRES_CONST_ACCESS_RIGHTS_ADD_REQUISITE_YES_CODE SYSRES_CONST_ACCESS_RIGHTS_CHANGE_REQUISITE_CODE SYSRES_CONST_ACCESS_RIGHTS_CHANGE_REQUISITE_YES_CODE SYSRES_CONST_ACCESS_RIGHTS_DELETE_REQUISITE_CODE SYSRES_CONST_ACCESS_RIGHTS_DELETE_REQUISITE_YES_CODE SYSRES_CONST_ACCESS_RIGHTS_EXECUTE_REQUISITE_CODE SYSRES_CONST_ACCESS_RIGHTS_EXECUTE_REQUISITE_YES_CODE SYSRES_CONST_ACCESS_RIGHTS_NO_ACCESS_REQUISITE_CODE SYSRES_CONST_ACCESS_RIGHTS_NO_ACCESS_REQUISITE_YES_CODE SYSRES_CONST_ACCESS_RIGHTS_RATIFY_REQUISITE_CODE SYSRES_CONST_ACCESS_RIGHTS_RATIFY_REQUISITE_YES_CODE SYSRES_CONST_ACCESS_RIGHTS_REQUISITE_CODE SYSRES_CONST_ACCESS_RIGHTS_VIEW SYSRES_CONST_ACCESS_RIGHTS_VIEW_CODE SYSRES_CONST_ACCESS_RIGHTS_VIEW_REQUISITE_CODE SYSRES_CONST_ACCESS_RIGHTS_VIEW_REQUISITE_YES_CODE SYSRES_CONST_ACCESS_TYPE_CHANGE SYSRES_CONST_ACCESS_TYPE_CHANGE_CODE SYSRES_CONST_ACCESS_TYPE_EXISTS SYSRES_CONST_ACCESS_TYPE_EXISTS_CODE SYSRES_CONST_ACCESS_TYPE_FULL SYSRES_CONST_ACCESS_TYPE_FULL_CODE SYSRES_CONST_ACCESS_TYPE_VIEW SYSRES_CONST_ACCESS_TYPE_VIEW_CODE SYSRES_CONST_ACTION_TYPE_ABORT SYSRES_CONST_ACTION_TYPE_ACCEPT SYSRES_CONST_ACTION_TYPE_ACCESS_RIGHTS SYSRES_CONST_ACTION_TYPE_ADD_ATTACHMENT SYSRES_CONST_ACTION_TYPE_CHANGE_CARD SYSRES_CONST_ACTION_TYPE_CHANGE_KIND SYSRES_CONST_ACTION_TYPE_CHANGE_STORAGE SYSRES_CONST_ACTION_TYPE_CONTINUE SYSRES_CONST_ACTION_TYPE_COPY SYSRES_CONST_ACTION_TYPE_CREATE SYSRES_CONST_ACTION_TYPE_CREATE_VERSION SYSRES_CONST_ACTION_TYPE_DELETE SYSRES_CONST_ACTION_TYPE_DELETE_ATTACHMENT SYSRES_CONST_ACTION_TYPE_DELETE_VERSION SYSRES_CONST_ACTION_TYPE_DISABLE_DELEGATE_ACCESS_RIGHTS SYSRES_CONST_ACTION_TYPE_ENABLE_DELEGATE_ACCESS_RIGHTS SYSRES_CONST_ACTION_TYPE_ENCRYPTION_BY_CERTIFICATE SYSRES_CONST_ACTION_TYPE_ENCRYPTION_BY_CERTIFICATE_AND_PASSWORD SYSRES_CONST_ACTION_TYPE_ENCRYPTION_BY_PASSWORD SYSRES_CONST_ACTION_TYPE_EXPORT_WITH_LOCK SYSRES_CONST_ACTION_TYPE_EXPORT_WITHOUT_LOCK SYSRES_CONST_ACTION_TYPE_IMPORT_WITH_UNLOCK SYSRES_CONST_ACTION_TYPE_IMPORT_WITHOUT_UNLOCK SYSRES_CONST_ACTION_TYPE_LIFE_CYCLE_STAGE SYSRES_CONST_ACTION_TYPE_LOCK SYSRES_CONST_ACTION_TYPE_LOCK_FOR_SERVER SYSRES_CONST_ACTION_TYPE_LOCK_MODIFY SYSRES_CONST_ACTION_TYPE_MARK_AS_READED SYSRES_CONST_ACTION_TYPE_MARK_AS_UNREADED SYSRES_CONST_ACTION_TYPE_MODIFY SYSRES_CONST_ACTION_TYPE_MODIFY_CARD SYSRES_CONST_ACTION_TYPE_MOVE_TO_ARCHIVE SYSRES_CONST_ACTION_TYPE_OFF_ENCRYPTION SYSRES_CONST_ACTION_TYPE_PASSWORD_CHANGE SYSRES_CONST_ACTION_TYPE_PERFORM SYSRES_CONST_ACTION_TYPE_RECOVER_FROM_LOCAL_COPY SYSRES_CONST_ACTION_TYPE_RESTART SYSRES_CONST_ACTION_TYPE_RESTORE_FROM_ARCHIVE SYSRES_CONST_ACTION_TYPE_REVISION SYSRES_CONST_ACTION_TYPE_SEND_BY_MAIL SYSRES_CONST_ACTION_TYPE_SIGN SYSRES_CONST_ACTION_TYPE_START SYSRES_CONST_ACTION_TYPE_UNLOCK SYSRES_CONST_ACTION_TYPE_UNLOCK_FROM_SERVER SYSRES_CONST_ACTION_TYPE_VERSION_STATE SYSRES_CONST_ACTION_TYPE_VERSION_VISIBILITY SYSRES_CONST_ACTION_TYPE_VIEW SYSRES_CONST_ACTION_TYPE_VIEW_SHADOW_COPY SYSRES_CONST_ACTION_TYPE_WORKFLOW_DESCRIPTION_MODIFY SYSRES_CONST_ACTION_TYPE_WRITE_HISTORY SYSRES_CONST_ACTIVE_VERSION_STATE_PICK_VALUE SYSRES_CONST_ADD_REFERENCE_MODE_NAME SYSRES_CONST_ADDITION_REQUISITE_CODE SYSRES_CONST_ADDITIONAL_PARAMS_REQUISITE_CODE SYSRES_CONST_ADITIONAL_JOB_END_DATE_REQUISITE_NAME SYSRES_CONST_ADITIONAL_JOB_READ_REQUISITE_NAME SYSRES_CONST_ADITIONAL_JOB_START_DATE_REQUISITE_NAME SYSRES_CONST_ADITIONAL_JOB_STATE_REQUISITE_NAME SYSRES_CONST_ADMINISTRATION_HISTORY_ADDING_USER_TO_GROUP_ACTION SYSRES_CONST_ADMINISTRATION_HISTORY_ADDING_USER_TO_GROUP_ACTION_CODE SYSRES_CONST_ADMINISTRATION_HISTORY_CREATION_COMP_ACTION SYSRES_CONST_ADMINISTRATION_HISTORY_CREATION_COMP_ACTION_CODE SYSRES_CONST_ADMINISTRATION_HISTORY_CREATION_GROUP_ACTION SYSRES_CONST_ADMINISTRATION_HISTORY_CREATION_GROUP_ACTION_CODE SYSRES_CONST_ADMINISTRATION_HISTORY_CREATION_USER_ACTION SYSRES_CONST_ADMINISTRATION_HISTORY_CREATION_USER_ACTION_CODE SYSRES_CONST_ADMINISTRATION_HISTORY_DATABASE_USER_CREATION SYSRES_CONST_ADMINISTRATION_HISTORY_DATABASE_USER_CREATION_ACTION SYSRES_CONST_ADMINISTRATION_HISTORY_DATABASE_USER_DELETION SYSRES_CONST_ADMINISTRATION_HISTORY_DATABASE_USER_DELETION_ACTION SYSRES_CONST_ADMINISTRATION_HISTORY_DELETION_COMP_ACTION SYSRES_CONST_ADMINISTRATION_HISTORY_DELETION_COMP_ACTION_CODE SYSRES_CONST_ADMINISTRATION_HISTORY_DELETION_GROUP_ACTION SYSRES_CONST_ADMINISTRATION_HISTORY_DELETION_GROUP_ACTION_CODE SYSRES_CONST_ADMINISTRATION_HISTORY_DELETION_USER_ACTION SYSRES_CONST_ADMINISTRATION_HISTORY_DELETION_USER_ACTION_CODE SYSRES_CONST_ADMINISTRATION_HISTORY_DELETION_USER_FROM_GROUP_ACTION SYSRES_CONST_ADMINISTRATION_HISTORY_DELETION_USER_FROM_GROUP_ACTION_CODE SYSRES_CONST_ADMINISTRATION_HISTORY_GRANTING_FILTERER_ACTION SYSRES_CONST_ADMINISTRATION_HISTORY_GRANTING_FILTERER_ACTION_CODE SYSRES_CONST_ADMINISTRATION_HISTORY_GRANTING_FILTERER_RESTRICTION_ACTION SYSRES_CONST_ADMINISTRATION_HISTORY_GRANTING_FILTERER_RESTRICTION_ACTION_CODE SYSRES_CONST_ADMINISTRATION_HISTORY_GRANTING_PRIVILEGE_ACTION SYSRES_CONST_ADMINISTRATION_HISTORY_GRANTING_PRIVILEGE_ACTION_CODE SYSRES_CONST_ADMINISTRATION_HISTORY_GRANTING_RIGHTS_ACTION SYSRES_CONST_ADMINISTRATION_HISTORY_GRANTING_RIGHTS_ACTION_CODE SYSRES_CONST_ADMINISTRATION_HISTORY_IS_MAIN_SERVER_CHANGED_ACTION SYSRES_CONST_ADMINISTRATION_HISTORY_IS_MAIN_SERVER_CHANGED_ACTION_CODE SYSRES_CONST_ADMINISTRATION_HISTORY_IS_PUBLIC_CHANGED_ACTION SYSRES_CONST_ADMINISTRATION_HISTORY_IS_PUBLIC_CHANGED_ACTION_CODE SYSRES_CONST_ADMINISTRATION_HISTORY_REMOVING_FILTERER_ACTION SYSRES_CONST_ADMINISTRATION_HISTORY_REMOVING_FILTERER_ACTION_CODE SYSRES_CONST_ADMINISTRATION_HISTORY_REMOVING_FILTERER_RESTRICTION_ACTION SYSRES_CONST_ADMINISTRATION_HISTORY_REMOVING_FILTERER_RESTRICTION_ACTION_CODE SYSRES_CONST_ADMINISTRATION_HISTORY_REMOVING_PRIVILEGE_ACTION SYSRES_CONST_ADMINISTRATION_HISTORY_REMOVING_PRIVILEGE_ACTION_CODE SYSRES_CONST_ADMINISTRATION_HISTORY_REMOVING_RIGHTS_ACTION SYSRES_CONST_ADMINISTRATION_HISTORY_REMOVING_RIGHTS_ACTION_CODE SYSRES_CONST_ADMINISTRATION_HISTORY_SERVER_LOGIN_CREATION SYSRES_CONST_ADMINISTRATION_HISTORY_SERVER_LOGIN_CREATION_ACTION SYSRES_CONST_ADMINISTRATION_HISTORY_SERVER_LOGIN_DELETION SYSRES_CONST_ADMINISTRATION_HISTORY_SERVER_LOGIN_DELETION_ACTION SYSRES_CONST_ADMINISTRATION_HISTORY_UPDATING_CATEGORY_ACTION SYSRES_CONST_ADMINISTRATION_HISTORY_UPDATING_CATEGORY_ACTION_CODE SYSRES_CONST_ADMINISTRATION_HISTORY_UPDATING_COMP_TITLE_ACTION SYSRES_CONST_ADMINISTRATION_HISTORY_UPDATING_COMP_TITLE_ACTION_CODE SYSRES_CONST_ADMINISTRATION_HISTORY_UPDATING_FULL_NAME_ACTION SYSRES_CONST_ADMINISTRATION_HISTORY_UPDATING_FULL_NAME_ACTION_CODE SYSRES_CONST_ADMINISTRATION_HISTORY_UPDATING_GROUP_ACTION SYSRES_CONST_ADMINISTRATION_HISTORY_UPDATING_GROUP_ACTION_CODE SYSRES_CONST_ADMINISTRATION_HISTORY_UPDATING_PARENT_GROUP_ACTION SYSRES_CONST_ADMINISTRATION_HISTORY_UPDATING_PARENT_GROUP_ACTION_CODE SYSRES_CONST_ADMINISTRATION_HISTORY_UPDATING_USER_AUTH_TYPE_ACTION SYSRES_CONST_ADMINISTRATION_HISTORY_UPDATING_USER_AUTH_TYPE_ACTION_CODE SYSRES_CONST_ADMINISTRATION_HISTORY_UPDATING_USER_LOGIN_ACTION SYSRES_CONST_ADMINISTRATION_HISTORY_UPDATING_USER_LOGIN_ACTION_CODE SYSRES_CONST_ADMINISTRATION_HISTORY_UPDATING_USER_STATUS_ACTION SYSRES_CONST_ADMINISTRATION_HISTORY_UPDATING_USER_STATUS_ACTION_CODE SYSRES_CONST_ADMINISTRATION_HISTORY_USER_PASSWORD_CHANGE SYSRES_CONST_ADMINISTRATION_HISTORY_USER_PASSWORD_CHANGE_ACTION SYSRES_CONST_ALL_ACCEPT_CONDITION_RUS SYSRES_CONST_ALL_USERS_GROUP SYSRES_CONST_ALL_USERS_GROUP_NAME SYSRES_CONST_ALL_USERS_SERVER_GROUP_NAME SYSRES_CONST_ALLOWED_ACCESS_TYPE_CODE SYSRES_CONST_ALLOWED_ACCESS_TYPE_NAME SYSRES_CONST_APP_VIEWER_TYPE_REQUISITE_CODE SYSRES_CONST_APPROVING_SIGNATURE_NAME SYSRES_CONST_APPROVING_SIGNATURE_REQUISITE_CODE SYSRES_CONST_ASSISTANT_SUBSTITUE_TYPE SYSRES_CONST_ASSISTANT_SUBSTITUE_TYPE_CODE SYSRES_CONST_ATTACH_TYPE_COMPONENT_TOKEN SYSRES_CONST_ATTACH_TYPE_DOC SYSRES_CONST_ATTACH_TYPE_EDOC SYSRES_CONST_ATTACH_TYPE_FOLDER SYSRES_CONST_ATTACH_TYPE_JOB SYSRES_CONST_ATTACH_TYPE_REFERENCE SYSRES_CONST_ATTACH_TYPE_TASK SYSRES_CONST_AUTH_ENCODED_PASSWORD SYSRES_CONST_AUTH_ENCODED_PASSWORD_CODE SYSRES_CONST_AUTH_NOVELL SYSRES_CONST_AUTH_PASSWORD SYSRES_CONST_AUTH_PASSWORD_CODE SYSRES_CONST_AUTH_WINDOWS SYSRES_CONST_AUTHENTICATING_SIGNATURE_NAME SYSRES_CONST_AUTHENTICATING_SIGNATURE_REQUISITE_CODE SYSRES_CONST_AUTO_ENUM_METHOD_FLAG SYSRES_CONST_AUTO_NUMERATION_CODE SYSRES_CONST_AUTO_STRONG_ENUM_METHOD_FLAG SYSRES_CONST_AUTOTEXT_NAME_REQUISITE_CODE SYSRES_CONST_AUTOTEXT_TEXT_REQUISITE_CODE SYSRES_CONST_AUTOTEXT_USAGE_ALL SYSRES_CONST_AUTOTEXT_USAGE_ALL_CODE SYSRES_CONST_AUTOTEXT_USAGE_SIGN SYSRES_CONST_AUTOTEXT_USAGE_SIGN_CODE SYSRES_CONST_AUTOTEXT_USAGE_WORK SYSRES_CONST_AUTOTEXT_USAGE_WORK_CODE SYSRES_CONST_AUTOTEXT_USE_ANYWHERE_CODE SYSRES_CONST_AUTOTEXT_USE_ON_SIGNING_CODE SYSRES_CONST_AUTOTEXT_USE_ON_WORK_CODE SYSRES_CONST_BEGIN_DATE_REQUISITE_CODE SYSRES_CONST_BLACK_LIFE_CYCLE_STAGE_FONT_COLOR SYSRES_CONST_BLUE_LIFE_CYCLE_STAGE_FONT_COLOR SYSRES_CONST_BTN_PART SYSRES_CONST_CALCULATED_ROLE_TYPE_CODE SYSRES_CONST_CALL_TYPE_VARIABLE_BUTTON_VALUE SYSRES_CONST_CALL_TYPE_VARIABLE_PROGRAM_VALUE SYSRES_CONST_CANCEL_MESSAGE_FUNCTION_RESULT SYSRES_CONST_CARD_PART SYSRES_CONST_CARD_REFERENCE_MODE_NAME SYSRES_CONST_CERTIFICATE_TYPE_REQUISITE_ENCRYPT_VALUE SYSRES_CONST_CERTIFICATE_TYPE_REQUISITE_SIGN_AND_ENCRYPT_VALUE SYSRES_CONST_CERTIFICATE_TYPE_REQUISITE_SIGN_VALUE SYSRES_CONST_CHECK_PARAM_VALUE_DATE_PARAM_TYPE SYSRES_CONST_CHECK_PARAM_VALUE_FLOAT_PARAM_TYPE SYSRES_CONST_CHECK_PARAM_VALUE_INTEGER_PARAM_TYPE SYSRES_CONST_CHECK_PARAM_VALUE_PICK_PARAM_TYPE SYSRES_CONST_CHECK_PARAM_VALUE_REEFRENCE_PARAM_TYPE SYSRES_CONST_CLOSED_RECORD_FLAG_VALUE_FEMININE SYSRES_CONST_CLOSED_RECORD_FLAG_VALUE_MASCULINE SYSRES_CONST_CODE_COMPONENT_TYPE_ADMIN SYSRES_CONST_CODE_COMPONENT_TYPE_DEVELOPER SYSRES_CONST_CODE_COMPONENT_TYPE_DOCS SYSRES_CONST_CODE_COMPONENT_TYPE_EDOC_CARDS SYSRES_CONST_CODE_COMPONENT_TYPE_EXTERNAL_EXECUTABLE SYSRES_CONST_CODE_COMPONENT_TYPE_OTHER SYSRES_CONST_CODE_COMPONENT_TYPE_REFERENCE SYSRES_CONST_CODE_COMPONENT_TYPE_REPORT SYSRES_CONST_CODE_COMPONENT_TYPE_SCRIPT SYSRES_CONST_CODE_COMPONENT_TYPE_URL SYSRES_CONST_CODE_REQUISITE_ACCESS SYSRES_CONST_CODE_REQUISITE_CODE SYSRES_CONST_CODE_REQUISITE_COMPONENT SYSRES_CONST_CODE_REQUISITE_DESCRIPTION SYSRES_CONST_CODE_REQUISITE_EXCLUDE_COMPONENT SYSRES_CONST_CODE_REQUISITE_RECORD SYSRES_CONST_COMMENT_REQ_CODE SYSRES_CONST_COMMON_SETTINGS_REQUISITE_CODE SYSRES_CONST_COMP_CODE_GRD SYSRES_CONST_COMPONENT_GROUP_TYPE_REQUISITE_CODE SYSRES_CONST_COMPONENT_TYPE_ADMIN_COMPONENTS SYSRES_CONST_COMPONENT_TYPE_DEVELOPER_COMPONENTS SYSRES_CONST_COMPONENT_TYPE_DOCS SYSRES_CONST_COMPONENT_TYPE_EDOC_CARDS SYSRES_CONST_COMPONENT_TYPE_EDOCS SYSRES_CONST_COMPONENT_TYPE_EXTERNAL_EXECUTABLE SYSRES_CONST_COMPONENT_TYPE_OTHER SYSRES_CONST_COMPONENT_TYPE_REFERENCE_TYPES SYSRES_CONST_COMPONENT_TYPE_REFERENCES SYSRES_CONST_COMPONENT_TYPE_REPORTS SYSRES_CONST_COMPONENT_TYPE_SCRIPTS SYSRES_CONST_COMPONENT_TYPE_URL SYSRES_CONST_COMPONENTS_REMOTE_SERVERS_VIEW_CODE SYSRES_CONST_CONDITION_BLOCK_DESCRIPTION SYSRES_CONST_CONST_FIRM_STATUS_COMMON SYSRES_CONST_CONST_FIRM_STATUS_INDIVIDUAL SYSRES_CONST_CONST_NEGATIVE_VALUE SYSRES_CONST_CONST_POSITIVE_VALUE SYSRES_CONST_CONST_SERVER_STATUS_DONT_REPLICATE SYSRES_CONST_CONST_SERVER_STATUS_REPLICATE SYSRES_CONST_CONTENTS_REQUISITE_CODE SYSRES_CONST_DATA_TYPE_BOOLEAN SYSRES_CONST_DATA_TYPE_DATE SYSRES_CONST_DATA_TYPE_FLOAT SYSRES_CONST_DATA_TYPE_INTEGER SYSRES_CONST_DATA_TYPE_PICK SYSRES_CONST_DATA_TYPE_REFERENCE SYSRES_CONST_DATA_TYPE_STRING SYSRES_CONST_DATA_TYPE_TEXT SYSRES_CONST_DATA_TYPE_VARIANT SYSRES_CONST_DATE_CLOSE_REQ_CODE SYSRES_CONST_DATE_FORMAT_DATE_ONLY_CHAR SYSRES_CONST_DATE_OPEN_REQ_CODE SYSRES_CONST_DATE_REQUISITE SYSRES_CONST_DATE_REQUISITE_CODE SYSRES_CONST_DATE_REQUISITE_NAME SYSRES_CONST_DATE_REQUISITE_TYPE SYSRES_CONST_DATE_TYPE_CHAR SYSRES_CONST_DATETIME_FORMAT_VALUE SYSRES_CONST_DEA_ACCESS_RIGHTS_ACTION_CODE SYSRES_CONST_DESCRIPTION_LOCALIZE_ID_REQUISITE_CODE SYSRES_CONST_DESCRIPTION_REQUISITE_CODE SYSRES_CONST_DET1_PART SYSRES_CONST_DET2_PART SYSRES_CONST_DET3_PART SYSRES_CONST_DET4_PART SYSRES_CONST_DET5_PART SYSRES_CONST_DET6_PART SYSRES_CONST_DETAIL_DATASET_KEY_REQUISITE_CODE SYSRES_CONST_DETAIL_PICK_REQUISITE_CODE SYSRES_CONST_DETAIL_REQ_CODE SYSRES_CONST_DO_NOT_USE_ACCESS_TYPE_CODE SYSRES_CONST_DO_NOT_USE_ACCESS_TYPE_NAME SYSRES_CONST_DO_NOT_USE_ON_VIEW_ACCESS_TYPE_CODE SYSRES_CONST_DO_NOT_USE_ON_VIEW_ACCESS_TYPE_NAME SYSRES_CONST_DOCUMENT_STORAGES_CODE SYSRES_CONST_DOCUMENT_TEMPLATES_TYPE_NAME SYSRES_CONST_DOUBLE_REQUISITE_CODE SYSRES_CONST_EDITOR_CLOSE_FILE_OBSERV_TYPE_CODE SYSRES_CONST_EDITOR_CLOSE_PROCESS_OBSERV_TYPE_CODE SYSRES_CONST_EDITOR_TYPE_REQUISITE_CODE SYSRES_CONST_EDITORS_APPLICATION_NAME_REQUISITE_CODE SYSRES_CONST_EDITORS_CREATE_SEVERAL_PROCESSES_REQUISITE_CODE SYSRES_CONST_EDITORS_EXTENSION_REQUISITE_CODE SYSRES_CONST_EDITORS_OBSERVER_BY_PROCESS_TYPE SYSRES_CONST_EDITORS_REFERENCE_CODE SYSRES_CONST_EDITORS_REPLACE_SPEC_CHARS_REQUISITE_CODE SYSRES_CONST_EDITORS_USE_PLUGINS_REQUISITE_CODE SYSRES_CONST_EDITORS_VIEW_DOCUMENT_OPENED_TO_EDIT_CODE SYSRES_CONST_EDOC_CARD_TYPE_REQUISITE_CODE SYSRES_CONST_EDOC_CARD_TYPES_LINK_REQUISITE_CODE SYSRES_CONST_EDOC_CERTIFICATE_AND_PASSWORD_ENCODE_CODE SYSRES_CONST_EDOC_CERTIFICATE_ENCODE_CODE SYSRES_CONST_EDOC_DATE_REQUISITE_CODE SYSRES_CONST_EDOC_KIND_REFERENCE_CODE SYSRES_CONST_EDOC_KINDS_BY_TEMPLATE_ACTION_CODE SYSRES_CONST_EDOC_MANAGE_ACCESS_CODE SYSRES_CONST_EDOC_NONE_ENCODE_CODE SYSRES_CONST_EDOC_NUMBER_REQUISITE_CODE SYSRES_CONST_EDOC_PASSWORD_ENCODE_CODE SYSRES_CONST_EDOC_READONLY_ACCESS_CODE SYSRES_CONST_EDOC_SHELL_LIFE_TYPE_VIEW_VALUE SYSRES_CONST_EDOC_SIZE_RESTRICTION_PRIORITY_REQUISITE_CODE SYSRES_CONST_EDOC_STORAGE_CHECK_ACCESS_RIGHTS_REQUISITE_CODE SYSRES_CONST_EDOC_STORAGE_COMPUTER_NAME_REQUISITE_CODE SYSRES_CONST_EDOC_STORAGE_DATABASE_NAME_REQUISITE_CODE SYSRES_CONST_EDOC_STORAGE_EDIT_IN_STORAGE_REQUISITE_CODE SYSRES_CONST_EDOC_STORAGE_LOCAL_PATH_REQUISITE_CODE SYSRES_CONST_EDOC_STORAGE_SHARED_SOURCE_NAME_REQUISITE_CODE SYSRES_CONST_EDOC_TEMPLATE_REQUISITE_CODE SYSRES_CONST_EDOC_TYPES_REFERENCE_CODE SYSRES_CONST_EDOC_VERSION_ACTIVE_STAGE_CODE SYSRES_CONST_EDOC_VERSION_DESIGN_STAGE_CODE SYSRES_CONST_EDOC_VERSION_OBSOLETE_STAGE_CODE SYSRES_CONST_EDOC_WRITE_ACCES_CODE SYSRES_CONST_EDOCUMENT_CARD_REQUISITES_REFERENCE_CODE_SELECTED_REQUISITE SYSRES_CONST_ENCODE_CERTIFICATE_TYPE_CODE SYSRES_CONST_END_DATE_REQUISITE_CODE SYSRES_CONST_ENUMERATION_TYPE_REQUISITE_CODE SYSRES_CONST_EXECUTE_ACCESS_RIGHTS_TYPE_CODE SYSRES_CONST_EXECUTIVE_FILE_STORAGE_TYPE SYSRES_CONST_EXIST_CONST SYSRES_CONST_EXIST_VALUE SYSRES_CONST_EXPORT_LOCK_TYPE_ASK SYSRES_CONST_EXPORT_LOCK_TYPE_WITH_LOCK SYSRES_CONST_EXPORT_LOCK_TYPE_WITHOUT_LOCK SYSRES_CONST_EXPORT_VERSION_TYPE_ASK SYSRES_CONST_EXPORT_VERSION_TYPE_LAST SYSRES_CONST_EXPORT_VERSION_TYPE_LAST_ACTIVE SYSRES_CONST_EXTENSION_REQUISITE_CODE SYSRES_CONST_FILTER_NAME_REQUISITE_CODE SYSRES_CONST_FILTER_REQUISITE_CODE SYSRES_CONST_FILTER_TYPE_COMMON_CODE SYSRES_CONST_FILTER_TYPE_COMMON_NAME SYSRES_CONST_FILTER_TYPE_USER_CODE SYSRES_CONST_FILTER_TYPE_USER_NAME SYSRES_CONST_FILTER_VALUE_REQUISITE_NAME SYSRES_CONST_FLOAT_NUMBER_FORMAT_CHAR SYSRES_CONST_FLOAT_REQUISITE_TYPE SYSRES_CONST_FOLDER_AUTHOR_VALUE SYSRES_CONST_FOLDER_KIND_ANY_OBJECTS SYSRES_CONST_FOLDER_KIND_COMPONENTS SYSRES_CONST_FOLDER_KIND_EDOCS SYSRES_CONST_FOLDER_KIND_JOBS SYSRES_CONST_FOLDER_KIND_TASKS SYSRES_CONST_FOLDER_TYPE_COMMON SYSRES_CONST_FOLDER_TYPE_COMPONENT SYSRES_CONST_FOLDER_TYPE_FAVORITES SYSRES_CONST_FOLDER_TYPE_INBOX SYSRES_CONST_FOLDER_TYPE_OUTBOX SYSRES_CONST_FOLDER_TYPE_QUICK_LAUNCH SYSRES_CONST_FOLDER_TYPE_SEARCH SYSRES_CONST_FOLDER_TYPE_SHORTCUTS SYSRES_CONST_FOLDER_TYPE_USER SYSRES_CONST_FROM_DICTIONARY_ENUM_METHOD_FLAG SYSRES_CONST_FULL_SUBSTITUTE_TYPE SYSRES_CONST_FULL_SUBSTITUTE_TYPE_CODE SYSRES_CONST_FUNCTION_CANCEL_RESULT SYSRES_CONST_FUNCTION_CATEGORY_SYSTEM SYSRES_CONST_FUNCTION_CATEGORY_USER SYSRES_CONST_FUNCTION_FAILURE_RESULT SYSRES_CONST_FUNCTION_SAVE_RESULT SYSRES_CONST_GENERATED_REQUISITE SYSRES_CONST_GREEN_LIFE_CYCLE_STAGE_FONT_COLOR SYSRES_CONST_GROUP_ACCOUNT_TYPE_VALUE_CODE SYSRES_CONST_GROUP_CATEGORY_NORMAL_CODE SYSRES_CONST_GROUP_CATEGORY_NORMAL_NAME SYSRES_CONST_GROUP_CATEGORY_SERVICE_CODE SYSRES_CONST_GROUP_CATEGORY_SERVICE_NAME SYSRES_CONST_GROUP_COMMON_CATEGORY_FIELD_VALUE SYSRES_CONST_GROUP_FULL_NAME_REQUISITE_CODE SYSRES_CONST_GROUP_NAME_REQUISITE_CODE SYSRES_CONST_GROUP_RIGHTS_T_REQUISITE_CODE SYSRES_CONST_GROUP_SERVER_CODES_REQUISITE_CODE SYSRES_CONST_GROUP_SERVER_NAME_REQUISITE_CODE SYSRES_CONST_GROUP_SERVICE_CATEGORY_FIELD_VALUE SYSRES_CONST_GROUP_USER_REQUISITE_CODE SYSRES_CONST_GROUPS_REFERENCE_CODE SYSRES_CONST_GROUPS_REQUISITE_CODE SYSRES_CONST_HIDDEN_MODE_NAME SYSRES_CONST_HIGH_LVL_REQUISITE_CODE SYSRES_CONST_HISTORY_ACTION_CREATE_CODE SYSRES_CONST_HISTORY_ACTION_DELETE_CODE SYSRES_CONST_HISTORY_ACTION_EDIT_CODE SYSRES_CONST_HOUR_CHAR SYSRES_CONST_ID_REQUISITE_CODE SYSRES_CONST_IDSPS_REQUISITE_CODE SYSRES_CONST_IMAGE_MODE_COLOR SYSRES_CONST_IMAGE_MODE_GREYSCALE SYSRES_CONST_IMAGE_MODE_MONOCHROME SYSRES_CONST_IMPORTANCE_HIGH SYSRES_CONST_IMPORTANCE_LOW SYSRES_CONST_IMPORTANCE_NORMAL SYSRES_CONST_IN_DESIGN_VERSION_STATE_PICK_VALUE SYSRES_CONST_INCOMING_WORK_RULE_TYPE_CODE SYSRES_CONST_INT_REQUISITE SYSRES_CONST_INT_REQUISITE_TYPE SYSRES_CONST_INTEGER_NUMBER_FORMAT_CHAR SYSRES_CONST_INTEGER_TYPE_CHAR SYSRES_CONST_IS_GENERATED_REQUISITE_NEGATIVE_VALUE SYSRES_CONST_IS_PUBLIC_ROLE_REQUISITE_CODE SYSRES_CONST_IS_REMOTE_USER_NEGATIVE_VALUE SYSRES_CONST_IS_REMOTE_USER_POSITIVE_VALUE SYSRES_CONST_IS_STORED_REQUISITE_NEGATIVE_VALUE SYSRES_CONST_IS_STORED_REQUISITE_STORED_VALUE SYSRES_CONST_ITALIC_LIFE_CYCLE_STAGE_DRAW_STYLE SYSRES_CONST_JOB_BLOCK_DESCRIPTION SYSRES_CONST_JOB_KIND_CONTROL_JOB SYSRES_CONST_JOB_KIND_JOB SYSRES_CONST_JOB_KIND_NOTICE SYSRES_CONST_JOB_STATE_ABORTED SYSRES_CONST_JOB_STATE_COMPLETE SYSRES_CONST_JOB_STATE_WORKING SYSRES_CONST_KIND_REQUISITE_CODE SYSRES_CONST_KIND_REQUISITE_NAME SYSRES_CONST_KINDS_CREATE_SHADOW_COPIES_REQUISITE_CODE SYSRES_CONST_KINDS_DEFAULT_EDOC_LIFE_STAGE_REQUISITE_CODE SYSRES_CONST_KINDS_EDOC_ALL_TEPLATES_ALLOWED_REQUISITE_CODE SYSRES_CONST_KINDS_EDOC_ALLOW_LIFE_CYCLE_STAGE_CHANGING_REQUISITE_CODE SYSRES_CONST_KINDS_EDOC_ALLOW_MULTIPLE_ACTIVE_VERSIONS_REQUISITE_CODE SYSRES_CONST_KINDS_EDOC_SHARE_ACCES_RIGHTS_BY_DEFAULT_CODE SYSRES_CONST_KINDS_EDOC_TEMPLATE_REQUISITE_CODE SYSRES_CONST_KINDS_EDOC_TYPE_REQUISITE_CODE SYSRES_CONST_KINDS_SIGNERS_REQUISITES_CODE SYSRES_CONST_KOD_INPUT_TYPE SYSRES_CONST_LAST_UPDATE_DATE_REQUISITE_CODE SYSRES_CONST_LIFE_CYCLE_START_STAGE_REQUISITE_CODE SYSRES_CONST_LILAC_LIFE_CYCLE_STAGE_FONT_COLOR SYSRES_CONST_LINK_OBJECT_KIND_COMPONENT SYSRES_CONST_LINK_OBJECT_KIND_DOCUMENT SYSRES_CONST_LINK_OBJECT_KIND_EDOC SYSRES_CONST_LINK_OBJECT_KIND_FOLDER SYSRES_CONST_LINK_OBJECT_KIND_JOB SYSRES_CONST_LINK_OBJECT_KIND_REFERENCE SYSRES_CONST_LINK_OBJECT_KIND_TASK SYSRES_CONST_LINK_REF_TYPE_REQUISITE_CODE SYSRES_CONST_LIST_REFERENCE_MODE_NAME SYSRES_CONST_LOCALIZATION_DICTIONARY_MAIN_VIEW_CODE SYSRES_CONST_MAIN_VIEW_CODE SYSRES_CONST_MANUAL_ENUM_METHOD_FLAG SYSRES_CONST_MASTER_COMP_TYPE_REQUISITE_CODE SYSRES_CONST_MASTER_TABLE_REC_ID_REQUISITE_CODE SYSRES_CONST_MAXIMIZED_MODE_NAME SYSRES_CONST_ME_VALUE SYSRES_CONST_MESSAGE_ATTENTION_CAPTION SYSRES_CONST_MESSAGE_CONFIRMATION_CAPTION SYSRES_CONST_MESSAGE_ERROR_CAPTION SYSRES_CONST_MESSAGE_INFORMATION_CAPTION SYSRES_CONST_MINIMIZED_MODE_NAME SYSRES_CONST_MINUTE_CHAR SYSRES_CONST_MODULE_REQUISITE_CODE SYSRES_CONST_MONITORING_BLOCK_DESCRIPTION SYSRES_CONST_MONTH_FORMAT_VALUE SYSRES_CONST_NAME_LOCALIZE_ID_REQUISITE_CODE SYSRES_CONST_NAME_REQUISITE_CODE SYSRES_CONST_NAME_SINGULAR_REQUISITE_CODE SYSRES_CONST_NAMEAN_INPUT_TYPE SYSRES_CONST_NEGATIVE_PICK_VALUE SYSRES_CONST_NEGATIVE_VALUE SYSRES_CONST_NO SYSRES_CONST_NO_PICK_VALUE SYSRES_CONST_NO_SIGNATURE_REQUISITE_CODE SYSRES_CONST_NO_VALUE SYSRES_CONST_NONE_ACCESS_RIGHTS_TYPE_CODE SYSRES_CONST_NONOPERATING_RECORD_FLAG_VALUE SYSRES_CONST_NONOPERATING_RECORD_FLAG_VALUE_MASCULINE SYSRES_CONST_NORMAL_ACCESS_RIGHTS_TYPE_CODE SYSRES_CONST_NORMAL_LIFE_CYCLE_STAGE_DRAW_STYLE SYSRES_CONST_NORMAL_MODE_NAME SYSRES_CONST_NOT_ALLOWED_ACCESS_TYPE_CODE SYSRES_CONST_NOT_ALLOWED_ACCESS_TYPE_NAME SYSRES_CONST_NOTE_REQUISITE_CODE SYSRES_CONST_NOTICE_BLOCK_DESCRIPTION SYSRES_CONST_NUM_REQUISITE SYSRES_CONST_NUM_STR_REQUISITE_CODE SYSRES_CONST_NUMERATION_AUTO_NOT_STRONG SYSRES_CONST_NUMERATION_AUTO_STRONG SYSRES_CONST_NUMERATION_FROM_DICTONARY SYSRES_CONST_NUMERATION_MANUAL SYSRES_CONST_NUMERIC_TYPE_CHAR SYSRES_CONST_NUMREQ_REQUISITE_CODE SYSRES_CONST_OBSOLETE_VERSION_STATE_PICK_VALUE SYSRES_CONST_OPERATING_RECORD_FLAG_VALUE SYSRES_CONST_OPERATING_RECORD_FLAG_VALUE_CODE SYSRES_CONST_OPERATING_RECORD_FLAG_VALUE_FEMININE SYSRES_CONST_OPERATING_RECORD_FLAG_VALUE_MASCULINE SYSRES_CONST_OPTIONAL_FORM_COMP_REQCODE_PREFIX SYSRES_CONST_ORANGE_LIFE_CYCLE_STAGE_FONT_COLOR SYSRES_CONST_ORIGINALREF_REQUISITE_CODE SYSRES_CONST_OURFIRM_REF_CODE SYSRES_CONST_OURFIRM_REQUISITE_CODE SYSRES_CONST_OURFIRM_VAR SYSRES_CONST_OUTGOING_WORK_RULE_TYPE_CODE SYSRES_CONST_PICK_NEGATIVE_RESULT SYSRES_CONST_PICK_POSITIVE_RESULT SYSRES_CONST_PICK_REQUISITE SYSRES_CONST_PICK_REQUISITE_TYPE SYSRES_CONST_PICK_TYPE_CHAR SYSRES_CONST_PLAN_STATUS_REQUISITE_CODE SYSRES_CONST_PLATFORM_VERSION_COMMENT SYSRES_CONST_PLUGINS_SETTINGS_DESCRIPTION_REQUISITE_CODE SYSRES_CONST_POSITIVE_PICK_VALUE SYSRES_CONST_POWER_TO_CREATE_ACTION_CODE SYSRES_CONST_POWER_TO_SIGN_ACTION_CODE SYSRES_CONST_PRIORITY_REQUISITE_CODE SYSRES_CONST_QUALIFIED_TASK_TYPE SYSRES_CONST_QUALIFIED_TASK_TYPE_CODE SYSRES_CONST_RECSTAT_REQUISITE_CODE SYSRES_CONST_RED_LIFE_CYCLE_STAGE_FONT_COLOR SYSRES_CONST_REF_ID_T_REF_TYPE_REQUISITE_CODE SYSRES_CONST_REF_REQUISITE SYSRES_CONST_REF_REQUISITE_TYPE SYSRES_CONST_REF_REQUISITES_REFERENCE_CODE_SELECTED_REQUISITE SYSRES_CONST_REFERENCE_RECORD_HISTORY_CREATE_ACTION_CODE SYSRES_CONST_REFERENCE_RECORD_HISTORY_DELETE_ACTION_CODE SYSRES_CONST_REFERENCE_RECORD_HISTORY_MODIFY_ACTION_CODE SYSRES_CONST_REFERENCE_TYPE_CHAR SYSRES_CONST_REFERENCE_TYPE_REQUISITE_NAME SYSRES_CONST_REFERENCES_ADD_PARAMS_REQUISITE_CODE SYSRES_CONST_REFERENCES_DISPLAY_REQUISITE_REQUISITE_CODE SYSRES_CONST_REMOTE_SERVER_STATUS_WORKING SYSRES_CONST_REMOTE_SERVER_TYPE_MAIN SYSRES_CONST_REMOTE_SERVER_TYPE_SECONDARY SYSRES_CONST_REMOTE_USER_FLAG_VALUE_CODE SYSRES_CONST_REPORT_APP_EDITOR_INTERNAL SYSRES_CONST_REPORT_BASE_REPORT_ID_REQUISITE_CODE SYSRES_CONST_REPORT_BASE_REPORT_REQUISITE_CODE SYSRES_CONST_REPORT_SCRIPT_REQUISITE_CODE SYSRES_CONST_REPORT_TEMPLATE_REQUISITE_CODE SYSRES_CONST_REPORT_VIEWER_CODE_REQUISITE_CODE SYSRES_CONST_REQ_ALLOW_COMPONENT_DEFAULT_VALUE SYSRES_CONST_REQ_ALLOW_RECORD_DEFAULT_VALUE SYSRES_CONST_REQ_ALLOW_SERVER_COMPONENT_DEFAULT_VALUE SYSRES_CONST_REQ_MODE_AVAILABLE_CODE SYSRES_CONST_REQ_MODE_EDIT_CODE SYSRES_CONST_REQ_MODE_HIDDEN_CODE SYSRES_CONST_REQ_MODE_NOT_AVAILABLE_CODE SYSRES_CONST_REQ_MODE_VIEW_CODE SYSRES_CONST_REQ_NUMBER_REQUISITE_CODE SYSRES_CONST_REQ_SECTION_VALUE SYSRES_CONST_REQ_TYPE_VALUE SYSRES_CONST_REQUISITE_FORMAT_BY_UNIT SYSRES_CONST_REQUISITE_FORMAT_DATE_FULL SYSRES_CONST_REQUISITE_FORMAT_DATE_TIME SYSRES_CONST_REQUISITE_FORMAT_LEFT SYSRES_CONST_REQUISITE_FORMAT_RIGHT SYSRES_CONST_REQUISITE_FORMAT_WITHOUT_UNIT SYSRES_CONST_REQUISITE_NUMBER_REQUISITE_CODE SYSRES_CONST_REQUISITE_SECTION_ACTIONS SYSRES_CONST_REQUISITE_SECTION_BUTTON SYSRES_CONST_REQUISITE_SECTION_BUTTONS SYSRES_CONST_REQUISITE_SECTION_CARD SYSRES_CONST_REQUISITE_SECTION_TABLE SYSRES_CONST_REQUISITE_SECTION_TABLE10 SYSRES_CONST_REQUISITE_SECTION_TABLE11 SYSRES_CONST_REQUISITE_SECTION_TABLE12 SYSRES_CONST_REQUISITE_SECTION_TABLE13 SYSRES_CONST_REQUISITE_SECTION_TABLE14 SYSRES_CONST_REQUISITE_SECTION_TABLE15 SYSRES_CONST_REQUISITE_SECTION_TABLE16 SYSRES_CONST_REQUISITE_SECTION_TABLE17 SYSRES_CONST_REQUISITE_SECTION_TABLE18 SYSRES_CONST_REQUISITE_SECTION_TABLE19 SYSRES_CONST_REQUISITE_SECTION_TABLE2 SYSRES_CONST_REQUISITE_SECTION_TABLE20 SYSRES_CONST_REQUISITE_SECTION_TABLE21 SYSRES_CONST_REQUISITE_SECTION_TABLE22 SYSRES_CONST_REQUISITE_SECTION_TABLE23 SYSRES_CONST_REQUISITE_SECTION_TABLE24 SYSRES_CONST_REQUISITE_SECTION_TABLE3 SYSRES_CONST_REQUISITE_SECTION_TABLE4 SYSRES_CONST_REQUISITE_SECTION_TABLE5 SYSRES_CONST_REQUISITE_SECTION_TABLE6 SYSRES_CONST_REQUISITE_SECTION_TABLE7 SYSRES_CONST_REQUISITE_SECTION_TABLE8 SYSRES_CONST_REQUISITE_SECTION_TABLE9 SYSRES_CONST_REQUISITES_PSEUDOREFERENCE_REQUISITE_NUMBER_REQUISITE_CODE SYSRES_CONST_RIGHT_ALIGNMENT_CODE SYSRES_CONST_ROLES_REFERENCE_CODE SYSRES_CONST_ROUTE_STEP_AFTER_RUS SYSRES_CONST_ROUTE_STEP_AND_CONDITION_RUS SYSRES_CONST_ROUTE_STEP_OR_CONDITION_RUS SYSRES_CONST_ROUTE_TYPE_COMPLEX SYSRES_CONST_ROUTE_TYPE_PARALLEL SYSRES_CONST_ROUTE_TYPE_SERIAL SYSRES_CONST_SBDATASETDESC_NEGATIVE_VALUE SYSRES_CONST_SBDATASETDESC_POSITIVE_VALUE SYSRES_CONST_SBVIEWSDESC_POSITIVE_VALUE SYSRES_CONST_SCRIPT_BLOCK_DESCRIPTION SYSRES_CONST_SEARCH_BY_TEXT_REQUISITE_CODE SYSRES_CONST_SEARCHES_COMPONENT_CONTENT SYSRES_CONST_SEARCHES_CRITERIA_ACTION_NAME SYSRES_CONST_SEARCHES_EDOC_CONTENT SYSRES_CONST_SEARCHES_FOLDER_CONTENT SYSRES_CONST_SEARCHES_JOB_CONTENT SYSRES_CONST_SEARCHES_REFERENCE_CODE SYSRES_CONST_SEARCHES_TASK_CONTENT SYSRES_CONST_SECOND_CHAR SYSRES_CONST_SECTION_REQUISITE_ACTIONS_VALUE SYSRES_CONST_SECTION_REQUISITE_CARD_VALUE SYSRES_CONST_SECTION_REQUISITE_CODE SYSRES_CONST_SECTION_REQUISITE_DETAIL_1_VALUE SYSRES_CONST_SECTION_REQUISITE_DETAIL_2_VALUE SYSRES_CONST_SECTION_REQUISITE_DETAIL_3_VALUE SYSRES_CONST_SECTION_REQUISITE_DETAIL_4_VALUE SYSRES_CONST_SECTION_REQUISITE_DETAIL_5_VALUE SYSRES_CONST_SECTION_REQUISITE_DETAIL_6_VALUE SYSRES_CONST_SELECT_REFERENCE_MODE_NAME SYSRES_CONST_SELECT_TYPE_SELECTABLE SYSRES_CONST_SELECT_TYPE_SELECTABLE_ONLY_CHILD SYSRES_CONST_SELECT_TYPE_SELECTABLE_WITH_CHILD SYSRES_CONST_SELECT_TYPE_UNSLECTABLE SYSRES_CONST_SERVER_TYPE_MAIN SYSRES_CONST_SERVICE_USER_CATEGORY_FIELD_VALUE SYSRES_CONST_SETTINGS_USER_REQUISITE_CODE SYSRES_CONST_SIGNATURE_AND_ENCODE_CERTIFICATE_TYPE_CODE SYSRES_CONST_SIGNATURE_CERTIFICATE_TYPE_CODE SYSRES_CONST_SINGULAR_TITLE_REQUISITE_CODE SYSRES_CONST_SQL_SERVER_AUTHENTIFICATION_FLAG_VALUE_CODE SYSRES_CONST_SQL_SERVER_ENCODE_AUTHENTIFICATION_FLAG_VALUE_CODE SYSRES_CONST_STANDART_ROUTE_REFERENCE_CODE SYSRES_CONST_STANDART_ROUTE_REFERENCE_COMMENT_REQUISITE_CODE SYSRES_CONST_STANDART_ROUTES_GROUPS_REFERENCE_CODE SYSRES_CONST_STATE_REQ_NAME SYSRES_CONST_STATE_REQUISITE_ACTIVE_VALUE SYSRES_CONST_STATE_REQUISITE_CLOSED_VALUE SYSRES_CONST_STATE_REQUISITE_CODE SYSRES_CONST_STATIC_ROLE_TYPE_CODE SYSRES_CONST_STATUS_PLAN_DEFAULT_VALUE SYSRES_CONST_STATUS_VALUE_AUTOCLEANING SYSRES_CONST_STATUS_VALUE_BLUE_SQUARE SYSRES_CONST_STATUS_VALUE_COMPLETE SYSRES_CONST_STATUS_VALUE_GREEN_SQUARE SYSRES_CONST_STATUS_VALUE_ORANGE_SQUARE SYSRES_CONST_STATUS_VALUE_PURPLE_SQUARE SYSRES_CONST_STATUS_VALUE_RED_SQUARE SYSRES_CONST_STATUS_VALUE_SUSPEND SYSRES_CONST_STATUS_VALUE_YELLOW_SQUARE SYSRES_CONST_STDROUTE_SHOW_TO_USERS_REQUISITE_CODE SYSRES_CONST_STORAGE_TYPE_FILE SYSRES_CONST_STORAGE_TYPE_SQL_SERVER SYSRES_CONST_STR_REQUISITE SYSRES_CONST_STRIKEOUT_LIFE_CYCLE_STAGE_DRAW_STYLE SYSRES_CONST_STRING_FORMAT_LEFT_ALIGN_CHAR SYSRES_CONST_STRING_FORMAT_RIGHT_ALIGN_CHAR SYSRES_CONST_STRING_REQUISITE_CODE SYSRES_CONST_STRING_REQUISITE_TYPE SYSRES_CONST_STRING_TYPE_CHAR SYSRES_CONST_SUBSTITUTES_PSEUDOREFERENCE_CODE SYSRES_CONST_SUBTASK_BLOCK_DESCRIPTION SYSRES_CONST_SYSTEM_SETTING_CURRENT_USER_PARAM_VALUE SYSRES_CONST_SYSTEM_SETTING_EMPTY_VALUE_PARAM_VALUE SYSRES_CONST_SYSTEM_VERSION_COMMENT SYSRES_CONST_TASK_ACCESS_TYPE_ALL SYSRES_CONST_TASK_ACCESS_TYPE_ALL_MEMBERS SYSRES_CONST_TASK_ACCESS_TYPE_MANUAL SYSRES_CONST_TASK_ENCODE_TYPE_CERTIFICATION SYSRES_CONST_TASK_ENCODE_TYPE_CERTIFICATION_AND_PASSWORD SYSRES_CONST_TASK_ENCODE_TYPE_NONE SYSRES_CONST_TASK_ENCODE_TYPE_PASSWORD SYSRES_CONST_TASK_ROUTE_ALL_CONDITION SYSRES_CONST_TASK_ROUTE_AND_CONDITION SYSRES_CONST_TASK_ROUTE_OR_CONDITION SYSRES_CONST_TASK_STATE_ABORTED SYSRES_CONST_TASK_STATE_COMPLETE SYSRES_CONST_TASK_STATE_CONTINUED SYSRES_CONST_TASK_STATE_CONTROL SYSRES_CONST_TASK_STATE_INIT SYSRES_CONST_TASK_STATE_WORKING SYSRES_CONST_TASK_TITLE SYSRES_CONST_TASK_TYPES_GROUPS_REFERENCE_CODE SYSRES_CONST_TASK_TYPES_REFERENCE_CODE SYSRES_CONST_TEMPLATES_REFERENCE_CODE SYSRES_CONST_TEST_DATE_REQUISITE_NAME SYSRES_CONST_TEST_DEV_DATABASE_NAME SYSRES_CONST_TEST_DEV_SYSTEM_CODE SYSRES_CONST_TEST_EDMS_DATABASE_NAME SYSRES_CONST_TEST_EDMS_MAIN_CODE SYSRES_CONST_TEST_EDMS_MAIN_DB_NAME SYSRES_CONST_TEST_EDMS_SECOND_CODE SYSRES_CONST_TEST_EDMS_SECOND_DB_NAME SYSRES_CONST_TEST_EDMS_SYSTEM_CODE SYSRES_CONST_TEST_NUMERIC_REQUISITE_NAME SYSRES_CONST_TEXT_REQUISITE SYSRES_CONST_TEXT_REQUISITE_CODE SYSRES_CONST_TEXT_REQUISITE_TYPE SYSRES_CONST_TEXT_TYPE_CHAR SYSRES_CONST_TYPE_CODE_REQUISITE_CODE SYSRES_CONST_TYPE_REQUISITE_CODE SYSRES_CONST_UNDEFINED_LIFE_CYCLE_STAGE_FONT_COLOR SYSRES_CONST_UNITS_SECTION_ID_REQUISITE_CODE SYSRES_CONST_UNITS_SECTION_REQUISITE_CODE SYSRES_CONST_UNOPERATING_RECORD_FLAG_VALUE_CODE SYSRES_CONST_UNSTORED_DATA_REQUISITE_CODE SYSRES_CONST_UNSTORED_DATA_REQUISITE_NAME SYSRES_CONST_USE_ACCESS_TYPE_CODE SYSRES_CONST_USE_ACCESS_TYPE_NAME SYSRES_CONST_USER_ACCOUNT_TYPE_VALUE_CODE SYSRES_CONST_USER_ADDITIONAL_INFORMATION_REQUISITE_CODE SYSRES_CONST_USER_AND_GROUP_ID_FROM_PSEUDOREFERENCE_REQUISITE_CODE SYSRES_CONST_USER_CATEGORY_NORMAL SYSRES_CONST_USER_CERTIFICATE_REQUISITE_CODE SYSRES_CONST_USER_CERTIFICATE_STATE_REQUISITE_CODE SYSRES_CONST_USER_CERTIFICATE_SUBJECT_NAME_REQUISITE_CODE SYSRES_CONST_USER_CERTIFICATE_THUMBPRINT_REQUISITE_CODE SYSRES_CONST_USER_COMMON_CATEGORY SYSRES_CONST_USER_COMMON_CATEGORY_CODE SYSRES_CONST_USER_FULL_NAME_REQUISITE_CODE SYSRES_CONST_USER_GROUP_TYPE_REQUISITE_CODE SYSRES_CONST_USER_LOGIN_REQUISITE_CODE SYSRES_CONST_USER_REMOTE_CONTROLLER_REQUISITE_CODE SYSRES_CONST_USER_REMOTE_SYSTEM_REQUISITE_CODE SYSRES_CONST_USER_RIGHTS_T_REQUISITE_CODE SYSRES_CONST_USER_SERVER_NAME_REQUISITE_CODE SYSRES_CONST_USER_SERVICE_CATEGORY SYSRES_CONST_USER_SERVICE_CATEGORY_CODE SYSRES_CONST_USER_STATUS_ADMINISTRATOR_CODE SYSRES_CONST_USER_STATUS_ADMINISTRATOR_NAME SYSRES_CONST_USER_STATUS_DEVELOPER_CODE SYSRES_CONST_USER_STATUS_DEVELOPER_NAME SYSRES_CONST_USER_STATUS_DISABLED_CODE SYSRES_CONST_USER_STATUS_DISABLED_NAME SYSRES_CONST_USER_STATUS_SYSTEM_DEVELOPER_CODE SYSRES_CONST_USER_STATUS_USER_CODE SYSRES_CONST_USER_STATUS_USER_NAME SYSRES_CONST_USER_STATUS_USER_NAME_DEPRECATED SYSRES_CONST_USER_TYPE_FIELD_VALUE_USER SYSRES_CONST_USER_TYPE_REQUISITE_CODE SYSRES_CONST_USERS_CONTROLLER_REQUISITE_CODE SYSRES_CONST_USERS_IS_MAIN_SERVER_REQUISITE_CODE SYSRES_CONST_USERS_REFERENCE_CODE SYSRES_CONST_USERS_REGISTRATION_CERTIFICATES_ACTION_NAME SYSRES_CONST_USERS_REQUISITE_CODE SYSRES_CONST_USERS_SYSTEM_REQUISITE_CODE SYSRES_CONST_USERS_USER_ACCESS_RIGHTS_TYPR_REQUISITE_CODE SYSRES_CONST_USERS_USER_AUTHENTICATION_REQUISITE_CODE SYSRES_CONST_USERS_USER_COMPONENT_REQUISITE_CODE SYSRES_CONST_USERS_USER_GROUP_REQUISITE_CODE SYSRES_CONST_USERS_VIEW_CERTIFICATES_ACTION_NAME SYSRES_CONST_VIEW_DEFAULT_CODE SYSRES_CONST_VIEW_DEFAULT_NAME SYSRES_CONST_VIEWER_REQUISITE_CODE SYSRES_CONST_WAITING_BLOCK_DESCRIPTION SYSRES_CONST_WIZARD_FORM_LABEL_TEST_STRING SYSRES_CONST_WIZARD_QUERY_PARAM_HEIGHT_ETALON_STRING SYSRES_CONST_WIZARD_REFERENCE_COMMENT_REQUISITE_CODE SYSRES_CONST_WORK_RULES_DESCRIPTION_REQUISITE_CODE SYSRES_CONST_WORK_TIME_CALENDAR_REFERENCE_CODE SYSRES_CONST_WORK_WORKFLOW_HARD_ROUTE_TYPE_VALUE SYSRES_CONST_WORK_WORKFLOW_HARD_ROUTE_TYPE_VALUE_CODE SYSRES_CONST_WORK_WORKFLOW_HARD_ROUTE_TYPE_VALUE_CODE_RUS SYSRES_CONST_WORK_WORKFLOW_SOFT_ROUTE_TYPE_VALUE_CODE_RUS SYSRES_CONST_WORKFLOW_ROUTE_TYPR_HARD SYSRES_CONST_WORKFLOW_ROUTE_TYPR_SOFT SYSRES_CONST_XML_ENCODING SYSRES_CONST_XREC_STAT_REQUISITE_CODE SYSRES_CONST_XRECID_FIELD_NAME SYSRES_CONST_YES SYSRES_CONST_YES_NO_2_REQUISITE_CODE SYSRES_CONST_YES_NO_REQUISITE_CODE SYSRES_CONST_YES_NO_T_REF_TYPE_REQUISITE_CODE SYSRES_CONST_YES_PICK_VALUE SYSRES_CONST_YES_VALUE CR FALSE nil NO_VALUE NULL TAB TRUE YES_VALUE ADMINISTRATORS_GROUP_NAME CUSTOMIZERS_GROUP_NAME DEVELOPERS_GROUP_NAME SERVICE_USERS_GROUP_NAME DECISION_BLOCK_FIRST_OPERAND_PROPERTY DECISION_BLOCK_NAME_PROPERTY DECISION_BLOCK_OPERATION_PROPERTY DECISION_BLOCK_RESULT_TYPE_PROPERTY DECISION_BLOCK_SECOND_OPERAND_PROPERTY ANY_FILE_EXTENTION COMPRESSED_DOCUMENT_EXTENSION EXTENDED_DOCUMENT_EXTENSION SHORT_COMPRESSED_DOCUMENT_EXTENSION SHORT_EXTENDED_DOCUMENT_EXTENSION JOB_BLOCK_ABORT_DEADLINE_PROPERTY JOB_BLOCK_AFTER_FINISH_EVENT JOB_BLOCK_AFTER_QUERY_PARAMETERS_EVENT JOB_BLOCK_ATTACHMENT_PROPERTY JOB_BLOCK_ATTACHMENTS_RIGHTS_GROUP_PROPERTY JOB_BLOCK_ATTACHMENTS_RIGHTS_TYPE_PROPERTY JOB_BLOCK_BEFORE_QUERY_PARAMETERS_EVENT JOB_BLOCK_BEFORE_START_EVENT JOB_BLOCK_CREATED_JOBS_PROPERTY JOB_BLOCK_DEADLINE_PROPERTY JOB_BLOCK_EXECUTION_RESULTS_PROPERTY JOB_BLOCK_IS_PARALLEL_PROPERTY JOB_BLOCK_IS_RELATIVE_ABORT_DEADLINE_PROPERTY JOB_BLOCK_IS_RELATIVE_DEADLINE_PROPERTY JOB_BLOCK_JOB_TEXT_PROPERTY JOB_BLOCK_NAME_PROPERTY JOB_BLOCK_NEED_SIGN_ON_PERFORM_PROPERTY JOB_BLOCK_PERFORMER_PROPERTY JOB_BLOCK_RELATIVE_ABORT_DEADLINE_TYPE_PROPERTY JOB_BLOCK_RELATIVE_DEADLINE_TYPE_PROPERTY JOB_BLOCK_SUBJECT_PROPERTY ENGLISH_LANGUAGE_CODE RUSSIAN_LANGUAGE_CODE smHidden smMaximized smMinimized smNormal wmNo wmYes COMPONENT_TOKEN_LINK_KIND DOCUMENT_LINK_KIND EDOCUMENT_LINK_KIND FOLDER_LINK_KIND JOB_LINK_KIND REFERENCE_LINK_KIND TASK_LINK_KIND COMPONENT_TOKEN_LOCK_TYPE EDOCUMENT_VERSION_LOCK_TYPE MONITOR_BLOCK_AFTER_FINISH_EVENT MONITOR_BLOCK_BEFORE_START_EVENT MONITOR_BLOCK_DEADLINE_PROPERTY MONITOR_BLOCK_INTERVAL_PROPERTY MONITOR_BLOCK_INTERVAL_TYPE_PROPERTY MONITOR_BLOCK_IS_RELATIVE_DEADLINE_PROPERTY MONITOR_BLOCK_NAME_PROPERTY MONITOR_BLOCK_RELATIVE_DEADLINE_TYPE_PROPERTY MONITOR_BLOCK_SEARCH_SCRIPT_PROPERTY NOTICE_BLOCK_AFTER_FINISH_EVENT NOTICE_BLOCK_ATTACHMENT_PROPERTY NOTICE_BLOCK_ATTACHMENTS_RIGHTS_GROUP_PROPERTY NOTICE_BLOCK_ATTACHMENTS_RIGHTS_TYPE_PROPERTY NOTICE_BLOCK_BEFORE_START_EVENT NOTICE_BLOCK_CREATED_NOTICES_PROPERTY NOTICE_BLOCK_DEADLINE_PROPERTY NOTICE_BLOCK_IS_RELATIVE_DEADLINE_PROPERTY NOTICE_BLOCK_NAME_PROPERTY NOTICE_BLOCK_NOTICE_TEXT_PROPERTY NOTICE_BLOCK_PERFORMER_PROPERTY NOTICE_BLOCK_RELATIVE_DEADLINE_TYPE_PROPERTY NOTICE_BLOCK_SUBJECT_PROPERTY dseAfterCancel dseAfterClose dseAfterDelete dseAfterDeleteOutOfTransaction dseAfterInsert dseAfterOpen dseAfterScroll dseAfterUpdate dseAfterUpdateOutOfTransaction dseBeforeCancel dseBeforeClose dseBeforeDelete dseBeforeDetailUpdate dseBeforeInsert dseBeforeOpen dseBeforeUpdate dseOnAnyRequisiteChange dseOnCloseRecord dseOnDeleteError dseOnOpenRecord dseOnPrepareUpdate dseOnUpdateError dseOnUpdateRatifiedRecord dseOnValidDelete dseOnValidUpdate reOnChange reOnChangeValues SELECTION_BEGIN_ROUTE_EVENT SELECTION_END_ROUTE_EVENT CURRENT_PERIOD_IS_REQUIRED PREVIOUS_CARD_TYPE_NAME SHOW_RECORD_PROPERTIES_FORM ACCESS_RIGHTS_SETTING_DIALOG_CODE ADMINISTRATOR_USER_CODE ANALYTIC_REPORT_TYPE asrtHideLocal asrtHideRemote CALCULATED_ROLE_TYPE_CODE COMPONENTS_REFERENCE_DEVELOPER_VIEW_CODE DCTS_TEST_PROTOCOLS_FOLDER_PATH E_EDOC_VERSION_ALREADY_APPROVINGLY_SIGNED E_EDOC_VERSION_ALREADY_APPROVINGLY_SIGNED_BY_USER E_EDOC_VERSION_ALREDY_SIGNED E_EDOC_VERSION_ALREDY_SIGNED_BY_USER EDOC_TYPES_CODE_REQUISITE_FIELD_NAME EDOCUMENTS_ALIAS_NAME FILES_FOLDER_PATH FILTER_OPERANDS_DELIMITER FILTER_OPERATIONS_DELIMITER FORMCARD_NAME FORMLIST_NAME GET_EXTENDED_DOCUMENT_EXTENSION_CREATION_MODE GET_EXTENDED_DOCUMENT_EXTENSION_IMPORT_MODE INTEGRATED_REPORT_TYPE IS_BUILDER_APPLICATION_ROLE IS_BUILDER_APPLICATION_ROLE2 IS_BUILDER_USERS ISBSYSDEV LOG_FOLDER_PATH mbCancel mbNo mbNoToAll mbOK mbYes mbYesToAll MEMORY_DATASET_DESRIPTIONS_FILENAME mrNo mrNoToAll mrYes mrYesToAll MULTIPLE_SELECT_DIALOG_CODE NONOPERATING_RECORD_FLAG_FEMININE NONOPERATING_RECORD_FLAG_MASCULINE OPERATING_RECORD_FLAG_FEMININE OPERATING_RECORD_FLAG_MASCULINE PROFILING_SETTINGS_COMMON_SETTINGS_CODE_VALUE PROGRAM_INITIATED_LOOKUP_ACTION ratDelete ratEdit ratInsert REPORT_TYPE REQUIRED_PICK_VALUES_VARIABLE rmCard rmList SBRTE_PROGID_DEV SBRTE_PROGID_RELEASE STATIC_ROLE_TYPE_CODE SUPPRESS_EMPTY_TEMPLATE_CREATION SYSTEM_USER_CODE UPDATE_DIALOG_DATASET USED_IN_OBJECT_HINT_PARAM USER_INITIATED_LOOKUP_ACTION USER_NAME_FORMAT USER_SELECTION_RESTRICTIONS WORKFLOW_TEST_PROTOCOLS_FOLDER_PATH ELS_SUBTYPE_CONTROL_NAME ELS_FOLDER_KIND_CONTROL_NAME REPEAT_PROCESS_CURRENT_OBJECT_EXCEPTION_NAME PRIVILEGE_COMPONENT_FULL_ACCESS PRIVILEGE_DEVELOPMENT_EXPORT PRIVILEGE_DEVELOPMENT_IMPORT PRIVILEGE_DOCUMENT_DELETE PRIVILEGE_ESD PRIVILEGE_FOLDER_DELETE PRIVILEGE_MANAGE_ACCESS_RIGHTS PRIVILEGE_MANAGE_REPLICATION PRIVILEGE_MANAGE_SESSION_SERVER PRIVILEGE_OBJECT_FULL_ACCESS PRIVILEGE_OBJECT_VIEW PRIVILEGE_RESERVE_LICENSE PRIVILEGE_SYSTEM_CUSTOMIZE PRIVILEGE_SYSTEM_DEVELOP PRIVILEGE_SYSTEM_INSTALL PRIVILEGE_TASK_DELETE PRIVILEGE_USER_PLUGIN_SETTINGS_CUSTOMIZE PRIVILEGES_PSEUDOREFERENCE_CODE ACCESS_TYPES_PSEUDOREFERENCE_CODE ALL_AVAILABLE_COMPONENTS_PSEUDOREFERENCE_CODE ALL_AVAILABLE_PRIVILEGES_PSEUDOREFERENCE_CODE ALL_REPLICATE_COMPONENTS_PSEUDOREFERENCE_CODE AVAILABLE_DEVELOPERS_COMPONENTS_PSEUDOREFERENCE_CODE COMPONENTS_PSEUDOREFERENCE_CODE FILTRATER_SETTINGS_CONFLICTS_PSEUDOREFERENCE_CODE GROUPS_PSEUDOREFERENCE_CODE RECEIVE_PROTOCOL_PSEUDOREFERENCE_CODE REFERENCE_REQUISITE_PSEUDOREFERENCE_CODE REFERENCE_REQUISITES_PSEUDOREFERENCE_CODE REFTYPES_PSEUDOREFERENCE_CODE REPLICATION_SEANCES_DIARY_PSEUDOREFERENCE_CODE SEND_PROTOCOL_PSEUDOREFERENCE_CODE SUBSTITUTES_PSEUDOREFERENCE_CODE SYSTEM_SETTINGS_PSEUDOREFERENCE_CODE UNITS_PSEUDOREFERENCE_CODE USERS_PSEUDOREFERENCE_CODE VIEWERS_PSEUDOREFERENCE_CODE CERTIFICATE_TYPE_ENCRYPT CERTIFICATE_TYPE_SIGN CERTIFICATE_TYPE_SIGN_AND_ENCRYPT STORAGE_TYPE_FILE STORAGE_TYPE_NAS_CIFS STORAGE_TYPE_SAPERION STORAGE_TYPE_SQL_SERVER COMPTYPE2_REQUISITE_DOCUMENTS_VALUE COMPTYPE2_REQUISITE_TASKS_VALUE COMPTYPE2_REQUISITE_FOLDERS_VALUE COMPTYPE2_REQUISITE_REFERENCES_VALUE SYSREQ_CODE SYSREQ_COMPTYPE2 SYSREQ_CONST_AVAILABLE_FOR_WEB SYSREQ_CONST_COMMON_CODE SYSREQ_CONST_COMMON_VALUE SYSREQ_CONST_FIRM_CODE SYSREQ_CONST_FIRM_STATUS SYSREQ_CONST_FIRM_VALUE SYSREQ_CONST_SERVER_STATUS SYSREQ_CONTENTS SYSREQ_DATE_OPEN SYSREQ_DATE_CLOSE SYSREQ_DESCRIPTION SYSREQ_DESCRIPTION_LOCALIZE_ID SYSREQ_DOUBLE SYSREQ_EDOC_ACCESS_TYPE SYSREQ_EDOC_AUTHOR SYSREQ_EDOC_CREATED SYSREQ_EDOC_DELEGATE_RIGHTS_REQUISITE_CODE SYSREQ_EDOC_EDITOR SYSREQ_EDOC_ENCODE_TYPE SYSREQ_EDOC_ENCRYPTION_PLUGIN_NAME SYSREQ_EDOC_ENCRYPTION_PLUGIN_VERSION SYSREQ_EDOC_EXPORT_DATE SYSREQ_EDOC_EXPORTER SYSREQ_EDOC_KIND SYSREQ_EDOC_LIFE_STAGE_NAME SYSREQ_EDOC_LOCKED_FOR_SERVER_CODE SYSREQ_EDOC_MODIFIED SYSREQ_EDOC_NAME SYSREQ_EDOC_NOTE SYSREQ_EDOC_QUALIFIED_ID SYSREQ_EDOC_SESSION_KEY SYSREQ_EDOC_SESSION_KEY_ENCRYPTION_PLUGIN_NAME SYSREQ_EDOC_SESSION_KEY_ENCRYPTION_PLUGIN_VERSION SYSREQ_EDOC_SIGNATURE_TYPE SYSREQ_EDOC_SIGNED SYSREQ_EDOC_STORAGE SYSREQ_EDOC_STORAGES_ARCHIVE_STORAGE SYSREQ_EDOC_STORAGES_CHECK_RIGHTS SYSREQ_EDOC_STORAGES_COMPUTER_NAME SYSREQ_EDOC_STORAGES_EDIT_IN_STORAGE SYSREQ_EDOC_STORAGES_EXECUTIVE_STORAGE SYSREQ_EDOC_STORAGES_FUNCTION SYSREQ_EDOC_STORAGES_INITIALIZED SYSREQ_EDOC_STORAGES_LOCAL_PATH SYSREQ_EDOC_STORAGES_SAPERION_DATABASE_NAME SYSREQ_EDOC_STORAGES_SEARCH_BY_TEXT SYSREQ_EDOC_STORAGES_SERVER_NAME SYSREQ_EDOC_STORAGES_SHARED_SOURCE_NAME SYSREQ_EDOC_STORAGES_TYPE SYSREQ_EDOC_TEXT_MODIFIED SYSREQ_EDOC_TYPE_ACT_CODE SYSREQ_EDOC_TYPE_ACT_DESCRIPTION SYSREQ_EDOC_TYPE_ACT_DESCRIPTION_LOCALIZE_ID SYSREQ_EDOC_TYPE_ACT_ON_EXECUTE SYSREQ_EDOC_TYPE_ACT_ON_EXECUTE_EXISTS SYSREQ_EDOC_TYPE_ACT_SECTION SYSREQ_EDOC_TYPE_ADD_PARAMS SYSREQ_EDOC_TYPE_COMMENT SYSREQ_EDOC_TYPE_EVENT_TEXT SYSREQ_EDOC_TYPE_NAME_IN_SINGULAR SYSREQ_EDOC_TYPE_NAME_IN_SINGULAR_LOCALIZE_ID SYSREQ_EDOC_TYPE_NAME_LOCALIZE_ID SYSREQ_EDOC_TYPE_NUMERATION_METHOD SYSREQ_EDOC_TYPE_PSEUDO_REQUISITE_CODE SYSREQ_EDOC_TYPE_REQ_CODE SYSREQ_EDOC_TYPE_REQ_DESCRIPTION SYSREQ_EDOC_TYPE_REQ_DESCRIPTION_LOCALIZE_ID SYSREQ_EDOC_TYPE_REQ_IS_LEADING SYSREQ_EDOC_TYPE_REQ_IS_REQUIRED SYSREQ_EDOC_TYPE_REQ_NUMBER SYSREQ_EDOC_TYPE_REQ_ON_CHANGE SYSREQ_EDOC_TYPE_REQ_ON_CHANGE_EXISTS SYSREQ_EDOC_TYPE_REQ_ON_SELECT SYSREQ_EDOC_TYPE_REQ_ON_SELECT_KIND SYSREQ_EDOC_TYPE_REQ_SECTION SYSREQ_EDOC_TYPE_VIEW_CARD SYSREQ_EDOC_TYPE_VIEW_CODE SYSREQ_EDOC_TYPE_VIEW_COMMENT SYSREQ_EDOC_TYPE_VIEW_IS_MAIN SYSREQ_EDOC_TYPE_VIEW_NAME SYSREQ_EDOC_TYPE_VIEW_NAME_LOCALIZE_ID SYSREQ_EDOC_VERSION_AUTHOR SYSREQ_EDOC_VERSION_CRC SYSREQ_EDOC_VERSION_DATA SYSREQ_EDOC_VERSION_EDITOR SYSREQ_EDOC_VERSION_EXPORT_DATE SYSREQ_EDOC_VERSION_EXPORTER SYSREQ_EDOC_VERSION_HIDDEN SYSREQ_EDOC_VERSION_LIFE_STAGE SYSREQ_EDOC_VERSION_MODIFIED SYSREQ_EDOC_VERSION_NOTE SYSREQ_EDOC_VERSION_SIGNATURE_TYPE SYSREQ_EDOC_VERSION_SIGNED SYSREQ_EDOC_VERSION_SIZE SYSREQ_EDOC_VERSION_SOURCE SYSREQ_EDOC_VERSION_TEXT_MODIFIED SYSREQ_EDOCKIND_DEFAULT_VERSION_STATE_CODE SYSREQ_FOLDER_KIND SYSREQ_FUNC_CATEGORY SYSREQ_FUNC_COMMENT SYSREQ_FUNC_GROUP SYSREQ_FUNC_GROUP_COMMENT SYSREQ_FUNC_GROUP_NUMBER SYSREQ_FUNC_HELP SYSREQ_FUNC_PARAM_DEF_VALUE SYSREQ_FUNC_PARAM_IDENT SYSREQ_FUNC_PARAM_NUMBER SYSREQ_FUNC_PARAM_TYPE SYSREQ_FUNC_TEXT SYSREQ_GROUP_CATEGORY SYSREQ_ID SYSREQ_LAST_UPDATE SYSREQ_LEADER_REFERENCE SYSREQ_LINE_NUMBER SYSREQ_MAIN_RECORD_ID SYSREQ_NAME SYSREQ_NAME_LOCALIZE_ID SYSREQ_NOTE SYSREQ_ORIGINAL_RECORD SYSREQ_OUR_FIRM SYSREQ_PROFILING_SETTINGS_BATCH_LOGING SYSREQ_PROFILING_SETTINGS_BATCH_SIZE SYSREQ_PROFILING_SETTINGS_PROFILING_ENABLED SYSREQ_PROFILING_SETTINGS_SQL_PROFILING_ENABLED SYSREQ_PROFILING_SETTINGS_START_LOGGED SYSREQ_RECORD_STATUS SYSREQ_REF_REQ_FIELD_NAME SYSREQ_REF_REQ_FORMAT SYSREQ_REF_REQ_GENERATED SYSREQ_REF_REQ_LENGTH SYSREQ_REF_REQ_PRECISION SYSREQ_REF_REQ_REFERENCE SYSREQ_REF_REQ_SECTION SYSREQ_REF_REQ_STORED SYSREQ_REF_REQ_TOKENS SYSREQ_REF_REQ_TYPE SYSREQ_REF_REQ_VIEW SYSREQ_REF_TYPE_ACT_CODE SYSREQ_REF_TYPE_ACT_DESCRIPTION SYSREQ_REF_TYPE_ACT_DESCRIPTION_LOCALIZE_ID SYSREQ_REF_TYPE_ACT_ON_EXECUTE SYSREQ_REF_TYPE_ACT_ON_EXECUTE_EXISTS SYSREQ_REF_TYPE_ACT_SECTION SYSREQ_REF_TYPE_ADD_PARAMS SYSREQ_REF_TYPE_COMMENT SYSREQ_REF_TYPE_COMMON_SETTINGS SYSREQ_REF_TYPE_DISPLAY_REQUISITE_NAME SYSREQ_REF_TYPE_EVENT_TEXT SYSREQ_REF_TYPE_MAIN_LEADING_REF SYSREQ_REF_TYPE_NAME_IN_SINGULAR SYSREQ_REF_TYPE_NAME_IN_SINGULAR_LOCALIZE_ID SYSREQ_REF_TYPE_NAME_LOCALIZE_ID SYSREQ_REF_TYPE_NUMERATION_METHOD SYSREQ_REF_TYPE_REQ_CODE SYSREQ_REF_TYPE_REQ_DESCRIPTION SYSREQ_REF_TYPE_REQ_DESCRIPTION_LOCALIZE_ID SYSREQ_REF_TYPE_REQ_IS_CONTROL SYSREQ_REF_TYPE_REQ_IS_FILTER SYSREQ_REF_TYPE_REQ_IS_LEADING SYSREQ_REF_TYPE_REQ_IS_REQUIRED SYSREQ_REF_TYPE_REQ_NUMBER SYSREQ_REF_TYPE_REQ_ON_CHANGE SYSREQ_REF_TYPE_REQ_ON_CHANGE_EXISTS SYSREQ_REF_TYPE_REQ_ON_SELECT SYSREQ_REF_TYPE_REQ_ON_SELECT_KIND SYSREQ_REF_TYPE_REQ_SECTION SYSREQ_REF_TYPE_VIEW_CARD SYSREQ_REF_TYPE_VIEW_CODE SYSREQ_REF_TYPE_VIEW_COMMENT SYSREQ_REF_TYPE_VIEW_IS_MAIN SYSREQ_REF_TYPE_VIEW_NAME SYSREQ_REF_TYPE_VIEW_NAME_LOCALIZE_ID SYSREQ_REFERENCE_TYPE_ID SYSREQ_STATE SYSREQ_STATЕ SYSREQ_SYSTEM_SETTINGS_VALUE SYSREQ_TYPE SYSREQ_UNIT SYSREQ_UNIT_ID SYSREQ_USER_GROUPS_GROUP_FULL_NAME SYSREQ_USER_GROUPS_GROUP_NAME SYSREQ_USER_GROUPS_GROUP_SERVER_NAME SYSREQ_USERS_ACCESS_RIGHTS SYSREQ_USERS_AUTHENTICATION SYSREQ_USERS_CATEGORY SYSREQ_USERS_COMPONENT SYSREQ_USERS_COMPONENT_USER_IS_PUBLIC SYSREQ_USERS_DOMAIN SYSREQ_USERS_FULL_USER_NAME SYSREQ_USERS_GROUP SYSREQ_USERS_IS_MAIN_SERVER SYSREQ_USERS_LOGIN SYSREQ_USERS_REFERENCE_USER_IS_PUBLIC SYSREQ_USERS_STATUS SYSREQ_USERS_USER_CERTIFICATE SYSREQ_USERS_USER_CERTIFICATE_INFO SYSREQ_USERS_USER_CERTIFICATE_PLUGIN_NAME SYSREQ_USERS_USER_CERTIFICATE_PLUGIN_VERSION SYSREQ_USERS_USER_CERTIFICATE_STATE SYSREQ_USERS_USER_CERTIFICATE_SUBJECT_NAME SYSREQ_USERS_USER_CERTIFICATE_THUMBPRINT SYSREQ_USERS_USER_DEFAULT_CERTIFICATE SYSREQ_USERS_USER_DESCRIPTION SYSREQ_USERS_USER_GLOBAL_NAME SYSREQ_USERS_USER_LOGIN SYSREQ_USERS_USER_MAIN_SERVER SYSREQ_USERS_USER_TYPE SYSREQ_WORK_RULES_FOLDER_ID RESULT_VAR_NAME RESULT_VAR_NAME_ENG AUTO_NUMERATION_RULE_ID CANT_CHANGE_ID_REQUISITE_RULE_ID CANT_CHANGE_OURFIRM_REQUISITE_RULE_ID CHECK_CHANGING_REFERENCE_RECORD_USE_RULE_ID CHECK_CODE_REQUISITE_RULE_ID CHECK_DELETING_REFERENCE_RECORD_USE_RULE_ID CHECK_FILTRATER_CHANGES_RULE_ID CHECK_RECORD_INTERVAL_RULE_ID CHECK_REFERENCE_INTERVAL_RULE_ID CHECK_REQUIRED_DATA_FULLNESS_RULE_ID CHECK_REQUIRED_REQUISITES_FULLNESS_RULE_ID MAKE_RECORD_UNRATIFIED_RULE_ID RESTORE_AUTO_NUMERATION_RULE_ID SET_FIRM_CONTEXT_FROM_RECORD_RULE_ID SET_FIRST_RECORD_IN_LIST_FORM_RULE_ID SET_IDSPS_VALUE_RULE_ID SET_NEXT_CODE_VALUE_RULE_ID SET_OURFIRM_BOUNDS_RULE_ID SET_OURFIRM_REQUISITE_RULE_ID SCRIPT_BLOCK_AFTER_FINISH_EVENT SCRIPT_BLOCK_BEFORE_START_EVENT SCRIPT_BLOCK_EXECUTION_RESULTS_PROPERTY SCRIPT_BLOCK_NAME_PROPERTY SCRIPT_BLOCK_SCRIPT_PROPERTY SUBTASK_BLOCK_ABORT_DEADLINE_PROPERTY SUBTASK_BLOCK_AFTER_FINISH_EVENT SUBTASK_BLOCK_ASSIGN_PARAMS_EVENT SUBTASK_BLOCK_ATTACHMENTS_PROPERTY SUBTASK_BLOCK_ATTACHMENTS_RIGHTS_GROUP_PROPERTY SUBTASK_BLOCK_ATTACHMENTS_RIGHTS_TYPE_PROPERTY SUBTASK_BLOCK_BEFORE_START_EVENT SUBTASK_BLOCK_CREATED_TASK_PROPERTY SUBTASK_BLOCK_CREATION_EVENT SUBTASK_BLOCK_DEADLINE_PROPERTY SUBTASK_BLOCK_IMPORTANCE_PROPERTY SUBTASK_BLOCK_INITIATOR_PROPERTY SUBTASK_BLOCK_IS_RELATIVE_ABORT_DEADLINE_PROPERTY SUBTASK_BLOCK_IS_RELATIVE_DEADLINE_PROPERTY SUBTASK_BLOCK_JOBS_TYPE_PROPERTY SUBTASK_BLOCK_NAME_PROPERTY SUBTASK_BLOCK_PARALLEL_ROUTE_PROPERTY SUBTASK_BLOCK_PERFORMERS_PROPERTY SUBTASK_BLOCK_RELATIVE_ABORT_DEADLINE_TYPE_PROPERTY SUBTASK_BLOCK_RELATIVE_DEADLINE_TYPE_PROPERTY SUBTASK_BLOCK_REQUIRE_SIGN_PROPERTY SUBTASK_BLOCK_STANDARD_ROUTE_PROPERTY SUBTASK_BLOCK_START_EVENT SUBTASK_BLOCK_STEP_CONTROL_PROPERTY SUBTASK_BLOCK_SUBJECT_PROPERTY SUBTASK_BLOCK_TASK_CONTROL_PROPERTY SUBTASK_BLOCK_TEXT_PROPERTY SUBTASK_BLOCK_UNLOCK_ATTACHMENTS_ON_STOP_PROPERTY SUBTASK_BLOCK_USE_STANDARD_ROUTE_PROPERTY SUBTASK_BLOCK_WAIT_FOR_TASK_COMPLETE_PROPERTY SYSCOMP_CONTROL_JOBS SYSCOMP_FOLDERS SYSCOMP_JOBS SYSCOMP_NOTICES SYSCOMP_TASKS SYSDLG_CREATE_EDOCUMENT SYSDLG_CREATE_EDOCUMENT_VERSION SYSDLG_CURRENT_PERIOD SYSDLG_EDIT_FUNCTION_HELP SYSDLG_EDOCUMENT_KINDS_FOR_TEMPLATE SYSDLG_EXPORT_MULTIPLE_EDOCUMENTS SYSDLG_EXPORT_SINGLE_EDOCUMENT SYSDLG_IMPORT_EDOCUMENT SYSDLG_MULTIPLE_SELECT SYSDLG_SETUP_ACCESS_RIGHTS SYSDLG_SETUP_DEFAULT_RIGHTS SYSDLG_SETUP_FILTER_CONDITION SYSDLG_SETUP_SIGN_RIGHTS SYSDLG_SETUP_TASK_OBSERVERS SYSDLG_SETUP_TASK_ROUTE SYSDLG_SETUP_USERS_LIST SYSDLG_SIGN_EDOCUMENT SYSDLG_SIGN_MULTIPLE_EDOCUMENTS SYSREF_ACCESS_RIGHTS_TYPES SYSREF_ADMINISTRATION_HISTORY SYSREF_ALL_AVAILABLE_COMPONENTS SYSREF_ALL_AVAILABLE_PRIVILEGES SYSREF_ALL_REPLICATING_COMPONENTS SYSREF_AVAILABLE_DEVELOPERS_COMPONENTS SYSREF_CALENDAR_EVENTS SYSREF_COMPONENT_TOKEN_HISTORY SYSREF_COMPONENT_TOKENS SYSREF_COMPONENTS SYSREF_CONSTANTS SYSREF_DATA_RECEIVE_PROTOCOL SYSREF_DATA_SEND_PROTOCOL SYSREF_DIALOGS SYSREF_DIALOGS_REQUISITES SYSREF_EDITORS SYSREF_EDOC_CARDS SYSREF_EDOC_TYPES SYSREF_EDOCUMENT_CARD_REQUISITES SYSREF_EDOCUMENT_CARD_TYPES SYSREF_EDOCUMENT_CARD_TYPES_REFERENCE SYSREF_EDOCUMENT_CARDS SYSREF_EDOCUMENT_HISTORY SYSREF_EDOCUMENT_KINDS SYSREF_EDOCUMENT_REQUISITES SYSREF_EDOCUMENT_SIGNATURES SYSREF_EDOCUMENT_TEMPLATES SYSREF_EDOCUMENT_TEXT_STORAGES SYSREF_EDOCUMENT_VIEWS SYSREF_FILTERER_SETUP_CONFLICTS SYSREF_FILTRATER_SETTING_CONFLICTS SYSREF_FOLDER_HISTORY SYSREF_FOLDERS SYSREF_FUNCTION_GROUPS SYSREF_FUNCTION_PARAMS SYSREF_FUNCTIONS SYSREF_JOB_HISTORY SYSREF_LINKS SYSREF_LOCALIZATION_DICTIONARY SYSREF_LOCALIZATION_LANGUAGES SYSREF_MODULES SYSREF_PRIVILEGES SYSREF_RECORD_HISTORY SYSREF_REFERENCE_REQUISITES SYSREF_REFERENCE_TYPE_VIEWS SYSREF_REFERENCE_TYPES SYSREF_REFERENCES SYSREF_REFERENCES_REQUISITES SYSREF_REMOTE_SERVERS SYSREF_REPLICATION_SESSIONS_LOG SYSREF_REPLICATION_SESSIONS_PROTOCOL SYSREF_REPORTS SYSREF_ROLES SYSREF_ROUTE_BLOCK_GROUPS SYSREF_ROUTE_BLOCKS SYSREF_SCRIPTS SYSREF_SEARCHES SYSREF_SERVER_EVENTS SYSREF_SERVER_EVENTS_HISTORY SYSREF_STANDARD_ROUTE_GROUPS SYSREF_STANDARD_ROUTES SYSREF_STATUSES SYSREF_SYSTEM_SETTINGS SYSREF_TASK_HISTORY SYSREF_TASK_KIND_GROUPS SYSREF_TASK_KINDS SYSREF_TASK_RIGHTS SYSREF_TASK_SIGNATURES SYSREF_TASKS SYSREF_UNITS SYSREF_USER_GROUPS SYSREF_USER_GROUPS_REFERENCE SYSREF_USER_SUBSTITUTION SYSREF_USERS SYSREF_USERS_REFERENCE SYSREF_VIEWERS SYSREF_WORKING_TIME_CALENDARS ACCESS_RIGHTS_TABLE_NAME EDMS_ACCESS_TABLE_NAME EDOC_TYPES_TABLE_NAME TEST_DEV_DB_NAME TEST_DEV_SYSTEM_CODE TEST_EDMS_DB_NAME TEST_EDMS_MAIN_CODE TEST_EDMS_MAIN_DB_NAME TEST_EDMS_SECOND_CODE TEST_EDMS_SECOND_DB_NAME TEST_EDMS_SYSTEM_CODE TEST_ISB5_MAIN_CODE TEST_ISB5_SECOND_CODE TEST_SQL_SERVER_2005_NAME TEST_SQL_SERVER_NAME ATTENTION_CAPTION cbsCommandLinks cbsDefault CONFIRMATION_CAPTION ERROR_CAPTION INFORMATION_CAPTION mrCancel mrOk EDOC_VERSION_ACTIVE_STAGE_CODE EDOC_VERSION_DESIGN_STAGE_CODE EDOC_VERSION_OBSOLETE_STAGE_CODE cpDataEnciphermentEnabled cpDigitalSignatureEnabled cpID cpIssuer cpPluginVersion cpSerial cpSubjectName cpSubjSimpleName cpValidFromDate cpValidToDate ISBL_SYNTAX NO_SYNTAX XML_SYNTAX WAIT_BLOCK_AFTER_FINISH_EVENT WAIT_BLOCK_BEFORE_START_EVENT WAIT_BLOCK_DEADLINE_PROPERTY WAIT_BLOCK_IS_RELATIVE_DEADLINE_PROPERTY WAIT_BLOCK_NAME_PROPERTY WAIT_BLOCK_RELATIVE_DEADLINE_TYPE_PROPERTY SYSRES_COMMON SYSRES_CONST SYSRES_MBFUNC SYSRES_SBDATA SYSRES_SBGUI SYSRES_SBINTF SYSRES_SBREFDSC SYSRES_SQLERRORS SYSRES_SYSCOMP atUser atGroup atRole aemEnabledAlways aemDisabledAlways aemEnabledOnBrowse aemEnabledOnEdit aemDisabledOnBrowseEmpty apBegin apEnd alLeft alRight asmNever asmNoButCustomize asmAsLastTime asmYesButCustomize asmAlways cirCommon cirRevoked ctSignature ctEncode ctSignatureEncode clbUnchecked clbChecked clbGrayed ceISB ceAlways ceNever ctDocument ctReference ctScript ctUnknown ctReport ctDialog ctFunction ctFolder ctEDocument ctTask ctJob ctNotice ctControlJob cfInternal cfDisplay ciUnspecified ciWrite ciRead ckFolder ckEDocument ckTask ckJob ckComponentToken ckAny ckReference ckScript ckReport ckDialog ctISBLEditor ctBevel ctButton ctCheckListBox ctComboBox ctComboEdit ctGrid ctDBCheckBox ctDBComboBox ctDBEdit ctDBEllipsis ctDBMemo ctDBNavigator ctDBRadioGroup ctDBStatusLabel ctEdit ctGroupBox ctInplaceHint ctMemo ctPanel ctListBox ctRadioButton ctRichEdit ctTabSheet ctWebBrowser ctImage ctHyperLink ctLabel ctDBMultiEllipsis ctRibbon ctRichView ctInnerPanel ctPanelGroup ctBitButton cctDate cctInteger cctNumeric cctPick cctReference cctString cctText cltInternal cltPrimary cltGUI dseBeforeOpen dseAfterOpen dseBeforeClose dseAfterClose dseOnValidDelete dseBeforeDelete dseAfterDelete dseAfterDeleteOutOfTransaction dseOnDeleteError dseBeforeInsert dseAfterInsert dseOnValidUpdate dseBeforeUpdate dseOnUpdateRatifiedRecord dseAfterUpdate dseAfterUpdateOutOfTransaction dseOnUpdateError dseAfterScroll dseOnOpenRecord dseOnCloseRecord dseBeforeCancel dseAfterCancel dseOnUpdateDeadlockError dseBeforeDetailUpdate dseOnPrepareUpdate dseOnAnyRequisiteChange dssEdit dssInsert dssBrowse dssInActive dftDate dftShortDate dftDateTime dftTimeStamp dotDays dotHours dotMinutes dotSeconds dtkndLocal dtkndUTC arNone arView arEdit arFull ddaView ddaEdit emLock emEdit emSign emExportWithLock emImportWithUnlock emChangeVersionNote emOpenForModify emChangeLifeStage emDelete emCreateVersion emImport emUnlockExportedWithLock emStart emAbort emReInit emMarkAsReaded emMarkAsUnreaded emPerform emAccept emResume emChangeRights emEditRoute emEditObserver emRecoveryFromLocalCopy emChangeWorkAccessType emChangeEncodeTypeToCertificate emChangeEncodeTypeToPassword emChangeEncodeTypeToNone emChangeEncodeTypeToCertificatePassword emChangeStandardRoute emGetText emOpenForView emMoveToStorage emCreateObject emChangeVersionHidden emDeleteVersion emChangeLifeCycleStage emApprovingSign emExport emContinue emLockFromEdit emUnLockForEdit emLockForServer emUnlockFromServer emDelegateAccessRights emReEncode ecotFile ecotProcess eaGet eaCopy eaCreate eaCreateStandardRoute edltAll edltNothing edltQuery essmText essmCard esvtLast esvtLastActive esvtSpecified edsfExecutive edsfArchive edstSQLServer edstFile edvstNone edvstEDocumentVersionCopy edvstFile edvstTemplate edvstScannedFile vsDefault vsDesign vsActive vsObsolete etNone etCertificate etPassword etCertificatePassword ecException ecWarning ecInformation estAll estApprovingOnly evtLast evtLastActive evtQuery fdtString fdtNumeric fdtInteger fdtDate fdtText fdtUnknown fdtWideString fdtLargeInteger ftInbox ftOutbox ftFavorites ftCommonFolder ftUserFolder ftComponents ftQuickLaunch ftShortcuts ftSearch grhAuto grhX1 grhX2 grhX3 hltText hltRTF hltHTML iffBMP iffJPEG iffMultiPageTIFF iffSinglePageTIFF iffTIFF iffPNG im8bGrayscale im24bRGB im1bMonochrome itBMP itJPEG itWMF itPNG ikhInformation ikhWarning ikhError ikhNoIcon icUnknown icScript icFunction icIntegratedReport icAnalyticReport icDataSetEventHandler icActionHandler icFormEventHandler icLookUpEventHandler icRequisiteChangeEventHandler icBeforeSearchEventHandler icRoleCalculation icSelectRouteEventHandler icBlockPropertyCalculation icBlockQueryParamsEventHandler icChangeSearchResultEventHandler icBlockEventHandler icSubTaskInitEventHandler icEDocDataSetEventHandler icEDocLookUpEventHandler icEDocActionHandler icEDocFormEventHandler icEDocRequisiteChangeEventHandler icStructuredConversionRule icStructuredConversionEventBefore icStructuredConversionEventAfter icWizardEventHandler icWizardFinishEventHandler icWizardStepEventHandler icWizardStepFinishEventHandler icWizardActionEnableEventHandler icWizardActionExecuteEventHandler icCreateJobsHandler icCreateNoticesHandler icBeforeLookUpEventHandler icAfterLookUpEventHandler icTaskAbortEventHandler icWorkflowBlockActionHandler icDialogDataSetEventHandler icDialogActionHandler icDialogLookUpEventHandler icDialogRequisiteChangeEventHandler icDialogFormEventHandler icDialogValidCloseEventHandler icBlockFormEventHandler icTaskFormEventHandler icReferenceMethod icEDocMethod icDialogMethod icProcessMessageHandler isShow isHide isByUserSettings jkJob jkNotice jkControlJob jtInner jtLeft jtRight jtFull jtCross lbpAbove lbpBelow lbpLeft lbpRight eltPerConnection eltPerUser sfcUndefined sfcBlack sfcGreen sfcRed sfcBlue sfcOrange sfcLilac sfsItalic sfsStrikeout sfsNormal ldctStandardRoute ldctWizard ldctScript ldctFunction ldctRouteBlock ldctIntegratedReport ldctAnalyticReport ldctReferenceType ldctEDocumentType ldctDialog ldctServerEvents mrcrtNone mrcrtUser mrcrtMaximal mrcrtCustom vtEqual vtGreaterOrEqual vtLessOrEqual vtRange rdYesterday rdToday rdTomorrow rdThisWeek rdThisMonth rdThisYear rdNextMonth rdNextWeek rdLastWeek rdLastMonth rdWindow rdFile rdPrinter rdtString rdtNumeric rdtInteger rdtDate rdtReference rdtAccount rdtText rdtPick rdtUnknown rdtLargeInteger rdtDocument reOnChange reOnChangeValues ttGlobal ttLocal ttUser ttSystem ssmBrowse ssmSelect ssmMultiSelect ssmBrowseModal smSelect smLike smCard stNone stAuthenticating stApproving sctString sctStream sstAnsiSort sstNaturalSort svtEqual svtContain soatString soatNumeric soatInteger soatDatetime soatReferenceRecord soatText soatPick soatBoolean soatEDocument soatAccount soatIntegerCollection soatNumericCollection soatStringCollection soatPickCollection soatDatetimeCollection soatBooleanCollection soatReferenceRecordCollection soatEDocumentCollection soatAccountCollection soatContents soatUnknown tarAbortByUser tarAbortByWorkflowException tvtAllWords tvtExactPhrase tvtAnyWord usNone usCompleted usRedSquare usBlueSquare usYellowSquare usGreenSquare usOrangeSquare usPurpleSquare usFollowUp utUnknown utUser utDeveloper utAdministrator utSystemDeveloper utDisconnected btAnd btDetailAnd btOr btNotOr btOnly vmView vmSelect vmNavigation vsmSingle vsmMultiple vsmMultipleCheck vsmNoSelection wfatPrevious wfatNext wfatCancel wfatFinish wfepUndefined wfepText3 wfepText6 wfepText9 wfepSpinEdit wfepDropDown wfepRadioGroup wfepFlag wfepText12 wfepText15 wfepText18 wfepText21 wfepText24 wfepText27 wfepText30 wfepRadioGroupColumn1 wfepRadioGroupColumn2 wfepRadioGroupColumn3 wfetQueryParameter wfetText wfetDelimiter wfetLabel wptString wptInteger wptNumeric wptBoolean wptDateTime wptPick wptText wptUser wptUserList wptEDocumentInfo wptEDocumentInfoList wptReferenceRecordInfo wptReferenceRecordInfoList wptFolderInfo wptTaskInfo wptContents wptFileName wptDate wsrComplete wsrGoNext wsrGoPrevious wsrCustom wsrCancel wsrGoFinal wstForm wstEDocument wstTaskCard wstReferenceRecordCard wstFinal waAll waPerformers waManual wsbStart wsbFinish wsbNotice wsbStep wsbDecision wsbWait wsbMonitor wsbScript wsbConnector wsbSubTask wsbLifeCycleStage wsbPause wdtInteger wdtFloat wdtString wdtPick wdtDateTime wdtBoolean wdtTask wdtJob wdtFolder wdtEDocument wdtReferenceRecord wdtUser wdtGroup wdtRole wdtIntegerCollection wdtFloatCollection wdtStringCollection wdtPickCollection wdtDateTimeCollection wdtBooleanCollection wdtTaskCollection wdtJobCollection wdtFolderCollection wdtEDocumentCollection wdtReferenceRecordCollection wdtUserCollection wdtGroupCollection wdtRoleCollection wdtContents wdtUserList wdtSearchDescription wdtDeadLine wdtPickSet wdtAccountCollection wiLow wiNormal wiHigh wrtSoft wrtHard wsInit wsRunning wsDone wsControlled wsAborted wsContinued wtmFull wtmFromCurrent wtmOnlyCurrent ",class:"AltState Application CallType ComponentTokens CreatedJobs CreatedNotices ControlState DialogResult Dialogs EDocuments EDocumentVersionSource Folders GlobalIDs Job Jobs InputValue LookUpReference LookUpRequisiteNames LookUpSearch Object ParentComponent Processes References Requisite ReportName Reports Result Scripts Searches SelectedAttachments SelectedItems SelectMode Sender ServerEvents ServiceFactory ShiftState SubTask SystemDialogs Tasks Wizard Wizards Work ВызовСпособ ИмяОтчета РеквЗнач ",literal:"null true false nil "},s={begin:"\\.\\s*"+e.UNDERSCORE_IDENT_RE,keywords:o,relevance:0},l={className:"type",begin:":[ \\t]*("+"IApplication IAccessRights IAccountRepository IAccountSelectionRestrictions IAction IActionList IAdministrationHistoryDescription IAnchors IApplication IArchiveInfo IAttachment IAttachmentList ICheckListBox ICheckPointedList IColumn IComponent IComponentDescription IComponentToken IComponentTokenFactory IComponentTokenInfo ICompRecordInfo IConnection IContents IControl IControlJob IControlJobInfo IControlList ICrypto ICrypto2 ICustomJob ICustomJobInfo ICustomListBox ICustomObjectWizardStep ICustomWork ICustomWorkInfo IDataSet IDataSetAccessInfo IDataSigner IDateCriterion IDateRequisite IDateRequisiteDescription IDateValue IDeaAccessRights IDeaObjectInfo IDevelopmentComponentLock IDialog IDialogFactory IDialogPickRequisiteItems IDialogsFactory IDICSFactory IDocRequisite IDocumentInfo IDualListDialog IECertificate IECertificateInfo IECertificates IEditControl IEditorForm IEdmsExplorer IEdmsObject IEdmsObjectDescription IEdmsObjectFactory IEdmsObjectInfo IEDocument IEDocumentAccessRights IEDocumentDescription IEDocumentEditor IEDocumentFactory IEDocumentInfo IEDocumentStorage IEDocumentVersion IEDocumentVersionListDialog IEDocumentVersionSource IEDocumentWizardStep IEDocVerSignature IEDocVersionState IEnabledMode IEncodeProvider IEncrypter IEvent IEventList IException IExternalEvents IExternalHandler IFactory IField IFileDialog IFolder IFolderDescription IFolderDialog IFolderFactory IFolderInfo IForEach IForm IFormTitle IFormWizardStep IGlobalIDFactory IGlobalIDInfo IGrid IHasher IHistoryDescription IHyperLinkControl IImageButton IImageControl IInnerPanel IInplaceHint IIntegerCriterion IIntegerList IIntegerRequisite IIntegerValue IISBLEditorForm IJob IJobDescription IJobFactory IJobForm IJobInfo ILabelControl ILargeIntegerCriterion ILargeIntegerRequisite ILargeIntegerValue ILicenseInfo ILifeCycleStage IList IListBox ILocalIDInfo ILocalization ILock IMemoryDataSet IMessagingFactory IMetadataRepository INotice INoticeInfo INumericCriterion INumericRequisite INumericValue IObject IObjectDescription IObjectImporter IObjectInfo IObserver IPanelGroup IPickCriterion IPickProperty IPickRequisite IPickRequisiteDescription IPickRequisiteItem IPickRequisiteItems IPickValue IPrivilege IPrivilegeList IProcess IProcessFactory IProcessMessage IProgress IProperty IPropertyChangeEvent IQuery IReference IReferenceCriterion IReferenceEnabledMode IReferenceFactory IReferenceHistoryDescription IReferenceInfo IReferenceRecordCardWizardStep IReferenceRequisiteDescription IReferencesFactory IReferenceValue IRefRequisite IReport IReportFactory IRequisite IRequisiteDescription IRequisiteDescriptionList IRequisiteFactory IRichEdit IRouteStep IRule IRuleList ISchemeBlock IScript IScriptFactory ISearchCriteria ISearchCriterion ISearchDescription ISearchFactory ISearchFolderInfo ISearchForObjectDescription ISearchResultRestrictions ISecuredContext ISelectDialog IServerEvent IServerEventFactory IServiceDialog IServiceFactory ISignature ISignProvider ISignProvider2 ISignProvider3 ISimpleCriterion IStringCriterion IStringList IStringRequisite IStringRequisiteDescription IStringValue ISystemDialogsFactory ISystemInfo ITabSheet ITask ITaskAbortReasonInfo ITaskCardWizardStep ITaskDescription ITaskFactory ITaskInfo ITaskRoute ITextCriterion ITextRequisite ITextValue ITreeListSelectDialog IUser IUserList IValue IView IWebBrowserControl IWizard IWizardAction IWizardFactory IWizardFormElement IWizardParam IWizardPickParam IWizardReferenceParam IWizardStep IWorkAccessRights IWorkDescription IWorkflowAskableParam IWorkflowAskableParams IWorkflowBlock IWorkflowBlockResult IWorkflowEnabledMode IWorkflowParam IWorkflowPickParam IWorkflowReferenceParam IWorkState IWorkTreeCustomNode IWorkTreeJobNode IWorkTreeTaskNode IXMLEditorForm SBCrypto ".trim().replace(/\s/g,"|")+")",end:"[ \\t]*=",excludeEnd:!0},c={className:"variable",keywords:o,begin:t,relevance:0,contains:[l,s]},_="[A-Za-zА-Яа-яёЁ_][A-Za-zА-Яа-яёЁ_0-9]*\\(";return{name:"ISBL",case_insensitive:!0,keywords:o,illegal:"\\$|\\?|%|,|;$|~|#|@|({IMPORTANT:{scope:"meta",begin:"!important"},BLOCK_COMMENT:e.C_BLOCK_COMMENT_MODE,HEXCOLOR:{scope:"number",begin:/#(([0-9a-fA-F]{3,4})|(([0-9a-fA-F]{2}){3,4}))\b/},FUNCTION_DISPATCH:{className:"built_in",begin:/[\w-]+(?=\()/},ATTRIBUTE_SELECTOR_MODE:{scope:"selector-attr",begin:/\[/,end:/\]/,illegal:"$",contains:[e.APOS_STRING_MODE,e.QUOTE_STRING_MODE]},CSS_NUMBER_MODE:{scope:"number",begin:e.NUMBER_RE+"(%|em|ex|ch|rem|vw|vh|vmin|vmax|cm|mm|in|pt|pc|px|deg|grad|rad|turn|s|ms|Hz|kHz|dpi|dpcm|dppx)?",relevance:0},CSS_VARIABLE:{className:"attr",begin:/--[A-Za-z_][A-Za-z0-9_-]*/}}))(e),s={className:"variable",begin:"\\$"+e.IDENT_RE},l="(?=[.\\s\\n[:,(])";return{name:"Stylus",aliases:["styl"],case_insensitive:!1,keywords:"if else for in",illegal:"("+["\\?","(\\bReturn\\b)","(\\bEnd\\b)","(\\bend\\b)","(\\bdef\\b)",";","#\\s","\\*\\s","===\\s","\\|","%"].join("|")+")",contains:[e.QUOTE_STRING_MODE,e.APOS_STRING_MODE,e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE,o.HEXCOLOR,{begin:"\\.[a-zA-Z][a-zA-Z0-9_-]*"+l,className:"selector-class"},{begin:"#[a-zA-Z][a-zA-Z0-9_-]*"+l,className:"selector-id"},{begin:"\\b("+t.join("|")+")"+l,className:"selector-tag"},{className:"selector-pseudo",begin:"&?:("+a.join("|")+")"+l},{className:"selector-pseudo",begin:"&?:(:)?("+r.join("|")+")"+l},o.ATTRIBUTE_SELECTOR_MODE,{className:"keyword",begin:/@media/,starts:{end:/[{;}]/,keywords:{$pattern:/[a-z-]+/,keyword:"and or not only",attribute:n.join(" ")},contains:[o.CSS_NUMBER_MODE]}},{className:"keyword",begin:"@((-(o|moz|ms|webkit)-)?("+["charset","css","debug","extend","font-face","for","import","include","keyframes","media","mixin","page","warn","while"].join("|")+"))\\b"},s,o.CSS_NUMBER_MODE,{className:"function",begin:"^[a-zA-Z][a-zA-Z0-9_-]*\\(.*\\)",illegal:"[\\n]",returnBegin:!0,contains:[{className:"title",begin:"\\b[a-zA-Z][a-zA-Z0-9_-]*"},{className:"params",begin:/\(/,end:/\)/,contains:[o.HEXCOLOR,s,e.APOS_STRING_MODE,o.CSS_NUMBER_MODE,e.QUOTE_STRING_MODE]}]},o.CSS_VARIABLE,{className:"attribute",begin:"\\b("+i.join("|")+")\\b",starts:{end:/;|$/,contains:[o.HEXCOLOR,s,e.APOS_STRING_MODE,e.QUOTE_STRING_MODE,o.CSS_NUMBER_MODE,e.C_BLOCK_COMMENT_MODE,o.IMPORTANT,o.FUNCTION_DISPATCH],illegal:/\./,relevance:0}},o.FUNCTION_DISPATCH]}}},5914(e){const t=["as","in","of","if","for","while","finally","var","new","function","do","return","void","else","break","catch","instanceof","with","throw","case","default","try","switch","continue","typeof","delete","let","yield","const","class","debugger","async","await","static","import","from","export","extends","using"],n=["true","false","null","undefined","NaN","Infinity"],a=[].concat(["setInterval","setTimeout","clearInterval","clearTimeout","require","exports","eval","isFinite","isNaN","parseFloat","parseInt","decodeURI","decodeURIComponent","encodeURI","encodeURIComponent","escape","unescape"],["Object","Function","Boolean","Symbol","Math","Date","Number","BigInt","String","RegExp","Array","Float32Array","Float64Array","Int8Array","Uint8Array","Uint8ClampedArray","Int16Array","Int32Array","Uint16Array","Uint32Array","BigInt64Array","BigUint64Array","Set","Map","WeakSet","WeakMap","ArrayBuffer","SharedArrayBuffer","Atomics","DataView","JSON","Promise","Generator","GeneratorFunction","AsyncFunction","Reflect","Proxy","Intl","WebAssembly"],["Error","EvalError","InternalError","RangeError","ReferenceError","SyntaxError","TypeError","URIError"]);e.exports=function(e){const r={keyword:t.concat(["then","unless","until","loop","of","by","when","and","or","is","isnt","not","it","that","otherwise","from","to","til","fallthrough","case","enum","native","list","map","__hasProp","__extends","__slice","__bind","__indexOf"]),literal:n.concat(["yes","no","on","off","it","that","void"]),built_in:a.concat(["npm","print"])},i="[A-Za-z$_](?:-[0-9A-Za-z$_]|[0-9A-Za-z$_])*",o=e.inherit(e.TITLE_MODE,{begin:i}),s={className:"subst",begin:/#\{/,end:/\}/,keywords:r},l={className:"subst",begin:/#[A-Za-z$_]/,end:/(?:-[0-9A-Za-z$_]|[0-9A-Za-z$_])*/,keywords:r},c=[e.BINARY_NUMBER_MODE,{className:"number",begin:"(\\b0[xX][a-fA-F0-9_]+)|(\\b\\d(\\d|_\\d)*(\\.(\\d(\\d|_\\d)*)?)?(_*[eE]([-+]\\d(_\\d|\\d)*)?)?[_a-z]*)",relevance:0,starts:{end:"(\\s*/)?",relevance:0}},{className:"string",variants:[{begin:/'''/,end:/'''/,contains:[e.BACKSLASH_ESCAPE]},{begin:/'/,end:/'/,contains:[e.BACKSLASH_ESCAPE]},{begin:/"""/,end:/"""/,contains:[e.BACKSLASH_ESCAPE,s,l]},{begin:/"/,end:/"/,contains:[e.BACKSLASH_ESCAPE,s,l]},{begin:/\\/,end:/(\s|$)/,excludeEnd:!0}]},{className:"regexp",variants:[{begin:"//",end:"//[gim]*",contains:[s,e.HASH_COMMENT_MODE]},{begin:/\/(?![ *])(\\.|[^\\\n])*?\/[gim]*(?=\W)/}]},{begin:"@"+i},{begin:"``",end:"``",excludeBegin:!0,excludeEnd:!0,subLanguage:"javascript"}];s.contains=c;const _={className:"params",begin:"\\(",returnBegin:!0,contains:[{begin:/\(/,end:/\)/,keywords:r,contains:["self"].concat(c)}]},d={variants:[{match:[/class\s+/,i,/\s+extends\s+/,i]},{match:[/class\s+/,i]}],scope:{2:"title.class",4:"title.class.inherited"},keywords:r};return{name:"LiveScript",aliases:["ls"],keywords:r,illegal:/\/\*/,contains:c.concat([e.COMMENT("\\/\\*","\\*\\/"),e.HASH_COMMENT_MODE,{begin:"(#=>|=>|\\|>>|-?->|!->)"},{className:"function",contains:[o,_],returnBegin:!0,variants:[{begin:"("+i+"\\s*(?:=|:=)\\s*)?(\\(.*\\)\\s*)?\\B->\\*?",end:"->\\*?"},{begin:"("+i+"\\s*(?:=|:=)\\s*)?!?(\\(.*\\)\\s*)?\\B[-~]{1,2}>\\*?",end:"[-~]{1,2}>\\*?"},{begin:"("+i+"\\s*(?:=|:=)\\s*)?(\\(.*\\)\\s*)?\\B!?[-~]{1,2}>\\*?",end:"!?[-~]{1,2}>\\*?"}]},d,{begin:i+":",end:":",returnBegin:!0,returnEnd:!0,relevance:0}])}}},5975(e){e.exports=function(e){const t="[A-Za-zА-Яа-яёЁ_][A-Za-zА-Яа-яёЁ_0-9]+",n="далее возврат вызватьисключение выполнить для если и из или иначе иначеесли исключение каждого конецесли конецпопытки конеццикла не новый перейти перем по пока попытка прервать продолжить тогда цикл экспорт ",a="null истина ложь неопределено",r=e.inherit(e.NUMBER_MODE),i={className:"string",begin:'"|\\|',end:'"|$',contains:[{begin:'""'}]},o={begin:"'",end:"'",excludeBegin:!0,excludeEnd:!0,contains:[{className:"number",begin:"\\d{4}([\\.\\\\/:-]?\\d{2}){0,5}"}]},s=e.inherit(e.C_LINE_COMMENT_MODE);return{name:"1C:Enterprise",case_insensitive:!0,keywords:{$pattern:t,keyword:n,built_in:"разделительстраниц разделительстрок символтабуляции ansitooem oemtoansi ввестивидсубконто ввестиперечисление ввестипериод ввестиплансчетов выбранныйплансчетов датагод датамесяц датачисло заголовоксистемы значениевстроку значениеизстроки каталогиб каталогпользователя кодсимв конгода конецпериодаби конецрассчитанногопериодаби конецстандартногоинтервала конквартала конмесяца коннедели лог лог10 максимальноеколичествосубконто названиеинтерфейса названиенабораправ назначитьвид назначитьсчет найтиссылки началопериодаби началостандартногоинтервала начгода начквартала начмесяца начнедели номерднягода номерднянедели номернеделигода обработкаожидания основнойжурналрасчетов основнойплансчетов основнойязык очиститьокносообщений периодстр получитьвремята получитьдатута получитьдокументта получитьзначенияотбора получитьпозициюта получитьпустоезначение получитьта префиксавтонумерации пропись пустоезначение разм разобратьпозициюдокумента рассчитатьрегистрына рассчитатьрегистрыпо симв создатьобъект статусвозврата стрколичествострок сформироватьпозициюдокумента счетпокоду текущеевремя типзначения типзначениястр установитьтана установитьтапо фиксшаблон шаблон acos asin atan base64значение base64строка cos exp log log10 pow sin sqrt tan xmlзначение xmlстрока xmlтип xmlтипзнч активноеокно безопасныйрежим безопасныйрежимразделенияданных булево ввестидату ввестизначение ввестистроку ввестичисло возможностьчтенияxml вопрос восстановитьзначение врег выгрузитьжурналрегистрации выполнитьобработкуоповещения выполнитьпроверкуправдоступа вычислить год данныеформывзначение дата день деньгода деньнедели добавитьмесяц заблокироватьданныедляредактирования заблокироватьработупользователя завершитьработусистемы загрузитьвнешнююкомпоненту закрытьсправку записатьjson записатьxml записатьдатуjson записьжурналарегистрации заполнитьзначениясвойств запроситьразрешениепользователя запуститьприложение запуститьсистему зафиксироватьтранзакцию значениевданныеформы значениевстрокувнутр значениевфайл значениезаполнено значениеизстрокивнутр значениеизфайла изxmlтипа импортмоделиxdto имякомпьютера имяпользователя инициализироватьпредопределенныеданные информацияобошибке каталогбиблиотекимобильногоустройства каталогвременныхфайлов каталогдокументов каталогпрограммы кодироватьстроку кодлокализацииинформационнойбазы кодсимвола командасистемы конецгода конецдня конецквартала конецмесяца конецминуты конецнедели конецчаса конфигурациябазыданныхизмененадинамически конфигурацияизменена копироватьданныеформы копироватьфайл краткоепредставлениеошибки лев макс местноевремя месяц мин минута монопольныйрежим найти найтинедопустимыесимволыxml найтиокнопонавигационнойссылке найтипомеченныенаудаление найтипоссылкам найтифайлы началогода началодня началоквартала началомесяца началоминуты началонедели началочаса начатьзапросразрешенияпользователя начатьзапускприложения начатькопированиефайла начатьперемещениефайла начатьподключениевнешнейкомпоненты начатьподключениерасширенияработыскриптографией начатьподключениерасширенияработысфайлами начатьпоискфайлов начатьполучениекаталогавременныхфайлов начатьполучениекаталогадокументов начатьполучениерабочегокаталогаданныхпользователя начатьполучениефайлов начатьпомещениефайла начатьпомещениефайлов начатьсозданиедвоичныхданныхизфайла начатьсозданиекаталога начатьтранзакцию начатьудалениефайлов начатьустановкувнешнейкомпоненты начатьустановкурасширенияработыскриптографией начатьустановкурасширенияработысфайлами неделягода необходимостьзавершениясоединения номерсеансаинформационнойбазы номерсоединенияинформационнойбазы нрег нстр обновитьинтерфейс обновитьнумерациюобъектов обновитьповторноиспользуемыезначения обработкапрерыванияпользователя объединитьфайлы окр описаниеошибки оповестить оповеститьобизменении отключитьобработчикзапросанастроекклиенталицензирования отключитьобработчикожидания отключитьобработчикоповещения открытьзначение открытьиндекссправки открытьсодержаниесправки открытьсправку открытьформу открытьформумодально отменитьтранзакцию очиститьжурналрегистрации очиститьнастройкипользователя очиститьсообщения параметрыдоступа перейтипонавигационнойссылке переместитьфайл подключитьвнешнююкомпоненту подключитьобработчикзапросанастроекклиенталицензирования подключитьобработчикожидания подключитьобработчикоповещения подключитьрасширениеработыскриптографией подключитьрасширениеработысфайлами подробноепредставлениеошибки показатьвводдаты показатьвводзначения показатьвводстроки показатьвводчисла показатьвопрос показатьзначение показатьинформациюобошибке показатьнакарте показатьоповещениепользователя показатьпредупреждение полноеимяпользователя получитьcomобъект получитьxmlтип получитьадреспоместоположению получитьблокировкусеансов получитьвремязавершенияспящегосеанса получитьвремязасыпанияпассивногосеанса получитьвремяожиданияблокировкиданных получитьданныевыбора получитьдополнительныйпараметрклиенталицензирования получитьдопустимыекодылокализации получитьдопустимыечасовыепояса получитьзаголовокклиентскогоприложения получитьзаголовоксистемы получитьзначенияотборажурналарегистрации получитьидентификаторконфигурации получитьизвременногохранилища получитьимявременногофайла получитьимяклиенталицензирования получитьинформациюэкрановклиента получитьиспользованиежурналарегистрации получитьиспользованиесобытияжурналарегистрации получитькраткийзаголовокприложения получитьмакетоформления получитьмаскувсефайлы получитьмаскувсефайлыклиента получитьмаскувсефайлысервера получитьместоположениепоадресу получитьминимальнуюдлинупаролейпользователей получитьнавигационнуюссылку получитьнавигационнуюссылкуинформационнойбазы получитьобновлениеконфигурациибазыданных получитьобновлениепредопределенныхданныхинформационнойбазы получитьобщиймакет получитьобщуюформу получитьокна получитьоперативнуюотметкувремени получитьотключениебезопасногорежима получитьпараметрыфункциональныхопцийинтерфейса получитьполноеимяпредопределенногозначения получитьпредставлениянавигационныхссылок получитьпроверкусложностипаролейпользователей получитьразделительпути получитьразделительпутиклиента получитьразделительпутисервера получитьсеансыинформационнойбазы получитьскоростьклиентскогосоединения получитьсоединенияинформационнойбазы получитьсообщенияпользователю получитьсоответствиеобъектаиформы получитьсоставстандартногоинтерфейсаodata получитьструктурухранениябазыданных получитьтекущийсеансинформационнойбазы получитьфайл получитьфайлы получитьформу получитьфункциональнуюопцию получитьфункциональнуюопциюинтерфейса получитьчасовойпоясинформационнойбазы пользователиос поместитьвовременноехранилище поместитьфайл поместитьфайлы прав праводоступа предопределенноезначение представлениекодалокализации представлениепериода представлениеправа представлениеприложения представлениесобытияжурналарегистрации представлениечасовогопояса предупреждение прекратитьработусистемы привилегированныйрежим продолжитьвызов прочитатьjson прочитатьxml прочитатьдатуjson пустаястрока рабочийкаталогданныхпользователя разблокироватьданныедляредактирования разделитьфайл разорватьсоединениесвнешнимисточникомданных раскодироватьстроку рольдоступна секунда сигнал символ скопироватьжурналрегистрации смещениелетнеговремени смещениестандартноговремени соединитьбуферыдвоичныхданных создатькаталог создатьфабрикуxdto сокрл сокрлп сокрп сообщить состояние сохранитьзначение сохранитьнастройкипользователя сред стрдлина стрзаканчиваетсяна стрзаменить стрнайти стрначинаетсяс строка строкасоединенияинформационнойбазы стрполучитьстроку стрразделить стрсоединить стрсравнить стрчисловхождений стрчислострок стршаблон текущаядата текущаядатасеанса текущаяуниверсальнаядата текущаяуниверсальнаядатавмиллисекундах текущийвариантинтерфейсаклиентскогоприложения текущийвариантосновногошрифтаклиентскогоприложения текущийкодлокализации текущийрежимзапуска текущийязык текущийязыксистемы тип типзнч транзакцияактивна трег удалитьданныеинформационнойбазы удалитьизвременногохранилища удалитьобъекты удалитьфайлы универсальноевремя установитьбезопасныйрежим установитьбезопасныйрежимразделенияданных установитьблокировкусеансов установитьвнешнююкомпоненту установитьвремязавершенияспящегосеанса установитьвремязасыпанияпассивногосеанса установитьвремяожиданияблокировкиданных установитьзаголовокклиентскогоприложения установитьзаголовоксистемы установитьиспользованиежурналарегистрации установитьиспользованиесобытияжурналарегистрации установитькраткийзаголовокприложения установитьминимальнуюдлинупаролейпользователей установитьмонопольныйрежим установитьнастройкиклиенталицензирования установитьобновлениепредопределенныхданныхинформационнойбазы установитьотключениебезопасногорежима установитьпараметрыфункциональныхопцийинтерфейса установитьпривилегированныйрежим установитьпроверкусложностипаролейпользователей установитьрасширениеработыскриптографией установитьрасширениеработысфайлами установитьсоединениесвнешнимисточникомданных установитьсоответствиеобъектаиформы установитьсоставстандартногоинтерфейсаodata установитьчасовойпоясинформационнойбазы установитьчасовойпояссеанса формат цел час часовойпояс часовойпояссеанса число числопрописью этоадресвременногохранилища wsссылки библиотекакартинок библиотекамакетовоформлениякомпоновкиданных библиотекастилей бизнеспроцессы внешниеисточникиданных внешниеобработки внешниеотчеты встроенныепокупки главныйинтерфейс главныйстиль документы доставляемыеуведомления журналыдокументов задачи информацияобинтернетсоединении использованиерабочейдаты историяработыпользователя константы критерииотбора метаданные обработки отображениерекламы отправкадоставляемыхуведомлений отчеты панельзадачос параметрзапуска параметрысеанса перечисления планывидоврасчета планывидовхарактеристик планыобмена планысчетов полнотекстовыйпоиск пользователиинформационнойбазы последовательности проверкавстроенныхпокупок рабочаядата расширенияконфигурации регистрыбухгалтерии регистрынакопления регистрырасчета регистрысведений регламентныезадания сериализаторxdto справочники средствагеопозиционирования средствакриптографии средствамультимедиа средстваотображениярекламы средствапочты средствателефонии фабрикаxdto файловыепотоки фоновыезадания хранилищанастроек хранилищевариантовотчетов хранилищенастроекданныхформ хранилищеобщихнастроек хранилищепользовательскихнастроекдинамическихсписков хранилищепользовательскихнастроекотчетов хранилищесистемныхнастроек ",class:"webцвета windowsцвета windowsшрифты библиотекакартинок рамкистиля символы цветастиля шрифтыстиля автоматическоесохранениеданныхформывнастройках автонумерациявформе автораздвижениесерий анимациядиаграммы вариантвыравниванияэлементовизаголовков вариантуправлениявысотойтаблицы вертикальнаяпрокруткаформы вертикальноеположение вертикальноеположениеэлемента видгруппыформы виддекорацииформы виддополненияэлементаформы видизмененияданных видкнопкиформы видпереключателя видподписейкдиаграмме видполяформы видфлажка влияниеразмеранапузырекдиаграммы горизонтальноеположение горизонтальноеположениеэлемента группировкаколонок группировкаподчиненныхэлементовформы группыиэлементы действиеперетаскивания дополнительныйрежимотображения допустимыедействияперетаскивания интервалмеждуэлементамиформы использованиевывода использованиеполосыпрокрутки используемоезначениеточкибиржевойдиаграммы историявыборапривводе источникзначенийоситочекдиаграммы источникзначенияразмерапузырькадиаграммы категориягруппыкоманд максимумсерий начальноеотображениедерева начальноеотображениесписка обновлениетекстаредактирования ориентациядендрограммы ориентациядиаграммы ориентацияметокдиаграммы ориентацияметоксводнойдиаграммы ориентацияэлементаформы отображениевдиаграмме отображениевлегендедиаграммы отображениегруппыкнопок отображениезаголовкашкалыдиаграммы отображениезначенийсводнойдиаграммы отображениезначенияизмерительнойдиаграммы отображениеинтерваладиаграммыганта отображениекнопки отображениекнопкивыбора отображениеобсужденийформы отображениеобычнойгруппы отображениеотрицательныхзначенийпузырьковойдиаграммы отображениепанелипоиска отображениеподсказки отображениепредупрежденияприредактировании отображениеразметкиполосырегулирования отображениестраницформы отображениетаблицы отображениетекстазначениядиаграммыганта отображениеуправленияобычнойгруппы отображениефигурыкнопки палитрацветовдиаграммы поведениеобычнойгруппы поддержкамасштабадендрограммы поддержкамасштабадиаграммыганта поддержкамасштабасводнойдиаграммы поисквтаблицепривводе положениезаголовкаэлементаформы положениекартинкикнопкиформы положениекартинкиэлементаграфическойсхемы положениекоманднойпанелиформы положениекоманднойпанелиэлементаформы положениеопорнойточкиотрисовки положениеподписейкдиаграмме положениеподписейшкалызначенийизмерительнойдиаграммы положениесостоянияпросмотра положениестрокипоиска положениетекстасоединительнойлинии положениеуправленияпоиском положениешкалывремени порядокотображенияточекгоризонтальнойгистограммы порядоксерийвлегендедиаграммы размеркартинки расположениезаголовкашкалыдиаграммы растягиваниеповертикалидиаграммыганта режимавтоотображениясостояния режимвводастроктаблицы режимвыборанезаполненного режимвыделениядаты режимвыделениястрокитаблицы режимвыделениятаблицы режимизмененияразмера режимизменениясвязанногозначения режимиспользованиядиалогапечати режимиспользованияпараметракоманды режиммасштабированияпросмотра режимосновногоокнаклиентскогоприложения режимоткрытияокнаформы режимотображениявыделения режимотображениягеографическойсхемы режимотображениязначенийсерии режимотрисовкисеткиграфическойсхемы режимполупрозрачностидиаграммы режимпробеловдиаграммы режимразмещениянастранице режимредактированияколонки режимсглаживаниядиаграммы режимсглаживанияиндикатора режимсписказадач сквозноевыравнивание сохранениеданныхформывнастройках способзаполнениятекстазаголовкашкалыдиаграммы способопределенияограничивающегозначениядиаграммы стандартнаягруппакоманд стандартноеоформление статусоповещенияпользователя стильстрелки типаппроксимациилиниитрендадиаграммы типдиаграммы типединицышкалывремени типимпортасерийслоягеографическойсхемы типлиниигеографическойсхемы типлиниидиаграммы типмаркерагеографическойсхемы типмаркерадиаграммы типобластиоформления типорганизацииисточникаданныхгеографическойсхемы типотображениясериислоягеографическойсхемы типотображенияточечногообъектагеографическойсхемы типотображенияшкалыэлементалегендыгеографическойсхемы типпоискаобъектовгеографическойсхемы типпроекциигеографическойсхемы типразмещенияизмерений типразмещенияреквизитовизмерений типрамкиэлементауправления типсводнойдиаграммы типсвязидиаграммыганта типсоединениязначенийпосериямдиаграммы типсоединенияточекдиаграммы типсоединительнойлинии типстороныэлементаграфическойсхемы типформыотчета типшкалырадарнойдиаграммы факторлиниитрендадиаграммы фигуракнопки фигурыграфическойсхемы фиксациявтаблице форматдняшкалывремени форматкартинки ширинаподчиненныхэлементовформы виддвижениябухгалтерии виддвижениянакопления видпериодарегистрарасчета видсчета видточкимаршрутабизнеспроцесса использованиеагрегатарегистранакопления использованиегруппиэлементов использованиережимапроведения использованиесреза периодичностьагрегатарегистранакопления режимавтовремя режимзаписидокумента режимпроведениядокумента авторегистрацияизменений допустимыйномерсообщения отправкаэлементаданных получениеэлементаданных использованиерасшифровкитабличногодокумента ориентациястраницы положениеитоговколоноксводнойтаблицы положениеитоговстроксводнойтаблицы положениетекстаотносительнокартинки расположениезаголовкагруппировкитабличногодокумента способчтениязначенийтабличногодокумента типдвустороннейпечати типзаполненияобластитабличногодокумента типкурсоровтабличногодокумента типлиниирисункатабличногодокумента типлинииячейкитабличногодокумента типнаправленияпереходатабличногодокумента типотображениявыделениятабличногодокумента типотображениялинийсводнойтаблицы типразмещениятекстатабличногодокумента типрисункатабличногодокумента типсмещениятабличногодокумента типузоратабличногодокумента типфайлатабличногодокумента точностьпечати чередованиерасположениястраниц отображениевремениэлементовпланировщика типфайлаформатированногодокумента обходрезультатазапроса типзаписизапроса видзаполнениярасшифровкипостроителяотчета типдобавленияпредставлений типизмеренияпостроителяотчета типразмещенияитогов доступкфайлу режимдиалогавыборафайла режимоткрытияфайла типизмеренияпостроителязапроса видданныханализа методкластеризации типединицыинтервалавременианализаданных типзаполнениятаблицырезультатаанализаданных типиспользованиячисловыхзначенийанализаданных типисточникаданныхпоискаассоциаций типколонкианализаданныхдереворешений типколонкианализаданныхкластеризация типколонкианализаданныхобщаястатистика типколонкианализаданныхпоискассоциаций типколонкианализаданныхпоискпоследовательностей типколонкимоделипрогноза типмерырасстоянияанализаданных типотсеченияправилассоциации типполяанализаданных типстандартизациианализаданных типупорядочиванияправилассоциациианализаданных типупорядочиванияшаблоновпоследовательностейанализаданных типупрощениядереварешений wsнаправлениепараметра вариантxpathxs вариантзаписидатыjson вариантпростоготипаxs видгруппымоделиxs видфасетаxdto действиепостроителяdom завершенностьпростоготипаxs завершенностьсоставноготипаxs завершенностьсхемыxs запрещенныеподстановкиxs исключениягруппподстановкиxs категорияиспользованияатрибутаxs категорияограниченияидентичностиxs категорияограниченияпространствименxs методнаследованияxs модельсодержимогоxs назначениетипаxml недопустимыеподстановкиxs обработкапробельныхсимволовxs обработкасодержимогоxs ограничениезначенияxs параметрыотбораузловdom переносстрокjson позициявдокументеdom пробельныесимволыxml типатрибутаxml типзначенияjson типканоническогоxml типкомпонентыxs типпроверкиxml типрезультатаdomxpath типузлаdom типузлаxml формаxml формапредставленияxs форматдатыjson экранированиесимволовjson видсравнениякомпоновкиданных действиеобработкирасшифровкикомпоновкиданных направлениесортировкикомпоновкиданных расположениевложенныхэлементоврезультатакомпоновкиданных расположениеитоговкомпоновкиданных расположениегруппировкикомпоновкиданных расположениеполейгруппировкикомпоновкиданных расположениеполякомпоновкиданных расположениереквизитовкомпоновкиданных расположениересурсовкомпоновкиданных типбухгалтерскогоостаткакомпоновкиданных типвыводатекстакомпоновкиданных типгруппировкикомпоновкиданных типгруппыэлементовотборакомпоновкиданных типдополненияпериодакомпоновкиданных типзаголовкаполейкомпоновкиданных типмакетагруппировкикомпоновкиданных типмакетаобластикомпоновкиданных типостаткакомпоновкиданных типпериодакомпоновкиданных типразмещениятекстакомпоновкиданных типсвязинаборовданныхкомпоновкиданных типэлементарезультатакомпоновкиданных расположениелегендыдиаграммыкомпоновкиданных типпримененияотборакомпоновкиданных режимотображенияэлементанастройкикомпоновкиданных режимотображениянастроеккомпоновкиданных состояниеэлементанастройкикомпоновкиданных способвосстановлениянастроеккомпоновкиданных режимкомпоновкирезультата использованиепараметракомпоновкиданных автопозицияресурсовкомпоновкиданных вариантиспользованиягруппировкикомпоновкиданных расположениересурсоввдиаграммекомпоновкиданных фиксациякомпоновкиданных использованиеусловногооформлениякомпоновкиданных важностьинтернетпочтовогосообщения обработкатекстаинтернетпочтовогосообщения способкодированияинтернетпочтовоговложения способкодированиянеasciiсимволовинтернетпочтовогосообщения типтекстапочтовогосообщения протоколинтернетпочты статусразборапочтовогосообщения режимтранзакциизаписижурналарегистрации статустранзакциизаписижурналарегистрации уровеньжурналарегистрации расположениехранилищасертификатовкриптографии режимвключениясертификатовкриптографии режимпроверкисертификатакриптографии типхранилищасертификатовкриптографии кодировкаименфайловвzipфайле методсжатияzip методшифрованияzip режимвосстановленияпутейфайловzip режимобработкиподкаталоговzip режимсохраненияпутейzip уровеньсжатияzip звуковоеоповещение направлениепереходакстроке позициявпотоке порядокбайтов режимблокировкиданных режимуправленияблокировкойданных сервисвстроенныхпокупок состояниефоновогозадания типподписчикадоставляемыхуведомлений уровеньиспользованиязащищенногосоединенияftp направлениепорядкасхемызапроса типдополненияпериодамисхемызапроса типконтрольнойточкисхемызапроса типобъединениясхемызапроса типпараметрадоступнойтаблицысхемызапроса типсоединениясхемызапроса httpметод автоиспользованиеобщегореквизита автопрефиксномеразадачи вариантвстроенногоязыка видиерархии видрегистранакопления видтаблицывнешнегоисточникаданных записьдвиженийприпроведении заполнениепоследовательностей индексирование использованиебазыпланавидоврасчета использованиебыстроговыбора использованиеобщегореквизита использованиеподчинения использованиеполнотекстовогопоиска использованиеразделяемыхданныхобщегореквизита использованиереквизита назначениеиспользованияприложения назначениерасширенияконфигурации направлениепередачи обновлениепредопределенныхданных оперативноепроведение основноепредставлениевидарасчета основноепредставлениевидахарактеристики основноепредставлениезадачи основноепредставлениепланаобмена основноепредставлениесправочника основноепредставлениесчета перемещениеграницыприпроведении периодичностьномерабизнеспроцесса периодичностьномерадокумента периодичностьрегистрарасчета периодичностьрегистрасведений повторноеиспользованиевозвращаемыхзначений полнотекстовыйпоискпривводепостроке принадлежностьобъекта проведение разделениеаутентификацииобщегореквизита разделениеданныхобщегореквизита разделениерасширенийконфигурацииобщегореквизита режимавтонумерацииобъектов режимзаписирегистра режимиспользованиямодальности режимиспользованиясинхронныхвызововрасширенийплатформыивнешнихкомпонент режимповторногоиспользованиясеансов режимполученияданныхвыборапривводепостроке режимсовместимости режимсовместимостиинтерфейса режимуправленияблокировкойданныхпоумолчанию сериикодовпланавидовхарактеристик сериикодовпланасчетов сериикодовсправочника созданиепривводе способвыбора способпоискастрокипривводепостроке способредактирования типданныхтаблицывнешнегоисточникаданных типкодапланавидоврасчета типкодасправочника типмакета типномерабизнеспроцесса типномерадокумента типномеразадачи типформы удалениедвижений важностьпроблемыприменениярасширенияконфигурации вариантинтерфейсаклиентскогоприложения вариантмасштабаформклиентскогоприложения вариантосновногошрифтаклиентскогоприложения вариантстандартногопериода вариантстандартнойдатыначала видграницы видкартинки видотображенияполнотекстовогопоиска видрамки видсравнения видцвета видчисловогозначения видшрифта допустимаядлина допустимыйзнак использованиеbyteordermark использованиеметаданныхполнотекстовогопоиска источникрасширенийконфигурации клавиша кодвозвратадиалога кодировкаxbase кодировкатекста направлениепоиска направлениесортировки обновлениепредопределенныхданных обновлениеприизмененииданных отображениепанелиразделов проверказаполнения режимдиалогавопрос режимзапускаклиентскогоприложения режимокругления режимоткрытияформприложения режимполнотекстовогопоиска скоростьклиентскогосоединения состояниевнешнегоисточникаданных состояниеобновленияконфигурациибазыданных способвыборасертификатаwindows способкодированиястроки статуссообщения типвнешнейкомпоненты типплатформы типповеденияклавишиenter типэлементаинформацииовыполненииобновленияконфигурациибазыданных уровеньизоляциитранзакций хешфункция частидаты",type:"comобъект ftpсоединение httpзапрос httpсервисответ httpсоединение wsопределения wsпрокси xbase анализданных аннотацияxs блокировкаданных буфердвоичныхданных включениеxs выражениекомпоновкиданных генераторслучайныхчисел географическаясхема географическиекоординаты графическаясхема группамоделиxs данныерасшифровкикомпоновкиданных двоичныеданные дендрограмма диаграмма диаграммаганта диалогвыборафайла диалогвыборацвета диалогвыборашрифта диалограсписаниярегламентногозадания диалогредактированиястандартногопериода диапазон документdom документhtml документацияxs доставляемоеуведомление записьdom записьfastinfoset записьhtml записьjson записьxml записьzipфайла записьданных записьтекста записьузловdom запрос защищенноесоединениеopenssl значенияполейрасшифровкикомпоновкиданных извлечениетекста импортxs интернетпочта интернетпочтовоесообщение интернетпочтовыйпрофиль интернетпрокси интернетсоединение информациядляприложенияxs использованиеатрибутаxs использованиесобытияжурналарегистрации источникдоступныхнастроеккомпоновкиданных итераторузловdom картинка квалификаторыдаты квалификаторыдвоичныхданных квалификаторыстроки квалификаторычисла компоновщикмакетакомпоновкиданных компоновщикнастроеккомпоновкиданных конструктормакетаоформлениякомпоновкиданных конструкторнастроеккомпоновкиданных конструкторформатнойстроки линия макеткомпоновкиданных макетобластикомпоновкиданных макетоформлениякомпоновкиданных маскаxs менеджеркриптографии наборсхемxml настройкикомпоновкиданных настройкисериализацииjson обработкакартинок обработкарасшифровкикомпоновкиданных обходдереваdom объявлениеатрибутаxs объявлениенотацииxs объявлениеэлементаxs описаниеиспользованиясобытиядоступжурналарегистрации описаниеиспользованиясобытияотказвдоступежурналарегистрации описаниеобработкирасшифровкикомпоновкиданных описаниепередаваемогофайла описаниетипов определениегруппыатрибутовxs определениегруппымоделиxs определениеограниченияидентичностиxs определениепростоготипаxs определениесоставноготипаxs определениетипадокументаdom определенияxpathxs отборкомпоновкиданных пакетотображаемыхдокументов параметрвыбора параметркомпоновкиданных параметрызаписиjson параметрызаписиxml параметрычтенияxml переопределениеxs планировщик полеанализаданных полекомпоновкиданных построительdom построительзапроса построительотчета построительотчетаанализаданных построительсхемxml поток потоквпамяти почта почтовоесообщение преобразованиеxsl преобразованиекканоническомуxml процессорвыводарезультатакомпоновкиданныхвколлекциюзначений процессорвыводарезультатакомпоновкиданныхвтабличныйдокумент процессоркомпоновкиданных разыменовательпространствименdom рамка расписаниерегламентногозадания расширенноеимяxml результатчтенияданных своднаядиаграмма связьпараметравыбора связьпотипу связьпотипукомпоновкиданных сериализаторxdto сертификатклиентаwindows сертификатклиентафайл сертификаткриптографии сертификатыудостоверяющихцентровwindows сертификатыудостоверяющихцентровфайл сжатиеданных системнаяинформация сообщениепользователю сочетаниеклавиш сравнениезначений стандартнаядатаначала стандартныйпериод схемаxml схемакомпоновкиданных табличныйдокумент текстовыйдокумент тестируемоеприложение типданныхxml уникальныйидентификатор фабрикаxdto файл файловыйпоток фасетдлиныxs фасетколичестваразрядовдробнойчастиxs фасетмаксимальноговключающегозначенияxs фасетмаксимальногоисключающегозначенияxs фасетмаксимальнойдлиныxs фасетминимальноговключающегозначенияxs фасетминимальногоисключающегозначенияxs фасетминимальнойдлиныxs фасетобразцаxs фасетобщегоколичестваразрядовxs фасетперечисленияxs фасетпробельныхсимволовxs фильтрузловdom форматированнаястрока форматированныйдокумент фрагментxs хешированиеданных хранилищезначения цвет чтениеfastinfoset чтениеhtml чтениеjson чтениеxml чтениеzipфайла чтениеданных чтениетекста чтениеузловdom шрифт элементрезультатакомпоновкиданных comsafearray деревозначений массив соответствие списокзначений структура таблицазначений фиксированнаяструктура фиксированноесоответствие фиксированныймассив ",literal:a},contains:[{className:"meta",begin:"#|&",end:"$",keywords:{$pattern:t,keyword:n+"загрузитьизфайла вебклиент вместо внешнеесоединение клиент конецобласти мобильноеприложениеклиент мобильноеприложениесервер наклиенте наклиентенасервере наклиентенасерверебезконтекста насервере насерверебезконтекста область перед после сервер толстыйклиентобычноеприложение толстыйклиентуправляемоеприложение тонкийклиент "},contains:[s]},{className:"function",variants:[{begin:"процедура|функция",end:"\\)",keywords:"процедура функция"},{begin:"конецпроцедуры|конецфункции",keywords:"конецпроцедуры конецфункции"}],contains:[{begin:"\\(",end:"\\)",endsParent:!0,contains:[{className:"params",begin:t,end:",",excludeEnd:!0,endsWithParent:!0,keywords:{$pattern:t,keyword:"знач",literal:a},contains:[r,i,o]},s]},e.inherit(e.TITLE_MODE,{begin:t})]},s,{className:"symbol",begin:"~",end:";|:",excludeEnd:!0},r,i,o,{match:/[;()+\-:=,]/,className:"punctuation",relevance:0}]}}},5980(e){e.exports=function(e){const t="foreach do while for if from to step else on-error and or not in",n="true false yes no nothing nil null",a={className:"variable",variants:[{begin:/\$[\w\d#@][\w\d_]*/},{begin:/\$\{(.*?)\}/}]},r={className:"string",begin:/"/,end:/"/,contains:[e.BACKSLASH_ESCAPE,a,{className:"variable",begin:/\$\(/,end:/\)/,contains:[e.BACKSLASH_ESCAPE]}]},i={className:"string",begin:/'/,end:/'/};return{name:"MikroTik RouterOS script",aliases:["mikrotik"],case_insensitive:!0,keywords:{$pattern:/:?[\w-]+/,literal:n,keyword:t+" :"+t.split(" ").join(" :")+" :"+"global local beep delay put len typeof pick log time set find environment terminal error execute parse resolve toarray tobool toid toip toip6 tonum tostr totime".split(" ").join(" :")},contains:[{variants:[{begin:/\/\*/,end:/\*\//},{begin:/\/\//,end:/$/},{begin:/<\//,end:/>/}],illegal:/./},e.COMMENT("^#","$"),r,i,a,{begin:/[\w-]+=([^\s{}[\]()>]+)/,relevance:0,returnBegin:!0,contains:[{className:"attribute",begin:/[^=]+/},{begin:/=/,endsWithParent:!0,relevance:0,contains:[r,i,a,{className:"literal",begin:"\\b("+n.split(" ").join("|")+")\\b"},{begin:/("[^"]*"|[^\s{}[\]]+)/}]}]},{className:"number",begin:/\*[0-9a-fA-F]+/},{begin:"\\b("+"add remove enable disable set get print export edit find run debug error info warning".split(" ").join("|")+")([\\s[(\\]|])",returnBegin:!0,contains:[{className:"built_in",begin:/\w+/}]},{className:"built_in",variants:[{begin:"(\\.\\./|/|\\s)(("+"traffic-flow traffic-generator firewall scheduler aaa accounting address-list address align area bandwidth-server bfd bgp bridge client clock community config connection console customer default dhcp-client dhcp-server discovery dns e-mail ethernet filter firmware gps graphing group hardware health hotspot identity igmp-proxy incoming instance interface ip ipsec ipv6 irq l2tp-server lcd ldp logging mac-server mac-winbox mangle manual mirror mme mpls nat nd neighbor network note ntp ospf ospf-v3 ovpn-server page peer pim ping policy pool port ppp pppoe-client pptp-server prefix profile proposal proxy queue radius resource rip ripng route routing screen script security-profiles server service service-port settings shares smb sms sniffer snmp snooper socks sstp-server system tool tracking type upgrade upnp user-manager users user vlan secret vrrp watchdog web-access wireless pptp pppoe lan wan layer7-protocol lease simple raw".split(" ").join("|")+");?\\s)+"},{begin:/\.\./,relevance:0}]}]}}},6035(e){const t="[A-Za-z$_][0-9A-Za-z$_]*",n=["as","in","of","if","for","while","finally","var","new","function","do","return","void","else","break","catch","instanceof","with","throw","case","default","try","switch","continue","typeof","delete","let","yield","const","class","debugger","async","await","static","import","from","export","extends","using"],a=["true","false","null","undefined","NaN","Infinity"],r=["Object","Function","Boolean","Symbol","Math","Date","Number","BigInt","String","RegExp","Array","Float32Array","Float64Array","Int8Array","Uint8Array","Uint8ClampedArray","Int16Array","Int32Array","Uint16Array","Uint32Array","BigInt64Array","BigUint64Array","Set","Map","WeakSet","WeakMap","ArrayBuffer","SharedArrayBuffer","Atomics","DataView","JSON","Promise","Generator","GeneratorFunction","AsyncFunction","Reflect","Proxy","Intl","WebAssembly"],i=["Error","EvalError","InternalError","RangeError","ReferenceError","SyntaxError","TypeError","URIError"],o=["setInterval","setTimeout","clearInterval","clearTimeout","require","exports","eval","isFinite","isNaN","parseFloat","parseInt","decodeURI","decodeURIComponent","encodeURI","encodeURIComponent","escape","unescape"],s=["arguments","this","super","console","window","document","localStorage","sessionStorage","module","global"],l=[].concat(o,r,i);e.exports=function(e){const c=e.regex,_=t,d={begin:/<[A-Za-z0-9\\._:-]+/,end:/\/[A-Za-z0-9\\._:-]+>|\/>/,isTrulyOpeningTag:(e,t)=>{const n=e[0].length+e.index,a=e.input[n];if("<"===a||","===a)return void t.ignoreMatch();let r;">"===a&&(((e,{after:t})=>{const n="`${e}\\s*\\(`)),c.concat("(?!",x.join("|"),")")),_,c.lookahead(/\s*\(/)),className:"title.function",relevance:0};var x;const w={begin:c.concat(/\./,c.lookahead(c.concat(_,/(?![0-9A-Za-z$_(])/))),end:_,excludeBegin:!0,keywords:"prototype",className:"property",relevance:0},M={match:[/get|set/,/\s+/,_,/(?=\()/],className:{1:"keyword",3:"title.function"},contains:[{begin:/\(\)/},v]},L="(\\([^()]*(\\([^()]*(\\([^()]*\\)[^()]*)*\\)[^()]*)*\\)|"+e.UNDERSCORE_IDENT_RE+")\\s*=>",P={match:[/const|var|let/,/\s+/,_,/\s*/,/=\s*/,/(async\s*)?/,c.lookahead(L)],keywords:"async",className:{1:"keyword",3:"title.function"},contains:[v]};return{name:"JavaScript",aliases:["js","jsx","mjs","cjs"],keywords:p,exports:{PARAMS_CONTAINS:R,CLASS_REFERENCE:A},illegal:/#(?![$_A-z])/,contains:[e.SHEBANG({label:"shebang",binary:"node",relevance:5}),{label:"use_strict",className:"meta",relevance:10,begin:/^\s*['"]use (strict|asm)['"]/},e.APOS_STRING_MODE,e.QUOTE_STRING_MODE,b,f,T,h,C,{match:/\$\d+/},E,A,{scope:"attr",match:_+c.lookahead(":"),relevance:0},P,{begin:"("+e.RE_STARTERS_RE+"|\\b(case|return|throw)\\b)\\s*",keywords:"return throw case",relevance:0,contains:[C,e.REGEXP_MODE,{className:"function",begin:L,returnBegin:!0,end:"\\s*=>",contains:[{className:"params",variants:[{begin:e.UNDERSCORE_IDENT_RE,relevance:0},{className:null,begin:/\(\s*\)/,skip:!0},{begin:/(\s*)\(/,end:/\)/,excludeBegin:!0,excludeEnd:!0,keywords:p,contains:R}]}]},{begin:/,/,relevance:0},{match:/\s+/,relevance:0},{variants:[{begin:"<>",end:""},{match:/<[A-Za-z0-9\\._:-]+\s*\/>/},{begin:d.begin,"on:begin":d.isTrulyOpeningTag,end:d.end}],subLanguage:"xml",contains:[{begin:d.begin,end:d.end,skip:!0,contains:["self"]}]}]},I,{beginKeywords:"while if switch catch for"},{begin:"\\b(?!function)"+e.UNDERSCORE_IDENT_RE+"\\([^()]*(\\([^()]*(\\([^()]*\\)[^()]*)*\\)[^()]*)*\\)\\s*\\{",returnBegin:!0,label:"func.def",contains:[v,e.inherit(e.TITLE_MODE,{begin:_,className:"title.function"})]},{match:/\.\.\./,relevance:0},w,{match:"\\$"+_,relevance:0},{match:[/\bconstructor(?=\s*\()/],className:{1:"title.function"},contains:[v]},D,{relevance:0,match:/\b[A-Z][A-Z_0-9]+\b/,className:"variable.constant"},y,M,{match:/\$[(.]/}]}}},6078(e){e.exports=function(e){return{name:"Mojolicious",subLanguage:"xml",contains:[{className:"meta",begin:"^__(END|DATA)__$"},{begin:"^\\s*%{1,2}={0,2}",end:"$",subLanguage:"perl"},{begin:"<%{1,2}={0,2}",end:"={0,1}%>",subLanguage:"perl",excludeBegin:!0,excludeEnd:!0}]}}},6089(e){e.exports=function(e){const t="[a-zA-Z_][\\w.]*",n="<\\?(lasso(script)?|=)",a="\\]|\\?>",r={$pattern:t+"|&[lg]t;",literal:"true false none minimal full all void and or not bw nbw ew new cn ncn lt lte gt gte eq neq rx nrx ft",built_in:"array date decimal duration integer map pair string tag xml null boolean bytes keyword list locale queue set stack staticarray local var variable global data self inherited currentcapture givenblock",keyword:"cache database_names database_schemanames database_tablenames define_tag define_type email_batch encode_set html_comment handle handle_error header if inline iterate ljax_target link link_currentaction link_currentgroup link_currentrecord link_detail link_firstgroup link_firstrecord link_lastgroup link_lastrecord link_nextgroup link_nextrecord link_prevgroup link_prevrecord log loop namespace_using output_none portal private protect records referer referrer repeating resultset rows search_args search_arguments select sort_args sort_arguments thread_atomic value_list while abort case else fail_if fail_ifnot fail if_empty if_false if_null if_true loop_abort loop_continue loop_count params params_up return return_value run_children soap_definetag soap_lastrequest soap_lastresponse tag_name ascending average by define descending do equals frozen group handle_failure import in into join let match max min on order parent protected provide public require returnhome skip split_thread sum take thread to trait type where with yield yieldhome"},i=e.COMMENT("\x3c!--","--\x3e",{relevance:0}),o={className:"meta",begin:"\\[noprocess\\]",starts:{end:"\\[/noprocess\\]",returnEnd:!0,contains:[i]}},s={className:"meta",begin:"\\[/noprocess|"+n},l={className:"symbol",begin:"'"+t+"'"},c=[e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE,e.inherit(e.C_NUMBER_MODE,{begin:e.C_NUMBER_RE+"|(-?infinity|NaN)\\b"}),e.inherit(e.APOS_STRING_MODE,{illegal:null}),e.inherit(e.QUOTE_STRING_MODE,{illegal:null}),{className:"string",begin:"`",end:"`"},{variants:[{begin:"[#$]"+t},{begin:"#",end:"\\d+",illegal:"\\W"}]},{className:"type",begin:"::\\s*",end:t,illegal:"\\W"},{className:"params",variants:[{begin:"-(?!infinity)"+t,relevance:0},{begin:"(\\.\\.\\.)"}]},{begin:/(->|\.)\s*/,relevance:0,contains:[l]},{className:"class",beginKeywords:"define",returnEnd:!0,end:"\\(|=>",contains:[e.inherit(e.TITLE_MODE,{begin:t+"(=(?!>))?|[-+*/%](?!>)"})]}];return{name:"Lasso",aliases:["ls","lassoscript"],case_insensitive:!0,keywords:r,contains:[{className:"meta",begin:a,relevance:0,starts:{end:"\\[|"+n,returnEnd:!0,relevance:0,contains:[i]}},o,s,{className:"meta",begin:"\\[no_square_brackets",starts:{end:"\\[/no_square_brackets\\]",keywords:r,contains:[{className:"meta",begin:a,relevance:0,starts:{end:"\\[noprocess\\]|"+n,returnEnd:!0,contains:[i]}},o,s].concat(c)}},{className:"meta",begin:"\\[",relevance:0},{className:"meta",begin:"^#!",end:"lasso9$",relevance:10}].concat(c)}}},6166(e,t,n){"use strict";var a,r=this&&this.__createBinding||(Object.create?function(e,t,n,a){void 0===a&&(a=n);var r=Object.getOwnPropertyDescriptor(t,n);r&&!("get"in r?!t.__esModule:r.writable||r.configurable)||(r={enumerable:!0,get:function(){return t[n]}}),Object.defineProperty(e,a,r)}:function(e,t,n,a){void 0===a&&(a=n),e[a]=t[n]}),i=this&&this.__setModuleDefault||(Object.create?function(e,t){Object.defineProperty(e,"default",{enumerable:!0,value:t})}:function(e,t){e.default=t}),o=this&&this.__importStar||(a=function(e){return a=Object.getOwnPropertyNames||function(e){var t=[];for(var n in e)Object.prototype.hasOwnProperty.call(e,n)&&(t[t.length]=n);return t},a(e)},function(e){if(e&&e.__esModule)return e;var t={};if(null!=e)for(var n=a(e),o=0;o > >= ` abs acos angle append apply asin assoc assq assv atan boolean? caar cadr call-with-input-file call-with-output-file call-with-values car cdddar cddddr cdr ceiling char->integer char-alphabetic? char-ci<=? char-ci=? char-ci>? char-downcase char-lower-case? char-numeric? char-ready? char-upcase char-upper-case? char-whitespace? char<=? char=? char>? char? close-input-port close-output-port complex? cons cos current-input-port current-output-port denominator display eof-object? eq? equal? eqv? eval even? exact->inexact exact? exp expt floor force gcd imag-part inexact->exact inexact? input-port? integer->char integer? interaction-environment lcm length list list->string list->vector list-ref list-tail list? load log magnitude make-polar make-rectangular make-string make-vector max member memq memv min modulo negative? newline not null-environment null? number->string number? numerator odd? open-input-file open-output-file output-port? pair? peek-char port? positive? procedure? quasiquote quote quotient rational? rationalize read read-char real-part real? remainder reverse round scheme-report-environment set! set-car! set-cdr! sin sqrt string string->list string->number string->symbol string-append string-ci<=? string-ci=? string-ci>? string-copy string-fill! string-length string-ref string-set! string<=? string=? string>? string? substring symbol->string symbol? tan transcript-off transcript-on truncate values vector vector->list vector-fill! vector-length vector-ref vector-set! with-input-from-file with-output-to-file write write-char zero?"},r={className:"literal",begin:"(#t|#f|#\\\\"+t+"|#\\\\.)"},i={className:"number",variants:[{begin:n,relevance:0},{begin:n+"[+\\-]"+n+"i",relevance:0},{begin:"#b[0-1]+(/[0-1]+)?"},{begin:"#o[0-7]+(/[0-7]+)?"},{begin:"#x[0-9a-f]+(/[0-9a-f]+)?"}]},o=e.QUOTE_STRING_MODE,s=[e.COMMENT(";","$",{relevance:0}),e.COMMENT("#\\|","\\|#")],l={begin:t,relevance:0},c={className:"symbol",begin:"'"+t},_={endsWithParent:!0,relevance:0},d={variants:[{begin:/'/},{begin:"`"}],contains:[{begin:"\\(",end:"\\)",contains:["self",r,o,i,l,c]}]},p={className:"name",relevance:0,begin:t,keywords:a},u={variants:[{begin:"\\(",end:"\\)"},{begin:"\\[",end:"\\]"}],contains:[{begin:/lambda/,endsWithParent:!0,returnBegin:!0,contains:[p,{endsParent:!0,variants:[{begin:/\(/,end:/\)/},{begin:/\[/,end:/\]/}],contains:[l]}]},p,_]};return _.contains=[r,i,o,l,c,d,u].concat(s),{name:"Scheme",aliases:["scm"],illegal:/\S/,contains:[e.SHEBANG(),i,o,c,d,u].concat(s)}}},6325(e){e.exports=function(e){const t={className:"string",variants:[{begin:'"',end:'"',contains:[{begin:'""',relevance:0}]},{begin:"'",end:"'",contains:[{begin:"''",relevance:0}]}]},n={className:"meta",begin:/#\s*[a-z]+\b/,end:/$/,keywords:"define undef ifdef ifndef else endif include if",contains:[{begin:/\\\n/,relevance:0},e.inherit(t,{className:"string"}),{begin:/<[^\n>]*>/,end:/$/,illegal:"\\n"},e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE]};return{name:"SQF",case_insensitive:!0,keywords:{keyword:["break","breakWith","breakOut","breakTo","case","catch","continue","continueWith","default","do","else","exit","exitWith","for","forEach","from","if","local","private","switch","step","then","throw","to","try","waitUntil","while","with"],built_in:["abs","accTime","acos","action","actionIDs","actionKeys","actionKeysEx","actionKeysImages","actionKeysNames","actionKeysNamesArray","actionName","actionParams","activateAddons","activatedAddons","activateKey","activeTitleEffectParams","add3DENConnection","add3DENEventHandler","add3DENLayer","addAction","addBackpack","addBackpackCargo","addBackpackCargoGlobal","addBackpackGlobal","addBinocularItem","addCamShake","addCuratorAddons","addCuratorCameraArea","addCuratorEditableObjects","addCuratorEditingArea","addCuratorPoints","addEditorObject","addEventHandler","addForce","addForceGeneratorRTD","addGoggles","addGroupIcon","addHandgunItem","addHeadgear","addItem","addItemCargo","addItemCargoGlobal","addItemPool","addItemToBackpack","addItemToUniform","addItemToVest","addLiveStats","addMagazine","addMagazineAmmoCargo","addMagazineCargo","addMagazineCargoGlobal","addMagazineGlobal","addMagazinePool","addMagazines","addMagazineTurret","addMenu","addMenuItem","addMissionEventHandler","addMPEventHandler","addMusicEventHandler","addonFiles","addOwnedMine","addPlayerScores","addPrimaryWeaponItem","addPublicVariableEventHandler","addRating","addResources","addScore","addScoreSide","addSecondaryWeaponItem","addSwitchableUnit","addTeamMember","addToRemainsCollector","addTorque","addUniform","addUserActionEventHandler","addVehicle","addVest","addWaypoint","addWeapon","addWeaponCargo","addWeaponCargoGlobal","addWeaponGlobal","addWeaponItem","addWeaponPool","addWeaponTurret","addWeaponWithAttachmentsCargo","addWeaponWithAttachmentsCargoGlobal","admin","agent","agents","AGLToASL","aimedAtTarget","aimPos","airDensityCurveRTD","airDensityRTD","airplaneThrottle","airportSide","AISFinishHeal","alive","all3DENEntities","allActiveTitleEffects","allAddonsInfo","allAirports","allControls","allCurators","allCutLayers","allDead","allDeadMen","allDiaryRecords","allDiarySubjects","allDisplays","allEnv3DSoundSources","allGroups","allLODs","allMapMarkers","allMines","allMissionObjects","allObjects","allow3DMode","allowCrewInImmobile","allowCuratorLogicIgnoreAreas","allowDamage","allowDammage","allowedService","allowFileOperations","allowFleeing","allowGetIn","allowService","allowSprint","allPlayers","allSimpleObjects","allSites","allTurrets","allUnits","allUnitsUAV","allUsers","allVariables","ambientTemperature","ammo","ammoOnPylon","and","animate","animateBay","animateDoor","animatePylon","animateSource","animationNames","animationPhase","animationSourcePhase","animationState","apertureParams","append","apply","armoryPoints","arrayIntersect","asin","ASLToAGL","ASLToATL","assert","assignAsCargo","assignAsCargoIndex","assignAsCommander","assignAsDriver","assignAsGunner","assignAsTurret","assignCurator","assignedCargo","assignedCommander","assignedDriver","assignedGroup","assignedGunner","assignedItems","assignedTarget","assignedTeam","assignedVehicle","assignedVehicleRole","assignedVehicles","assignItem","assignTeam","assignToAirport","atan","atan2","atg","ATLToASL","attachedObject","attachedObjects","attachedTo","attachObject","attachTo","attackEnabled","awake","backpack","backpackCargo","backpackContainer","backpackItems","backpackMagazines","backpackSpaceFor","behaviour","benchmark","bezierInterpolation","binocular","binocularItems","binocularMagazine","boundingBox","boundingBoxReal","boundingCenter","brakesDisabled","briefingName","buildingExit","buildingPos","buldozer_EnableRoadDiag","buldozer_IsEnabledRoadDiag","buldozer_LoadNewRoads","buldozer_reloadOperMap","buttonAction","buttonSetAction","cadetMode","calculatePath","calculatePlayerVisibilityByFriendly","call","callExtension","camCommand","camCommit","camCommitPrepared","camCommitted","camConstuctionSetParams","camCreate","camDestroy","cameraEffect","cameraEffectEnableHUD","cameraInterest","cameraOn","cameraView","campaignConfigFile","camPreload","camPreloaded","camPrepareBank","camPrepareDir","camPrepareDive","camPrepareFocus","camPrepareFov","camPrepareFovRange","camPreparePos","camPrepareRelPos","camPrepareTarget","camSetBank","camSetDir","camSetDive","camSetFocus","camSetFov","camSetFovRange","camSetPos","camSetRelPos","camSetTarget","camTarget","camUseNVG","canAdd","canAddItemToBackpack","canAddItemToUniform","canAddItemToVest","cancelSimpleTaskDestination","canDeployWeapon","canFire","canMove","canSlingLoad","canStand","canSuspend","canTriggerDynamicSimulation","canUnloadInCombat","canVehicleCargo","captive","captiveNum","cbChecked","cbSetChecked","ceil","channelEnabled","cheatsEnabled","checkAIFeature","checkVisibility","className","clear3DENAttribute","clear3DENInventory","clearAllItemsFromBackpack","clearBackpackCargo","clearBackpackCargoGlobal","clearForcesRTD","clearGroupIcons","clearItemCargo","clearItemCargoGlobal","clearItemPool","clearMagazineCargo","clearMagazineCargoGlobal","clearMagazinePool","clearOverlay","clearRadio","clearWeaponCargo","clearWeaponCargoGlobal","clearWeaponPool","clientOwner","closeDialog","closeDisplay","closeOverlay","collapseObjectTree","collect3DENHistory","collectiveRTD","collisionDisabledWith","combatBehaviour","combatMode","commandArtilleryFire","commandChat","commander","commandFire","commandFollow","commandFSM","commandGetOut","commandingMenu","commandMove","commandRadio","commandStop","commandSuppressiveFire","commandTarget","commandWatch","comment","commitOverlay","compatibleItems","compatibleMagazines","compile","compileFinal","compileScript","completedFSM","composeText","configClasses","configFile","configHierarchy","configName","configOf","configProperties","configSourceAddonList","configSourceMod","configSourceModList","confirmSensorTarget","connectTerminalToUAV","connectToServer","controlsGroupCtrl","conversationDisabled","copyFromClipboard","copyToClipboard","copyWaypoints","cos","count","countEnemy","countFriendly","countSide","countType","countUnknown","create3DENComposition","create3DENEntity","createAgent","createCenter","createDialog","createDiaryLink","createDiaryRecord","createDiarySubject","createDisplay","createGearDialog","createGroup","createGuardedPoint","createHashMap","createHashMapFromArray","createLocation","createMarker","createMarkerLocal","createMenu","createMine","createMissionDisplay","createMPCampaignDisplay","createSimpleObject","createSimpleTask","createSite","createSoundSource","createTask","createTeam","createTrigger","createUnit","createVehicle","createVehicleCrew","createVehicleLocal","crew","ctAddHeader","ctAddRow","ctClear","ctCurSel","ctData","ctFindHeaderRows","ctFindRowHeader","ctHeaderControls","ctHeaderCount","ctRemoveHeaders","ctRemoveRows","ctrlActivate","ctrlAddEventHandler","ctrlAngle","ctrlAnimateModel","ctrlAnimationPhaseModel","ctrlAt","ctrlAutoScrollDelay","ctrlAutoScrollRewind","ctrlAutoScrollSpeed","ctrlBackgroundColor","ctrlChecked","ctrlClassName","ctrlCommit","ctrlCommitted","ctrlCreate","ctrlDelete","ctrlEnable","ctrlEnabled","ctrlFade","ctrlFontHeight","ctrlForegroundColor","ctrlHTMLLoaded","ctrlIDC","ctrlIDD","ctrlMapAnimAdd","ctrlMapAnimClear","ctrlMapAnimCommit","ctrlMapAnimDone","ctrlMapCursor","ctrlMapMouseOver","ctrlMapPosition","ctrlMapScale","ctrlMapScreenToWorld","ctrlMapSetPosition","ctrlMapWorldToScreen","ctrlModel","ctrlModelDirAndUp","ctrlModelScale","ctrlMousePosition","ctrlParent","ctrlParentControlsGroup","ctrlPosition","ctrlRemoveAllEventHandlers","ctrlRemoveEventHandler","ctrlScale","ctrlScrollValues","ctrlSetActiveColor","ctrlSetAngle","ctrlSetAutoScrollDelay","ctrlSetAutoScrollRewind","ctrlSetAutoScrollSpeed","ctrlSetBackgroundColor","ctrlSetChecked","ctrlSetDisabledColor","ctrlSetEventHandler","ctrlSetFade","ctrlSetFocus","ctrlSetFont","ctrlSetFontH1","ctrlSetFontH1B","ctrlSetFontH2","ctrlSetFontH2B","ctrlSetFontH3","ctrlSetFontH3B","ctrlSetFontH4","ctrlSetFontH4B","ctrlSetFontH5","ctrlSetFontH5B","ctrlSetFontH6","ctrlSetFontH6B","ctrlSetFontHeight","ctrlSetFontHeightH1","ctrlSetFontHeightH2","ctrlSetFontHeightH3","ctrlSetFontHeightH4","ctrlSetFontHeightH5","ctrlSetFontHeightH6","ctrlSetFontHeightSecondary","ctrlSetFontP","ctrlSetFontPB","ctrlSetFontSecondary","ctrlSetForegroundColor","ctrlSetModel","ctrlSetModelDirAndUp","ctrlSetModelScale","ctrlSetMousePosition","ctrlSetPixelPrecision","ctrlSetPosition","ctrlSetPositionH","ctrlSetPositionW","ctrlSetPositionX","ctrlSetPositionY","ctrlSetScale","ctrlSetScrollValues","ctrlSetShadow","ctrlSetStructuredText","ctrlSetText","ctrlSetTextColor","ctrlSetTextColorSecondary","ctrlSetTextSecondary","ctrlSetTextSelection","ctrlSetTooltip","ctrlSetTooltipColorBox","ctrlSetTooltipColorShade","ctrlSetTooltipColorText","ctrlSetTooltipMaxWidth","ctrlSetURL","ctrlSetURLOverlayMode","ctrlShadow","ctrlShow","ctrlShown","ctrlStyle","ctrlText","ctrlTextColor","ctrlTextHeight","ctrlTextSecondary","ctrlTextSelection","ctrlTextWidth","ctrlTooltip","ctrlType","ctrlURL","ctrlURLOverlayMode","ctrlVisible","ctRowControls","ctRowCount","ctSetCurSel","ctSetData","ctSetHeaderTemplate","ctSetRowTemplate","ctSetValue","ctValue","curatorAddons","curatorCamera","curatorCameraArea","curatorCameraAreaCeiling","curatorCoef","curatorEditableObjects","curatorEditingArea","curatorEditingAreaType","curatorMouseOver","curatorPoints","curatorRegisteredObjects","curatorSelected","curatorWaypointCost","current3DENOperation","currentChannel","currentCommand","currentMagazine","currentMagazineDetail","currentMagazineDetailTurret","currentMagazineTurret","currentMuzzle","currentNamespace","currentPilot","currentTask","currentTasks","currentThrowable","currentVisionMode","currentWaypoint","currentWeapon","currentWeaponMode","currentWeaponTurret","currentZeroing","cursorObject","cursorTarget","customChat","customRadio","customWaypointPosition","cutFadeOut","cutObj","cutRsc","cutText","damage","date","dateToNumber","dayTime","deActivateKey","debriefingText","debugFSM","debugLog","decayGraphValues","deg","delete3DENEntities","deleteAt","deleteCenter","deleteCollection","deleteEditorObject","deleteGroup","deleteGroupWhenEmpty","deleteIdentity","deleteLocation","deleteMarker","deleteMarkerLocal","deleteRange","deleteResources","deleteSite","deleteStatus","deleteTeam","deleteVehicle","deleteVehicleCrew","deleteWaypoint","detach","detectedMines","diag_activeMissionFSMs","diag_activeScripts","diag_activeSQFScripts","diag_activeSQSScripts","diag_allMissionEventHandlers","diag_captureFrame","diag_captureFrameToFile","diag_captureSlowFrame","diag_codePerformance","diag_deltaTime","diag_drawmode","diag_dumpCalltraceToLog","diag_dumpScriptAssembly","diag_dumpTerrainSynth","diag_dynamicSimulationEnd","diag_enable","diag_enabled","diag_exportConfig","diag_exportTerrainSVG","diag_fps","diag_fpsmin","diag_frameno","diag_getTerrainSegmentOffset","diag_lightNewLoad","diag_list","diag_localized","diag_log","diag_logSlowFrame","diag_mergeConfigFile","diag_recordTurretLimits","diag_resetFSM","diag_resetshapes","diag_scope","diag_setLightNew","diag_stacktrace","diag_tickTime","diag_toggle","dialog","diarySubjectExists","didJIP","didJIPOwner","difficulty","difficultyEnabled","difficultyEnabledRTD","difficultyOption","direction","directionStabilizationEnabled","directSay","disableAI","disableBrakes","disableCollisionWith","disableConversation","disableDebriefingStats","disableMapIndicators","disableNVGEquipment","disableRemoteSensors","disableSerialization","disableTIEquipment","disableUAVConnectability","disableUserInput","displayAddEventHandler","displayChild","displayCtrl","displayParent","displayRemoveAllEventHandlers","displayRemoveEventHandler","displaySetEventHandler","displayUniqueName","displayUpdate","dissolveTeam","distance","distance2D","distanceSqr","distributionRegion","do3DENAction","doArtilleryFire","doFire","doFollow","doFSM","doGetOut","doMove","doorPhase","doStop","doSuppressiveFire","doTarget","doWatch","drawArrow","drawEllipse","drawIcon","drawIcon3D","drawLaser","drawLine","drawLine3D","drawLink","drawLocation","drawPolygon","drawRectangle","drawTriangle","driver","drop","dynamicSimulationDistance","dynamicSimulationDistanceCoef","dynamicSimulationEnabled","dynamicSimulationSystemEnabled","echo","edit3DENMissionAttributes","editObject","editorSetEventHandler","effectiveCommander","elevatePeriscope","emptyPositions","enableAI","enableAIFeature","enableAimPrecision","enableAttack","enableAudioFeature","enableAutoStartUpRTD","enableAutoTrimRTD","enableCamShake","enableCaustics","enableChannel","enableCollisionWith","enableCopilot","enableDebriefingStats","enableDiagLegend","enableDirectionStabilization","enableDynamicSimulation","enableDynamicSimulationSystem","enableEndDialog","enableEngineArtillery","enableEnvironment","enableFatigue","enableGunLights","enableInfoPanelComponent","enableIRLasers","enableMimics","enablePersonTurret","enableRadio","enableReload","enableRopeAttach","enableSatNormalOnDetail","enableSaving","enableSentences","enableSimulation","enableSimulationGlobal","enableStamina","enableStressDamage","enableTeamSwitch","enableTraffic","enableUAVConnectability","enableUAVWaypoints","enableVehicleCargo","enableVehicleSensor","enableWeaponDisassembly","endLoadingScreen","endMission","engineOn","enginesIsOnRTD","enginesPowerRTD","enginesRpmRTD","enginesTorqueRTD","entities","environmentEnabled","environmentVolume","equipmentDisabled","estimatedEndServerTime","estimatedTimeLeft","evalObjectArgument","everyBackpack","everyContainer","exec","execEditorScript","execFSM","execVM","exp","expectedDestination","exportJIPMessages","eyeDirection","eyePos","face","faction","fadeEnvironment","fadeMusic","fadeRadio","fadeSound","fadeSpeech","failMission","fileExists","fillWeaponsFromPool","find","findAny","findCover","findDisplay","findEditorObject","findEmptyPosition","findEmptyPositionReady","findIf","findNearestEnemy","finishMissionInit","finite","fire","fireAtTarget","firstBackpack","flag","flagAnimationPhase","flagOwner","flagSide","flagTexture","flatten","fleeing","floor","flyInHeight","flyInHeightASL","focusedCtrl","fog","fogForecast","fogParams","forceAddUniform","forceAtPositionRTD","forceCadetDifficulty","forcedMap","forceEnd","forceFlagTexture","forceFollowRoad","forceGeneratorRTD","forceMap","forceRespawn","forceSpeed","forceUnicode","forceWalk","forceWeaponFire","forceWeatherChange","forEachMember","forEachMemberAgent","forEachMemberTeam","forgetTarget","format","formation","formationDirection","formationLeader","formationMembers","formationPosition","formationTask","formatText","formLeader","freeExtension","freeLook","fromEditor","fuel","fullCrew","gearIDCAmmoCount","gearSlotAmmoCount","gearSlotData","gestureState","get","get3DENActionState","get3DENAttribute","get3DENCamera","get3DENConnections","get3DENEntity","get3DENEntityID","get3DENGrid","get3DENIconsVisible","get3DENLayerEntities","get3DENLinesVisible","get3DENMissionAttribute","get3DENMouseOver","get3DENSelected","getAimingCoef","getAllEnv3DSoundControllers","getAllEnvSoundControllers","getAllHitPointsDamage","getAllOwnedMines","getAllPylonsInfo","getAllSoundControllers","getAllUnitTraits","getAmmoCargo","getAnimAimPrecision","getAnimSpeedCoef","getArray","getArtilleryAmmo","getArtilleryComputerSettings","getArtilleryETA","getAssetDLCInfo","getAssignedCuratorLogic","getAssignedCuratorUnit","getAttackTarget","getAudioOptionVolumes","getBackpackCargo","getBleedingRemaining","getBurningValue","getCalculatePlayerVisibilityByFriendly","getCameraViewDirection","getCargoIndex","getCenterOfMass","getClientState","getClientStateNumber","getCompatiblePylonMagazines","getConnectedUAV","getConnectedUAVUnit","getContainerMaxLoad","getCorpse","getCruiseControl","getCursorObjectParams","getCustomAimCoef","getCustomSoundController","getCustomSoundControllerCount","getDammage","getDebriefingText","getDescription","getDir","getDirVisual","getDiverState","getDLCAssetsUsage","getDLCAssetsUsageByName","getDLCs","getDLCUsageTime","getEditorCamera","getEditorMode","getEditorObjectScope","getElevationOffset","getEngineTargetRPMRTD","getEnv3DSoundController","getEnvSoundController","getEventHandlerInfo","getFatigue","getFieldManualStartPage","getForcedFlagTexture","getForcedSpeed","getFriend","getFSMVariable","getFuelCargo","getGraphValues","getGroupIcon","getGroupIconParams","getGroupIcons","getHideFrom","getHit","getHitIndex","getHitPointDamage","getItemCargo","getLighting","getLightingAt","getLoadedModsInfo","getMagazineCargo","getMarkerColor","getMarkerPos","getMarkerSize","getMarkerType","getMass","getMissionConfig","getMissionConfigValue","getMissionDLCs","getMissionLayerEntities","getMissionLayers","getMissionPath","getModelInfo","getMousePosition","getMusicPlayedTime","getNumber","getObjectArgument","getObjectChildren","getObjectDLC","getObjectFOV","getObjectID","getObjectMaterials","getObjectProxy","getObjectScale","getObjectTextures","getObjectType","getObjectViewDistance","getOpticsMode","getOrDefault","getOrDefaultCall","getOxygenRemaining","getPersonUsedDLCs","getPilotCameraDirection","getPilotCameraPosition","getPilotCameraRotation","getPilotCameraTarget","getPiPViewDistance","getPlateNumber","getPlayerChannel","getPlayerID","getPlayerScores","getPlayerUID","getPlayerVoNVolume","getPos","getPosASL","getPosASLVisual","getPosASLW","getPosATL","getPosATLVisual","getPosVisual","getPosWorld","getPosWorldVisual","getPylonMagazines","getRelDir","getRelPos","getRemoteSensorsDisabled","getRepairCargo","getResolution","getRoadInfo","getRotorBrakeRTD","getSensorTargets","getSensorThreats","getShadowDistance","getShotParents","getSlingLoad","getSoundController","getSoundControllerResult","getSpeed","getStamina","getStatValue","getSteamFriendsServers","getSubtitleOptions","getSuppression","getTerrainGrid","getTerrainHeight","getTerrainHeightASL","getTerrainInfo","getText","getTextRaw","getTextureInfo","getTextWidth","getTiParameters","getTotalDLCUsageTime","getTrimOffsetRTD","getTurretLimits","getTurretOpticsMode","getUnitFreefallInfo","getUnitLoadout","getUnitTrait","getUnloadInCombat","getUserInfo","getUserMFDText","getUserMFDValue","getVariable","getVehicleCargo","getVehicleTiPars","getWeaponCargo","getWeaponSway","getWingsOrientationRTD","getWingsPositionRTD","getWPPos","glanceAt","globalChat","globalRadio","goggles","goto","group","groupChat","groupFromNetId","groupIconSelectable","groupIconsVisible","groupID","groupOwner","groupRadio","groups","groupSelectedUnits","groupSelectUnit","gunner","gusts","halt","handgunItems","handgunMagazine","handgunWeapon","handsHit","hashValue","hasInterface","hasPilotCamera","hasWeapon","hcAllGroups","hcGroupParams","hcLeader","hcRemoveAllGroups","hcRemoveGroup","hcSelected","hcSelectGroup","hcSetGroup","hcShowBar","hcShownBar","headgear","hideBody","hideObject","hideObjectGlobal","hideSelection","hint","hintC","hintCadet","hintSilent","hmd","hostMission","htmlLoad","HUDMovementLevels","humidity","image","importAllGroups","importance","in","inArea","inAreaArray","incapacitatedState","inflame","inflamed","infoPanel","infoPanelComponentEnabled","infoPanelComponents","infoPanels","inGameUISetEventHandler","inheritsFrom","initAmbientLife","inPolygon","inputAction","inputController","inputMouse","inRangeOfArtillery","insert","insertEditorObject","intersect","is3DEN","is3DENMultiplayer","is3DENPreview","isAbleToBreathe","isActionMenuVisible","isAgent","isAimPrecisionEnabled","isAllowedCrewInImmobile","isArray","isAutoHoverOn","isAutonomous","isAutoStartUpEnabledRTD","isAutotest","isAutoTrimOnRTD","isAwake","isBleeding","isBurning","isClass","isCollisionLightOn","isCopilotEnabled","isDamageAllowed","isDedicated","isDLCAvailable","isEngineOn","isEqualRef","isEqualTo","isEqualType","isEqualTypeAll","isEqualTypeAny","isEqualTypeArray","isEqualTypeParams","isFilePatchingEnabled","isFinal","isFlashlightOn","isFlatEmpty","isForcedWalk","isFormationLeader","isGameFocused","isGamePaused","isGroupDeletedWhenEmpty","isHidden","isInRemainsCollector","isInstructorFigureEnabled","isIRLaserOn","isKeyActive","isKindOf","isLaserOn","isLightOn","isLocalized","isManualFire","isMarkedForCollection","isMissionProfileNamespaceLoaded","isMultiplayer","isMultiplayerSolo","isNil","isNotEqualRef","isNotEqualTo","isNull","isNumber","isObjectHidden","isObjectRTD","isOnRoad","isPiPEnabled","isPlayer","isRealTime","isRemoteExecuted","isRemoteExecutedJIP","isSaving","isSensorTargetConfirmed","isServer","isShowing3DIcons","isSimpleObject","isSprintAllowed","isStaminaEnabled","isSteamMission","isSteamOverlayEnabled","isStreamFriendlyUIEnabled","isStressDamageEnabled","isText","isTouchingGround","isTurnedOut","isTutHintsEnabled","isUAVConnectable","isUAVConnected","isUIContext","isUniformAllowed","isVehicleCargo","isVehicleRadarOn","isVehicleSensorEnabled","isWalking","isWeaponDeployed","isWeaponRested","itemCargo","items","itemsWithMagazines","join","joinAs","joinAsSilent","joinSilent","joinString","kbAddDatabase","kbAddDatabaseTargets","kbAddTopic","kbHasTopic","kbReact","kbRemoveTopic","kbTell","kbWasSaid","keyImage","keyName","keys","knowsAbout","land","landAt","landResult","language","laserTarget","lbAdd","lbClear","lbColor","lbColorRight","lbCurSel","lbData","lbDelete","lbIsSelected","lbPicture","lbPictureRight","lbSelection","lbSetColor","lbSetColorRight","lbSetCurSel","lbSetData","lbSetPicture","lbSetPictureColor","lbSetPictureColorDisabled","lbSetPictureColorSelected","lbSetPictureRight","lbSetPictureRightColor","lbSetPictureRightColorDisabled","lbSetPictureRightColorSelected","lbSetSelectColor","lbSetSelectColorRight","lbSetSelected","lbSetText","lbSetTextRight","lbSetTooltip","lbSetValue","lbSize","lbSort","lbSortBy","lbSortByValue","lbText","lbTextRight","lbTooltip","lbValue","leader","leaderboardDeInit","leaderboardGetRows","leaderboardInit","leaderboardRequestRowsFriends","leaderboardRequestRowsGlobal","leaderboardRequestRowsGlobalAroundUser","leaderboardsRequestUploadScore","leaderboardsRequestUploadScoreKeepBest","leaderboardState","leaveVehicle","libraryCredits","libraryDisclaimers","lifeState","lightAttachObject","lightDetachObject","lightIsOn","lightnings","limitSpeed","linearConversion","lineIntersects","lineIntersectsObjs","lineIntersectsSurfaces","lineIntersectsWith","linkItem","list","listObjects","listRemoteTargets","listVehicleSensors","ln","lnbAddArray","lnbAddColumn","lnbAddRow","lnbClear","lnbColor","lnbColorRight","lnbCurSelRow","lnbData","lnbDeleteColumn","lnbDeleteRow","lnbGetColumnsPosition","lnbPicture","lnbPictureRight","lnbSetColor","lnbSetColorRight","lnbSetColumnsPos","lnbSetCurSelRow","lnbSetData","lnbSetPicture","lnbSetPictureColor","lnbSetPictureColorRight","lnbSetPictureColorSelected","lnbSetPictureColorSelectedRight","lnbSetPictureRight","lnbSetText","lnbSetTextRight","lnbSetTooltip","lnbSetValue","lnbSize","lnbSort","lnbSortBy","lnbSortByValue","lnbText","lnbTextRight","lnbValue","load","loadAbs","loadBackpack","loadConfig","loadFile","loadGame","loadIdentity","loadMagazine","loadOverlay","loadStatus","loadUniform","loadVest","localize","localNamespace","locationPosition","lock","lockCameraTo","lockCargo","lockDriver","locked","lockedCameraTo","lockedCargo","lockedDriver","lockedInventory","lockedTurret","lockIdentity","lockInventory","lockTurret","lockWp","log","logEntities","logNetwork","logNetworkTerminate","lookAt","lookAtPos","magazineCargo","magazines","magazinesAllTurrets","magazinesAmmo","magazinesAmmoCargo","magazinesAmmoFull","magazinesDetail","magazinesDetailBackpack","magazinesDetailUniform","magazinesDetailVest","magazinesTurret","magazineTurretAmmo","mapAnimAdd","mapAnimClear","mapAnimCommit","mapAnimDone","mapCenterOnCamera","mapGridPosition","markAsFinishedOnSteam","markerAlpha","markerBrush","markerChannel","markerColor","markerDir","markerPolyline","markerPos","markerShadow","markerShape","markerSize","markerText","markerType","matrixMultiply","matrixTranspose","max","maxLoad","members","menuAction","menuAdd","menuChecked","menuClear","menuCollapse","menuData","menuDelete","menuEnable","menuEnabled","menuExpand","menuHover","menuPicture","menuSetAction","menuSetCheck","menuSetData","menuSetPicture","menuSetShortcut","menuSetText","menuSetURL","menuSetValue","menuShortcut","menuShortcutText","menuSize","menuSort","menuText","menuURL","menuValue","merge","min","mineActive","mineDetectedBy","missileTarget","missileTargetPos","missionConfigFile","missionDifficulty","missionEnd","missionName","missionNameSource","missionNamespace","missionProfileNamespace","missionStart","missionVersion","mod","modelToWorld","modelToWorldVisual","modelToWorldVisualWorld","modelToWorldWorld","modParams","moonIntensity","moonPhase","morale","move","move3DENCamera","moveInAny","moveInCargo","moveInCommander","moveInDriver","moveInGunner","moveInTurret","moveObjectToEnd","moveOut","moveTime","moveTo","moveToCompleted","moveToFailed","musicVolume","name","namedProperties","nameSound","nearEntities","nearestBuilding","nearestLocation","nearestLocations","nearestLocationWithDubbing","nearestMines","nearestObject","nearestObjects","nearestTerrainObjects","nearObjects","nearObjectsReady","nearRoads","nearSupplies","nearTargets","needReload","needService","netId","netObjNull","newOverlay","nextMenuItemIndex","nextWeatherChange","nMenuItems","not","numberOfEnginesRTD","numberToDate","objectCurators","objectFromNetId","objectParent","objStatus","onBriefingGroup","onBriefingNotes","onBriefingPlan","onBriefingTeamSwitch","onCommandModeChanged","onDoubleClick","onEachFrame","onGroupIconClick","onGroupIconOverEnter","onGroupIconOverLeave","onHCGroupSelectionChanged","onMapSingleClick","onPlayerConnected","onPlayerDisconnected","onPreloadFinished","onPreloadStarted","onShowNewObject","onTeamSwitch","openCuratorInterface","openDLCPage","openGPS","openMap","openSteamApp","openYoutubeVideo","or","orderGetIn","overcast","overcastForecast","owner","param","params","parseNumber","parseSimpleArray","parseText","parsingNamespace","particlesQuality","periscopeElevation","pickWeaponPool","pitch","pixelGrid","pixelGridBase","pixelGridNoUIScale","pixelH","pixelW","playableSlotsNumber","playableUnits","playAction","playActionNow","player","playerRespawnTime","playerSide","playersNumber","playGesture","playMission","playMove","playMoveNow","playMusic","playScriptedMission","playSound","playSound3D","playSoundUI","pose","position","positionCameraToWorld","posScreenToWorld","posWorldToScreen","ppEffectAdjust","ppEffectCommit","ppEffectCommitted","ppEffectCreate","ppEffectDestroy","ppEffectEnable","ppEffectEnabled","ppEffectForceInNVG","precision","preloadCamera","preloadObject","preloadSound","preloadTitleObj","preloadTitleRsc","preprocessFile","preprocessFileLineNumbers","primaryWeapon","primaryWeaponItems","primaryWeaponMagazine","priority","processDiaryLink","productVersion","profileName","profileNamespace","profileNameSteam","progressLoadingScreen","progressPosition","progressSetPosition","publicVariable","publicVariableClient","publicVariableServer","pushBack","pushBackUnique","putWeaponPool","queryItemsPool","queryMagazinePool","queryWeaponPool","rad","radioChannelAdd","radioChannelCreate","radioChannelInfo","radioChannelRemove","radioChannelSetCallSign","radioChannelSetLabel","radioEnabled","radioVolume","rain","rainbow","rainParams","random","rank","rankId","rating","rectangular","regexFind","regexMatch","regexReplace","registeredTasks","registerTask","reload","reloadEnabled","remoteControl","remoteExec","remoteExecCall","remoteExecutedOwner","remove3DENConnection","remove3DENEventHandler","remove3DENLayer","removeAction","removeAll3DENEventHandlers","removeAllActions","removeAllAssignedItems","removeAllBinocularItems","removeAllContainers","removeAllCuratorAddons","removeAllCuratorCameraAreas","removeAllCuratorEditingAreas","removeAllEventHandlers","removeAllHandgunItems","removeAllItems","removeAllItemsWithMagazines","removeAllMissionEventHandlers","removeAllMPEventHandlers","removeAllMusicEventHandlers","removeAllOwnedMines","removeAllPrimaryWeaponItems","removeAllSecondaryWeaponItems","removeAllUserActionEventHandlers","removeAllWeapons","removeBackpack","removeBackpackGlobal","removeBinocularItem","removeCuratorAddons","removeCuratorCameraArea","removeCuratorEditableObjects","removeCuratorEditingArea","removeDiaryRecord","removeDiarySubject","removeDrawIcon","removeDrawLinks","removeEventHandler","removeFromRemainsCollector","removeGoggles","removeGroupIcon","removeHandgunItem","removeHeadgear","removeItem","removeItemFromBackpack","removeItemFromUniform","removeItemFromVest","removeItems","removeMagazine","removeMagazineGlobal","removeMagazines","removeMagazinesTurret","removeMagazineTurret","removeMenuItem","removeMissionEventHandler","removeMPEventHandler","removeMusicEventHandler","removeOwnedMine","removePrimaryWeaponItem","removeSecondaryWeaponItem","removeSimpleTask","removeSwitchableUnit","removeTeamMember","removeUniform","removeUserActionEventHandler","removeVest","removeWeapon","removeWeaponAttachmentCargo","removeWeaponCargo","removeWeaponGlobal","removeWeaponTurret","reportRemoteTarget","requiredVersion","resetCamShake","resetSubgroupDirection","resize","resources","respawnVehicle","restartEditorCamera","reveal","revealMine","reverse","reversedMouseY","roadAt","roadsConnectedTo","roleDescription","ropeAttachedObjects","ropeAttachedTo","ropeAttachEnabled","ropeAttachTo","ropeCreate","ropeCut","ropeDestroy","ropeDetach","ropeEndPosition","ropeLength","ropes","ropesAttachedTo","ropeSegments","ropeUnwind","ropeUnwound","rotorsForcesRTD","rotorsRpmRTD","round","runInitScript","safeZoneH","safeZoneW","safeZoneWAbs","safeZoneX","safeZoneXAbs","safeZoneY","save3DENInventory","saveGame","saveIdentity","saveJoysticks","saveMissionProfileNamespace","saveOverlay","saveProfileNamespace","saveStatus","saveVar","savingEnabled","say","say2D","say3D","scopeName","score","scoreSide","screenshot","screenToWorld","scriptDone","scriptName","scudState","secondaryWeapon","secondaryWeaponItems","secondaryWeaponMagazine","select","selectBestPlaces","selectDiarySubject","selectedEditorObjects","selectEditorObject","selectionNames","selectionPosition","selectionVectorDirAndUp","selectLeader","selectMax","selectMin","selectNoPlayer","selectPlayer","selectRandom","selectRandomWeighted","selectWeapon","selectWeaponTurret","sendAUMessage","sendSimpleCommand","sendTask","sendTaskResult","sendUDPMessage","sentencesEnabled","serverCommand","serverCommandAvailable","serverCommandExecutable","serverName","serverNamespace","serverTime","set","set3DENAttribute","set3DENAttributes","set3DENGrid","set3DENIconsVisible","set3DENLayer","set3DENLinesVisible","set3DENLogicType","set3DENMissionAttribute","set3DENMissionAttributes","set3DENModelsVisible","set3DENObjectType","set3DENSelected","setAccTime","setActualCollectiveRTD","setAirplaneThrottle","setAirportSide","setAmmo","setAmmoCargo","setAmmoOnPylon","setAnimSpeedCoef","setAperture","setApertureNew","setArmoryPoints","setAttributes","setAutonomous","setBehaviour","setBehaviourStrong","setBleedingRemaining","setBrakesRTD","setCameraInterest","setCamShakeDefParams","setCamShakeParams","setCamUseTi","setCaptive","setCenterOfMass","setCollisionLight","setCombatBehaviour","setCombatMode","setCompassOscillation","setConvoySeparation","setCruiseControl","setCuratorCameraAreaCeiling","setCuratorCoef","setCuratorEditingAreaType","setCuratorWaypointCost","setCurrentChannel","setCurrentTask","setCurrentWaypoint","setCustomAimCoef","SetCustomMissionData","setCustomSoundController","setCustomWeightRTD","setDamage","setDammage","setDate","setDebriefingText","setDefaultCamera","setDestination","setDetailMapBlendPars","setDiaryRecordText","setDiarySubjectPicture","setDir","setDirection","setDrawIcon","setDriveOnPath","setDropInterval","setDynamicSimulationDistance","setDynamicSimulationDistanceCoef","setEditorMode","setEditorObjectScope","setEffectCondition","setEffectiveCommander","setEngineRpmRTD","setFace","setFaceanimation","setFatigue","setFeatureType","setFlagAnimationPhase","setFlagOwner","setFlagSide","setFlagTexture","setFog","setForceGeneratorRTD","setFormation","setFormationTask","setFormDir","setFriend","setFromEditor","setFSMVariable","setFuel","setFuelCargo","setGroupIcon","setGroupIconParams","setGroupIconsSelectable","setGroupIconsVisible","setGroupid","setGroupIdGlobal","setGroupOwner","setGusts","setHideBehind","setHit","setHitIndex","setHitPointDamage","setHorizonParallaxCoef","setHUDMovementLevels","setHumidity","setIdentity","setImportance","setInfoPanel","setLeader","setLightAmbient","setLightAttenuation","setLightBrightness","setLightColor","setLightConePars","setLightDayLight","setLightFlareMaxDistance","setLightFlareSize","setLightIntensity","setLightIR","setLightnings","setLightUseFlare","setLightVolumeShape","setLocalWindParams","setMagazineTurretAmmo","setMarkerAlpha","setMarkerAlphaLocal","setMarkerBrush","setMarkerBrushLocal","setMarkerColor","setMarkerColorLocal","setMarkerDir","setMarkerDirLocal","setMarkerPolyline","setMarkerPolylineLocal","setMarkerPos","setMarkerPosLocal","setMarkerShadow","setMarkerShadowLocal","setMarkerShape","setMarkerShapeLocal","setMarkerSize","setMarkerSizeLocal","setMarkerText","setMarkerTextLocal","setMarkerType","setMarkerTypeLocal","setMass","setMaxLoad","setMimic","setMissileTarget","setMissileTargetPos","setMousePosition","setMusicEffect","setMusicEventHandler","setName","setNameSound","setObjectArguments","setObjectMaterial","setObjectMaterialGlobal","setObjectProxy","setObjectScale","setObjectTexture","setObjectTextureGlobal","setObjectViewDistance","setOpticsMode","setOvercast","setOwner","setOxygenRemaining","setParticleCircle","setParticleClass","setParticleFire","setParticleParams","setParticleRandom","setPilotCameraDirection","setPilotCameraRotation","setPilotCameraTarget","setPilotLight","setPiPEffect","setPiPViewDistance","setPitch","setPlateNumber","setPlayable","setPlayerRespawnTime","setPlayerVoNVolume","setPos","setPosASL","setPosASL2","setPosASLW","setPosATL","setPosition","setPosWorld","setPylonLoadout","setPylonsPriority","setRadioMsg","setRain","setRainbow","setRandomLip","setRank","setRectangular","setRepairCargo","setRotorBrakeRTD","setShadowDistance","setShotParents","setSide","setSimpleTaskAlwaysVisible","setSimpleTaskCustomData","setSimpleTaskDescription","setSimpleTaskDestination","setSimpleTaskTarget","setSimpleTaskType","setSimulWeatherLayers","setSize","setSkill","setSlingLoad","setSoundEffect","setSpeaker","setSpeech","setSpeedMode","setStamina","setStaminaScheme","setStatValue","setSuppression","setSystemOfUnits","setTargetAge","setTaskMarkerOffset","setTaskResult","setTaskState","setTerrainGrid","setTerrainHeight","setText","setTimeMultiplier","setTiParameter","setTitleEffect","setTowParent","setTrafficDensity","setTrafficDistance","setTrafficGap","setTrafficSpeed","setTriggerActivation","setTriggerArea","setTriggerInterval","setTriggerStatements","setTriggerText","setTriggerTimeout","setTriggerType","setTurretLimits","setTurretOpticsMode","setType","setUnconscious","setUnitAbility","setUnitCombatMode","setUnitFreefallHeight","setUnitLoadout","setUnitPos","setUnitPosWeak","setUnitRank","setUnitRecoilCoefficient","setUnitTrait","setUnloadInCombat","setUserActionText","setUserMFDText","setUserMFDValue","setVariable","setVectorDir","setVectorDirAndUp","setVectorUp","setVehicleAmmo","setVehicleAmmoDef","setVehicleArmor","setVehicleCargo","setVehicleId","setVehicleLock","setVehiclePosition","setVehicleRadar","setVehicleReceiveRemoteTargets","setVehicleReportOwnPosition","setVehicleReportRemoteTargets","setVehicleTiPars","setVehicleVarName","setVelocity","setVelocityModelSpace","setVelocityTransformation","setViewDistance","setVisibleIfTreeCollapsed","setWantedRPMRTD","setWaves","setWaypointBehaviour","setWaypointCombatMode","setWaypointCompletionRadius","setWaypointDescription","setWaypointForceBehaviour","setWaypointFormation","setWaypointHousePosition","setWaypointLoiterAltitude","setWaypointLoiterRadius","setWaypointLoiterType","setWaypointName","setWaypointPosition","setWaypointScript","setWaypointSpeed","setWaypointStatements","setWaypointTimeout","setWaypointType","setWaypointVisible","setWeaponReloadingTime","setWeaponZeroing","setWind","setWindDir","setWindForce","setWindStr","setWingForceScaleRTD","setWPPos","show3DIcons","showChat","showCinemaBorder","showCommandingMenu","showCompass","showCuratorCompass","showGps","showHUD","showLegend","showMap","shownArtilleryComputer","shownChat","shownCompass","shownCuratorCompass","showNewEditorObject","shownGps","shownHUD","shownMap","shownPad","shownRadio","shownScoretable","shownSubtitles","shownUAVFeed","shownWarrant","shownWatch","showPad","showRadio","showScoretable","showSubtitles","showUAVFeed","showWarrant","showWatch","showWaypoint","showWaypoints","side","sideChat","sideRadio","simpleTasks","simulationEnabled","simulCloudDensity","simulCloudOcclusion","simulInClouds","simulWeatherSync","sin","size","sizeOf","skill","skillFinal","skipTime","sleep","sliderPosition","sliderRange","sliderSetPosition","sliderSetRange","sliderSetSpeed","sliderSpeed","slingLoadAssistantShown","soldierMagazines","someAmmo","sort","soundVolume","spawn","speaker","speechVolume","speed","speedMode","splitString","sqrt","squadParams","stance","startLoadingScreen","stop","stopEngineRTD","stopped","str","sunOrMoon","supportInfo","suppressFor","surfaceIsWater","surfaceNormal","surfaceTexture","surfaceType","swimInDepth","switchableUnits","switchAction","switchCamera","switchGesture","switchLight","switchMove","synchronizedObjects","synchronizedTriggers","synchronizedWaypoints","synchronizeObjectsAdd","synchronizeObjectsRemove","synchronizeTrigger","synchronizeWaypoint","systemChat","systemOfUnits","systemTime","systemTimeUTC","tan","targetKnowledge","targets","targetsAggregate","targetsQuery","taskAlwaysVisible","taskChildren","taskCompleted","taskCustomData","taskDescription","taskDestination","taskHint","taskMarkerOffset","taskName","taskParent","taskResult","taskState","taskType","teamMember","teamName","teams","teamSwitch","teamSwitchEnabled","teamType","terminate","terrainIntersect","terrainIntersectASL","terrainIntersectAtASL","text","textLog","textLogFormat","tg","time","timeMultiplier","titleCut","titleFadeOut","titleObj","titleRsc","titleText","toArray","toFixed","toLower","toLowerANSI","toString","toUpper","toUpperANSI","triggerActivated","triggerActivation","triggerAmmo","triggerArea","triggerAttachedVehicle","triggerAttachObject","triggerAttachVehicle","triggerDynamicSimulation","triggerInterval","triggerStatements","triggerText","triggerTimeout","triggerTimeoutCurrent","triggerType","trim","turretLocal","turretOwner","turretUnit","tvAdd","tvClear","tvCollapse","tvCollapseAll","tvCount","tvCurSel","tvData","tvDelete","tvExpand","tvExpandAll","tvIsSelected","tvPicture","tvPictureRight","tvSelection","tvSetColor","tvSetCurSel","tvSetData","tvSetPicture","tvSetPictureColor","tvSetPictureColorDisabled","tvSetPictureColorSelected","tvSetPictureRight","tvSetPictureRightColor","tvSetPictureRightColorDisabled","tvSetPictureRightColorSelected","tvSetSelectColor","tvSetSelected","tvSetText","tvSetTooltip","tvSetValue","tvSort","tvSortAll","tvSortByValue","tvSortByValueAll","tvText","tvTooltip","tvValue","type","typeName","typeOf","UAVControl","uiNamespace","uiSleep","unassignCurator","unassignItem","unassignTeam","unassignVehicle","underwater","uniform","uniformContainer","uniformItems","uniformMagazines","uniqueUnitItems","unitAddons","unitAimPosition","unitAimPositionVisual","unitBackpack","unitCombatMode","unitIsUAV","unitPos","unitReady","unitRecoilCoefficient","units","unitsBelowHeight","unitTurret","unlinkItem","unlockAchievement","unregisterTask","updateDrawIcon","updateMenuItem","updateObjectTree","useAIOperMapObstructionTest","useAISteeringComponent","useAudioTimeForMoves","userInputDisabled","values","vectorAdd","vectorCos","vectorCrossProduct","vectorDiff","vectorDir","vectorDirVisual","vectorDistance","vectorDistanceSqr","vectorDotProduct","vectorFromTo","vectorLinearConversion","vectorMagnitude","vectorMagnitudeSqr","vectorModelToWorld","vectorModelToWorldVisual","vectorMultiply","vectorNormalized","vectorUp","vectorUpVisual","vectorWorldToModel","vectorWorldToModelVisual","vehicle","vehicleCargoEnabled","vehicleChat","vehicleMoveInfo","vehicleRadio","vehicleReceiveRemoteTargets","vehicleReportOwnPosition","vehicleReportRemoteTargets","vehicles","vehicleVarName","velocity","velocityModelSpace","verifySignature","vest","vestContainer","vestItems","vestMagazines","viewDistance","visibleCompass","visibleGps","visibleMap","visiblePosition","visiblePositionASL","visibleScoretable","visibleWatch","waves","waypointAttachedObject","waypointAttachedVehicle","waypointAttachObject","waypointAttachVehicle","waypointBehaviour","waypointCombatMode","waypointCompletionRadius","waypointDescription","waypointForceBehaviour","waypointFormation","waypointHousePosition","waypointLoiterAltitude","waypointLoiterRadius","waypointLoiterType","waypointName","waypointPosition","waypoints","waypointScript","waypointsEnabledUAV","waypointShow","waypointSpeed","waypointStatements","waypointTimeout","waypointTimeoutCurrent","waypointType","waypointVisible","weaponAccessories","weaponAccessoriesCargo","weaponCargo","weaponDirection","weaponInertia","weaponLowered","weaponReloadingTime","weapons","weaponsInfo","weaponsItems","weaponsItemsCargo","weaponState","weaponsTurret","weightRTD","WFSideText","wind","windDir","windRTD","windStr","wingsForcesRTD","worldName","worldSize","worldToModel","worldToModelVisual","worldToScreen"],literal:["blufor","civilian","configNull","controlNull","displayNull","diaryRecordNull","east","endl","false","grpNull","independent","lineBreak","locationNull","nil","objNull","opfor","pi","resistance","scriptNull","sideAmbientLife","sideEmpty","sideEnemy","sideFriendly","sideLogic","sideUnknown","taskNull","teamMemberNull","true","west"]},contains:[e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE,e.NUMBER_MODE,{className:"variable",begin:/\b_+[a-zA-Z]\w*/},{className:"title",begin:/[a-zA-Z][a-zA-Z_0-9]*_fnc_[a-zA-Z_0-9]+/},t,n],illegal:[/\$[^a-fA-F0-9]/,/\w\$/,/\?/,/@/,/ \| /,/[a-zA-Z_]\./,/\:\=/,/\[\:/]}}},6398(e){e.exports=function(e){const t=e.regex,n=/([-a-zA-Z$._][\w$.-]*)/,a={className:"variable",variants:[{begin:t.concat(/%/,n)},{begin:/%\d+/},{begin:/#\d+/}]},r={className:"title",variants:[{begin:t.concat(/@/,n)},{begin:/@\d+/},{begin:t.concat(/!/,n)},{begin:t.concat(/!\d+/,n)},{begin:/!\d+/}]};return{name:"LLVM IR",keywords:{keyword:"begin end true false declare define global constant private linker_private internal available_externally linkonce linkonce_odr weak weak_odr appending dllimport dllexport common default hidden protected extern_weak external thread_local zeroinitializer undef null to tail target triple datalayout volatile nuw nsw nnan ninf nsz arcp fast exact inbounds align addrspace section alias module asm sideeffect gc dbg linker_private_weak attributes blockaddress initialexec localdynamic localexec prefix unnamed_addr ccc fastcc coldcc x86_stdcallcc x86_fastcallcc arm_apcscc arm_aapcscc arm_aapcs_vfpcc ptx_device ptx_kernel intel_ocl_bicc msp430_intrcc spir_func spir_kernel x86_64_sysvcc x86_64_win64cc x86_thiscallcc cc c signext zeroext inreg sret nounwind noreturn noalias nocapture byval nest readnone readonly inlinehint noinline alwaysinline optsize ssp sspreq noredzone noimplicitfloat naked builtin cold nobuiltin noduplicate nonlazybind optnone returns_twice sanitize_address sanitize_memory sanitize_thread sspstrong uwtable returned type opaque eq ne slt sgt sle sge ult ugt ule uge oeq one olt ogt ole oge ord uno ueq une x acq_rel acquire alignstack atomic catch cleanup filter inteldialect max min monotonic nand personality release seq_cst singlethread umax umin unordered xchg add fadd sub fsub mul fmul udiv sdiv fdiv urem srem frem shl lshr ashr and or xor icmp fcmp phi call trunc zext sext fptrunc fpext uitofp sitofp fptoui fptosi inttoptr ptrtoint bitcast addrspacecast select va_arg ret br switch invoke unwind unreachable indirectbr landingpad resume malloc alloca free load store getelementptr extractelement insertelement shufflevector getresult extractvalue insertvalue atomicrmw cmpxchg fence argmemonly",type:"void half bfloat float double fp128 x86_fp80 ppc_fp128 x86_amx x86_mmx ptr label token metadata opaque"},contains:[{className:"type",begin:/\bi\d+(?=\s|\b)/},e.COMMENT(/;\s*$/,null,{relevance:0}),e.COMMENT(/;/,/$/),{className:"string",begin:/"/,end:/"/,contains:[{className:"char.escape",match:/\\\d\d/}]},r,{className:"punctuation",relevance:0,begin:/,/},{className:"operator",relevance:0,begin:/=/},a,{className:"symbol",variants:[{begin:/^\s*[a-z]+:/}],relevance:0},{className:"number",variants:[{begin:/[su]?0[xX][KMLHR]?[a-fA-F0-9]+/},{begin:/[-+]?\d+(?:[.]\d+)?(?:[eE][-+]?\d+(?:[.]\d+)?)?/}],relevance:0}]}}},6446(e){e.exports=function(e){return{name:"ReasonML",aliases:["re"],keywords:{$pattern:/[a-z_]\w*!?/,keyword:["and","as","asr","assert","begin","class","constraint","do","done","downto","else","end","esfun","exception","external","for","fun","function","functor","if","in","include","inherit","initializer","land","lazy","let","lor","lsl","lsr","lxor","mod","module","mutable","new","nonrec","object","of","open","or","pri","pub","rec","sig","struct","switch","then","to","try","type","val","virtual","when","while","with"],built_in:["array","bool","bytes","char","exn|5","float","int","int32","int64","list","lazy_t|5","nativeint|5","ref","string","unit"],literal:["true","false"]},illegal:/(:-|:=|\$\{|\+=)/,contains:[{scope:"literal",match:/\[(\|\|)?\]|\(\)/,relevance:0},e.C_LINE_COMMENT_MODE,e.COMMENT(/\/\*/,/\*\//,{illegal:/^(#,\/\/)/}),{scope:"symbol",match:/\'[A-Za-z_](?!\')[\w\']*/},{scope:"type",match:/`[A-Z][\w\']*/},{scope:"type",match:/\b[A-Z][\w\']*/,relevance:0},{match:/[a-z_]\w*\'[\w\']*/,relevance:0},{scope:"operator",match:/\s+(\|\||\+[\+\.]?|\*[\*\/\.]?|\/[\.]?|\.\.\.|\|>|&&|===?)\s+/,relevance:0},e.inherit(e.APOS_STRING_MODE,{scope:"string",relevance:0}),e.inherit(e.QUOTE_STRING_MODE,{illegal:null}),{scope:"number",variants:[{match:/\b0[xX][a-fA-F0-9_]+[Lln]?/},{match:/\b0[oO][0-7_]+[Lln]?/},{match:/\b0[bB][01_]+[Lln]?/},{match:/\b[0-9][0-9_]*([Lln]|(\.[0-9_]*)?([eE][-+]?[0-9_]+)?)/}],relevance:0}]}}},6455(e){const t=["AASTriangle","AbelianGroup","Abort","AbortKernels","AbortProtect","AbortScheduledTask","Above","Abs","AbsArg","AbsArgPlot","Absolute","AbsoluteCorrelation","AbsoluteCorrelationFunction","AbsoluteCurrentValue","AbsoluteDashing","AbsoluteFileName","AbsoluteOptions","AbsolutePointSize","AbsoluteThickness","AbsoluteTime","AbsoluteTiming","AcceptanceThreshold","AccountingForm","Accumulate","Accuracy","AccuracyGoal","AcousticAbsorbingValue","AcousticImpedanceValue","AcousticNormalVelocityValue","AcousticPDEComponent","AcousticPressureCondition","AcousticRadiationValue","AcousticSoundHardValue","AcousticSoundSoftCondition","ActionDelay","ActionMenu","ActionMenuBox","ActionMenuBoxOptions","Activate","Active","ActiveClassification","ActiveClassificationObject","ActiveItem","ActivePrediction","ActivePredictionObject","ActiveStyle","AcyclicGraphQ","AddOnHelpPath","AddSides","AddTo","AddToSearchIndex","AddUsers","AdjacencyGraph","AdjacencyList","AdjacencyMatrix","AdjacentMeshCells","Adjugate","AdjustmentBox","AdjustmentBoxOptions","AdjustTimeSeriesForecast","AdministrativeDivisionData","AffineHalfSpace","AffineSpace","AffineStateSpaceModel","AffineTransform","After","AggregatedEntityClass","AggregationLayer","AircraftData","AirportData","AirPressureData","AirSoundAttenuation","AirTemperatureData","AiryAi","AiryAiPrime","AiryAiZero","AiryBi","AiryBiPrime","AiryBiZero","AlgebraicIntegerQ","AlgebraicNumber","AlgebraicNumberDenominator","AlgebraicNumberNorm","AlgebraicNumberPolynomial","AlgebraicNumberTrace","AlgebraicRules","AlgebraicRulesData","Algebraics","AlgebraicUnitQ","Alignment","AlignmentMarker","AlignmentPoint","All","AllowAdultContent","AllowChatServices","AllowedCloudExtraParameters","AllowedCloudParameterExtensions","AllowedDimensions","AllowedFrequencyRange","AllowedHeads","AllowGroupClose","AllowIncomplete","AllowInlineCells","AllowKernelInitialization","AllowLooseGrammar","AllowReverseGroupClose","AllowScriptLevelChange","AllowVersionUpdate","AllTrue","Alphabet","AlphabeticOrder","AlphabeticSort","AlphaChannel","AlternateImage","AlternatingFactorial","AlternatingGroup","AlternativeHypothesis","Alternatives","AltitudeMethod","AmbientLight","AmbiguityFunction","AmbiguityList","Analytic","AnatomyData","AnatomyForm","AnatomyPlot3D","AnatomySkinStyle","AnatomyStyling","AnchoredSearch","And","AndersonDarlingTest","AngerJ","AngleBisector","AngleBracket","AnglePath","AnglePath3D","AngleVector","AngularGauge","Animate","AnimatedImage","AnimationCycleOffset","AnimationCycleRepetitions","AnimationDirection","AnimationDisplayTime","AnimationRate","AnimationRepetitions","AnimationRunning","AnimationRunTime","AnimationTimeIndex","AnimationVideo","Animator","AnimatorBox","AnimatorBoxOptions","AnimatorElements","Annotate","Annotation","AnnotationDelete","AnnotationKeys","AnnotationRules","AnnotationValue","Annuity","AnnuityDue","Annulus","AnomalyDetection","AnomalyDetector","AnomalyDetectorFunction","Anonymous","Antialiasing","Antihermitian","AntihermitianMatrixQ","Antisymmetric","AntisymmetricMatrixQ","Antonyms","AnyOrder","AnySubset","AnyTrue","Apart","ApartSquareFree","APIFunction","Appearance","AppearanceElements","AppearanceRules","AppellF1","Append","AppendCheck","AppendLayer","AppendTo","Application","Apply","ApplyReaction","ApplySides","ApplyTo","ArcCos","ArcCosh","ArcCot","ArcCoth","ArcCsc","ArcCsch","ArcCurvature","ARCHProcess","ArcLength","ArcSec","ArcSech","ArcSin","ArcSinDistribution","ArcSinh","ArcTan","ArcTanh","Area","Arg","ArgMax","ArgMin","ArgumentCountQ","ArgumentsOptions","ARIMAProcess","ArithmeticGeometricMean","ARMAProcess","Around","AroundReplace","ARProcess","Array","ArrayComponents","ArrayDepth","ArrayFilter","ArrayFlatten","ArrayMesh","ArrayPad","ArrayPlot","ArrayPlot3D","ArrayQ","ArrayReduce","ArrayResample","ArrayReshape","ArrayRules","Arrays","Arrow","Arrow3DBox","ArrowBox","Arrowheads","ASATriangle","Ask","AskAppend","AskConfirm","AskDisplay","AskedQ","AskedValue","AskFunction","AskState","AskTemplateDisplay","AspectRatio","AspectRatioFixed","Assert","AssessmentFunction","AssessmentResultObject","AssociateTo","Association","AssociationFormat","AssociationMap","AssociationQ","AssociationThread","AssumeDeterministic","Assuming","Assumptions","AstroAngularSeparation","AstroBackground","AstroCenter","AstroDistance","AstroGraphics","AstroGridLines","AstroGridLinesStyle","AstronomicalData","AstroPosition","AstroProjection","AstroRange","AstroRangePadding","AstroReferenceFrame","AstroStyling","AstroZoomLevel","Asymptotic","AsymptoticDSolveValue","AsymptoticEqual","AsymptoticEquivalent","AsymptoticExpectation","AsymptoticGreater","AsymptoticGreaterEqual","AsymptoticIntegrate","AsymptoticLess","AsymptoticLessEqual","AsymptoticOutputTracker","AsymptoticProbability","AsymptoticProduct","AsymptoticRSolveValue","AsymptoticSolve","AsymptoticSum","Asynchronous","AsynchronousTaskObject","AsynchronousTasks","Atom","AtomCoordinates","AtomCount","AtomDiagramCoordinates","AtomLabels","AtomLabelStyle","AtomList","AtomQ","AttachCell","AttachedCell","AttentionLayer","Attributes","Audio","AudioAmplify","AudioAnnotate","AudioAnnotationLookup","AudioBlockMap","AudioCapture","AudioChannelAssignment","AudioChannelCombine","AudioChannelMix","AudioChannels","AudioChannelSeparate","AudioData","AudioDelay","AudioDelete","AudioDevice","AudioDistance","AudioEncoding","AudioFade","AudioFrequencyShift","AudioGenerator","AudioIdentify","AudioInputDevice","AudioInsert","AudioInstanceQ","AudioIntervals","AudioJoin","AudioLabel","AudioLength","AudioLocalMeasurements","AudioLooping","AudioLoudness","AudioMeasurements","AudioNormalize","AudioOutputDevice","AudioOverlay","AudioPad","AudioPan","AudioPartition","AudioPause","AudioPitchShift","AudioPlay","AudioPlot","AudioQ","AudioRecord","AudioReplace","AudioResample","AudioReverb","AudioReverse","AudioSampleRate","AudioSpectralMap","AudioSpectralTransformation","AudioSplit","AudioStop","AudioStream","AudioStreams","AudioTimeStretch","AudioTrackApply","AudioTrackSelection","AudioTrim","AudioType","AugmentedPolyhedron","AugmentedSymmetricPolynomial","Authenticate","Authentication","AuthenticationDialog","AutoAction","Autocomplete","AutocompletionFunction","AutoCopy","AutocorrelationTest","AutoDelete","AutoEvaluateEvents","AutoGeneratedPackage","AutoIndent","AutoIndentSpacings","AutoItalicWords","AutoloadPath","AutoMatch","Automatic","AutomaticImageSize","AutoMultiplicationSymbol","AutoNumberFormatting","AutoOpenNotebooks","AutoOpenPalettes","AutoOperatorRenderings","AutoQuoteCharacters","AutoRefreshed","AutoRemove","AutorunSequencing","AutoScaling","AutoScroll","AutoSpacing","AutoStyleOptions","AutoStyleWords","AutoSubmitting","Axes","AxesEdge","AxesLabel","AxesOrigin","AxesStyle","AxiomaticTheory","Axis","Axis3DBox","Axis3DBoxOptions","AxisBox","AxisBoxOptions","AxisLabel","AxisObject","AxisStyle","BabyMonsterGroupB","Back","BackFaceColor","BackFaceGlowColor","BackFaceOpacity","BackFaceSpecularColor","BackFaceSpecularExponent","BackFaceSurfaceAppearance","BackFaceTexture","Background","BackgroundAppearance","BackgroundTasksSettings","Backslash","Backsubstitution","Backward","Ball","Band","BandpassFilter","BandstopFilter","BarabasiAlbertGraphDistribution","BarChart","BarChart3D","BarcodeImage","BarcodeRecognize","BaringhausHenzeTest","BarLegend","BarlowProschanImportance","BarnesG","BarOrigin","BarSpacing","BartlettHannWindow","BartlettWindow","BaseDecode","BaseEncode","BaseForm","Baseline","BaselinePosition","BaseStyle","BasicRecurrentLayer","BatchNormalizationLayer","BatchSize","BatesDistribution","BattleLemarieWavelet","BayesianMaximization","BayesianMaximizationObject","BayesianMinimization","BayesianMinimizationObject","Because","BeckmannDistribution","Beep","Before","Begin","BeginDialogPacket","BeginPackage","BellB","BellY","Below","BenfordDistribution","BeniniDistribution","BenktanderGibratDistribution","BenktanderWeibullDistribution","BernoulliB","BernoulliDistribution","BernoulliGraphDistribution","BernoulliProcess","BernsteinBasis","BesagL","BesselFilterModel","BesselI","BesselJ","BesselJZero","BesselK","BesselY","BesselYZero","Beta","BetaBinomialDistribution","BetaDistribution","BetaNegativeBinomialDistribution","BetaPrimeDistribution","BetaRegularized","Between","BetweennessCentrality","Beveled","BeveledPolyhedron","BezierCurve","BezierCurve3DBox","BezierCurve3DBoxOptions","BezierCurveBox","BezierCurveBoxOptions","BezierFunction","BilateralFilter","BilateralLaplaceTransform","BilateralZTransform","Binarize","BinaryDeserialize","BinaryDistance","BinaryFormat","BinaryImageQ","BinaryRead","BinaryReadList","BinarySerialize","BinaryWrite","BinCounts","BinLists","BinnedVariogramList","Binomial","BinomialDistribution","BinomialPointProcess","BinomialProcess","BinormalDistribution","BiorthogonalSplineWavelet","BioSequence","BioSequenceBackTranslateList","BioSequenceComplement","BioSequenceInstances","BioSequenceModify","BioSequencePlot","BioSequenceQ","BioSequenceReverseComplement","BioSequenceTranscribe","BioSequenceTranslate","BipartiteGraphQ","BiquadraticFilterModel","BirnbaumImportance","BirnbaumSaundersDistribution","BitAnd","BitClear","BitGet","BitLength","BitNot","BitOr","BitRate","BitSet","BitShiftLeft","BitShiftRight","BitXor","BiweightLocation","BiweightMidvariance","Black","BlackmanHarrisWindow","BlackmanNuttallWindow","BlackmanWindow","Blank","BlankForm","BlankNullSequence","BlankSequence","Blend","Block","BlockchainAddressData","BlockchainBase","BlockchainBlockData","BlockchainContractValue","BlockchainData","BlockchainGet","BlockchainKeyEncode","BlockchainPut","BlockchainTokenData","BlockchainTransaction","BlockchainTransactionData","BlockchainTransactionSign","BlockchainTransactionSubmit","BlockDiagonalMatrix","BlockLowerTriangularMatrix","BlockMap","BlockRandom","BlockUpperTriangularMatrix","BlomqvistBeta","BlomqvistBetaTest","Blue","Blur","Blurring","BodePlot","BohmanWindow","Bold","Bond","BondCount","BondLabels","BondLabelStyle","BondList","BondQ","Bookmarks","Boole","BooleanConsecutiveFunction","BooleanConvert","BooleanCountingFunction","BooleanFunction","BooleanGraph","BooleanMaxterms","BooleanMinimize","BooleanMinterms","BooleanQ","BooleanRegion","Booleans","BooleanStrings","BooleanTable","BooleanVariables","BorderDimensions","BorelTannerDistribution","Bottom","BottomHatTransform","BoundaryDiscretizeGraphics","BoundaryDiscretizeRegion","BoundaryMesh","BoundaryMeshRegion","BoundaryMeshRegionQ","BoundaryStyle","BoundedRegionQ","BoundingRegion","Bounds","Box","BoxBaselineShift","BoxData","BoxDimensions","Boxed","Boxes","BoxForm","BoxFormFormatTypes","BoxFrame","BoxID","BoxMargins","BoxMatrix","BoxObject","BoxRatios","BoxRotation","BoxRotationPoint","BoxStyle","BoxWhiskerChart","Bra","BracketingBar","BraKet","BrayCurtisDistance","BreadthFirstScan","Break","BridgeData","BrightnessEqualize","BroadcastStationData","Brown","BrownForsytheTest","BrownianBridgeProcess","BrowserCategory","BSplineBasis","BSplineCurve","BSplineCurve3DBox","BSplineCurve3DBoxOptions","BSplineCurveBox","BSplineCurveBoxOptions","BSplineFunction","BSplineSurface","BSplineSurface3DBox","BSplineSurface3DBoxOptions","BubbleChart","BubbleChart3D","BubbleScale","BubbleSizes","BuckyballGraph","BuildCompiledComponent","BuildingData","BulletGauge","BusinessDayQ","ButterflyGraph","ButterworthFilterModel","Button","ButtonBar","ButtonBox","ButtonBoxOptions","ButtonCell","ButtonContents","ButtonData","ButtonEvaluator","ButtonExpandable","ButtonFrame","ButtonFunction","ButtonMargins","ButtonMinHeight","ButtonNote","ButtonNotebook","ButtonSource","ButtonStyle","ButtonStyleMenuListing","Byte","ByteArray","ByteArrayFormat","ByteArrayFormatQ","ByteArrayQ","ByteArrayToString","ByteCount","ByteOrdering","C","CachedValue","CacheGraphics","CachePersistence","CalendarConvert","CalendarData","CalendarType","Callout","CalloutMarker","CalloutStyle","CallPacket","CanberraDistance","Cancel","CancelButton","CandlestickChart","CanonicalGraph","CanonicalizePolygon","CanonicalizePolyhedron","CanonicalizeRegion","CanonicalName","CanonicalWarpingCorrespondence","CanonicalWarpingDistance","CantorMesh","CantorStaircase","Canvas","Cap","CapForm","CapitalDifferentialD","Capitalize","CapsuleShape","CaptureRunning","CaputoD","CardinalBSplineBasis","CarlemanLinearize","CarlsonRC","CarlsonRD","CarlsonRE","CarlsonRF","CarlsonRG","CarlsonRJ","CarlsonRK","CarlsonRM","CarmichaelLambda","CaseOrdering","Cases","CaseSensitive","Cashflow","Casoratian","Cast","Catalan","CatalanNumber","Catch","CategoricalDistribution","Catenate","CatenateLayer","CauchyDistribution","CauchyMatrix","CauchyPointProcess","CauchyWindow","CayleyGraph","CDF","CDFDeploy","CDFInformation","CDFWavelet","Ceiling","CelestialSystem","Cell","CellAutoOverwrite","CellBaseline","CellBoundingBox","CellBracketOptions","CellChangeTimes","CellContents","CellContext","CellDingbat","CellDingbatMargin","CellDynamicExpression","CellEditDuplicate","CellElementsBoundingBox","CellElementSpacings","CellEpilog","CellEvaluationDuplicate","CellEvaluationFunction","CellEvaluationLanguage","CellEventActions","CellFrame","CellFrameColor","CellFrameLabelMargins","CellFrameLabels","CellFrameMargins","CellFrameStyle","CellGroup","CellGroupData","CellGrouping","CellGroupingRules","CellHorizontalScrolling","CellID","CellInsertionPointCell","CellLabel","CellLabelAutoDelete","CellLabelMargins","CellLabelPositioning","CellLabelStyle","CellLabelTemplate","CellMargins","CellObject","CellOpen","CellPrint","CellProlog","Cells","CellSize","CellStyle","CellTags","CellTrayPosition","CellTrayWidgets","CellularAutomaton","CensoredDistribution","Censoring","Center","CenterArray","CenterDot","CenteredInterval","CentralFeature","CentralMoment","CentralMomentGeneratingFunction","Cepstrogram","CepstrogramArray","CepstrumArray","CForm","ChampernowneNumber","ChangeOptions","ChannelBase","ChannelBrokerAction","ChannelDatabin","ChannelHistoryLength","ChannelListen","ChannelListener","ChannelListeners","ChannelListenerWait","ChannelObject","ChannelPreSendFunction","ChannelReceiverFunction","ChannelSend","ChannelSubscribers","ChanVeseBinarize","Character","CharacterCounts","CharacterEncoding","CharacterEncodingsPath","CharacteristicFunction","CharacteristicPolynomial","CharacterName","CharacterNormalize","CharacterRange","Characters","ChartBaseStyle","ChartElementData","ChartElementDataFunction","ChartElementFunction","ChartElements","ChartLabels","ChartLayout","ChartLegends","ChartStyle","Chebyshev1FilterModel","Chebyshev2FilterModel","ChebyshevDistance","ChebyshevT","ChebyshevU","Check","CheckAbort","CheckAll","CheckArguments","Checkbox","CheckboxBar","CheckboxBox","CheckboxBoxOptions","ChemicalConvert","ChemicalData","ChemicalFormula","ChemicalInstance","ChemicalReaction","ChessboardDistance","ChiDistribution","ChineseRemainder","ChiSquareDistribution","ChoiceButtons","ChoiceDialog","CholeskyDecomposition","Chop","ChromaticityPlot","ChromaticityPlot3D","ChromaticPolynomial","Circle","CircleBox","CircleDot","CircleMinus","CirclePlus","CirclePoints","CircleThrough","CircleTimes","CirculantGraph","CircularArcThrough","CircularOrthogonalMatrixDistribution","CircularQuaternionMatrixDistribution","CircularRealMatrixDistribution","CircularSymplecticMatrixDistribution","CircularUnitaryMatrixDistribution","Circumsphere","CityData","ClassifierFunction","ClassifierInformation","ClassifierMeasurements","ClassifierMeasurementsObject","Classify","ClassPriors","Clear","ClearAll","ClearAttributes","ClearCookies","ClearPermissions","ClearSystemCache","ClebschGordan","ClickPane","ClickToCopy","ClickToCopyEnabled","Clip","ClipboardNotebook","ClipFill","ClippingStyle","ClipPlanes","ClipPlanesStyle","ClipRange","Clock","ClockGauge","ClockwiseContourIntegral","Close","Closed","CloseKernels","ClosenessCentrality","Closing","ClosingAutoSave","ClosingEvent","CloudAccountData","CloudBase","CloudConnect","CloudConnections","CloudDeploy","CloudDirectory","CloudDisconnect","CloudEvaluate","CloudExport","CloudExpression","CloudExpressions","CloudFunction","CloudGet","CloudImport","CloudLoggingData","CloudObject","CloudObjectInformation","CloudObjectInformationData","CloudObjectNameFormat","CloudObjects","CloudObjectURLType","CloudPublish","CloudPut","CloudRenderingMethod","CloudSave","CloudShare","CloudSubmit","CloudSymbol","CloudUnshare","CloudUserID","ClusterClassify","ClusterDissimilarityFunction","ClusteringComponents","ClusteringMeasurements","ClusteringTree","CMYKColor","Coarse","CodeAssistOptions","Coefficient","CoefficientArrays","CoefficientDomain","CoefficientList","CoefficientRules","CoifletWavelet","Collect","CollinearPoints","Colon","ColonForm","ColorBalance","ColorCombine","ColorConvert","ColorCoverage","ColorData","ColorDataFunction","ColorDetect","ColorDistance","ColorFunction","ColorFunctionBinning","ColorFunctionScaling","Colorize","ColorNegate","ColorOutput","ColorProfileData","ColorQ","ColorQuantize","ColorReplace","ColorRules","ColorSelectorSettings","ColorSeparate","ColorSetter","ColorSetterBox","ColorSetterBoxOptions","ColorSlider","ColorsNear","ColorSpace","ColorToneMapping","Column","ColumnAlignments","ColumnBackgrounds","ColumnForm","ColumnLines","ColumnsEqual","ColumnSpacings","ColumnWidths","CombinatorB","CombinatorC","CombinatorI","CombinatorK","CombinatorS","CombinatorW","CombinatorY","CombinedEntityClass","CombinerFunction","CometData","CommonDefaultFormatTypes","Commonest","CommonestFilter","CommonName","CommonUnits","CommunityBoundaryStyle","CommunityGraphPlot","CommunityLabels","CommunityRegionStyle","CompanyData","CompatibleUnitQ","CompilationOptions","CompilationTarget","Compile","Compiled","CompiledCodeFunction","CompiledComponent","CompiledExpressionDeclaration","CompiledFunction","CompiledLayer","CompilerCallback","CompilerEnvironment","CompilerEnvironmentAppend","CompilerEnvironmentAppendTo","CompilerEnvironmentObject","CompilerOptions","Complement","ComplementedEntityClass","CompleteGraph","CompleteGraphQ","CompleteIntegral","CompleteKaryTree","CompletionsListPacket","Complex","ComplexArrayPlot","ComplexContourPlot","Complexes","ComplexExpand","ComplexInfinity","ComplexityFunction","ComplexListPlot","ComplexPlot","ComplexPlot3D","ComplexRegionPlot","ComplexStreamPlot","ComplexVectorPlot","ComponentMeasurements","ComponentwiseContextMenu","Compose","ComposeList","ComposeSeries","CompositeQ","Composition","CompoundElement","CompoundExpression","CompoundPoissonDistribution","CompoundPoissonProcess","CompoundRenewalProcess","Compress","CompressedData","CompressionLevel","ComputeUncertainty","ConcaveHullMesh","Condition","ConditionalExpression","Conditioned","Cone","ConeBox","ConfidenceLevel","ConfidenceRange","ConfidenceTransform","ConfigurationPath","Confirm","ConfirmAssert","ConfirmBy","ConfirmMatch","ConfirmQuiet","ConformationMethod","ConformAudio","ConformImages","Congruent","ConicGradientFilling","ConicHullRegion","ConicHullRegion3DBox","ConicHullRegion3DBoxOptions","ConicHullRegionBox","ConicHullRegionBoxOptions","ConicOptimization","Conjugate","ConjugateTranspose","Conjunction","Connect","ConnectedComponents","ConnectedGraphComponents","ConnectedGraphQ","ConnectedMeshComponents","ConnectedMoleculeComponents","ConnectedMoleculeQ","ConnectionSettings","ConnectLibraryCallbackFunction","ConnectSystemModelComponents","ConnectSystemModelController","ConnesWindow","ConoverTest","ConservativeConvectionPDETerm","ConsoleMessage","Constant","ConstantArray","ConstantArrayLayer","ConstantImage","ConstantPlusLayer","ConstantRegionQ","Constants","ConstantTimesLayer","ConstellationData","ConstrainedMax","ConstrainedMin","Construct","Containing","ContainsAll","ContainsAny","ContainsExactly","ContainsNone","ContainsOnly","ContentDetectorFunction","ContentFieldOptions","ContentLocationFunction","ContentObject","ContentPadding","ContentsBoundingBox","ContentSelectable","ContentSize","Context","ContextMenu","Contexts","ContextToFileName","Continuation","Continue","ContinuedFraction","ContinuedFractionK","ContinuousAction","ContinuousMarkovProcess","ContinuousTask","ContinuousTimeModelQ","ContinuousWaveletData","ContinuousWaveletTransform","ContourDetect","ContourGraphics","ContourIntegral","ContourLabels","ContourLines","ContourPlot","ContourPlot3D","Contours","ContourShading","ContourSmoothing","ContourStyle","ContraharmonicMean","ContrastiveLossLayer","Control","ControlActive","ControlAlignment","ControlGroupContentsBox","ControllabilityGramian","ControllabilityMatrix","ControllableDecomposition","ControllableModelQ","ControllerDuration","ControllerInformation","ControllerInformationData","ControllerLinking","ControllerManipulate","ControllerMethod","ControllerPath","ControllerState","ControlPlacement","ControlsRendering","ControlType","ConvectionPDETerm","Convergents","ConversionOptions","ConversionRules","ConvertToPostScript","ConvertToPostScriptPacket","ConvexHullMesh","ConvexHullRegion","ConvexOptimization","ConvexPolygonQ","ConvexPolyhedronQ","ConvexRegionQ","ConvolutionLayer","Convolve","ConwayGroupCo1","ConwayGroupCo2","ConwayGroupCo3","CookieFunction","Cookies","CoordinateBoundingBox","CoordinateBoundingBoxArray","CoordinateBounds","CoordinateBoundsArray","CoordinateChartData","CoordinatesToolOptions","CoordinateTransform","CoordinateTransformData","CoplanarPoints","CoprimeQ","Coproduct","CopulaDistribution","Copyable","CopyDatabin","CopyDirectory","CopyFile","CopyFunction","CopyTag","CopyToClipboard","CoreNilpotentDecomposition","CornerFilter","CornerNeighbors","Correlation","CorrelationDistance","CorrelationFunction","CorrelationTest","Cos","Cosh","CoshIntegral","CosineDistance","CosineWindow","CosIntegral","Cot","Coth","CoulombF","CoulombG","CoulombH1","CoulombH2","Count","CountDistinct","CountDistinctBy","CounterAssignments","CounterBox","CounterBoxOptions","CounterClockwiseContourIntegral","CounterEvaluator","CounterFunction","CounterIncrements","CounterStyle","CounterStyleMenuListing","CountRoots","CountryData","Counts","CountsBy","Covariance","CovarianceEstimatorFunction","CovarianceFunction","CoxianDistribution","CoxIngersollRossProcess","CoxModel","CoxModelFit","CramerVonMisesTest","CreateArchive","CreateCellID","CreateChannel","CreateCloudExpression","CreateCompilerEnvironment","CreateDatabin","CreateDataStructure","CreateDataSystemModel","CreateDialog","CreateDirectory","CreateDocument","CreateFile","CreateIntermediateDirectories","CreateLicenseEntitlement","CreateManagedLibraryExpression","CreateNotebook","CreatePacletArchive","CreatePalette","CreatePermissionsGroup","CreateScheduledTask","CreateSearchIndex","CreateSystemModel","CreateTemporary","CreateTypeInstance","CreateUUID","CreateWindow","CriterionFunction","CriticalityFailureImportance","CriticalitySuccessImportance","CriticalSection","Cross","CrossEntropyLossLayer","CrossingCount","CrossingDetect","CrossingPolygon","CrossMatrix","Csc","Csch","CSGRegion","CSGRegionQ","CSGRegionTree","CTCLossLayer","Cube","CubeRoot","Cubics","Cuboid","CuboidBox","CuboidBoxOptions","Cumulant","CumulantGeneratingFunction","CumulativeFeatureImpactPlot","Cup","CupCap","Curl","CurlyDoubleQuote","CurlyQuote","CurrencyConvert","CurrentDate","CurrentImage","CurrentNotebookImage","CurrentScreenImage","CurrentValue","Curry","CurryApplied","CurvatureFlowFilter","CurveClosed","Cyan","CycleGraph","CycleIndexPolynomial","Cycles","CyclicGroup","Cyclotomic","Cylinder","CylinderBox","CylinderBoxOptions","CylindricalDecomposition","CylindricalDecompositionFunction","D","DagumDistribution","DamData","DamerauLevenshteinDistance","DampingFactor","Darker","Dashed","Dashing","DatabaseConnect","DatabaseDisconnect","DatabaseReference","Databin","DatabinAdd","DatabinRemove","Databins","DatabinSubmit","DatabinUpload","DataCompression","DataDistribution","DataRange","DataReversed","Dataset","DatasetDisplayPanel","DatasetTheme","DataStructure","DataStructureQ","Date","DateBounds","Dated","DateDelimiters","DateDifference","DatedUnit","DateFormat","DateFunction","DateGranularity","DateHistogram","DateInterval","DateList","DateListLogPlot","DateListPlot","DateListStepPlot","DateObject","DateObjectQ","DateOverlapsQ","DatePattern","DatePlus","DateRange","DateReduction","DateScale","DateSelect","DateString","DateTicksFormat","DateValue","DateWithinQ","DaubechiesWavelet","DavisDistribution","DawsonF","DayCount","DayCountConvention","DayHemisphere","DaylightQ","DayMatchQ","DayName","DayNightTerminator","DayPlus","DayRange","DayRound","DeBruijnGraph","DeBruijnSequence","Debug","DebugTag","Decapitalize","Decimal","DecimalForm","DeclareCompiledComponent","DeclareKnownSymbols","DeclarePackage","Decompose","DeconvolutionLayer","Decrement","Decrypt","DecryptFile","DedekindEta","DeepSpaceProbeData","Default","Default2DTool","Default3DTool","DefaultAttachedCellStyle","DefaultAxesStyle","DefaultBaseStyle","DefaultBoxStyle","DefaultButton","DefaultColor","DefaultControlPlacement","DefaultDockedCellStyle","DefaultDuplicateCellStyle","DefaultDuration","DefaultElement","DefaultFaceGridsStyle","DefaultFieldHintStyle","DefaultFont","DefaultFontProperties","DefaultFormatType","DefaultFrameStyle","DefaultFrameTicksStyle","DefaultGridLinesStyle","DefaultInlineFormatType","DefaultInputFormatType","DefaultLabelStyle","DefaultMenuStyle","DefaultNaturalLanguage","DefaultNewCellStyle","DefaultNewInlineCellStyle","DefaultNotebook","DefaultOptions","DefaultOutputFormatType","DefaultPrintPrecision","DefaultStyle","DefaultStyleDefinitions","DefaultTextFormatType","DefaultTextInlineFormatType","DefaultTicksStyle","DefaultTooltipStyle","DefaultValue","DefaultValues","Defer","DefineExternal","DefineInputStreamMethod","DefineOutputStreamMethod","DefineResourceFunction","Definition","Degree","DegreeCentrality","DegreeGraphDistribution","DegreeLexicographic","DegreeReverseLexicographic","DEigensystem","DEigenvalues","Deinitialization","Del","DelaunayMesh","Delayed","Deletable","Delete","DeleteAdjacentDuplicates","DeleteAnomalies","DeleteBorderComponents","DeleteCases","DeleteChannel","DeleteCloudExpression","DeleteContents","DeleteDirectory","DeleteDuplicates","DeleteDuplicatesBy","DeleteElements","DeleteFile","DeleteMissing","DeleteObject","DeletePermissionsKey","DeleteSearchIndex","DeleteSmallComponents","DeleteStopwords","DeleteWithContents","DeletionWarning","DelimitedArray","DelimitedSequence","Delimiter","DelimiterAutoMatching","DelimiterFlashTime","DelimiterMatching","Delimiters","DeliveryFunction","Dendrogram","Denominator","DensityGraphics","DensityHistogram","DensityPlot","DensityPlot3D","DependentVariables","Deploy","Deployed","Depth","DepthFirstScan","Derivative","DerivativeFilter","DerivativePDETerm","DerivedKey","DescriptorStateSpace","DesignMatrix","DestroyAfterEvaluation","Det","DeviceClose","DeviceConfigure","DeviceExecute","DeviceExecuteAsynchronous","DeviceObject","DeviceOpen","DeviceOpenQ","DeviceRead","DeviceReadBuffer","DeviceReadLatest","DeviceReadList","DeviceReadTimeSeries","Devices","DeviceStreams","DeviceWrite","DeviceWriteBuffer","DGaussianWavelet","DiacriticalPositioning","Diagonal","DiagonalizableMatrixQ","DiagonalMatrix","DiagonalMatrixQ","Dialog","DialogIndent","DialogInput","DialogLevel","DialogNotebook","DialogProlog","DialogReturn","DialogSymbols","Diamond","DiamondMatrix","DiceDissimilarity","DictionaryLookup","DictionaryWordQ","DifferenceDelta","DifferenceOrder","DifferenceQuotient","DifferenceRoot","DifferenceRootReduce","Differences","DifferentialD","DifferentialRoot","DifferentialRootReduce","DifferentiatorFilter","DiffusionPDETerm","DiggleGatesPointProcess","DiggleGrattonPointProcess","DigitalSignature","DigitBlock","DigitBlockMinimum","DigitCharacter","DigitCount","DigitQ","DihedralAngle","DihedralGroup","Dilation","DimensionalCombinations","DimensionalMeshComponents","DimensionReduce","DimensionReducerFunction","DimensionReduction","Dimensions","DiracComb","DiracDelta","DirectedEdge","DirectedEdges","DirectedGraph","DirectedGraphQ","DirectedInfinity","Direction","DirectionalLight","Directive","Directory","DirectoryName","DirectoryQ","DirectoryStack","DirichletBeta","DirichletCharacter","DirichletCondition","DirichletConvolve","DirichletDistribution","DirichletEta","DirichletL","DirichletLambda","DirichletTransform","DirichletWindow","DisableConsolePrintPacket","DisableFormatting","DiscreteAsymptotic","DiscreteChirpZTransform","DiscreteConvolve","DiscreteDelta","DiscreteHadamardTransform","DiscreteIndicator","DiscreteInputOutputModel","DiscreteLimit","DiscreteLQEstimatorGains","DiscreteLQRegulatorGains","DiscreteLyapunovSolve","DiscreteMarkovProcess","DiscreteMaxLimit","DiscreteMinLimit","DiscretePlot","DiscretePlot3D","DiscreteRatio","DiscreteRiccatiSolve","DiscreteShift","DiscreteTimeModelQ","DiscreteUniformDistribution","DiscreteVariables","DiscreteWaveletData","DiscreteWaveletPacketTransform","DiscreteWaveletTransform","DiscretizeGraphics","DiscretizeRegion","Discriminant","DisjointQ","Disjunction","Disk","DiskBox","DiskBoxOptions","DiskMatrix","DiskSegment","Dispatch","DispatchQ","DispersionEstimatorFunction","Display","DisplayAllSteps","DisplayEndPacket","DisplayForm","DisplayFunction","DisplayPacket","DisplayRules","DisplayString","DisplayTemporary","DisplayWith","DisplayWithRef","DisplayWithVariable","DistanceFunction","DistanceMatrix","DistanceTransform","Distribute","Distributed","DistributedContexts","DistributeDefinitions","DistributionChart","DistributionDomain","DistributionFitTest","DistributionParameterAssumptions","DistributionParameterQ","Dithering","Div","Divergence","Divide","DivideBy","Dividers","DivideSides","Divisible","Divisors","DivisorSigma","DivisorSum","DMSList","DMSString","Do","DockedCell","DockedCells","DocumentGenerator","DocumentGeneratorInformation","DocumentGeneratorInformationData","DocumentGenerators","DocumentNotebook","DocumentWeightingRules","Dodecahedron","DomainRegistrationInformation","DominantColors","DominatorTreeGraph","DominatorVertexList","DOSTextFormat","Dot","DotDashed","DotEqual","DotLayer","DotPlusLayer","Dotted","DoubleBracketingBar","DoubleContourIntegral","DoubleDownArrow","DoubleLeftArrow","DoubleLeftRightArrow","DoubleLeftTee","DoubleLongLeftArrow","DoubleLongLeftRightArrow","DoubleLongRightArrow","DoubleRightArrow","DoubleRightTee","DoubleUpArrow","DoubleUpDownArrow","DoubleVerticalBar","DoublyInfinite","Down","DownArrow","DownArrowBar","DownArrowUpArrow","DownLeftRightVector","DownLeftTeeVector","DownLeftVector","DownLeftVectorBar","DownRightTeeVector","DownRightVector","DownRightVectorBar","Downsample","DownTee","DownTeeArrow","DownValues","DownValuesFunction","DragAndDrop","DrawBackFaces","DrawEdges","DrawFrontFaces","DrawHighlighted","DrazinInverse","Drop","DropoutLayer","DropShadowing","DSolve","DSolveChangeVariables","DSolveValue","Dt","DualLinearProgramming","DualPlanarGraph","DualPolyhedron","DualSystemsModel","DumpGet","DumpSave","DuplicateFreeQ","Duration","Dynamic","DynamicBox","DynamicBoxOptions","DynamicEvaluationTimeout","DynamicGeoGraphics","DynamicImage","DynamicLocation","DynamicModule","DynamicModuleBox","DynamicModuleBoxOptions","DynamicModuleParent","DynamicModuleValues","DynamicName","DynamicNamespace","DynamicReference","DynamicSetting","DynamicUpdating","DynamicWrapper","DynamicWrapperBox","DynamicWrapperBoxOptions","E","EarthImpactData","EarthquakeData","EccentricityCentrality","Echo","EchoEvaluation","EchoFunction","EchoLabel","EchoTiming","EclipseType","EdgeAdd","EdgeBetweennessCentrality","EdgeCapacity","EdgeCapForm","EdgeChromaticNumber","EdgeColor","EdgeConnectivity","EdgeContract","EdgeCost","EdgeCount","EdgeCoverQ","EdgeCycleMatrix","EdgeDashing","EdgeDelete","EdgeDetect","EdgeForm","EdgeIndex","EdgeJoinForm","EdgeLabeling","EdgeLabels","EdgeLabelStyle","EdgeList","EdgeOpacity","EdgeQ","EdgeRenderingFunction","EdgeRules","EdgeShapeFunction","EdgeStyle","EdgeTaggedGraph","EdgeTaggedGraphQ","EdgeTags","EdgeThickness","EdgeTransitiveGraphQ","EdgeValueRange","EdgeValueSizes","EdgeWeight","EdgeWeightedGraphQ","Editable","EditButtonSettings","EditCellTagsSettings","EditDistance","EffectiveInterest","Eigensystem","Eigenvalues","EigenvectorCentrality","Eigenvectors","Element","ElementData","ElementwiseLayer","ElidedForms","Eliminate","EliminationOrder","Ellipsoid","EllipticE","EllipticExp","EllipticExpPrime","EllipticF","EllipticFilterModel","EllipticK","EllipticLog","EllipticNomeQ","EllipticPi","EllipticReducedHalfPeriods","EllipticTheta","EllipticThetaPrime","EmbedCode","EmbeddedHTML","EmbeddedService","EmbeddedSQLEntityClass","EmbeddedSQLExpression","EmbeddingLayer","EmbeddingObject","EmitSound","EmphasizeSyntaxErrors","EmpiricalDistribution","Empty","EmptyGraphQ","EmptyRegion","EmptySpaceF","EnableConsolePrintPacket","Enabled","Enclose","Encode","Encrypt","EncryptedObject","EncryptFile","End","EndAdd","EndDialogPacket","EndOfBuffer","EndOfFile","EndOfLine","EndOfString","EndPackage","EngineEnvironment","EngineeringForm","Enter","EnterExpressionPacket","EnterTextPacket","Entity","EntityClass","EntityClassList","EntityCopies","EntityFunction","EntityGroup","EntityInstance","EntityList","EntityPrefetch","EntityProperties","EntityProperty","EntityPropertyClass","EntityRegister","EntityStore","EntityStores","EntityTypeName","EntityUnregister","EntityValue","Entropy","EntropyFilter","Environment","Epilog","EpilogFunction","Equal","EqualColumns","EqualRows","EqualTilde","EqualTo","EquatedTo","Equilibrium","EquirippleFilterKernel","Equivalent","Erf","Erfc","Erfi","ErlangB","ErlangC","ErlangDistribution","Erosion","ErrorBox","ErrorBoxOptions","ErrorNorm","ErrorPacket","ErrorsDialogSettings","EscapeRadius","EstimatedBackground","EstimatedDistribution","EstimatedPointNormals","EstimatedPointProcess","EstimatedProcess","EstimatedVariogramModel","EstimatorGains","EstimatorRegulator","EuclideanDistance","EulerAngles","EulerCharacteristic","EulerE","EulerGamma","EulerianGraphQ","EulerMatrix","EulerPhi","Evaluatable","Evaluate","Evaluated","EvaluatePacket","EvaluateScheduledTask","EvaluationBox","EvaluationCell","EvaluationCompletionAction","EvaluationData","EvaluationElements","EvaluationEnvironment","EvaluationMode","EvaluationMonitor","EvaluationNotebook","EvaluationObject","EvaluationOrder","EvaluationPrivileges","EvaluationRateLimit","Evaluator","EvaluatorNames","EvenQ","EventData","EventEvaluator","EventHandler","EventHandlerTag","EventLabels","EventSeries","ExactBlackmanWindow","ExactNumberQ","ExactRootIsolation","ExampleData","Except","ExcludedContexts","ExcludedForms","ExcludedLines","ExcludedPhysicalQuantities","ExcludePods","Exclusions","ExclusionsStyle","Exists","Exit","ExitDialog","ExoplanetData","Exp","Expand","ExpandAll","ExpandDenominator","ExpandFileName","ExpandNumerator","Expectation","ExpectationE","ExpectedValue","ExpGammaDistribution","ExpIntegralE","ExpIntegralEi","ExpirationDate","Exponent","ExponentFunction","ExponentialDistribution","ExponentialFamily","ExponentialGeneratingFunction","ExponentialMovingAverage","ExponentialPowerDistribution","ExponentPosition","ExponentStep","Export","ExportAutoReplacements","ExportByteArray","ExportForm","ExportPacket","ExportString","Expression","ExpressionCell","ExpressionGraph","ExpressionPacket","ExpressionTree","ExpressionUUID","ExpToTrig","ExtendedEntityClass","ExtendedGCD","Extension","ExtentElementFunction","ExtentMarkers","ExtentSize","ExternalBundle","ExternalCall","ExternalDataCharacterEncoding","ExternalEvaluate","ExternalFunction","ExternalFunctionName","ExternalIdentifier","ExternalObject","ExternalOptions","ExternalSessionObject","ExternalSessions","ExternalStorageBase","ExternalStorageDownload","ExternalStorageGet","ExternalStorageObject","ExternalStoragePut","ExternalStorageUpload","ExternalTypeSignature","ExternalValue","Extract","ExtractArchive","ExtractLayer","ExtractPacletArchive","ExtremeValueDistribution","FaceAlign","FaceForm","FaceGrids","FaceGridsStyle","FaceRecognize","FacialFeatures","Factor","FactorComplete","Factorial","Factorial2","FactorialMoment","FactorialMomentGeneratingFunction","FactorialPower","FactorInteger","FactorList","FactorSquareFree","FactorSquareFreeList","FactorTerms","FactorTermsList","Fail","Failure","FailureAction","FailureDistribution","FailureQ","False","FareySequence","FARIMAProcess","FeatureDistance","FeatureExtract","FeatureExtraction","FeatureExtractor","FeatureExtractorFunction","FeatureImpactPlot","FeatureNames","FeatureNearest","FeatureSpacePlot","FeatureSpacePlot3D","FeatureTypes","FeatureValueDependencyPlot","FeatureValueImpactPlot","FEDisableConsolePrintPacket","FeedbackLinearize","FeedbackSector","FeedbackSectorStyle","FeedbackType","FEEnableConsolePrintPacket","FetalGrowthData","Fibonacci","Fibonorial","FieldCompletionFunction","FieldHint","FieldHintStyle","FieldMasked","FieldSize","File","FileBaseName","FileByteCount","FileConvert","FileDate","FileExistsQ","FileExtension","FileFormat","FileFormatProperties","FileFormatQ","FileHandler","FileHash","FileInformation","FileName","FileNameDepth","FileNameDialogSettings","FileNameDrop","FileNameForms","FileNameJoin","FileNames","FileNameSetter","FileNameSplit","FileNameTake","FileNameToFormatList","FilePrint","FileSize","FileSystemMap","FileSystemScan","FileSystemTree","FileTemplate","FileTemplateApply","FileType","FilledCurve","FilledCurveBox","FilledCurveBoxOptions","FilledTorus","FillForm","Filling","FillingStyle","FillingTransform","FilteredEntityClass","FilterRules","FinancialBond","FinancialData","FinancialDerivative","FinancialIndicator","Find","FindAnomalies","FindArgMax","FindArgMin","FindChannels","FindClique","FindClusters","FindCookies","FindCurvePath","FindCycle","FindDevices","FindDistribution","FindDistributionParameters","FindDivisions","FindEdgeColoring","FindEdgeCover","FindEdgeCut","FindEdgeIndependentPaths","FindEquationalProof","FindEulerianCycle","FindExternalEvaluators","FindFaces","FindFile","FindFit","FindFormula","FindFundamentalCycles","FindGeneratingFunction","FindGeoLocation","FindGeometricConjectures","FindGeometricTransform","FindGraphCommunities","FindGraphIsomorphism","FindGraphPartition","FindHamiltonianCycle","FindHamiltonianPath","FindHiddenMarkovStates","FindImageText","FindIndependentEdgeSet","FindIndependentVertexSet","FindInstance","FindIntegerNullVector","FindIsomers","FindIsomorphicSubgraph","FindKClan","FindKClique","FindKClub","FindKPlex","FindLibrary","FindLinearRecurrence","FindList","FindMatchingColor","FindMaximum","FindMaximumCut","FindMaximumFlow","FindMaxValue","FindMeshDefects","FindMinimum","FindMinimumCostFlow","FindMinimumCut","FindMinValue","FindMoleculeSubstructure","FindPath","FindPeaks","FindPermutation","FindPlanarColoring","FindPointProcessParameters","FindPostmanTour","FindProcessParameters","FindRegionTransform","FindRepeat","FindRoot","FindSequenceFunction","FindSettings","FindShortestPath","FindShortestTour","FindSpanningTree","FindSubgraphIsomorphism","FindSystemModelEquilibrium","FindTextualAnswer","FindThreshold","FindTransientRepeat","FindVertexColoring","FindVertexCover","FindVertexCut","FindVertexIndependentPaths","Fine","FinishDynamic","FiniteAbelianGroupCount","FiniteGroupCount","FiniteGroupData","First","FirstCase","FirstPassageTimeDistribution","FirstPosition","FischerGroupFi22","FischerGroupFi23","FischerGroupFi24Prime","FisherHypergeometricDistribution","FisherRatioTest","FisherZDistribution","Fit","FitAll","FitRegularization","FittedModel","FixedOrder","FixedPoint","FixedPointList","FlashSelection","Flat","FlatShading","Flatten","FlattenAt","FlattenLayer","FlatTopWindow","FlightData","FlipView","Floor","FlowPolynomial","Fold","FoldList","FoldPair","FoldPairList","FoldWhile","FoldWhileList","FollowRedirects","Font","FontColor","FontFamily","FontForm","FontName","FontOpacity","FontPostScriptName","FontProperties","FontReencoding","FontSize","FontSlant","FontSubstitutions","FontTracking","FontVariations","FontWeight","For","ForAll","ForAllType","ForceVersionInstall","Format","FormatRules","FormatType","FormatTypeAutoConvert","FormatValues","FormBox","FormBoxOptions","FormControl","FormFunction","FormLayoutFunction","FormObject","FormPage","FormProtectionMethod","FormTheme","FormulaData","FormulaLookup","FortranForm","Forward","ForwardBackward","ForwardCloudCredentials","Fourier","FourierCoefficient","FourierCosCoefficient","FourierCosSeries","FourierCosTransform","FourierDCT","FourierDCTFilter","FourierDCTMatrix","FourierDST","FourierDSTMatrix","FourierMatrix","FourierParameters","FourierSequenceTransform","FourierSeries","FourierSinCoefficient","FourierSinSeries","FourierSinTransform","FourierTransform","FourierTrigSeries","FoxH","FoxHReduce","FractionalBrownianMotionProcess","FractionalD","FractionalGaussianNoiseProcess","FractionalPart","FractionBox","FractionBoxOptions","FractionLine","Frame","FrameBox","FrameBoxOptions","Framed","FrameInset","FrameLabel","Frameless","FrameListVideo","FrameMargins","FrameRate","FrameStyle","FrameTicks","FrameTicksStyle","FRatioDistribution","FrechetDistribution","FreeQ","FrenetSerretSystem","FrequencySamplingFilterKernel","FresnelC","FresnelF","FresnelG","FresnelS","Friday","FrobeniusNumber","FrobeniusSolve","FromAbsoluteTime","FromCharacterCode","FromCoefficientRules","FromContinuedFraction","FromDate","FromDateString","FromDigits","FromDMS","FromEntity","FromJulianDate","FromLetterNumber","FromPolarCoordinates","FromRawPointer","FromRomanNumeral","FromSphericalCoordinates","FromUnixTime","Front","FrontEndDynamicExpression","FrontEndEventActions","FrontEndExecute","FrontEndObject","FrontEndResource","FrontEndResourceString","FrontEndStackSize","FrontEndToken","FrontEndTokenExecute","FrontEndValueCache","FrontEndVersion","FrontFaceColor","FrontFaceGlowColor","FrontFaceOpacity","FrontFaceSpecularColor","FrontFaceSpecularExponent","FrontFaceSurfaceAppearance","FrontFaceTexture","Full","FullAxes","FullDefinition","FullForm","FullGraphics","FullInformationOutputRegulator","FullOptions","FullRegion","FullSimplify","Function","FunctionAnalytic","FunctionBijective","FunctionCompile","FunctionCompileExport","FunctionCompileExportByteArray","FunctionCompileExportLibrary","FunctionCompileExportString","FunctionContinuous","FunctionConvexity","FunctionDeclaration","FunctionDiscontinuities","FunctionDomain","FunctionExpand","FunctionInjective","FunctionInterpolation","FunctionLayer","FunctionMeromorphic","FunctionMonotonicity","FunctionPeriod","FunctionPoles","FunctionRange","FunctionSign","FunctionSingularities","FunctionSpace","FunctionSurjective","FussellVeselyImportance","GaborFilter","GaborMatrix","GaborWavelet","GainMargins","GainPhaseMargins","GalaxyData","GalleryView","Gamma","GammaDistribution","GammaRegularized","GapPenalty","GARCHProcess","GatedRecurrentLayer","Gather","GatherBy","GaugeFaceElementFunction","GaugeFaceStyle","GaugeFrameElementFunction","GaugeFrameSize","GaugeFrameStyle","GaugeLabels","GaugeMarkers","GaugeStyle","GaussianFilter","GaussianIntegers","GaussianMatrix","GaussianOrthogonalMatrixDistribution","GaussianSymplecticMatrixDistribution","GaussianUnitaryMatrixDistribution","GaussianWindow","GCD","GegenbauerC","General","GeneralizedLinearModelFit","GenerateAsymmetricKeyPair","GenerateConditions","GeneratedAssetFormat","GeneratedAssetLocation","GeneratedCell","GeneratedCellStyles","GeneratedDocumentBinding","GenerateDerivedKey","GenerateDigitalSignature","GenerateDocument","GeneratedParameters","GeneratedQuantityMagnitudes","GenerateFileSignature","GenerateHTTPResponse","GenerateSecuredAuthenticationKey","GenerateSymmetricKey","GeneratingFunction","GeneratorDescription","GeneratorHistoryLength","GeneratorOutputType","Generic","GenericCylindricalDecomposition","GenomeData","GenomeLookup","GeoAntipode","GeoArea","GeoArraySize","GeoBackground","GeoBoundary","GeoBoundingBox","GeoBounds","GeoBoundsRegion","GeoBoundsRegionBoundary","GeoBubbleChart","GeoCenter","GeoCircle","GeoContourPlot","GeoDensityPlot","GeodesicClosing","GeodesicDilation","GeodesicErosion","GeodesicOpening","GeodesicPolyhedron","GeoDestination","GeodesyData","GeoDirection","GeoDisk","GeoDisplacement","GeoDistance","GeoDistanceList","GeoElevationData","GeoEntities","GeoGraphics","GeoGraphPlot","GeoGraphValuePlot","GeogravityModelData","GeoGridDirectionDifference","GeoGridLines","GeoGridLinesStyle","GeoGridPosition","GeoGridRange","GeoGridRangePadding","GeoGridUnitArea","GeoGridUnitDistance","GeoGridVector","GeoGroup","GeoHemisphere","GeoHemisphereBoundary","GeoHistogram","GeoIdentify","GeoImage","GeoLabels","GeoLength","GeoListPlot","GeoLocation","GeologicalPeriodData","GeomagneticModelData","GeoMarker","GeometricAssertion","GeometricBrownianMotionProcess","GeometricDistribution","GeometricMean","GeometricMeanFilter","GeometricOptimization","GeometricScene","GeometricStep","GeometricStylingRules","GeometricTest","GeometricTransformation","GeometricTransformation3DBox","GeometricTransformation3DBoxOptions","GeometricTransformationBox","GeometricTransformationBoxOptions","GeoModel","GeoNearest","GeoOrientationData","GeoPath","GeoPolygon","GeoPosition","GeoPositionENU","GeoPositionXYZ","GeoProjection","GeoProjectionData","GeoRange","GeoRangePadding","GeoRegionValuePlot","GeoResolution","GeoScaleBar","GeoServer","GeoSmoothHistogram","GeoStreamPlot","GeoStyling","GeoStylingImageFunction","GeoVariant","GeoVector","GeoVectorENU","GeoVectorPlot","GeoVectorXYZ","GeoVisibleRegion","GeoVisibleRegionBoundary","GeoWithinQ","GeoZoomLevel","GestureHandler","GestureHandlerTag","Get","GetContext","GetEnvironment","GetFileName","GetLinebreakInformationPacket","GibbsPointProcess","Glaisher","GlobalClusteringCoefficient","GlobalPreferences","GlobalSession","Glow","GoldenAngle","GoldenRatio","GompertzMakehamDistribution","GoochShading","GoodmanKruskalGamma","GoodmanKruskalGammaTest","Goto","GouraudShading","Grad","Gradient","GradientFilter","GradientFittedMesh","GradientOrientationFilter","GrammarApply","GrammarRules","GrammarToken","Graph","Graph3D","GraphAssortativity","GraphAutomorphismGroup","GraphCenter","GraphComplement","GraphData","GraphDensity","GraphDiameter","GraphDifference","GraphDisjointUnion","GraphDistance","GraphDistanceMatrix","GraphEmbedding","GraphHighlight","GraphHighlightStyle","GraphHub","Graphics","Graphics3D","Graphics3DBox","Graphics3DBoxOptions","GraphicsArray","GraphicsBaseline","GraphicsBox","GraphicsBoxOptions","GraphicsColor","GraphicsColumn","GraphicsComplex","GraphicsComplex3DBox","GraphicsComplex3DBoxOptions","GraphicsComplexBox","GraphicsComplexBoxOptions","GraphicsContents","GraphicsData","GraphicsGrid","GraphicsGridBox","GraphicsGroup","GraphicsGroup3DBox","GraphicsGroup3DBoxOptions","GraphicsGroupBox","GraphicsGroupBoxOptions","GraphicsGrouping","GraphicsHighlightColor","GraphicsRow","GraphicsSpacing","GraphicsStyle","GraphIntersection","GraphJoin","GraphLayerLabels","GraphLayers","GraphLayerStyle","GraphLayout","GraphLinkEfficiency","GraphPeriphery","GraphPlot","GraphPlot3D","GraphPower","GraphProduct","GraphPropertyDistribution","GraphQ","GraphRadius","GraphReciprocity","GraphRoot","GraphStyle","GraphSum","GraphTree","GraphUnion","Gray","GrayLevel","Greater","GreaterEqual","GreaterEqualLess","GreaterEqualThan","GreaterFullEqual","GreaterGreater","GreaterLess","GreaterSlantEqual","GreaterThan","GreaterTilde","GreekStyle","Green","GreenFunction","Grid","GridBaseline","GridBox","GridBoxAlignment","GridBoxBackground","GridBoxDividers","GridBoxFrame","GridBoxItemSize","GridBoxItemStyle","GridBoxOptions","GridBoxSpacings","GridCreationSettings","GridDefaultElement","GridElementStyleOptions","GridFrame","GridFrameMargins","GridGraph","GridLines","GridLinesStyle","GridVideo","GroebnerBasis","GroupActionBase","GroupBy","GroupCentralizer","GroupElementFromWord","GroupElementPosition","GroupElementQ","GroupElements","GroupElementToWord","GroupGenerators","Groupings","GroupMultiplicationTable","GroupOpenerColor","GroupOpenerInsideFrame","GroupOrbits","GroupOrder","GroupPageBreakWithin","GroupSetwiseStabilizer","GroupStabilizer","GroupStabilizerChain","GroupTogetherGrouping","GroupTogetherNestedGrouping","GrowCutComponents","Gudermannian","GuidedFilter","GumbelDistribution","HaarWavelet","HadamardMatrix","HalfLine","HalfNormalDistribution","HalfPlane","HalfSpace","HalftoneShading","HamiltonianGraphQ","HammingDistance","HammingWindow","HandlerFunctions","HandlerFunctionsKeys","HankelH1","HankelH2","HankelMatrix","HankelTransform","HannPoissonWindow","HannWindow","HaradaNortonGroupHN","HararyGraph","HardcorePointProcess","HarmonicMean","HarmonicMeanFilter","HarmonicNumber","Hash","HatchFilling","HatchShading","Haversine","HazardFunction","Head","HeadCompose","HeaderAlignment","HeaderBackground","HeaderDisplayFunction","HeaderLines","Headers","HeaderSize","HeaderStyle","Heads","HeatFluxValue","HeatInsulationValue","HeatOutflowValue","HeatRadiationValue","HeatSymmetryValue","HeatTemperatureCondition","HeatTransferPDEComponent","HeatTransferValue","HeavisideLambda","HeavisidePi","HeavisideTheta","HeldGroupHe","HeldPart","HelmholtzPDEComponent","HelpBrowserLookup","HelpBrowserNotebook","HelpBrowserSettings","HelpViewerSettings","Here","HermiteDecomposition","HermiteH","Hermitian","HermitianMatrixQ","HessenbergDecomposition","Hessian","HeunB","HeunBPrime","HeunC","HeunCPrime","HeunD","HeunDPrime","HeunG","HeunGPrime","HeunT","HeunTPrime","HexadecimalCharacter","Hexahedron","HexahedronBox","HexahedronBoxOptions","HiddenItems","HiddenMarkovProcess","HiddenSurface","Highlighted","HighlightGraph","HighlightImage","HighlightMesh","HighlightString","HighpassFilter","HigmanSimsGroupHS","HilbertCurve","HilbertFilter","HilbertMatrix","Histogram","Histogram3D","HistogramDistribution","HistogramList","HistogramPointDensity","HistogramTransform","HistogramTransformInterpolation","HistoricalPeriodData","HitMissTransform","HITSCentrality","HjorthDistribution","HodgeDual","HoeffdingD","HoeffdingDTest","Hold","HoldAll","HoldAllComplete","HoldComplete","HoldFirst","HoldForm","HoldPattern","HoldRest","HolidayCalendar","HomeDirectory","HomePage","Horizontal","HorizontalForm","HorizontalGauge","HorizontalScrollPosition","HornerForm","HostLookup","HotellingTSquareDistribution","HoytDistribution","HTMLSave","HTTPErrorResponse","HTTPRedirect","HTTPRequest","HTTPRequestData","HTTPResponse","Hue","HumanGrowthData","HumpDownHump","HumpEqual","HurwitzLerchPhi","HurwitzZeta","HyperbolicDistribution","HypercubeGraph","HyperexponentialDistribution","Hyperfactorial","Hypergeometric0F1","Hypergeometric0F1Regularized","Hypergeometric1F1","Hypergeometric1F1Regularized","Hypergeometric2F1","Hypergeometric2F1Regularized","HypergeometricDistribution","HypergeometricPFQ","HypergeometricPFQRegularized","HypergeometricU","Hyperlink","HyperlinkAction","HyperlinkCreationSettings","Hyperplane","Hyphenation","HyphenationOptions","HypoexponentialDistribution","HypothesisTestData","I","IconData","Iconize","IconizedObject","IconRules","Icosahedron","Identity","IdentityMatrix","If","IfCompiled","IgnoreCase","IgnoreDiacritics","IgnoreIsotopes","IgnorePunctuation","IgnoreSpellCheck","IgnoreStereochemistry","IgnoringInactive","Im","Image","Image3D","Image3DProjection","Image3DSlices","ImageAccumulate","ImageAdd","ImageAdjust","ImageAlign","ImageApply","ImageApplyIndexed","ImageAspectRatio","ImageAssemble","ImageAugmentationLayer","ImageBoundingBoxes","ImageCache","ImageCacheValid","ImageCapture","ImageCaptureFunction","ImageCases","ImageChannels","ImageClip","ImageCollage","ImageColorSpace","ImageCompose","ImageContainsQ","ImageContents","ImageConvolve","ImageCooccurrence","ImageCorners","ImageCorrelate","ImageCorrespondingPoints","ImageCrop","ImageData","ImageDeconvolve","ImageDemosaic","ImageDifference","ImageDimensions","ImageDisplacements","ImageDistance","ImageEditMode","ImageEffect","ImageExposureCombine","ImageFeatureTrack","ImageFileApply","ImageFileFilter","ImageFileScan","ImageFilter","ImageFocusCombine","ImageForestingComponents","ImageFormattingWidth","ImageForwardTransformation","ImageGraphics","ImageHistogram","ImageIdentify","ImageInstanceQ","ImageKeypoints","ImageLabels","ImageLegends","ImageLevels","ImageLines","ImageMargins","ImageMarker","ImageMarkers","ImageMeasurements","ImageMesh","ImageMultiply","ImageOffset","ImagePad","ImagePadding","ImagePartition","ImagePeriodogram","ImagePerspectiveTransformation","ImagePosition","ImagePreviewFunction","ImagePyramid","ImagePyramidApply","ImageQ","ImageRangeCache","ImageRecolor","ImageReflect","ImageRegion","ImageResize","ImageResolution","ImageRestyle","ImageRotate","ImageRotated","ImageSaliencyFilter","ImageScaled","ImageScan","ImageSize","ImageSizeAction","ImageSizeCache","ImageSizeMultipliers","ImageSizeRaw","ImageStitch","ImageSubtract","ImageTake","ImageTransformation","ImageTrim","ImageType","ImageValue","ImageValuePositions","ImageVectorscopePlot","ImageWaveformPlot","ImagingDevice","ImplicitD","ImplicitRegion","Implies","Import","ImportAutoReplacements","ImportByteArray","ImportedObject","ImportOptions","ImportString","ImprovementImportance","In","Inactivate","Inactive","InactiveStyle","IncidenceGraph","IncidenceList","IncidenceMatrix","IncludeAromaticBonds","IncludeConstantBasis","IncludedContexts","IncludeDefinitions","IncludeDirectories","IncludeFileExtension","IncludeGeneratorTasks","IncludeHydrogens","IncludeInflections","IncludeMetaInformation","IncludePods","IncludeQuantities","IncludeRelatedTables","IncludeSingularSolutions","IncludeSingularTerm","IncludeWindowTimes","Increment","IndefiniteMatrixQ","Indent","IndentingNewlineSpacings","IndentMaxFraction","IndependenceTest","IndependentEdgeSetQ","IndependentPhysicalQuantity","IndependentUnit","IndependentUnitDimension","IndependentVertexSetQ","Indeterminate","IndeterminateThreshold","IndexCreationOptions","Indexed","IndexEdgeTaggedGraph","IndexGraph","IndexTag","Inequality","InertEvaluate","InertExpression","InexactNumberQ","InexactNumbers","InfiniteFuture","InfiniteLine","InfiniteLineThrough","InfinitePast","InfinitePlane","Infinity","Infix","InflationAdjust","InflationMethod","Information","InformationData","InformationDataGrid","Inherited","InheritScope","InhomogeneousPoissonPointProcess","InhomogeneousPoissonProcess","InitialEvaluationHistory","Initialization","InitializationCell","InitializationCellEvaluation","InitializationCellWarning","InitializationObject","InitializationObjects","InitializationValue","Initialize","InitialSeeding","InlineCounterAssignments","InlineCounterIncrements","InlineRules","Inner","InnerPolygon","InnerPolyhedron","Inpaint","Input","InputAliases","InputAssumptions","InputAutoReplacements","InputField","InputFieldBox","InputFieldBoxOptions","InputForm","InputGrouping","InputNamePacket","InputNotebook","InputPacket","InputPorts","InputSettings","InputStream","InputString","InputStringPacket","InputToBoxFormPacket","Insert","InsertionFunction","InsertionPointObject","InsertLinebreaks","InsertResults","Inset","Inset3DBox","Inset3DBoxOptions","InsetBox","InsetBoxOptions","Insphere","Install","InstallService","InstanceNormalizationLayer","InString","Integer","IntegerDigits","IntegerExponent","IntegerLength","IntegerName","IntegerPart","IntegerPartitions","IntegerQ","IntegerReverse","Integers","IntegerString","Integral","Integrate","IntegrateChangeVariables","Interactive","InteractiveTradingChart","InterfaceSwitched","Interlaced","Interleaving","InternallyBalancedDecomposition","InterpolatingFunction","InterpolatingPolynomial","Interpolation","InterpolationOrder","InterpolationPoints","InterpolationPrecision","Interpretation","InterpretationBox","InterpretationBoxOptions","InterpretationFunction","Interpreter","InterpretTemplate","InterquartileRange","Interrupt","InterruptSettings","IntersectedEntityClass","IntersectingQ","Intersection","Interval","IntervalIntersection","IntervalMarkers","IntervalMarkersStyle","IntervalMemberQ","IntervalSlider","IntervalUnion","Into","Inverse","InverseBetaRegularized","InverseBilateralLaplaceTransform","InverseBilateralZTransform","InverseCDF","InverseChiSquareDistribution","InverseContinuousWaveletTransform","InverseDistanceTransform","InverseEllipticNomeQ","InverseErf","InverseErfc","InverseFourier","InverseFourierCosTransform","InverseFourierSequenceTransform","InverseFourierSinTransform","InverseFourierTransform","InverseFunction","InverseFunctions","InverseGammaDistribution","InverseGammaRegularized","InverseGaussianDistribution","InverseGudermannian","InverseHankelTransform","InverseHaversine","InverseImagePyramid","InverseJacobiCD","InverseJacobiCN","InverseJacobiCS","InverseJacobiDC","InverseJacobiDN","InverseJacobiDS","InverseJacobiNC","InverseJacobiND","InverseJacobiNS","InverseJacobiSC","InverseJacobiSD","InverseJacobiSN","InverseLaplaceTransform","InverseMellinTransform","InversePermutation","InverseRadon","InverseRadonTransform","InverseSeries","InverseShortTimeFourier","InverseSpectrogram","InverseSurvivalFunction","InverseTransformedRegion","InverseWaveletTransform","InverseWeierstrassP","InverseWishartMatrixDistribution","InverseZTransform","Invisible","InvisibleApplication","InvisibleTimes","IPAddress","IrreduciblePolynomialQ","IslandData","IsolatingInterval","IsomorphicGraphQ","IsomorphicSubgraphQ","IsotopeData","Italic","Item","ItemAspectRatio","ItemBox","ItemBoxOptions","ItemDisplayFunction","ItemSize","ItemStyle","ItoProcess","JaccardDissimilarity","JacobiAmplitude","Jacobian","JacobiCD","JacobiCN","JacobiCS","JacobiDC","JacobiDN","JacobiDS","JacobiEpsilon","JacobiNC","JacobiND","JacobiNS","JacobiP","JacobiSC","JacobiSD","JacobiSN","JacobiSymbol","JacobiZeta","JacobiZN","JankoGroupJ1","JankoGroupJ2","JankoGroupJ3","JankoGroupJ4","JarqueBeraALMTest","JohnsonDistribution","Join","JoinAcross","Joined","JoinedCurve","JoinedCurveBox","JoinedCurveBoxOptions","JoinForm","JordanDecomposition","JordanModelDecomposition","JulianDate","JuliaSetBoettcher","JuliaSetIterationCount","JuliaSetPlot","JuliaSetPoints","K","KagiChart","KaiserBesselWindow","KaiserWindow","KalmanEstimator","KalmanFilter","KarhunenLoeveDecomposition","KaryTree","KatzCentrality","KCoreComponents","KDistribution","KEdgeConnectedComponents","KEdgeConnectedGraphQ","KeepExistingVersion","KelvinBei","KelvinBer","KelvinKei","KelvinKer","KendallTau","KendallTauTest","KernelConfiguration","KernelExecute","KernelFunction","KernelMixtureDistribution","KernelObject","Kernels","Ket","Key","KeyCollisionFunction","KeyComplement","KeyDrop","KeyDropFrom","KeyExistsQ","KeyFreeQ","KeyIntersection","KeyMap","KeyMemberQ","KeypointStrength","Keys","KeySelect","KeySort","KeySortBy","KeyTake","KeyUnion","KeyValueMap","KeyValuePattern","Khinchin","KillProcess","KirchhoffGraph","KirchhoffMatrix","KleinInvariantJ","KnapsackSolve","KnightTourGraph","KnotData","KnownUnitQ","KochCurve","KolmogorovSmirnovTest","KroneckerDelta","KroneckerModelDecomposition","KroneckerProduct","KroneckerSymbol","KuiperTest","KumaraswamyDistribution","Kurtosis","KuwaharaFilter","KVertexConnectedComponents","KVertexConnectedGraphQ","LABColor","Label","Labeled","LabeledSlider","LabelingFunction","LabelingSize","LabelStyle","LabelVisibility","LaguerreL","LakeData","LambdaComponents","LambertW","LameC","LameCPrime","LameEigenvalueA","LameEigenvalueB","LameS","LameSPrime","LaminaData","LanczosWindow","LandauDistribution","Language","LanguageCategory","LanguageData","LanguageIdentify","LanguageOptions","LaplaceDistribution","LaplaceTransform","Laplacian","LaplacianFilter","LaplacianGaussianFilter","LaplacianPDETerm","Large","Larger","Last","Latitude","LatitudeLongitude","LatticeData","LatticeReduce","Launch","LaunchKernels","LayeredGraphPlot","LayeredGraphPlot3D","LayerSizeFunction","LayoutInformation","LCHColor","LCM","LeaderSize","LeafCount","LeapVariant","LeapYearQ","LearnDistribution","LearnedDistribution","LearningRate","LearningRateMultipliers","LeastSquares","LeastSquaresFilterKernel","Left","LeftArrow","LeftArrowBar","LeftArrowRightArrow","LeftDownTeeVector","LeftDownVector","LeftDownVectorBar","LeftRightArrow","LeftRightVector","LeftTee","LeftTeeArrow","LeftTeeVector","LeftTriangle","LeftTriangleBar","LeftTriangleEqual","LeftUpDownVector","LeftUpTeeVector","LeftUpVector","LeftUpVectorBar","LeftVector","LeftVectorBar","LegendAppearance","Legended","LegendFunction","LegendLabel","LegendLayout","LegendMargins","LegendMarkers","LegendMarkerSize","LegendreP","LegendreQ","LegendreType","Length","LengthWhile","LerchPhi","Less","LessEqual","LessEqualGreater","LessEqualThan","LessFullEqual","LessGreater","LessLess","LessSlantEqual","LessThan","LessTilde","LetterCharacter","LetterCounts","LetterNumber","LetterQ","Level","LeveneTest","LeviCivitaTensor","LevyDistribution","Lexicographic","LexicographicOrder","LexicographicSort","LibraryDataType","LibraryFunction","LibraryFunctionDeclaration","LibraryFunctionError","LibraryFunctionInformation","LibraryFunctionLoad","LibraryFunctionUnload","LibraryLoad","LibraryUnload","LicenseEntitlementObject","LicenseEntitlements","LicenseID","LicensingSettings","LiftingFilterData","LiftingWaveletTransform","LightBlue","LightBrown","LightCyan","Lighter","LightGray","LightGreen","Lighting","LightingAngle","LightMagenta","LightOrange","LightPink","LightPurple","LightRed","LightSources","LightYellow","Likelihood","Limit","LimitsPositioning","LimitsPositioningTokens","LindleyDistribution","Line","Line3DBox","Line3DBoxOptions","LinearFilter","LinearFractionalOptimization","LinearFractionalTransform","LinearGradientFilling","LinearGradientImage","LinearizingTransformationData","LinearLayer","LinearModelFit","LinearOffsetFunction","LinearOptimization","LinearProgramming","LinearRecurrence","LinearSolve","LinearSolveFunction","LineBox","LineBoxOptions","LineBreak","LinebreakAdjustments","LineBreakChart","LinebreakSemicolonWeighting","LineBreakWithin","LineColor","LineGraph","LineIndent","LineIndentMaxFraction","LineIntegralConvolutionPlot","LineIntegralConvolutionScale","LineLegend","LineOpacity","LineSpacing","LineWrapParts","LinkActivate","LinkClose","LinkConnect","LinkConnectedQ","LinkCreate","LinkError","LinkFlush","LinkFunction","LinkHost","LinkInterrupt","LinkLaunch","LinkMode","LinkObject","LinkOpen","LinkOptions","LinkPatterns","LinkProtocol","LinkRankCentrality","LinkRead","LinkReadHeld","LinkReadyQ","Links","LinkService","LinkWrite","LinkWriteHeld","LiouvilleLambda","List","Listable","ListAnimate","ListContourPlot","ListContourPlot3D","ListConvolve","ListCorrelate","ListCurvePathPlot","ListDeconvolve","ListDensityPlot","ListDensityPlot3D","Listen","ListFormat","ListFourierSequenceTransform","ListInterpolation","ListLineIntegralConvolutionPlot","ListLinePlot","ListLinePlot3D","ListLogLinearPlot","ListLogLogPlot","ListLogPlot","ListPicker","ListPickerBox","ListPickerBoxBackground","ListPickerBoxOptions","ListPlay","ListPlot","ListPlot3D","ListPointPlot3D","ListPolarPlot","ListQ","ListSliceContourPlot3D","ListSliceDensityPlot3D","ListSliceVectorPlot3D","ListStepPlot","ListStreamDensityPlot","ListStreamPlot","ListStreamPlot3D","ListSurfacePlot3D","ListVectorDensityPlot","ListVectorDisplacementPlot","ListVectorDisplacementPlot3D","ListVectorPlot","ListVectorPlot3D","ListZTransform","Literal","LiteralSearch","LiteralType","LoadCompiledComponent","LocalAdaptiveBinarize","LocalCache","LocalClusteringCoefficient","LocalEvaluate","LocalizeDefinitions","LocalizeVariables","LocalObject","LocalObjects","LocalResponseNormalizationLayer","LocalSubmit","LocalSymbol","LocalTime","LocalTimeZone","LocationEquivalenceTest","LocationTest","Locator","LocatorAutoCreate","LocatorBox","LocatorBoxOptions","LocatorCentering","LocatorPane","LocatorPaneBox","LocatorPaneBoxOptions","LocatorRegion","Locked","Log","Log10","Log2","LogBarnesG","LogGamma","LogGammaDistribution","LogicalExpand","LogIntegral","LogisticDistribution","LogisticSigmoid","LogitModelFit","LogLikelihood","LogLinearPlot","LogLogisticDistribution","LogLogPlot","LogMultinormalDistribution","LogNormalDistribution","LogPlot","LogRankTest","LogSeriesDistribution","LongEqual","Longest","LongestCommonSequence","LongestCommonSequencePositions","LongestCommonSubsequence","LongestCommonSubsequencePositions","LongestMatch","LongestOrderedSequence","LongForm","Longitude","LongLeftArrow","LongLeftRightArrow","LongRightArrow","LongShortTermMemoryLayer","Lookup","Loopback","LoopFreeGraphQ","Looping","LossFunction","LowerCaseQ","LowerLeftArrow","LowerRightArrow","LowerTriangularize","LowerTriangularMatrix","LowerTriangularMatrixQ","LowpassFilter","LQEstimatorGains","LQGRegulator","LQOutputRegulatorGains","LQRegulatorGains","LUBackSubstitution","LucasL","LuccioSamiComponents","LUDecomposition","LunarEclipse","LUVColor","LyapunovSolve","LyonsGroupLy","MachineID","MachineName","MachineNumberQ","MachinePrecision","MacintoshSystemPageSetup","Magenta","Magnification","Magnify","MailAddressValidation","MailExecute","MailFolder","MailItem","MailReceiverFunction","MailResponseFunction","MailSearch","MailServerConnect","MailServerConnection","MailSettings","MainSolve","MaintainDynamicCaches","Majority","MakeBoxes","MakeExpression","MakeRules","ManagedLibraryExpressionID","ManagedLibraryExpressionQ","MandelbrotSetBoettcher","MandelbrotSetDistance","MandelbrotSetIterationCount","MandelbrotSetMemberQ","MandelbrotSetPlot","MangoldtLambda","ManhattanDistance","Manipulate","Manipulator","MannedSpaceMissionData","MannWhitneyTest","MantissaExponent","Manual","Map","MapAll","MapApply","MapAt","MapIndexed","MAProcess","MapThread","MarchenkoPasturDistribution","MarcumQ","MardiaCombinedTest","MardiaKurtosisTest","MardiaSkewnessTest","MarginalDistribution","MarkovProcessProperties","Masking","MassConcentrationCondition","MassFluxValue","MassImpermeableBoundaryValue","MassOutflowValue","MassSymmetryValue","MassTransferValue","MassTransportPDEComponent","MatchingDissimilarity","MatchLocalNameQ","MatchLocalNames","MatchQ","Material","MaterialShading","MaternPointProcess","MathematicalFunctionData","MathematicaNotation","MathieuC","MathieuCharacteristicA","MathieuCharacteristicB","MathieuCharacteristicExponent","MathieuCPrime","MathieuGroupM11","MathieuGroupM12","MathieuGroupM22","MathieuGroupM23","MathieuGroupM24","MathieuS","MathieuSPrime","MathMLForm","MathMLText","Matrices","MatrixExp","MatrixForm","MatrixFunction","MatrixLog","MatrixNormalDistribution","MatrixPlot","MatrixPower","MatrixPropertyDistribution","MatrixQ","MatrixRank","MatrixTDistribution","Max","MaxBend","MaxCellMeasure","MaxColorDistance","MaxDate","MaxDetect","MaxDisplayedChildren","MaxDuration","MaxExtraBandwidths","MaxExtraConditions","MaxFeatureDisplacement","MaxFeatures","MaxFilter","MaximalBy","Maximize","MaxItems","MaxIterations","MaxLimit","MaxMemoryUsed","MaxMixtureKernels","MaxOverlapFraction","MaxPlotPoints","MaxPoints","MaxRecursion","MaxStableDistribution","MaxStepFraction","MaxSteps","MaxStepSize","MaxTrainingRounds","MaxValue","MaxwellDistribution","MaxWordGap","McLaughlinGroupMcL","Mean","MeanAbsoluteLossLayer","MeanAround","MeanClusteringCoefficient","MeanDegreeConnectivity","MeanDeviation","MeanFilter","MeanGraphDistance","MeanNeighborDegree","MeanPointDensity","MeanShift","MeanShiftFilter","MeanSquaredLossLayer","Median","MedianDeviation","MedianFilter","MedicalTestData","Medium","MeijerG","MeijerGReduce","MeixnerDistribution","MellinConvolve","MellinTransform","MemberQ","MemoryAvailable","MemoryConstrained","MemoryConstraint","MemoryInUse","MengerMesh","Menu","MenuAppearance","MenuCommandKey","MenuEvaluator","MenuItem","MenuList","MenuPacket","MenuSortingValue","MenuStyle","MenuView","Merge","MergeDifferences","MergingFunction","MersennePrimeExponent","MersennePrimeExponentQ","Mesh","MeshCellCentroid","MeshCellCount","MeshCellHighlight","MeshCellIndex","MeshCellLabel","MeshCellMarker","MeshCellMeasure","MeshCellQuality","MeshCells","MeshCellShapeFunction","MeshCellStyle","MeshConnectivityGraph","MeshCoordinates","MeshFunctions","MeshPrimitives","MeshQualityGoal","MeshRange","MeshRefinementFunction","MeshRegion","MeshRegionQ","MeshShading","MeshStyle","Message","MessageDialog","MessageList","MessageName","MessageObject","MessageOptions","MessagePacket","Messages","MessagesNotebook","MetaCharacters","MetaInformation","MeteorShowerData","Method","MethodOptions","MexicanHatWavelet","MeyerWavelet","Midpoint","MIMETypeToFormatList","Min","MinColorDistance","MinDate","MinDetect","MineralData","MinFilter","MinimalBy","MinimalPolynomial","MinimalStateSpaceModel","Minimize","MinimumTimeIncrement","MinIntervalSize","MinkowskiQuestionMark","MinLimit","MinMax","MinorPlanetData","Minors","MinPointSeparation","MinRecursion","MinSize","MinStableDistribution","Minus","MinusPlus","MinValue","Missing","MissingBehavior","MissingDataMethod","MissingDataRules","MissingQ","MissingString","MissingStyle","MissingValuePattern","MissingValueSynthesis","MittagLefflerE","MixedFractionParts","MixedGraphQ","MixedMagnitude","MixedRadix","MixedRadixQuantity","MixedUnit","MixtureDistribution","Mod","Modal","Mode","ModelPredictiveController","Modular","ModularInverse","ModularLambda","Module","Modulus","MoebiusMu","Molecule","MoleculeAlign","MoleculeContainsQ","MoleculeDraw","MoleculeEquivalentQ","MoleculeFreeQ","MoleculeGraph","MoleculeMatchQ","MoleculeMaximumCommonSubstructure","MoleculeModify","MoleculeName","MoleculePattern","MoleculePlot","MoleculePlot3D","MoleculeProperty","MoleculeQ","MoleculeRecognize","MoleculeSubstructureCount","MoleculeValue","Moment","MomentConvert","MomentEvaluate","MomentGeneratingFunction","MomentOfInertia","Monday","Monitor","MonomialList","MonomialOrder","MonsterGroupM","MoonPhase","MoonPosition","MorletWavelet","MorphologicalBinarize","MorphologicalBranchPoints","MorphologicalComponents","MorphologicalEulerNumber","MorphologicalGraph","MorphologicalPerimeter","MorphologicalTransform","MortalityData","Most","MountainData","MouseAnnotation","MouseAppearance","MouseAppearanceTag","MouseButtons","Mouseover","MousePointerNote","MousePosition","MovieData","MovingAverage","MovingMap","MovingMedian","MoyalDistribution","MultiaxisArrangement","Multicolumn","MultiedgeStyle","MultigraphQ","MultilaunchWarning","MultiLetterItalics","MultiLetterStyle","MultilineFunction","Multinomial","MultinomialDistribution","MultinormalDistribution","MultiplicativeOrder","Multiplicity","MultiplySides","MultiscriptBoxOptions","Multiselection","MultivariateHypergeometricDistribution","MultivariatePoissonDistribution","MultivariateTDistribution","N","NakagamiDistribution","NameQ","Names","NamespaceBox","NamespaceBoxOptions","Nand","NArgMax","NArgMin","NBernoulliB","NBodySimulation","NBodySimulationData","NCache","NCaputoD","NDEigensystem","NDEigenvalues","NDSolve","NDSolveValue","Nearest","NearestFunction","NearestMeshCells","NearestNeighborG","NearestNeighborGraph","NearestTo","NebulaData","NeedlemanWunschSimilarity","Needs","Negative","NegativeBinomialDistribution","NegativeDefiniteMatrixQ","NegativeIntegers","NegativelyOrientedPoints","NegativeMultinomialDistribution","NegativeRationals","NegativeReals","NegativeSemidefiniteMatrixQ","NeighborhoodData","NeighborhoodGraph","Nest","NestedGreaterGreater","NestedLessLess","NestedScriptRules","NestGraph","NestList","NestTree","NestWhile","NestWhileList","NetAppend","NetArray","NetArrayLayer","NetBidirectionalOperator","NetChain","NetDecoder","NetDelete","NetDrop","NetEncoder","NetEvaluationMode","NetExternalObject","NetExtract","NetFlatten","NetFoldOperator","NetGANOperator","NetGraph","NetInformation","NetInitialize","NetInsert","NetInsertSharedArrays","NetJoin","NetMapOperator","NetMapThreadOperator","NetMeasurements","NetModel","NetNestOperator","NetPairEmbeddingOperator","NetPort","NetPortGradient","NetPrepend","NetRename","NetReplace","NetReplacePart","NetSharedArray","NetStateObject","NetTake","NetTrain","NetTrainResultsObject","NetUnfold","NetworkPacketCapture","NetworkPacketRecording","NetworkPacketRecordingDuring","NetworkPacketTrace","NeumannValue","NevilleThetaC","NevilleThetaD","NevilleThetaN","NevilleThetaS","NewPrimitiveStyle","NExpectation","Next","NextCell","NextDate","NextPrime","NextScheduledTaskTime","NeymanScottPointProcess","NFractionalD","NHoldAll","NHoldFirst","NHoldRest","NicholsGridLines","NicholsPlot","NightHemisphere","NIntegrate","NMaximize","NMaxValue","NMinimize","NMinValue","NominalScale","NominalVariables","NonAssociative","NoncentralBetaDistribution","NoncentralChiSquareDistribution","NoncentralFRatioDistribution","NoncentralStudentTDistribution","NonCommutativeMultiply","NonConstants","NondimensionalizationTransform","None","NoneTrue","NonlinearModelFit","NonlinearStateSpaceModel","NonlocalMeansFilter","NonNegative","NonNegativeIntegers","NonNegativeRationals","NonNegativeReals","NonPositive","NonPositiveIntegers","NonPositiveRationals","NonPositiveReals","Nor","NorlundB","Norm","Normal","NormalDistribution","NormalGrouping","NormalizationLayer","Normalize","Normalized","NormalizedSquaredEuclideanDistance","NormalMatrixQ","NormalsFunction","NormFunction","Not","NotCongruent","NotCupCap","NotDoubleVerticalBar","Notebook","NotebookApply","NotebookAutoSave","NotebookBrowseDirectory","NotebookClose","NotebookConvertSettings","NotebookCreate","NotebookDefault","NotebookDelete","NotebookDirectory","NotebookDynamicExpression","NotebookEvaluate","NotebookEventActions","NotebookFileName","NotebookFind","NotebookGet","NotebookImport","NotebookInformation","NotebookInterfaceObject","NotebookLocate","NotebookObject","NotebookOpen","NotebookPath","NotebookPrint","NotebookPut","NotebookRead","Notebooks","NotebookSave","NotebookSelection","NotebooksMenu","NotebookTemplate","NotebookWrite","NotElement","NotEqualTilde","NotExists","NotGreater","NotGreaterEqual","NotGreaterFullEqual","NotGreaterGreater","NotGreaterLess","NotGreaterSlantEqual","NotGreaterTilde","Nothing","NotHumpDownHump","NotHumpEqual","NotificationFunction","NotLeftTriangle","NotLeftTriangleBar","NotLeftTriangleEqual","NotLess","NotLessEqual","NotLessFullEqual","NotLessGreater","NotLessLess","NotLessSlantEqual","NotLessTilde","NotNestedGreaterGreater","NotNestedLessLess","NotPrecedes","NotPrecedesEqual","NotPrecedesSlantEqual","NotPrecedesTilde","NotReverseElement","NotRightTriangle","NotRightTriangleBar","NotRightTriangleEqual","NotSquareSubset","NotSquareSubsetEqual","NotSquareSuperset","NotSquareSupersetEqual","NotSubset","NotSubsetEqual","NotSucceeds","NotSucceedsEqual","NotSucceedsSlantEqual","NotSucceedsTilde","NotSuperset","NotSupersetEqual","NotTilde","NotTildeEqual","NotTildeFullEqual","NotTildeTilde","NotVerticalBar","Now","NoWhitespace","NProbability","NProduct","NProductFactors","NRoots","NSolve","NSolveValues","NSum","NSumTerms","NuclearExplosionData","NuclearReactorData","Null","NullRecords","NullSpace","NullWords","Number","NumberCompose","NumberDecompose","NumberDigit","NumberExpand","NumberFieldClassNumber","NumberFieldDiscriminant","NumberFieldFundamentalUnits","NumberFieldIntegralBasis","NumberFieldNormRepresentatives","NumberFieldRegulator","NumberFieldRootsOfUnity","NumberFieldSignature","NumberForm","NumberFormat","NumberLinePlot","NumberMarks","NumberMultiplier","NumberPadding","NumberPoint","NumberQ","NumberSeparator","NumberSigns","NumberString","Numerator","NumeratorDenominator","NumericalOrder","NumericalSort","NumericArray","NumericArrayQ","NumericArrayType","NumericFunction","NumericQ","NuttallWindow","NValues","NyquistGridLines","NyquistPlot","O","ObjectExistsQ","ObservabilityGramian","ObservabilityMatrix","ObservableDecomposition","ObservableModelQ","OceanData","Octahedron","OddQ","Off","Offset","OLEData","On","ONanGroupON","Once","OneIdentity","Opacity","OpacityFunction","OpacityFunctionScaling","Open","OpenAppend","Opener","OpenerBox","OpenerBoxOptions","OpenerView","OpenFunctionInspectorPacket","Opening","OpenRead","OpenSpecialOptions","OpenTemporary","OpenWrite","Operate","OperatingSystem","OperatorApplied","OptimumFlowData","Optional","OptionalElement","OptionInspectorSettings","OptionQ","Options","OptionsPacket","OptionsPattern","OptionValue","OptionValueBox","OptionValueBoxOptions","Or","Orange","Order","OrderDistribution","OrderedQ","Ordering","OrderingBy","OrderingLayer","Orderless","OrderlessPatternSequence","OrdinalScale","OrnsteinUhlenbeckProcess","Orthogonalize","OrthogonalMatrixQ","Out","Outer","OuterPolygon","OuterPolyhedron","OutputAutoOverwrite","OutputControllabilityMatrix","OutputControllableModelQ","OutputForm","OutputFormData","OutputGrouping","OutputMathEditExpression","OutputNamePacket","OutputPorts","OutputResponse","OutputSizeLimit","OutputStream","Over","OverBar","OverDot","Overflow","OverHat","Overlaps","Overlay","OverlayBox","OverlayBoxOptions","OverlayVideo","Overscript","OverscriptBox","OverscriptBoxOptions","OverTilde","OverVector","OverwriteTarget","OwenT","OwnValues","Package","PackingMethod","PackPaclet","PacletDataRebuild","PacletDirectoryAdd","PacletDirectoryLoad","PacletDirectoryRemove","PacletDirectoryUnload","PacletDisable","PacletEnable","PacletFind","PacletFindRemote","PacletInformation","PacletInstall","PacletInstallSubmit","PacletNewerQ","PacletObject","PacletObjectQ","PacletSite","PacletSiteObject","PacletSiteRegister","PacletSites","PacletSiteUnregister","PacletSiteUpdate","PacletSymbol","PacletUninstall","PacletUpdate","PaddedForm","Padding","PaddingLayer","PaddingSize","PadeApproximant","PadLeft","PadRight","PageBreakAbove","PageBreakBelow","PageBreakWithin","PageFooterLines","PageFooters","PageHeaderLines","PageHeaders","PageHeight","PageRankCentrality","PageTheme","PageWidth","Pagination","PairCorrelationG","PairedBarChart","PairedHistogram","PairedSmoothHistogram","PairedTTest","PairedZTest","PaletteNotebook","PalettePath","PalettesMenuSettings","PalindromeQ","Pane","PaneBox","PaneBoxOptions","Panel","PanelBox","PanelBoxOptions","Paneled","PaneSelector","PaneSelectorBox","PaneSelectorBoxOptions","PaperWidth","ParabolicCylinderD","ParagraphIndent","ParagraphSpacing","ParallelArray","ParallelAxisPlot","ParallelCombine","ParallelDo","Parallelepiped","ParallelEvaluate","Parallelization","Parallelize","ParallelKernels","ParallelMap","ParallelNeeds","Parallelogram","ParallelProduct","ParallelSubmit","ParallelSum","ParallelTable","ParallelTry","Parameter","ParameterEstimator","ParameterMixtureDistribution","ParameterVariables","ParametricConvexOptimization","ParametricFunction","ParametricNDSolve","ParametricNDSolveValue","ParametricPlot","ParametricPlot3D","ParametricRampLayer","ParametricRegion","ParentBox","ParentCell","ParentConnect","ParentDirectory","ParentEdgeLabel","ParentEdgeLabelFunction","ParentEdgeLabelStyle","ParentEdgeShapeFunction","ParentEdgeStyle","ParentEdgeStyleFunction","ParentForm","Parenthesize","ParentList","ParentNotebook","ParetoDistribution","ParetoPickandsDistribution","ParkData","Part","PartBehavior","PartialCorrelationFunction","PartialD","ParticleAcceleratorData","ParticleData","Partition","PartitionGranularity","PartitionsP","PartitionsQ","PartLayer","PartOfSpeech","PartProtection","ParzenWindow","PascalDistribution","PassEventsDown","PassEventsUp","Paste","PasteAutoQuoteCharacters","PasteBoxFormInlineCells","PasteButton","Path","PathGraph","PathGraphQ","Pattern","PatternFilling","PatternReaction","PatternSequence","PatternTest","PauliMatrix","PaulWavelet","Pause","PausedTime","PDF","PeakDetect","PeanoCurve","PearsonChiSquareTest","PearsonCorrelationTest","PearsonDistribution","PenttinenPointProcess","PercentForm","PerfectNumber","PerfectNumberQ","PerformanceGoal","Perimeter","PeriodicBoundaryCondition","PeriodicInterpolation","Periodogram","PeriodogramArray","Permanent","Permissions","PermissionsGroup","PermissionsGroupMemberQ","PermissionsGroups","PermissionsKey","PermissionsKeys","PermutationCycles","PermutationCyclesQ","PermutationGroup","PermutationLength","PermutationList","PermutationListQ","PermutationMatrix","PermutationMax","PermutationMin","PermutationOrder","PermutationPower","PermutationProduct","PermutationReplace","Permutations","PermutationSupport","Permute","PeronaMalikFilter","Perpendicular","PerpendicularBisector","PersistenceLocation","PersistenceTime","PersistentObject","PersistentObjects","PersistentSymbol","PersistentValue","PersonData","PERTDistribution","PetersenGraph","PhaseMargins","PhaseRange","PhongShading","PhysicalSystemData","Pi","Pick","PickedElements","PickMode","PIDData","PIDDerivativeFilter","PIDFeedforward","PIDTune","Piecewise","PiecewiseExpand","PieChart","PieChart3D","PillaiTrace","PillaiTraceTest","PingTime","Pink","PitchRecognize","Pivoting","PixelConstrained","PixelValue","PixelValuePositions","Placed","Placeholder","PlaceholderLayer","PlaceholderReplace","Plain","PlanarAngle","PlanarFaceList","PlanarGraph","PlanarGraphQ","PlanckRadiationLaw","PlaneCurveData","PlanetaryMoonData","PlanetData","PlantData","Play","PlaybackSettings","PlayRange","Plot","Plot3D","Plot3Matrix","PlotDivision","PlotJoined","PlotLabel","PlotLabels","PlotLayout","PlotLegends","PlotMarkers","PlotPoints","PlotRange","PlotRangeClipping","PlotRangeClipPlanesStyle","PlotRangePadding","PlotRegion","PlotStyle","PlotTheme","Pluralize","Plus","PlusMinus","Pochhammer","PodStates","PodWidth","Point","Point3DBox","Point3DBoxOptions","PointBox","PointBoxOptions","PointCountDistribution","PointDensity","PointDensityFunction","PointFigureChart","PointLegend","PointLight","PointProcessEstimator","PointProcessFitTest","PointProcessParameterAssumptions","PointProcessParameterQ","PointSize","PointStatisticFunction","PointValuePlot","PoissonConsulDistribution","PoissonDistribution","PoissonPDEComponent","PoissonPointProcess","PoissonProcess","PoissonWindow","PolarAxes","PolarAxesOrigin","PolarGridLines","PolarPlot","PolarTicks","PoleZeroMarkers","PolyaAeppliDistribution","PolyGamma","Polygon","Polygon3DBox","Polygon3DBoxOptions","PolygonalNumber","PolygonAngle","PolygonBox","PolygonBoxOptions","PolygonCoordinates","PolygonDecomposition","PolygonHoleScale","PolygonIntersections","PolygonScale","Polyhedron","PolyhedronAngle","PolyhedronBox","PolyhedronBoxOptions","PolyhedronCoordinates","PolyhedronData","PolyhedronDecomposition","PolyhedronGenus","PolyLog","PolynomialExpressionQ","PolynomialExtendedGCD","PolynomialForm","PolynomialGCD","PolynomialLCM","PolynomialMod","PolynomialQ","PolynomialQuotient","PolynomialQuotientRemainder","PolynomialReduce","PolynomialRemainder","Polynomials","PolynomialSumOfSquaresList","PoolingLayer","PopupMenu","PopupMenuBox","PopupMenuBoxOptions","PopupView","PopupWindow","Position","PositionIndex","PositionLargest","PositionSmallest","Positive","PositiveDefiniteMatrixQ","PositiveIntegers","PositivelyOrientedPoints","PositiveRationals","PositiveReals","PositiveSemidefiniteMatrixQ","PossibleZeroQ","Postfix","PostScript","Power","PowerDistribution","PowerExpand","PowerMod","PowerModList","PowerRange","PowerSpectralDensity","PowersRepresentations","PowerSymmetricPolynomial","Precedence","PrecedenceForm","Precedes","PrecedesEqual","PrecedesSlantEqual","PrecedesTilde","Precision","PrecisionGoal","PreDecrement","Predict","PredictionRoot","PredictorFunction","PredictorInformation","PredictorMeasurements","PredictorMeasurementsObject","PreemptProtect","PreferencesPath","PreferencesSettings","Prefix","PreIncrement","Prepend","PrependLayer","PrependTo","PreprocessingRules","PreserveColor","PreserveImageOptions","Previous","PreviousCell","PreviousDate","PriceGraphDistribution","PrimaryPlaceholder","Prime","PrimeNu","PrimeOmega","PrimePi","PrimePowerQ","PrimeQ","Primes","PrimeZetaP","PrimitivePolynomialQ","PrimitiveRoot","PrimitiveRootList","PrincipalComponents","PrincipalValue","Print","PrintableASCIIQ","PrintAction","PrintForm","PrintingCopies","PrintingOptions","PrintingPageRange","PrintingStartingPageNumber","PrintingStyleEnvironment","Printout3D","Printout3DPreviewer","PrintPrecision","PrintTemporary","Prism","PrismBox","PrismBoxOptions","PrivateCellOptions","PrivateEvaluationOptions","PrivateFontOptions","PrivateFrontEndOptions","PrivateKey","PrivateNotebookOptions","PrivatePaths","Probability","ProbabilityDistribution","ProbabilityPlot","ProbabilityPr","ProbabilityScalePlot","ProbitModelFit","ProcessConnection","ProcessDirectory","ProcessEnvironment","Processes","ProcessEstimator","ProcessInformation","ProcessObject","ProcessParameterAssumptions","ProcessParameterQ","ProcessStateDomain","ProcessStatus","ProcessTimeDomain","Product","ProductDistribution","ProductLog","ProgressIndicator","ProgressIndicatorBox","ProgressIndicatorBoxOptions","ProgressReporting","Projection","Prolog","PromptForm","ProofObject","PropagateAborts","Properties","Property","PropertyList","PropertyValue","Proportion","Proportional","Protect","Protected","ProteinData","Pruning","PseudoInverse","PsychrometricPropertyData","PublicKey","PublisherID","PulsarData","PunctuationCharacter","Purple","Put","PutAppend","Pyramid","PyramidBox","PyramidBoxOptions","QBinomial","QFactorial","QGamma","QHypergeometricPFQ","QnDispersion","QPochhammer","QPolyGamma","QRDecomposition","QuadraticIrrationalQ","QuadraticOptimization","Quantile","QuantilePlot","Quantity","QuantityArray","QuantityDistribution","QuantityForm","QuantityMagnitude","QuantityQ","QuantityUnit","QuantityVariable","QuantityVariableCanonicalUnit","QuantityVariableDimensions","QuantityVariableIdentifier","QuantityVariablePhysicalQuantity","Quartics","QuartileDeviation","Quartiles","QuartileSkewness","Query","QuestionGenerator","QuestionInterface","QuestionObject","QuestionSelector","QueueingNetworkProcess","QueueingProcess","QueueProperties","Quiet","QuietEcho","Quit","Quotient","QuotientRemainder","RadialAxisPlot","RadialGradientFilling","RadialGradientImage","RadialityCentrality","RadicalBox","RadicalBoxOptions","RadioButton","RadioButtonBar","RadioButtonBox","RadioButtonBoxOptions","Radon","RadonTransform","RamanujanTau","RamanujanTauL","RamanujanTauTheta","RamanujanTauZ","Ramp","Random","RandomArrayLayer","RandomChoice","RandomColor","RandomComplex","RandomDate","RandomEntity","RandomFunction","RandomGeneratorState","RandomGeoPosition","RandomGraph","RandomImage","RandomInstance","RandomInteger","RandomPermutation","RandomPoint","RandomPointConfiguration","RandomPolygon","RandomPolyhedron","RandomPrime","RandomReal","RandomSample","RandomSeed","RandomSeeding","RandomTime","RandomTree","RandomVariate","RandomWalkProcess","RandomWord","Range","RangeFilter","RangeSpecification","RankedMax","RankedMin","RarerProbability","Raster","Raster3D","Raster3DBox","Raster3DBoxOptions","RasterArray","RasterBox","RasterBoxOptions","Rasterize","RasterSize","Rational","RationalExpressionQ","RationalFunctions","Rationalize","Rationals","Ratios","RawArray","RawBoxes","RawData","RawMedium","RayleighDistribution","Re","ReactionBalance","ReactionBalancedQ","ReactionPDETerm","Read","ReadByteArray","ReadLine","ReadList","ReadProtected","ReadString","Real","RealAbs","RealBlockDiagonalForm","RealDigits","RealExponent","Reals","RealSign","Reap","RebuildPacletData","RecalibrationFunction","RecognitionPrior","RecognitionThreshold","ReconstructionMesh","Record","RecordLists","RecordSeparators","Rectangle","RectangleBox","RectangleBoxOptions","RectangleChart","RectangleChart3D","RectangularRepeatingElement","RecurrenceFilter","RecurrenceTable","RecurringDigitsForm","Red","Reduce","RefBox","ReferenceLineStyle","ReferenceMarkers","ReferenceMarkerStyle","Refine","ReflectionMatrix","ReflectionTransform","Refresh","RefreshRate","Region","RegionBinarize","RegionBoundary","RegionBoundaryStyle","RegionBounds","RegionCentroid","RegionCongruent","RegionConvert","RegionDifference","RegionDilation","RegionDimension","RegionDisjoint","RegionDistance","RegionDistanceFunction","RegionEmbeddingDimension","RegionEqual","RegionErosion","RegionFillingStyle","RegionFit","RegionFunction","RegionImage","RegionIntersection","RegionMeasure","RegionMember","RegionMemberFunction","RegionMoment","RegionNearest","RegionNearestFunction","RegionPlot","RegionPlot3D","RegionProduct","RegionQ","RegionResize","RegionSimilar","RegionSize","RegionSymmetricDifference","RegionUnion","RegionWithin","RegisterExternalEvaluator","RegularExpression","Regularization","RegularlySampledQ","RegularPolygon","ReIm","ReImLabels","ReImPlot","ReImStyle","Reinstall","RelationalDatabase","RelationGraph","Release","ReleaseHold","ReliabilityDistribution","ReliefImage","ReliefPlot","RemoteAuthorizationCaching","RemoteBatchJobAbort","RemoteBatchJobObject","RemoteBatchJobs","RemoteBatchMapSubmit","RemoteBatchSubmissionEnvironment","RemoteBatchSubmit","RemoteConnect","RemoteConnectionObject","RemoteEvaluate","RemoteFile","RemoteInputFiles","RemoteKernelObject","RemoteProviderSettings","RemoteRun","RemoteRunProcess","RemovalConditions","Remove","RemoveAlphaChannel","RemoveAsynchronousTask","RemoveAudioStream","RemoveBackground","RemoveChannelListener","RemoveChannelSubscribers","Removed","RemoveDiacritics","RemoveInputStreamMethod","RemoveOutputStreamMethod","RemoveProperty","RemoveScheduledTask","RemoveUsers","RemoveVideoStream","RenameDirectory","RenameFile","RenderAll","RenderingOptions","RenewalProcess","RenkoChart","RepairMesh","Repeated","RepeatedNull","RepeatedString","RepeatedTiming","RepeatingElement","Replace","ReplaceAll","ReplaceAt","ReplaceHeldPart","ReplaceImageValue","ReplaceList","ReplacePart","ReplacePixelValue","ReplaceRepeated","ReplicateLayer","RequiredPhysicalQuantities","Resampling","ResamplingAlgorithmData","ResamplingMethod","Rescale","RescalingTransform","ResetDirectory","ResetScheduledTask","ReshapeLayer","Residue","ResidueSum","ResizeLayer","Resolve","ResolveContextAliases","ResourceAcquire","ResourceData","ResourceFunction","ResourceObject","ResourceRegister","ResourceRemove","ResourceSearch","ResourceSubmissionObject","ResourceSubmit","ResourceSystemBase","ResourceSystemPath","ResourceUpdate","ResourceVersion","ResponseForm","Rest","RestartInterval","Restricted","Resultant","ResumePacket","Return","ReturnCreatesNewCell","ReturnEntersInput","ReturnExpressionPacket","ReturnInputFormPacket","ReturnPacket","ReturnReceiptFunction","ReturnTextPacket","Reverse","ReverseApplied","ReverseBiorthogonalSplineWavelet","ReverseElement","ReverseEquilibrium","ReverseGraph","ReverseSort","ReverseSortBy","ReverseUpEquilibrium","RevolutionAxis","RevolutionPlot3D","RGBColor","RiccatiSolve","RiceDistribution","RidgeFilter","RiemannR","RiemannSiegelTheta","RiemannSiegelZ","RiemannXi","Riffle","Right","RightArrow","RightArrowBar","RightArrowLeftArrow","RightComposition","RightCosetRepresentative","RightDownTeeVector","RightDownVector","RightDownVectorBar","RightTee","RightTeeArrow","RightTeeVector","RightTriangle","RightTriangleBar","RightTriangleEqual","RightUpDownVector","RightUpTeeVector","RightUpVector","RightUpVectorBar","RightVector","RightVectorBar","RipleyK","RipleyRassonRegion","RiskAchievementImportance","RiskReductionImportance","RobustConvexOptimization","RogersTanimotoDissimilarity","RollPitchYawAngles","RollPitchYawMatrix","RomanNumeral","Root","RootApproximant","RootIntervals","RootLocusPlot","RootMeanSquare","RootOfUnityQ","RootReduce","Roots","RootSum","RootTree","Rotate","RotateLabel","RotateLeft","RotateRight","RotationAction","RotationBox","RotationBoxOptions","RotationMatrix","RotationTransform","Round","RoundImplies","RoundingRadius","Row","RowAlignments","RowBackgrounds","RowBox","RowHeights","RowLines","RowMinHeight","RowReduce","RowsEqual","RowSpacings","RSolve","RSolveValue","RudinShapiro","RudvalisGroupRu","Rule","RuleCondition","RuleDelayed","RuleForm","RulePlot","RulerUnits","RulesTree","Run","RunProcess","RunScheduledTask","RunThrough","RuntimeAttributes","RuntimeOptions","RussellRaoDissimilarity","SameAs","SameQ","SameTest","SameTestProperties","SampledEntityClass","SampleDepth","SampledSoundFunction","SampledSoundList","SampleRate","SamplingPeriod","SARIMAProcess","SARMAProcess","SASTriangle","SatelliteData","SatisfiabilityCount","SatisfiabilityInstances","SatisfiableQ","Saturday","Save","Saveable","SaveAutoDelete","SaveConnection","SaveDefinitions","SavitzkyGolayMatrix","SawtoothWave","Scale","Scaled","ScaleDivisions","ScaledMousePosition","ScaleOrigin","ScalePadding","ScaleRanges","ScaleRangeStyle","ScalingFunctions","ScalingMatrix","ScalingTransform","Scan","ScheduledTask","ScheduledTaskActiveQ","ScheduledTaskInformation","ScheduledTaskInformationData","ScheduledTaskObject","ScheduledTasks","SchurDecomposition","ScientificForm","ScientificNotationThreshold","ScorerGi","ScorerGiPrime","ScorerHi","ScorerHiPrime","ScreenRectangle","ScreenStyleEnvironment","ScriptBaselineShifts","ScriptForm","ScriptLevel","ScriptMinSize","ScriptRules","ScriptSizeMultipliers","Scrollbars","ScrollingOptions","ScrollPosition","SearchAdjustment","SearchIndexObject","SearchIndices","SearchQueryString","SearchResultObject","Sec","Sech","SechDistribution","SecondOrderConeOptimization","SectionGrouping","SectorChart","SectorChart3D","SectorOrigin","SectorSpacing","SecuredAuthenticationKey","SecuredAuthenticationKeys","SecurityCertificate","SeedRandom","Select","Selectable","SelectComponents","SelectedCells","SelectedNotebook","SelectFirst","Selection","SelectionAnimate","SelectionCell","SelectionCellCreateCell","SelectionCellDefaultStyle","SelectionCellParentStyle","SelectionCreateCell","SelectionDebuggerTag","SelectionEvaluate","SelectionEvaluateCreateCell","SelectionMove","SelectionPlaceholder","SelectWithContents","SelfLoops","SelfLoopStyle","SemanticImport","SemanticImportString","SemanticInterpretation","SemialgebraicComponentInstances","SemidefiniteOptimization","SendMail","SendMessage","Sequence","SequenceAlignment","SequenceAttentionLayer","SequenceCases","SequenceCount","SequenceFold","SequenceFoldList","SequenceForm","SequenceHold","SequenceIndicesLayer","SequenceLastLayer","SequenceMostLayer","SequencePosition","SequencePredict","SequencePredictorFunction","SequenceReplace","SequenceRestLayer","SequenceReverseLayer","SequenceSplit","Series","SeriesCoefficient","SeriesData","SeriesTermGoal","ServiceConnect","ServiceDisconnect","ServiceExecute","ServiceObject","ServiceRequest","ServiceResponse","ServiceSubmit","SessionSubmit","SessionTime","Set","SetAccuracy","SetAlphaChannel","SetAttributes","Setbacks","SetCloudDirectory","SetCookies","SetDelayed","SetDirectory","SetEnvironment","SetFileDate","SetFileFormatProperties","SetOptions","SetOptionsPacket","SetPermissions","SetPrecision","SetProperty","SetSecuredAuthenticationKey","SetSelectedNotebook","SetSharedFunction","SetSharedVariable","SetStreamPosition","SetSystemModel","SetSystemOptions","Setter","SetterBar","SetterBox","SetterBoxOptions","Setting","SetUsers","Shading","Shallow","ShannonWavelet","ShapiroWilkTest","Share","SharingList","Sharpen","ShearingMatrix","ShearingTransform","ShellRegion","ShenCastanMatrix","ShiftedGompertzDistribution","ShiftRegisterSequence","Short","ShortDownArrow","Shortest","ShortestMatch","ShortestPathFunction","ShortLeftArrow","ShortRightArrow","ShortTimeFourier","ShortTimeFourierData","ShortUpArrow","Show","ShowAutoConvert","ShowAutoSpellCheck","ShowAutoStyles","ShowCellBracket","ShowCellLabel","ShowCellTags","ShowClosedCellArea","ShowCodeAssist","ShowContents","ShowControls","ShowCursorTracker","ShowGroupOpenCloseIcon","ShowGroupOpener","ShowInvisibleCharacters","ShowPageBreaks","ShowPredictiveInterface","ShowSelection","ShowShortBoxForm","ShowSpecialCharacters","ShowStringCharacters","ShowSyntaxStyles","ShrinkingDelay","ShrinkWrapBoundingBox","SiderealTime","SiegelTheta","SiegelTukeyTest","SierpinskiCurve","SierpinskiMesh","Sign","Signature","SignedRankTest","SignedRegionDistance","SignificanceLevel","SignPadding","SignTest","SimilarityRules","SimpleGraph","SimpleGraphQ","SimplePolygonQ","SimplePolyhedronQ","Simplex","Simplify","Sin","Sinc","SinghMaddalaDistribution","SingleEvaluation","SingleLetterItalics","SingleLetterStyle","SingularValueDecomposition","SingularValueList","SingularValuePlot","SingularValues","Sinh","SinhIntegral","SinIntegral","SixJSymbol","Skeleton","SkeletonTransform","SkellamDistribution","Skewness","SkewNormalDistribution","SkinStyle","Skip","SliceContourPlot3D","SliceDensityPlot3D","SliceDistribution","SliceVectorPlot3D","Slider","Slider2D","Slider2DBox","Slider2DBoxOptions","SliderBox","SliderBoxOptions","SlideShowVideo","SlideView","Slot","SlotSequence","Small","SmallCircle","Smaller","SmithDecomposition","SmithDelayCompensator","SmithWatermanSimilarity","SmoothDensityHistogram","SmoothHistogram","SmoothHistogram3D","SmoothKernelDistribution","SmoothPointDensity","SnDispersion","Snippet","SnippetsVideo","SnubPolyhedron","SocialMediaData","Socket","SocketConnect","SocketListen","SocketListener","SocketObject","SocketOpen","SocketReadMessage","SocketReadyQ","Sockets","SocketWaitAll","SocketWaitNext","SoftmaxLayer","SokalSneathDissimilarity","SolarEclipse","SolarSystemFeatureData","SolarTime","SolidAngle","SolidBoundaryLoadValue","SolidData","SolidDisplacementCondition","SolidFixedCondition","SolidMechanicsPDEComponent","SolidMechanicsStrain","SolidMechanicsStress","SolidRegionQ","Solve","SolveAlways","SolveDelayed","SolveValues","Sort","SortBy","SortedBy","SortedEntityClass","Sound","SoundAndGraphics","SoundNote","SoundVolume","SourceLink","SourcePDETerm","Sow","Space","SpaceCurveData","SpaceForm","Spacer","Spacings","Span","SpanAdjustments","SpanCharacterRounding","SpanFromAbove","SpanFromBoth","SpanFromLeft","SpanLineThickness","SpanMaxSize","SpanMinSize","SpanningCharacters","SpanSymmetric","SparseArray","SparseArrayQ","SpatialBinnedPointData","SpatialBoundaryCorrection","SpatialEstimate","SpatialEstimatorFunction","SpatialGraphDistribution","SpatialJ","SpatialMedian","SpatialNoiseLevel","SpatialObservationRegionQ","SpatialPointData","SpatialPointSelect","SpatialRandomnessTest","SpatialTransformationLayer","SpatialTrendFunction","Speak","SpeakerMatchQ","SpearmanRankTest","SpearmanRho","SpeciesData","SpecificityGoal","SpectralLineData","Spectrogram","SpectrogramArray","Specularity","SpeechCases","SpeechInterpreter","SpeechRecognize","SpeechSynthesize","SpellingCorrection","SpellingCorrectionList","SpellingDictionaries","SpellingDictionariesPath","SpellingOptions","Sphere","SphereBox","SphereBoxOptions","SpherePoints","SphericalBesselJ","SphericalBesselY","SphericalHankelH1","SphericalHankelH2","SphericalHarmonicY","SphericalPlot3D","SphericalRegion","SphericalShell","SpheroidalEigenvalue","SpheroidalJoiningFactor","SpheroidalPS","SpheroidalPSPrime","SpheroidalQS","SpheroidalQSPrime","SpheroidalRadialFactor","SpheroidalS1","SpheroidalS1Prime","SpheroidalS2","SpheroidalS2Prime","Splice","SplicedDistribution","SplineClosed","SplineDegree","SplineKnots","SplineWeights","Split","SplitBy","SpokenString","SpotLight","Sqrt","SqrtBox","SqrtBoxOptions","Square","SquaredEuclideanDistance","SquareFreeQ","SquareIntersection","SquareMatrixQ","SquareRepeatingElement","SquaresR","SquareSubset","SquareSubsetEqual","SquareSuperset","SquareSupersetEqual","SquareUnion","SquareWave","SSSTriangle","StabilityMargins","StabilityMarginsStyle","StableDistribution","Stack","StackBegin","StackComplete","StackedDateListPlot","StackedListPlot","StackInhibit","StadiumShape","StandardAtmosphereData","StandardDeviation","StandardDeviationFilter","StandardForm","Standardize","Standardized","StandardOceanData","StandbyDistribution","Star","StarClusterData","StarData","StarGraph","StartAsynchronousTask","StartExternalSession","StartingStepSize","StartOfLine","StartOfString","StartProcess","StartScheduledTask","StartupSound","StartWebSession","StateDimensions","StateFeedbackGains","StateOutputEstimator","StateResponse","StateSpaceModel","StateSpaceRealization","StateSpaceTransform","StateTransformationLinearize","StationaryDistribution","StationaryWaveletPacketTransform","StationaryWaveletTransform","StatusArea","StatusCentrality","StepMonitor","StereochemistryElements","StieltjesGamma","StippleShading","StirlingS1","StirlingS2","StopAsynchronousTask","StoppingPowerData","StopScheduledTask","StrataVariables","StratonovichProcess","StraussHardcorePointProcess","StraussPointProcess","StreamColorFunction","StreamColorFunctionScaling","StreamDensityPlot","StreamMarkers","StreamPlot","StreamPlot3D","StreamPoints","StreamPosition","Streams","StreamScale","StreamStyle","StrictInequalities","String","StringBreak","StringByteCount","StringCases","StringContainsQ","StringCount","StringDelete","StringDrop","StringEndsQ","StringExpression","StringExtract","StringForm","StringFormat","StringFormatQ","StringFreeQ","StringInsert","StringJoin","StringLength","StringMatchQ","StringPadLeft","StringPadRight","StringPart","StringPartition","StringPosition","StringQ","StringRepeat","StringReplace","StringReplaceList","StringReplacePart","StringReverse","StringRiffle","StringRotateLeft","StringRotateRight","StringSkeleton","StringSplit","StringStartsQ","StringTake","StringTakeDrop","StringTemplate","StringToByteArray","StringToStream","StringTrim","StripBoxes","StripOnInput","StripStyleOnPaste","StripWrapperBoxes","StrokeForm","Struckthrough","StructuralImportance","StructuredArray","StructuredArrayHeadQ","StructuredSelection","StruveH","StruveL","Stub","StudentTDistribution","Style","StyleBox","StyleBoxAutoDelete","StyleData","StyleDefinitions","StyleForm","StyleHints","StyleKeyMapping","StyleMenuListing","StyleNameDialogSettings","StyleNames","StylePrint","StyleSheetPath","Subdivide","Subfactorial","Subgraph","SubMinus","SubPlus","SubresultantPolynomialRemainders","SubresultantPolynomials","Subresultants","Subscript","SubscriptBox","SubscriptBoxOptions","Subscripted","Subsequences","Subset","SubsetCases","SubsetCount","SubsetEqual","SubsetMap","SubsetPosition","SubsetQ","SubsetReplace","Subsets","SubStar","SubstitutionSystem","Subsuperscript","SubsuperscriptBox","SubsuperscriptBoxOptions","SubtitleEncoding","SubtitleTrackSelection","Subtract","SubtractFrom","SubtractSides","SubValues","Succeeds","SucceedsEqual","SucceedsSlantEqual","SucceedsTilde","Success","SuchThat","Sum","SumConvergence","SummationLayer","Sunday","SunPosition","Sunrise","Sunset","SuperDagger","SuperMinus","SupernovaData","SuperPlus","Superscript","SuperscriptBox","SuperscriptBoxOptions","Superset","SupersetEqual","SuperStar","Surd","SurdForm","SurfaceAppearance","SurfaceArea","SurfaceColor","SurfaceData","SurfaceGraphics","SurvivalDistribution","SurvivalFunction","SurvivalModel","SurvivalModelFit","SuspendPacket","SuzukiDistribution","SuzukiGroupSuz","SwatchLegend","Switch","Symbol","SymbolName","SymletWavelet","Symmetric","SymmetricDifference","SymmetricGroup","SymmetricKey","SymmetricMatrixQ","SymmetricPolynomial","SymmetricReduction","Symmetrize","SymmetrizedArray","SymmetrizedArrayRules","SymmetrizedDependentComponents","SymmetrizedIndependentComponents","SymmetrizedReplacePart","SynchronousInitialization","SynchronousUpdating","Synonyms","Syntax","SyntaxForm","SyntaxInformation","SyntaxLength","SyntaxPacket","SyntaxQ","SynthesizeMissingValues","SystemCredential","SystemCredentialData","SystemCredentialKey","SystemCredentialKeys","SystemCredentialStoreObject","SystemDialogInput","SystemException","SystemGet","SystemHelpPath","SystemInformation","SystemInformationData","SystemInstall","SystemModel","SystemModeler","SystemModelExamples","SystemModelLinearize","SystemModelMeasurements","SystemModelParametricSimulate","SystemModelPlot","SystemModelProgressReporting","SystemModelReliability","SystemModels","SystemModelSimulate","SystemModelSimulateSensitivity","SystemModelSimulationData","SystemOpen","SystemOptions","SystemProcessData","SystemProcesses","SystemsConnectionsModel","SystemsModelControllerData","SystemsModelDelay","SystemsModelDelayApproximate","SystemsModelDelete","SystemsModelDimensions","SystemsModelExtract","SystemsModelFeedbackConnect","SystemsModelLabels","SystemsModelLinearity","SystemsModelMerge","SystemsModelOrder","SystemsModelParallelConnect","SystemsModelSeriesConnect","SystemsModelStateFeedbackConnect","SystemsModelVectorRelativeOrders","SystemStub","SystemTest","Tab","TabFilling","Table","TableAlignments","TableDepth","TableDirections","TableForm","TableHeadings","TableSpacing","TableView","TableViewBox","TableViewBoxAlignment","TableViewBoxBackground","TableViewBoxHeaders","TableViewBoxItemSize","TableViewBoxItemStyle","TableViewBoxOptions","TabSpacings","TabView","TabViewBox","TabViewBoxOptions","TagBox","TagBoxNote","TagBoxOptions","TaggingRules","TagSet","TagSetDelayed","TagStyle","TagUnset","Take","TakeDrop","TakeLargest","TakeLargestBy","TakeList","TakeSmallest","TakeSmallestBy","TakeWhile","Tally","Tan","Tanh","TargetDevice","TargetFunctions","TargetSystem","TargetUnits","TaskAbort","TaskExecute","TaskObject","TaskRemove","TaskResume","Tasks","TaskSuspend","TaskWait","TautologyQ","TelegraphProcess","TemplateApply","TemplateArgBox","TemplateBox","TemplateBoxOptions","TemplateEvaluate","TemplateExpression","TemplateIf","TemplateObject","TemplateSequence","TemplateSlot","TemplateSlotSequence","TemplateUnevaluated","TemplateVerbatim","TemplateWith","TemporalData","TemporalRegularity","Temporary","TemporaryVariable","TensorContract","TensorDimensions","TensorExpand","TensorProduct","TensorQ","TensorRank","TensorReduce","TensorSymmetry","TensorTranspose","TensorWedge","TerminatedEvaluation","TernaryListPlot","TernaryPlotCorners","TestID","TestReport","TestReportObject","TestResultObject","Tetrahedron","TetrahedronBox","TetrahedronBoxOptions","TeXForm","TeXSave","Text","Text3DBox","Text3DBoxOptions","TextAlignment","TextBand","TextBoundingBox","TextBox","TextCases","TextCell","TextClipboardType","TextContents","TextData","TextElement","TextForm","TextGrid","TextJustification","TextLine","TextPacket","TextParagraph","TextPosition","TextRecognize","TextSearch","TextSearchReport","TextSentences","TextString","TextStructure","TextStyle","TextTranslation","Texture","TextureCoordinateFunction","TextureCoordinateScaling","TextWords","Therefore","ThermodynamicData","ThermometerGauge","Thick","Thickness","Thin","Thinning","ThisLink","ThomasPointProcess","ThompsonGroupTh","Thread","Threaded","ThreadingLayer","ThreeJSymbol","Threshold","Through","Throw","ThueMorse","Thumbnail","Thursday","TickDirection","TickLabelOrientation","TickLabelPositioning","TickLabels","TickLengths","TickPositions","Ticks","TicksStyle","TideData","Tilde","TildeEqual","TildeFullEqual","TildeTilde","TimeConstrained","TimeConstraint","TimeDirection","TimeFormat","TimeGoal","TimelinePlot","TimeObject","TimeObjectQ","TimeRemaining","Times","TimesBy","TimeSeries","TimeSeriesAggregate","TimeSeriesForecast","TimeSeriesInsert","TimeSeriesInvertibility","TimeSeriesMap","TimeSeriesMapThread","TimeSeriesModel","TimeSeriesModelFit","TimeSeriesResample","TimeSeriesRescale","TimeSeriesShift","TimeSeriesThread","TimeSeriesWindow","TimeSystem","TimeSystemConvert","TimeUsed","TimeValue","TimeWarpingCorrespondence","TimeWarpingDistance","TimeZone","TimeZoneConvert","TimeZoneOffset","Timing","Tiny","TitleGrouping","TitsGroupT","ToBoxes","ToCharacterCode","ToColor","ToContinuousTimeModel","ToDate","Today","ToDiscreteTimeModel","ToEntity","ToeplitzMatrix","ToExpression","ToFileName","Together","Toggle","ToggleFalse","Toggler","TogglerBar","TogglerBox","TogglerBoxOptions","ToHeldExpression","ToInvertibleTimeSeries","TokenWords","Tolerance","ToLowerCase","Tomorrow","ToNumberField","TooBig","Tooltip","TooltipBox","TooltipBoxOptions","TooltipDelay","TooltipStyle","ToonShading","Top","TopHatTransform","ToPolarCoordinates","TopologicalSort","ToRadicals","ToRawPointer","ToRules","Torus","TorusGraph","ToSphericalCoordinates","ToString","Total","TotalHeight","TotalLayer","TotalVariationFilter","TotalWidth","TouchPosition","TouchscreenAutoZoom","TouchscreenControlPlacement","ToUpperCase","TourVideo","Tr","Trace","TraceAbove","TraceAction","TraceBackward","TraceDepth","TraceDialog","TraceForward","TraceInternal","TraceLevel","TraceOff","TraceOn","TraceOriginal","TracePrint","TraceScan","TrackCellChangeTimes","TrackedSymbols","TrackingFunction","TracyWidomDistribution","TradingChart","TraditionalForm","TraditionalFunctionNotation","TraditionalNotation","TraditionalOrder","TrainImageContentDetector","TrainingProgressCheckpointing","TrainingProgressFunction","TrainingProgressMeasurements","TrainingProgressReporting","TrainingStoppingCriterion","TrainingUpdateSchedule","TrainTextContentDetector","TransferFunctionCancel","TransferFunctionExpand","TransferFunctionFactor","TransferFunctionModel","TransferFunctionPoles","TransferFunctionTransform","TransferFunctionZeros","TransformationClass","TransformationFunction","TransformationFunctions","TransformationMatrix","TransformedDistribution","TransformedField","TransformedProcess","TransformedRegion","TransitionDirection","TransitionDuration","TransitionEffect","TransitiveClosureGraph","TransitiveReductionGraph","Translate","TranslationOptions","TranslationTransform","Transliterate","Transparent","TransparentColor","Transpose","TransposeLayer","TrapEnterKey","TrapSelection","TravelDirections","TravelDirectionsData","TravelDistance","TravelDistanceList","TravelMethod","TravelTime","Tree","TreeCases","TreeChildren","TreeCount","TreeData","TreeDelete","TreeDepth","TreeElementCoordinates","TreeElementLabel","TreeElementLabelFunction","TreeElementLabelStyle","TreeElementShape","TreeElementShapeFunction","TreeElementSize","TreeElementSizeFunction","TreeElementStyle","TreeElementStyleFunction","TreeExpression","TreeExtract","TreeFold","TreeForm","TreeGraph","TreeGraphQ","TreeInsert","TreeLayout","TreeLeafCount","TreeLeafQ","TreeLeaves","TreeLevel","TreeMap","TreeMapAt","TreeOutline","TreePlot","TreePosition","TreeQ","TreeReplacePart","TreeRules","TreeScan","TreeSelect","TreeSize","TreeTraversalOrder","TrendStyle","Triangle","TriangleCenter","TriangleConstruct","TriangleMeasurement","TriangleWave","TriangularDistribution","TriangulateMesh","Trig","TrigExpand","TrigFactor","TrigFactorList","Trigger","TrigReduce","TrigToExp","TrimmedMean","TrimmedVariance","TropicalStormData","True","TrueQ","TruncatedDistribution","TruncatedPolyhedron","TsallisQExponentialDistribution","TsallisQGaussianDistribution","TTest","Tube","TubeBezierCurveBox","TubeBezierCurveBoxOptions","TubeBox","TubeBoxOptions","TubeBSplineCurveBox","TubeBSplineCurveBoxOptions","Tuesday","TukeyLambdaDistribution","TukeyWindow","TunnelData","Tuples","TuranGraph","TuringMachine","TuttePolynomial","TwoWayRule","Typed","TypeDeclaration","TypeEvaluate","TypeHint","TypeOf","TypeSpecifier","UnateQ","Uncompress","UnconstrainedParameters","Undefined","UnderBar","Underflow","Underlined","Underoverscript","UnderoverscriptBox","UnderoverscriptBoxOptions","Underscript","UnderscriptBox","UnderscriptBoxOptions","UnderseaFeatureData","UndirectedEdge","UndirectedGraph","UndirectedGraphQ","UndoOptions","UndoTrackedVariables","Unequal","UnequalTo","Unevaluated","UniformDistribution","UniformGraphDistribution","UniformPolyhedron","UniformSumDistribution","Uninstall","Union","UnionedEntityClass","UnionPlus","Unique","UniqueElements","UnitaryMatrixQ","UnitBox","UnitConvert","UnitDimensions","Unitize","UnitRootTest","UnitSimplify","UnitStep","UnitSystem","UnitTriangle","UnitVector","UnitVectorLayer","UnityDimensions","UniverseModelData","UniversityData","UnixTime","UnlabeledTree","UnmanageObject","Unprotect","UnregisterExternalEvaluator","UnsameQ","UnsavedVariables","Unset","UnsetShared","Until","UntrackedVariables","Up","UpArrow","UpArrowBar","UpArrowDownArrow","Update","UpdateDynamicObjects","UpdateDynamicObjectsSynchronous","UpdateInterval","UpdatePacletSites","UpdateSearchIndex","UpDownArrow","UpEquilibrium","UpperCaseQ","UpperLeftArrow","UpperRightArrow","UpperTriangularize","UpperTriangularMatrix","UpperTriangularMatrixQ","Upsample","UpSet","UpSetDelayed","UpTee","UpTeeArrow","UpTo","UpValues","URL","URLBuild","URLDecode","URLDispatcher","URLDownload","URLDownloadSubmit","URLEncode","URLExecute","URLExpand","URLFetch","URLFetchAsynchronous","URLParse","URLQueryDecode","URLQueryEncode","URLRead","URLResponseTime","URLSave","URLSaveAsynchronous","URLShorten","URLSubmit","UseEmbeddedLibrary","UseGraphicsRange","UserDefinedWavelet","Using","UsingFrontEnd","UtilityFunction","V2Get","ValenceErrorHandling","ValenceFilling","ValidationLength","ValidationSet","ValueBox","ValueBoxOptions","ValueDimensions","ValueForm","ValuePreprocessingFunction","ValueQ","Values","ValuesData","VandermondeMatrix","Variables","Variance","VarianceEquivalenceTest","VarianceEstimatorFunction","VarianceGammaDistribution","VarianceGammaPointProcess","VarianceTest","VariogramFunction","VariogramModel","VectorAngle","VectorAround","VectorAspectRatio","VectorColorFunction","VectorColorFunctionScaling","VectorDensityPlot","VectorDisplacementPlot","VectorDisplacementPlot3D","VectorGlyphData","VectorGreater","VectorGreaterEqual","VectorLess","VectorLessEqual","VectorMarkers","VectorPlot","VectorPlot3D","VectorPoints","VectorQ","VectorRange","Vectors","VectorScale","VectorScaling","VectorSizes","VectorStyle","Vee","Verbatim","Verbose","VerificationTest","VerifyConvergence","VerifyDerivedKey","VerifyDigitalSignature","VerifyFileSignature","VerifyInterpretation","VerifySecurityCertificates","VerifySolutions","VerifyTestAssumptions","VersionedPreferences","VertexAdd","VertexCapacity","VertexChromaticNumber","VertexColors","VertexComponent","VertexConnectivity","VertexContract","VertexCoordinateRules","VertexCoordinates","VertexCorrelationSimilarity","VertexCosineSimilarity","VertexCount","VertexCoverQ","VertexDataCoordinates","VertexDegree","VertexDelete","VertexDiceSimilarity","VertexEccentricity","VertexInComponent","VertexInComponentGraph","VertexInDegree","VertexIndex","VertexJaccardSimilarity","VertexLabeling","VertexLabels","VertexLabelStyle","VertexList","VertexNormals","VertexOutComponent","VertexOutComponentGraph","VertexOutDegree","VertexQ","VertexRenderingFunction","VertexReplace","VertexShape","VertexShapeFunction","VertexSize","VertexStyle","VertexTextureCoordinates","VertexTransitiveGraphQ","VertexWeight","VertexWeightedGraphQ","Vertical","VerticalBar","VerticalForm","VerticalGauge","VerticalSeparator","VerticalSlider","VerticalTilde","Video","VideoCapture","VideoCombine","VideoDelete","VideoEncoding","VideoExtractFrames","VideoFrameList","VideoFrameMap","VideoGenerator","VideoInsert","VideoIntervals","VideoJoin","VideoMap","VideoMapList","VideoMapTimeSeries","VideoPadding","VideoPause","VideoPlay","VideoQ","VideoRecord","VideoReplace","VideoScreenCapture","VideoSplit","VideoStop","VideoStream","VideoStreams","VideoTimeStretch","VideoTrackSelection","VideoTranscode","VideoTransparency","VideoTrim","ViewAngle","ViewCenter","ViewMatrix","ViewPoint","ViewPointSelectorSettings","ViewPort","ViewProjection","ViewRange","ViewVector","ViewVertical","VirtualGroupData","Visible","VisibleCell","VoiceStyleData","VoigtDistribution","VolcanoData","Volume","VonMisesDistribution","VoronoiMesh","WaitAll","WaitAsynchronousTask","WaitNext","WaitUntil","WakebyDistribution","WalleniusHypergeometricDistribution","WaringYuleDistribution","WarpingCorrespondence","WarpingDistance","WatershedComponents","WatsonUSquareTest","WattsStrogatzGraphDistribution","WaveletBestBasis","WaveletFilterCoefficients","WaveletImagePlot","WaveletListPlot","WaveletMapIndexed","WaveletMatrixPlot","WaveletPhi","WaveletPsi","WaveletScale","WaveletScalogram","WaveletThreshold","WavePDEComponent","WeaklyConnectedComponents","WeaklyConnectedGraphComponents","WeaklyConnectedGraphQ","WeakStationarity","WeatherData","WeatherForecastData","WebAudioSearch","WebColumn","WebElementObject","WeberE","WebExecute","WebImage","WebImageSearch","WebItem","WebPageMetaInformation","WebRow","WebSearch","WebSessionObject","WebSessions","WebWindowObject","Wedge","Wednesday","WeibullDistribution","WeierstrassE1","WeierstrassE2","WeierstrassE3","WeierstrassEta1","WeierstrassEta2","WeierstrassEta3","WeierstrassHalfPeriods","WeierstrassHalfPeriodW1","WeierstrassHalfPeriodW2","WeierstrassHalfPeriodW3","WeierstrassInvariantG2","WeierstrassInvariantG3","WeierstrassInvariants","WeierstrassP","WeierstrassPPrime","WeierstrassSigma","WeierstrassZeta","WeightedAdjacencyGraph","WeightedAdjacencyMatrix","WeightedData","WeightedGraphQ","Weights","WelchWindow","WheelGraph","WhenEvent","Which","While","White","WhiteNoiseProcess","WhitePoint","Whitespace","WhitespaceCharacter","WhittakerM","WhittakerW","WholeCellGroupOpener","WienerFilter","WienerProcess","WignerD","WignerSemicircleDistribution","WikidataData","WikidataSearch","WikipediaData","WikipediaSearch","WilksW","WilksWTest","WindDirectionData","WindingCount","WindingPolygon","WindowClickSelect","WindowElements","WindowFloating","WindowFrame","WindowFrameElements","WindowMargins","WindowMovable","WindowOpacity","WindowPersistentStyles","WindowSelected","WindowSize","WindowStatusArea","WindowTitle","WindowToolbars","WindowWidth","WindSpeedData","WindVectorData","WinsorizedMean","WinsorizedVariance","WishartMatrixDistribution","With","WithCleanup","WithLock","WolframAlpha","WolframAlphaDate","WolframAlphaQuantity","WolframAlphaResult","WolframCloudSettings","WolframLanguageData","Word","WordBoundary","WordCharacter","WordCloud","WordCount","WordCounts","WordData","WordDefinition","WordFrequency","WordFrequencyData","WordList","WordOrientation","WordSearch","WordSelectionFunction","WordSeparators","WordSpacings","WordStem","WordTranslation","WorkingPrecision","WrapAround","Write","WriteLine","WriteString","Wronskian","XMLElement","XMLObject","XMLTemplate","Xnor","Xor","XYZColor","Yellow","Yesterday","YuleDissimilarity","ZernikeR","ZeroSymmetric","ZeroTest","ZeroWidthTimes","Zeta","ZetaZero","ZIPCodeData","ZipfDistribution","ZoomCenter","ZoomFactor","ZTest","ZTransform","$Aborted","$ActivationGroupID","$ActivationKey","$ActivationUserRegistered","$AddOnsDirectory","$AllowDataUpdates","$AllowExternalChannelFunctions","$AllowInternet","$AssertFunction","$Assumptions","$AsynchronousTask","$AudioDecoders","$AudioEncoders","$AudioInputDevices","$AudioOutputDevices","$BaseDirectory","$BasePacletsDirectory","$BatchInput","$BatchOutput","$BlockchainBase","$BoxForms","$ByteOrdering","$CacheBaseDirectory","$Canceled","$ChannelBase","$CharacterEncoding","$CharacterEncodings","$CloudAccountName","$CloudBase","$CloudConnected","$CloudConnection","$CloudCreditsAvailable","$CloudEvaluation","$CloudExpressionBase","$CloudObjectNameFormat","$CloudObjectURLType","$CloudRootDirectory","$CloudSymbolBase","$CloudUserID","$CloudUserUUID","$CloudVersion","$CloudVersionNumber","$CloudWolframEngineVersionNumber","$CommandLine","$CompilationTarget","$CompilerEnvironment","$ConditionHold","$ConfiguredKernels","$Context","$ContextAliases","$ContextPath","$ControlActiveSetting","$Cookies","$CookieStore","$CreationDate","$CryptographicEllipticCurveNames","$CurrentLink","$CurrentTask","$CurrentWebSession","$DataStructures","$DateStringFormat","$DefaultAudioInputDevice","$DefaultAudioOutputDevice","$DefaultFont","$DefaultFrontEnd","$DefaultImagingDevice","$DefaultKernels","$DefaultLocalBase","$DefaultLocalKernel","$DefaultMailbox","$DefaultNetworkInterface","$DefaultPath","$DefaultProxyRules","$DefaultRemoteBatchSubmissionEnvironment","$DefaultRemoteKernel","$DefaultSystemCredentialStore","$Display","$DisplayFunction","$DistributedContexts","$DynamicEvaluation","$Echo","$EmbedCodeEnvironments","$EmbeddableServices","$EntityStores","$Epilog","$EvaluationCloudBase","$EvaluationCloudObject","$EvaluationEnvironment","$ExportFormats","$ExternalIdentifierTypes","$ExternalStorageBase","$Failed","$FinancialDataSource","$FontFamilies","$FormatType","$FrontEnd","$FrontEndSession","$GeneratedAssetLocation","$GeoEntityTypes","$GeoLocation","$GeoLocationCity","$GeoLocationCountry","$GeoLocationPrecision","$GeoLocationSource","$HistoryLength","$HomeDirectory","$HTMLExportRules","$HTTPCookies","$HTTPRequest","$IgnoreEOF","$ImageFormattingWidth","$ImageResolution","$ImagingDevice","$ImagingDevices","$ImportFormats","$IncomingMailSettings","$InitialDirectory","$Initialization","$InitializationContexts","$Input","$InputFileName","$InputStreamMethods","$Inspector","$InstallationDate","$InstallationDirectory","$InterfaceEnvironment","$InterpreterTypes","$IterationLimit","$KernelCount","$KernelID","$Language","$LaunchDirectory","$LibraryPath","$LicenseExpirationDate","$LicenseID","$LicenseProcesses","$LicenseServer","$LicenseSubprocesses","$LicenseType","$Line","$Linked","$LinkSupported","$LoadedFiles","$LocalBase","$LocalSymbolBase","$MachineAddresses","$MachineDomain","$MachineDomains","$MachineEpsilon","$MachineID","$MachineName","$MachinePrecision","$MachineType","$MaxDisplayedChildren","$MaxExtraPrecision","$MaxLicenseProcesses","$MaxLicenseSubprocesses","$MaxMachineNumber","$MaxNumber","$MaxPiecewiseCases","$MaxPrecision","$MaxRootDegree","$MessageGroups","$MessageList","$MessagePrePrint","$Messages","$MinMachineNumber","$MinNumber","$MinorReleaseNumber","$MinPrecision","$MobilePhone","$ModuleNumber","$NetworkConnected","$NetworkInterfaces","$NetworkLicense","$NewMessage","$NewSymbol","$NotebookInlineStorageLimit","$Notebooks","$NoValue","$NumberMarks","$Off","$OperatingSystem","$Output","$OutputForms","$OutputSizeLimit","$OutputStreamMethods","$Packages","$ParentLink","$ParentProcessID","$PasswordFile","$PatchLevelID","$Path","$PathnameSeparator","$PerformanceGoal","$Permissions","$PermissionsGroupBase","$PersistenceBase","$PersistencePath","$PipeSupported","$PlotTheme","$Post","$Pre","$PreferencesDirectory","$PreInitialization","$PrePrint","$PreRead","$PrintForms","$PrintLiteral","$Printout3DPreviewer","$ProcessID","$ProcessorCount","$ProcessorType","$ProductInformation","$ProgramName","$ProgressReporting","$PublisherID","$RandomGeneratorState","$RandomState","$RecursionLimit","$RegisteredDeviceClasses","$RegisteredUserName","$ReleaseNumber","$RequesterAddress","$RequesterCloudUserID","$RequesterCloudUserUUID","$RequesterWolframID","$RequesterWolframUUID","$ResourceSystemBase","$ResourceSystemPath","$RootDirectory","$ScheduledTask","$ScriptCommandLine","$ScriptInputString","$SecuredAuthenticationKeyTokens","$ServiceCreditsAvailable","$Services","$SessionID","$SetParentLink","$SharedFunctions","$SharedVariables","$SoundDisplay","$SoundDisplayFunction","$SourceLink","$SSHAuthentication","$SubtitleDecoders","$SubtitleEncoders","$SummaryBoxDataSizeLimit","$SuppressInputFormHeads","$SynchronousEvaluation","$SyntaxHandler","$System","$SystemCharacterEncoding","$SystemCredentialStore","$SystemID","$SystemMemory","$SystemShell","$SystemTimeZone","$SystemWordLength","$TargetSystems","$TemplatePath","$TemporaryDirectory","$TemporaryPrefix","$TestFileName","$TextStyle","$TimedOut","$TimeUnit","$TimeZone","$TimeZoneEntity","$TopDirectory","$TraceOff","$TraceOn","$TracePattern","$TracePostAction","$TracePreAction","$UnitSystem","$Urgent","$UserAddOnsDirectory","$UserAgentLanguages","$UserAgentMachine","$UserAgentName","$UserAgentOperatingSystem","$UserAgentString","$UserAgentVersion","$UserBaseDirectory","$UserBasePacletsDirectory","$UserDocumentsDirectory","$Username","$UserName","$UserURLBase","$Version","$VersionNumber","$VideoDecoders","$VideoEncoders","$VoiceStyles","$WolframDocumentsDirectory","$WolframID","$WolframUUID"];e.exports=function(e){const n=e.regex,a=n.either(n.concat(/([2-9]|[1-2]\d|[3][0-5])\^\^/,/(\w*\.\w+|\w+\.\w*|\w+)/),/(\d*\.\d+|\d+\.\d*|\d+)/),r=n.either(/``[+-]?(\d*\.\d+|\d+\.\d*|\d+)/,/`([+-]?(\d*\.\d+|\d+\.\d*|\d+))?/),i={className:"number",relevance:0,begin:n.concat(a,n.optional(r),n.optional(/\*\^[+-]?\d+/))},o=/[a-zA-Z$][a-zA-Z0-9$]*/,s=new Set(t),l={variants:[{className:"builtin-symbol",begin:o,"on:begin":(e,t)=>{s.has(e[0])||t.ignoreMatch()}},{className:"symbol",relevance:0,begin:o}]},c={className:"message-name",relevance:0,begin:n.concat("::",o)};return{name:"Mathematica",aliases:["mma","wl"],classNameAliases:{brace:"punctuation",pattern:"type",slot:"type",symbol:"variable","named-character":"variable","builtin-symbol":"built_in","message-name":"string"},contains:[e.COMMENT(/\(\*/,/\*\)/,{contains:["self"]}),{className:"pattern",relevance:0,begin:/([a-zA-Z$][a-zA-Z0-9$]*)?_+([a-zA-Z$][a-zA-Z0-9$]*)?/},{className:"slot",relevance:0,begin:/#[a-zA-Z$][a-zA-Z0-9$]*|#+[0-9]?/},c,l,{className:"named-character",begin:/\\\[[$a-zA-Z][$a-zA-Z0-9]+\]/},e.QUOTE_STRING_MODE,i,{className:"operator",relevance:0,begin:/[+\-*/,;.:@~=><&|_`'^?!%]+/},{className:"brace",relevance:0,begin:/[[\](){}]/}]}}},6475(e){e.exports=function(e){return{name:"SubUnit",case_insensitive:!0,contains:[{className:"string",begin:"\\[\n(multipart)?",end:"\\]\n"},{className:"string",begin:"\\d{4}-\\d{2}-\\d{2}(\\s+)\\d{2}:\\d{2}:\\d{2}.\\d+Z"},{className:"string",begin:"(\\+|-)\\d+"},{className:"keyword",relevance:10,variants:[{begin:"^(test|testing|success|successful|failure|error|skip|xfail|uxsuccess)(:?)\\s+(test)?"},{begin:"^progress(:?)(\\s+)?(pop|push)?"},{begin:"^tags:"},{begin:"^time:"}]}]}}},6545(e){e.exports=function(e){return{name:"BASIC",case_insensitive:!0,illegal:"^.",keywords:{$pattern:"[a-zA-Z][a-zA-Z0-9_$%!#]*",keyword:["ABS","ASC","AND","ATN","AUTO|0","BEEP","BLOAD|10","BSAVE|10","CALL","CALLS","CDBL","CHAIN","CHDIR","CHR$|10","CINT","CIRCLE","CLEAR","CLOSE","CLS","COLOR","COM","COMMON","CONT","COS","CSNG","CSRLIN","CVD","CVI","CVS","DATA","DATE$","DEFDBL","DEFINT","DEFSNG","DEFSTR","DEF|0","SEG","USR","DELETE","DIM","DRAW","EDIT","END","ENVIRON","ENVIRON$","EOF","EQV","ERASE","ERDEV","ERDEV$","ERL","ERR","ERROR","EXP","FIELD","FILES","FIX","FOR|0","FRE","GET","GOSUB|10","GOTO","HEX$","IF","THEN","ELSE|0","INKEY$","INP","INPUT","INPUT#","INPUT$","INSTR","IMP","INT","IOCTL","IOCTL$","KEY","ON","OFF","LIST","KILL","LEFT$","LEN","LET","LINE","LLIST","LOAD","LOC","LOCATE","LOF","LOG","LPRINT","USING","LSET","MERGE","MID$","MKDIR","MKD$","MKI$","MKS$","MOD","NAME","NEW","NEXT","NOISE","NOT","OCT$","ON","OR","PEN","PLAY","STRIG","OPEN","OPTION","BASE","OUT","PAINT","PALETTE","PCOPY","PEEK","PMAP","POINT","POKE","POS","PRINT","PRINT]","PSET","PRESET","PUT","RANDOMIZE","READ","REM","RENUM","RESET|0","RESTORE","RESUME","RETURN|0","RIGHT$","RMDIR","RND","RSET","RUN","SAVE","SCREEN","SGN","SHELL","SIN","SOUND","SPACE$","SPC","SQR","STEP","STICK","STOP","STR$","STRING$","SWAP","SYSTEM","TAB","TAN","TIME$","TIMER","TROFF","TRON","TO","USR","VAL","VARPTR","VARPTR$","VIEW","WAIT","WHILE","WEND","WIDTH","WINDOW","WRITE","XOR"]},contains:[{scope:"string",begin:/"/,end:/"|$/,contains:[e.BACKSLASH_ESCAPE]},e.COMMENT("REM","$",{relevance:10}),e.COMMENT("'","$",{relevance:0}),{className:"symbol",begin:"^[0-9]+ ",relevance:10},{className:"number",begin:"\\b\\d+(\\.\\d+)?([edED]\\d+)?[#!]?",relevance:0},{className:"number",begin:"(&[hH][0-9a-fA-F]{1,4})"},{className:"number",begin:"(&[oO][0-7]{1,6})"}]}}},6570(e){e.exports=function(e){const t=e.regex,n=e.COMMENT("//","$",{contains:[{begin:/\\\n/}]}),a="decltype\\(auto\\)",r="[a-zA-Z_]\\w*::",i="(?!struct)("+a+"|"+t.optional(r)+"[a-zA-Z_]\\w*"+t.optional("<[^<>]+>")+")",o={className:"type",begin:"\\b[a-z\\d_]*_t\\b"},s={className:"string",variants:[{begin:'(u8?|U|L)?"',end:'"',illegal:"\\n",contains:[e.BACKSLASH_ESCAPE]},{begin:"(u8?|U|L)?'(\\\\(x[0-9A-Fa-f]{2}|u[0-9A-Fa-f]{4,8}|[0-7]{3}|\\S)|.)",end:"'",illegal:"."},e.END_SAME_AS_BEGIN({begin:/(?:u8?|U|L)?R"([^()\\ ]{0,16})\(/,end:/\)([^()\\ ]{0,16})"/})]},l={className:"number",variants:[{begin:"[+-]?(?:(?:[0-9](?:'?[0-9])*\\.(?:[0-9](?:'?[0-9])*)?|\\.[0-9](?:'?[0-9])*)(?:[Ee][+-]?[0-9](?:'?[0-9])*)?|[0-9](?:'?[0-9])*[Ee][+-]?[0-9](?:'?[0-9])*|0[Xx](?:[0-9A-Fa-f](?:'?[0-9A-Fa-f])*(?:\\.(?:[0-9A-Fa-f](?:'?[0-9A-Fa-f])*)?)?|\\.[0-9A-Fa-f](?:'?[0-9A-Fa-f])*)[Pp][+-]?[0-9](?:'?[0-9])*)(?:[Ff](?:16|32|64|128)?|(BF|bf)16|[Ll]|)"},{begin:"[+-]?\\b(?:0[Bb][01](?:'?[01])*|0[Xx][0-9A-Fa-f](?:'?[0-9A-Fa-f])*|0(?:'?[0-7])*|[1-9](?:'?[0-9])*)(?:[Uu](?:LL?|ll?)|[Uu][Zz]?|(?:LL?|ll?)[Uu]?|[Zz][Uu]|)"}],relevance:0},c={className:"meta",begin:/#\s*[a-z]+\b/,end:/$/,keywords:{keyword:"if else elif endif define undef warning error line pragma _Pragma ifdef ifndef include"},contains:[{begin:/\\\n/,relevance:0},e.inherit(s,{className:"string"}),{className:"string",begin:/<.*?>/},n,e.C_BLOCK_COMMENT_MODE]},_={className:"title",begin:t.optional(r)+e.IDENT_RE,relevance:0},d=t.optional(r)+e.IDENT_RE+"\\s*\\(",p={type:["bool","char","char16_t","char32_t","char8_t","double","float","int","long","short","void","wchar_t","unsigned","signed","const","static"],keyword:["alignas","alignof","and","and_eq","asm","atomic_cancel","atomic_commit","atomic_noexcept","auto","bitand","bitor","break","case","catch","class","co_await","co_return","co_yield","compl","concept","const_cast|10","consteval","constexpr","constinit","continue","decltype","default","delete","do","dynamic_cast|10","else","enum","explicit","export","extern","false","final","for","friend","goto","if","import","inline","module","mutable","namespace","new","noexcept","not","not_eq","nullptr","operator","or","or_eq","override","private","protected","public","reflexpr","register","reinterpret_cast|10","requires","return","sizeof","static_assert","static_cast|10","struct","switch","synchronized","template","this","thread_local","throw","transaction_safe","transaction_safe_dynamic","true","try","typedef","typeid","typename","union","using","virtual","volatile","while","xor","xor_eq"],literal:["NULL","false","nullopt","nullptr","true"],built_in:["_Pragma"],_type_hints:["any","auto_ptr","barrier","binary_semaphore","bitset","complex","condition_variable","condition_variable_any","counting_semaphore","deque","false_type","flat_map","flat_set","future","imaginary","initializer_list","istringstream","jthread","latch","lock_guard","multimap","multiset","mutex","optional","ostringstream","packaged_task","pair","promise","priority_queue","queue","recursive_mutex","recursive_timed_mutex","scoped_lock","set","shared_future","shared_lock","shared_mutex","shared_timed_mutex","shared_ptr","stack","string_view","stringstream","timed_mutex","thread","true_type","tuple","unique_lock","unique_ptr","unordered_map","unordered_multimap","unordered_multiset","unordered_set","variant","vector","weak_ptr","wstring","wstring_view"]},u={className:"function.dispatch",relevance:0,keywords:{_hint:["abort","abs","acos","apply","as_const","asin","atan","atan2","calloc","ceil","cerr","cin","clog","cos","cosh","cout","declval","endl","exchange","exit","exp","fabs","floor","fmod","forward","fprintf","fputs","free","frexp","fscanf","future","invoke","isalnum","isalpha","iscntrl","isdigit","isgraph","islower","isprint","ispunct","isspace","isupper","isxdigit","labs","launder","ldexp","log","log10","make_pair","make_shared","make_shared_for_overwrite","make_tuple","make_unique","malloc","memchr","memcmp","memcpy","memset","modf","move","pow","printf","putchar","puts","realloc","scanf","sin","sinh","snprintf","sprintf","sqrt","sscanf","std","stderr","stdin","stdout","strcat","strchr","strcmp","strcpy","strcspn","strlen","strncat","strncmp","strncpy","strpbrk","strrchr","strspn","strstr","swap","tan","tanh","terminate","to_underlying","tolower","toupper","vfprintf","visit","vprintf","vsprintf"]},begin:t.concat(/\b/,/(?!decltype)/,/(?!if)/,/(?!for)/,/(?!switch)/,/(?!while)/,e.IDENT_RE,t.lookahead(/(<[^<>]+>|)\s*\(/))},m=[u,c,o,n,e.C_BLOCK_COMMENT_MODE,l,s],g={variants:[{begin:/=/,end:/;/},{begin:/\(/,end:/\)/},{beginKeywords:"new throw return else",end:/;/}],keywords:p,contains:m.concat([{begin:/\(/,end:/\)/,keywords:p,contains:m.concat(["self"]),relevance:0}]),relevance:0},E={className:"function",begin:"("+i+"[\\*&\\s]+)+"+d,returnBegin:!0,end:/[{;=]/,excludeEnd:!0,keywords:p,illegal:/[^\w\s\*&:<>.]/,contains:[{begin:a,keywords:p,relevance:0},{begin:d,returnBegin:!0,contains:[_],relevance:0},{begin:/::/,relevance:0},{begin:/:/,endsWithParent:!0,contains:[s,l]},{relevance:0,match:/,/},{className:"params",begin:/\(/,end:/\)/,keywords:p,relevance:0,contains:[n,e.C_BLOCK_COMMENT_MODE,s,l,o,{begin:/\(/,end:/\)/,keywords:p,relevance:0,contains:["self",n,e.C_BLOCK_COMMENT_MODE,s,l,o]}]},o,n,e.C_BLOCK_COMMENT_MODE,c]};return{name:"C++",aliases:["cc","c++","h++","hpp","hh","hxx","cxx"],keywords:p,illegal:"",keywords:p,contains:["self",o]},{begin:e.IDENT_RE+"::",keywords:p},{match:[/\b(?:enum(?:\s+(?:class|struct))?|class|struct|union)/,/\s+/,/\w+/],className:{1:"keyword",3:"title.class"}}])}}},6571(e){e.exports=function(e){const t=e.regex,n=["div","mod","in","and","or","not","xor","asserterror","begin","case","do","downto","else","end","exit","for","local","if","of","repeat","then","to","until","while","with","var"],a=[e.C_LINE_COMMENT_MODE,e.COMMENT(/\{/,/\}/,{relevance:0}),e.COMMENT(/\(\*/,/\*\)/,{relevance:10})],r={className:"string",begin:/'/,end:/'/,contains:[{begin:/''/}]},i={className:"string",begin:/(#\d+)+/},o={match:[/procedure/,/\s+/,/[a-zA-Z_][\w@]*/,/\s*/],scope:{1:"keyword",3:"title.function"},contains:[{className:"params",begin:/\(/,end:/\)/,keywords:n,contains:[r,i,e.NUMBER_MODE]},...a]},s={match:[/OBJECT/,/\s+/,t.either("Table","Form","Report","Dataport","Codeunit","XMLport","MenuSuite","Page","Query"),/\s+/,/\d+/,/\s+(?=[^\s])/,/.*/,/$/],relevance:3,scope:{1:"keyword",3:"type",5:"number",7:"title"}};return{name:"C/AL",case_insensitive:!0,keywords:{keyword:n,literal:"false true"},illegal:/\/\*/,contains:[{match:/[\w]+(?=\=)/,scope:"attribute",relevance:0},r,i,{className:"number",begin:"\\b\\d+(\\.\\d+)?(DT|D|T)",relevance:0},{className:"string",begin:'"',end:'"'},e.NUMBER_MODE,s,o]}}},6650(e){e.exports=function(e){const t={match:[/(surface|displacement|light|volume|imager)/,/\s+/,e.IDENT_RE],scope:{1:"keyword",3:"title.class"}};return{name:"RenderMan RSL",keywords:{keyword:["while","for","if","do","return","else","break","extern","continue"],built_in:["abs","acos","ambient","area","asin","atan","atmosphere","attribute","calculatenormal","ceil","cellnoise","clamp","comp","concat","cos","degrees","depth","Deriv","diffuse","distance","Du","Dv","environment","exp","faceforward","filterstep","floor","format","fresnel","incident","length","lightsource","log","match","max","min","mod","noise","normalize","ntransform","opposite","option","phong","pnoise","pow","printf","ptlined","radians","random","reflect","refract","renderinfo","round","setcomp","setxcomp","setycomp","setzcomp","shadow","sign","sin","smoothstep","specular","specularbrdf","spline","sqrt","step","tan","texture","textureinfo","trace","transform","vtransform","xcomp","ycomp","zcomp"],type:["matrix","float","color","point","normal","vector"]},illegal:"t.length&&(n=e.length-t.length);var a=t.length;e.length0&&t[o]!=t[i];)i=r[i];t[o]==t[i]&&i++}i=0;for(var s=n;s0&&e[s]!=t[i];)i=r[i];e[s]==t[i]&&i++}return i}(e,t))},t.hasOnlyWinLineEndings=function(e){return e.includes("\r\n")&&!e.startsWith("\n")&&!e.match(/[^\r]\n/)},t.hasOnlyUnixLineEndings=function(e){return!e.includes("\r\n")&&e.includes("\n")},t.trailingWs=function(e){var t;for(t=e.length-1;t>=0&&e[t].match(/\s/);t--);return e.substring(t+1)},t.leadingWs=function(e){var t=e.match(/^\s*/);return t?t[0]:""}},6672(e){e.exports=function(e){const t={className:"keyword",begin:"\\$(f[asn]|t|vp[rtd]|children)"},n={className:"number",begin:"\\b\\d+(\\.\\d+)?(e-?\\d+)?",relevance:0},a=e.inherit(e.QUOTE_STRING_MODE,{illegal:null}),r={className:"function",beginKeywords:"module function",end:/=|\{/,contains:[{className:"params",begin:"\\(",end:"\\)",contains:["self",n,a,t,{className:"literal",begin:"false|true|PI|undef"}]},e.UNDERSCORE_TITLE_MODE]};return{name:"OpenSCAD",aliases:["scad"],keywords:{keyword:"function module include use for intersection_for if else \\%",literal:"false true PI undef",built_in:"circle square polygon text sphere cube cylinder polyhedron translate rotate scale resize mirror multmatrix color offset hull minkowski union difference intersection abs sign sin cos tan acos asin atan atan2 floor round ceil ln log pow sqrt exp rands min max concat lookup str chr search version version_num norm cross parent_module echo import import_dxf dxf_linear_extrude linear_extrude rotate_extrude surface projection render children dxf_cross dxf_dim let assign"},contains:[e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE,n,{className:"meta",keywords:{keyword:"include use"},begin:"include|use <",end:">"},a,t,{begin:"[*!#%]",relevance:0},r]}}},6673(e){e.exports=function(e){return{name:"Node REPL",contains:[{className:"meta.prompt",starts:{end:/ |$/,starts:{end:"$",subLanguage:"javascript"}},variants:[{begin:/^>(?=[ ]|$)/},{begin:/^\.\.\.(?=[ ]|$)/}]}]}}},6683(e){e.exports=function(e){const t={$pattern:e.UNDERSCORE_IDENT_RE,keyword:"abstract alias align asm assert auto body break byte case cast catch class const continue debug default delete deprecated do else enum export extern final finally for foreach foreach_reverse|10 goto if immutable import in inout int interface invariant is lazy macro mixin module new nothrow out override package pragma private protected public pure ref return scope shared static struct super switch synchronized template this throw try typedef typeid typeof union unittest version void volatile while with __FILE__ __LINE__ __gshared|10 __thread __traits __DATE__ __EOF__ __TIME__ __TIMESTAMP__ __VENDOR__ __VERSION__",built_in:"bool cdouble cent cfloat char creal dchar delegate double dstring float function idouble ifloat ireal long real short string ubyte ucent uint ulong ushort wchar wstring",literal:"false null true"},n="(0|[1-9][\\d_]*)",a="(0|[1-9][\\d_]*|\\d[\\d_]*|[\\d_]+?\\d)",r="([\\da-fA-F][\\da-fA-F_]*|_[\\da-fA-F][\\da-fA-F_]*)",i="([eE][+-]?"+a+")",o="("+n+"|0[bB][01_]+|0[xX]"+r+")",s="\\\\(['\"\\?\\\\abfnrtv]|u[\\dA-Fa-f]{4}|[0-7]{1,3}|x[\\dA-Fa-f]{2}|U[\\dA-Fa-f]{8})|&[a-zA-Z\\d]{2,};",l={className:"number",begin:"\\b"+o+"(L|u|U|Lu|LU|uL|UL)?",relevance:0},c={className:"number",begin:"\\b(((0[xX]("+r+"\\."+r+"|\\.?"+r+")[pP][+-]?"+a+")|("+a+"(\\.\\d*|"+i+")|\\d+\\."+a+"|\\."+n+i+"?))([fF]|L|i|[fF]i|Li)?|"+o+"(i|[fF]i|Li))",relevance:0},_={className:"string",begin:"'("+s+"|.)",end:"'",illegal:"."},d={className:"string",begin:'"',contains:[{begin:s,relevance:0}],end:'"[cwd]?'},p=e.COMMENT("\\/\\+","\\+\\/",{contains:["self"],relevance:10});return{name:"D",keywords:t,contains:[e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE,p,{className:"string",begin:'x"[\\da-fA-F\\s\\n\\r]*"[cwd]?',relevance:10},d,{className:"string",begin:'[rq]"',end:'"[cwd]?',relevance:5},{className:"string",begin:"`",end:"`[cwd]?"},{className:"string",begin:'q"\\{',end:'\\}"'},c,l,_,{className:"meta",begin:"^#!",end:"$",relevance:5},{className:"meta",begin:"#(line)",end:"$",relevance:5},{className:"keyword",begin:"@[a-zA-Z_][a-zA-Z_\\d]*"}]}}},6703(e){e.exports=function(e){const t=/([A-Za-z_][A-Za-z_0-9]*)?/,n={scope:"params",begin:/\(/,end:/\)(?=\:?)/,endsParent:!0,relevance:7,contains:[{scope:"string",begin:'"',end:'"'},{scope:"keyword",match:["true","false","in"].join("|")},{scope:"variable",match:/[A-Za-z_][A-Za-z_0-9]*/},{scope:"operator",match:/\+|\-|\*|\/|\%|\=\=|\=|\!|\>|\<|\&\&|\|\|/}]},a={match:[t,/(?=\()/],scope:{1:"keyword"},contains:[n]};return n.contains.unshift(a),{name:"Leaf",contains:[{match:[/#+/,t,/(?=\()/],scope:{1:"punctuation",2:"keyword"},starts:{contains:[{match:/\:/,scope:"punctuation"}]},contains:[n]},{match:[/#+/,t,/:?/],scope:{1:"punctuation",2:"keyword",3:"punctuation"}}]}}},6797(e){e.exports=function(e){return{name:"Backus–Naur Form",contains:[{className:"attribute",begin://},{begin:/::=/,end:/$/,contains:[{begin://},e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE,e.APOS_STRING_MODE,e.QUOTE_STRING_MODE]}]}}},6801(e,t,n){"use strict";Object.defineProperty(t,"__esModule",{value:!0}),t.canonicalize=t.convertChangesToXML=t.convertChangesToDMP=t.reversePatch=t.parsePatch=t.applyPatches=t.applyPatch=t.OMIT_HEADERS=t.FILE_HEADERS_ONLY=t.INCLUDE_HEADERS=t.formatPatch=t.createPatch=t.createTwoFilesPatch=t.structuredPatch=t.arrayDiff=t.diffArrays=t.jsonDiff=t.diffJson=t.cssDiff=t.diffCss=t.sentenceDiff=t.diffSentences=t.diffTrimmedLines=t.lineDiff=t.diffLines=t.wordsWithSpaceDiff=t.diffWordsWithSpace=t.wordDiff=t.diffWords=t.characterDiff=t.diffChars=t.Diff=void 0;var a=n(2188);t.Diff=a.default;var r=n(3622);Object.defineProperty(t,"diffChars",{enumerable:!0,get:function(){return r.diffChars}}),Object.defineProperty(t,"characterDiff",{enumerable:!0,get:function(){return r.characterDiff}});var i=n(9025);Object.defineProperty(t,"diffWords",{enumerable:!0,get:function(){return i.diffWords}}),Object.defineProperty(t,"diffWordsWithSpace",{enumerable:!0,get:function(){return i.diffWordsWithSpace}}),Object.defineProperty(t,"wordDiff",{enumerable:!0,get:function(){return i.wordDiff}}),Object.defineProperty(t,"wordsWithSpaceDiff",{enumerable:!0,get:function(){return i.wordsWithSpaceDiff}});var o=n(9839);Object.defineProperty(t,"diffLines",{enumerable:!0,get:function(){return o.diffLines}}),Object.defineProperty(t,"diffTrimmedLines",{enumerable:!0,get:function(){return o.diffTrimmedLines}}),Object.defineProperty(t,"lineDiff",{enumerable:!0,get:function(){return o.lineDiff}});var s=n(4484);Object.defineProperty(t,"diffSentences",{enumerable:!0,get:function(){return s.diffSentences}}),Object.defineProperty(t,"sentenceDiff",{enumerable:!0,get:function(){return s.sentenceDiff}});var l=n(3042);Object.defineProperty(t,"diffCss",{enumerable:!0,get:function(){return l.diffCss}}),Object.defineProperty(t,"cssDiff",{enumerable:!0,get:function(){return l.cssDiff}});var c=n(8843);Object.defineProperty(t,"diffJson",{enumerable:!0,get:function(){return c.diffJson}}),Object.defineProperty(t,"canonicalize",{enumerable:!0,get:function(){return c.canonicalize}}),Object.defineProperty(t,"jsonDiff",{enumerable:!0,get:function(){return c.jsonDiff}});var _=n(4968);Object.defineProperty(t,"diffArrays",{enumerable:!0,get:function(){return _.diffArrays}}),Object.defineProperty(t,"arrayDiff",{enumerable:!0,get:function(){return _.arrayDiff}});var d=n(7904);Object.defineProperty(t,"applyPatch",{enumerable:!0,get:function(){return d.applyPatch}}),Object.defineProperty(t,"applyPatches",{enumerable:!0,get:function(){return d.applyPatches}});var p=n(9363);Object.defineProperty(t,"parsePatch",{enumerable:!0,get:function(){return p.parsePatch}});var u=n(3406);Object.defineProperty(t,"reversePatch",{enumerable:!0,get:function(){return u.reversePatch}});var m=n(6822);Object.defineProperty(t,"structuredPatch",{enumerable:!0,get:function(){return m.structuredPatch}}),Object.defineProperty(t,"createTwoFilesPatch",{enumerable:!0,get:function(){return m.createTwoFilesPatch}}),Object.defineProperty(t,"createPatch",{enumerable:!0,get:function(){return m.createPatch}}),Object.defineProperty(t,"formatPatch",{enumerable:!0,get:function(){return m.formatPatch}}),Object.defineProperty(t,"INCLUDE_HEADERS",{enumerable:!0,get:function(){return m.INCLUDE_HEADERS}}),Object.defineProperty(t,"FILE_HEADERS_ONLY",{enumerable:!0,get:function(){return m.FILE_HEADERS_ONLY}}),Object.defineProperty(t,"OMIT_HEADERS",{enumerable:!0,get:function(){return m.OMIT_HEADERS}});var g=n(2868);Object.defineProperty(t,"convertChangesToDMP",{enumerable:!0,get:function(){return g.convertChangesToDMP}});var E=n(9660);Object.defineProperty(t,"convertChangesToXML",{enumerable:!0,get:function(){return E.convertChangesToXML}})},6813(e){e.exports=function(e){return{name:"CSP",case_insensitive:!1,keywords:{$pattern:"[a-zA-Z][a-zA-Z0-9_-]*",keyword:["base-uri","child-src","connect-src","default-src","font-src","form-action","frame-ancestors","frame-src","img-src","manifest-src","media-src","object-src","plugin-types","report-uri","sandbox","script-src","style-src","trusted-types","unsafe-hashes","worker-src"]},contains:[{className:"string",begin:"'",end:"'"},{className:"attribute",begin:"^Content",end:":",excludeEnd:!0}]}}},6822(e,t,n){"use strict";var a=this&&this.__assign||function(){return a=Object.assign||function(e){for(var t,n=1,a=arguments.length;n0?D(g.lines.slice(-_)):[]).length,i-=l.length)}for(var E=0,S=m;E1&&!n.includeFileHeaders)throw new Error("Cannot omit file headers on a multi-file patch. (The result would be unparseable; how would a tool trying to apply the patch know which changes are to which file?)");return e.map((function(e){return o(e,n)})).join("\n")}var a=[];n.includeIndex&&e.oldFileName==e.newFileName&&a.push("Index: "+e.oldFileName),n.includeUnderline&&a.push("==================================================================="),n.includeFileHeaders&&(a.push("--- "+e.oldFileName+(void 0===e.oldHeader?"":"\t"+e.oldHeader)),a.push("+++ "+e.newFileName+(void 0===e.newHeader?"":"\t"+e.newHeader)));for(var r=0;r"}]},e.APOS_STRING_MODE,e.QUOTE_STRING_MODE,e.BINARY_NUMBER_MODE,e.C_NUMBER_MODE,a].concat(n),illegal:";$|^\\[|^=|&|\\{"}}},6876(e){e.exports=function(e){const t=e.COMMENT("--","$"),n="\\$([a-zA-Z_]?|[a-zA-Z_][a-zA-Z_0-9]*)\\$",a="BIGINT INT8 BIGSERIAL SERIAL8 BIT VARYING VARBIT BOOLEAN BOOL BOX BYTEA CHARACTER CHAR VARCHAR CIDR CIRCLE DATE DOUBLE PRECISION FLOAT8 FLOAT INET INTEGER INT INT4 INTERVAL JSON JSONB LINE LSEG|10 MACADDR MACADDR8 MONEY NUMERIC DEC DECIMAL PATH POINT POLYGON REAL FLOAT4 SMALLINT INT2 SMALLSERIAL|10 SERIAL2|10 SERIAL|10 SERIAL4|10 TEXT TIME ZONE TIMETZ|10 TIMESTAMP TIMESTAMPTZ|10 TSQUERY|10 TSVECTOR|10 TXID_SNAPSHOT|10 UUID XML NATIONAL NCHAR INT4RANGE|10 INT8RANGE|10 NUMRANGE|10 TSRANGE|10 TSTZRANGE|10 DATERANGE|10 ANYELEMENT ANYARRAY ANYNONARRAY ANYENUM ANYRANGE CSTRING INTERNAL RECORD PG_DDL_COMMAND VOID UNKNOWN OPAQUE REFCURSOR NAME OID REGPROC|10 REGPROCEDURE|10 REGOPER|10 REGOPERATOR|10 REGCLASS|10 REGTYPE|10 REGROLE|10 REGNAMESPACE|10 REGCONFIG|10 REGDICTIONARY|10 ",r=a.trim().split(" ").map((function(e){return e.split("|")[0]})).join("|"),i="ARRAY_AGG AVG BIT_AND BIT_OR BOOL_AND BOOL_OR COUNT EVERY JSON_AGG JSONB_AGG JSON_OBJECT_AGG JSONB_OBJECT_AGG MAX MIN MODE STRING_AGG SUM XMLAGG CORR COVAR_POP COVAR_SAMP REGR_AVGX REGR_AVGY REGR_COUNT REGR_INTERCEPT REGR_R2 REGR_SLOPE REGR_SXX REGR_SXY REGR_SYY STDDEV STDDEV_POP STDDEV_SAMP VARIANCE VAR_POP VAR_SAMP PERCENTILE_CONT PERCENTILE_DISC ROW_NUMBER RANK DENSE_RANK PERCENT_RANK CUME_DIST NTILE LAG LEAD FIRST_VALUE LAST_VALUE NTH_VALUE NUM_NONNULLS NUM_NULLS ABS CBRT CEIL CEILING DEGREES DIV EXP FLOOR LN LOG MOD PI POWER RADIANS ROUND SCALE SIGN SQRT TRUNC WIDTH_BUCKET RANDOM SETSEED ACOS ACOSD ASIN ASIND ATAN ATAND ATAN2 ATAN2D COS COSD COT COTD SIN SIND TAN TAND BIT_LENGTH CHAR_LENGTH CHARACTER_LENGTH LOWER OCTET_LENGTH OVERLAY POSITION SUBSTRING TREAT TRIM UPPER ASCII BTRIM CHR CONCAT CONCAT_WS CONVERT CONVERT_FROM CONVERT_TO DECODE ENCODE INITCAP LEFT LENGTH LPAD LTRIM MD5 PARSE_IDENT PG_CLIENT_ENCODING QUOTE_IDENT|10 QUOTE_LITERAL|10 QUOTE_NULLABLE|10 REGEXP_MATCH REGEXP_MATCHES REGEXP_REPLACE REGEXP_SPLIT_TO_ARRAY REGEXP_SPLIT_TO_TABLE REPEAT REPLACE REVERSE RIGHT RPAD RTRIM SPLIT_PART STRPOS SUBSTR TO_ASCII TO_HEX TRANSLATE OCTET_LENGTH GET_BIT GET_BYTE SET_BIT SET_BYTE TO_CHAR TO_DATE TO_NUMBER TO_TIMESTAMP AGE CLOCK_TIMESTAMP|10 DATE_PART DATE_TRUNC ISFINITE JUSTIFY_DAYS JUSTIFY_HOURS JUSTIFY_INTERVAL MAKE_DATE MAKE_INTERVAL|10 MAKE_TIME MAKE_TIMESTAMP|10 MAKE_TIMESTAMPTZ|10 NOW STATEMENT_TIMESTAMP|10 TIMEOFDAY TRANSACTION_TIMESTAMP|10 ENUM_FIRST ENUM_LAST ENUM_RANGE AREA CENTER DIAMETER HEIGHT ISCLOSED ISOPEN NPOINTS PCLOSE POPEN RADIUS WIDTH BOX BOUND_BOX CIRCLE LINE LSEG PATH POLYGON ABBREV BROADCAST HOST HOSTMASK MASKLEN NETMASK NETWORK SET_MASKLEN TEXT INET_SAME_FAMILY INET_MERGE MACADDR8_SET7BIT ARRAY_TO_TSVECTOR GET_CURRENT_TS_CONFIG NUMNODE PLAINTO_TSQUERY PHRASETO_TSQUERY WEBSEARCH_TO_TSQUERY QUERYTREE SETWEIGHT STRIP TO_TSQUERY TO_TSVECTOR JSON_TO_TSVECTOR JSONB_TO_TSVECTOR TS_DELETE TS_FILTER TS_HEADLINE TS_RANK TS_RANK_CD TS_REWRITE TSQUERY_PHRASE TSVECTOR_TO_ARRAY TSVECTOR_UPDATE_TRIGGER TSVECTOR_UPDATE_TRIGGER_COLUMN XMLCOMMENT XMLCONCAT XMLELEMENT XMLFOREST XMLPI XMLROOT XMLEXISTS XML_IS_WELL_FORMED XML_IS_WELL_FORMED_DOCUMENT XML_IS_WELL_FORMED_CONTENT XPATH XPATH_EXISTS XMLTABLE XMLNAMESPACES TABLE_TO_XML TABLE_TO_XMLSCHEMA TABLE_TO_XML_AND_XMLSCHEMA QUERY_TO_XML QUERY_TO_XMLSCHEMA QUERY_TO_XML_AND_XMLSCHEMA CURSOR_TO_XML CURSOR_TO_XMLSCHEMA SCHEMA_TO_XML SCHEMA_TO_XMLSCHEMA SCHEMA_TO_XML_AND_XMLSCHEMA DATABASE_TO_XML DATABASE_TO_XMLSCHEMA DATABASE_TO_XML_AND_XMLSCHEMA XMLATTRIBUTES TO_JSON TO_JSONB ARRAY_TO_JSON ROW_TO_JSON JSON_BUILD_ARRAY JSONB_BUILD_ARRAY JSON_BUILD_OBJECT JSONB_BUILD_OBJECT JSON_OBJECT JSONB_OBJECT JSON_ARRAY_LENGTH JSONB_ARRAY_LENGTH JSON_EACH JSONB_EACH JSON_EACH_TEXT JSONB_EACH_TEXT JSON_EXTRACT_PATH JSONB_EXTRACT_PATH JSON_OBJECT_KEYS JSONB_OBJECT_KEYS JSON_POPULATE_RECORD JSONB_POPULATE_RECORD JSON_POPULATE_RECORDSET JSONB_POPULATE_RECORDSET JSON_ARRAY_ELEMENTS JSONB_ARRAY_ELEMENTS JSON_ARRAY_ELEMENTS_TEXT JSONB_ARRAY_ELEMENTS_TEXT JSON_TYPEOF JSONB_TYPEOF JSON_TO_RECORD JSONB_TO_RECORD JSON_TO_RECORDSET JSONB_TO_RECORDSET JSON_STRIP_NULLS JSONB_STRIP_NULLS JSONB_SET JSONB_INSERT JSONB_PRETTY CURRVAL LASTVAL NEXTVAL SETVAL COALESCE NULLIF GREATEST LEAST ARRAY_APPEND ARRAY_CAT ARRAY_NDIMS ARRAY_DIMS ARRAY_FILL ARRAY_LENGTH ARRAY_LOWER ARRAY_POSITION ARRAY_POSITIONS ARRAY_PREPEND ARRAY_REMOVE ARRAY_REPLACE ARRAY_TO_STRING ARRAY_UPPER CARDINALITY STRING_TO_ARRAY UNNEST ISEMPTY LOWER_INC UPPER_INC LOWER_INF UPPER_INF RANGE_MERGE GENERATE_SERIES GENERATE_SUBSCRIPTS CURRENT_DATABASE CURRENT_QUERY CURRENT_SCHEMA|10 CURRENT_SCHEMAS|10 INET_CLIENT_ADDR INET_CLIENT_PORT INET_SERVER_ADDR INET_SERVER_PORT ROW_SECURITY_ACTIVE FORMAT_TYPE TO_REGCLASS TO_REGPROC TO_REGPROCEDURE TO_REGOPER TO_REGOPERATOR TO_REGTYPE TO_REGNAMESPACE TO_REGROLE COL_DESCRIPTION OBJ_DESCRIPTION SHOBJ_DESCRIPTION TXID_CURRENT TXID_CURRENT_IF_ASSIGNED TXID_CURRENT_SNAPSHOT TXID_SNAPSHOT_XIP TXID_SNAPSHOT_XMAX TXID_SNAPSHOT_XMIN TXID_VISIBLE_IN_SNAPSHOT TXID_STATUS CURRENT_SETTING SET_CONFIG BRIN_SUMMARIZE_NEW_VALUES BRIN_SUMMARIZE_RANGE BRIN_DESUMMARIZE_RANGE GIN_CLEAN_PENDING_LIST SUPPRESS_REDUNDANT_UPDATES_TRIGGER LO_FROM_BYTEA LO_PUT LO_GET LO_CREAT LO_CREATE LO_UNLINK LO_IMPORT LO_EXPORT LOREAD LOWRITE GROUPING CAST ".trim().split(" ").map((function(e){return e.split("|")[0]})).join("|");return{name:"PostgreSQL",aliases:["postgres","postgresql"],supersetOf:"sql",case_insensitive:!0,keywords:{keyword:"ABORT ALTER ANALYZE BEGIN CALL CHECKPOINT|10 CLOSE CLUSTER COMMENT COMMIT COPY CREATE DEALLOCATE DECLARE DELETE DISCARD DO DROP END EXECUTE EXPLAIN FETCH GRANT IMPORT INSERT LISTEN LOAD LOCK MOVE NOTIFY PREPARE REASSIGN|10 REFRESH REINDEX RELEASE RESET REVOKE ROLLBACK SAVEPOINT SECURITY SELECT SET SHOW START TRUNCATE UNLISTEN|10 UPDATE VACUUM|10 VALUES AGGREGATE COLLATION CONVERSION|10 DATABASE DEFAULT PRIVILEGES DOMAIN TRIGGER EXTENSION FOREIGN WRAPPER|10 TABLE FUNCTION GROUP LANGUAGE LARGE OBJECT MATERIALIZED VIEW OPERATOR CLASS FAMILY POLICY PUBLICATION|10 ROLE RULE SCHEMA SEQUENCE SERVER STATISTICS SUBSCRIPTION SYSTEM TABLESPACE CONFIGURATION DICTIONARY PARSER TEMPLATE TYPE USER MAPPING PREPARED ACCESS METHOD CAST AS TRANSFORM TRANSACTION OWNED TO INTO SESSION AUTHORIZATION INDEX PROCEDURE ASSERTION ALL ANALYSE AND ANY ARRAY ASC ASYMMETRIC|10 BOTH CASE CHECK COLLATE COLUMN CONCURRENTLY|10 CONSTRAINT CROSS DEFERRABLE RANGE DESC DISTINCT ELSE EXCEPT FOR FREEZE|10 FROM FULL HAVING ILIKE IN INITIALLY INNER INTERSECT IS ISNULL JOIN LATERAL LEADING LIKE LIMIT NATURAL NOT NOTNULL NULL OFFSET ON ONLY OR ORDER OUTER OVERLAPS PLACING PRIMARY REFERENCES RETURNING SIMILAR SOME SYMMETRIC TABLESAMPLE THEN TRAILING UNION UNIQUE USING VARIADIC|10 VERBOSE WHEN WHERE WINDOW WITH BY RETURNS INOUT OUT SETOF|10 IF STRICT CURRENT CONTINUE OWNER LOCATION OVER PARTITION WITHIN BETWEEN ESCAPE EXTERNAL INVOKER DEFINER WORK RENAME VERSION CONNECTION CONNECT TABLES TEMP TEMPORARY FUNCTIONS SEQUENCES TYPES SCHEMAS OPTION CASCADE RESTRICT ADD ADMIN EXISTS VALID VALIDATE ENABLE DISABLE REPLICA|10 ALWAYS PASSING COLUMNS PATH REF VALUE OVERRIDING IMMUTABLE STABLE VOLATILE BEFORE AFTER EACH ROW PROCEDURAL ROUTINE NO HANDLER VALIDATOR OPTIONS STORAGE OIDS|10 WITHOUT INHERIT DEPENDS CALLED INPUT LEAKPROOF|10 COST ROWS NOWAIT SEARCH UNTIL ENCRYPTED|10 PASSWORD CONFLICT|10 INSTEAD INHERITS CHARACTERISTICS WRITE CURSOR ALSO STATEMENT SHARE EXCLUSIVE INLINE ISOLATION REPEATABLE READ COMMITTED SERIALIZABLE UNCOMMITTED LOCAL GLOBAL SQL PROCEDURES RECURSIVE SNAPSHOT ROLLUP CUBE TRUSTED|10 INCLUDE FOLLOWING PRECEDING UNBOUNDED RANGE GROUPS UNENCRYPTED|10 SYSID FORMAT DELIMITER HEADER QUOTE ENCODING FILTER OFF FORCE_QUOTE FORCE_NOT_NULL FORCE_NULL COSTS BUFFERS TIMING SUMMARY DISABLE_PAGE_SKIPPING RESTART CYCLE GENERATED IDENTITY DEFERRED IMMEDIATE LEVEL LOGGED UNLOGGED OF NOTHING NONE EXCLUDE ATTRIBUTE USAGE ROUTINES TRUE FALSE NAN INFINITY ALIAS BEGIN CONSTANT DECLARE END EXCEPTION RETURN PERFORM|10 RAISE GET DIAGNOSTICS STACKED|10 FOREACH LOOP ELSIF EXIT WHILE REVERSE SLICE DEBUG LOG INFO NOTICE WARNING ASSERT OPEN SUPERUSER NOSUPERUSER CREATEDB NOCREATEDB CREATEROLE NOCREATEROLE INHERIT NOINHERIT LOGIN NOLOGIN REPLICATION NOREPLICATION BYPASSRLS NOBYPASSRLS ",built_in:"CURRENT_TIME CURRENT_TIMESTAMP CURRENT_USER CURRENT_CATALOG|10 CURRENT_DATE LOCALTIME LOCALTIMESTAMP CURRENT_ROLE|10 CURRENT_SCHEMA|10 SESSION_USER PUBLIC FOUND NEW OLD TG_NAME|10 TG_WHEN|10 TG_LEVEL|10 TG_OP|10 TG_RELID|10 TG_RELNAME|10 TG_TABLE_NAME|10 TG_TABLE_SCHEMA|10 TG_NARGS|10 TG_ARGV|10 TG_EVENT|10 TG_TAG|10 ROW_COUNT RESULT_OID|10 PG_CONTEXT|10 RETURNED_SQLSTATE COLUMN_NAME CONSTRAINT_NAME PG_DATATYPE_NAME|10 MESSAGE_TEXT TABLE_NAME SCHEMA_NAME PG_EXCEPTION_DETAIL|10 PG_EXCEPTION_HINT|10 PG_EXCEPTION_CONTEXT|10 SQLSTATE SQLERRM|10 SUCCESSFUL_COMPLETION WARNING DYNAMIC_RESULT_SETS_RETURNED IMPLICIT_ZERO_BIT_PADDING NULL_VALUE_ELIMINATED_IN_SET_FUNCTION PRIVILEGE_NOT_GRANTED PRIVILEGE_NOT_REVOKED STRING_DATA_RIGHT_TRUNCATION DEPRECATED_FEATURE NO_DATA NO_ADDITIONAL_DYNAMIC_RESULT_SETS_RETURNED SQL_STATEMENT_NOT_YET_COMPLETE CONNECTION_EXCEPTION CONNECTION_DOES_NOT_EXIST CONNECTION_FAILURE SQLCLIENT_UNABLE_TO_ESTABLISH_SQLCONNECTION SQLSERVER_REJECTED_ESTABLISHMENT_OF_SQLCONNECTION TRANSACTION_RESOLUTION_UNKNOWN PROTOCOL_VIOLATION TRIGGERED_ACTION_EXCEPTION FEATURE_NOT_SUPPORTED INVALID_TRANSACTION_INITIATION LOCATOR_EXCEPTION INVALID_LOCATOR_SPECIFICATION INVALID_GRANTOR INVALID_GRANT_OPERATION INVALID_ROLE_SPECIFICATION DIAGNOSTICS_EXCEPTION STACKED_DIAGNOSTICS_ACCESSED_WITHOUT_ACTIVE_HANDLER CASE_NOT_FOUND CARDINALITY_VIOLATION DATA_EXCEPTION ARRAY_SUBSCRIPT_ERROR CHARACTER_NOT_IN_REPERTOIRE DATETIME_FIELD_OVERFLOW DIVISION_BY_ZERO ERROR_IN_ASSIGNMENT ESCAPE_CHARACTER_CONFLICT INDICATOR_OVERFLOW INTERVAL_FIELD_OVERFLOW INVALID_ARGUMENT_FOR_LOGARITHM INVALID_ARGUMENT_FOR_NTILE_FUNCTION INVALID_ARGUMENT_FOR_NTH_VALUE_FUNCTION INVALID_ARGUMENT_FOR_POWER_FUNCTION INVALID_ARGUMENT_FOR_WIDTH_BUCKET_FUNCTION INVALID_CHARACTER_VALUE_FOR_CAST INVALID_DATETIME_FORMAT INVALID_ESCAPE_CHARACTER INVALID_ESCAPE_OCTET INVALID_ESCAPE_SEQUENCE NONSTANDARD_USE_OF_ESCAPE_CHARACTER INVALID_INDICATOR_PARAMETER_VALUE INVALID_PARAMETER_VALUE INVALID_REGULAR_EXPRESSION INVALID_ROW_COUNT_IN_LIMIT_CLAUSE INVALID_ROW_COUNT_IN_RESULT_OFFSET_CLAUSE INVALID_TABLESAMPLE_ARGUMENT INVALID_TABLESAMPLE_REPEAT INVALID_TIME_ZONE_DISPLACEMENT_VALUE INVALID_USE_OF_ESCAPE_CHARACTER MOST_SPECIFIC_TYPE_MISMATCH NULL_VALUE_NOT_ALLOWED NULL_VALUE_NO_INDICATOR_PARAMETER NUMERIC_VALUE_OUT_OF_RANGE SEQUENCE_GENERATOR_LIMIT_EXCEEDED STRING_DATA_LENGTH_MISMATCH STRING_DATA_RIGHT_TRUNCATION SUBSTRING_ERROR TRIM_ERROR UNTERMINATED_C_STRING ZERO_LENGTH_CHARACTER_STRING FLOATING_POINT_EXCEPTION INVALID_TEXT_REPRESENTATION INVALID_BINARY_REPRESENTATION BAD_COPY_FILE_FORMAT UNTRANSLATABLE_CHARACTER NOT_AN_XML_DOCUMENT INVALID_XML_DOCUMENT INVALID_XML_CONTENT INVALID_XML_COMMENT INVALID_XML_PROCESSING_INSTRUCTION INTEGRITY_CONSTRAINT_VIOLATION RESTRICT_VIOLATION NOT_NULL_VIOLATION FOREIGN_KEY_VIOLATION UNIQUE_VIOLATION CHECK_VIOLATION EXCLUSION_VIOLATION INVALID_CURSOR_STATE INVALID_TRANSACTION_STATE ACTIVE_SQL_TRANSACTION BRANCH_TRANSACTION_ALREADY_ACTIVE HELD_CURSOR_REQUIRES_SAME_ISOLATION_LEVEL INAPPROPRIATE_ACCESS_MODE_FOR_BRANCH_TRANSACTION INAPPROPRIATE_ISOLATION_LEVEL_FOR_BRANCH_TRANSACTION NO_ACTIVE_SQL_TRANSACTION_FOR_BRANCH_TRANSACTION READ_ONLY_SQL_TRANSACTION SCHEMA_AND_DATA_STATEMENT_MIXING_NOT_SUPPORTED NO_ACTIVE_SQL_TRANSACTION IN_FAILED_SQL_TRANSACTION IDLE_IN_TRANSACTION_SESSION_TIMEOUT INVALID_SQL_STATEMENT_NAME TRIGGERED_DATA_CHANGE_VIOLATION INVALID_AUTHORIZATION_SPECIFICATION INVALID_PASSWORD DEPENDENT_PRIVILEGE_DESCRIPTORS_STILL_EXIST DEPENDENT_OBJECTS_STILL_EXIST INVALID_TRANSACTION_TERMINATION SQL_ROUTINE_EXCEPTION FUNCTION_EXECUTED_NO_RETURN_STATEMENT MODIFYING_SQL_DATA_NOT_PERMITTED PROHIBITED_SQL_STATEMENT_ATTEMPTED READING_SQL_DATA_NOT_PERMITTED INVALID_CURSOR_NAME EXTERNAL_ROUTINE_EXCEPTION CONTAINING_SQL_NOT_PERMITTED MODIFYING_SQL_DATA_NOT_PERMITTED PROHIBITED_SQL_STATEMENT_ATTEMPTED READING_SQL_DATA_NOT_PERMITTED EXTERNAL_ROUTINE_INVOCATION_EXCEPTION INVALID_SQLSTATE_RETURNED NULL_VALUE_NOT_ALLOWED TRIGGER_PROTOCOL_VIOLATED SRF_PROTOCOL_VIOLATED EVENT_TRIGGER_PROTOCOL_VIOLATED SAVEPOINT_EXCEPTION INVALID_SAVEPOINT_SPECIFICATION INVALID_CATALOG_NAME INVALID_SCHEMA_NAME TRANSACTION_ROLLBACK TRANSACTION_INTEGRITY_CONSTRAINT_VIOLATION SERIALIZATION_FAILURE STATEMENT_COMPLETION_UNKNOWN DEADLOCK_DETECTED SYNTAX_ERROR_OR_ACCESS_RULE_VIOLATION SYNTAX_ERROR INSUFFICIENT_PRIVILEGE CANNOT_COERCE GROUPING_ERROR WINDOWING_ERROR INVALID_RECURSION INVALID_FOREIGN_KEY INVALID_NAME NAME_TOO_LONG RESERVED_NAME DATATYPE_MISMATCH INDETERMINATE_DATATYPE COLLATION_MISMATCH INDETERMINATE_COLLATION WRONG_OBJECT_TYPE GENERATED_ALWAYS UNDEFINED_COLUMN UNDEFINED_FUNCTION UNDEFINED_TABLE UNDEFINED_PARAMETER UNDEFINED_OBJECT DUPLICATE_COLUMN DUPLICATE_CURSOR DUPLICATE_DATABASE DUPLICATE_FUNCTION DUPLICATE_PREPARED_STATEMENT DUPLICATE_SCHEMA DUPLICATE_TABLE DUPLICATE_ALIAS DUPLICATE_OBJECT AMBIGUOUS_COLUMN AMBIGUOUS_FUNCTION AMBIGUOUS_PARAMETER AMBIGUOUS_ALIAS INVALID_COLUMN_REFERENCE INVALID_COLUMN_DEFINITION INVALID_CURSOR_DEFINITION INVALID_DATABASE_DEFINITION INVALID_FUNCTION_DEFINITION INVALID_PREPARED_STATEMENT_DEFINITION INVALID_SCHEMA_DEFINITION INVALID_TABLE_DEFINITION INVALID_OBJECT_DEFINITION WITH_CHECK_OPTION_VIOLATION INSUFFICIENT_RESOURCES DISK_FULL OUT_OF_MEMORY TOO_MANY_CONNECTIONS CONFIGURATION_LIMIT_EXCEEDED PROGRAM_LIMIT_EXCEEDED STATEMENT_TOO_COMPLEX TOO_MANY_COLUMNS TOO_MANY_ARGUMENTS OBJECT_NOT_IN_PREREQUISITE_STATE OBJECT_IN_USE CANT_CHANGE_RUNTIME_PARAM LOCK_NOT_AVAILABLE OPERATOR_INTERVENTION QUERY_CANCELED ADMIN_SHUTDOWN CRASH_SHUTDOWN CANNOT_CONNECT_NOW DATABASE_DROPPED SYSTEM_ERROR IO_ERROR UNDEFINED_FILE DUPLICATE_FILE SNAPSHOT_TOO_OLD CONFIG_FILE_ERROR LOCK_FILE_EXISTS FDW_ERROR FDW_COLUMN_NAME_NOT_FOUND FDW_DYNAMIC_PARAMETER_VALUE_NEEDED FDW_FUNCTION_SEQUENCE_ERROR FDW_INCONSISTENT_DESCRIPTOR_INFORMATION FDW_INVALID_ATTRIBUTE_VALUE FDW_INVALID_COLUMN_NAME FDW_INVALID_COLUMN_NUMBER FDW_INVALID_DATA_TYPE FDW_INVALID_DATA_TYPE_DESCRIPTORS FDW_INVALID_DESCRIPTOR_FIELD_IDENTIFIER FDW_INVALID_HANDLE FDW_INVALID_OPTION_INDEX FDW_INVALID_OPTION_NAME FDW_INVALID_STRING_LENGTH_OR_BUFFER_LENGTH FDW_INVALID_STRING_FORMAT FDW_INVALID_USE_OF_NULL_POINTER FDW_TOO_MANY_HANDLES FDW_OUT_OF_MEMORY FDW_NO_SCHEMAS FDW_OPTION_NAME_NOT_FOUND FDW_REPLY_HANDLE FDW_SCHEMA_NOT_FOUND FDW_TABLE_NOT_FOUND FDW_UNABLE_TO_CREATE_EXECUTION FDW_UNABLE_TO_CREATE_REPLY FDW_UNABLE_TO_ESTABLISH_CONNECTION PLPGSQL_ERROR RAISE_EXCEPTION NO_DATA_FOUND TOO_MANY_ROWS ASSERT_FAILURE INTERNAL_ERROR DATA_CORRUPTED INDEX_CORRUPTED "},illegal:/:==|\W\s*\(\*|(^|\s)\$[a-z]|\{\{|[a-z]:\s*$|\.\.\.|TO:|DO:/,contains:[{className:"keyword",variants:[{begin:/\bTEXT\s*SEARCH\b/},{begin:/\b(PRIMARY|FOREIGN|FOR(\s+NO)?)\s+KEY\b/},{begin:/\bPARALLEL\s+(UNSAFE|RESTRICTED|SAFE)\b/},{begin:/\bSTORAGE\s+(PLAIN|EXTERNAL|EXTENDED|MAIN)\b/},{begin:/\bMATCH\s+(FULL|PARTIAL|SIMPLE)\b/},{begin:/\bNULLS\s+(FIRST|LAST)\b/},{begin:/\bEVENT\s+TRIGGER\b/},{begin:/\b(MAPPING|OR)\s+REPLACE\b/},{begin:/\b(FROM|TO)\s+(PROGRAM|STDIN|STDOUT)\b/},{begin:/\b(SHARE|EXCLUSIVE)\s+MODE\b/},{begin:/\b(LEFT|RIGHT)\s+(OUTER\s+)?JOIN\b/},{begin:/\b(FETCH|MOVE)\s+(NEXT|PRIOR|FIRST|LAST|ABSOLUTE|RELATIVE|FORWARD|BACKWARD)\b/},{begin:/\bPRESERVE\s+ROWS\b/},{begin:/\bDISCARD\s+PLANS\b/},{begin:/\bREFERENCING\s+(OLD|NEW)\b/},{begin:/\bSKIP\s+LOCKED\b/},{begin:/\bGROUPING\s+SETS\b/},{begin:/\b(BINARY|INSENSITIVE|SCROLL|NO\s+SCROLL)\s+(CURSOR|FOR)\b/},{begin:/\b(WITH|WITHOUT)\s+HOLD\b/},{begin:/\bWITH\s+(CASCADED|LOCAL)\s+CHECK\s+OPTION\b/},{begin:/\bEXCLUDE\s+(TIES|NO\s+OTHERS)\b/},{begin:/\bFORMAT\s+(TEXT|XML|JSON|YAML)\b/},{begin:/\bSET\s+((SESSION|LOCAL)\s+)?NAMES\b/},{begin:/\bIS\s+(NOT\s+)?UNKNOWN\b/},{begin:/\bSECURITY\s+LABEL\b/},{begin:/\bSTANDALONE\s+(YES|NO|NO\s+VALUE)\b/},{begin:/\bWITH\s+(NO\s+)?DATA\b/},{begin:/\b(FOREIGN|SET)\s+DATA\b/},{begin:/\bSET\s+(CATALOG|CONSTRAINTS)\b/},{begin:/\b(WITH|FOR)\s+ORDINALITY\b/},{begin:/\bIS\s+(NOT\s+)?DOCUMENT\b/},{begin:/\bXML\s+OPTION\s+(DOCUMENT|CONTENT)\b/},{begin:/\b(STRIP|PRESERVE)\s+WHITESPACE\b/},{begin:/\bNO\s+(ACTION|MAXVALUE|MINVALUE)\b/},{begin:/\bPARTITION\s+BY\s+(RANGE|LIST|HASH)\b/},{begin:/\bAT\s+TIME\s+ZONE\b/},{begin:/\bGRANTED\s+BY\b/},{begin:/\bRETURN\s+(QUERY|NEXT)\b/},{begin:/\b(ATTACH|DETACH)\s+PARTITION\b/},{begin:/\bFORCE\s+ROW\s+LEVEL\s+SECURITY\b/},{begin:/\b(INCLUDING|EXCLUDING)\s+(COMMENTS|CONSTRAINTS|DEFAULTS|IDENTITY|INDEXES|STATISTICS|STORAGE|ALL)\b/},{begin:/\bAS\s+(ASSIGNMENT|IMPLICIT|PERMISSIVE|RESTRICTIVE|ENUM|RANGE)\b/}]},{begin:/\b(FORMAT|FAMILY|VERSION)\s*\(/},{begin:/\bINCLUDE\s*\(/,keywords:"INCLUDE"},{begin:/\bRANGE(?!\s*(BETWEEN|UNBOUNDED|CURRENT|[-0-9]+))/},{begin:/\b(VERSION|OWNER|TEMPLATE|TABLESPACE|CONNECTION\s+LIMIT|PROCEDURE|RESTRICT|JOIN|PARSER|COPY|START|END|COLLATION|INPUT|ANALYZE|STORAGE|LIKE|DEFAULT|DELIMITER|ENCODING|COLUMN|CONSTRAINT|TABLE|SCHEMA)\s*=/},{begin:/\b(PG_\w+?|HAS_[A-Z_]+_PRIVILEGE)\b/,relevance:10},{begin:/\bEXTRACT\s*\(/,end:/\bFROM\b/,returnEnd:!0,keywords:{type:"CENTURY DAY DECADE DOW DOY EPOCH HOUR ISODOW ISOYEAR MICROSECONDS MILLENNIUM MILLISECONDS MINUTE MONTH QUARTER SECOND TIMEZONE TIMEZONE_HOUR TIMEZONE_MINUTE WEEK YEAR"}},{begin:/\b(XMLELEMENT|XMLPI)\s*\(\s*NAME/,keywords:{keyword:"NAME"}},{begin:/\b(XMLPARSE|XMLSERIALIZE)\s*\(\s*(DOCUMENT|CONTENT)/,keywords:{keyword:"DOCUMENT CONTENT"}},{beginKeywords:"CACHE INCREMENT MAXVALUE MINVALUE",end:e.C_NUMBER_RE,returnEnd:!0,keywords:"BY CACHE INCREMENT MAXVALUE MINVALUE"},{className:"type",begin:/\b(WITH|WITHOUT)\s+TIME\s+ZONE\b/},{className:"type",begin:/\bINTERVAL\s+(YEAR|MONTH|DAY|HOUR|MINUTE|SECOND)(\s+TO\s+(MONTH|HOUR|MINUTE|SECOND))?\b/},{begin:/\bRETURNS\s+(LANGUAGE_HANDLER|TRIGGER|EVENT_TRIGGER|FDW_HANDLER|INDEX_AM_HANDLER|TSM_HANDLER)\b/,keywords:{keyword:"RETURNS",type:"LANGUAGE_HANDLER TRIGGER EVENT_TRIGGER FDW_HANDLER INDEX_AM_HANDLER TSM_HANDLER"}},{begin:"\\b("+i+")\\s*\\("},{begin:"\\.("+r+")\\b"},{begin:"\\b("+r+")\\s+PATH\\b",keywords:{keyword:"PATH",type:a.replace("PATH ","")}},{className:"type",begin:"\\b("+r+")\\b"},{className:"string",begin:"'",end:"'",contains:[{begin:"''"}]},{className:"string",begin:"(e|E|u&|U&)'",end:"'",contains:[{begin:"\\\\."}],relevance:10},e.END_SAME_AS_BEGIN({begin:n,end:n,contains:[{subLanguage:["pgsql","perl","python","tcl","r","lua","java","php","ruby","bash","scheme","xml","json"],endsWithParent:!0}]}),{begin:'"',end:'"',contains:[{begin:'""'}]},e.C_NUMBER_MODE,e.C_BLOCK_COMMENT_MODE,t,{className:"meta",variants:[{begin:"%(ROW)?TYPE",relevance:10},{begin:"\\$\\d+"},{begin:"^#\\w",end:"$"}]},{className:"symbol",begin:"<<\\s*[a-zA-Z_][a-zA-Z_0-9$]*\\s*>>",relevance:10}]}}},6903(e){e.exports=function(e){return{name:"Pony",keywords:{keyword:"actor addressof and as be break class compile_error compile_intrinsic consume continue delegate digestof do else elseif embed end error for fun if ifdef in interface is isnt lambda let match new not object or primitive recover repeat return struct then trait try type until use var where while with xor",meta:"iso val tag trn box ref",literal:"this false true"},contains:[{className:"type",begin:"\\b_?[A-Z][\\w]*",relevance:0},{className:"string",begin:'"""',end:'"""',relevance:10},{className:"string",begin:'"',end:'"',contains:[e.BACKSLASH_ESCAPE]},{className:"string",begin:"'",end:"'",contains:[e.BACKSLASH_ESCAPE],relevance:0},{begin:e.IDENT_RE+"'",relevance:0},{className:"number",begin:"(-?)(\\b0[xX][a-fA-F0-9]+|\\b0[bB][01]+|(\\b\\d+(_\\d+)?(\\.\\d*)?|\\.\\d+)([eE][-+]?\\d+)?)",relevance:0},e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE]}}},6959(e){e.exports=function(e){return{name:"SML (Standard ML)",aliases:["ml"],keywords:{$pattern:"[a-z_]\\w*!?",keyword:"abstype and andalso as case datatype do else end eqtype exception fn fun functor handle if in include infix infixr let local nonfix of op open orelse raise rec sharing sig signature struct structure then type val with withtype where while",built_in:"array bool char exn int list option order real ref string substring vector unit word",literal:"true false NONE SOME LESS EQUAL GREATER nil"},illegal:/\/\/|>>/,contains:[{className:"literal",begin:/\[(\|\|)?\]|\(\)/,relevance:0},e.COMMENT("\\(\\*","\\*\\)",{contains:["self"]}),{className:"symbol",begin:"'[A-Za-z_](?!')[\\w']*"},{className:"type",begin:"`[A-Z][\\w']*"},{className:"type",begin:"\\b[A-Z][\\w']*",relevance:0},{begin:"[a-z_]\\w*'[\\w']*"},e.inherit(e.APOS_STRING_MODE,{className:"string",relevance:0}),e.inherit(e.QUOTE_STRING_MODE,{illegal:null}),{className:"number",begin:"\\b(0[xX][a-fA-F0-9_]+[Lln]?|0[oO][0-7_]+[Lln]?|0[bB][01_]+[Lln]?|[0-9][0-9_]*([Lln]|(\\.[0-9_]*)?([eE][-+]?[0-9_]+)?)?)",relevance:0},{begin:/[-=]>/}]}}},6969(e){e.exports=function(e){return{name:"Inform 7",aliases:["i7"],case_insensitive:!0,keywords:{keyword:"thing room person man woman animal container supporter backdrop door scenery open closed locked inside gender is are say understand kind of rule"},contains:[{className:"string",begin:'"',end:'"',relevance:0,contains:[{className:"subst",begin:"\\[",end:"\\]"}]},{className:"section",begin:/^(Volume|Book|Part|Chapter|Section|Table)\b/,end:"$"},{begin:/^(Check|Carry out|Report|Instead of|To|Rule|When|Before|After)\b/,end:":",contains:[{begin:"\\(This",end:"\\)"}]},{className:"comment",begin:"\\[",end:"\\]",contains:["self"]}]}}},7120(e){e.exports=function(e){const t={keyword:["abstract","as","base","break","case","catch","class","const","continue","do","else","event","explicit","extern","finally","fixed","for","foreach","goto","if","implicit","in","interface","internal","is","lock","namespace","new","operator","out","override","params","private","protected","public","readonly","record","ref","return","scoped","sealed","sizeof","stackalloc","static","struct","switch","this","throw","try","typeof","unchecked","unsafe","using","virtual","void","volatile","while"].concat(["add","alias","and","ascending","args","async","await","by","descending","dynamic","equals","file","from","get","global","group","init","into","join","let","nameof","not","notnull","on","or","orderby","partial","record","remove","required","scoped","select","set","unmanaged","value|0","var","when","where","with","yield"]),built_in:["bool","byte","char","decimal","delegate","double","dynamic","enum","float","int","long","nint","nuint","object","sbyte","short","string","ulong","uint","ushort"],literal:["default","false","null","true"]},n=e.inherit(e.TITLE_MODE,{begin:"[a-zA-Z](\\.?\\w)*"}),a={className:"number",variants:[{begin:"\\b(0b[01']+)"},{begin:"(-?)\\b([\\d']+(\\.[\\d']*)?|\\.[\\d']+)(u|U|l|L|ul|UL|f|F|b|B)"},{begin:"(-?)(\\b0[xX][a-fA-F0-9']+|(\\b[\\d']+(\\.[\\d']*)?|\\.[\\d']+)([eE][-+]?[\\d']+)?)"}],relevance:0},r={className:"string",begin:'@"',end:'"',contains:[{begin:'""'}]},i=e.inherit(r,{illegal:/\n/}),o={className:"subst",begin:/\{/,end:/\}/,keywords:t},s=e.inherit(o,{illegal:/\n/}),l={className:"string",begin:/\$"/,end:'"',illegal:/\n/,contains:[{begin:/\{\{/},{begin:/\}\}/},e.BACKSLASH_ESCAPE,s]},c={className:"string",begin:/\$@"/,end:'"',contains:[{begin:/\{\{/},{begin:/\}\}/},{begin:'""'},o]},_=e.inherit(c,{illegal:/\n/,contains:[{begin:/\{\{/},{begin:/\}\}/},{begin:'""'},s]});o.contains=[c,l,r,e.APOS_STRING_MODE,e.QUOTE_STRING_MODE,a,e.C_BLOCK_COMMENT_MODE],s.contains=[_,l,i,e.APOS_STRING_MODE,e.QUOTE_STRING_MODE,a,e.inherit(e.C_BLOCK_COMMENT_MODE,{illegal:/\n/})];const d={variants:[{className:"string",begin:/"""("*)(?!")(.|\n)*?"""\1/,relevance:1},c,l,r,e.APOS_STRING_MODE,e.QUOTE_STRING_MODE]},p={begin:"<",end:">",contains:[{beginKeywords:"in out"},n]},u=e.IDENT_RE+"(<"+e.IDENT_RE+"(\\s*,\\s*"+e.IDENT_RE+")*>)?(\\[\\])?",m={begin:"@"+e.IDENT_RE,relevance:0};return{name:"C#",aliases:["cs","c#"],keywords:t,illegal:/::/,contains:[e.COMMENT("///","$",{returnBegin:!0,contains:[{className:"doctag",variants:[{begin:"///",relevance:0},{begin:"\x3c!--|--\x3e"},{begin:""}]}]}),e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE,{className:"meta",begin:"#",end:"$",keywords:{keyword:"if else elif endif define undef warning error line region endregion pragma checksum"}},d,a,{beginKeywords:"class interface",relevance:0,end:/[{;=]/,illegal:/[^\s:,]/,contains:[{beginKeywords:"where class"},n,p,e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE]},{beginKeywords:"namespace",relevance:0,end:/[{;=]/,illegal:/[^\s:]/,contains:[n,e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE]},{beginKeywords:"record",relevance:0,end:/[{;=]/,illegal:/[^\s:]/,contains:[n,p,e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE]},{className:"meta",begin:"^\\s*\\[(?=[\\w])",excludeBegin:!0,end:"\\]",excludeEnd:!0,contains:[{className:"string",begin:/"/,end:/"/}]},{beginKeywords:"new return throw await else",relevance:0},{className:"function",begin:"("+u+"\\s+)+"+e.IDENT_RE+"\\s*(<[^=]+>\\s*)?\\(",returnBegin:!0,end:/\s*[{;=]/,excludeEnd:!0,keywords:t,contains:[{beginKeywords:["public","private","protected","static","internal","protected","abstract","async","extern","override","unsafe","virtual","new","sealed","partial"].join(" "),relevance:0},{begin:e.IDENT_RE+"\\s*(<[^=]+>\\s*)?\\(",returnBegin:!0,contains:[e.TITLE_MODE,p],relevance:0},{match:/\(\)/},{className:"params",begin:/\(/,end:/\)/,excludeBegin:!0,excludeEnd:!0,keywords:t,relevance:0,contains:[d,a,e.C_BLOCK_COMMENT_MODE]},e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE]},m]}}},7172(e){e.exports=function(e){return{name:"LDIF",contains:[{className:"attribute",match:"^dn(?=:)",relevance:10},{className:"attribute",match:"^\\w+(?=:)"},{className:"literal",match:"^-"},e.HASH_COMMENT_MODE]}}},7240(e){e.exports=function(e){return{name:"Golo",keywords:{keyword:["println","readln","print","import","module","function","local","return","let","var","while","for","foreach","times","in","case","when","match","with","break","continue","augment","augmentation","each","find","filter","reduce","if","then","else","otherwise","try","catch","finally","raise","throw","orIfNull","DynamicObject|10","DynamicVariable","struct","Observable","map","set","vector","list","array"],literal:["true","false","null"]},contains:[e.HASH_COMMENT_MODE,e.QUOTE_STRING_MODE,e.C_NUMBER_MODE,{className:"meta",begin:"@[A-Za-z]+"}]}}},7259(e){e.exports=function(e){const t=e.regex,n={className:"variable",variants:[{begin:/\$\d+/},{begin:/\$\{\w+\}/},{begin:t.concat(/[$@]/,e.UNDERSCORE_IDENT_RE)}]},a={endsWithParent:!0,keywords:{$pattern:/[a-z_]{2,}|\/dev\/poll/,literal:["on","off","yes","no","true","false","none","blocked","debug","info","notice","warn","error","crit","select","break","last","permanent","redirect","kqueue","rtsig","epoll","poll","/dev/poll"]},relevance:0,illegal:"=>",contains:[e.HASH_COMMENT_MODE,{className:"string",contains:[e.BACKSLASH_ESCAPE,n],variants:[{begin:/"/,end:/"/},{begin:/'/,end:/'/}]},{begin:"([a-z]+):/",end:"\\s",endsWithParent:!0,excludeEnd:!0,contains:[n]},{className:"regexp",contains:[e.BACKSLASH_ESCAPE,n],variants:[{begin:"\\s\\^",end:"\\s|\\{|;",returnEnd:!0},{begin:"~\\*?\\s+",end:"\\s|\\{|;",returnEnd:!0},{begin:"\\*(\\.[a-z\\-]+)+"},{begin:"([a-z\\-]+\\.)+\\*"}]},{className:"number",begin:"\\b\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}(:\\d{1,5})?\\b"},{className:"number",begin:"\\b\\d+[kKmMgGdshdwy]?\\b",relevance:0},n]};return{name:"Nginx config",aliases:["nginxconf"],contains:[e.HASH_COMMENT_MODE,{beginKeywords:"upstream location",end:/;|\{/,contains:a.contains,keywords:{section:"upstream location"}},{className:"section",begin:t.concat(e.UNDERSCORE_IDENT_RE+t.lookahead(/\s+\{/)),relevance:0},{begin:t.lookahead(e.UNDERSCORE_IDENT_RE+"\\s"),end:";|\\{",contains:[{className:"attribute",begin:e.UNDERSCORE_IDENT_RE,starts:a}],relevance:0}],illegal:"[^\\s\\}\\{]"}}},7288(e){e.exports=function(e){return{name:"Excel formulae",aliases:["xlsx","xls"],case_insensitive:!0,keywords:{$pattern:/[a-zA-Z][\w\.]*/,built_in:["ABS","ACCRINT","ACCRINTM","ACOS","ACOSH","ACOT","ACOTH","AGGREGATE","ADDRESS","AMORDEGRC","AMORLINC","AND","ARABIC","AREAS","ARRAYTOTEXT","ASC","ASIN","ASINH","ATAN","ATAN2","ATANH","AVEDEV","AVERAGE","AVERAGEA","AVERAGEIF","AVERAGEIFS","BAHTTEXT","BASE","BESSELI","BESSELJ","BESSELK","BESSELY","BETADIST","BETA.DIST","BETAINV","BETA.INV","BIN2DEC","BIN2HEX","BIN2OCT","BINOMDIST","BINOM.DIST","BINOM.DIST.RANGE","BINOM.INV","BITAND","BITLSHIFT","BITOR","BITRSHIFT","BITXOR","BYCOL","BYROW","CALL","CEILING","CEILING.MATH","CEILING.PRECISE","CELL","CHAR","CHIDIST","CHIINV","CHITEST","CHISQ.DIST","CHISQ.DIST.RT","CHISQ.INV","CHISQ.INV.RT","CHISQ.TEST","CHOOSE","CHOOSECOLS","CHOOSEROWS","CLEAN","CODE","COLUMN","COLUMNS","COMBIN","COMBINA","COMPLEX","CONCAT","CONCATENATE","CONFIDENCE","CONFIDENCE.NORM","CONFIDENCE.T","CONVERT","CORREL","COS","COSH","COT","COTH","COUNT","COUNTA","COUNTBLANK","COUNTIF","COUNTIFS","COUPDAYBS","COUPDAYS","COUPDAYSNC","COUPNCD","COUPNUM","COUPPCD","COVAR","COVARIANCE.P","COVARIANCE.S","CRITBINOM","CSC","CSCH","CUBEKPIMEMBER","CUBEMEMBER","CUBEMEMBERPROPERTY","CUBERANKEDMEMBER","CUBESET","CUBESETCOUNT","CUBEVALUE","CUMIPMT","CUMPRINC","DATE","DATEDIF","DATEVALUE","DAVERAGE","DAY","DAYS","DAYS360","DB","DBCS","DCOUNT","DCOUNTA","DDB","DEC2BIN","DEC2HEX","DEC2OCT","DECIMAL","DEGREES","DELTA","DEVSQ","DGET","DISC","DMAX","DMIN","DOLLAR","DOLLARDE","DOLLARFR","DPRODUCT","DROP","DSTDEV","DSTDEVP","DSUM","DURATION","DVAR","DVARP","EDATE","EFFECT","ENCODEURL","EOMONTH","ERF","ERF.PRECISE","ERFC","ERFC.PRECISE","ERROR.TYPE","EUROCONVERT","EVEN","EXACT","EXP","EXPAND","EXPON.DIST","EXPONDIST","FACT","FACTDOUBLE","FALSE","F.DIST","FDIST","F.DIST.RT","FILTER","FILTERXML","FIND","FINDB","F.INV","F.INV.RT","FINV","FISHER","FISHERINV","FIXED","FLOOR","FLOOR.MATH","FLOOR.PRECISE","FORECAST","FORECAST.ETS","FORECAST.ETS.CONFINT","FORECAST.ETS.SEASONALITY","FORECAST.ETS.STAT","FORECAST.LINEAR","FORMULATEXT","FREQUENCY","F.TEST","FTEST","FV","FVSCHEDULE","GAMMA","GAMMA.DIST","GAMMADIST","GAMMA.INV","GAMMAINV","GAMMALN","GAMMALN.PRECISE","GAUSS","GCD","GEOMEAN","GESTEP","GETPIVOTDATA","GROWTH","HARMEAN","HEX2BIN","HEX2DEC","HEX2OCT","HLOOKUP","HOUR","HSTACK","HYPERLINK","HYPGEOM.DIST","HYPGEOMDIST","IF","IFERROR","IFNA","IFS","IMABS","IMAGE","IMAGINARY","IMARGUMENT","IMCONJUGATE","IMCOS","IMCOSH","IMCOT","IMCSC","IMCSCH","IMDIV","IMEXP","IMLN","IMLOG10","IMLOG2","IMPOWER","IMPRODUCT","IMREAL","IMSEC","IMSECH","IMSIN","IMSINH","IMSQRT","IMSUB","IMSUM","IMTAN","INDEX","INDIRECT","INFO","INT","INTERCEPT","INTRATE","IPMT","IRR","ISBLANK","ISERR","ISERROR","ISEVEN","ISFORMULA","ISLOGICAL","ISNA","ISNONTEXT","ISNUMBER","ISODD","ISOMITTED","ISREF","ISTEXT","ISO.CEILING","ISOWEEKNUM","ISPMT","JIS","KURT","LAMBDA","LARGE","LCM","LEFT","LEFTB","LEN","LENB","LET","LINEST","LN","LOG","LOG10","LOGEST","LOGINV","LOGNORM.DIST","LOGNORMDIST","LOGNORM.INV","LOOKUP","LOWER","MAKEARRAY","MAP","MATCH","MAX","MAXA","MAXIFS","MDETERM","MDURATION","MEDIAN","MID","MIDB","MIN","MINIFS","MINA","MINUTE","MINVERSE","MIRR","MMULT","MOD","MODE","MODE.MULT","MODE.SNGL","MONTH","MROUND","MULTINOMIAL","MUNIT","N","NA","NEGBINOM.DIST","NEGBINOMDIST","NETWORKDAYS","NETWORKDAYS.INTL","NOMINAL","NORM.DIST","NORMDIST","NORMINV","NORM.INV","NORM.S.DIST","NORMSDIST","NORM.S.INV","NORMSINV","NOT","NOW","NPER","NPV","NUMBERVALUE","OCT2BIN","OCT2DEC","OCT2HEX","ODD","ODDFPRICE","ODDFYIELD","ODDLPRICE","ODDLYIELD","OFFSET","OR","PDURATION","PEARSON","PERCENTILE.EXC","PERCENTILE.INC","PERCENTILE","PERCENTRANK.EXC","PERCENTRANK.INC","PERCENTRANK","PERMUT","PERMUTATIONA","PHI","PHONETIC","PI","PMT","POISSON.DIST","POISSON","POWER","PPMT","PRICE","PRICEDISC","PRICEMAT","PROB","PRODUCT","PROPER","PV","QUARTILE","QUARTILE.EXC","QUARTILE.INC","QUOTIENT","RADIANS","RAND","RANDARRAY","RANDBETWEEN","RANK.AVG","RANK.EQ","RANK","RATE","RECEIVED","REDUCE","REGISTER.ID","REPLACE","REPLACEB","REPT","RIGHT","RIGHTB","ROMAN","ROUND","ROUNDDOWN","ROUNDUP","ROW","ROWS","RRI","RSQ","RTD","SCAN","SEARCH","SEARCHB","SEC","SECH","SECOND","SEQUENCE","SERIESSUM","SHEET","SHEETS","SIGN","SIN","SINH","SKEW","SKEW.P","SLN","SLOPE","SMALL","SORT","SORTBY","SQRT","SQRTPI","SQL.REQUEST","STANDARDIZE","STOCKHISTORY","STDEV","STDEV.P","STDEV.S","STDEVA","STDEVP","STDEVPA","STEYX","SUBSTITUTE","SUBTOTAL","SUM","SUMIF","SUMIFS","SUMPRODUCT","SUMSQ","SUMX2MY2","SUMX2PY2","SUMXMY2","SWITCH","SYD","T","TAN","TANH","TAKE","TBILLEQ","TBILLPRICE","TBILLYIELD","T.DIST","T.DIST.2T","T.DIST.RT","TDIST","TEXT","TEXTAFTER","TEXTBEFORE","TEXTJOIN","TEXTSPLIT","TIME","TIMEVALUE","T.INV","T.INV.2T","TINV","TOCOL","TOROW","TODAY","TRANSPOSE","TREND","TRIM","TRIMMEAN","TRUE","TRUNC","T.TEST","TTEST","TYPE","UNICHAR","UNICODE","UNIQUE","UPPER","VALUE","VALUETOTEXT","VAR","VAR.P","VAR.S","VARA","VARP","VARPA","VDB","VLOOKUP","VSTACK","WEBSERVICE","WEEKDAY","WEEKNUM","WEIBULL","WEIBULL.DIST","WORKDAY","WORKDAY.INTL","WRAPCOLS","WRAPROWS","XIRR","XLOOKUP","XMATCH","XNPV","XOR","YEAR","YEARFRAC","YIELD","YIELDDISC","YIELDMAT","Z.TEST","ZTEST"]},contains:[{begin:/^=/,end:/[^=]/,returnEnd:!0,illegal:/=/,relevance:10},{className:"symbol",begin:/\b[A-Z]{1,2}\d+\b/,end:/[^\d]/,excludeEnd:!0,relevance:0},{className:"symbol",begin:/[A-Z]{0,2}\d*:[A-Z]{0,2}\d*/,relevance:0},e.BACKSLASH_ESCAPE,e.QUOTE_STRING_MODE,{className:"number",begin:e.NUMBER_RE+"(%)?",relevance:0},e.COMMENT(/\bN\(/,/\)/,{excludeBegin:!0,excludeEnd:!0,illegal:/\n/})]}}},7300(e){e.exports=function(e){const t={match:[/(message|enum|service)\s+/,e.IDENT_RE],scope:{1:"keyword",2:"title.class"}};return{name:"Protocol Buffers",aliases:["proto"],keywords:{keyword:["package","import","option","optional","required","repeated","group","oneof"],type:["double","float","int32","int64","uint32","uint64","sint32","sint64","fixed32","fixed64","sfixed32","sfixed64","bool","string","bytes"],literal:["true","false"]},contains:[e.QUOTE_STRING_MODE,e.NUMBER_MODE,e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE,t,{className:"function",beginKeywords:"rpc",end:/[{;]/,excludeEnd:!0,keywords:"rpc returns"},{begin:/^\s*[A-Z_]+(?=\s*=[^\n]+;$)/}]}}},7453(e,t){!function(e){var t=/\S/,n=/\"/g,a=/\n/g,r=/\r/g,i=/\\/g,o=/\u2028/,s=/\u2029/;function l(e){return e.trim?e.trim():e.replace(/^\s*|\s*$/g,"")}function c(e,t,n){if(t.charAt(n)!=e.charAt(0))return!1;for(var a=1,r=e.length;a":7,"=":8,_v:9,"{":10,"&":11,_t:12},e.scan=function(n,a){var r,i=n.length,o=0,s=null,_=null,d="",p=[],u=!1,m=0,g=0,E="{{",S="}}";function b(){d.length>0&&(p.push({tag:"_t",text:new String(d)}),d="")}function f(n,a){if(b(),n&&function(){for(var n=!0,a=g;a"==r.tag&&(r.indent=p[i].text.toString()),p.splice(i,1));else a||p.push({tag:"\n"});u=!1,g=p.length}function T(e,t){var n="="+S,a=e.indexOf(n,t),r=l(e.substring(e.indexOf("=",t)+1,a)).split(" ");return E=r[0],S=r[r.length-1],a+n.length-1}for(a&&(a=a.split(" "),E=a[0],S=a[1]),m=0;m0;){if(l=t.shift(),i&&"<"==i.tag&&!(l.tag in _))throw new Error("Illegal content in < super tag.");if(e.tags[l.tag]<=e.tags.$||p(l,r))a.push(l),l.nodes=d(t,l.tag,a,r);else{if("/"==l.tag){if(0===a.length)throw new Error("Closing tag without opener: /"+l.n);if(s=a.pop(),l.n!=s.n&&!u(l.n,s.n,r))throw new Error("Nesting error: "+s.n+" vs. "+l.n);return s.end=l.i,o}"\n"==l.tag&&(l.last=0==t.length||"\n"==t[0].tag)}o.push(l)}if(a.length>0)throw new Error("missing closing tag: "+a.pop().n);return o}function p(e,t){for(var n=0,a=t.length;n":b,"<":function(t,n){var a={partials:{},code:"",subs:{},inPartial:!0};e.walk(t.nodes,a);var r=n.partials[b(t,n)];r.subs=a.subs,r.partials=a.partials},$:function(t,n){var a={subs:{},code:"",partials:n.partials,prefix:t.n};e.walk(t.nodes,a),n.subs[t.n]=a.code,n.inPartial||(n.code+='t.sub("'+E(t.n)+'",c,p,i);')},"\n":function(e,t){t.code+=T('"\\n"'+(e.last?"":" + i"))},_v:function(e,t){t.code+="t.b(t.v(t."+S(e.n)+'("'+E(e.n)+'",c,p,0)));'},_t:function(e,t){t.code+=T('"'+E(e.text)+'"')},"{":f,"&":f},e.walk=function(t,n){for(var a,r=0,i=t.length;r<:,()$[\]_.{}!+%^-]+/,s={begin:/[a-z][a-z0-9_]*(\([a-z0-9_, ]*\))?[ \t]+/,excludeBegin:!0,end:"$",endsWithParent:!0,contains:[r,i,{className:"comment",begin:t.concat(o,t.anyNumberOfTimes(t.concat(/[ ]+/,o))),relevance:0}]};return{name:"GAMS",aliases:["gms"],case_insensitive:!0,keywords:n,contains:[e.COMMENT(/^\$ontext/,/^\$offtext/),{className:"meta",begin:"^\\$[a-z0-9]+",end:"$",returnBegin:!0,contains:[{className:"keyword",begin:"^\\$[a-z0-9]+"}]},e.COMMENT("^\\*","$"),e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE,e.QUOTE_STRING_MODE,e.APOS_STRING_MODE,{beginKeywords:"set sets parameter parameters variable variables scalar scalars equation equations",end:";",contains:[e.COMMENT("^\\*","$"),e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE,e.QUOTE_STRING_MODE,e.APOS_STRING_MODE,i,s]},{beginKeywords:"table",end:";",returnBegin:!0,contains:[{beginKeywords:"table",end:"$",contains:[s]},e.COMMENT("^\\*","$"),e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE,e.QUOTE_STRING_MODE,e.APOS_STRING_MODE,e.C_NUMBER_MODE]},{className:"function",begin:/^[a-z][a-z0-9_,\-+' ()$]+\.{2}/,returnBegin:!0,contains:[{className:"title",begin:/^[a-z0-9_]+/},{className:"params",begin:/\(/,end:/\)/,excludeBegin:!0,excludeEnd:!0},a]},e.C_NUMBER_MODE,a]}}},7514(e){e.exports=function(e){return{name:"Intel x86 Assembly",case_insensitive:!0,keywords:{$pattern:"[.%]?"+e.IDENT_RE,keyword:"lock rep repe repz repne repnz xaquire xrelease bnd nobnd aaa aad aam aas adc add and arpl bb0_reset bb1_reset bound bsf bsr bswap bt btc btr bts call cbw cdq cdqe clc cld cli clts cmc cmp cmpsb cmpsd cmpsq cmpsw cmpxchg cmpxchg486 cmpxchg8b cmpxchg16b cpuid cpu_read cpu_write cqo cwd cwde daa das dec div dmint emms enter equ f2xm1 fabs fadd faddp fbld fbstp fchs fclex fcmovb fcmovbe fcmove fcmovnb fcmovnbe fcmovne fcmovnu fcmovu fcom fcomi fcomip fcomp fcompp fcos fdecstp fdisi fdiv fdivp fdivr fdivrp femms feni ffree ffreep fiadd ficom ficomp fidiv fidivr fild fimul fincstp finit fist fistp fisttp fisub fisubr fld fld1 fldcw fldenv fldl2e fldl2t fldlg2 fldln2 fldpi fldz fmul fmulp fnclex fndisi fneni fninit fnop fnsave fnstcw fnstenv fnstsw fpatan fprem fprem1 fptan frndint frstor fsave fscale fsetpm fsin fsincos fsqrt fst fstcw fstenv fstp fstsw fsub fsubp fsubr fsubrp ftst fucom fucomi fucomip fucomp fucompp fxam fxch fxtract fyl2x fyl2xp1 hlt ibts icebp idiv imul in inc incbin insb insd insw int int01 int1 int03 int3 into invd invpcid invlpg invlpga iret iretd iretq iretw jcxz jecxz jrcxz jmp jmpe lahf lar lds lea leave les lfence lfs lgdt lgs lidt lldt lmsw loadall loadall286 lodsb lodsd lodsq lodsw loop loope loopne loopnz loopz lsl lss ltr mfence monitor mov movd movq movsb movsd movsq movsw movsx movsxd movzx mul mwait neg nop not or out outsb outsd outsw packssdw packsswb packuswb paddb paddd paddsb paddsiw paddsw paddusb paddusw paddw pand pandn pause paveb pavgusb pcmpeqb pcmpeqd pcmpeqw pcmpgtb pcmpgtd pcmpgtw pdistib pf2id pfacc pfadd pfcmpeq pfcmpge pfcmpgt pfmax pfmin pfmul pfrcp pfrcpit1 pfrcpit2 pfrsqit1 pfrsqrt pfsub pfsubr pi2fd pmachriw pmaddwd pmagw pmulhriw pmulhrwa pmulhrwc pmulhw pmullw pmvgezb pmvlzb pmvnzb pmvzb pop popa popad popaw popf popfd popfq popfw por prefetch prefetchw pslld psllq psllw psrad psraw psrld psrlq psrlw psubb psubd psubsb psubsiw psubsw psubusb psubusw psubw punpckhbw punpckhdq punpckhwd punpcklbw punpckldq punpcklwd push pusha pushad pushaw pushf pushfd pushfq pushfw pxor rcl rcr rdshr rdmsr rdpmc rdtsc rdtscp ret retf retn rol ror rdm rsdc rsldt rsm rsts sahf sal salc sar sbb scasb scasd scasq scasw sfence sgdt shl shld shr shrd sidt sldt skinit smi smint smintold smsw stc std sti stosb stosd stosq stosw str sub svdc svldt svts swapgs syscall sysenter sysexit sysret test ud0 ud1 ud2b ud2 ud2a umov verr verw fwait wbinvd wrshr wrmsr xadd xbts xchg xlatb xlat xor cmove cmovz cmovne cmovnz cmova cmovnbe cmovae cmovnb cmovb cmovnae cmovbe cmovna cmovg cmovnle cmovge cmovnl cmovl cmovnge cmovle cmovng cmovc cmovnc cmovo cmovno cmovs cmovns cmovp cmovpe cmovnp cmovpo je jz jne jnz ja jnbe jae jnb jb jnae jbe jna jg jnle jge jnl jl jnge jle jng jc jnc jo jno js jns jpo jnp jpe jp sete setz setne setnz seta setnbe setae setnb setnc setb setnae setcset setbe setna setg setnle setge setnl setl setnge setle setng sets setns seto setno setpe setp setpo setnp addps addss andnps andps cmpeqps cmpeqss cmpleps cmpless cmpltps cmpltss cmpneqps cmpneqss cmpnleps cmpnless cmpnltps cmpnltss cmpordps cmpordss cmpunordps cmpunordss cmpps cmpss comiss cvtpi2ps cvtps2pi cvtsi2ss cvtss2si cvttps2pi cvttss2si divps divss ldmxcsr maxps maxss minps minss movaps movhps movlhps movlps movhlps movmskps movntps movss movups mulps mulss orps rcpps rcpss rsqrtps rsqrtss shufps sqrtps sqrtss stmxcsr subps subss ucomiss unpckhps unpcklps xorps fxrstor fxrstor64 fxsave fxsave64 xgetbv xsetbv xsave xsave64 xsaveopt xsaveopt64 xrstor xrstor64 prefetchnta prefetcht0 prefetcht1 prefetcht2 maskmovq movntq pavgb pavgw pextrw pinsrw pmaxsw pmaxub pminsw pminub pmovmskb pmulhuw psadbw pshufw pf2iw pfnacc pfpnacc pi2fw pswapd maskmovdqu clflush movntdq movnti movntpd movdqa movdqu movdq2q movq2dq paddq pmuludq pshufd pshufhw pshuflw pslldq psrldq psubq punpckhqdq punpcklqdq addpd addsd andnpd andpd cmpeqpd cmpeqsd cmplepd cmplesd cmpltpd cmpltsd cmpneqpd cmpneqsd cmpnlepd cmpnlesd cmpnltpd cmpnltsd cmpordpd cmpordsd cmpunordpd cmpunordsd cmppd comisd cvtdq2pd cvtdq2ps cvtpd2dq cvtpd2pi cvtpd2ps cvtpi2pd cvtps2dq cvtps2pd cvtsd2si cvtsd2ss cvtsi2sd cvtss2sd cvttpd2pi cvttpd2dq cvttps2dq cvttsd2si divpd divsd maxpd maxsd minpd minsd movapd movhpd movlpd movmskpd movupd mulpd mulsd orpd shufpd sqrtpd sqrtsd subpd subsd ucomisd unpckhpd unpcklpd xorpd addsubpd addsubps haddpd haddps hsubpd hsubps lddqu movddup movshdup movsldup clgi stgi vmcall vmclear vmfunc vmlaunch vmload vmmcall vmptrld vmptrst vmread vmresume vmrun vmsave vmwrite vmxoff vmxon invept invvpid pabsb pabsw pabsd palignr phaddw phaddd phaddsw phsubw phsubd phsubsw pmaddubsw pmulhrsw pshufb psignb psignw psignd extrq insertq movntsd movntss lzcnt blendpd blendps blendvpd blendvps dppd dpps extractps insertps movntdqa mpsadbw packusdw pblendvb pblendw pcmpeqq pextrb pextrd pextrq phminposuw pinsrb pinsrd pinsrq pmaxsb pmaxsd pmaxud pmaxuw pminsb pminsd pminud pminuw pmovsxbw pmovsxbd pmovsxbq pmovsxwd pmovsxwq pmovsxdq pmovzxbw pmovzxbd pmovzxbq pmovzxwd pmovzxwq pmovzxdq pmuldq pmulld ptest roundpd roundps roundsd roundss crc32 pcmpestri pcmpestrm pcmpistri pcmpistrm pcmpgtq popcnt getsec pfrcpv pfrsqrtv movbe aesenc aesenclast aesdec aesdeclast aesimc aeskeygenassist vaesenc vaesenclast vaesdec vaesdeclast vaesimc vaeskeygenassist vaddpd vaddps vaddsd vaddss vaddsubpd vaddsubps vandpd vandps vandnpd vandnps vblendpd vblendps vblendvpd vblendvps vbroadcastss vbroadcastsd vbroadcastf128 vcmpeq_ospd vcmpeqpd vcmplt_ospd vcmpltpd vcmple_ospd vcmplepd vcmpunord_qpd vcmpunordpd vcmpneq_uqpd vcmpneqpd vcmpnlt_uspd vcmpnltpd vcmpnle_uspd vcmpnlepd vcmpord_qpd vcmpordpd vcmpeq_uqpd vcmpnge_uspd vcmpngepd vcmpngt_uspd vcmpngtpd vcmpfalse_oqpd vcmpfalsepd vcmpneq_oqpd vcmpge_ospd vcmpgepd vcmpgt_ospd vcmpgtpd vcmptrue_uqpd vcmptruepd vcmplt_oqpd vcmple_oqpd vcmpunord_spd vcmpneq_uspd vcmpnlt_uqpd vcmpnle_uqpd vcmpord_spd vcmpeq_uspd vcmpnge_uqpd vcmpngt_uqpd vcmpfalse_ospd vcmpneq_ospd vcmpge_oqpd vcmpgt_oqpd vcmptrue_uspd vcmppd vcmpeq_osps vcmpeqps vcmplt_osps vcmpltps vcmple_osps vcmpleps vcmpunord_qps vcmpunordps vcmpneq_uqps vcmpneqps vcmpnlt_usps vcmpnltps vcmpnle_usps vcmpnleps vcmpord_qps vcmpordps vcmpeq_uqps vcmpnge_usps vcmpngeps vcmpngt_usps vcmpngtps vcmpfalse_oqps vcmpfalseps vcmpneq_oqps vcmpge_osps vcmpgeps vcmpgt_osps vcmpgtps vcmptrue_uqps vcmptrueps vcmplt_oqps vcmple_oqps vcmpunord_sps vcmpneq_usps vcmpnlt_uqps vcmpnle_uqps vcmpord_sps vcmpeq_usps vcmpnge_uqps vcmpngt_uqps vcmpfalse_osps vcmpneq_osps vcmpge_oqps vcmpgt_oqps vcmptrue_usps vcmpps vcmpeq_ossd vcmpeqsd vcmplt_ossd vcmpltsd vcmple_ossd vcmplesd vcmpunord_qsd vcmpunordsd vcmpneq_uqsd vcmpneqsd vcmpnlt_ussd vcmpnltsd vcmpnle_ussd vcmpnlesd vcmpord_qsd vcmpordsd vcmpeq_uqsd vcmpnge_ussd vcmpngesd vcmpngt_ussd vcmpngtsd vcmpfalse_oqsd vcmpfalsesd vcmpneq_oqsd vcmpge_ossd vcmpgesd vcmpgt_ossd vcmpgtsd vcmptrue_uqsd vcmptruesd vcmplt_oqsd vcmple_oqsd vcmpunord_ssd vcmpneq_ussd vcmpnlt_uqsd vcmpnle_uqsd vcmpord_ssd vcmpeq_ussd vcmpnge_uqsd vcmpngt_uqsd vcmpfalse_ossd vcmpneq_ossd vcmpge_oqsd vcmpgt_oqsd vcmptrue_ussd vcmpsd vcmpeq_osss vcmpeqss vcmplt_osss vcmpltss vcmple_osss vcmpless vcmpunord_qss vcmpunordss vcmpneq_uqss vcmpneqss vcmpnlt_usss vcmpnltss vcmpnle_usss vcmpnless vcmpord_qss vcmpordss vcmpeq_uqss vcmpnge_usss vcmpngess vcmpngt_usss vcmpngtss vcmpfalse_oqss vcmpfalsess vcmpneq_oqss vcmpge_osss vcmpgess vcmpgt_osss vcmpgtss vcmptrue_uqss vcmptruess vcmplt_oqss vcmple_oqss vcmpunord_sss vcmpneq_usss vcmpnlt_uqss vcmpnle_uqss vcmpord_sss vcmpeq_usss vcmpnge_uqss vcmpngt_uqss vcmpfalse_osss vcmpneq_osss vcmpge_oqss vcmpgt_oqss vcmptrue_usss vcmpss vcomisd vcomiss vcvtdq2pd vcvtdq2ps vcvtpd2dq vcvtpd2ps vcvtps2dq vcvtps2pd vcvtsd2si vcvtsd2ss vcvtsi2sd vcvtsi2ss vcvtss2sd vcvtss2si vcvttpd2dq vcvttps2dq vcvttsd2si vcvttss2si vdivpd vdivps vdivsd vdivss vdppd vdpps vextractf128 vextractps vhaddpd vhaddps vhsubpd vhsubps vinsertf128 vinsertps vlddqu vldqqu vldmxcsr vmaskmovdqu vmaskmovps vmaskmovpd vmaxpd vmaxps vmaxsd vmaxss vminpd vminps vminsd vminss vmovapd vmovaps vmovd vmovq vmovddup vmovdqa vmovqqa vmovdqu vmovqqu vmovhlps vmovhpd vmovhps vmovlhps vmovlpd vmovlps vmovmskpd vmovmskps vmovntdq vmovntqq vmovntdqa vmovntpd vmovntps vmovsd vmovshdup vmovsldup vmovss vmovupd vmovups vmpsadbw vmulpd vmulps vmulsd vmulss vorpd vorps vpabsb vpabsw vpabsd vpacksswb vpackssdw vpackuswb vpackusdw vpaddb vpaddw vpaddd vpaddq vpaddsb vpaddsw vpaddusb vpaddusw vpalignr vpand vpandn vpavgb vpavgw vpblendvb vpblendw vpcmpestri vpcmpestrm vpcmpistri vpcmpistrm vpcmpeqb vpcmpeqw vpcmpeqd vpcmpeqq vpcmpgtb vpcmpgtw vpcmpgtd vpcmpgtq vpermilpd vpermilps vperm2f128 vpextrb vpextrw vpextrd vpextrq vphaddw vphaddd vphaddsw vphminposuw vphsubw vphsubd vphsubsw vpinsrb vpinsrw vpinsrd vpinsrq vpmaddwd vpmaddubsw vpmaxsb vpmaxsw vpmaxsd vpmaxub vpmaxuw vpmaxud vpminsb vpminsw vpminsd vpminub vpminuw vpminud vpmovmskb vpmovsxbw vpmovsxbd vpmovsxbq vpmovsxwd vpmovsxwq vpmovsxdq vpmovzxbw vpmovzxbd vpmovzxbq vpmovzxwd vpmovzxwq vpmovzxdq vpmulhuw vpmulhrsw vpmulhw vpmullw vpmulld vpmuludq vpmuldq vpor vpsadbw vpshufb vpshufd vpshufhw vpshuflw vpsignb vpsignw vpsignd vpslldq vpsrldq vpsllw vpslld vpsllq vpsraw vpsrad vpsrlw vpsrld vpsrlq vptest vpsubb vpsubw vpsubd vpsubq vpsubsb vpsubsw vpsubusb vpsubusw vpunpckhbw vpunpckhwd vpunpckhdq vpunpckhqdq vpunpcklbw vpunpcklwd vpunpckldq vpunpcklqdq vpxor vrcpps vrcpss vrsqrtps vrsqrtss vroundpd vroundps vroundsd vroundss vshufpd vshufps vsqrtpd vsqrtps vsqrtsd vsqrtss vstmxcsr vsubpd vsubps vsubsd vsubss vtestps vtestpd vucomisd vucomiss vunpckhpd vunpckhps vunpcklpd vunpcklps vxorpd vxorps vzeroall vzeroupper pclmullqlqdq pclmulhqlqdq pclmullqhqdq pclmulhqhqdq pclmulqdq vpclmullqlqdq vpclmulhqlqdq vpclmullqhqdq vpclmulhqhqdq vpclmulqdq vfmadd132ps vfmadd132pd vfmadd312ps vfmadd312pd vfmadd213ps vfmadd213pd vfmadd123ps vfmadd123pd vfmadd231ps vfmadd231pd vfmadd321ps vfmadd321pd vfmaddsub132ps vfmaddsub132pd vfmaddsub312ps vfmaddsub312pd vfmaddsub213ps vfmaddsub213pd vfmaddsub123ps vfmaddsub123pd vfmaddsub231ps vfmaddsub231pd vfmaddsub321ps vfmaddsub321pd vfmsub132ps vfmsub132pd vfmsub312ps vfmsub312pd vfmsub213ps vfmsub213pd vfmsub123ps vfmsub123pd vfmsub231ps vfmsub231pd vfmsub321ps vfmsub321pd vfmsubadd132ps vfmsubadd132pd vfmsubadd312ps vfmsubadd312pd vfmsubadd213ps vfmsubadd213pd vfmsubadd123ps vfmsubadd123pd vfmsubadd231ps vfmsubadd231pd vfmsubadd321ps vfmsubadd321pd vfnmadd132ps vfnmadd132pd vfnmadd312ps vfnmadd312pd vfnmadd213ps vfnmadd213pd vfnmadd123ps vfnmadd123pd vfnmadd231ps vfnmadd231pd vfnmadd321ps vfnmadd321pd vfnmsub132ps vfnmsub132pd vfnmsub312ps vfnmsub312pd vfnmsub213ps vfnmsub213pd vfnmsub123ps vfnmsub123pd vfnmsub231ps vfnmsub231pd vfnmsub321ps vfnmsub321pd vfmadd132ss vfmadd132sd vfmadd312ss vfmadd312sd vfmadd213ss vfmadd213sd vfmadd123ss vfmadd123sd vfmadd231ss vfmadd231sd vfmadd321ss vfmadd321sd vfmsub132ss vfmsub132sd vfmsub312ss vfmsub312sd vfmsub213ss vfmsub213sd vfmsub123ss vfmsub123sd vfmsub231ss vfmsub231sd vfmsub321ss vfmsub321sd vfnmadd132ss vfnmadd132sd vfnmadd312ss vfnmadd312sd vfnmadd213ss vfnmadd213sd vfnmadd123ss vfnmadd123sd vfnmadd231ss vfnmadd231sd vfnmadd321ss vfnmadd321sd vfnmsub132ss vfnmsub132sd vfnmsub312ss vfnmsub312sd vfnmsub213ss vfnmsub213sd vfnmsub123ss vfnmsub123sd vfnmsub231ss vfnmsub231sd vfnmsub321ss vfnmsub321sd rdfsbase rdgsbase rdrand wrfsbase wrgsbase vcvtph2ps vcvtps2ph adcx adox rdseed clac stac xstore xcryptecb xcryptcbc xcryptctr xcryptcfb xcryptofb montmul xsha1 xsha256 llwpcb slwpcb lwpval lwpins vfmaddpd vfmaddps vfmaddsd vfmaddss vfmaddsubpd vfmaddsubps vfmsubaddpd vfmsubaddps vfmsubpd vfmsubps vfmsubsd vfmsubss vfnmaddpd vfnmaddps vfnmaddsd vfnmaddss vfnmsubpd vfnmsubps vfnmsubsd vfnmsubss vfrczpd vfrczps vfrczsd vfrczss vpcmov vpcomb vpcomd vpcomq vpcomub vpcomud vpcomuq vpcomuw vpcomw vphaddbd vphaddbq vphaddbw vphadddq vphaddubd vphaddubq vphaddubw vphaddudq vphadduwd vphadduwq vphaddwd vphaddwq vphsubbw vphsubdq vphsubwd vpmacsdd vpmacsdqh vpmacsdql vpmacssdd vpmacssdqh vpmacssdql vpmacsswd vpmacssww vpmacswd vpmacsww vpmadcsswd vpmadcswd vpperm vprotb vprotd vprotq vprotw vpshab vpshad vpshaq vpshaw vpshlb vpshld vpshlq vpshlw vbroadcasti128 vpblendd vpbroadcastb vpbroadcastw vpbroadcastd vpbroadcastq vpermd vpermpd vpermps vpermq vperm2i128 vextracti128 vinserti128 vpmaskmovd vpmaskmovq vpsllvd vpsllvq vpsravd vpsrlvd vpsrlvq vgatherdpd vgatherqpd vgatherdps vgatherqps vpgatherdd vpgatherqd vpgatherdq vpgatherqq xabort xbegin xend xtest andn bextr blci blcic blsi blsic blcfill blsfill blcmsk blsmsk blsr blcs bzhi mulx pdep pext rorx sarx shlx shrx tzcnt tzmsk t1mskc valignd valignq vblendmpd vblendmps vbroadcastf32x4 vbroadcastf64x4 vbroadcasti32x4 vbroadcasti64x4 vcompresspd vcompressps vcvtpd2udq vcvtps2udq vcvtsd2usi vcvtss2usi vcvttpd2udq vcvttps2udq vcvttsd2usi vcvttss2usi vcvtudq2pd vcvtudq2ps vcvtusi2sd vcvtusi2ss vexpandpd vexpandps vextractf32x4 vextractf64x4 vextracti32x4 vextracti64x4 vfixupimmpd vfixupimmps vfixupimmsd vfixupimmss vgetexppd vgetexpps vgetexpsd vgetexpss vgetmantpd vgetmantps vgetmantsd vgetmantss vinsertf32x4 vinsertf64x4 vinserti32x4 vinserti64x4 vmovdqa32 vmovdqa64 vmovdqu32 vmovdqu64 vpabsq vpandd vpandnd vpandnq vpandq vpblendmd vpblendmq vpcmpltd vpcmpled vpcmpneqd vpcmpnltd vpcmpnled vpcmpd vpcmpltq vpcmpleq vpcmpneqq vpcmpnltq vpcmpnleq vpcmpq vpcmpequd vpcmpltud vpcmpleud vpcmpnequd vpcmpnltud vpcmpnleud vpcmpud vpcmpequq vpcmpltuq vpcmpleuq vpcmpnequq vpcmpnltuq vpcmpnleuq vpcmpuq vpcompressd vpcompressq vpermi2d vpermi2pd vpermi2ps vpermi2q vpermt2d vpermt2pd vpermt2ps vpermt2q vpexpandd vpexpandq vpmaxsq vpmaxuq vpminsq vpminuq vpmovdb vpmovdw vpmovqb vpmovqd vpmovqw vpmovsdb vpmovsdw vpmovsqb vpmovsqd vpmovsqw vpmovusdb vpmovusdw vpmovusqb vpmovusqd vpmovusqw vpord vporq vprold vprolq vprolvd vprolvq vprord vprorq vprorvd vprorvq vpscatterdd vpscatterdq vpscatterqd vpscatterqq vpsraq vpsravq vpternlogd vpternlogq vptestmd vptestmq vptestnmd vptestnmq vpxord vpxorq vrcp14pd vrcp14ps vrcp14sd vrcp14ss vrndscalepd vrndscaleps vrndscalesd vrndscaless vrsqrt14pd vrsqrt14ps vrsqrt14sd vrsqrt14ss vscalefpd vscalefps vscalefsd vscalefss vscatterdpd vscatterdps vscatterqpd vscatterqps vshuff32x4 vshuff64x2 vshufi32x4 vshufi64x2 kandnw kandw kmovw knotw kortestw korw kshiftlw kshiftrw kunpckbw kxnorw kxorw vpbroadcastmb2q vpbroadcastmw2d vpconflictd vpconflictq vplzcntd vplzcntq vexp2pd vexp2ps vrcp28pd vrcp28ps vrcp28sd vrcp28ss vrsqrt28pd vrsqrt28ps vrsqrt28sd vrsqrt28ss vgatherpf0dpd vgatherpf0dps vgatherpf0qpd vgatherpf0qps vgatherpf1dpd vgatherpf1dps vgatherpf1qpd vgatherpf1qps vscatterpf0dpd vscatterpf0dps vscatterpf0qpd vscatterpf0qps vscatterpf1dpd vscatterpf1dps vscatterpf1qpd vscatterpf1qps prefetchwt1 bndmk bndcl bndcu bndcn bndmov bndldx bndstx sha1rnds4 sha1nexte sha1msg1 sha1msg2 sha256rnds2 sha256msg1 sha256msg2 hint_nop0 hint_nop1 hint_nop2 hint_nop3 hint_nop4 hint_nop5 hint_nop6 hint_nop7 hint_nop8 hint_nop9 hint_nop10 hint_nop11 hint_nop12 hint_nop13 hint_nop14 hint_nop15 hint_nop16 hint_nop17 hint_nop18 hint_nop19 hint_nop20 hint_nop21 hint_nop22 hint_nop23 hint_nop24 hint_nop25 hint_nop26 hint_nop27 hint_nop28 hint_nop29 hint_nop30 hint_nop31 hint_nop32 hint_nop33 hint_nop34 hint_nop35 hint_nop36 hint_nop37 hint_nop38 hint_nop39 hint_nop40 hint_nop41 hint_nop42 hint_nop43 hint_nop44 hint_nop45 hint_nop46 hint_nop47 hint_nop48 hint_nop49 hint_nop50 hint_nop51 hint_nop52 hint_nop53 hint_nop54 hint_nop55 hint_nop56 hint_nop57 hint_nop58 hint_nop59 hint_nop60 hint_nop61 hint_nop62 hint_nop63",built_in:"ip eip rip al ah bl bh cl ch dl dh sil dil bpl spl r8b r9b r10b r11b r12b r13b r14b r15b ax bx cx dx si di bp sp r8w r9w r10w r11w r12w r13w r14w r15w eax ebx ecx edx esi edi ebp esp eip r8d r9d r10d r11d r12d r13d r14d r15d rax rbx rcx rdx rsi rdi rbp rsp r8 r9 r10 r11 r12 r13 r14 r15 cs ds es fs gs ss st st0 st1 st2 st3 st4 st5 st6 st7 mm0 mm1 mm2 mm3 mm4 mm5 mm6 mm7 xmm0 xmm1 xmm2 xmm3 xmm4 xmm5 xmm6 xmm7 xmm8 xmm9 xmm10 xmm11 xmm12 xmm13 xmm14 xmm15 xmm16 xmm17 xmm18 xmm19 xmm20 xmm21 xmm22 xmm23 xmm24 xmm25 xmm26 xmm27 xmm28 xmm29 xmm30 xmm31 ymm0 ymm1 ymm2 ymm3 ymm4 ymm5 ymm6 ymm7 ymm8 ymm9 ymm10 ymm11 ymm12 ymm13 ymm14 ymm15 ymm16 ymm17 ymm18 ymm19 ymm20 ymm21 ymm22 ymm23 ymm24 ymm25 ymm26 ymm27 ymm28 ymm29 ymm30 ymm31 zmm0 zmm1 zmm2 zmm3 zmm4 zmm5 zmm6 zmm7 zmm8 zmm9 zmm10 zmm11 zmm12 zmm13 zmm14 zmm15 zmm16 zmm17 zmm18 zmm19 zmm20 zmm21 zmm22 zmm23 zmm24 zmm25 zmm26 zmm27 zmm28 zmm29 zmm30 zmm31 k0 k1 k2 k3 k4 k5 k6 k7 bnd0 bnd1 bnd2 bnd3 cr0 cr1 cr2 cr3 cr4 cr8 dr0 dr1 dr2 dr3 dr8 tr3 tr4 tr5 tr6 tr7 r0 r1 r2 r3 r4 r5 r6 r7 r0b r1b r2b r3b r4b r5b r6b r7b r0w r1w r2w r3w r4w r5w r6w r7w r0d r1d r2d r3d r4d r5d r6d r7d r0h r1h r2h r3h r0l r1l r2l r3l r4l r5l r6l r7l r8l r9l r10l r11l r12l r13l r14l r15l db dw dd dq dt ddq do dy dz resb resw resd resq rest resdq reso resy resz incbin equ times byte word dword qword nosplit rel abs seg wrt strict near far a32 ptr",meta:"%define %xdefine %+ %undef %defstr %deftok %assign %strcat %strlen %substr %rotate %elif %else %endif %if %ifmacro %ifctx %ifidn %ifidni %ifid %ifnum %ifstr %iftoken %ifempty %ifenv %error %warning %fatal %rep %endrep %include %push %pop %repl %pathsearch %depend %use %arg %stacksize %local %line %comment %endcomment .nolist __FILE__ __LINE__ __SECT__ __BITS__ __OUTPUT_FORMAT__ __DATE__ __TIME__ __DATE_NUM__ __TIME_NUM__ __UTC_DATE__ __UTC_TIME__ __UTC_DATE_NUM__ __UTC_TIME_NUM__ __PASS__ struc endstruc istruc at iend align alignb sectalign daz nodaz up down zero default option assume public bits use16 use32 use64 default section segment absolute extern global common cpu float __utf16__ __utf16le__ __utf16be__ __utf32__ __utf32le__ __utf32be__ __float8__ __float16__ __float32__ __float64__ __float80m__ __float80e__ __float128l__ __float128h__ __Infinity__ __QNaN__ __SNaN__ Inf NaN QNaN SNaN float8 float16 float32 float64 float80m float80e float128l float128h __FLOAT_DAZ__ __FLOAT_ROUND__ __FLOAT__"},contains:[e.COMMENT(";","$",{relevance:0}),{className:"number",variants:[{begin:"\\b(?:([0-9][0-9_]*)?\\.[0-9_]*(?:[eE][+-]?[0-9_]+)?|(0[Xx])?[0-9][0-9_]*(\\.[0-9_]*)?(?:[pP](?:[+-]?[0-9_]+)?)?)\\b",relevance:0},{begin:"\\$[0-9][0-9A-Fa-f]*",relevance:0},{begin:"\\b(?:[0-9A-Fa-f][0-9A-Fa-f_]*[Hh]|[0-9][0-9_]*[DdTt]?|[0-7][0-7_]*[QqOo]|[0-1][0-1_]*[BbYy])\\b"},{begin:"\\b(?:0[Xx][0-9A-Fa-f_]+|0[DdTt][0-9_]+|0[QqOo][0-7_]+|0[BbYy][0-1_]+)\\b"}]},e.QUOTE_STRING_MODE,{className:"string",variants:[{begin:"'",end:"[^\\\\]'"},{begin:"`",end:"[^\\\\]`"}],relevance:0},{className:"symbol",variants:[{begin:"^\\s*[A-Za-z._?][A-Za-z0-9_$#@~.?]*(:|\\s+label)"},{begin:"^\\s*%%[A-Za-z0-9_$#@~.?]*:"}],relevance:0},{className:"subst",begin:"%[0-9]+",relevance:0},{className:"subst",begin:"%!S+",relevance:0},{className:"meta",begin:/^\s*\.[\w_-]+/}]}}},7572(e){e.exports=function(e){const t=e.regex;return{name:"Erlang REPL",keywords:{built_in:"spawn spawn_link self",keyword:"after and andalso|10 band begin bnot bor bsl bsr bxor case catch cond div end fun if let not of or orelse|10 query receive rem try when xor"},contains:[{className:"meta.prompt",begin:"^[0-9]+> ",relevance:10},e.COMMENT("%","$"),{className:"number",begin:"\\b(\\d+(_\\d+)*#[a-fA-F0-9]+(_[a-fA-F0-9]+)*|\\d+(_\\d+)*(\\.\\d+(_\\d+)*)?([eE][-+]?\\d+)?)",relevance:0},e.APOS_STRING_MODE,e.QUOTE_STRING_MODE,{begin:t.concat(/\?(::)?/,/([A-Z]\w*)/,/((::)[A-Z]\w*)*/)},{begin:"->"},{begin:"ok"},{begin:"!"},{begin:"(\\b[a-z'][a-zA-Z0-9_']*:[a-z'][a-zA-Z0-9_']*)|(\\b[a-z'][a-zA-Z0-9_']*)",relevance:0},{begin:"[A-Z][a-zA-Z0-9_']*",relevance:0}]}}},7579(e){e.exports=function(e){const t="a-zA-Z_\\-!.?+*=<>&'",n="[#]?["+t+"]["+t+"0-9/;:$#]*",a="def defonce defprotocol defstruct defmulti defmethod defn- defn defmacro deftype defrecord",r={$pattern:n,built_in:a+" cond apply if-not if-let if not not= =|0 <|0 >|0 <=|0 >=|0 ==|0 +|0 /|0 *|0 -|0 rem quot neg? pos? delay? symbol? keyword? true? false? integer? empty? coll? list? set? ifn? fn? associative? sequential? sorted? counted? reversible? number? decimal? class? distinct? isa? float? rational? reduced? ratio? odd? even? char? seq? vector? string? map? nil? contains? zero? instance? not-every? not-any? libspec? -> ->> .. . inc compare do dotimes mapcat take remove take-while drop letfn drop-last take-last drop-while while intern condp case reduced cycle split-at split-with repeat replicate iterate range merge zipmap declare line-seq sort comparator sort-by dorun doall nthnext nthrest partition eval doseq await await-for let agent atom send send-off release-pending-sends add-watch mapv filterv remove-watch agent-error restart-agent set-error-handler error-handler set-error-mode! error-mode shutdown-agents quote var fn loop recur throw try monitor-enter monitor-exit macroexpand macroexpand-1 for dosync and or when when-not when-let comp juxt partial sequence memoize constantly complement identity assert peek pop doto proxy first rest cons cast coll last butlast sigs reify second ffirst fnext nfirst nnext meta with-meta ns in-ns create-ns import refer keys select-keys vals key val rseq name namespace promise into transient persistent! conj! assoc! dissoc! pop! disj! use class type num float double short byte boolean bigint biginteger bigdec print-method print-dup throw-if printf format load compile get-in update-in pr pr-on newline flush read slurp read-line subvec with-open memfn time re-find re-groups rand-int rand mod locking assert-valid-fdecl alias resolve ref deref refset swap! reset! set-validator! compare-and-set! alter-meta! reset-meta! commute get-validator alter ref-set ref-history-count ref-min-history ref-max-history ensure sync io! new next conj set! to-array future future-call into-array aset gen-class reduce map filter find empty hash-map hash-set sorted-map sorted-map-by sorted-set sorted-set-by vec vector seq flatten reverse assoc dissoc list disj get union difference intersection extend extend-type extend-protocol int nth delay count concat chunk chunk-buffer chunk-append chunk-first chunk-rest max min dec unchecked-inc-int unchecked-inc unchecked-dec-inc unchecked-dec unchecked-negate unchecked-add-int unchecked-add unchecked-subtract-int unchecked-subtract chunk-next chunk-cons chunked-seq? prn vary-meta lazy-seq spread list* str find-keyword keyword symbol gensym force rationalize"},i={begin:n,relevance:0},o={scope:"number",relevance:0,variants:[{match:/[-+]?0[xX][0-9a-fA-F]+N?/},{match:/[-+]?0[0-7]+N?/},{match:/[-+]?[1-9][0-9]?[rR][0-9a-zA-Z]+N?/},{match:/[-+]?[0-9]+\/[0-9]+N?/},{match:/[-+]?[0-9]+((\.[0-9]*([eE][+-]?[0-9]+)?M?)|([eE][+-]?[0-9]+M?|M))/},{match:/[-+]?([1-9][0-9]*|0)N?/}]},s={scope:"character",variants:[{match:/\\o[0-3]?[0-7]{1,2}/},{match:/\\u[0-9a-fA-F]{4}/},{match:/\\(newline|space|tab|formfeed|backspace|return)/},{match:/\\\S/,relevance:0}]},l={scope:"regex",begin:/#"/,end:/"/,contains:[e.BACKSLASH_ESCAPE]},c=e.inherit(e.QUOTE_STRING_MODE,{illegal:null}),_={scope:"punctuation",match:/,/,relevance:0},d=e.COMMENT(";","$",{relevance:0}),p={className:"literal",begin:/\b(true|false|nil)\b/},u={begin:"\\[|(#::?"+n+")?\\{",end:"[\\]\\}]",relevance:0},m={className:"symbol",begin:"[:]{1,2}"+n},g={begin:"\\(",end:"\\)"},E={endsWithParent:!0,relevance:0},S={keywords:r,className:"name",begin:n,relevance:0,starts:E},b=[_,g,s,l,c,d,m,u,o,p,i],f={beginKeywords:a,keywords:{$pattern:n,keyword:a},end:'(\\[|#|\\d|"|:|\\{|\\)|\\(|$)',contains:[{className:"title",begin:n,relevance:0,excludeEnd:!0,endsParent:!0}].concat(b)};return g.contains=[f,S,E],E.contains=b,u.contains=b,{name:"Clojure",aliases:["clj","edn"],illegal:/\S/,contains:[_,g,s,l,c,d,m,u,o,p]}}},7646(e){e.exports=function(e){const t="[ \\t\\f]*",n=t+"[:=]"+t,a="[ \\t\\f]+",r="([^\\\\:= \\t\\f\\n]|\\\\.)+",i={end:"("+n+"|"+a+")",relevance:0,starts:{className:"string",end:/$/,relevance:0,contains:[{begin:"\\\\\\\\"},{begin:"\\\\\\n"}]}};return{name:".properties",disableAutodetect:!0,case_insensitive:!0,illegal:/\S/,contains:[e.COMMENT("^\\s*[!#]","$"),{returnBegin:!0,variants:[{begin:r+n},{begin:r+a}],contains:[{className:"attr",begin:r,endsParent:!0}],starts:i},{className:"attr",begin:r+t+"$"}]}}},7667(e){e.exports=function(e){const t={className:"variable",variants:[{begin:"\\$\\("+e.UNDERSCORE_IDENT_RE+"\\)",contains:[e.BACKSLASH_ESCAPE]},{begin:/\$[@%=0;l--)if(void 0!==(i=t(e,n[l],s))){o=!0;break}return o?(r||"function"!=typeof i||(i=this.mv(i,n,a)),i):!r&&""},ls:function(e,t,n,a,r,i){var o=this.options.delimiters;return this.options.delimiters=i,this.b(this.ct(l(e.call(t,r,n)),t,a)),this.options.delimiters=o,!1},ct:function(e,t,n){if(this.options.disableLambda)throw new Error("Lambda features disabled.");return this.c.compile(e,this.options).render(t,n)},b:function(e){this.buf+=e},fl:function(){var e=this.buf;return this.buf="",e},ms:function(e,t,n,a,r,i,o){var s,l=t[t.length-1],c=e.call(l);return"function"==typeof c?!!a||(s=this.activeSub&&this.subsText&&this.subsText[this.activeSub]?this.subsText[this.activeSub]:this.text,this.ls(c,l,t,n,s.substring(r,i),o)):c},mv:function(e,t,n){var a=t[t.length-1],r=e.call(a);return"function"==typeof r?this.ct(l(r.call(a)),a,n):r},sub:function(e,t,n,a){var r=this.subs[e];r&&(this.activeSub=e,r(t,n,this,a),this.activeSub=!1)}};var n=/&/g,a=//g,i=/\'/g,o=/\"/g,s=/[&<>\"\']/;function l(e){return String(null==e?"":e)}var c=Array.isArray||function(e){return"[object Array]"===Object.prototype.toString.call(e)}}(t)},7826(e){e.exports=function(e){const t={variants:[e.COMMENT("^[ \\t]*(?=#)","$",{relevance:0,excludeBegin:!0}),e.COMMENT("[;@]","$",{relevance:0}),e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE]};return{name:"ARM Assembly",case_insensitive:!0,aliases:["arm"],keywords:{$pattern:"\\.?"+e.IDENT_RE,meta:".2byte .4byte .align .ascii .asciz .balign .byte .code .data .else .end .endif .endm .endr .equ .err .exitm .extern .global .hword .if .ifdef .ifndef .include .irp .long .macro .rept .req .section .set .skip .space .text .word .arm .thumb .code16 .code32 .force_thumb .thumb_func .ltorg ALIAS ALIGN ARM AREA ASSERT ATTR CN CODE CODE16 CODE32 COMMON CP DATA DCB DCD DCDU DCDO DCFD DCFDU DCI DCQ DCQU DCW DCWU DN ELIF ELSE END ENDFUNC ENDIF ENDP ENTRY EQU EXPORT EXPORTAS EXTERN FIELD FILL FUNCTION GBLA GBLL GBLS GET GLOBAL IF IMPORT INCBIN INCLUDE INFO KEEP LCLA LCLL LCLS LTORG MACRO MAP MEND MEXIT NOFP OPT PRESERVE8 PROC QN READONLY RELOC REQUIRE REQUIRE8 RLIST FN ROUT SETA SETL SETS SN SPACE SUBT THUMB THUMBX TTL WHILE WEND ",built_in:"r0 r1 r2 r3 r4 r5 r6 r7 r8 r9 r10 r11 r12 r13 r14 r15 w0 w1 w2 w3 w4 w5 w6 w7 w8 w9 w10 w11 w12 w13 w14 w15 w16 w17 w18 w19 w20 w21 w22 w23 w24 w25 w26 w27 w28 w29 w30 x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 x10 x11 x12 x13 x14 x15 x16 x17 x18 x19 x20 x21 x22 x23 x24 x25 x26 x27 x28 x29 x30 pc lr sp ip sl sb fp a1 a2 a3 a4 v1 v2 v3 v4 v5 v6 v7 v8 f0 f1 f2 f3 f4 f5 f6 f7 p0 p1 p2 p3 p4 p5 p6 p7 p8 p9 p10 p11 p12 p13 p14 p15 c0 c1 c2 c3 c4 c5 c6 c7 c8 c9 c10 c11 c12 c13 c14 c15 q0 q1 q2 q3 q4 q5 q6 q7 q8 q9 q10 q11 q12 q13 q14 q15 cpsr_c cpsr_x cpsr_s cpsr_f cpsr_cx cpsr_cxs cpsr_xs cpsr_xsf cpsr_sf cpsr_cxsf spsr_c spsr_x spsr_s spsr_f spsr_cx spsr_cxs spsr_xs spsr_xsf spsr_sf spsr_cxsf s0 s1 s2 s3 s4 s5 s6 s7 s8 s9 s10 s11 s12 s13 s14 s15 s16 s17 s18 s19 s20 s21 s22 s23 s24 s25 s26 s27 s28 s29 s30 s31 d0 d1 d2 d3 d4 d5 d6 d7 d8 d9 d10 d11 d12 d13 d14 d15 d16 d17 d18 d19 d20 d21 d22 d23 d24 d25 d26 d27 d28 d29 d30 d31 {PC} {VAR} {TRUE} {FALSE} {OPT} {CONFIG} {ENDIAN} {CODESIZE} {CPU} {FPU} {ARCHITECTURE} {PCSTOREOFFSET} {ARMASM_VERSION} {INTER} {ROPI} {RWPI} {SWST} {NOSWST} . @"},contains:[{className:"keyword",begin:"\\b(adc|(qd?|sh?|u[qh]?)?add(8|16)?|usada?8|(q|sh?|u[qh]?)?(as|sa)x|and|adrl?|sbc|rs[bc]|asr|b[lx]?|blx|bxj|cbn?z|tb[bh]|bic|bfc|bfi|[su]bfx|bkpt|cdp2?|clz|clrex|cmp|cmn|cpsi[ed]|cps|setend|dbg|dmb|dsb|eor|isb|it[te]{0,3}|lsl|lsr|ror|rrx|ldm(([id][ab])|f[ds])?|ldr((s|ex)?[bhd])?|movt?|mvn|mra|mar|mul|[us]mull|smul[bwt][bt]|smu[as]d|smmul|smmla|mla|umlaal|smlal?([wbt][bt]|d)|mls|smlsl?[ds]|smc|svc|sev|mia([bt]{2}|ph)?|mrr?c2?|mcrr2?|mrs|msr|orr|orn|pkh(tb|bt)|rbit|rev(16|sh)?|sel|[su]sat(16)?|nop|pop|push|rfe([id][ab])?|stm([id][ab])?|str(ex)?[bhd]?|(qd?)?sub|(sh?|q|u[qh]?)?sub(8|16)|[su]xt(a?h|a?b(16)?)|srs([id][ab])?|swpb?|swi|smi|tst|teq|wfe|wfi|yield)(eq|ne|cs|cc|mi|pl|vs|vc|hi|ls|ge|lt|gt|le|al|hs|lo)?[sptrx]?(?=\\s)"},t,e.QUOTE_STRING_MODE,{className:"string",begin:"'",end:"[^\\\\]'",relevance:0},{className:"title",begin:"\\|",end:"\\|",illegal:"\\n",relevance:0},{className:"number",variants:[{begin:"[#$=]?0x[0-9a-f]+"},{begin:"[#$=]?0b[01]+"},{begin:"[#$=]\\d+"},{begin:"\\b\\d+"}],relevance:0},{className:"symbol",variants:[{begin:"^[ \\t]*[a-z_\\.\\$][a-z0-9_\\.\\$]+:"},{begin:"^[a-z_\\.\\$][a-z0-9_\\.\\$]+"},{begin:"[=#]\\w+"}],relevance:0}]}}},7850(e){e.exports=function(e){const t={className:"number",relevance:0,variants:[{begin:"[$][a-fA-F0-9]+"},e.NUMBER_MODE]},n={variants:[{match:[/(function|method)/,/\s+/,e.UNDERSCORE_IDENT_RE]}],scope:{1:"keyword",3:"title.function"}},a={variants:[{match:[/(class|interface|extends|implements)/,/\s+/,e.UNDERSCORE_IDENT_RE]}],scope:{1:"keyword",3:"title.class"}};return{name:"Monkey",case_insensitive:!0,keywords:{keyword:["public","private","property","continue","exit","extern","new","try","catch","eachin","not","abstract","final","select","case","default","const","local","global","field","end","if","then","else","elseif","endif","while","wend","repeat","until","forever","for","to","step","next","return","module","inline","throw","import","and","or","shl","shr","mod"],built_in:["DebugLog","DebugStop","Error","Print","ACos","ACosr","ASin","ASinr","ATan","ATan2","ATan2r","ATanr","Abs","Abs","Ceil","Clamp","Clamp","Cos","Cosr","Exp","Floor","Log","Max","Max","Min","Min","Pow","Sgn","Sgn","Sin","Sinr","Sqrt","Tan","Tanr","Seed","PI","HALFPI","TWOPI"],literal:["true","false","null"]},illegal:/\/\*/,contains:[e.COMMENT("#rem","#end"),e.COMMENT("'","$",{relevance:0}),n,a,{className:"variable.language",begin:/\b(self|super)\b/},{className:"meta",begin:/\s*#/,end:"$",keywords:{keyword:"if else elseif endif end then"}},{match:[/^\s*/,/strict\b/],scope:{2:"meta"}},{beginKeywords:"alias",end:"=",contains:[e.UNDERSCORE_TITLE_MODE]},e.QUOTE_STRING_MODE,t]}}},7869(e){e.exports=function(e){const t={className:"number",begin:/\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}(:\d{1,5})?/};return{name:"Apache config",aliases:["apacheconf"],case_insensitive:!0,contains:[e.HASH_COMMENT_MODE,{className:"section",begin:/<\/?/,end:/>/,contains:[t,{className:"number",begin:/:\d{1,5}/},e.inherit(e.QUOTE_STRING_MODE,{relevance:0})]},{className:"attribute",begin:/\w+/,relevance:0,keywords:{_:["order","deny","allow","setenv","rewriterule","rewriteengine","rewritecond","documentroot","sethandler","errordocument","loadmodule","options","header","listen","serverroot","servername"]},starts:{end:/$/,relevance:0,keywords:{literal:"on off all deny allow"},contains:[{scope:"punctuation",match:/\\\n/},{className:"meta",begin:/\s\[/,end:/\]$/},{className:"variable",begin:/[\$%]\{/,end:/\}/,contains:["self",{className:"number",begin:/[$%]\d+/}]},t,{className:"number",begin:/\b\d+/},e.QUOTE_STRING_MODE]}}],illegal:/\S/}}},7891(e){e.exports=function(e){const t=e.regex,n=["bernoulli","bernoulli_logit","bernoulli_logit_glm","beta","beta_binomial","beta_proportion","binomial","binomial_logit","categorical","categorical_logit","categorical_logit_glm","cauchy","chi_square","dirichlet","discrete_range","double_exponential","exp_mod_normal","exponential","frechet","gamma","gaussian_dlm_obs","gumbel","hmm_latent","hypergeometric","inv_chi_square","inv_gamma","inv_wishart","inv_wishart_cholesky","lkj_corr","lkj_corr_cholesky","logistic","loglogistic","lognormal","multi_gp","multi_gp_cholesky","multinomial","multinomial_logit","multi_normal","multi_normal_cholesky","multi_normal_prec","multi_student_cholesky_t","multi_student_t","multi_student_t_cholesky","neg_binomial","neg_binomial_2","neg_binomial_2_log","neg_binomial_2_log_glm","normal","normal_id_glm","ordered_logistic","ordered_logistic_glm","ordered_probit","pareto","pareto_type_2","poisson","poisson_log","poisson_log_glm","rayleigh","scaled_inv_chi_square","skew_double_exponential","skew_normal","std_normal","std_normal_log","student_t","uniform","von_mises","weibull","wiener","wishart","wishart_cholesky"],a=e.COMMENT(/\/\*/,/\*\//,{relevance:0,contains:[{scope:"doctag",match:/@(return|param)/}]}),r={scope:"meta",begin:/#include\b/,end:/$/,contains:[{match:/[a-z][a-z-._]+/,scope:"string"},e.C_LINE_COMMENT_MODE]},i=["lower","upper","offset","multiplier"];return{name:"Stan",aliases:["stanfuncs"],keywords:{$pattern:e.IDENT_RE,title:["functions","model","data","parameters","quantities","transformed","generated"],type:["array","tuple","complex","int","real","vector","complex_vector","ordered","positive_ordered","simplex","unit_vector","row_vector","complex_row_vector","matrix","complex_matrix","cholesky_factor_corr|10","cholesky_factor_cov|10","corr_matrix|10","cov_matrix|10","void"],keyword:["for","in","if","else","while","break","continue","return"],built_in:["abs","acos","acosh","add_diag","algebra_solver","algebra_solver_newton","append_array","append_col","append_row","asin","asinh","atan","atan2","atanh","bessel_first_kind","bessel_second_kind","binary_log_loss","block","cbrt","ceil","chol2inv","cholesky_decompose","choose","col","cols","columns_dot_product","columns_dot_self","complex_schur_decompose","complex_schur_decompose_t","complex_schur_decompose_u","conj","cos","cosh","cov_exp_quad","crossprod","csr_extract","csr_extract_u","csr_extract_v","csr_extract_w","csr_matrix_times_vector","csr_to_dense_matrix","cumulative_sum","dae","dae_tol","determinant","diag_matrix","diagonal","diag_post_multiply","diag_pre_multiply","digamma","dims","distance","dot_product","dot_self","eigendecompose","eigendecompose_sym","eigenvalues","eigenvalues_sym","eigenvectors","eigenvectors_sym","erf","erfc","exp","exp2","expm1","falling_factorial","fdim","fft","fft2","floor","fma","fmax","fmin","fmod","gamma_p","gamma_q","generalized_inverse","get_imag","get_real","head","hmm_hidden_state_prob","hmm_marginal","hypot","identity_matrix","inc_beta","integrate_1d","integrate_ode","integrate_ode_adams","integrate_ode_bdf","integrate_ode_rk45","int_step","inv","inv_cloglog","inv_erfc","inverse","inverse_spd","inv_fft","inv_fft2","inv_inc_beta","inv_logit","inv_Phi","inv_sqrt","inv_square","is_inf","is_nan","lambert_w0","lambert_wm1","lbeta","lchoose","ldexp","lgamma","linspaced_array","linspaced_int_array","linspaced_row_vector","linspaced_vector","lmgamma","lmultiply","log","log1m","log1m_exp","log1m_inv_logit","log1p","log1p_exp","log_determinant","log_diff_exp","log_falling_factorial","log_inv_logit","log_inv_logit_diff","logit","log_mix","log_modified_bessel_first_kind","log_rising_factorial","log_softmax","log_sum_exp","machine_precision","map_rect","matrix_exp","matrix_exp_multiply","matrix_power","max","mdivide_left_spd","mdivide_left_tri_low","mdivide_right_spd","mdivide_right_tri_low","mean","min","modified_bessel_first_kind","modified_bessel_second_kind","multiply_lower_tri_self_transpose","negative_infinity","norm","norm1","norm2","not_a_number","num_elements","ode_adams","ode_adams_tol","ode_adjoint_tol_ctl","ode_bdf","ode_bdf_tol","ode_ckrk","ode_ckrk_tol","ode_rk45","ode_rk45_tol","one_hot_array","one_hot_int_array","one_hot_row_vector","one_hot_vector","ones_array","ones_int_array","ones_row_vector","ones_vector","owens_t","Phi","Phi_approx","polar","positive_infinity","pow","print","prod","proj","qr","qr_Q","qr_R","qr_thin","qr_thin_Q","qr_thin_R","quad_form","quad_form_diag","quad_form_sym","quantile","rank","reduce_sum","reject","rep_array","rep_matrix","rep_row_vector","rep_vector","reverse","rising_factorial","round","row","rows","rows_dot_product","rows_dot_self","scale_matrix_exp_multiply","sd","segment","sin","singular_values","sinh","size","softmax","sort_asc","sort_desc","sort_indices_asc","sort_indices_desc","sqrt","square","squared_distance","step","sub_col","sub_row","sum","svd","svd_U","svd_V","symmetrize_from_lower_tri","tail","tan","tanh","target","tcrossprod","tgamma","to_array_1d","to_array_2d","to_complex","to_int","to_matrix","to_row_vector","to_vector","trace","trace_gen_quad_form","trace_quad_form","trigamma","trunc","uniform_simplex","variance","zeros_array","zeros_int_array","zeros_row_vector"]},contains:[e.C_LINE_COMMENT_MODE,r,e.HASH_COMMENT_MODE,a,{scope:"built_in",match:/\s(pi|e|sqrt2|log2|log10)(?=\()/,relevance:0},{match:t.concat(/[<,]\s*/,t.either(...i),/\s*=/),keywords:i},{scope:"keyword",match:/\btarget(?=\s*\+=)/},{match:[/~\s*/,t.either(...n),/(?:\(\))/,/\s*T(?=\s*\[)/],scope:{2:"built_in",4:"keyword"}},{scope:"built_in",keywords:n,begin:t.concat(/\w*/,t.either(...n),/(_lpdf|_lupdf|_lpmf|_cdf|_lcdf|_lccdf|_qf)(?=\s*[\(.*\)])/)},{begin:[/~/,/\s*/,t.concat(t.either(...n),/(?=\s*[\(.*\)])/)],scope:{3:"built_in"}},{begin:[/~/,/\s*\w+(?=\s*[\(.*\)])/,"(?!.*/\b("+t.either(...n)+")\b)"],scope:{2:"title.function"}},{scope:"title.function",begin:/\w*(_lpdf|_lupdf|_lpmf|_cdf|_lcdf|_lccdf|_qf)(?=\s*[\(.*\)])/},{scope:"number",match:t.concat(/(?:\b\d+(?:_\d+)*(?:\.(?:\d+(?:_\d+)*)?)?|\B\.\d+(?:_\d+)*)/,/(?:[eE][+-]?\d+(?:_\d+)*)?i?(?!\w)/),relevance:0},{scope:"string",begin:/"/,end:/"/}]}}},7904(e,t,n){"use strict";Object.defineProperty(t,"__esModule",{value:!0}),t.applyPatch=s,t.applyPatches=function(e,t){var n="string"==typeof e?(0,i.parsePatch)(e):e,a=0;!function e(){var r=n[a++];if(!r)return t.complete();t.loadFile(r,(function(n,a){if(n)return t.complete(n);var i=s(a,r,t);t.patched(r,i,(function(n){if(n)return t.complete(n);e()}))}))}()};var a=n(6665),r=n(8725),i=n(9363),o=n(2738);function s(e,t,n){var s;if(void 0===n&&(n={}),(s="string"==typeof t?(0,i.parsePatch)(t):Array.isArray(t)?t:[t]).length>1)throw new Error("applyPatch only works with a single input.");return function(e,t,n){void 0===n&&(n={}),(n.autoConvertLineEndings||null==n.autoConvertLineEndings)&&((0,a.hasOnlyWinLineEndings)(e)&&(0,r.isUnix)(t)?t=(0,r.unixToWin)(t):(0,a.hasOnlyUnixLineEndings)(e)&&(0,r.isWin)(t)&&(t=(0,r.winToUnix)(t)));var i=e.split("\n"),s=t.hunks,l=n.compareLine||function(e,t,n,a){return t===a},c=n.fuzzFactor||0,_=0;if(c<0||!Number.isInteger(c))throw new Error("fuzzFactor must be a non-negative integer");if(!s.length)return e;for(var d="",p=!1,u=!1,m=0;m0?d[0]:" ",u=d.length>0?d.substr(1):d;if("-"===p){if(!l(t+1,i[t],p,u))return n&&null!=i[t]?(o[s]=i[t],g(e,t+1,n-1,a,!1,o,s+1)):null;t++,c=0}if("+"===p){if(!r)return null;o[s]=u,s++,c=0,_=!0}if(" "===p){if(c++,o[s]=i[t],!l(t+1,i[t],p,u))return _||!n?null:i[t]&&(g(e,t+1,n-1,a+1,!1,o,s+1)||g(e,t+1,n-1,a,!1,o,s+1))||g(e,t,n-1,a+1,!1,o,s);s++,r=!0,_=!1,t++}}return s-=c,t-=c,o.length=s,{patchedLines:o,oldLineLastI:t-1}}var E=[],S=0;for(m=0;m:]=|\|\||&&|:::?|<-|<<-|->>|->|\|>|[-+*\/?!$&|:<=>@^~]|\*\*/,i=t.either(/[()]/,/[{}]/,/\[\[/,/[[\]]/,/\\/,/,/);return{name:"R",keywords:{$pattern:n,keyword:"function if in break next repeat else for while",literal:"NULL NA TRUE FALSE Inf NaN NA_integer_|10 NA_real_|10 NA_character_|10 NA_complex_|10",built_in:"LETTERS letters month.abb month.name pi T F abs acos acosh all any anyNA Arg as.call as.character as.complex as.double as.environment as.integer as.logical as.null.default as.numeric as.raw asin asinh atan atanh attr attributes baseenv browser c call ceiling class Conj cos cosh cospi cummax cummin cumprod cumsum digamma dim dimnames emptyenv exp expression floor forceAndCall gamma gc.time globalenv Im interactive invisible is.array is.atomic is.call is.character is.complex is.double is.environment is.expression is.finite is.function is.infinite is.integer is.language is.list is.logical is.matrix is.na is.name is.nan is.null is.numeric is.object is.pairlist is.raw is.recursive is.single is.symbol lazyLoadDBfetch length lgamma list log max min missing Mod names nargs nzchar oldClass on.exit pos.to.env proc.time prod quote range Re rep retracemem return round seq_along seq_len seq.int sign signif sin sinh sinpi sqrt standardGeneric substitute sum switch tan tanh tanpi tracemem trigamma trunc unclass untracemem UseMethod xtfrm"},contains:[e.COMMENT(/#'/,/$/,{contains:[{scope:"doctag",match:/@examples/,starts:{end:t.lookahead(t.either(/\n^#'\s*(?=@[a-zA-Z]+)/,/\n^(?!#')/)),endsParent:!0}},{scope:"doctag",begin:"@param",end:/$/,contains:[{scope:"variable",variants:[{match:n},{match:/`(?:\\.|[^`\\])+`/}],endsParent:!0}]},{scope:"doctag",match:/@[a-zA-Z]+/},{scope:"keyword",match:/\\[a-zA-Z]+/}]}),e.HASH_COMMENT_MODE,{scope:"string",contains:[e.BACKSLASH_ESCAPE],variants:[e.END_SAME_AS_BEGIN({begin:/[rR]"(-*)\(/,end:/\)(-*)"/}),e.END_SAME_AS_BEGIN({begin:/[rR]"(-*)\{/,end:/\}(-*)"/}),e.END_SAME_AS_BEGIN({begin:/[rR]"(-*)\[/,end:/\](-*)"/}),e.END_SAME_AS_BEGIN({begin:/[rR]'(-*)\(/,end:/\)(-*)'/}),e.END_SAME_AS_BEGIN({begin:/[rR]'(-*)\{/,end:/\}(-*)'/}),e.END_SAME_AS_BEGIN({begin:/[rR]'(-*)\[/,end:/\](-*)'/}),{begin:'"',end:'"',relevance:0},{begin:"'",end:"'",relevance:0}]},{relevance:0,variants:[{scope:{1:"operator",2:"number"},match:[r,a]},{scope:{1:"operator",2:"number"},match:[/%[^%]*%/,a]},{scope:{1:"punctuation",2:"number"},match:[i,a]},{scope:{2:"number"},match:[/[^a-zA-Z0-9._]|^/,a]}]},{scope:{3:"operator"},match:[n,/\s+/,/<-/,/\s+/]},{scope:"operator",relevance:0,variants:[{match:r},{match:/%[^%]*%/}]},{scope:"punctuation",relevance:0,match:i},{begin:"`",end:"`",contains:[{begin:/\\./}]}]}}},8155(e){e.exports=function(e){const t={variants:[e.COMMENT("--","$"),e.COMMENT(/\{-/,/-\}/,{contains:["self"]})]},n={className:"type",begin:"\\b[A-Z][\\w']*",relevance:0},a={begin:"\\(",end:"\\)",illegal:'"',contains:[{className:"type",begin:"\\b[A-Z][\\w]*(\\((\\.\\.|,|\\w+)\\))?"},t]};return{name:"Elm",keywords:["let","in","if","then","else","case","of","where","module","import","exposing","type","alias","as","infix","infixl","infixr","port","effect","command","subscription"],contains:[{beginKeywords:"port effect module",end:"exposing",keywords:"port effect module where command subscription exposing",contains:[a,t],illegal:"\\W\\.|;"},{begin:"import",end:"$",keywords:"import as exposing",contains:[a,t],illegal:"\\W\\.|;"},{begin:"type",end:"$",keywords:"type alias",contains:[n,a,{begin:/\{/,end:/\}/,contains:a.contains},t]},{beginKeywords:"infix infixl infixr",end:"$",contains:[e.C_NUMBER_MODE,t]},{begin:"port",end:"$",keywords:"port",contains:[t]},{className:"string",begin:"'\\\\?.",end:"'",illegal:"."},e.QUOTE_STRING_MODE,e.C_NUMBER_MODE,n,e.inherit(e.TITLE_MODE,{begin:"^[_a-z][\\w']*"}),t,{begin:"->|<-"}],illegal:/;/}}},8188(e){e.exports=function(e){const t="[a-z'][a-zA-Z0-9_']*",n="("+t+":"+t+"|"+t+")",a={keyword:"after and andalso|10 band begin bnot bor bsl bzr bxor case catch cond div end fun if let not of orelse|10 query receive rem try when xor maybe else",literal:"false true"},r=e.COMMENT("%","$"),i={className:"number",begin:"\\b(\\d+(_\\d+)*#[a-fA-F0-9]+(_[a-fA-F0-9]+)*|\\d+(_\\d+)*(\\.\\d+(_\\d+)*)?([eE][-+]?\\d+)?)",relevance:0},o={begin:"fun\\s+"+t+"/\\d+"},s={begin:n+"\\(",end:"\\)",returnBegin:!0,relevance:0,contains:[{begin:n,relevance:0},{begin:"\\(",end:"\\)",endsWithParent:!0,returnEnd:!0,relevance:0}]},l={begin:/\{/,end:/\}/,relevance:0},c={begin:"\\b_([A-Z][A-Za-z0-9_]*)?",relevance:0},_={begin:"[A-Z][a-zA-Z0-9_]*",relevance:0},d={begin:"#"+e.UNDERSCORE_IDENT_RE,relevance:0,returnBegin:!0,contains:[{begin:"#"+e.UNDERSCORE_IDENT_RE,relevance:0},{begin:/\{/,end:/\}/,relevance:0}]},p={scope:"string",match:/\$(\\([^0-9]|[0-9]{1,3}|)|.)/},u={scope:"string",match:/"""("*)(?!")[\s\S]*?"""\1/},m={scope:"string",contains:[e.BACKSLASH_ESCAPE],variants:[{match:/~\w?"""("*)(?!")[\s\S]*?"""\1/},{begin:/~\w?\(/,end:/\)/},{begin:/~\w?\[/,end:/\]/},{begin:/~\w?{/,end:/}/},{begin:/~\w?/},{begin:/~\w?\//,end:/\//},{begin:/~\w?\|/,end:/\|/},{begin:/~\w?'/,end:/'/},{begin:/~\w?"/,end:/"/},{begin:/~\w?`/,end:/`/},{begin:/~\w?#/,end:/#/}]},g={beginKeywords:"fun receive if try case maybe",end:"end",keywords:a};g.contains=[r,o,e.inherit(e.APOS_STRING_MODE,{className:""}),g,s,m,u,e.QUOTE_STRING_MODE,i,l,c,_,d,p];const E=[r,o,g,s,m,u,e.QUOTE_STRING_MODE,i,l,c,_,d,p];s.contains[1].contains=E,l.contains=E,d.contains[1].contains=E;const S={className:"params",begin:"\\(",end:"\\)",contains:E};return{name:"Erlang",aliases:["erl"],keywords:a,illegal:"(",returnBegin:!0,illegal:"\\(|#|//|/\\*|\\\\|:|;",contains:[S,e.inherit(e.TITLE_MODE,{begin:t})],starts:{end:";|\\.",keywords:a,contains:E}},r,{begin:"^-",end:"\\.",relevance:0,excludeEnd:!0,returnBegin:!0,keywords:{$pattern:"-"+e.IDENT_RE,keyword:["-module","-record","-undef","-export","-ifdef","-ifndef","-author","-copyright","-doc","-moduledoc","-vsn","-import","-include","-include_lib","-compile","-define","-else","-endif","-file","-behaviour","-behavior","-spec","-on_load","-nifs"].map((e=>`${e}|1.5`)).join(" ")},contains:[S,m,u,e.QUOTE_STRING_MODE]},i,m,u,e.QUOTE_STRING_MODE,d,c,_,l,p,{begin:/\.$/}]}}},8216(e){e.exports=function(e){return{name:"Maxima",keywords:{$pattern:"[A-Za-z_%][0-9A-Za-z_%]*",keyword:"if then else elseif for thru do while unless step in and or not",literal:"true false unknown inf minf ind und %e %i %pi %phi %gamma",built_in:" abasep abs absint absolute_real_time acos acosh acot acoth acsc acsch activate addcol add_edge add_edges addmatrices addrow add_vertex add_vertices adjacency_matrix adjoin adjoint af agd airy airy_ai airy_bi airy_dai airy_dbi algsys alg_type alias allroots alphacharp alphanumericp amortization %and annuity_fv annuity_pv antid antidiff AntiDifference append appendfile apply apply1 apply2 applyb1 apropos args arit_amortization arithmetic arithsum array arrayapply arrayinfo arraymake arraysetapply ascii asec asech asin asinh askinteger asksign assoc assoc_legendre_p assoc_legendre_q assume assume_external_byte_order asympa at atan atan2 atanh atensimp atom atvalue augcoefmatrix augmented_lagrangian_method av average_degree backtrace bars barsplot barsplot_description base64 base64_decode bashindices batch batchload bc2 bdvac belln benefit_cost bern bernpoly bernstein_approx bernstein_expand bernstein_poly bessel bessel_i bessel_j bessel_k bessel_simplify bessel_y beta beta_incomplete beta_incomplete_generalized beta_incomplete_regularized bezout bfallroots bffac bf_find_root bf_fmin_cobyla bfhzeta bfloat bfloatp bfpsi bfpsi0 bfzeta biconnected_components bimetric binomial bipartition block blockmatrixp bode_gain bode_phase bothcoef box boxplot boxplot_description break bug_report build_info|10 buildq build_sample burn cabs canform canten cardinality carg cartan cartesian_product catch cauchy_matrix cbffac cdf_bernoulli cdf_beta cdf_binomial cdf_cauchy cdf_chi2 cdf_continuous_uniform cdf_discrete_uniform cdf_exp cdf_f cdf_gamma cdf_general_finite_discrete cdf_geometric cdf_gumbel cdf_hypergeometric cdf_laplace cdf_logistic cdf_lognormal cdf_negative_binomial cdf_noncentral_chi2 cdf_noncentral_student_t cdf_normal cdf_pareto cdf_poisson cdf_rank_sum cdf_rayleigh cdf_signed_rank cdf_student_t cdf_weibull cdisplay ceiling central_moment cequal cequalignore cf cfdisrep cfexpand cgeodesic cgreaterp cgreaterpignore changename changevar chaosgame charat charfun charfun2 charlist charp charpoly chdir chebyshev_t chebyshev_u checkdiv check_overlaps chinese cholesky christof chromatic_index chromatic_number cint circulant_graph clear_edge_weight clear_rules clear_vertex_label clebsch_gordan clebsch_graph clessp clesspignore close closefile cmetric coeff coefmatrix cograd col collapse collectterms columnop columnspace columnswap columnvector combination combine comp2pui compare compfile compile compile_file complement_graph complete_bipartite_graph complete_graph complex_number_p components compose_functions concan concat conjugate conmetderiv connected_components connect_vertices cons constant constantp constituent constvalue cont2part content continuous_freq contortion contour_plot contract contract_edge contragrad contrib_ode convert coord copy copy_file copy_graph copylist copymatrix cor cos cosh cot coth cov cov1 covdiff covect covers crc24sum create_graph create_list csc csch csetup cspline ctaylor ct_coordsys ctransform ctranspose cube_graph cuboctahedron_graph cunlisp cv cycle_digraph cycle_graph cylindrical days360 dblint deactivate declare declare_constvalue declare_dimensions declare_fundamental_dimensions declare_fundamental_units declare_qty declare_translated declare_unit_conversion declare_units declare_weights decsym defcon define define_alt_display define_variable defint defmatch defrule defstruct deftaylor degree_sequence del delete deleten delta demo demoivre denom depends derivdegree derivlist describe desolve determinant dfloat dgauss_a dgauss_b dgeev dgemm dgeqrf dgesv dgesvd diag diagmatrix diag_matrix diagmatrixp diameter diff digitcharp dimacs_export dimacs_import dimension dimensionless dimensions dimensions_as_list direct directory discrete_freq disjoin disjointp disolate disp dispcon dispform dispfun dispJordan display disprule dispterms distrib divide divisors divsum dkummer_m dkummer_u dlange dodecahedron_graph dotproduct dotsimp dpart draw draw2d draw3d drawdf draw_file draw_graph dscalar echelon edge_coloring edge_connectivity edges eigens_by_jacobi eigenvalues eigenvectors eighth einstein eivals eivects elapsed_real_time elapsed_run_time ele2comp ele2polynome ele2pui elem elementp elevation_grid elim elim_allbut eliminate eliminate_using ellipse elliptic_e elliptic_ec elliptic_eu elliptic_f elliptic_kc elliptic_pi ematrix empty_graph emptyp endcons entermatrix entertensor entier equal equalp equiv_classes erf erfc erf_generalized erfi errcatch error errormsg errors euler ev eval_string evenp every evolution evolution2d evundiff example exp expand expandwrt expandwrt_factored expint expintegral_chi expintegral_ci expintegral_e expintegral_e1 expintegral_ei expintegral_e_simplify expintegral_li expintegral_shi expintegral_si explicit explose exponentialize express expt exsec extdiff extract_linear_equations extremal_subset ezgcd %f f90 facsum factcomb factor factorfacsum factorial factorout factorsum facts fast_central_elements fast_linsolve fasttimes featurep fernfale fft fib fibtophi fifth filename_merge file_search file_type fillarray findde find_root find_root_abs find_root_error find_root_rel first fix flatten flength float floatnump floor flower_snark flush flush1deriv flushd flushnd flush_output fmin_cobyla forget fortran fourcos fourexpand fourier fourier_elim fourint fourintcos fourintsin foursimp foursin fourth fposition frame_bracket freeof freshline fresnel_c fresnel_s from_adjacency_matrix frucht_graph full_listify fullmap fullmapl fullratsimp fullratsubst fullsetify funcsolve fundamental_dimensions fundamental_units fundef funmake funp fv g0 g1 gamma gamma_greek gamma_incomplete gamma_incomplete_generalized gamma_incomplete_regularized gauss gauss_a gauss_b gaussprob gcd gcdex gcdivide gcfac gcfactor gd generalized_lambert_w genfact gen_laguerre genmatrix gensym geo_amortization geo_annuity_fv geo_annuity_pv geomap geometric geometric_mean geosum get getcurrentdirectory get_edge_weight getenv get_lu_factors get_output_stream_string get_pixel get_plot_option get_tex_environment get_tex_environment_default get_vertex_label gfactor gfactorsum ggf girth global_variances gn gnuplot_close gnuplot_replot gnuplot_reset gnuplot_restart gnuplot_start go Gosper GosperSum gr2d gr3d gradef gramschmidt graph6_decode graph6_encode graph6_export graph6_import graph_center graph_charpoly graph_eigenvalues graph_flow graph_order graph_periphery graph_product graph_size graph_union great_rhombicosidodecahedron_graph great_rhombicuboctahedron_graph grid_graph grind grobner_basis grotzch_graph hamilton_cycle hamilton_path hankel hankel_1 hankel_2 harmonic harmonic_mean hav heawood_graph hermite hessian hgfred hilbertmap hilbert_matrix hipow histogram histogram_description hodge horner hypergeometric i0 i1 %ibes ic1 ic2 ic_convert ichr1 ichr2 icosahedron_graph icosidodecahedron_graph icurvature ident identfor identity idiff idim idummy ieqn %if ifactors iframes ifs igcdex igeodesic_coords ilt image imagpart imetric implicit implicit_derivative implicit_plot indexed_tensor indices induced_subgraph inferencep inference_result infix info_display init_atensor init_ctensor in_neighbors innerproduct inpart inprod inrt integerp integer_partitions integrate intersect intersection intervalp intopois intosum invariant1 invariant2 inverse_fft inverse_jacobi_cd inverse_jacobi_cn inverse_jacobi_cs inverse_jacobi_dc inverse_jacobi_dn inverse_jacobi_ds inverse_jacobi_nc inverse_jacobi_nd inverse_jacobi_ns inverse_jacobi_sc inverse_jacobi_sd inverse_jacobi_sn invert invert_by_adjoint invert_by_lu inv_mod irr is is_biconnected is_bipartite is_connected is_digraph is_edge_in_graph is_graph is_graph_or_digraph ishow is_isomorphic isolate isomorphism is_planar isqrt isreal_p is_sconnected is_tree is_vertex_in_graph items_inference %j j0 j1 jacobi jacobian jacobi_cd jacobi_cn jacobi_cs jacobi_dc jacobi_dn jacobi_ds jacobi_nc jacobi_nd jacobi_ns jacobi_p jacobi_sc jacobi_sd jacobi_sn JF jn join jordan julia julia_set julia_sin %k kdels kdelta kill killcontext kostka kron_delta kronecker_product kummer_m kummer_u kurtosis kurtosis_bernoulli kurtosis_beta kurtosis_binomial kurtosis_chi2 kurtosis_continuous_uniform kurtosis_discrete_uniform kurtosis_exp kurtosis_f kurtosis_gamma kurtosis_general_finite_discrete kurtosis_geometric kurtosis_gumbel kurtosis_hypergeometric kurtosis_laplace kurtosis_logistic kurtosis_lognormal kurtosis_negative_binomial kurtosis_noncentral_chi2 kurtosis_noncentral_student_t kurtosis_normal kurtosis_pareto kurtosis_poisson kurtosis_rayleigh kurtosis_student_t kurtosis_weibull label labels lagrange laguerre lambda lambert_w laplace laplacian_matrix last lbfgs lc2kdt lcharp lc_l lcm lc_u ldefint ldisp ldisplay legendre_p legendre_q leinstein length let letrules letsimp levi_civita lfreeof lgtreillis lhs li liediff limit Lindstedt linear linearinterpol linear_program linear_regression line_graph linsolve listarray list_correlations listify list_matrix_entries list_nc_monomials listoftens listofvars listp lmax lmin load loadfile local locate_matrix_entry log logcontract log_gamma lopow lorentz_gauge lowercasep lpart lratsubst lreduce lriemann lsquares_estimates lsquares_estimates_approximate lsquares_estimates_exact lsquares_mse lsquares_residual_mse lsquares_residuals lsum ltreillis lu_backsub lucas lu_factor %m macroexpand macroexpand1 make_array makebox makefact makegamma make_graph make_level_picture makelist makeOrders make_poly_continent make_poly_country make_polygon make_random_state make_rgb_picture makeset make_string_input_stream make_string_output_stream make_transform mandelbrot mandelbrot_set map mapatom maplist matchdeclare matchfix mat_cond mat_fullunblocker mat_function mathml_display mat_norm matrix matrixmap matrixp matrix_size mattrace mat_trace mat_unblocker max max_clique max_degree max_flow maximize_lp max_independent_set max_matching maybe md5sum mean mean_bernoulli mean_beta mean_binomial mean_chi2 mean_continuous_uniform mean_deviation mean_discrete_uniform mean_exp mean_f mean_gamma mean_general_finite_discrete mean_geometric mean_gumbel mean_hypergeometric mean_laplace mean_logistic mean_lognormal mean_negative_binomial mean_noncentral_chi2 mean_noncentral_student_t mean_normal mean_pareto mean_poisson mean_rayleigh mean_student_t mean_weibull median median_deviation member mesh metricexpandall mgf1_sha1 min min_degree min_edge_cut minfactorial minimalPoly minimize_lp minimum_spanning_tree minor minpack_lsquares minpack_solve min_vertex_cover min_vertex_cut mkdir mnewton mod mode_declare mode_identity ModeMatrix moebius mon2schur mono monomial_dimensions multibernstein_poly multi_display_for_texinfo multi_elem multinomial multinomial_coeff multi_orbit multiplot_mode multi_pui multsym multthru mycielski_graph nary natural_unit nc_degree ncexpt ncharpoly negative_picture neighbors new newcontext newdet new_graph newline newton new_variable next_prime nicedummies niceindices ninth nofix nonarray noncentral_moment nonmetricity nonnegintegerp nonscalarp nonzeroandfreeof notequal nounify nptetrad npv nroots nterms ntermst nthroot nullity nullspace num numbered_boundaries numberp number_to_octets num_distinct_partitions numerval numfactor num_partitions nusum nzeta nzetai nzetar octets_to_number octets_to_oid odd_girth oddp ode2 ode_check odelin oid_to_octets op opena opena_binary openr openr_binary openw openw_binary operatorp opsubst optimize %or orbit orbits ordergreat ordergreatp orderless orderlessp orthogonal_complement orthopoly_recur orthopoly_weight outermap out_neighbors outofpois pade parabolic_cylinder_d parametric parametric_surface parg parGosper parse_string parse_timedate part part2cont partfrac partition partition_set partpol path_digraph path_graph pathname_directory pathname_name pathname_type pdf_bernoulli pdf_beta pdf_binomial pdf_cauchy pdf_chi2 pdf_continuous_uniform pdf_discrete_uniform pdf_exp pdf_f pdf_gamma pdf_general_finite_discrete pdf_geometric pdf_gumbel pdf_hypergeometric pdf_laplace pdf_logistic pdf_lognormal pdf_negative_binomial pdf_noncentral_chi2 pdf_noncentral_student_t pdf_normal pdf_pareto pdf_poisson pdf_rank_sum pdf_rayleigh pdf_signed_rank pdf_student_t pdf_weibull pearson_skewness permanent permut permutation permutations petersen_graph petrov pickapart picture_equalp picturep piechart piechart_description planar_embedding playback plog plot2d plot3d plotdf ploteq plsquares pochhammer points poisdiff poisexpt poisint poismap poisplus poissimp poissubst poistimes poistrim polar polarform polartorect polar_to_xy poly_add poly_buchberger poly_buchberger_criterion poly_colon_ideal poly_content polydecomp poly_depends_p poly_elimination_ideal poly_exact_divide poly_expand poly_expt poly_gcd polygon poly_grobner poly_grobner_equal poly_grobner_member poly_grobner_subsetp poly_ideal_intersection poly_ideal_polysaturation poly_ideal_polysaturation1 poly_ideal_saturation poly_ideal_saturation1 poly_lcm poly_minimization polymod poly_multiply polynome2ele polynomialp poly_normal_form poly_normalize poly_normalize_list poly_polysaturation_extension poly_primitive_part poly_pseudo_divide poly_reduced_grobner poly_reduction poly_saturation_extension poly_s_polynomial poly_subtract polytocompanion pop postfix potential power_mod powerseries powerset prefix prev_prime primep primes principal_components print printf printfile print_graph printpois printprops prodrac product properties propvars psi psubst ptriangularize pui pui2comp pui2ele pui2polynome pui_direct puireduc push put pv qput qrange qty quad_control quad_qag quad_qagi quad_qagp quad_qags quad_qawc quad_qawf quad_qawo quad_qaws quadrilateral quantile quantile_bernoulli quantile_beta quantile_binomial quantile_cauchy quantile_chi2 quantile_continuous_uniform quantile_discrete_uniform quantile_exp quantile_f quantile_gamma quantile_general_finite_discrete quantile_geometric quantile_gumbel quantile_hypergeometric quantile_laplace quantile_logistic quantile_lognormal quantile_negative_binomial quantile_noncentral_chi2 quantile_noncentral_student_t quantile_normal quantile_pareto quantile_poisson quantile_rayleigh quantile_student_t quantile_weibull quartile_skewness quit qunit quotient racah_v racah_w radcan radius random random_bernoulli random_beta random_binomial random_bipartite_graph random_cauchy random_chi2 random_continuous_uniform random_digraph random_discrete_uniform random_exp random_f random_gamma random_general_finite_discrete random_geometric random_graph random_graph1 random_gumbel random_hypergeometric random_laplace random_logistic random_lognormal random_negative_binomial random_network random_noncentral_chi2 random_noncentral_student_t random_normal random_pareto random_permutation random_poisson random_rayleigh random_regular_graph random_student_t random_tournament random_tree random_weibull range rank rat ratcoef ratdenom ratdiff ratdisrep ratexpand ratinterpol rational rationalize ratnumer ratnump ratp ratsimp ratsubst ratvars ratweight read read_array read_binary_array read_binary_list read_binary_matrix readbyte readchar read_hashed_array readline read_list read_matrix read_nested_list readonly read_xpm real_imagpart_to_conjugate realpart realroots rearray rectangle rectform rectform_log_if_constant recttopolar rediff reduce_consts reduce_order region region_boundaries region_boundaries_plus rem remainder remarray rembox remcomps remcon remcoord remfun remfunction remlet remove remove_constvalue remove_dimensions remove_edge remove_fundamental_dimensions remove_fundamental_units remove_plot_option remove_vertex rempart remrule remsym remvalue rename rename_file reset reset_displays residue resolvante resolvante_alternee1 resolvante_bipartite resolvante_diedrale resolvante_klein resolvante_klein3 resolvante_produit_sym resolvante_unitaire resolvante_vierer rest resultant return reveal reverse revert revert2 rgb2level rhs ricci riemann rinvariant risch rk rmdir rncombine romberg room rootscontract round row rowop rowswap rreduce run_testsuite %s save saving scalarp scaled_bessel_i scaled_bessel_i0 scaled_bessel_i1 scalefactors scanmap scatterplot scatterplot_description scene schur2comp sconcat scopy scsimp scurvature sdowncase sec sech second sequal sequalignore set_alt_display setdifference set_draw_defaults set_edge_weight setelmx setequalp setify setp set_partitions set_plot_option set_prompt set_random_state set_tex_environment set_tex_environment_default setunits setup_autoload set_up_dot_simplifications set_vertex_label seventh sexplode sf sha1sum sha256sum shortest_path shortest_weighted_path show showcomps showratvars sierpinskiale sierpinskimap sign signum similaritytransform simp_inequality simplify_sum simplode simpmetderiv simtran sin sinh sinsert sinvertcase sixth skewness skewness_bernoulli skewness_beta skewness_binomial skewness_chi2 skewness_continuous_uniform skewness_discrete_uniform skewness_exp skewness_f skewness_gamma skewness_general_finite_discrete skewness_geometric skewness_gumbel skewness_hypergeometric skewness_laplace skewness_logistic skewness_lognormal skewness_negative_binomial skewness_noncentral_chi2 skewness_noncentral_student_t skewness_normal skewness_pareto skewness_poisson skewness_rayleigh skewness_student_t skewness_weibull slength smake small_rhombicosidodecahedron_graph small_rhombicuboctahedron_graph smax smin smismatch snowmap snub_cube_graph snub_dodecahedron_graph solve solve_rec solve_rec_rat some somrac sort sparse6_decode sparse6_encode sparse6_export sparse6_import specint spherical spherical_bessel_j spherical_bessel_y spherical_hankel1 spherical_hankel2 spherical_harmonic spherical_to_xyz splice split sposition sprint sqfr sqrt sqrtdenest sremove sremovefirst sreverse ssearch ssort sstatus ssubst ssubstfirst staircase standardize standardize_inverse_trig starplot starplot_description status std std1 std_bernoulli std_beta std_binomial std_chi2 std_continuous_uniform std_discrete_uniform std_exp std_f std_gamma std_general_finite_discrete std_geometric std_gumbel std_hypergeometric std_laplace std_logistic std_lognormal std_negative_binomial std_noncentral_chi2 std_noncentral_student_t std_normal std_pareto std_poisson std_rayleigh std_student_t std_weibull stemplot stirling stirling1 stirling2 strim striml strimr string stringout stringp strong_components struve_h struve_l sublis sublist sublist_indices submatrix subsample subset subsetp subst substinpart subst_parallel substpart substring subvar subvarp sum sumcontract summand_to_rec supcase supcontext symbolp symmdifference symmetricp system take_channel take_inference tan tanh taylor taylorinfo taylorp taylor_simplifier taytorat tcl_output tcontract tellrat tellsimp tellsimpafter tentex tenth test_mean test_means_difference test_normality test_proportion test_proportions_difference test_rank_sum test_sign test_signed_rank test_variance test_variance_ratio tex tex1 tex_display texput %th third throw time timedate timer timer_info tldefint tlimit todd_coxeter toeplitz tokens to_lisp topological_sort to_poly to_poly_solve totaldisrep totalfourier totient tpartpol trace tracematrix trace_options transform_sample translate translate_file transpose treefale tree_reduce treillis treinat triangle triangularize trigexpand trigrat trigreduce trigsimp trunc truncate truncated_cube_graph truncated_dodecahedron_graph truncated_icosahedron_graph truncated_tetrahedron_graph tr_warnings_get tube tutte_graph ueivects uforget ultraspherical underlying_graph undiff union unique uniteigenvectors unitp units unit_step unitvector unorder unsum untellrat untimer untrace uppercasep uricci uriemann uvect vandermonde_matrix var var1 var_bernoulli var_beta var_binomial var_chi2 var_continuous_uniform var_discrete_uniform var_exp var_f var_gamma var_general_finite_discrete var_geometric var_gumbel var_hypergeometric var_laplace var_logistic var_lognormal var_negative_binomial var_noncentral_chi2 var_noncentral_student_t var_normal var_pareto var_poisson var_rayleigh var_student_t var_weibull vector vectorpotential vectorsimp verbify vers vertex_coloring vertex_connectivity vertex_degree vertex_distance vertex_eccentricity vertex_in_degree vertex_out_degree vertices vertices_to_cycle vertices_to_path %w weyl wheel_graph wiener_index wigner_3j wigner_6j wigner_9j with_stdout write_binary_data writebyte write_data writefile wronskian xreduce xthru %y Zeilberger zeroequiv zerofor zeromatrix zeromatrixp zeta zgeev zheev zlange zn_add_table zn_carmichael_lambda zn_characteristic_factors zn_determinant zn_factor_generators zn_invert_by_lu zn_log zn_mult_table absboxchar activecontexts adapt_depth additive adim aform algebraic algepsilon algexact aliases allbut all_dotsimp_denoms allocation allsym alphabetic animation antisymmetric arrays askexp assume_pos assume_pos_pred assumescalar asymbol atomgrad atrig1 axes axis_3d axis_bottom axis_left axis_right axis_top azimuth background background_color backsubst berlefact bernstein_explicit besselexpand beta_args_sum_to_integer beta_expand bftorat bftrunc bindtest border boundaries_array box boxchar breakup %c capping cauchysum cbrange cbtics center cflength cframe_flag cnonmet_flag color color_bar color_bar_tics colorbox columns commutative complex cone context contexts contour contour_levels cosnpiflag ctaypov ctaypt ctayswitch ctayvar ct_coords ctorsion_flag ctrgsimp cube current_let_rule_package cylinder data_file_name debugmode decreasing default_let_rule_package delay dependencies derivabbrev derivsubst detout diagmetric diff dim dimensions dispflag display2d|10 display_format_internal distribute_over doallmxops domain domxexpt domxmxops domxnctimes dontfactor doscmxops doscmxplus dot0nscsimp dot0simp dot1simp dotassoc dotconstrules dotdistrib dotexptsimp dotident dotscrules draw_graph_program draw_realpart edge_color edge_coloring edge_partition edge_type edge_width %edispflag elevation %emode endphi endtheta engineering_format_floats enhanced3d %enumer epsilon_lp erfflag erf_representation errormsg error_size error_syms error_type %e_to_numlog eval even evenfun evflag evfun ev_point expandwrt_denom expintexpand expintrep expon expop exptdispflag exptisolate exptsubst facexpand facsum_combine factlim factorflag factorial_expand factors_only fb feature features file_name file_output_append file_search_demo file_search_lisp file_search_maxima|10 file_search_tests file_search_usage file_type_lisp file_type_maxima|10 fill_color fill_density filled_func fixed_vertices flipflag float2bf font font_size fortindent fortspaces fpprec fpprintprec functions gamma_expand gammalim gdet genindex gensumnum GGFCFMAX GGFINFINITY globalsolve gnuplot_command gnuplot_curve_styles gnuplot_curve_titles gnuplot_default_term_command gnuplot_dumb_term_command gnuplot_file_args gnuplot_file_name gnuplot_out_file gnuplot_pdf_term_command gnuplot_pm3d gnuplot_png_term_command gnuplot_postamble gnuplot_preamble gnuplot_ps_term_command gnuplot_svg_term_command gnuplot_term gnuplot_view_args Gosper_in_Zeilberger gradefs grid grid2d grind halfangles head_angle head_both head_length head_type height hypergeometric_representation %iargs ibase icc1 icc2 icounter idummyx ieqnprint ifb ifc1 ifc2 ifg ifgi ifr iframe_bracket_form ifri igeowedge_flag ikt1 ikt2 imaginary inchar increasing infeval infinity inflag infolists inm inmc1 inmc2 intanalysis integer integervalued integrate_use_rootsof integration_constant integration_constant_counter interpolate_color intfaclim ip_grid ip_grid_in irrational isolate_wrt_times iterations itr julia_parameter %k1 %k2 keepfloat key key_pos kinvariant kt label label_alignment label_orientation labels lassociative lbfgs_ncorrections lbfgs_nfeval_max leftjust legend letrat let_rule_packages lfg lg lhospitallim limsubst linear linear_solver linechar linel|10 linenum line_type linewidth line_width linsolve_params linsolvewarn lispdisp listarith listconstvars listdummyvars lmxchar load_pathname loadprint logabs logarc logcb logconcoeffp logexpand lognegint logsimp logx logx_secondary logy logy_secondary logz lriem m1pbranch macroexpansion macros mainvar manual_demo maperror mapprint matrix_element_add matrix_element_mult matrix_element_transpose maxapplydepth maxapplyheight maxima_tempdir|10 maxima_userdir|10 maxnegex MAX_ORD maxposex maxpsifracdenom maxpsifracnum maxpsinegint maxpsiposint maxtayorder mesh_lines_color method mod_big_prime mode_check_errorp mode_checkp mode_check_warnp mod_test mod_threshold modular_linear_solver modulus multiplicative multiplicities myoptions nary negdistrib negsumdispflag newline newtonepsilon newtonmaxiter nextlayerfactor niceindicespref nm nmc noeval nolabels nonegative_lp noninteger nonscalar noun noundisp nouns np npi nticks ntrig numer numer_pbranch obase odd oddfun opacity opproperties opsubst optimprefix optionset orientation origin orthopoly_returns_intervals outative outchar packagefile palette partswitch pdf_file pfeformat phiresolution %piargs piece pivot_count_sx pivot_max_sx plot_format plot_options plot_realpart png_file pochhammer_max_index points pointsize point_size points_joined point_type poislim poisson poly_coefficient_ring poly_elimination_order polyfactor poly_grobner_algorithm poly_grobner_debug poly_monomial_order poly_primary_elimination_order poly_return_term_list poly_secondary_elimination_order poly_top_reduction_only posfun position powerdisp pred prederror primep_number_of_tests product_use_gamma program programmode promote_float_to_bigfloat prompt proportional_axes props psexpand ps_file radexpand radius radsubstflag rassociative ratalgdenom ratchristof ratdenomdivide rateinstein ratepsilon ratfac rational ratmx ratprint ratriemann ratsimpexpons ratvarswitch ratweights ratweyl ratwtlvl real realonly redraw refcheck resolution restart resultant ric riem rmxchar %rnum_list rombergabs rombergit rombergmin rombergtol rootsconmode rootsepsilon run_viewer same_xy same_xyz savedef savefactors scalar scalarmatrixp scale scale_lp setcheck setcheckbreak setval show_edge_color show_edges show_edge_type show_edge_width show_id show_label showtime show_vertex_color show_vertex_size show_vertex_type show_vertices show_weight simp simplified_output simplify_products simpproduct simpsum sinnpiflag solvedecomposes solveexplicit solvefactors solvenullwarn solveradcan solvetrigwarn space sparse sphere spring_embedding_depth sqrtdispflag stardisp startphi starttheta stats_numer stringdisp structures style sublis_apply_lambda subnumsimp sumexpand sumsplitfact surface surface_hide svg_file symmetric tab taylordepth taylor_logexpand taylor_order_coefficients taylor_truncate_polynomials tensorkill terminal testsuite_files thetaresolution timer_devalue title tlimswitch tr track transcompile transform transform_xy translate_fast_arrays transparent transrun tr_array_as_ref tr_bound_function_applyp tr_file_tty_messagesp tr_float_can_branch_complex tr_function_call_default trigexpandplus trigexpandtimes triginverses trigsign trivial_solutions tr_numer tr_optimize_max_loop tr_semicompile tr_state_vars tr_warn_bad_function_calls tr_warn_fexpr tr_warn_meval tr_warn_mode tr_warn_undeclared tr_warn_undefined_variable tstep ttyoff tube_extremes ufg ug %unitexpand unit_vectors uric uriem use_fast_arrays user_preamble usersetunits values vect_cross verbose vertex_color vertex_coloring vertex_partition vertex_size vertex_type view warnings weyl width windowname windowtitle wired_surface wireframe xaxis xaxis_color xaxis_secondary xaxis_type xaxis_width xlabel xlabel_secondary xlength xrange xrange_secondary xtics xtics_axis xtics_rotate xtics_rotate_secondary xtics_secondary xtics_secondary_axis xu_grid x_voxel xy_file xyplane xy_scale yaxis yaxis_color yaxis_secondary yaxis_type yaxis_width ylabel ylabel_secondary ylength yrange yrange_secondary ytics ytics_axis ytics_rotate ytics_rotate_secondary ytics_secondary ytics_secondary_axis yv_grid y_voxel yx_ratio zaxis zaxis_color zaxis_type zaxis_width zeroa zerob zerobern zeta%pi zlabel zlabel_rotate zlength zmin zn_primroot_limit zn_primroot_pretest",symbol:"_ __ %|0 %%|0"},contains:[{className:"comment",begin:"/\\*",end:"\\*/",contains:["self"]},e.QUOTE_STRING_MODE,{className:"number",relevance:0,variants:[{begin:"\\b(\\d+|\\d+\\.|\\.\\d+|\\d+\\.\\d+)[Ee][-+]?\\d+\\b"},{begin:"\\b(\\d+|\\d+\\.|\\.\\d+|\\d+\\.\\d+)[Bb][-+]?\\d+\\b",relevance:10},{begin:"\\b(\\.\\d+|\\d+\\.\\d+)\\b"},{begin:"\\b(\\d+|0[0-9A-Za-z]+)\\.?\\b"}]}],illegal:/@/}}},8249(e){e.exports=function(e){const t=e.regex,n=["begin_keywords","celldefine","default_nettype","default_decay_time","default_trireg_strength","define","delay_mode_distributed","delay_mode_path","delay_mode_unit","delay_mode_zero","else","elsif","end_keywords","endcelldefine","endif","ifdef","ifndef","include","line","nounconnected_drive","pragma","resetall","timescale","unconnected_drive","undef","undefineall"];return{name:"Verilog",aliases:["v","sv","svh"],case_insensitive:!1,keywords:{$pattern:/\$?[\w]+(\$[\w]+)*/,keyword:["accept_on","alias","always","always_comb","always_ff","always_latch","and","assert","assign","assume","automatic","before","begin","bind","bins","binsof","bit","break","buf|0","bufif0","bufif1","byte","case","casex","casez","cell","chandle","checker","class","clocking","cmos","config","const","constraint","context","continue","cover","covergroup","coverpoint","cross","deassign","default","defparam","design","disable","dist","do","edge","else","end","endcase","endchecker","endclass","endclocking","endconfig","endfunction","endgenerate","endgroup","endinterface","endmodule","endpackage","endprimitive","endprogram","endproperty","endspecify","endsequence","endtable","endtask","enum","event","eventually","expect","export","extends","extern","final","first_match","for","force","foreach","forever","fork","forkjoin","function","generate|5","genvar","global","highz0","highz1","if","iff","ifnone","ignore_bins","illegal_bins","implements","implies","import","incdir","include","initial","inout","input","inside","instance","int","integer","interconnect","interface","intersect","join","join_any","join_none","large","let","liblist","library","local","localparam","logic","longint","macromodule","matches","medium","modport","module","nand","negedge","nettype","new","nexttime","nmos","nor","noshowcancelled","not","notif0","notif1","or","output","package","packed","parameter","pmos","posedge","primitive","priority","program","property","protected","pull0","pull1","pulldown","pullup","pulsestyle_ondetect","pulsestyle_onevent","pure","rand","randc","randcase","randsequence","rcmos","real","realtime","ref","reg","reject_on","release","repeat","restrict","return","rnmos","rpmos","rtran","rtranif0","rtranif1","s_always","s_eventually","s_nexttime","s_until","s_until_with","scalared","sequence","shortint","shortreal","showcancelled","signed","small","soft","solve","specify","specparam","static","string","strong","strong0","strong1","struct","super","supply0","supply1","sync_accept_on","sync_reject_on","table","tagged","task","this","throughout","time","timeprecision","timeunit","tran","tranif0","tranif1","tri","tri0","tri1","triand","trior","trireg","type","typedef","union","unique","unique0","unsigned","until","until_with","untyped","use","uwire","var","vectored","virtual","void","wait","wait_order","wand","weak","weak0","weak1","while","wildcard","wire","with","within","wor","xnor","xor"],literal:["null"],built_in:["$finish","$stop","$exit","$fatal","$error","$warning","$info","$realtime","$time","$printtimescale","$bitstoreal","$bitstoshortreal","$itor","$signed","$cast","$bits","$stime","$timeformat","$realtobits","$shortrealtobits","$rtoi","$unsigned","$asserton","$assertkill","$assertpasson","$assertfailon","$assertnonvacuouson","$assertoff","$assertcontrol","$assertpassoff","$assertfailoff","$assertvacuousoff","$isunbounded","$sampled","$fell","$changed","$past_gclk","$fell_gclk","$changed_gclk","$rising_gclk","$steady_gclk","$coverage_control","$coverage_get","$coverage_save","$set_coverage_db_name","$rose","$stable","$past","$rose_gclk","$stable_gclk","$future_gclk","$falling_gclk","$changing_gclk","$display","$coverage_get_max","$coverage_merge","$get_coverage","$load_coverage_db","$typename","$unpacked_dimensions","$left","$low","$increment","$clog2","$ln","$log10","$exp","$sqrt","$pow","$floor","$ceil","$sin","$cos","$tan","$countbits","$onehot","$isunknown","$fatal","$warning","$dimensions","$right","$high","$size","$asin","$acos","$atan","$atan2","$hypot","$sinh","$cosh","$tanh","$asinh","$acosh","$atanh","$countones","$onehot0","$error","$info","$random","$dist_chi_square","$dist_erlang","$dist_exponential","$dist_normal","$dist_poisson","$dist_t","$dist_uniform","$q_initialize","$q_remove","$q_exam","$async$and$array","$async$nand$array","$async$or$array","$async$nor$array","$sync$and$array","$sync$nand$array","$sync$or$array","$sync$nor$array","$q_add","$q_full","$psprintf","$async$and$plane","$async$nand$plane","$async$or$plane","$async$nor$plane","$sync$and$plane","$sync$nand$plane","$sync$or$plane","$sync$nor$plane","$system","$display","$displayb","$displayh","$displayo","$strobe","$strobeb","$strobeh","$strobeo","$write","$readmemb","$readmemh","$writememh","$value$plusargs","$dumpvars","$dumpon","$dumplimit","$dumpports","$dumpportson","$dumpportslimit","$writeb","$writeh","$writeo","$monitor","$monitorb","$monitorh","$monitoro","$writememb","$dumpfile","$dumpoff","$dumpall","$dumpflush","$dumpportsoff","$dumpportsall","$dumpportsflush","$fclose","$fdisplay","$fdisplayb","$fdisplayh","$fdisplayo","$fstrobe","$fstrobeb","$fstrobeh","$fstrobeo","$swrite","$swriteb","$swriteh","$swriteo","$fscanf","$fread","$fseek","$fflush","$feof","$fopen","$fwrite","$fwriteb","$fwriteh","$fwriteo","$fmonitor","$fmonitorb","$fmonitorh","$fmonitoro","$sformat","$sformatf","$fgetc","$ungetc","$fgets","$sscanf","$rewind","$ftell","$ferror"]},contains:[e.C_BLOCK_COMMENT_MODE,e.C_LINE_COMMENT_MODE,e.QUOTE_STRING_MODE,{scope:"number",contains:[e.BACKSLASH_ESCAPE],variants:[{begin:/\b((\d+'([bhodBHOD]))[0-9xzXZa-fA-F_]+)/},{begin:/\B(('([bhodBHOD]))[0-9xzXZa-fA-F_]+)/},{begin:/\b[0-9][0-9_]*/,relevance:0}]},{scope:"variable",variants:[{begin:"#\\((?!parameter).+\\)"},{begin:"\\.\\w+",relevance:0}]},{scope:"variable.constant",match:t.concat(/`/,t.either("__FILE__","__LINE__"))},{scope:"meta",begin:t.concat(/`/,t.either(...n)),end:/$|\/\/|\/\*/,returnEnd:!0,keywords:n}]}}},8269(e){e.exports=function(e){return{name:"MIPS Assembly",case_insensitive:!0,aliases:["mips"],keywords:{$pattern:"\\.?"+e.IDENT_RE,meta:".2byte .4byte .align .ascii .asciz .balign .byte .code .data .else .end .endif .endm .endr .equ .err .exitm .extern .global .hword .if .ifdef .ifndef .include .irp .long .macro .rept .req .section .set .skip .space .text .word .ltorg ",built_in:"$0 $1 $2 $3 $4 $5 $6 $7 $8 $9 $10 $11 $12 $13 $14 $15 $16 $17 $18 $19 $20 $21 $22 $23 $24 $25 $26 $27 $28 $29 $30 $31 zero at v0 v1 a0 a1 a2 a3 a4 a5 a6 a7 t0 t1 t2 t3 t4 t5 t6 t7 t8 t9 s0 s1 s2 s3 s4 s5 s6 s7 s8 k0 k1 gp sp fp ra $f0 $f1 $f2 $f2 $f4 $f5 $f6 $f7 $f8 $f9 $f10 $f11 $f12 $f13 $f14 $f15 $f16 $f17 $f18 $f19 $f20 $f21 $f22 $f23 $f24 $f25 $f26 $f27 $f28 $f29 $f30 $f31 Context Random EntryLo0 EntryLo1 Context PageMask Wired EntryHi HWREna BadVAddr Count Compare SR IntCtl SRSCtl SRSMap Cause EPC PRId EBase Config Config1 Config2 Config3 LLAddr Debug DEPC DESAVE CacheErr ECC ErrorEPC TagLo DataLo TagHi DataHi WatchLo WatchHi PerfCtl PerfCnt "},contains:[{className:"keyword",begin:"\\b(addi?u?|andi?|b(al)?|beql?|bgez(al)?l?|bgtzl?|blezl?|bltz(al)?l?|bnel?|cl[oz]|divu?|ext|ins|j(al)?|jalr(\\.hb)?|jr(\\.hb)?|lbu?|lhu?|ll|lui|lw[lr]?|maddu?|mfhi|mflo|movn|movz|move|msubu?|mthi|mtlo|mul|multu?|nop|nor|ori?|rotrv?|sb|sc|se[bh]|sh|sllv?|slti?u?|srav?|srlv?|subu?|sw[lr]?|xori?|wsbh|abs\\.[sd]|add\\.[sd]|alnv.ps|bc1[ft]l?|c\\.(s?f|un|u?eq|[ou]lt|[ou]le|ngle?|seq|l[et]|ng[et])\\.[sd]|(ceil|floor|round|trunc)\\.[lw]\\.[sd]|cfc1|cvt\\.d\\.[lsw]|cvt\\.l\\.[dsw]|cvt\\.ps\\.s|cvt\\.s\\.[dlw]|cvt\\.s\\.p[lu]|cvt\\.w\\.[dls]|div\\.[ds]|ldx?c1|luxc1|lwx?c1|madd\\.[sd]|mfc1|mov[fntz]?\\.[ds]|msub\\.[sd]|mth?c1|mul\\.[ds]|neg\\.[ds]|nmadd\\.[ds]|nmsub\\.[ds]|p[lu][lu]\\.ps|recip\\.fmt|r?sqrt\\.[ds]|sdx?c1|sub\\.[ds]|suxc1|swx?c1|break|cache|d?eret|[de]i|ehb|mfc0|mtc0|pause|prefx?|rdhwr|rdpgpr|sdbbp|ssnop|synci?|syscall|teqi?|tgei?u?|tlb(p|r|w[ir])|tlti?u?|tnei?|wait|wrpgpr)",end:"\\s"},e.COMMENT("[;#](?!\\s*$)","$"),e.C_BLOCK_COMMENT_MODE,e.QUOTE_STRING_MODE,{className:"string",begin:"'",end:"[^\\\\]'",relevance:0},{className:"title",begin:"\\|",end:"\\|",illegal:"\\n",relevance:0},{className:"number",variants:[{begin:"0x[0-9a-f]+"},{begin:"\\b-?\\d+"}],relevance:0},{className:"symbol",variants:[{begin:"^\\s*[a-z_\\.\\$][a-z0-9_\\.\\$]+:"},{begin:"^\\s*[0-9]+:"},{begin:"[0-9]+[bf]"}],relevance:0}],illegal:/\//}}},8284(e){e.exports=function(e){return{keywords:"dsconfig",contains:[{className:"keyword",begin:"^dsconfig",end:/\s/,excludeEnd:!0,relevance:10},{className:"built_in",begin:/(list|create|get|set|delete)-(\w+)/,end:/\s/,excludeEnd:!0,illegal:"!@#$%^&*()",relevance:10},{className:"built_in",begin:/--(\w+)/,end:/\s/,excludeEnd:!0},{className:"string",begin:/"/,end:/"/},{className:"string",begin:/'/,end:/'/},{className:"string",begin:/[\w\-?]+:\w+/,end:/\W/,relevance:0},{className:"string",begin:/\w+(\-\w+)*/,end:/(?=\W)/,relevance:0},e.HASH_COMMENT_MODE]}}},8330(e){const t=["a","abbr","address","article","aside","audio","b","blockquote","body","button","canvas","caption","cite","code","dd","del","details","dfn","div","dl","dt","em","fieldset","figcaption","figure","footer","form","h1","h2","h3","h4","h5","h6","header","hgroup","html","i","iframe","img","input","ins","kbd","label","legend","li","main","mark","menu","nav","object","ol","optgroup","option","p","picture","q","quote","samp","section","select","source","span","strong","summary","sup","table","tbody","td","textarea","tfoot","th","thead","time","tr","ul","var","video","defs","g","marker","mask","pattern","svg","switch","symbol","feBlend","feColorMatrix","feComponentTransfer","feComposite","feConvolveMatrix","feDiffuseLighting","feDisplacementMap","feFlood","feGaussianBlur","feImage","feMerge","feMorphology","feOffset","feSpecularLighting","feTile","feTurbulence","linearGradient","radialGradient","stop","circle","ellipse","image","line","path","polygon","polyline","rect","text","use","textPath","tspan","foreignObject","clipPath"],n=["any-hover","any-pointer","aspect-ratio","color","color-gamut","color-index","device-aspect-ratio","device-height","device-width","display-mode","forced-colors","grid","height","hover","inverted-colors","monochrome","orientation","overflow-block","overflow-inline","pointer","prefers-color-scheme","prefers-contrast","prefers-reduced-motion","prefers-reduced-transparency","resolution","scan","scripting","update","width","min-width","max-width","min-height","max-height"].sort().reverse(),a=["active","any-link","blank","checked","current","default","defined","dir","disabled","drop","empty","enabled","first","first-child","first-of-type","fullscreen","future","focus","focus-visible","focus-within","has","host","host-context","hover","indeterminate","in-range","invalid","is","lang","last-child","last-of-type","left","link","local-link","not","nth-child","nth-col","nth-last-child","nth-last-col","nth-last-of-type","nth-of-type","only-child","only-of-type","optional","out-of-range","past","placeholder-shown","read-only","read-write","required","right","root","scope","target","target-within","user-invalid","valid","visited","where"].sort().reverse(),r=["after","backdrop","before","cue","cue-region","first-letter","first-line","grammar-error","marker","part","placeholder","selection","slotted","spelling-error"].sort().reverse(),i=["accent-color","align-content","align-items","align-self","alignment-baseline","all","anchor-name","animation","animation-composition","animation-delay","animation-direction","animation-duration","animation-fill-mode","animation-iteration-count","animation-name","animation-play-state","animation-range","animation-range-end","animation-range-start","animation-timeline","animation-timing-function","appearance","aspect-ratio","backdrop-filter","backface-visibility","background","background-attachment","background-blend-mode","background-clip","background-color","background-image","background-origin","background-position","background-position-x","background-position-y","background-repeat","background-size","baseline-shift","block-size","border","border-block","border-block-color","border-block-end","border-block-end-color","border-block-end-style","border-block-end-width","border-block-start","border-block-start-color","border-block-start-style","border-block-start-width","border-block-style","border-block-width","border-bottom","border-bottom-color","border-bottom-left-radius","border-bottom-right-radius","border-bottom-style","border-bottom-width","border-collapse","border-color","border-end-end-radius","border-end-start-radius","border-image","border-image-outset","border-image-repeat","border-image-slice","border-image-source","border-image-width","border-inline","border-inline-color","border-inline-end","border-inline-end-color","border-inline-end-style","border-inline-end-width","border-inline-start","border-inline-start-color","border-inline-start-style","border-inline-start-width","border-inline-style","border-inline-width","border-left","border-left-color","border-left-style","border-left-width","border-radius","border-right","border-right-color","border-right-style","border-right-width","border-spacing","border-start-end-radius","border-start-start-radius","border-style","border-top","border-top-color","border-top-left-radius","border-top-right-radius","border-top-style","border-top-width","border-width","bottom","box-align","box-decoration-break","box-direction","box-flex","box-flex-group","box-lines","box-ordinal-group","box-orient","box-pack","box-shadow","box-sizing","break-after","break-before","break-inside","caption-side","caret-color","clear","clip","clip-path","clip-rule","color","color-interpolation","color-interpolation-filters","color-profile","color-rendering","color-scheme","column-count","column-fill","column-gap","column-rule","column-rule-color","column-rule-style","column-rule-width","column-span","column-width","columns","contain","contain-intrinsic-block-size","contain-intrinsic-height","contain-intrinsic-inline-size","contain-intrinsic-size","contain-intrinsic-width","container","container-name","container-type","content","content-visibility","counter-increment","counter-reset","counter-set","cue","cue-after","cue-before","cursor","cx","cy","direction","display","dominant-baseline","empty-cells","enable-background","field-sizing","fill","fill-opacity","fill-rule","filter","flex","flex-basis","flex-direction","flex-flow","flex-grow","flex-shrink","flex-wrap","float","flood-color","flood-opacity","flow","font","font-display","font-family","font-feature-settings","font-kerning","font-language-override","font-optical-sizing","font-palette","font-size","font-size-adjust","font-smooth","font-smoothing","font-stretch","font-style","font-synthesis","font-synthesis-position","font-synthesis-small-caps","font-synthesis-style","font-synthesis-weight","font-variant","font-variant-alternates","font-variant-caps","font-variant-east-asian","font-variant-emoji","font-variant-ligatures","font-variant-numeric","font-variant-position","font-variation-settings","font-weight","forced-color-adjust","gap","glyph-orientation-horizontal","glyph-orientation-vertical","grid","grid-area","grid-auto-columns","grid-auto-flow","grid-auto-rows","grid-column","grid-column-end","grid-column-start","grid-gap","grid-row","grid-row-end","grid-row-start","grid-template","grid-template-areas","grid-template-columns","grid-template-rows","hanging-punctuation","height","hyphenate-character","hyphenate-limit-chars","hyphens","icon","image-orientation","image-rendering","image-resolution","ime-mode","initial-letter","initial-letter-align","inline-size","inset","inset-area","inset-block","inset-block-end","inset-block-start","inset-inline","inset-inline-end","inset-inline-start","isolation","justify-content","justify-items","justify-self","kerning","left","letter-spacing","lighting-color","line-break","line-height","line-height-step","list-style","list-style-image","list-style-position","list-style-type","margin","margin-block","margin-block-end","margin-block-start","margin-bottom","margin-inline","margin-inline-end","margin-inline-start","margin-left","margin-right","margin-top","margin-trim","marker","marker-end","marker-mid","marker-start","marks","mask","mask-border","mask-border-mode","mask-border-outset","mask-border-repeat","mask-border-slice","mask-border-source","mask-border-width","mask-clip","mask-composite","mask-image","mask-mode","mask-origin","mask-position","mask-repeat","mask-size","mask-type","masonry-auto-flow","math-depth","math-shift","math-style","max-block-size","max-height","max-inline-size","max-width","min-block-size","min-height","min-inline-size","min-width","mix-blend-mode","nav-down","nav-index","nav-left","nav-right","nav-up","none","normal","object-fit","object-position","offset","offset-anchor","offset-distance","offset-path","offset-position","offset-rotate","opacity","order","orphans","outline","outline-color","outline-offset","outline-style","outline-width","overflow","overflow-anchor","overflow-block","overflow-clip-margin","overflow-inline","overflow-wrap","overflow-x","overflow-y","overlay","overscroll-behavior","overscroll-behavior-block","overscroll-behavior-inline","overscroll-behavior-x","overscroll-behavior-y","padding","padding-block","padding-block-end","padding-block-start","padding-bottom","padding-inline","padding-inline-end","padding-inline-start","padding-left","padding-right","padding-top","page","page-break-after","page-break-before","page-break-inside","paint-order","pause","pause-after","pause-before","perspective","perspective-origin","place-content","place-items","place-self","pointer-events","position","position-anchor","position-visibility","print-color-adjust","quotes","r","resize","rest","rest-after","rest-before","right","rotate","row-gap","ruby-align","ruby-position","scale","scroll-behavior","scroll-margin","scroll-margin-block","scroll-margin-block-end","scroll-margin-block-start","scroll-margin-bottom","scroll-margin-inline","scroll-margin-inline-end","scroll-margin-inline-start","scroll-margin-left","scroll-margin-right","scroll-margin-top","scroll-padding","scroll-padding-block","scroll-padding-block-end","scroll-padding-block-start","scroll-padding-bottom","scroll-padding-inline","scroll-padding-inline-end","scroll-padding-inline-start","scroll-padding-left","scroll-padding-right","scroll-padding-top","scroll-snap-align","scroll-snap-stop","scroll-snap-type","scroll-timeline","scroll-timeline-axis","scroll-timeline-name","scrollbar-color","scrollbar-gutter","scrollbar-width","shape-image-threshold","shape-margin","shape-outside","shape-rendering","speak","speak-as","src","stop-color","stop-opacity","stroke","stroke-dasharray","stroke-dashoffset","stroke-linecap","stroke-linejoin","stroke-miterlimit","stroke-opacity","stroke-width","tab-size","table-layout","text-align","text-align-all","text-align-last","text-anchor","text-combine-upright","text-decoration","text-decoration-color","text-decoration-line","text-decoration-skip","text-decoration-skip-ink","text-decoration-style","text-decoration-thickness","text-emphasis","text-emphasis-color","text-emphasis-position","text-emphasis-style","text-indent","text-justify","text-orientation","text-overflow","text-rendering","text-shadow","text-size-adjust","text-transform","text-underline-offset","text-underline-position","text-wrap","text-wrap-mode","text-wrap-style","timeline-scope","top","touch-action","transform","transform-box","transform-origin","transform-style","transition","transition-behavior","transition-delay","transition-duration","transition-property","transition-timing-function","translate","unicode-bidi","user-modify","user-select","vector-effect","vertical-align","view-timeline","view-timeline-axis","view-timeline-inset","view-timeline-name","view-transition-name","visibility","voice-balance","voice-duration","voice-family","voice-pitch","voice-range","voice-rate","voice-stress","voice-volume","white-space","white-space-collapse","widows","width","will-change","word-break","word-spacing","word-wrap","writing-mode","x","y","z-index","zoom"].sort().reverse(),o=a.concat(r).sort().reverse();e.exports=function(e){const s=(e=>({IMPORTANT:{scope:"meta",begin:"!important"},BLOCK_COMMENT:e.C_BLOCK_COMMENT_MODE,HEXCOLOR:{scope:"number",begin:/#(([0-9a-fA-F]{3,4})|(([0-9a-fA-F]{2}){3,4}))\b/},FUNCTION_DISPATCH:{className:"built_in",begin:/[\w-]+(?=\()/},ATTRIBUTE_SELECTOR_MODE:{scope:"selector-attr",begin:/\[/,end:/\]/,illegal:"$",contains:[e.APOS_STRING_MODE,e.QUOTE_STRING_MODE]},CSS_NUMBER_MODE:{scope:"number",begin:e.NUMBER_RE+"(%|em|ex|ch|rem|vw|vh|vmin|vmax|cm|mm|in|pt|pc|px|deg|grad|rad|turn|s|ms|Hz|kHz|dpi|dpcm|dppx)?",relevance:0},CSS_VARIABLE:{className:"attr",begin:/--[A-Za-z_][A-Za-z0-9_-]*/}}))(e),l=o,c="[\\w-]+",_="("+c+"|@\\{"+c+"\\})",d=[],p=[],u=function(e){return{className:"string",begin:"~?"+e+".*?"+e}},m=function(e,t,n){return{className:e,begin:t,relevance:n}},g={$pattern:/[a-z-]+/,keyword:"and or not only",attribute:n.join(" ")},E={begin:"\\(",end:"\\)",contains:p,keywords:g,relevance:0};p.push(e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE,u("'"),u('"'),s.CSS_NUMBER_MODE,{begin:"(url|data-uri)\\(",starts:{className:"string",end:"[\\)\\n]",excludeEnd:!0}},s.HEXCOLOR,E,m("variable","@@?"+c,10),m("variable","@\\{"+c+"\\}"),m("built_in","~?`[^`]*?`"),{className:"attribute",begin:c+"\\s*:",end:":",returnBegin:!0,excludeEnd:!0},s.IMPORTANT,{beginKeywords:"and not"},s.FUNCTION_DISPATCH);const S=p.concat({begin:/\{/,end:/\}/,contains:d}),b={beginKeywords:"when",endsWithParent:!0,contains:[{beginKeywords:"and not"}].concat(p)},f={begin:_+"\\s*:",returnBegin:!0,end:/[;}]/,relevance:0,contains:[{begin:/-(webkit|moz|ms|o)-/},s.CSS_VARIABLE,{className:"attribute",begin:"\\b("+i.join("|")+")\\b",end:/(?=:)/,starts:{endsWithParent:!0,illegal:"[<=$]",relevance:0,contains:p}}]},T={className:"keyword",begin:"@(import|media|charset|font-face|(-[a-z]+-)?keyframes|supports|document|namespace|page|viewport|host)\\b",starts:{end:"[;{}]",keywords:g,returnEnd:!0,contains:p,relevance:0}},h={className:"variable",variants:[{begin:"@"+c+"\\s*:",relevance:15},{begin:"@"+c}],starts:{end:"[;}]",returnEnd:!0,contains:S}},C={variants:[{begin:"[\\.#:&\\[>]",end:"[;{}]"},{begin:_,end:/\{/}],returnBegin:!0,returnEnd:!0,illegal:"[<='$\"]",relevance:0,contains:[e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE,b,m("keyword","all\\b"),m("variable","@\\{"+c+"\\}"),{begin:"\\b("+t.join("|")+")\\b",className:"selector-tag"},s.CSS_NUMBER_MODE,m("selector-tag",_,0),m("selector-id","#"+_),m("selector-class","\\."+_,0),m("selector-tag","&",0),s.ATTRIBUTE_SELECTOR_MODE,{className:"selector-pseudo",begin:":("+a.join("|")+")"},{className:"selector-pseudo",begin:":(:)?("+r.join("|")+")"},{begin:/\(/,end:/\)/,relevance:0,contains:S},{begin:"!important"},s.FUNCTION_DISPATCH]},O={begin:c+":(:)?"+`(${l.join("|")})`,returnBegin:!0,contains:[C]};return d.push(e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE,T,h,O,f,C,b,s.FUNCTION_DISPATCH),{name:"Less",case_insensitive:!0,illegal:"[=>'/<($\"]",contains:d}}},8340(e){e.exports=function(e){const t=e.COMMENT(/\(\*/,/\*\)/);return{name:"Extended Backus-Naur Form",illegal:/\S/,contains:[t,{className:"attribute",begin:/^[ ]*[a-zA-Z]+([\s_-]+[a-zA-Z]+)*/},{begin:/=/,end:/[.;]/,contains:[t,{className:"meta",begin:/\?.*\?/},{className:"string",variants:[e.APOS_STRING_MODE,e.QUOTE_STRING_MODE,{begin:"`",end:"`"}]}]}]}}},8364(e){const t=["as","in","of","if","for","while","finally","var","new","function","do","return","void","else","break","catch","instanceof","with","throw","case","default","try","switch","continue","typeof","delete","let","yield","const","class","debugger","async","await","static","import","from","export","extends","using"],n=["true","false","null","undefined","NaN","Infinity"],a=[].concat(["setInterval","setTimeout","clearInterval","clearTimeout","require","exports","eval","isFinite","isNaN","parseFloat","parseInt","decodeURI","decodeURIComponent","encodeURI","encodeURIComponent","escape","unescape"],["Object","Function","Boolean","Symbol","Math","Date","Number","BigInt","String","RegExp","Array","Float32Array","Float64Array","Int8Array","Uint8Array","Uint8ClampedArray","Int16Array","Int32Array","Uint16Array","Uint32Array","BigInt64Array","BigUint64Array","Set","Map","WeakSet","WeakMap","ArrayBuffer","SharedArrayBuffer","Atomics","DataView","JSON","Promise","Generator","GeneratorFunction","AsyncFunction","Reflect","Proxy","Intl","WebAssembly"],["Error","EvalError","InternalError","RangeError","ReferenceError","SyntaxError","TypeError","URIError"]);e.exports=function(e){const r={keyword:t.concat(["then","unless","until","loop","by","when","and","or","is","isnt","not"]).filter((i=["var","const","let","function","static"],e=>!i.includes(e))),literal:n.concat(["yes","no","on","off"]),built_in:a.concat(["npm","print"])};var i;const o="[A-Za-z$_][0-9A-Za-z$_]*",s={className:"subst",begin:/#\{/,end:/\}/,keywords:r},l=[e.BINARY_NUMBER_MODE,e.inherit(e.C_NUMBER_MODE,{starts:{end:"(\\s*/)?",relevance:0}}),{className:"string",variants:[{begin:/'''/,end:/'''/,contains:[e.BACKSLASH_ESCAPE]},{begin:/'/,end:/'/,contains:[e.BACKSLASH_ESCAPE]},{begin:/"""/,end:/"""/,contains:[e.BACKSLASH_ESCAPE,s]},{begin:/"/,end:/"/,contains:[e.BACKSLASH_ESCAPE,s]}]},{className:"regexp",variants:[{begin:"///",end:"///",contains:[s,e.HASH_COMMENT_MODE]},{begin:"//[gim]{0,3}(?=\\W)",relevance:0},{begin:/\/(?![ *]).*?(?![\\]).\/[gim]{0,3}(?=\W)/}]},{begin:"@"+o},{subLanguage:"javascript",excludeBegin:!0,excludeEnd:!0,variants:[{begin:"```",end:"```"},{begin:"`",end:"`"}]}];s.contains=l;const c=e.inherit(e.TITLE_MODE,{begin:o}),_="(\\(.*\\)\\s*)?\\B[-=]>",d={className:"params",begin:"\\([^\\(]",returnBegin:!0,contains:[{begin:/\(/,end:/\)/,keywords:r,contains:["self"].concat(l)}]},p={variants:[{match:[/class\s+/,o,/\s+extends\s+/,o]},{match:[/class\s+/,o]}],scope:{2:"title.class",4:"title.class.inherited"},keywords:r};return{name:"CoffeeScript",aliases:["coffee","cson","iced"],keywords:r,illegal:/\/\*/,contains:[...l,e.COMMENT("###","###"),e.HASH_COMMENT_MODE,{className:"function",begin:"^\\s*"+o+"\\s*=\\s*"+_,end:"[-=]>",returnBegin:!0,contains:[c,d]},{begin:/[:\(,=]\s*/,relevance:0,contains:[{className:"function",begin:_,end:"[-=]>",returnBegin:!0,contains:[d]}]},p,{begin:o+":",end:":",returnBegin:!0,returnEnd:!0,relevance:0}]}}},8416(e){function t(e){return e instanceof Map?e.clear=e.delete=e.set=function(){throw new Error("map is read-only")}:e instanceof Set&&(e.add=e.clear=e.delete=function(){throw new Error("set is read-only")}),Object.freeze(e),Object.getOwnPropertyNames(e).forEach((n=>{const a=e[n],r=typeof a;"object"!==r&&"function"!==r||Object.isFrozen(a)||t(a)})),e}class n{constructor(e){void 0===e.data&&(e.data={}),this.data=e.data,this.isMatchIgnored=!1}ignoreMatch(){this.isMatchIgnored=!0}}function a(e){return e.replace(/&/g,"&").replace(//g,">").replace(/"/g,""").replace(/'/g,"'")}function r(e,...t){const n=Object.create(null);for(const t in e)n[t]=e[t];return t.forEach((function(e){for(const t in e)n[t]=e[t]})),n}const i=e=>!!e.scope;class o{constructor(e,t){this.buffer="",this.classPrefix=t.classPrefix,e.walk(this)}addText(e){this.buffer+=a(e)}openNode(e){if(!i(e))return;const t=((e,{prefix:t})=>{if(e.startsWith("language:"))return e.replace("language:","language-");if(e.includes(".")){const n=e.split(".");return[`${t}${n.shift()}`,...n.map(((e,t)=>`${e}${"_".repeat(t+1)}`))].join(" ")}return`${t}${e}`})(e.scope,{prefix:this.classPrefix});this.span(t)}closeNode(e){i(e)&&(this.buffer+="")}value(){return this.buffer}span(e){this.buffer+=``}}const s=(e={})=>{const t={children:[]};return Object.assign(t,e),t};class l{constructor(){this.rootNode=s(),this.stack=[this.rootNode]}get top(){return this.stack[this.stack.length-1]}get root(){return this.rootNode}add(e){this.top.children.push(e)}openNode(e){const t=s({scope:e});this.add(t),this.stack.push(t)}closeNode(){if(this.stack.length>1)return this.stack.pop()}closeAllNodes(){for(;this.closeNode(););}toJSON(){return JSON.stringify(this.rootNode,null,4)}walk(e){return this.constructor._walk(e,this.rootNode)}static _walk(e,t){return"string"==typeof t?e.addText(t):t.children&&(e.openNode(t),t.children.forEach((t=>this._walk(e,t))),e.closeNode(t)),e}static _collapse(e){"string"!=typeof e&&e.children&&(e.children.every((e=>"string"==typeof e))?e.children=[e.children.join("")]:e.children.forEach((e=>{l._collapse(e)})))}}class c extends l{constructor(e){super(),this.options=e}addText(e){""!==e&&this.add(e)}startScope(e){this.openNode(e)}endScope(){this.closeNode()}__addSublanguage(e,t){const n=e.root;t&&(n.scope=`language:${t}`),this.add(n)}toHTML(){return new o(this,this.options).value()}finalize(){return this.closeAllNodes(),!0}}function _(e){return e?"string"==typeof e?e:e.source:null}function d(e){return m("(?=",e,")")}function p(e){return m("(?:",e,")*")}function u(e){return m("(?:",e,")?")}function m(...e){return e.map((e=>_(e))).join("")}function g(...e){const t=function(e){const t=e[e.length-1];return"object"==typeof t&&t.constructor===Object?(e.splice(e.length-1,1),t):{}}(e);return"("+(t.capture?"":"?:")+e.map((e=>_(e))).join("|")+")"}function E(e){return new RegExp(e.toString()+"|").exec("").length-1}const S=/\[(?:[^\\\]]|\\.)*\]|\(\??|\\([1-9][0-9]*)|\\./;function b(e,{joinWith:t}){let n=0;return e.map((e=>{n+=1;const t=n;let a=_(e),r="";for(;a.length>0;){const e=S.exec(a);if(!e){r+=a;break}r+=a.substring(0,e.index),a=a.substring(e.index+e[0].length),"\\"===e[0][0]&&e[1]?r+="\\"+String(Number(e[1])+t):(r+=e[0],"("===e[0]&&n++)}return r})).map((e=>`(${e})`)).join(t)}const f="[a-zA-Z]\\w*",T="[a-zA-Z_]\\w*",h="\\b\\d+(\\.\\d+)?",C="(-?)(\\b0[xX][a-fA-F0-9]+|(\\b\\d+(\\.\\d*)?|\\.\\d+)([eE][-+]?\\d+)?)",O="\\b(0b[01]+)",N={begin:"\\\\[\\s\\S]",relevance:0},R={scope:"string",begin:"'",end:"'",illegal:"\\n",contains:[N]},v={scope:"string",begin:'"',end:'"',illegal:"\\n",contains:[N]},y=function(e,t,n={}){const a=r({scope:"comment",begin:e,end:t,contains:[]},n);a.contains.push({scope:"doctag",begin:"[ ]*(?=(TODO|FIXME|NOTE|BUG|OPTIMIZE|HACK|XXX):)",end:/(TODO|FIXME|NOTE|BUG|OPTIMIZE|HACK|XXX):/,excludeBegin:!0,relevance:0});const i=g("I","a","is","so","us","to","at","if","in","it","on",/[A-Za-z]+['](d|ve|re|ll|t|s|n)/,/[A-Za-z]+[-][a-z]+/,/[A-Za-z][a-z]{2,}/);return a.contains.push({begin:m(/[ ]+/,"(",i,/[.]?[:]?([.][ ]|[ ])/,"){3}")}),a},A=y("//","$"),I=y("/\\*","\\*/"),D=y("#","$"),x={scope:"number",begin:h,relevance:0},w={scope:"number",begin:C,relevance:0},M={scope:"number",begin:O,relevance:0},L={scope:"regexp",begin:/\/(?=[^/\n]*\/)/,end:/\/[gimuy]*/,contains:[N,{begin:/\[/,end:/\]/,relevance:0,contains:[N]}]},P={scope:"title",begin:f,relevance:0},k={scope:"title",begin:T,relevance:0},F={begin:"\\.\\s*"+T,relevance:0};var U=Object.freeze({__proto__:null,APOS_STRING_MODE:R,BACKSLASH_ESCAPE:N,BINARY_NUMBER_MODE:M,BINARY_NUMBER_RE:O,COMMENT:y,C_BLOCK_COMMENT_MODE:I,C_LINE_COMMENT_MODE:A,C_NUMBER_MODE:w,C_NUMBER_RE:C,END_SAME_AS_BEGIN:function(e){return Object.assign(e,{"on:begin":(e,t)=>{t.data._beginMatch=e[1]},"on:end":(e,t)=>{t.data._beginMatch!==e[1]&&t.ignoreMatch()}})},HASH_COMMENT_MODE:D,IDENT_RE:f,MATCH_NOTHING_RE:/\b\B/,METHOD_GUARD:F,NUMBER_MODE:x,NUMBER_RE:h,PHRASAL_WORDS_MODE:{begin:/\b(a|an|the|are|I'm|isn't|don't|doesn't|won't|but|just|should|pretty|simply|enough|gonna|going|wtf|so|such|will|you|your|they|like|more)\b/},QUOTE_STRING_MODE:v,REGEXP_MODE:L,RE_STARTERS_RE:"!|!=|!==|%|%=|&|&&|&=|\\*|\\*=|\\+|\\+=|,|-|-=|/=|/|:|;|<<|<<=|<=|<|===|==|=|>>>=|>>=|>=|>>>|>>|>|\\?|\\[|\\{|\\(|\\^|\\^=|\\||\\|=|\\|\\||~",SHEBANG:(e={})=>{const t=/^#![ ]*\//;return e.binary&&(e.begin=m(t,/.*\b/,e.binary,/\b.*/)),r({scope:"meta",begin:t,end:/$/,relevance:0,"on:begin":(e,t)=>{0!==e.index&&t.ignoreMatch()}},e)},TITLE_MODE:P,UNDERSCORE_IDENT_RE:T,UNDERSCORE_TITLE_MODE:k});function B(e,t){"."===e.input[e.index-1]&&t.ignoreMatch()}function G(e,t){void 0!==e.className&&(e.scope=e.className,delete e.className)}function Y(e,t){t&&e.beginKeywords&&(e.begin="\\b("+e.beginKeywords.split(" ").join("|")+")(?!\\.)(?=\\b|\\s)",e.__beforeBegin=B,e.keywords=e.keywords||e.beginKeywords,delete e.beginKeywords,void 0===e.relevance&&(e.relevance=0))}function H(e,t){Array.isArray(e.illegal)&&(e.illegal=g(...e.illegal))}function V(e,t){if(e.match){if(e.begin||e.end)throw new Error("begin & end are not supported with match");e.begin=e.match,delete e.match}}function q(e,t){void 0===e.relevance&&(e.relevance=1)}const z=(e,t)=>{if(!e.beforeMatch)return;if(e.starts)throw new Error("beforeMatch cannot be used with starts");const n=Object.assign({},e);Object.keys(e).forEach((t=>{delete e[t]})),e.keywords=n.keywords,e.begin=m(n.beforeMatch,d(n.begin)),e.starts={relevance:0,contains:[Object.assign(n,{endsParent:!0})]},e.relevance=0,delete n.beforeMatch},j=["of","and","for","in","not","or","if","then","parent","list","value"];function $(e,t,n="keyword"){const a=Object.create(null);return"string"==typeof e?r(n,e.split(" ")):Array.isArray(e)?r(n,e):Object.keys(e).forEach((function(n){Object.assign(a,$(e[n],t,n))})),a;function r(e,n){t&&(n=n.map((e=>e.toLowerCase()))),n.forEach((function(t){const n=t.split("|");a[n[0]]=[e,W(n[0],n[1])]}))}}function W(e,t){return t?Number(t):function(e){return j.includes(e.toLowerCase())}(e)?0:1}const Q={},K=e=>{console.error(e)},X=(e,...t)=>{console.log(`WARN: ${e}`,...t)},Z=(e,t)=>{Q[`${e}/${t}`]||(console.log(`Deprecated as of ${e}. ${t}`),Q[`${e}/${t}`]=!0)},J=new Error;function ee(e,t,{key:n}){let a=0;const r=e[n],i={},o={};for(let e=1;e<=t.length;e++)o[e+a]=r[e],i[e+a]=!0,a+=E(t[e-1]);e[n]=o,e[n]._emit=i,e[n]._multi=!0}function te(e){!function(e){e.scope&&"object"==typeof e.scope&&null!==e.scope&&(e.beginScope=e.scope,delete e.scope)}(e),"string"==typeof e.beginScope&&(e.beginScope={_wrap:e.beginScope}),"string"==typeof e.endScope&&(e.endScope={_wrap:e.endScope}),function(e){if(Array.isArray(e.begin)){if(e.skip||e.excludeBegin||e.returnBegin)throw K("skip, excludeBegin, returnBegin not compatible with beginScope: {}"),J;if("object"!=typeof e.beginScope||null===e.beginScope)throw K("beginScope must be object"),J;ee(e,e.begin,{key:"beginScope"}),e.begin=b(e.begin,{joinWith:""})}}(e),function(e){if(Array.isArray(e.end)){if(e.skip||e.excludeEnd||e.returnEnd)throw K("skip, excludeEnd, returnEnd not compatible with endScope: {}"),J;if("object"!=typeof e.endScope||null===e.endScope)throw K("endScope must be object"),J;ee(e,e.end,{key:"endScope"}),e.end=b(e.end,{joinWith:""})}}(e)}function ne(e){function t(t,n){return new RegExp(_(t),"m"+(e.case_insensitive?"i":"")+(e.unicodeRegex?"u":"")+(n?"g":""))}class n{constructor(){this.matchIndexes={},this.regexes=[],this.matchAt=1,this.position=0}addRule(e,t){t.position=this.position++,this.matchIndexes[this.matchAt]=t,this.regexes.push([t,e]),this.matchAt+=E(e)+1}compile(){0===this.regexes.length&&(this.exec=()=>null);const e=this.regexes.map((e=>e[1]));this.matcherRe=t(b(e,{joinWith:"|"}),!0),this.lastIndex=0}exec(e){this.matcherRe.lastIndex=this.lastIndex;const t=this.matcherRe.exec(e);if(!t)return null;const n=t.findIndex(((e,t)=>t>0&&void 0!==e)),a=this.matchIndexes[n];return t.splice(0,n),Object.assign(t,a)}}class a{constructor(){this.rules=[],this.multiRegexes=[],this.count=0,this.lastIndex=0,this.regexIndex=0}getMatcher(e){if(this.multiRegexes[e])return this.multiRegexes[e];const t=new n;return this.rules.slice(e).forEach((([e,n])=>t.addRule(e,n))),t.compile(),this.multiRegexes[e]=t,t}resumingScanAtSamePosition(){return 0!==this.regexIndex}considerAll(){this.regexIndex=0}addRule(e,t){this.rules.push([e,t]),"begin"===t.type&&this.count++}exec(e){const t=this.getMatcher(this.regexIndex);t.lastIndex=this.lastIndex;let n=t.exec(e);if(this.resumingScanAtSamePosition())if(n&&n.index===this.lastIndex);else{const t=this.getMatcher(0);t.lastIndex=this.lastIndex+1,n=t.exec(e)}return n&&(this.regexIndex+=n.position+1,this.regexIndex===this.count&&this.considerAll()),n}}if(e.compilerExtensions||(e.compilerExtensions=[]),e.contains&&e.contains.includes("self"))throw new Error("ERR: contains `self` is not supported at the top-level of a language. See documentation.");return e.classNameAliases=r(e.classNameAliases||{}),function n(i,o){const s=i;if(i.isCompiled)return s;[G,V,te,z].forEach((e=>e(i,o))),e.compilerExtensions.forEach((e=>e(i,o))),i.__beforeBegin=null,[Y,H,q].forEach((e=>e(i,o))),i.isCompiled=!0;let l=null;return"object"==typeof i.keywords&&i.keywords.$pattern&&(i.keywords=Object.assign({},i.keywords),l=i.keywords.$pattern,delete i.keywords.$pattern),l=l||/\w+/,i.keywords&&(i.keywords=$(i.keywords,e.case_insensitive)),s.keywordPatternRe=t(l,!0),o&&(i.begin||(i.begin=/\B|\b/),s.beginRe=t(s.begin),i.end||i.endsWithParent||(i.end=/\B|\b/),i.end&&(s.endRe=t(s.end)),s.terminatorEnd=_(s.end)||"",i.endsWithParent&&o.terminatorEnd&&(s.terminatorEnd+=(i.end?"|":"")+o.terminatorEnd)),i.illegal&&(s.illegalRe=t(i.illegal)),i.contains||(i.contains=[]),i.contains=[].concat(...i.contains.map((function(e){return function(e){return e.variants&&!e.cachedVariants&&(e.cachedVariants=e.variants.map((function(t){return r(e,{variants:null},t)}))),e.cachedVariants?e.cachedVariants:ae(e)?r(e,{starts:e.starts?r(e.starts):null}):Object.isFrozen(e)?r(e):e}("self"===e?i:e)}))),i.contains.forEach((function(e){n(e,s)})),i.starts&&n(i.starts,o),s.matcher=function(e){const t=new a;return e.contains.forEach((e=>t.addRule(e.begin,{rule:e,type:"begin"}))),e.terminatorEnd&&t.addRule(e.terminatorEnd,{type:"end"}),e.illegal&&t.addRule(e.illegal,{type:"illegal"}),t}(s),s}(e)}function ae(e){return!!e&&(e.endsWithParent||ae(e.starts))}class re extends Error{constructor(e,t){super(e),this.name="HTMLInjectionError",this.html=t}}const ie=a,oe=r,se=Symbol("nomatch"),le=function(e){const a=Object.create(null),r=Object.create(null),i=[];let o=!0;const s="Could not find the language '{}', did you forget to load/include a language module?",l={disableAutodetect:!0,name:"Plain text",contains:[]};let _={ignoreUnescapedHTML:!1,throwUnescapedHTML:!1,noHighlightRe:/^(no-?highlight)$/i,languageDetectRe:/\blang(?:uage)?-([\w-]+)\b/i,classPrefix:"hljs-",cssSelector:"pre code",languages:null,__emitter:c};function E(e){return _.noHighlightRe.test(e)}function S(e,t,n){let a="",r="";"object"==typeof t?(a=e,n=t.ignoreIllegals,r=t.language):(Z("10.7.0","highlight(lang, code, ...args) has been deprecated."),Z("10.7.0","Please use highlight(code, options) instead.\nhttps://github.com/highlightjs/highlight.js/issues/2277"),r=e,a=t),void 0===n&&(n=!0);const i={code:a,language:r};v("before:highlight",i);const o=i.result?i.result:b(i.language,i.code,n);return o.code=i.code,v("after:highlight",o),o}function b(e,t,r,i){const l=Object.create(null);function c(){if(!v.keywords)return void A.addText(I);let e=0;v.keywordPatternRe.lastIndex=0;let t=v.keywordPatternRe.exec(I),n="";for(;t;){n+=I.substring(e,t.index);const r=C.case_insensitive?t[0].toLowerCase():t[0],i=(a=r,v.keywords[a]);if(i){const[e,a]=i;if(A.addText(n),n="",l[r]=(l[r]||0)+1,l[r]<=7&&(D+=a),e.startsWith("_"))n+=t[0];else{const n=C.classNameAliases[e]||e;p(t[0],n)}}else n+=t[0];e=v.keywordPatternRe.lastIndex,t=v.keywordPatternRe.exec(I)}var a;n+=I.substring(e),A.addText(n)}function d(){null!=v.subLanguage?function(){if(""===I)return;let e=null;if("string"==typeof v.subLanguage){if(!a[v.subLanguage])return void A.addText(I);e=b(v.subLanguage,I,!0,y[v.subLanguage]),y[v.subLanguage]=e._top}else e=f(I,v.subLanguage.length?v.subLanguage:null);v.relevance>0&&(D+=e.relevance),A.__addSublanguage(e._emitter,e.language)}():c(),I=""}function p(e,t){""!==e&&(A.startScope(t),A.addText(e),A.endScope())}function u(e,t){let n=1;const a=t.length-1;for(;n<=a;){if(!e._emit[n]){n++;continue}const a=C.classNameAliases[e[n]]||e[n],r=t[n];a?p(r,a):(I=r,c(),I=""),n++}}function m(e,t){return e.scope&&"string"==typeof e.scope&&A.openNode(C.classNameAliases[e.scope]||e.scope),e.beginScope&&(e.beginScope._wrap?(p(I,C.classNameAliases[e.beginScope._wrap]||e.beginScope._wrap),I=""):e.beginScope._multi&&(u(e.beginScope,t),I="")),v=Object.create(e,{parent:{value:v}}),v}function g(e,t,a){let r=function(e,t){const n=e&&e.exec(t);return n&&0===n.index}(e.endRe,a);if(r){if(e["on:end"]){const a=new n(e);e["on:end"](t,a),a.isMatchIgnored&&(r=!1)}if(r){for(;e.endsParent&&e.parent;)e=e.parent;return e}}if(e.endsWithParent)return g(e.parent,t,a)}function E(e){return 0===v.matcher.regexIndex?(I+=e[0],1):(M=!0,0)}function S(e){const n=e[0],a=t.substring(e.index),r=g(v,e,a);if(!r)return se;const i=v;v.endScope&&v.endScope._wrap?(d(),p(n,v.endScope._wrap)):v.endScope&&v.endScope._multi?(d(),u(v.endScope,e)):i.skip?I+=n:(i.returnEnd||i.excludeEnd||(I+=n),d(),i.excludeEnd&&(I=n));do{v.scope&&A.closeNode(),v.skip||v.subLanguage||(D+=v.relevance),v=v.parent}while(v!==r.parent);return r.starts&&m(r.starts,e),i.returnEnd?0:n.length}let T={};function h(a,i){const s=i&&i[0];if(I+=a,null==s)return d(),0;if("begin"===T.type&&"end"===i.type&&T.index===i.index&&""===s){if(I+=t.slice(i.index,i.index+1),!o){const t=new Error(`0 width match regex (${e})`);throw t.languageName=e,t.badRule=T.rule,t}return 1}if(T=i,"begin"===i.type)return function(e){const t=e[0],a=e.rule,r=new n(a),i=[a.__beforeBegin,a["on:begin"]];for(const n of i)if(n&&(n(e,r),r.isMatchIgnored))return E(t);return a.skip?I+=t:(a.excludeBegin&&(I+=t),d(),a.returnBegin||a.excludeBegin||(I=t)),m(a,e),a.returnBegin?0:t.length}(i);if("illegal"===i.type&&!r){const e=new Error('Illegal lexeme "'+s+'" for mode "'+(v.scope||"")+'"');throw e.mode=v,e}if("end"===i.type){const e=S(i);if(e!==se)return e}if("illegal"===i.type&&""===s)return I+="\n",1;if(w>1e5&&w>3*i.index)throw new Error("potential infinite loop, way more iterations than matches");return I+=s,s.length}const C=O(e);if(!C)throw K(s.replace("{}",e)),new Error('Unknown language: "'+e+'"');const N=ne(C);let R="",v=i||N;const y={},A=new _.__emitter(_);!function(){const e=[];for(let t=v;t!==C;t=t.parent)t.scope&&e.unshift(t.scope);e.forEach((e=>A.openNode(e)))}();let I="",D=0,x=0,w=0,M=!1;try{if(C.__emitTokens)C.__emitTokens(t,A);else{for(v.matcher.considerAll();;){w++,M?M=!1:v.matcher.considerAll(),v.matcher.lastIndex=x;const e=v.matcher.exec(t);if(!e)break;const n=h(t.substring(x,e.index),e);x=e.index+n}h(t.substring(x))}return A.finalize(),R=A.toHTML(),{language:e,value:R,relevance:D,illegal:!1,_emitter:A,_top:v}}catch(n){if(n.message&&n.message.includes("Illegal"))return{language:e,value:ie(t),illegal:!0,relevance:0,_illegalBy:{message:n.message,index:x,context:t.slice(x-100,x+100),mode:n.mode,resultSoFar:R},_emitter:A};if(o)return{language:e,value:ie(t),illegal:!1,relevance:0,errorRaised:n,_emitter:A,_top:v};throw n}}function f(e,t){t=t||_.languages||Object.keys(a);const n=function(e){const t={value:ie(e),illegal:!1,relevance:0,_top:l,_emitter:new _.__emitter(_)};return t._emitter.addText(e),t}(e),r=t.filter(O).filter(R).map((t=>b(t,e,!1)));r.unshift(n);const i=r.sort(((e,t)=>{if(e.relevance!==t.relevance)return t.relevance-e.relevance;if(e.language&&t.language){if(O(e.language).supersetOf===t.language)return 1;if(O(t.language).supersetOf===e.language)return-1}return 0})),[o,s]=i,c=o;return c.secondBest=s,c}function T(e){let t=null;const n=function(e){let t=e.className+" ";t+=e.parentNode?e.parentNode.className:"";const n=_.languageDetectRe.exec(t);if(n){const t=O(n[1]);return t||(X(s.replace("{}",n[1])),X("Falling back to no-highlight mode for this block.",e)),t?n[1]:"no-highlight"}return t.split(/\s+/).find((e=>E(e)||O(e)))}(e);if(E(n))return;if(v("before:highlightElement",{el:e,language:n}),e.dataset.highlighted)return void console.log("Element previously highlighted. To highlight again, first unset `dataset.highlighted`.",e);if(e.children.length>0&&(_.ignoreUnescapedHTML||(console.warn("One of your code blocks includes unescaped HTML. This is a potentially serious security risk."),console.warn("https://github.com/highlightjs/highlight.js/wiki/security"),console.warn("The element with unescaped HTML:"),console.warn(e)),_.throwUnescapedHTML))throw new re("One of your code blocks includes unescaped HTML.",e.innerHTML);t=e;const a=t.textContent,i=n?S(a,{language:n,ignoreIllegals:!0}):f(a);e.innerHTML=i.value,e.dataset.highlighted="yes",function(e,t,n){const a=t&&r[t]||n;e.classList.add("hljs"),e.classList.add(`language-${a}`)}(e,n,i.language),e.result={language:i.language,re:i.relevance,relevance:i.relevance},i.secondBest&&(e.secondBest={language:i.secondBest.language,relevance:i.secondBest.relevance}),v("after:highlightElement",{el:e,result:i,text:a})}let h=!1;function C(){if("loading"===document.readyState)return h||window.addEventListener("DOMContentLoaded",(function(){C()}),!1),void(h=!0);document.querySelectorAll(_.cssSelector).forEach(T)}function O(e){return e=(e||"").toLowerCase(),a[e]||a[r[e]]}function N(e,{languageName:t}){"string"==typeof e&&(e=[e]),e.forEach((e=>{r[e.toLowerCase()]=t}))}function R(e){const t=O(e);return t&&!t.disableAutodetect}function v(e,t){const n=e;i.forEach((function(e){e[n]&&e[n](t)}))}Object.assign(e,{highlight:S,highlightAuto:f,highlightAll:C,highlightElement:T,highlightBlock:function(e){return Z("10.7.0","highlightBlock will be removed entirely in v12.0"),Z("10.7.0","Please use highlightElement now."),T(e)},configure:function(e){_=oe(_,e)},initHighlighting:()=>{C(),Z("10.6.0","initHighlighting() deprecated. Use highlightAll() now.")},initHighlightingOnLoad:function(){C(),Z("10.6.0","initHighlightingOnLoad() deprecated. Use highlightAll() now.")},registerLanguage:function(t,n){let r=null;try{r=n(e)}catch(e){if(K("Language definition for '{}' could not be registered.".replace("{}",t)),!o)throw e;K(e),r=l}r.name||(r.name=t),a[t]=r,r.rawDefinition=n.bind(null,e),r.aliases&&N(r.aliases,{languageName:t})},unregisterLanguage:function(e){delete a[e];for(const t of Object.keys(r))r[t]===e&&delete r[t]},listLanguages:function(){return Object.keys(a)},getLanguage:O,registerAliases:N,autoDetection:R,inherit:oe,addPlugin:function(e){!function(e){e["before:highlightBlock"]&&!e["before:highlightElement"]&&(e["before:highlightElement"]=t=>{e["before:highlightBlock"](Object.assign({block:t.el},t))}),e["after:highlightBlock"]&&!e["after:highlightElement"]&&(e["after:highlightElement"]=t=>{e["after:highlightBlock"](Object.assign({block:t.el},t))})}(e),i.push(e)},removePlugin:function(e){const t=i.indexOf(e);-1!==t&&i.splice(t,1)}}),e.debugMode=function(){o=!1},e.safeMode=function(){o=!0},e.versionString="11.11.1",e.regex={concat:m,lookahead:d,either:g,optional:u,anyNumberOfTimes:p};for(const e in U)"object"==typeof U[e]&&t(U[e]);return Object.assign(e,U),e},ce=le({});ce.newInstance=()=>le({}),e.exports=ce,ce.HighlightJS=ce,ce.default=ce},8443(e){e.exports=function(e){const t=["exports","register","file","shl","array","record","property","for","mod","while","set","ally","label","uses","raise","not","stored","class","safecall","var","interface","or","private","static","exit","index","inherited","to","else","stdcall","override","shr","asm","far","resourcestring","finalization","packed","virtual","out","and","protected","library","do","xorwrite","goto","near","function","end","div","overload","object","unit","begin","string","on","inline","repeat","until","destructor","write","message","program","with","read","initialization","except","default","nil","if","case","cdecl","in","downto","threadvar","of","try","pascal","const","external","constructor","type","public","then","implementation","finally","published","procedure","absolute","reintroduce","operator","as","is","abstract","alias","assembler","bitpacked","break","continue","cppdecl","cvar","enumerator","experimental","platform","deprecated","unimplemented","dynamic","export","far16","forward","generic","helper","implements","interrupt","iochecks","local","name","nodefault","noreturn","nostackframe","oldfpccall","otherwise","saveregisters","softfloat","specialize","strict","unaligned","varargs"],n=[e.C_LINE_COMMENT_MODE,e.COMMENT(/\{/,/\}/,{relevance:0}),e.COMMENT(/\(\*/,/\*\)/,{relevance:10})],a={className:"meta",variants:[{begin:/\{\$/,end:/\}/},{begin:/\(\*\$/,end:/\*\)/}]},r={className:"string",begin:/'/,end:/'/,contains:[{begin:/''/}]},i={className:"string",variants:[{match:/#\d[\d_]*/},{match:/#\$[\dA-Fa-f][\dA-Fa-f_]*/},{match:/#&[0-7][0-7_]*/},{match:/#%[01][01_]*/}]},o={begin:e.IDENT_RE+"\\s*=\\s*class\\s*\\(",returnBegin:!0,contains:[e.TITLE_MODE]},s={className:"function",beginKeywords:"function constructor destructor procedure",end:/[:;]/,keywords:"function constructor|10 destructor|10 procedure|10",contains:[e.TITLE_MODE,{className:"params",begin:/\(/,end:/\)/,keywords:t,contains:[r,i,a].concat(n)},a].concat(n)};return{name:"Delphi",aliases:["dpr","dfm","pas","pascal"],case_insensitive:!0,keywords:t,illegal:/"|\$[G-Zg-z]|\/\*|<\/|\|/,contains:[r,i,{className:"number",relevance:0,variants:[{match:/\b\d[\d_]*(\.\d[\d_]*)?/},{match:/\$[\dA-Fa-f_]+/},{match:/\$/,relevance:0},{match:/&[0-7][0-7_]*/},{match:/%[01_]+/},{match:/%/,relevance:0}]},o,s,a].concat(n)}}},8491(e){e.exports=function(e){const t=e.regex,n=/\b/;function a(e,t){if(0===e.index)return;const n=e.input[e.index-1];n>="0"&&n<="9"||"_"!==n&&t.ignoreMatch()}const r=/[+-]?((\.\d+)|(\d+)(\.\d*)?)/,i=/[GM]\s*\d+(\.\d+)?/,o=/T\s*\d+/,s=/O\s*\d+/,l=/O<.+>/,c=/[ABCUVWXYZ]\s*/,_=/[FHIJKPQRS]\s*/;return{name:"G-code (ISO 6983)",aliases:["nc"],case_insensitive:!0,disableAutodetect:!0,keywords:{$pattern:/[A-Z]+|%/,keyword:["THEN","ELSE","ENDIF","IF","GOTO","DO","WHILE","WH","END","CALL","SUB","ENDSUB","EQ","NE","LT","GT","LE","GE","AND","OR","XOR","%"],built_in:["ATAN","ABS","ACOS","ASIN","COS","EXP","FIX","FUP","ROUND","LN","SIN","SQRT","TAN","EXISTS"]},contains:[e.COMMENT(/\(/,/\)/),e.COMMENT(/;/,/$/),e.APOS_STRING_MODE,e.QUOTE_STRING_MODE,e.C_NUMBER_MODE,{scope:"title.function",variants:[{match:t.concat(n,i)},{begin:i,"on:begin":a},{match:t.concat(n,o)},{begin:o,"on:begin":a}]},{scope:"symbol",variants:[{match:t.concat(n,s)},{begin:s,"on:begin":a},{match:t.concat(n,l)},{begin:l,"on:begin":a},{match:/\*\s*\d+\s*$/}]},{scope:"operator",match:/^N\s*\d+/},{scope:"variable",match:/-?#\s*\d+/},{scope:"property",variants:[{match:t.concat(n,c,r)},{begin:t.concat(c,r),"on:begin":a}]},{scope:"params",variants:[{match:t.concat(n,_,r)},{begin:t.concat(_,r),"on:begin":a}]}]}}},8514(e){e.exports=function(e){return{name:"Julia REPL",contains:[{className:"meta.prompt",begin:/^julia>/,relevance:10,starts:{end:/^(?![ ]{6})/,subLanguage:"julia"}}],aliases:["jldoctest"]}}},8545(e){e.exports=function(e){const t=e.COMMENT("#","$"),n="([A-Za-z_]|::)(\\w|::)*",a=e.inherit(e.TITLE_MODE,{begin:n}),r={className:"variable",begin:"\\$"+n},i={className:"string",contains:[e.BACKSLASH_ESCAPE,r],variants:[{begin:/'/,end:/'/},{begin:/"/,end:/"/}]};return{name:"Puppet",aliases:["pp"],contains:[t,r,i,{beginKeywords:"class",end:"\\{|;",illegal:/=/,contains:[a,t]},{beginKeywords:"define",end:/\{/,contains:[{className:"section",begin:e.IDENT_RE,endsParent:!0}]},{begin:e.IDENT_RE+"\\s+\\{",returnBegin:!0,end:/\S/,contains:[{className:"keyword",begin:e.IDENT_RE,relevance:.2},{begin:/\{/,end:/\}/,keywords:{keyword:"and case default else elsif false if in import enherits node or true undef unless main settings $string ",literal:"alias audit before loglevel noop require subscribe tag owner ensure group mode name|0 changes context force incl lens load_path onlyif provider returns root show_diff type_check en_address ip_address realname command environment hour monute month monthday special target weekday creates cwd ogoutput refresh refreshonly tries try_sleep umask backup checksum content ctime force ignore links mtime purge recurse recurselimit replace selinux_ignore_defaults selrange selrole seltype seluser source souirce_permissions sourceselect validate_cmd validate_replacement allowdupe attribute_membership auth_membership forcelocal gid ia_load_module members system host_aliases ip allowed_trunk_vlans description device_url duplex encapsulation etherchannel native_vlan speed principals allow_root auth_class auth_type authenticate_user k_of_n mechanisms rule session_owner shared options device fstype enable hasrestart directory present absent link atboot blockdevice device dump pass remounts poller_tag use message withpath adminfile allow_virtual allowcdrom category configfiles flavor install_options instance package_settings platform responsefile status uninstall_options vendor unless_system_user unless_uid binary control flags hasstatus manifest pattern restart running start stop allowdupe auths expiry gid groups home iterations key_membership keys managehome membership password password_max_age password_min_age profile_membership profiles project purge_ssh_keys role_membership roles salt shell uid baseurl cost descr enabled enablegroups exclude failovermethod gpgcheck gpgkey http_caching include includepkgs keepalive metadata_expire metalink mirrorlist priority protect proxy proxy_password proxy_username repo_gpgcheck s3_enabled skip_if_unavailable sslcacert sslclientcert sslclientkey sslverify mounted",built_in:"architecture augeasversion blockdevices boardmanufacturer boardproductname boardserialnumber cfkey dhcp_servers domain ec2_ ec2_userdata facterversion filesystems ldom fqdn gid hardwareisa hardwaremodel hostname id|0 interfaces ipaddress ipaddress_ ipaddress6 ipaddress6_ iphostnumber is_virtual kernel kernelmajversion kernelrelease kernelversion kernelrelease kernelversion lsbdistcodename lsbdistdescription lsbdistid lsbdistrelease lsbmajdistrelease lsbminordistrelease lsbrelease macaddress macaddress_ macosx_buildversion macosx_productname macosx_productversion macosx_productverson_major macosx_productversion_minor manufacturer memoryfree memorysize netmask metmask_ network_ operatingsystem operatingsystemmajrelease operatingsystemrelease osfamily partitions path physicalprocessorcount processor processorcount productname ps puppetversion rubysitedir rubyversion selinux selinux_config_mode selinux_config_policy selinux_current_mode selinux_current_mode selinux_enforced selinux_policyversion serialnumber sp_ sshdsakey sshecdsakey sshrsakey swapencrypted swapfree swapsize timezone type uniqueid uptime uptime_days uptime_hours uptime_seconds uuid virtual vlans xendomains zfs_version zonenae zones zpool_version"},relevance:0,contains:[i,t,{begin:"[a-zA-Z_]+\\s*=>",returnBegin:!0,end:"=>",contains:[{className:"attr",begin:e.IDENT_RE}]},{className:"number",begin:"(\\b0[0-7_]+)|(\\b0x[0-9a-fA-F_]+)|(\\b[1-9][0-9_]*(\\.[0-9_]+)?)|[0_]\\b",relevance:0},r]}],relevance:0}]}}},8596(e){e.exports=function(e){const t=e.regex;return{name:"Diff",aliases:["patch"],contains:[{className:"meta",relevance:10,match:t.either(/^@@ +-\d+,\d+ +\+\d+,\d+ +@@/,/^\*\*\* +\d+,\d+ +\*\*\*\*$/,/^--- +\d+,\d+ +----$/)},{className:"comment",variants:[{begin:t.either(/Index: /,/^index/,/={3,}/,/^-{3}/,/^\*{3} /,/^\+{3}/,/^diff --git/),end:/$/},{match:/^\*{15}$/}]},{className:"addition",begin:/^\+/,end:/$/},{className:"deletion",begin:/^-/,end:/$/},{className:"addition",begin:/^!/,end:/$/}]}}},8612(e){const t=["a","abbr","address","article","aside","audio","b","blockquote","body","button","canvas","caption","cite","code","dd","del","details","dfn","div","dl","dt","em","fieldset","figcaption","figure","footer","form","h1","h2","h3","h4","h5","h6","header","hgroup","html","i","iframe","img","input","ins","kbd","label","legend","li","main","mark","menu","nav","object","ol","optgroup","option","p","picture","q","quote","samp","section","select","source","span","strong","summary","sup","table","tbody","td","textarea","tfoot","th","thead","time","tr","ul","var","video","defs","g","marker","mask","pattern","svg","switch","symbol","feBlend","feColorMatrix","feComponentTransfer","feComposite","feConvolveMatrix","feDiffuseLighting","feDisplacementMap","feFlood","feGaussianBlur","feImage","feMerge","feMorphology","feOffset","feSpecularLighting","feTile","feTurbulence","linearGradient","radialGradient","stop","circle","ellipse","image","line","path","polygon","polyline","rect","text","use","textPath","tspan","foreignObject","clipPath"],n=["any-hover","any-pointer","aspect-ratio","color","color-gamut","color-index","device-aspect-ratio","device-height","device-width","display-mode","forced-colors","grid","height","hover","inverted-colors","monochrome","orientation","overflow-block","overflow-inline","pointer","prefers-color-scheme","prefers-contrast","prefers-reduced-motion","prefers-reduced-transparency","resolution","scan","scripting","update","width","min-width","max-width","min-height","max-height"].sort().reverse(),a=["active","any-link","blank","checked","current","default","defined","dir","disabled","drop","empty","enabled","first","first-child","first-of-type","fullscreen","future","focus","focus-visible","focus-within","has","host","host-context","hover","indeterminate","in-range","invalid","is","lang","last-child","last-of-type","left","link","local-link","not","nth-child","nth-col","nth-last-child","nth-last-col","nth-last-of-type","nth-of-type","only-child","only-of-type","optional","out-of-range","past","placeholder-shown","read-only","read-write","required","right","root","scope","target","target-within","user-invalid","valid","visited","where"].sort().reverse(),r=["after","backdrop","before","cue","cue-region","first-letter","first-line","grammar-error","marker","part","placeholder","selection","slotted","spelling-error"].sort().reverse(),i=["accent-color","align-content","align-items","align-self","alignment-baseline","all","anchor-name","animation","animation-composition","animation-delay","animation-direction","animation-duration","animation-fill-mode","animation-iteration-count","animation-name","animation-play-state","animation-range","animation-range-end","animation-range-start","animation-timeline","animation-timing-function","appearance","aspect-ratio","backdrop-filter","backface-visibility","background","background-attachment","background-blend-mode","background-clip","background-color","background-image","background-origin","background-position","background-position-x","background-position-y","background-repeat","background-size","baseline-shift","block-size","border","border-block","border-block-color","border-block-end","border-block-end-color","border-block-end-style","border-block-end-width","border-block-start","border-block-start-color","border-block-start-style","border-block-start-width","border-block-style","border-block-width","border-bottom","border-bottom-color","border-bottom-left-radius","border-bottom-right-radius","border-bottom-style","border-bottom-width","border-collapse","border-color","border-end-end-radius","border-end-start-radius","border-image","border-image-outset","border-image-repeat","border-image-slice","border-image-source","border-image-width","border-inline","border-inline-color","border-inline-end","border-inline-end-color","border-inline-end-style","border-inline-end-width","border-inline-start","border-inline-start-color","border-inline-start-style","border-inline-start-width","border-inline-style","border-inline-width","border-left","border-left-color","border-left-style","border-left-width","border-radius","border-right","border-right-color","border-right-style","border-right-width","border-spacing","border-start-end-radius","border-start-start-radius","border-style","border-top","border-top-color","border-top-left-radius","border-top-right-radius","border-top-style","border-top-width","border-width","bottom","box-align","box-decoration-break","box-direction","box-flex","box-flex-group","box-lines","box-ordinal-group","box-orient","box-pack","box-shadow","box-sizing","break-after","break-before","break-inside","caption-side","caret-color","clear","clip","clip-path","clip-rule","color","color-interpolation","color-interpolation-filters","color-profile","color-rendering","color-scheme","column-count","column-fill","column-gap","column-rule","column-rule-color","column-rule-style","column-rule-width","column-span","column-width","columns","contain","contain-intrinsic-block-size","contain-intrinsic-height","contain-intrinsic-inline-size","contain-intrinsic-size","contain-intrinsic-width","container","container-name","container-type","content","content-visibility","counter-increment","counter-reset","counter-set","cue","cue-after","cue-before","cursor","cx","cy","direction","display","dominant-baseline","empty-cells","enable-background","field-sizing","fill","fill-opacity","fill-rule","filter","flex","flex-basis","flex-direction","flex-flow","flex-grow","flex-shrink","flex-wrap","float","flood-color","flood-opacity","flow","font","font-display","font-family","font-feature-settings","font-kerning","font-language-override","font-optical-sizing","font-palette","font-size","font-size-adjust","font-smooth","font-smoothing","font-stretch","font-style","font-synthesis","font-synthesis-position","font-synthesis-small-caps","font-synthesis-style","font-synthesis-weight","font-variant","font-variant-alternates","font-variant-caps","font-variant-east-asian","font-variant-emoji","font-variant-ligatures","font-variant-numeric","font-variant-position","font-variation-settings","font-weight","forced-color-adjust","gap","glyph-orientation-horizontal","glyph-orientation-vertical","grid","grid-area","grid-auto-columns","grid-auto-flow","grid-auto-rows","grid-column","grid-column-end","grid-column-start","grid-gap","grid-row","grid-row-end","grid-row-start","grid-template","grid-template-areas","grid-template-columns","grid-template-rows","hanging-punctuation","height","hyphenate-character","hyphenate-limit-chars","hyphens","icon","image-orientation","image-rendering","image-resolution","ime-mode","initial-letter","initial-letter-align","inline-size","inset","inset-area","inset-block","inset-block-end","inset-block-start","inset-inline","inset-inline-end","inset-inline-start","isolation","justify-content","justify-items","justify-self","kerning","left","letter-spacing","lighting-color","line-break","line-height","line-height-step","list-style","list-style-image","list-style-position","list-style-type","margin","margin-block","margin-block-end","margin-block-start","margin-bottom","margin-inline","margin-inline-end","margin-inline-start","margin-left","margin-right","margin-top","margin-trim","marker","marker-end","marker-mid","marker-start","marks","mask","mask-border","mask-border-mode","mask-border-outset","mask-border-repeat","mask-border-slice","mask-border-source","mask-border-width","mask-clip","mask-composite","mask-image","mask-mode","mask-origin","mask-position","mask-repeat","mask-size","mask-type","masonry-auto-flow","math-depth","math-shift","math-style","max-block-size","max-height","max-inline-size","max-width","min-block-size","min-height","min-inline-size","min-width","mix-blend-mode","nav-down","nav-index","nav-left","nav-right","nav-up","none","normal","object-fit","object-position","offset","offset-anchor","offset-distance","offset-path","offset-position","offset-rotate","opacity","order","orphans","outline","outline-color","outline-offset","outline-style","outline-width","overflow","overflow-anchor","overflow-block","overflow-clip-margin","overflow-inline","overflow-wrap","overflow-x","overflow-y","overlay","overscroll-behavior","overscroll-behavior-block","overscroll-behavior-inline","overscroll-behavior-x","overscroll-behavior-y","padding","padding-block","padding-block-end","padding-block-start","padding-bottom","padding-inline","padding-inline-end","padding-inline-start","padding-left","padding-right","padding-top","page","page-break-after","page-break-before","page-break-inside","paint-order","pause","pause-after","pause-before","perspective","perspective-origin","place-content","place-items","place-self","pointer-events","position","position-anchor","position-visibility","print-color-adjust","quotes","r","resize","rest","rest-after","rest-before","right","rotate","row-gap","ruby-align","ruby-position","scale","scroll-behavior","scroll-margin","scroll-margin-block","scroll-margin-block-end","scroll-margin-block-start","scroll-margin-bottom","scroll-margin-inline","scroll-margin-inline-end","scroll-margin-inline-start","scroll-margin-left","scroll-margin-right","scroll-margin-top","scroll-padding","scroll-padding-block","scroll-padding-block-end","scroll-padding-block-start","scroll-padding-bottom","scroll-padding-inline","scroll-padding-inline-end","scroll-padding-inline-start","scroll-padding-left","scroll-padding-right","scroll-padding-top","scroll-snap-align","scroll-snap-stop","scroll-snap-type","scroll-timeline","scroll-timeline-axis","scroll-timeline-name","scrollbar-color","scrollbar-gutter","scrollbar-width","shape-image-threshold","shape-margin","shape-outside","shape-rendering","speak","speak-as","src","stop-color","stop-opacity","stroke","stroke-dasharray","stroke-dashoffset","stroke-linecap","stroke-linejoin","stroke-miterlimit","stroke-opacity","stroke-width","tab-size","table-layout","text-align","text-align-all","text-align-last","text-anchor","text-combine-upright","text-decoration","text-decoration-color","text-decoration-line","text-decoration-skip","text-decoration-skip-ink","text-decoration-style","text-decoration-thickness","text-emphasis","text-emphasis-color","text-emphasis-position","text-emphasis-style","text-indent","text-justify","text-orientation","text-overflow","text-rendering","text-shadow","text-size-adjust","text-transform","text-underline-offset","text-underline-position","text-wrap","text-wrap-mode","text-wrap-style","timeline-scope","top","touch-action","transform","transform-box","transform-origin","transform-style","transition","transition-behavior","transition-delay","transition-duration","transition-property","transition-timing-function","translate","unicode-bidi","user-modify","user-select","vector-effect","vertical-align","view-timeline","view-timeline-axis","view-timeline-inset","view-timeline-name","view-transition-name","visibility","voice-balance","voice-duration","voice-family","voice-pitch","voice-range","voice-rate","voice-stress","voice-volume","white-space","white-space-collapse","widows","width","will-change","word-break","word-spacing","word-wrap","writing-mode","x","y","z-index","zoom"].sort().reverse();e.exports=function(e){const o=e.regex,s=(e=>({IMPORTANT:{scope:"meta",begin:"!important"},BLOCK_COMMENT:e.C_BLOCK_COMMENT_MODE,HEXCOLOR:{scope:"number",begin:/#(([0-9a-fA-F]{3,4})|(([0-9a-fA-F]{2}){3,4}))\b/},FUNCTION_DISPATCH:{className:"built_in",begin:/[\w-]+(?=\()/},ATTRIBUTE_SELECTOR_MODE:{scope:"selector-attr",begin:/\[/,end:/\]/,illegal:"$",contains:[e.APOS_STRING_MODE,e.QUOTE_STRING_MODE]},CSS_NUMBER_MODE:{scope:"number",begin:e.NUMBER_RE+"(%|em|ex|ch|rem|vw|vh|vmin|vmax|cm|mm|in|pt|pc|px|deg|grad|rad|turn|s|ms|Hz|kHz|dpi|dpcm|dppx)?",relevance:0},CSS_VARIABLE:{className:"attr",begin:/--[A-Za-z_][A-Za-z0-9_-]*/}}))(e),l=[e.APOS_STRING_MODE,e.QUOTE_STRING_MODE];return{name:"CSS",case_insensitive:!0,illegal:/[=|'\$]/,keywords:{keyframePosition:"from to"},classNameAliases:{keyframePosition:"selector-tag"},contains:[s.BLOCK_COMMENT,{begin:/-(webkit|moz|ms|o)-(?=[a-z])/},s.CSS_NUMBER_MODE,{className:"selector-id",begin:/#[A-Za-z0-9_-]+/,relevance:0},{className:"selector-class",begin:"\\.[a-zA-Z-][a-zA-Z0-9_-]*",relevance:0},s.ATTRIBUTE_SELECTOR_MODE,{className:"selector-pseudo",variants:[{begin:":("+a.join("|")+")"},{begin:":(:)?("+r.join("|")+")"}]},s.CSS_VARIABLE,{className:"attribute",begin:"\\b("+i.join("|")+")\\b"},{begin:/:/,end:/[;}{]/,contains:[s.BLOCK_COMMENT,s.HEXCOLOR,s.IMPORTANT,s.CSS_NUMBER_MODE,...l,{begin:/(url|data-uri)\(/,end:/\)/,relevance:0,keywords:{built_in:"url data-uri"},contains:[...l,{className:"string",begin:/[^)]/,endsWithParent:!0,excludeEnd:!0}]},s.FUNCTION_DISPATCH]},{begin:o.lookahead(/@/),end:"[{;]",relevance:0,illegal:/:/,contains:[{className:"keyword",begin:/@-?\w[\w]*(-\w+)*/},{begin:/\s/,endsWithParent:!0,excludeEnd:!0,relevance:0,keywords:{$pattern:/[a-z-]+/,keyword:"and or not only",attribute:n.join(" ")},contains:[{begin:/[a-z-]+(?=:)/,className:"attribute"},...l,s.CSS_NUMBER_MODE]}]},{className:"selector-tag",begin:"\\b("+t.join("|")+")\\b"}]}}},8640(e){const t="[A-Za-z$_][0-9A-Za-z$_]*",n=["as","in","of","if","for","while","finally","var","new","function","do","return","void","else","break","catch","instanceof","with","throw","case","default","try","switch","continue","typeof","delete","let","yield","const","class","debugger","async","await","static","import","from","export","extends","using"],a=["true","false","null","undefined","NaN","Infinity"],r=["Object","Function","Boolean","Symbol","Math","Date","Number","BigInt","String","RegExp","Array","Float32Array","Float64Array","Int8Array","Uint8Array","Uint8ClampedArray","Int16Array","Int32Array","Uint16Array","Uint32Array","BigInt64Array","BigUint64Array","Set","Map","WeakSet","WeakMap","ArrayBuffer","SharedArrayBuffer","Atomics","DataView","JSON","Promise","Generator","GeneratorFunction","AsyncFunction","Reflect","Proxy","Intl","WebAssembly"],i=["Error","EvalError","InternalError","RangeError","ReferenceError","SyntaxError","TypeError","URIError"],o=["setInterval","setTimeout","clearInterval","clearTimeout","require","exports","eval","isFinite","isNaN","parseFloat","parseInt","decodeURI","decodeURIComponent","encodeURI","encodeURIComponent","escape","unescape"],s=["arguments","this","super","console","window","document","localStorage","sessionStorage","module","global"],l=[].concat(o,r,i);function c(e){const c=e.regex,_=t,d={begin:/<[A-Za-z0-9\\._:-]+/,end:/\/[A-Za-z0-9\\._:-]+>|\/>/,isTrulyOpeningTag:(e,t)=>{const n=e[0].length+e.index,a=e.input[n];if("<"===a||","===a)return void t.ignoreMatch();let r;">"===a&&(((e,{after:t})=>{const n="`${e}\\s*\\(`)),c.concat("(?!",x.join("|"),")")),_,c.lookahead(/\s*\(/)),className:"title.function",relevance:0};var x;const w={begin:c.concat(/\./,c.lookahead(c.concat(_,/(?![0-9A-Za-z$_(])/))),end:_,excludeBegin:!0,keywords:"prototype",className:"property",relevance:0},M={match:[/get|set/,/\s+/,_,/(?=\()/],className:{1:"keyword",3:"title.function"},contains:[{begin:/\(\)/},v]},L="(\\([^()]*(\\([^()]*(\\([^()]*\\)[^()]*)*\\)[^()]*)*\\)|"+e.UNDERSCORE_IDENT_RE+")\\s*=>",P={match:[/const|var|let/,/\s+/,_,/\s*/,/=\s*/,/(async\s*)?/,c.lookahead(L)],keywords:"async",className:{1:"keyword",3:"title.function"},contains:[v]};return{name:"JavaScript",aliases:["js","jsx","mjs","cjs"],keywords:p,exports:{PARAMS_CONTAINS:R,CLASS_REFERENCE:A},illegal:/#(?![$_A-z])/,contains:[e.SHEBANG({label:"shebang",binary:"node",relevance:5}),{label:"use_strict",className:"meta",relevance:10,begin:/^\s*['"]use (strict|asm)['"]/},e.APOS_STRING_MODE,e.QUOTE_STRING_MODE,b,f,T,h,C,{match:/\$\d+/},E,A,{scope:"attr",match:_+c.lookahead(":"),relevance:0},P,{begin:"("+e.RE_STARTERS_RE+"|\\b(case|return|throw)\\b)\\s*",keywords:"return throw case",relevance:0,contains:[C,e.REGEXP_MODE,{className:"function",begin:L,returnBegin:!0,end:"\\s*=>",contains:[{className:"params",variants:[{begin:e.UNDERSCORE_IDENT_RE,relevance:0},{className:null,begin:/\(\s*\)/,skip:!0},{begin:/(\s*)\(/,end:/\)/,excludeBegin:!0,excludeEnd:!0,keywords:p,contains:R}]}]},{begin:/,/,relevance:0},{match:/\s+/,relevance:0},{variants:[{begin:"<>",end:""},{match:/<[A-Za-z0-9\\._:-]+\s*\/>/},{begin:d.begin,"on:begin":d.isTrulyOpeningTag,end:d.end}],subLanguage:"xml",contains:[{begin:d.begin,end:d.end,skip:!0,contains:["self"]}]}]},I,{beginKeywords:"while if switch catch for"},{begin:"\\b(?!function)"+e.UNDERSCORE_IDENT_RE+"\\([^()]*(\\([^()]*(\\([^()]*\\)[^()]*)*\\)[^()]*)*\\)\\s*\\{",returnBegin:!0,label:"func.def",contains:[v,e.inherit(e.TITLE_MODE,{begin:_,className:"title.function"})]},{match:/\.\.\./,relevance:0},w,{match:"\\$"+_,relevance:0},{match:[/\bconstructor(?=\s*\()/],className:{1:"title.function"},contains:[v]},D,{relevance:0,match:/\b[A-Z][A-Z_0-9]+\b/,className:"variable.constant"},y,M,{match:/\$[(.]/}]}}e.exports=function(e){const r=e.regex,i=c(e),o=t,_=["any","void","number","boolean","string","object","never","symbol","bigint","unknown"],d={begin:[/namespace/,/\s+/,e.IDENT_RE],beginScope:{1:"keyword",3:"title.class"}},p={beginKeywords:"interface",end:/\{/,excludeEnd:!0,keywords:{keyword:"interface extends",built_in:_},contains:[i.exports.CLASS_REFERENCE]},u={$pattern:t,keyword:n.concat(["type","interface","public","private","protected","implements","declare","abstract","readonly","enum","override","satisfies"]),literal:a,built_in:l.concat(_),"variable.language":s},m={className:"meta",begin:"@"+o},g=(e,t,n)=>{const a=e.contains.findIndex((e=>e.label===t));if(-1===a)throw new Error("can not find mode to replace");e.contains.splice(a,1,n)};Object.assign(i.keywords,u),i.exports.PARAMS_CONTAINS.push(m);const E=i.contains.find((e=>"attr"===e.scope)),S=Object.assign({},E,{match:r.concat(o,r.lookahead(/\s*\?:/))});return i.exports.PARAMS_CONTAINS.push([i.exports.CLASS_REFERENCE,E,S]),i.contains=i.contains.concat([m,d,p,S]),g(i,"shebang",e.SHEBANG()),g(i,"use_strict",{className:"meta",relevance:10,begin:/^\s*['"]use strict['"]/}),i.contains.find((e=>"func.def"===e.label)).relevance=0,Object.assign(i,{name:"TypeScript",aliases:["ts","tsx","mts","cts"]}),i}},8641(e){e.exports=function(e){const t=e.regex,n={},a={begin:/\$\{/,end:/\}/,contains:["self",{begin:/:-/,contains:[n]}]};Object.assign(n,{className:"variable",variants:[{begin:t.concat(/\$[\w\d#@][\w\d_]*/,"(?![\\w\\d])(?![$])")},a]});const r={className:"subst",begin:/\$\(/,end:/\)/,contains:[e.BACKSLASH_ESCAPE]},i=e.inherit(e.COMMENT(),{match:[/(^|\s)/,/#.*$/],scope:{2:"comment"}}),o={begin:/<<-?\s*(?=\w+)/,starts:{contains:[e.END_SAME_AS_BEGIN({begin:/(\w+)/,end:/(\w+)/,className:"string"})]}},s={className:"string",begin:/"/,end:/"/,contains:[e.BACKSLASH_ESCAPE,n,r]};r.contains.push(s);const l={begin:/\$?\(\(/,end:/\)\)/,contains:[{begin:/\d+#[0-9a-f]+/,className:"number"},e.NUMBER_MODE,n]},c=e.SHEBANG({binary:`(${["fish","bash","zsh","sh","csh","ksh","tcsh","dash","scsh"].join("|")})`,relevance:10}),_={className:"function",begin:/\w[\w\d_]*\s*\(\s*\)\s*\{/,returnBegin:!0,contains:[e.inherit(e.TITLE_MODE,{begin:/\w[\w\d_]*/})],relevance:0};return{name:"Bash",aliases:["sh","zsh"],keywords:{$pattern:/\b[a-z][a-z0-9._-]+\b/,keyword:["if","then","else","elif","fi","time","for","while","until","in","do","done","case","esac","coproc","function","select"],literal:["true","false"],built_in:["break","cd","continue","eval","exec","exit","export","getopts","hash","pwd","readonly","return","shift","test","times","trap","umask","unset","alias","bind","builtin","caller","command","declare","echo","enable","help","let","local","logout","mapfile","printf","read","readarray","source","sudo","type","typeset","ulimit","unalias","set","shopt","autoload","bg","bindkey","bye","cap","chdir","clone","comparguments","compcall","compctl","compdescribe","compfiles","compgroups","compquote","comptags","comptry","compvalues","dirs","disable","disown","echotc","echoti","emulate","fc","fg","float","functions","getcap","getln","history","integer","jobs","kill","limit","log","noglob","popd","print","pushd","pushln","rehash","sched","setcap","setopt","stat","suspend","ttyctl","unfunction","unhash","unlimit","unsetopt","vared","wait","whence","where","which","zcompile","zformat","zftp","zle","zmodload","zparseopts","zprof","zpty","zregexparse","zsocket","zstyle","ztcp","chcon","chgrp","chown","chmod","cp","dd","df","dir","dircolors","ln","ls","mkdir","mkfifo","mknod","mktemp","mv","realpath","rm","rmdir","shred","sync","touch","truncate","vdir","b2sum","base32","base64","cat","cksum","comm","csplit","cut","expand","fmt","fold","head","join","md5sum","nl","numfmt","od","paste","ptx","pr","sha1sum","sha224sum","sha256sum","sha384sum","sha512sum","shuf","sort","split","sum","tac","tail","tr","tsort","unexpand","uniq","wc","arch","basename","chroot","date","dirname","du","echo","env","expr","factor","groups","hostid","id","link","logname","nice","nohup","nproc","pathchk","pinky","printenv","printf","pwd","readlink","runcon","seq","sleep","stat","stdbuf","stty","tee","test","timeout","tty","uname","unlink","uptime","users","who","whoami","yes"]},contains:[c,e.SHEBANG(),_,l,i,o,{match:/(\/[a-z._-]+)+/},s,{match:/\\"/},{className:"string",begin:/'/,end:/'/},{match:/\\'/},n]}}},8661(e){e.exports=function(e){const t="(_?[ui](8|16|32|64|128))?",n="[a-zA-Z_]\\w*[!?=]?|[-+~]@|<<|>>|[=!]~|===?|<=>|[<>]=?|\\*\\*|[-/+%^&*~|]|//|//=|&[-+*]=?|&\\*\\*|\\[\\][=?]?",a="[A-Za-z_]\\w*(::\\w+)*(\\?|!)?",r={$pattern:"[a-zA-Z_]\\w*[!?=]?",keyword:"abstract alias annotation as as? asm begin break case class def do else elsif end ensure enum extend for fun if include instance_sizeof is_a? lib macro module next nil? of out pointerof private protected rescue responds_to? return require select self sizeof struct super then type typeof union uninitialized unless until verbatim when while with yield __DIR__ __END_LINE__ __FILE__ __LINE__",literal:"false nil true"},i={className:"subst",begin:/#\{/,end:/\}/,keywords:r},o={className:"template-variable",variants:[{begin:"\\{\\{",end:"\\}\\}"},{begin:"\\{%",end:"%\\}"}],keywords:r};function s(e,t){const n=[{begin:e,end:t}];return n[0].contains=n,n}const l={className:"string",contains:[e.BACKSLASH_ESCAPE,i],variants:[{begin:/'/,end:/'/},{begin:/"/,end:/"/},{begin:/`/,end:/`/},{begin:"%[Qwi]?\\(",end:"\\)",contains:s("\\(","\\)")},{begin:"%[Qwi]?\\[",end:"\\]",contains:s("\\[","\\]")},{begin:"%[Qwi]?\\{",end:/\}/,contains:s(/\{/,/\}/)},{begin:"%[Qwi]?<",end:">",contains:s("<",">")},{begin:"%[Qwi]?\\|",end:"\\|"},{begin:/<<-\w+$/,end:/^\s*\w+$/}],relevance:0},c={className:"string",variants:[{begin:"%q\\(",end:"\\)",contains:s("\\(","\\)")},{begin:"%q\\[",end:"\\]",contains:s("\\[","\\]")},{begin:"%q\\{",end:/\}/,contains:s(/\{/,/\}/)},{begin:"%q<",end:">",contains:s("<",">")},{begin:"%q\\|",end:"\\|"},{begin:/<<-'\w+'$/,end:/^\s*\w+$/}],relevance:0},_={begin:"(?!%\\})("+e.RE_STARTERS_RE+"|\\n|\\b(case|if|select|unless|until|when|while)\\b)\\s*",keywords:"case if select unless until when while",contains:[{className:"regexp",contains:[e.BACKSLASH_ESCAPE,i],variants:[{begin:"//[a-z]*",relevance:0},{begin:"/(?!\\/)",end:"/[a-z]*"}]}],relevance:0},d=[o,l,c,{className:"regexp",contains:[e.BACKSLASH_ESCAPE,i],variants:[{begin:"%r\\(",end:"\\)",contains:s("\\(","\\)")},{begin:"%r\\[",end:"\\]",contains:s("\\[","\\]")},{begin:"%r\\{",end:/\}/,contains:s(/\{/,/\}/)},{begin:"%r<",end:">",contains:s("<",">")},{begin:"%r\\|",end:"\\|"}],relevance:0},_,{className:"meta",begin:"@\\[",end:"\\]",contains:[e.inherit(e.QUOTE_STRING_MODE,{className:"string"})]},{className:"variable",begin:"(\\$\\W)|((\\$|@@?)(\\w+))(?=[^@$?])(?![A-Za-z])(?![@$?'])"},e.HASH_COMMENT_MODE,{className:"class",beginKeywords:"class module struct",end:"$|;",illegal:/=/,contains:[e.HASH_COMMENT_MODE,e.inherit(e.TITLE_MODE,{begin:a}),{begin:"<"}]},{className:"class",beginKeywords:"lib enum union",end:"$|;",illegal:/=/,contains:[e.HASH_COMMENT_MODE,e.inherit(e.TITLE_MODE,{begin:a})]},{beginKeywords:"annotation",end:"$|;",illegal:/=/,contains:[e.HASH_COMMENT_MODE,e.inherit(e.TITLE_MODE,{begin:a})],relevance:2},{className:"function",beginKeywords:"def",end:/\B\b/,contains:[e.inherit(e.TITLE_MODE,{begin:n,endsParent:!0})]},{className:"function",beginKeywords:"fun macro",end:/\B\b/,contains:[e.inherit(e.TITLE_MODE,{begin:n,endsParent:!0})],relevance:2},{className:"symbol",begin:e.UNDERSCORE_IDENT_RE+"(!|\\?)?:",relevance:0},{className:"symbol",begin:":",contains:[l,{begin:n}],relevance:0},{className:"number",variants:[{begin:"\\b0b([01_]+)"+t},{begin:"\\b0o([0-7_]+)"+t},{begin:"\\b0x([A-Fa-f0-9_]+)"+t},{begin:"\\b([1-9][0-9_]*[0-9]|[0-9])(\\.[0-9][0-9_]*)?([eE]_?[-+]?[0-9_]*)?(_?f(32|64))?(?!_)"},{begin:"\\b([1-9][0-9_]*|0)"+t}],relevance:0}];return i.contains=d,o.contains=d.slice(1),{name:"Crystal",aliases:["cr"],keywords:r,contains:d}}},8675(e){e.exports=function(e){const t={$pattern:/[a-zA-Z][a-zA-Z0-9_?]*/,keyword:["if","then","else","do","while","until","for","loop","import","with","is","as","where","when","by","data","constant","integer","real","text","name","boolean","symbol","infix","prefix","postfix","block","tree"],literal:["true","false","nil"],built_in:["in","mod","rem","and","or","xor","not","abs","sign","floor","ceil","sqrt","sin","cos","tan","asin","acos","atan","exp","expm1","log","log2","log10","log1p","pi","at","text_length","text_range","text_find","text_replace","contains","page","slide","basic_slide","title_slide","title","subtitle","fade_in","fade_out","fade_at","clear_color","color","line_color","line_width","texture_wrap","texture_transform","texture","scale_?x","scale_?y","scale_?z?","translate_?x","translate_?y","translate_?z?","rotate_?x","rotate_?y","rotate_?z?","rectangle","circle","ellipse","sphere","path","line_to","move_to","quad_to","curve_to","theme","background","contents","locally","time","mouse_?x","mouse_?y","mouse_buttons"].concat(["ObjectLoader","Animate","MovieCredits","Slides","Filters","Shading","Materials","LensFlare","Mapping","VLCAudioVideo","StereoDecoder","PointCloud","NetworkAccess","RemoteControl","RegExp","ChromaKey","Snowfall","NodeJS","Speech","Charts"])},n={className:"string",begin:'"',end:'"',illegal:"\\n"},a={beginKeywords:"import",end:"$",keywords:t,contains:[n]},r={className:"function",begin:/[a-z][^\n]*->/,returnBegin:!0,end:/->/,contains:[e.inherit(e.TITLE_MODE,{starts:{endsWithParent:!0,keywords:t}})]};return{name:"XL",aliases:["tao"],keywords:t,contains:[e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE,n,{className:"string",begin:"'",end:"'",illegal:"\\n"},{className:"string",begin:"<<",end:">>"},r,a,{className:"number",begin:"[0-9]+#[0-9A-Z_]+(\\.[0-9-A-Z_]+)?#?([Ee][+-]?[0-9]+)?"},e.NUMBER_MODE]}}},8721(e){e.exports=function(e){const t="\\d(_|\\d)*",n="[eE][-+]?"+t,a="\\b("+t+"#\\w+(\\.\\w+)?#("+n+")?|"+t+"(\\."+t+")?("+n+")?)";return{name:"VHDL",case_insensitive:!0,keywords:{keyword:["abs","access","after","alias","all","and","architecture","array","assert","assume","assume_guarantee","attribute","begin","block","body","buffer","bus","case","component","configuration","constant","context","cover","disconnect","downto","default","else","elsif","end","entity","exit","fairness","file","for","force","function","generate","generic","group","guarded","if","impure","in","inertial","inout","is","label","library","linkage","literal","loop","map","mod","nand","new","next","nor","not","null","of","on","open","or","others","out","package","parameter","port","postponed","procedure","process","property","protected","pure","range","record","register","reject","release","rem","report","restrict","restrict_guarantee","return","rol","ror","select","sequence","severity","shared","signal","sla","sll","sra","srl","strong","subtype","then","to","transport","type","unaffected","units","until","use","variable","view","vmode","vprop","vunit","wait","when","while","with","xnor","xor"],built_in:["boolean","bit","character","integer","time","delay_length","natural","positive","string","bit_vector","file_open_kind","file_open_status","std_logic","std_logic_vector","unsigned","signed","boolean_vector","integer_vector","std_ulogic","std_ulogic_vector","unresolved_unsigned","u_unsigned","unresolved_signed","u_signed","real_vector","time_vector"],literal:["false","true","note","warning","error","failure","line","text","side","width"]},illegal:/\{/,contains:[e.C_BLOCK_COMMENT_MODE,e.COMMENT("--","$"),e.QUOTE_STRING_MODE,{className:"number",begin:a,relevance:0},{className:"string",begin:"'(U|X|0|1|Z|W|L|H|-)'",contains:[e.BACKSLASH_ESCAPE]},{className:"symbol",begin:"'[A-Za-z](_?[A-Za-z0-9])*",contains:[e.BACKSLASH_ESCAPE]}]}}},8722(e){e.exports=function(e){const t="('|\\.')+",n={relevance:0,contains:[{begin:t}]};return{name:"Matlab",keywords:{keyword:"arguments break case catch classdef continue else elseif end enumeration events for function global if methods otherwise parfor persistent properties return spmd switch try while",built_in:"sin sind sinh asin asind asinh cos cosd cosh acos acosd acosh tan tand tanh atan atand atan2 atanh sec secd sech asec asecd asech csc cscd csch acsc acscd acsch cot cotd coth acot acotd acoth hypot exp expm1 log log1p log10 log2 pow2 realpow reallog realsqrt sqrt nthroot nextpow2 abs angle complex conj imag real unwrap isreal cplxpair fix floor ceil round mod rem sign airy besselj bessely besselh besseli besselk beta betainc betaln ellipj ellipke erf erfc erfcx erfinv expint gamma gammainc gammaln psi legendre cross dot factor isprime primes gcd lcm rat rats perms nchoosek factorial cart2sph cart2pol pol2cart sph2cart hsv2rgb rgb2hsv zeros ones eye repmat rand randn linspace logspace freqspace meshgrid accumarray size length ndims numel disp isempty isequal isequalwithequalnans cat reshape diag blkdiag tril triu fliplr flipud flipdim rot90 find sub2ind ind2sub bsxfun ndgrid permute ipermute shiftdim circshift squeeze isscalar isvector ans eps realmax realmin pi i|0 inf nan isnan isinf isfinite j|0 why compan gallery hadamard hankel hilb invhilb magic pascal rosser toeplitz vander wilkinson max min nanmax nanmin mean nanmean type table readtable writetable sortrows sort figure plot plot3 scatter scatter3 cellfun legend intersect ismember procrustes hold num2cell "},illegal:'(//|"|#|/\\*|\\s+/\\w+)',contains:[{className:"function",beginKeywords:"function",end:"$",contains:[e.UNDERSCORE_TITLE_MODE,{className:"params",variants:[{begin:"\\(",end:"\\)"},{begin:"\\[",end:"\\]"}]}]},{className:"built_in",begin:/true|false/,relevance:0,starts:n},{begin:"[a-zA-Z][a-zA-Z_0-9]*"+t,relevance:0},{className:"number",begin:e.C_NUMBER_RE,relevance:0,starts:n},{className:"string",begin:"'",end:"'",contains:[{begin:"''"}]},{begin:/\]|\}|\)/,relevance:0,starts:n},{className:"string",begin:'"',end:'"',contains:[{begin:'""'}],starts:n},e.COMMENT("^\\s*%\\{\\s*$","^\\s*%\\}\\s*$"),e.COMMENT("%","$")]}}},8725(e,t){"use strict";var n=this&&this.__assign||function(){return n=Object.assign||function(e){for(var t,n=1,a=arguments.length;n)+[a-zA-Z_\x7f-\xff][a-zA-Z0-9_\x7f-\xff]*/},{className:"function",beginKeywords:"function fn",end:/[;{]/,excludeEnd:!0,illegal:/\$|\[|%/,contains:[n,{className:"params",begin:/\(/,end:/\)/,keywords:r,contains:["self",e.C_BLOCK_COMMENT_MODE,t,a]}]},{className:"class",beginKeywords:"class interface",end:/\{/,excludeEnd:!0,illegal:/[:($"]/,contains:[{beginKeywords:"extends implements"},n]},{beginKeywords:"namespace",end:/;/,illegal:/[.']/,contains:[n]},{beginKeywords:"use",end:/;/,contains:[n]},{begin:/=>/},t,a]}}},8813(e){e.exports=function(e){return{name:"Shell Session",aliases:["console","shellsession"],contains:[{className:"meta.prompt",begin:/^\s{0,3}[/~\w\d[\]()@-]*[>%$#][ ]?/,starts:{end:/[^\\](?=\s*$)/,subLanguage:"bash"}}]}}},8837(e){e.exports=function(e){return{name:"Packet Filter config",aliases:["pf.conf"],keywords:{$pattern:/[a-z0-9_<>-]+/,built_in:"block match pass load anchor|5 antispoof|10 set table",keyword:"in out log quick on rdomain inet inet6 proto from port os to route allow-opts divert-packet divert-reply divert-to flags group icmp-type icmp6-type label once probability recieved-on rtable prio queue tos tag tagged user keep fragment for os drop af-to|10 binat-to|10 nat-to|10 rdr-to|10 bitmask least-stats random round-robin source-hash static-port dup-to reply-to route-to parent bandwidth default min max qlimit block-policy debug fingerprints hostid limit loginterface optimization reassemble ruleset-optimization basic none profile skip state-defaults state-policy timeout const counters persist no modulate synproxy state|5 floating if-bound no-sync pflow|10 sloppy source-track global rule max-src-nodes max-src-states max-src-conn max-src-conn-rate overload flush scrub|5 max-mss min-ttl no-df|10 random-id",literal:"all any no-route self urpf-failed egress|5 unknown"},contains:[e.HASH_COMMENT_MODE,e.NUMBER_MODE,e.QUOTE_STRING_MODE,{className:"variable",begin:/\$[\w\d#@][\w\d_]*/,relevance:0},{className:"variable",begin:/<(?!\/)/,end:/>/}]}}},8843(e,t,n){"use strict";var a,r=this&&this.__extends||(a=function(e,t){return a=Object.setPrototypeOf||{__proto__:[]}instanceof Array&&function(e,t){e.__proto__=t}||function(e,t){for(var n in t)Object.prototype.hasOwnProperty.call(t,n)&&(e[n]=t[n])},a(e,t)},function(e,t){if("function"!=typeof t&&null!==t)throw new TypeError("Class extends value "+String(t)+" is not a constructor or null");function n(){this.constructor=e}a(e,t),e.prototype=null===t?Object.create(t):(n.prototype=t.prototype,new n)});Object.defineProperty(t,"__esModule",{value:!0}),t.jsonDiff=void 0,t.diffJson=function(e,n,a){return t.jsonDiff.diff(e,n,a)},t.canonicalize=l;var i=n(2188),o=n(9839),s=function(e){function t(){var t=null!==e&&e.apply(this,arguments)||this;return t.tokenize=o.tokenize,t}return r(t,e),Object.defineProperty(t.prototype,"useLongestToken",{get:function(){return!0},enumerable:!1,configurable:!0}),t.prototype.castInput=function(e,t){var n=t.undefinedReplacement,a=t.stringifyReplacer,r=void 0===a?function(e,t){return void 0===t?n:t}:a;return"string"==typeof e?e:JSON.stringify(l(e,null,null,r),null," ")},t.prototype.equals=function(t,n,a){return e.prototype.equals.call(this,t.replace(/,([\r\n])/g,"$1"),n.replace(/,([\r\n])/g,"$1"),a)},t}(i.default);function l(e,t,n,a,r){var i,o;for(t=t||[],n=n||[],a&&(e=a(void 0===r?"":r,e)),i=0;i/}]}}},8928(e){e.exports=function(e){const t=e.regex,n=/\d{1,2}\/\d{1,2}\/\d{4}/,a=/\d{4}-\d{1,2}-\d{1,2}/,r=/(\d|1[012])(:\d+){0,2} *(AM|PM)/,i=/\d{1,2}(:\d{1,2}){1,2}/,o={className:"literal",variants:[{begin:t.concat(/# */,t.either(a,n),/ *#/)},{begin:t.concat(/# */,i,/ *#/)},{begin:t.concat(/# */,r,/ *#/)},{begin:t.concat(/# */,t.either(a,n),/ +/,t.either(r,i),/ *#/)}]},s=e.COMMENT(/'''/,/$/,{contains:[{className:"doctag",begin:/<\/?/,end:/>/}]}),l=e.COMMENT(null,/$/,{variants:[{begin:/'/},{begin:/([\t ]|^)REM(?=\s)/}]});return{name:"Visual Basic .NET",aliases:["vb"],case_insensitive:!0,classNameAliases:{label:"symbol"},keywords:{keyword:"addhandler alias aggregate ansi as async assembly auto binary by byref byval call case catch class compare const continue custom declare default delegate dim distinct do each equals else elseif end enum erase error event exit explicit finally for friend from function get global goto group handles if implements imports in inherits interface into iterator join key let lib loop me mid module mustinherit mustoverride mybase myclass namespace narrowing new next notinheritable notoverridable of off on operator option optional order overloads overridable overrides paramarray partial preserve private property protected public raiseevent readonly redim removehandler resume return select set shadows shared skip static step stop structure strict sub synclock take text then throw to try unicode until using when where while widening with withevents writeonly yield",built_in:"addressof and andalso await directcast gettype getxmlnamespace is isfalse isnot istrue like mod nameof new not or orelse trycast typeof xor cbool cbyte cchar cdate cdbl cdec cint clng cobj csbyte cshort csng cstr cuint culng cushort",type:"boolean byte char date decimal double integer long object sbyte short single string uinteger ulong ushort",literal:"true false nothing"},illegal:"//|\\{|\\}|endif|gosub|variant|wend|^\\$ ",contains:[{className:"string",begin:/"(""|[^/n])"C\b/},{className:"string",begin:/"/,end:/"/,illegal:/\n/,contains:[{begin:/""/}]},o,{className:"number",relevance:0,variants:[{begin:/\b\d[\d_]*((\.[\d_]+(E[+-]?[\d_]+)?)|(E[+-]?[\d_]+))[RFD@!#]?/},{begin:/\b\d[\d_]*((U?[SIL])|[%&])?/},{begin:/&H[\dA-F_]+((U?[SIL])|[%&])?/},{begin:/&O[0-7_]+((U?[SIL])|[%&])?/},{begin:/&B[01_]+((U?[SIL])|[%&])?/}]},{className:"label",begin:/^\w+:/},s,l,{className:"meta",begin:/[\t ]*#(const|disable|else|elseif|enable|end|externalsource|if|region)\b/,end:/$/,keywords:{keyword:"const disable else elseif enable end externalsource if region then"},contains:[l]}]}}},8942(e){e.exports=function(e){return{name:"Test Anything Protocol",case_insensitive:!0,contains:[e.HASH_COMMENT_MODE,{className:"meta",variants:[{begin:"^TAP version (\\d+)$"},{begin:"^1\\.\\.(\\d+)$"}]},{begin:/---$/,end:"\\.\\.\\.$",subLanguage:"yaml",relevance:0},{className:"number",begin:" (\\d+) "},{className:"symbol",variants:[{begin:"^ok"},{begin:"^not ok"}]}]}}},9025(e,t,n){"use strict";var a,r=this&&this.__extends||(a=function(e,t){return a=Object.setPrototypeOf||{__proto__:[]}instanceof Array&&function(e,t){e.__proto__=t}||function(e,t){for(var n in t)Object.prototype.hasOwnProperty.call(t,n)&&(e[n]=t[n])},a(e,t)},function(e,t){if("function"!=typeof t&&null!==t)throw new TypeError("Class extends value "+String(t)+" is not a constructor or null");function n(){this.constructor=e}a(e,t),e.prototype=null===t?Object.create(t):(n.prototype=t.prototype,new n)});Object.defineProperty(t,"__esModule",{value:!0}),t.wordsWithSpaceDiff=t.wordDiff=void 0,t.diffWords=function(e,n,a){return null==(null==a?void 0:a.ignoreWhitespace)||a.ignoreWhitespace?t.wordDiff.diff(e,n,a):p(e,n,a)},t.diffWordsWithSpace=p;var i=n(2188),o=n(6665),s="a-zA-Z0-9_\\u{AD}\\u{C0}-\\u{D6}\\u{D8}-\\u{F6}\\u{F8}-\\u{2C6}\\u{2C8}-\\u{2D7}\\u{2DE}-\\u{2FF}\\u{1E00}-\\u{1EFF}",l=new RegExp("[".concat(s,"]+|\\s+|[^").concat(s,"]"),"ug"),c=function(e){function t(){return null!==e&&e.apply(this,arguments)||this}return r(t,e),t.prototype.equals=function(e,t,n){return n.ignoreCase&&(e=e.toLowerCase(),t=t.toLowerCase()),e.trim()===t.trim()},t.prototype.tokenize=function(e,t){var n;if(void 0===t&&(t={}),t.intlSegmenter){var a=t.intlSegmenter;if("word"!=a.resolvedOptions().granularity)throw new Error('The segmenter passed must have a granularity of "word"');n=[];for(var r=0,i=Array.from(a.segment(e));r",s={className:"params",begin:"\\([^\\(]",returnBegin:!0,contains:[{begin:/\(/,end:/\)/,keywords:t,contains:["self"].concat(r)}]};return{name:"MoonScript",aliases:["moon"],keywords:t,illegal:/\/\*/,contains:r.concat([e.COMMENT("--","$"),{className:"function",begin:"^\\s*"+n+"\\s*=\\s*"+o,end:"[-=]>",returnBegin:!0,contains:[i,s]},{begin:/[\(,:=]\s*/,relevance:0,contains:[{className:"function",begin:o,end:"[-=]>",returnBegin:!0,contains:[s]}]},{className:"class",beginKeywords:"class",end:"$",illegal:/[:="\[\]]/,contains:[{beginKeywords:"extends",endsWithParent:!0,illegal:/[:="\[\]]/,contains:[i]},i]},{className:"name",begin:n+":",end:":",returnBegin:!0,returnEnd:!0,relevance:0}])}}},9296(e){e.exports=function(e){return{name:"Caché Object Script",case_insensitive:!0,aliases:["cls"],keywords:"property parameter class classmethod clientmethod extends as break catch close continue do d|0 else elseif for goto halt hang h|0 if job j|0 kill k|0 lock l|0 merge new open quit q|0 read r|0 return set s|0 tcommit throw trollback try tstart use view while write w|0 xecute x|0 zkill znspace zn ztrap zwrite zw zzdump zzwrite print zbreak zinsert zload zprint zremove zsave zzprint mv mvcall mvcrt mvdim mvprint zquit zsync ascii",contains:[{className:"number",begin:"\\b(\\d+(\\.\\d*)?|\\.\\d+)",relevance:0},{className:"string",variants:[{begin:'"',end:'"',contains:[{begin:'""',relevance:0}]}]},e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE,{className:"comment",begin:/;/,end:"$",relevance:0},{className:"built_in",begin:/(?:\$\$?|\.\.)\^?[a-zA-Z]+/},{className:"built_in",begin:/\$\$\$[a-zA-Z]+/},{className:"built_in",begin:/%[a-z]+(?:\.[a-z]+)*/},{className:"symbol",begin:/\^%?[a-zA-Z][\w]*/},{className:"keyword",begin:/##class|##super|#define|#dim/},{begin:/&sql\(/,end:/\)/,excludeBegin:!0,excludeEnd:!0,subLanguage:"sql"},{begin:/&(js|jscript|javascript)/,excludeBegin:!0,excludeEnd:!0,subLanguage:"javascript"},{begin:/&html<\s*\s*>/,subLanguage:"xml"}]}}},9301(e){function t(e){return new RegExp(e.replace(/[-/\\^$*+?.()|[\]{}]/g,"\\$&"),"m")}function n(e){return e?"string"==typeof e?e:e.source:null}function a(e){return r("(?=",e,")")}function r(...e){return e.map((e=>n(e))).join("")}function i(...e){const t=function(e){const t=e[e.length-1];return"object"==typeof t&&t.constructor===Object?(e.splice(e.length-1,1),t):{}}(e);return"("+(t.capture?"":"?:")+e.map((e=>n(e))).join("|")+")"}e.exports=function(e){const n={scope:"keyword",match:/\b(yield|return|let|do|match|use)!/},o=["bool","byte","sbyte","int8","int16","int32","uint8","uint16","uint32","int","uint","int64","uint64","nativeint","unativeint","decimal","float","double","float32","single","char","string","unit","bigint","option","voption","list","array","seq","byref","exn","inref","nativeptr","obj","outref","voidptr","Result"],s={keyword:["abstract","and","as","assert","base","begin","class","default","delegate","do","done","downcast","downto","elif","else","end","exception","extern","finally","fixed","for","fun","function","global","if","in","inherit","inline","interface","internal","lazy","let","match","member","module","mutable","namespace","new","of","open","or","override","private","public","rec","return","static","struct","then","to","try","type","upcast","use","val","void","when","while","with","yield"],literal:["true","false","null","Some","None","Ok","Error","infinity","infinityf","nan","nanf"],built_in:["not","ref","raise","reraise","dict","readOnlyDict","set","get","enum","sizeof","typeof","typedefof","nameof","nullArg","invalidArg","invalidOp","id","fst","snd","ignore","lock","using","box","unbox","tryUnbox","printf","printfn","sprintf","eprintf","eprintfn","fprintf","fprintfn","failwith","failwithf"],"variable.constant":["__LINE__","__SOURCE_DIRECTORY__","__SOURCE_FILE__"]},l={variants:[e.COMMENT(/\(\*(?!\))/,/\*\)/,{contains:["self"]}),e.C_LINE_COMMENT_MODE]},c={scope:"variable",begin:/``/,end:/``/},_=/\B('|\^)/,d={scope:"symbol",variants:[{match:r(_,/``.*?``/)},{match:r(_,e.UNDERSCORE_IDENT_RE)}],relevance:0},p=function({includeEqual:e}){let n;n=e?"!%&*+-/<=>@^|~?":"!%&*+-/<>@^|~?";const o=r("[",...Array.from(n).map(t),"]"),s=i(o,/\./),l=r(s,a(s)),c=i(r(l,s,"*"),r(o,"+"));return{scope:"operator",match:i(c,/:\?>/,/:\?/,/:>/,/:=/,/::?/,/\$/),relevance:0}},u=p({includeEqual:!0}),m=p({includeEqual:!1}),g=function(t,n){return{begin:r(t,a(r(/\s*/,i(/\w/,/'/,/\^/,/#/,/``/,/\(/,/{\|/)))),beginScope:n,end:a(i(/\n/,/=/)),relevance:0,keywords:e.inherit(s,{type:o}),contains:[l,d,e.inherit(c,{scope:null}),m]}},E=g(/:/,"operator"),S=g(/\bof\b/,"keyword"),b={begin:[/(^|\s+)/,/type/,/\s+/,/[a-zA-Z_](\w|')*/],beginScope:{2:"keyword",4:"title.class"},end:a(/\(|=|$/),keywords:s,contains:[l,e.inherit(c,{scope:null}),d,{scope:"operator",match:/<|>/},E]},f={scope:"computation-expression",match:/\b[_a-z]\w*(?=\s*\{)/},T={begin:[/^\s*/,r(/#/,i("if","else","endif","line","nowarn","light","r","i","I","load","time","help","quit")),/\b/],beginScope:{2:"meta"},end:a(/\s|$/)},h={variants:[e.BINARY_NUMBER_MODE,e.C_NUMBER_MODE]},C={scope:"string",begin:/"/,end:/"/,contains:[e.BACKSLASH_ESCAPE]},O={scope:"string",begin:/@"/,end:/"/,contains:[{match:/""/},e.BACKSLASH_ESCAPE]},N={scope:"string",begin:/"""/,end:/"""/,relevance:2},R={scope:"subst",begin:/\{/,end:/\}/,keywords:s},v={scope:"string",begin:/\$"/,end:/"/,contains:[{match:/\{\{/},{match:/\}\}/},e.BACKSLASH_ESCAPE,R]},y={scope:"string",begin:/(\$@|@\$)"/,end:/"/,contains:[{match:/\{\{/},{match:/\}\}/},{match:/""/},e.BACKSLASH_ESCAPE,R]},A={scope:"string",begin:/\$"""/,end:/"""/,contains:[{match:/\{\{/},{match:/\}\}/},R],relevance:2},I={scope:"string",match:r(/'/,i(/[^\\']/,/\\(?:.|\d{3}|x[a-fA-F\d]{2}|u[a-fA-F\d]{4}|U[a-fA-F\d]{8})/),/'/)};return R.contains=[y,v,O,C,I,n,l,c,E,f,T,h,d,u],{name:"F#",aliases:["fs","f#"],keywords:s,illegal:/\/\*/,classNameAliases:{"computation-expression":"keyword"},contains:[n,{variants:[A,y,v,N,O,C,I]},l,c,b,{scope:"meta",begin:/\[\]/,relevance:2,contains:[c,N,O,C,I,h]},S,E,f,T,h,d,u]}}},9351(e){e.exports=function(e){e.regex;const t=e.COMMENT(/\(;/,/;\)/);return t.contains.push("self"),{name:"WebAssembly",keywords:{$pattern:/[\w.]+/,keyword:["anyfunc","block","br","br_if","br_table","call","call_indirect","data","drop","elem","else","end","export","func","global.get","global.set","local.get","local.set","local.tee","get_global","get_local","global","if","import","local","loop","memory","memory.grow","memory.size","module","mut","nop","offset","param","result","return","select","set_global","set_local","start","table","tee_local","then","type","unreachable"]},contains:[e.COMMENT(/;;/,/$/),t,{match:[/(?:offset|align)/,/\s*/,/=/],className:{1:"keyword",3:"operator"}},{className:"variable",begin:/\$[\w_]+/},{match:/(\((?!;)|\))+/,className:"punctuation",relevance:0},{begin:[/(?:func|call|call_indirect)/,/\s+/,/\$[^\s)]+/],className:{1:"keyword",3:"title.function"}},e.QUOTE_STRING_MODE,{match:/(i32|i64|f32|f64)(?!\.)/,className:"type"},{className:"keyword",match:/\b(f32|f64|i32|i64)(?:\.(?:abs|add|and|ceil|clz|const|convert_[su]\/i(?:32|64)|copysign|ctz|demote\/f64|div(?:_[su])?|eqz?|extend_[su]\/i32|floor|ge(?:_[su])?|gt(?:_[su])?|le(?:_[su])?|load(?:(?:8|16|32)_[su])?|lt(?:_[su])?|max|min|mul|nearest|neg?|or|popcnt|promote\/f32|reinterpret\/[fi](?:32|64)|rem_[su]|rot[lr]|shl|shr_[su]|store(?:8|16|32)?|sqrt|sub|trunc(?:_[su]\/f(?:32|64))?|wrap\/i64|xor))\b/},{className:"number",relevance:0,match:/[+-]?\b(?:\d(?:_?\d)*(?:\.\d(?:_?\d)*)?(?:[eE][+-]?\d(?:_?\d)*)?|0x[\da-fA-F](?:_?[\da-fA-F])*(?:\.[\da-fA-F](?:_?[\da-fA-D])*)?(?:[pP][+-]?\d(?:_?\d)*)?)\b|\binf\b|\bnan(?::0x[\da-fA-F](?:_?[\da-fA-D])*)?\b/}]}}},9363(e,t){"use strict";Object.defineProperty(t,"__esModule",{value:!0}),t.parsePatch=function(e){var t=e.split(/\n/),n=[],a=0;function r(){var e={};for(n.push(e);a/,starts:{end:/$/,subLanguage:"clojure"}}]}}},9632(e){e.exports=function(e){const t=e.regex,n="[a-zA-Z_][a-zA-Z0-9_.]*(!|\\?)?",a={$pattern:n,keyword:["after","alias","and","case","catch","cond","defstruct","defguard","do","else","end","fn","for","if","import","in","not","or","quote","raise","receive","require","reraise","rescue","try","unless","unquote","unquote_splicing","use","when","with|0"],literal:["false","nil","true"]},r={className:"subst",begin:/#\{/,end:/\}/,keywords:a},i={match:/\\[\s\S]/,scope:"char.escape",relevance:0},o="[/|([{<\"']",s=[{begin:/"/,end:/"/},{begin:/'/,end:/'/},{begin:/\//,end:/\//},{begin:/\|/,end:/\|/},{begin:/\(/,end:/\)/},{begin:/\[/,end:/\]/},{begin:/\{/,end:/\}/},{begin://}],l=e=>({scope:"char.escape",begin:t.concat(/\\/,e),relevance:0}),c={className:"string",begin:"~[a-z](?="+o+")",contains:s.map((t=>e.inherit(t,{contains:[l(t.end),i,r]})))},_={className:"string",begin:"~[A-Z](?="+o+")",contains:s.map((t=>e.inherit(t,{contains:[l(t.end)]})))},d={className:"regex",variants:[{begin:"~r(?="+o+")",contains:s.map((n=>e.inherit(n,{end:t.concat(n.end,/[uismxfU]{0,7}/),contains:[l(n.end),i,r]})))},{begin:"~R(?="+o+")",contains:s.map((n=>e.inherit(n,{end:t.concat(n.end,/[uismxfU]{0,7}/),contains:[l(n.end)]})))}]},p={className:"string",contains:[e.BACKSLASH_ESCAPE,r],variants:[{begin:/"""/,end:/"""/},{begin:/'''/,end:/'''/},{begin:/~S"""/,end:/"""/,contains:[]},{begin:/~S"/,end:/"/,contains:[]},{begin:/~S'''/,end:/'''/,contains:[]},{begin:/~S'/,end:/'/,contains:[]},{begin:/'/,end:/'/},{begin:/"/,end:/"/}]},u={className:"function",beginKeywords:"def defp defmacro defmacrop",end:/\B\b/,contains:[e.inherit(e.TITLE_MODE,{begin:n,endsParent:!0})]},m=e.inherit(u,{className:"class",beginKeywords:"defimpl defmodule defprotocol defrecord",end:/\bdo\b|$|;/}),g=[p,d,_,c,e.HASH_COMMENT_MODE,m,u,{begin:"::"},{className:"symbol",begin:":(?![\\s:])",contains:[p,{begin:"[a-zA-Z_]\\w*[!?=]?|[-+~]@|<<|>>|=~|===?|<=>|[<>]=?|\\*\\*|[-/+%^&*~`|]|\\[\\]=?"}],relevance:0},{className:"symbol",begin:n+":(?!:)",relevance:0},{className:"title.class",begin:/(\b[A-Z][a-zA-Z0-9_]+)/,relevance:0},{className:"number",begin:"(\\b0o[0-7_]+)|(\\b0b[01_]+)|(\\b0x[0-9a-fA-F_]+)|(-?\\b[0-9][0-9_]*(\\.[0-9_]+([eE][-+]?[0-9]+)?)?)",relevance:0},{className:"variable",begin:"(\\$\\W)|((\\$|@@?)(\\w+))"}];return r.contains=g,{name:"Elixir",aliases:["ex","exs"],keywords:a,contains:g}}},9639(e){e.exports=function(e){const t={variants:[{match:[/(struct|enum|interface)/,/\s+/,e.IDENT_RE]},{match:[/extends/,/\s*\(/,e.IDENT_RE,/\s*\)/]}],scope:{1:"keyword",3:"title.class"}};return{name:"Cap’n Proto",aliases:["capnp"],keywords:{keyword:["struct","enum","interface","union","group","import","using","const","annotation","extends","in","of","on","as","with","from","fixed"],type:["Void","Bool","Int8","Int16","Int32","Int64","UInt8","UInt16","UInt32","UInt64","Float32","Float64","Text","Data","AnyPointer","AnyStruct","Capability","List"],literal:["true","false"]},contains:[e.QUOTE_STRING_MODE,e.NUMBER_MODE,e.HASH_COMMENT_MODE,{className:"meta",begin:/@0x[\w\d]{16};/,illegal:/\n/},{className:"symbol",begin:/@\d+\b/},t]}}},9660(e,t){"use strict";Object.defineProperty(t,"__esModule",{value:!0}),t.convertChangesToXML=function(e){for(var t=[],n=0;n"):a.removed&&t.push(""),t.push(a.value.replace(/&/g,"&").replace(//g,">").replace(/"/g,""")),a.added?t.push(""):a.removed&&t.push("")}return t.join("")}},9722(e){e.exports=function(e){return{name:"Coq",keywords:{keyword:["_|0","as","at","cofix","else","end","exists","exists2","fix","for","forall","fun","if","IF","in","let","match","mod","Prop","return","Set","then","Type","using","where","with","Abort","About","Add","Admit","Admitted","All","Arguments","Assumptions","Axiom","Back","BackTo","Backtrack","Bind","Blacklist","Canonical","Cd","Check","Class","Classes","Close","Coercion","Coercions","CoFixpoint","CoInductive","Collection","Combined","Compute","Conjecture","Conjectures","Constant","constr","Constraint","Constructors","Context","Corollary","CreateHintDb","Cut","Declare","Defined","Definition","Delimit","Dependencies","Dependent","Derive","Drop","eauto","End","Equality","Eval","Example","Existential","Existentials","Existing","Export","exporting","Extern","Extract","Extraction","Fact","Field","Fields","File","Fixpoint","Focus","for","From","Function","Functional","Generalizable","Global","Goal","Grab","Grammar","Graph","Guarded","Heap","Hint","HintDb","Hints","Hypotheses","Hypothesis","ident","Identity","If","Immediate","Implicit","Import","Include","Inductive","Infix","Info","Initial","Inline","Inspect","Instance","Instances","Intro","Intros","Inversion","Inversion_clear","Language","Left","Lemma","Let","Libraries","Library","Load","LoadPath","Local","Locate","Ltac","ML","Mode","Module","Modules","Monomorphic","Morphism","Next","NoInline","Notation","Obligation","Obligations","Opaque","Open","Optimize","Options","Parameter","Parameters","Parametric","Path","Paths","pattern","Polymorphic","Preterm","Print","Printing","Program","Projections","Proof","Proposition","Pwd","Qed","Quit","Rec","Record","Recursive","Redirect","Relation","Remark","Remove","Require","Reserved","Reset","Resolve","Restart","Rewrite","Right","Ring","Rings","Save","Scheme","Scope","Scopes","Script","Search","SearchAbout","SearchHead","SearchPattern","SearchRewrite","Section","Separate","Set","Setoid","Show","Solve","Sorted","Step","Strategies","Strategy","Structure","SubClass","Table","Tables","Tactic","Term","Test","Theorem","Time","Timeout","Transparent","Type","Typeclasses","Types","Undelimit","Undo","Unfocus","Unfocused","Unfold","Universe","Universes","Unset","Unshelve","using","Variable","Variables","Variant","Verbose","Visibility","where","with"],built_in:["abstract","absurd","admit","after","apply","as","assert","assumption","at","auto","autorewrite","autounfold","before","bottom","btauto","by","case","case_eq","cbn","cbv","change","classical_left","classical_right","clear","clearbody","cofix","compare","compute","congruence","constr_eq","constructor","contradict","contradiction","cut","cutrewrite","cycle","decide","decompose","dependent","destruct","destruction","dintuition","discriminate","discrR","do","double","dtauto","eapply","eassumption","eauto","ecase","econstructor","edestruct","ediscriminate","eelim","eexact","eexists","einduction","einjection","eleft","elim","elimtype","enough","equality","erewrite","eright","esimplify_eq","esplit","evar","exact","exactly_once","exfalso","exists","f_equal","fail","field","field_simplify","field_simplify_eq","first","firstorder","fix","fold","fourier","functional","generalize","generalizing","gfail","give_up","has_evar","hnf","idtac","in","induction","injection","instantiate","intro","intro_pattern","intros","intuition","inversion","inversion_clear","is_evar","is_var","lapply","lazy","left","lia","lra","move","native_compute","nia","nsatz","omega","once","pattern","pose","progress","proof","psatz","quote","record","red","refine","reflexivity","remember","rename","repeat","replace","revert","revgoals","rewrite","rewrite_strat","right","ring","ring_simplify","rtauto","set","setoid_reflexivity","setoid_replace","setoid_rewrite","setoid_symmetry","setoid_transitivity","shelve","shelve_unifiable","simpl","simple","simplify_eq","solve","specialize","split","split_Rabs","split_Rmult","stepl","stepr","subst","sum","swap","symmetry","tactic","tauto","time","timeout","top","transitivity","trivial","try","tryif","unfold","unify","until","using","vm_compute","with"]},contains:[e.QUOTE_STRING_MODE,e.COMMENT("\\(\\*","\\*\\)"),e.C_NUMBER_MODE,{className:"type",excludeBegin:!0,begin:"\\|\\s*",end:"\\w+"},{begin:/[-=]>/}]}}},9723(e){e.exports=function(e){return{name:"Dust",aliases:["dst"],case_insensitive:!0,subLanguage:"xml",contains:[{className:"template-tag",begin:/\{[#\/]/,end:/\}/,illegal:/;/,contains:[{className:"name",begin:/[a-zA-Z\.-]+/,starts:{endsWithParent:!0,relevance:0,contains:[e.QUOTE_STRING_MODE]}}]},{className:"template-variable",begin:/\{/,end:/\}/,illegal:/;/,keywords:"if eq ne lt lte gt gte select default math sep"}]}}},9744(e){e.exports=function(e){return{name:"Flix",keywords:{keyword:["case","class","def","else","enum","if","impl","import","in","lat","rel","index","let","match","namespace","switch","type","yield","with"],literal:["true","false"]},contains:[e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE,{className:"string",begin:/'(.|\\[xXuU][a-zA-Z0-9]+)'/},{className:"string",variants:[{begin:'"',end:'"'}]},{className:"function",beginKeywords:"def",end:/[:={\[(\n;]/,excludeEnd:!0,contains:[{className:"title",relevance:0,begin:/[^0-9\n\t "'(),.`{}\[\]:;][^\n\t "'(),.`{}\[\]:;]+|[^0-9\n\t "'(),.`{}\[\]:;=]/}]},e.C_NUMBER_MODE]}}},9777(e){e.exports=function(e){const t={keyword:["break","case","chan","const","continue","default","defer","else","fallthrough","for","func","go","goto","if","import","interface","map","package","range","return","select","struct","switch","type","var"],type:["bool","byte","complex64","complex128","error","float32","float64","int8","int16","int32","int64","string","uint8","uint16","uint32","uint64","int","uint","uintptr","rune"],literal:["true","false","iota","nil"],built_in:["append","cap","close","complex","copy","imag","len","make","new","panic","print","println","real","recover","delete"]};return{name:"Go",aliases:["golang"],keywords:t,illegal:"\s*[);\]]/,relevance:0,subLanguage:"xml"}],relevance:0},{className:"keyword",begin:"\\bsignal\\b",starts:{className:"string",end:"(\\(|:|=|;|,|//|/\\*|$)",returnEnd:!0}},{className:"keyword",begin:"\\bproperty\\b",starts:{className:"string",end:"(:|=|;|,|//|/\\*|$)",returnEnd:!0}},{className:"function",beginKeywords:"function",end:/\{/,excludeEnd:!0,contains:[e.inherit(e.TITLE_MODE,{begin:/[A-Za-z$_][0-9A-Za-z$_]*/}),{className:"params",begin:/\(/,end:/\)/,excludeBegin:!0,excludeEnd:!0,contains:[e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE]}],illegal:/\[|%/},{begin:"\\."+e.IDENT_RE,relevance:0},n,a,r],illegal:/#/}}},9827(e){e.exports=function(e){return{name:"N1QL",case_insensitive:!0,contains:[{beginKeywords:"build create index delete drop explain infer|10 insert merge prepare select update upsert|10",end:/;/,keywords:{keyword:["all","alter","analyze","and","any","array","as","asc","begin","between","binary","boolean","break","bucket","build","by","call","case","cast","cluster","collate","collection","commit","connect","continue","correlate","cover","create","database","dataset","datastore","declare","decrement","delete","derived","desc","describe","distinct","do","drop","each","element","else","end","every","except","exclude","execute","exists","explain","fetch","first","flatten","for","force","from","function","grant","group","gsi","having","if","ignore","ilike","in","include","increment","index","infer","inline","inner","insert","intersect","into","is","join","key","keys","keyspace","known","last","left","let","letting","like","limit","lsm","map","mapping","matched","materialized","merge","minus","namespace","nest","not","number","object","offset","on","option","or","order","outer","over","parse","partition","password","path","pool","prepare","primary","private","privilege","procedure","public","raw","realm","reduce","rename","return","returning","revoke","right","role","rollback","satisfies","schema","select","self","semi","set","show","some","start","statistics","string","system","then","to","transaction","trigger","truncate","under","union","unique","unknown","unnest","unset","update","upsert","use","user","using","validate","value","valued","values","via","view","when","where","while","with","within","work","xor"],literal:["true","false","null","missing|5"],built_in:["array_agg","array_append","array_concat","array_contains","array_count","array_distinct","array_ifnull","array_length","array_max","array_min","array_position","array_prepend","array_put","array_range","array_remove","array_repeat","array_replace","array_reverse","array_sort","array_sum","avg","count","max","min","sum","greatest","least","ifmissing","ifmissingornull","ifnull","missingif","nullif","ifinf","ifnan","ifnanorinf","naninf","neginfif","posinfif","clock_millis","clock_str","date_add_millis","date_add_str","date_diff_millis","date_diff_str","date_part_millis","date_part_str","date_trunc_millis","date_trunc_str","duration_to_str","millis","str_to_millis","millis_to_str","millis_to_utc","millis_to_zone_name","now_millis","now_str","str_to_duration","str_to_utc","str_to_zone_name","decode_json","encode_json","encoded_size","poly_length","base64","base64_encode","base64_decode","meta","uuid","abs","acos","asin","atan","atan2","ceil","cos","degrees","e","exp","ln","log","floor","pi","power","radians","random","round","sign","sin","sqrt","tan","trunc","object_length","object_names","object_pairs","object_inner_pairs","object_values","object_inner_values","object_add","object_put","object_remove","object_unwrap","regexp_contains","regexp_like","regexp_position","regexp_replace","contains","initcap","length","lower","ltrim","position","repeat","replace","rtrim","split","substr","title","trim","upper","isarray","isatom","isboolean","isnumber","isobject","isstring","type","toarray","toatom","toboolean","tonumber","toobject","tostring"]},contains:[{className:"string",begin:"'",end:"'",contains:[e.BACKSLASH_ESCAPE]},{className:"string",begin:'"',end:'"',contains:[e.BACKSLASH_ESCAPE]},{className:"symbol",begin:"`",end:"`",contains:[e.BACKSLASH_ESCAPE]},e.C_NUMBER_MODE,e.C_BLOCK_COMMENT_MODE]},e.C_BLOCK_COMMENT_MODE]}}},9839(e,t,n){"use strict";var a,r=this&&this.__extends||(a=function(e,t){return a=Object.setPrototypeOf||{__proto__:[]}instanceof Array&&function(e,t){e.__proto__=t}||function(e,t){for(var n in t)Object.prototype.hasOwnProperty.call(t,n)&&(e[n]=t[n])},a(e,t)},function(e,t){if("function"!=typeof t&&null!==t)throw new TypeError("Class extends value "+String(t)+" is not a constructor or null");function n(){this.constructor=e}a(e,t),e.prototype=null===t?Object.create(t):(n.prototype=t.prototype,new n)});Object.defineProperty(t,"__esModule",{value:!0}),t.lineDiff=void 0,t.diffLines=function(e,n,a){return t.lineDiff.diff(e,n,a)},t.diffTrimmedLines=function(e,n,a){return a=(0,o.generateOptions)(a,{ignoreWhitespace:!0}),t.lineDiff.diff(e,n,a)},t.tokenize=l;var i=n(2188),o=n(2826),s=function(e){function t(){var t=null!==e&&e.apply(this,arguments)||this;return t.tokenize=l,t}return r(t,e),t.prototype.equals=function(t,n,a){return a.ignoreWhitespace?(a.newlineIsToken&&t.includes("\n")||(t=t.trim()),a.newlineIsToken&&n.includes("\n")||(n=n.trim())):a.ignoreNewlineAtEof&&!a.newlineIsToken&&(t.endsWith("\n")&&(t=t.slice(0,-1)),n.endsWith("\n")&&(n=n.slice(0,-1))),e.prototype.equals.call(this,t,n,a)},t}(i.default);function l(e,t){t.stripTrailingCr&&(e=e.replace(/\r\n/g,"\n"));var n=[],a=e.split(/(\n|\r\n)/);a[a.length-1]||a.pop();for(var r=0;r",contains:["self"]}]}}},9953(e){e.exports=function(e){return{name:"MEL",keywords:"int float string vector matrix if else switch case default while do for in break continue global proc return about abs addAttr addAttributeEditorNodeHelp addDynamic addNewShelfTab addPP addPanelCategory addPrefixToName advanceToNextDrivenKey affectedNet affects aimConstraint air alias aliasAttr align alignCtx alignCurve alignSurface allViewFit ambientLight angle angleBetween animCone animCurveEditor animDisplay animView annotate appendStringArray applicationName applyAttrPreset applyTake arcLenDimContext arcLengthDimension arclen arrayMapper art3dPaintCtx artAttrCtx artAttrPaintVertexCtx artAttrSkinPaintCtx artAttrTool artBuildPaintMenu artFluidAttrCtx artPuttyCtx artSelectCtx artSetPaintCtx artUserPaintCtx assignCommand assignInputDevice assignViewportFactories attachCurve attachDeviceAttr attachSurface attrColorSliderGrp attrCompatibility attrControlGrp attrEnumOptionMenu attrEnumOptionMenuGrp attrFieldGrp attrFieldSliderGrp attrNavigationControlGrp attrPresetEditWin attributeExists attributeInfo attributeMenu attributeQuery autoKeyframe autoPlace bakeClip bakeFluidShading bakePartialHistory bakeResults bakeSimulation basename basenameEx batchRender bessel bevel bevelPlus binMembership bindSkin blend2 blendShape blendShapeEditor blendShapePanel blendTwoAttr blindDataType boneLattice boundary boxDollyCtx boxZoomCtx bufferCurve buildBookmarkMenu buildKeyframeMenu button buttonManip CBG cacheFile cacheFileCombine cacheFileMerge cacheFileTrack camera cameraView canCreateManip canvas capitalizeString catch catchQuiet ceil changeSubdivComponentDisplayLevel changeSubdivRegion channelBox character characterMap characterOutlineEditor characterize chdir checkBox checkBoxGrp checkDefaultRenderGlobals choice circle circularFillet clamp clear clearCache clip clipEditor clipEditorCurrentTimeCtx clipSchedule clipSchedulerOutliner clipTrimBefore closeCurve closeSurface cluster cmdFileOutput cmdScrollFieldExecuter cmdScrollFieldReporter cmdShell coarsenSubdivSelectionList collision color colorAtPoint colorEditor colorIndex colorIndexSliderGrp colorSliderButtonGrp colorSliderGrp columnLayout commandEcho commandLine commandPort compactHairSystem componentEditor compositingInterop computePolysetVolume condition cone confirmDialog connectAttr connectControl connectDynamic connectJoint connectionInfo constrain constrainValue constructionHistory container containsMultibyte contextInfo control convertFromOldLayers convertIffToPsd convertLightmap convertSolidTx convertTessellation convertUnit copyArray copyFlexor copyKey copySkinWeights cos cpButton cpCache cpClothSet cpCollision cpConstraint cpConvClothToMesh cpForces cpGetSolverAttr cpPanel cpProperty cpRigidCollisionFilter cpSeam cpSetEdit cpSetSolverAttr cpSolver cpSolverTypes cpTool cpUpdateClothUVs createDisplayLayer createDrawCtx createEditor createLayeredPsdFile createMotionField createNewShelf createNode createRenderLayer createSubdivRegion cross crossProduct ctxAbort ctxCompletion ctxEditMode ctxTraverse currentCtx currentTime currentTimeCtx currentUnit curve curveAddPtCtx curveCVCtx curveEPCtx curveEditorCtx curveIntersect curveMoveEPCtx curveOnSurface curveSketchCtx cutKey cycleCheck cylinder dagPose date defaultLightListCheckBox defaultNavigation defineDataServer defineVirtualDevice deformer deg_to_rad delete deleteAttr deleteShadingGroupsAndMaterials deleteShelfTab deleteUI deleteUnusedBrushes delrandstr detachCurve detachDeviceAttr detachSurface deviceEditor devicePanel dgInfo dgdirty dgeval dgtimer dimWhen directKeyCtx directionalLight dirmap dirname disable disconnectAttr disconnectJoint diskCache displacementToPoly displayAffected displayColor displayCull displayLevelOfDetail displayPref displayRGBColor displaySmoothness displayStats displayString displaySurface distanceDimContext distanceDimension doBlur dolly dollyCtx dopeSheetEditor dot dotProduct doubleProfileBirailSurface drag dragAttrContext draggerContext dropoffLocator duplicate duplicateCurve duplicateSurface dynCache dynControl dynExport dynExpression dynGlobals dynPaintEditor dynParticleCtx dynPref dynRelEdPanel dynRelEditor dynamicLoad editAttrLimits editDisplayLayerGlobals editDisplayLayerMembers editRenderLayerAdjustment editRenderLayerGlobals editRenderLayerMembers editor editorTemplate effector emit emitter enableDevice encodeString endString endsWith env equivalent equivalentTol erf error eval evalDeferred evalEcho event exactWorldBoundingBox exclusiveLightCheckBox exec executeForEachObject exists exp expression expressionEditorListen extendCurve extendSurface extrude fcheck fclose feof fflush fgetline fgetword file fileBrowserDialog fileDialog fileExtension fileInfo filetest filletCurve filter filterCurve filterExpand filterStudioImport findAllIntersections findAnimCurves findKeyframe findMenuItem findRelatedSkinCluster finder firstParentOf fitBspline flexor floatEq floatField floatFieldGrp floatScrollBar floatSlider floatSlider2 floatSliderButtonGrp floatSliderGrp floor flow fluidCacheInfo fluidEmitter fluidVoxelInfo flushUndo fmod fontDialog fopen formLayout format fprint frameLayout fread freeFormFillet frewind fromNativePath fwrite gamma gauss geometryConstraint getApplicationVersionAsFloat getAttr getClassification getDefaultBrush getFileList getFluidAttr getInputDeviceRange getMayaPanelTypes getModifiers getPanel getParticleAttr getPluginResource getenv getpid glRender glRenderEditor globalStitch gmatch goal gotoBindPose grabColor gradientControl gradientControlNoAttr graphDollyCtx graphSelectContext graphTrackCtx gravity grid gridLayout group groupObjectsByName HfAddAttractorToAS HfAssignAS HfBuildEqualMap HfBuildFurFiles HfBuildFurImages HfCancelAFR HfConnectASToHF HfCreateAttractor HfDeleteAS HfEditAS HfPerformCreateAS HfRemoveAttractorFromAS HfSelectAttached HfSelectAttractors HfUnAssignAS hardenPointCurve hardware hardwareRenderPanel headsUpDisplay headsUpMessage help helpLine hermite hide hilite hitTest hotBox hotkey hotkeyCheck hsv_to_rgb hudButton hudSlider hudSliderButton hwReflectionMap hwRender hwRenderLoad hyperGraph hyperPanel hyperShade hypot iconTextButton iconTextCheckBox iconTextRadioButton iconTextRadioCollection iconTextScrollList iconTextStaticLabel ikHandle ikHandleCtx ikHandleDisplayScale ikSolver ikSplineHandleCtx ikSystem ikSystemInfo ikfkDisplayMethod illustratorCurves image imfPlugins inheritTransform insertJoint insertJointCtx insertKeyCtx insertKnotCurve insertKnotSurface instance instanceable instancer intField intFieldGrp intScrollBar intSlider intSliderGrp interToUI internalVar intersect iprEngine isAnimCurve isConnected isDirty isParentOf isSameObject isTrue isValidObjectName isValidString isValidUiName isolateSelect itemFilter itemFilterAttr itemFilterRender itemFilterType joint jointCluster jointCtx jointDisplayScale jointLattice keyTangent keyframe keyframeOutliner keyframeRegionCurrentTimeCtx keyframeRegionDirectKeyCtx keyframeRegionDollyCtx keyframeRegionInsertKeyCtx keyframeRegionMoveKeyCtx keyframeRegionScaleKeyCtx keyframeRegionSelectKeyCtx keyframeRegionSetKeyCtx keyframeRegionTrackCtx keyframeStats lassoContext lattice latticeDeformKeyCtx launch launchImageEditor layerButton layeredShaderPort layeredTexturePort layout layoutDialog lightList lightListEditor lightListPanel lightlink lineIntersection linearPrecision linstep listAnimatable listAttr listCameras listConnections listDeviceAttachments listHistory listInputDeviceAxes listInputDeviceButtons listInputDevices listMenuAnnotation listNodeTypes listPanelCategories listRelatives listSets listTransforms listUnselected listerEditor loadFluid loadNewShelf loadPlugin loadPluginLanguageResources loadPrefObjects localizedPanelLabel lockNode loft log longNameOf lookThru ls lsThroughFilter lsType lsUI Mayatomr mag makeIdentity makeLive makePaintable makeRoll makeSingleSurface makeTubeOn makebot manipMoveContext manipMoveLimitsCtx manipOptions manipRotateContext manipRotateLimitsCtx manipScaleContext manipScaleLimitsCtx marker match max memory menu menuBarLayout menuEditor menuItem menuItemToShelf menuSet menuSetPref messageLine min minimizeApp mirrorJoint modelCurrentTimeCtx modelEditor modelPanel mouse movIn movOut move moveIKtoFK moveKeyCtx moveVertexAlongDirection multiProfileBirailSurface mute nParticle nameCommand nameField namespace namespaceInfo newPanelItems newton nodeCast nodeIconButton nodeOutliner nodePreset nodeType noise nonLinear normalConstraint normalize nurbsBoolean nurbsCopyUVSet nurbsCube nurbsEditUV nurbsPlane nurbsSelect nurbsSquare nurbsToPoly nurbsToPolygonsPref nurbsToSubdiv nurbsToSubdivPref nurbsUVSet nurbsViewDirectionVector objExists objectCenter objectLayer objectType objectTypeUI obsoleteProc oceanNurbsPreviewPlane offsetCurve offsetCurveOnSurface offsetSurface openGLExtension openMayaPref optionMenu optionMenuGrp optionVar orbit orbitCtx orientConstraint outlinerEditor outlinerPanel overrideModifier paintEffectsDisplay pairBlend palettePort paneLayout panel panelConfiguration panelHistory paramDimContext paramDimension paramLocator parent parentConstraint particle particleExists particleInstancer particleRenderInfo partition pasteKey pathAnimation pause pclose percent performanceOptions pfxstrokes pickWalk picture pixelMove planarSrf plane play playbackOptions playblast plugAttr plugNode pluginInfo pluginResourceUtil pointConstraint pointCurveConstraint pointLight pointMatrixMult pointOnCurve pointOnSurface pointPosition poleVectorConstraint polyAppend polyAppendFacetCtx polyAppendVertex polyAutoProjection polyAverageNormal polyAverageVertex polyBevel polyBlendColor polyBlindData polyBoolOp polyBridgeEdge polyCacheMonitor polyCheck polyChipOff polyClipboard polyCloseBorder polyCollapseEdge polyCollapseFacet polyColorBlindData polyColorDel polyColorPerVertex polyColorSet polyCompare polyCone polyCopyUV polyCrease polyCreaseCtx polyCreateFacet polyCreateFacetCtx polyCube polyCut polyCutCtx polyCylinder polyCylindricalProjection polyDelEdge polyDelFacet polyDelVertex polyDuplicateAndConnect polyDuplicateEdge polyEditUV polyEditUVShell polyEvaluate polyExtrudeEdge polyExtrudeFacet polyExtrudeVertex polyFlipEdge polyFlipUV polyForceUV polyGeoSampler polyHelix polyInfo polyInstallAction polyLayoutUV polyListComponentConversion polyMapCut polyMapDel polyMapSew polyMapSewMove polyMergeEdge polyMergeEdgeCtx polyMergeFacet polyMergeFacetCtx polyMergeUV polyMergeVertex polyMirrorFace polyMoveEdge polyMoveFacet polyMoveFacetUV polyMoveUV polyMoveVertex polyNormal polyNormalPerVertex polyNormalizeUV polyOptUvs polyOptions polyOutput polyPipe polyPlanarProjection polyPlane polyPlatonicSolid polyPoke polyPrimitive polyPrism polyProjection polyPyramid polyQuad polyQueryBlindData polyReduce polySelect polySelectConstraint polySelectConstraintMonitor polySelectCtx polySelectEditCtx polySeparate polySetToFaceNormal polySewEdge polyShortestPathCtx polySmooth polySoftEdge polySphere polySphericalProjection polySplit polySplitCtx polySplitEdge polySplitRing polySplitVertex polyStraightenUVBorder polySubdivideEdge polySubdivideFacet polyToSubdiv polyTorus polyTransfer polyTriangulate polyUVSet polyUnite polyWedgeFace popen popupMenu pose pow preloadRefEd print progressBar progressWindow projFileViewer projectCurve projectTangent projectionContext projectionManip promptDialog propModCtx propMove psdChannelOutliner psdEditTextureFile psdExport psdTextureFile putenv pwd python querySubdiv quit rad_to_deg radial radioButton radioButtonGrp radioCollection radioMenuItemCollection rampColorPort rand randomizeFollicles randstate rangeControl readTake rebuildCurve rebuildSurface recordAttr recordDevice redo reference referenceEdit referenceQuery refineSubdivSelectionList refresh refreshAE registerPluginResource rehash reloadImage removeJoint removeMultiInstance removePanelCategory rename renameAttr renameSelectionList renameUI render renderGlobalsNode renderInfo renderLayerButton renderLayerParent renderLayerPostProcess renderLayerUnparent renderManip renderPartition renderQualityNode renderSettings renderThumbnailUpdate renderWindowEditor renderWindowSelectContext renderer reorder reorderDeformers requires reroot resampleFluid resetAE resetPfxToPolyCamera resetTool resolutionNode retarget reverseCurve reverseSurface revolve rgb_to_hsv rigidBody rigidSolver roll rollCtx rootOf rot rotate rotationInterpolation roundConstantRadius rowColumnLayout rowLayout runTimeCommand runup sampleImage saveAllShelves saveAttrPreset saveFluid saveImage saveInitialState saveMenu savePrefObjects savePrefs saveShelf saveToolSettings scale scaleBrushBrightness scaleComponents scaleConstraint scaleKey scaleKeyCtx sceneEditor sceneUIReplacement scmh scriptCtx scriptEditorInfo scriptJob scriptNode scriptTable scriptToShelf scriptedPanel scriptedPanelType scrollField scrollLayout sculpt searchPathArray seed selLoadSettings select selectContext selectCurveCV selectKey selectKeyCtx selectKeyframeRegionCtx selectMode selectPref selectPriority selectType selectedNodes selectionConnection separator setAttr setAttrEnumResource setAttrMapping setAttrNiceNameResource setConstraintRestPosition setDefaultShadingGroup setDrivenKeyframe setDynamic setEditCtx setEditor setFluidAttr setFocus setInfinity setInputDeviceMapping setKeyCtx setKeyPath setKeyframe setKeyframeBlendshapeTargetWts setMenuMode setNodeNiceNameResource setNodeTypeFlag setParent setParticleAttr setPfxToPolyCamera setPluginResource setProject setStampDensity setStartupMessage setState setToolTo setUITemplate setXformManip sets shadingConnection shadingGeometryRelCtx shadingLightRelCtx shadingNetworkCompare shadingNode shapeCompare shelfButton shelfLayout shelfTabLayout shellField shortNameOf showHelp showHidden showManipCtx showSelectionInTitle showShadingGroupAttrEditor showWindow sign simplify sin singleProfileBirailSurface size sizeBytes skinCluster skinPercent smoothCurve smoothTangentSurface smoothstep snap2to2 snapKey snapMode snapTogetherCtx snapshot soft softMod softModCtx sort sound soundControl source spaceLocator sphere sphrand spotLight spotLightPreviewPort spreadSheetEditor spring sqrt squareSurface srtContext stackTrace startString startsWith stitchAndExplodeShell stitchSurface stitchSurfacePoints strcmp stringArrayCatenate stringArrayContains stringArrayCount stringArrayInsertAtIndex stringArrayIntersector stringArrayRemove stringArrayRemoveAtIndex stringArrayRemoveDuplicates stringArrayRemoveExact stringArrayToString stringToStringArray strip stripPrefixFromName stroke subdAutoProjection subdCleanTopology subdCollapse subdDuplicateAndConnect subdEditUV subdListComponentConversion subdMapCut subdMapSewMove subdMatchTopology subdMirror subdToBlind subdToPoly subdTransferUVsToCache subdiv subdivCrease subdivDisplaySmoothness substitute substituteAllString substituteGeometry substring surface surfaceSampler surfaceShaderList swatchDisplayPort switchTable symbolButton symbolCheckBox sysFile system tabLayout tan tangentConstraint texLatticeDeformContext texManipContext texMoveContext texMoveUVShellContext texRotateContext texScaleContext texSelectContext texSelectShortestPathCtx texSmudgeUVContext texWinToolCtx text textCurves textField textFieldButtonGrp textFieldGrp textManip textScrollList textToShelf textureDisplacePlane textureHairColor texturePlacementContext textureWindow threadCount threePointArcCtx timeControl timePort timerX toNativePath toggle toggleAxis toggleWindowVisibility tokenize tokenizeList tolerance tolower toolButton toolCollection toolDropped toolHasOptions toolPropertyWindow torus toupper trace track trackCtx transferAttributes transformCompare transformLimits translator trim trunc truncateFluidCache truncateHairCache tumble tumbleCtx turbulence twoPointArcCtx uiRes uiTemplate unassignInputDevice undo undoInfo ungroup uniform unit unloadPlugin untangleUV untitledFileName untrim upAxis updateAE userCtx uvLink uvSnapshot validateShelfName vectorize view2dToolCtx viewCamera viewClipPlane viewFit viewHeadOn viewLookAt viewManip viewPlace viewSet visor volumeAxis vortex waitCursor warning webBrowser webBrowserPrefs whatIs window windowPref wire wireContext workspace wrinkle wrinkleContext writeTake xbmLangPathList xform",illegal:"li{border-bottom:1px solid #ddd;border-bottom:1px solid var(--d2h-border-color);margin:0;padding:5px 10px}.d2h-file-list>li:last-child{border-bottom:none}.d2h-file-switch{cursor:pointer;display:none;font-size:10px}.d2h-icon{fill:currentColor;margin-right:10px;vertical-align:middle}.d2h-deleted{color:#c33;color:var(--d2h-del-label-color)}.d2h-added{color:#399839;color:var(--d2h-ins-label-color)}.d2h-changed{color:#d0b44c;color:var(--d2h-change-label-color)}.d2h-moved{color:#3572b0;color:var(--d2h-moved-label-color)}.d2h-tag{background-color:#fff;background-color:var(--d2h-bg-color);display:-webkit-box;display:-ms-flexbox;display:flex;font-size:10px;margin-left:5px;padding:0 2px}.d2h-deleted-tag{border:1px solid #c33;border:1px solid var(--d2h-del-label-color)}.d2h-added-tag{border:1px solid #399839;border:1px solid var(--d2h-ins-label-color)}.d2h-changed-tag{border:1px solid #d0b44c;border:1px solid var(--d2h-change-label-color)}.d2h-moved-tag{border:1px solid #3572b0;border:1px solid var(--d2h-moved-label-color)}.d2h-dark-color-scheme{background-color:#0d1117;background-color:var(--d2h-dark-bg-color);color:#e6edf3;color:var(--d2h-dark-color)}.d2h-dark-color-scheme .d2h-file-header{background-color:#161b22;background-color:var(--d2h-dark-file-header-bg-color);border-bottom:#30363d;border-bottom:var(--d2h-dark-file-header-border-color)}.d2h-dark-color-scheme .d2h-lines-added{border:1px solid rgba(46,160,67,.4);border:1px solid var(--d2h-dark-ins-border-color);color:#3fb950;color:var(--d2h-dark-ins-label-color)}.d2h-dark-color-scheme .d2h-lines-deleted{border:1px solid rgba(248,81,73,.4);border:1px solid var(--d2h-dark-del-border-color);color:#f85149;color:var(--d2h-dark-del-label-color)}.d2h-dark-color-scheme .d2h-code-line del,.d2h-dark-color-scheme .d2h-code-side-line del{background-color:rgba(248,81,73,.4);background-color:var(--d2h-dark-del-highlight-bg-color)}.d2h-dark-color-scheme .d2h-code-line ins,.d2h-dark-color-scheme .d2h-code-side-line ins{background-color:rgba(46,160,67,.4);background-color:var(--d2h-dark-ins-highlight-bg-color)}.d2h-dark-color-scheme .d2h-diff-tbody{border-color:#30363d;border-color:var(--d2h-dark-border-color)}.d2h-dark-color-scheme .d2h-code-side-linenumber{background-color:#0d1117;background-color:var(--d2h-dark-bg-color);border-color:#21262d;border-color:var(--d2h-dark-line-border-color);color:#6e7681;color:var(--d2h-dark-dim-color)}.d2h-dark-color-scheme .d2h-files-diff .d2h-code-side-emptyplaceholder,.d2h-dark-color-scheme .d2h-files-diff .d2h-emptyplaceholder{background-color:hsla(215,8%,47%,.1);background-color:var(--d2h-dark-empty-placeholder-bg-color);border-color:#30363d;border-color:var(--d2h-dark-empty-placeholder-border-color)}.d2h-dark-color-scheme .d2h-code-linenumber{background-color:#0d1117;background-color:var(--d2h-dark-bg-color);border-color:#21262d;border-color:var(--d2h-dark-line-border-color);color:#6e7681;color:var(--d2h-dark-dim-color)}.d2h-dark-color-scheme .d2h-del{background-color:rgba(248,81,73,.1);background-color:var(--d2h-dark-del-bg-color);border-color:rgba(248,81,73,.4);border-color:var(--d2h-dark-del-border-color)}.d2h-dark-color-scheme .d2h-ins{background-color:rgba(46,160,67,.15);background-color:var(--d2h-dark-ins-bg-color);border-color:rgba(46,160,67,.4);border-color:var(--d2h-dark-ins-border-color)}.d2h-dark-color-scheme .d2h-info{background-color:rgba(56,139,253,.1);background-color:var(--d2h-dark-info-bg-color);border-color:rgba(56,139,253,.4);border-color:var(--d2h-dark-info-border-color);color:#6e7681;color:var(--d2h-dark-dim-color)}.d2h-dark-color-scheme .d2h-file-diff .d2h-del.d2h-change{background-color:rgba(210,153,34,.2);background-color:var(--d2h-dark-change-del-color)}.d2h-dark-color-scheme .d2h-file-diff .d2h-ins.d2h-change{background-color:rgba(46,160,67,.25);background-color:var(--d2h-dark-change-ins-color)}.d2h-dark-color-scheme .d2h-file-wrapper{border:1px solid #30363d;border:1px solid var(--d2h-dark-border-color)}.d2h-dark-color-scheme .d2h-file-collapse{border:1px solid #0d1117;border:1px solid var(--d2h-dark-bg-color)}.d2h-dark-color-scheme .d2h-file-collapse.d2h-selected{background-color:rgba(56,139,253,.1);background-color:var(--d2h-dark-selected-color)}.d2h-dark-color-scheme .d2h-file-list-wrapper a,.d2h-dark-color-scheme .d2h-file-list-wrapper a:visited{color:#3572b0;color:var(--d2h-dark-moved-label-color)}.d2h-dark-color-scheme .d2h-file-list>li{border-bottom:1px solid #0d1117;border-bottom:1px solid var(--d2h-dark-bg-color)}.d2h-dark-color-scheme .d2h-deleted{color:#f85149;color:var(--d2h-dark-del-label-color)}.d2h-dark-color-scheme .d2h-added{color:#3fb950;color:var(--d2h-dark-ins-label-color)}.d2h-dark-color-scheme .d2h-changed{color:#d29922;color:var(--d2h-dark-change-label-color)}.d2h-dark-color-scheme .d2h-moved{color:#3572b0;color:var(--d2h-dark-moved-label-color)}.d2h-dark-color-scheme .d2h-tag{background-color:#0d1117;background-color:var(--d2h-dark-bg-color)}.d2h-dark-color-scheme .d2h-deleted-tag{border:1px solid #f85149;border:1px solid var(--d2h-dark-del-label-color)}.d2h-dark-color-scheme .d2h-added-tag{border:1px solid #3fb950;border:1px solid var(--d2h-dark-ins-label-color)}.d2h-dark-color-scheme .d2h-changed-tag{border:1px solid #d29922;border:1px solid var(--d2h-dark-change-label-color)}.d2h-dark-color-scheme .d2h-moved-tag{border:1px solid #3572b0;border:1px solid var(--d2h-dark-moved-label-color)}@media (prefers-color-scheme:dark){.d2h-auto-color-scheme{background-color:#0d1117;background-color:var(--d2h-dark-bg-color);color:#e6edf3;color:var(--d2h-dark-color)}.d2h-auto-color-scheme .d2h-file-header{background-color:#161b22;background-color:var(--d2h-dark-file-header-bg-color);border-bottom:#30363d;border-bottom:var(--d2h-dark-file-header-border-color)}.d2h-auto-color-scheme .d2h-lines-added{border:1px solid rgba(46,160,67,.4);border:1px solid var(--d2h-dark-ins-border-color);color:#3fb950;color:var(--d2h-dark-ins-label-color)}.d2h-auto-color-scheme .d2h-lines-deleted{border:1px solid rgba(248,81,73,.4);border:1px solid var(--d2h-dark-del-border-color);color:#f85149;color:var(--d2h-dark-del-label-color)}.d2h-auto-color-scheme .d2h-code-line del,.d2h-auto-color-scheme .d2h-code-side-line del{background-color:rgba(248,81,73,.4);background-color:var(--d2h-dark-del-highlight-bg-color)}.d2h-auto-color-scheme .d2h-code-line ins,.d2h-auto-color-scheme .d2h-code-side-line ins{background-color:rgba(46,160,67,.4);background-color:var(--d2h-dark-ins-highlight-bg-color)}.d2h-auto-color-scheme .d2h-diff-tbody{border-color:#30363d;border-color:var(--d2h-dark-border-color)}.d2h-auto-color-scheme .d2h-code-side-linenumber{background-color:#0d1117;background-color:var(--d2h-dark-bg-color);border-color:#21262d;border-color:var(--d2h-dark-line-border-color);color:#6e7681;color:var(--d2h-dark-dim-color)}.d2h-auto-color-scheme .d2h-files-diff .d2h-code-side-emptyplaceholder,.d2h-auto-color-scheme .d2h-files-diff .d2h-emptyplaceholder{background-color:hsla(215,8%,47%,.1);background-color:var(--d2h-dark-empty-placeholder-bg-color);border-color:#30363d;border-color:var(--d2h-dark-empty-placeholder-border-color)}.d2h-auto-color-scheme .d2h-code-linenumber{background-color:#0d1117;background-color:var(--d2h-dark-bg-color);border-color:#21262d;border-color:var(--d2h-dark-line-border-color);color:#6e7681;color:var(--d2h-dark-dim-color)}.d2h-auto-color-scheme .d2h-del{background-color:rgba(248,81,73,.1);background-color:var(--d2h-dark-del-bg-color);border-color:rgba(248,81,73,.4);border-color:var(--d2h-dark-del-border-color)}.d2h-auto-color-scheme .d2h-ins{background-color:rgba(46,160,67,.15);background-color:var(--d2h-dark-ins-bg-color);border-color:rgba(46,160,67,.4);border-color:var(--d2h-dark-ins-border-color)}.d2h-auto-color-scheme .d2h-info{background-color:rgba(56,139,253,.1);background-color:var(--d2h-dark-info-bg-color);border-color:rgba(56,139,253,.4);border-color:var(--d2h-dark-info-border-color);color:#6e7681;color:var(--d2h-dark-dim-color)}.d2h-auto-color-scheme .d2h-file-diff .d2h-del.d2h-change{background-color:rgba(210,153,34,.2);background-color:var(--d2h-dark-change-del-color)}.d2h-auto-color-scheme .d2h-file-diff .d2h-ins.d2h-change{background-color:rgba(46,160,67,.25);background-color:var(--d2h-dark-change-ins-color)}.d2h-auto-color-scheme .d2h-file-wrapper{border:1px solid #30363d;border:1px solid var(--d2h-dark-border-color)}.d2h-auto-color-scheme .d2h-file-collapse{border:1px solid #0d1117;border:1px solid var(--d2h-dark-bg-color)}.d2h-auto-color-scheme .d2h-file-collapse.d2h-selected{background-color:rgba(56,139,253,.1);background-color:var(--d2h-dark-selected-color)}.d2h-auto-color-scheme .d2h-file-list-wrapper a,.d2h-auto-color-scheme .d2h-file-list-wrapper a:visited{color:#3572b0;color:var(--d2h-dark-moved-label-color)}.d2h-auto-color-scheme .d2h-file-list>li{border-bottom:1px solid #0d1117;border-bottom:1px solid var(--d2h-dark-bg-color)}.d2h-dark-color-scheme .d2h-deleted{color:#f85149;color:var(--d2h-dark-del-label-color)}.d2h-auto-color-scheme .d2h-added{color:#3fb950;color:var(--d2h-dark-ins-label-color)}.d2h-auto-color-scheme .d2h-changed{color:#d29922;color:var(--d2h-dark-change-label-color)}.d2h-auto-color-scheme .d2h-moved{color:#3572b0;color:var(--d2h-dark-moved-label-color)}.d2h-auto-color-scheme .d2h-tag{background-color:#0d1117;background-color:var(--d2h-dark-bg-color)}.d2h-auto-color-scheme .d2h-deleted-tag{border:1px solid #f85149;border:1px solid var(--d2h-dark-del-label-color)}.d2h-auto-color-scheme .d2h-added-tag{border:1px solid #3fb950;border:1px solid var(--d2h-dark-ins-label-color)}.d2h-auto-color-scheme .d2h-changed-tag{border:1px solid #d29922;border:1px solid var(--d2h-dark-change-label-color)}.d2h-auto-color-scheme .d2h-moved-tag{border:1px solid #3572b0;border:1px solid var(--d2h-dark-moved-label-color)}} \ No newline at end of file diff --git a/internal/server/templates/pages/compare_versions.html b/internal/server/templates/pages/compare_versions.html index 0c3c314..79cd89b 100644 --- a/internal/server/templates/pages/compare_versions.html +++ b/internal/server/templates/pages/compare_versions.html @@ -66,10 +66,10 @@
- + - + ") + })) + t.Cleanup(upstream.Close) + + db, err := database.Create(t.TempDir() + "/test.db") + if err != nil { + t.Fatalf("creating database: %v", err) + } + t.Cleanup(func() { _ = db.Close() }) + store, err := storage.OpenBucket(context.Background(), "file://"+t.TempDir()) + if err != nil { + t.Fatalf("opening storage: %v", err) + } + fetcher := fetch.NewFetcher(fetch.WithHTTPClient(upstream.Client()), fetch.WithMaxRetries(0)) + t.Cleanup(func() { _ = fetcher.Close() }) + proxy := handler.NewProxy(db, store, fetcher, fetch.NewResolver(), slog.New(slog.NewTextHandler(io.Discard, nil))) + proxy.HTTPClient = upstream.Client() + + cfg := config.Default() + cfg.BaseURL = "http://proxy.example" + cfg.Upstream.Generic = map[string]string{"site": upstream.URL} + cfg.Upstream.APK = map[string]string{"alpine": upstream.URL} + s := &Server{cfg: cfg} + r := chi.NewRouter() + s.mountProtocolHandlers(r, proxy) + + for _, target := range []string{ + "/generic/site/files/page-1.0.html", // cached artifact + "/generic/site/files/page-1.0.html", // the same, from the cache + "/apk/alpine/v3.22/main/x86_64/README.html", // relayed as is + "/npm/does-not-exist%2F..", // an error response + } { + w := httptest.NewRecorder() + r.ServeHTTP(w, httptest.NewRequest(http.MethodGet, target, nil)) + if got := w.Header().Get("X-Content-Type-Options"); got != "nosniff" { + t.Errorf("%s (status %d): X-Content-Type-Options = %q, want nosniff", target, w.Code, got) + } + if got := w.Header().Get("Content-Security-Policy"); got != "sandbox" { + t.Errorf("%s (status %d): Content-Security-Policy = %q, want sandbox", target, w.Code, got) + } + } +} From 6a90fb411fef610caf510bf405da52c9f8843be6 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Wed, 14 Oct 2026 17:56:18 +0200 Subject: [PATCH 292/633] Only send credentialed registry tokens to trusted registries Credentials configured for a token service were applied to any challenge naming it. A host the proxy fetches from, such as one in a Composer dist URL or a Helm index, could answer a /v2/ request with a Bearer challenge naming that realm, and the proxy fetched a token with the configured credentials and sent it to that host. A token cached from a legitimate pull was handed over the same way. - Follow a challenge to a realm with configured credentials only for a registry that has configured credentials of its own, or is on the same site (registrable domain) as the realm, as registry-1.docker.io is with auth.docker.io; otherwise pass the 401 back - Keep following anonymous challenges for any registry, as before - Document the rule --- docs/configuration.md | 2 + internal/httpclient/token_scope_test.go | 173 ++++++++++++++++++++++++ internal/httpclient/transport.go | 42 +++++- 3 files changed, 216 insertions(+), 1 deletion(-) create mode 100644 internal/httpclient/token_scope_test.go diff --git a/docs/configuration.md b/docs/configuration.md index e2a9178..c735c34 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -588,6 +588,8 @@ Configure authentication for private upstream registries. The same authenticatio OCI registries that return a Bearer challenge from a `/v2/{repository}/…` endpoint are handled automatically. The proxy discovers the token realm from `WWW-Authenticate`, applies any configured credentials for the token URL, and reuses the scoped token until shortly before it expires. +A token fetched with credentials configured for the token URL is only fetched for, and sent to, a registry that is trusted with it: one with configured credentials of its own, or one on the same site (registrable domain) as the token service, as `registry-1.docker.io` is with `auth.docker.io`. Any other host that answers with a challenge naming that token service gets its 401 back, so it cannot obtain a token minted from your credentials. Tokens from a token service with no configured credentials are anonymous and are fetched for any registry, as before. If your registry and its token service are on different sites, configure credentials for the registry URL as well. + ### Bearer Token Used by npm, GitHub Package Registry, and many other registries: diff --git a/internal/httpclient/token_scope_test.go b/internal/httpclient/token_scope_test.go new file mode 100644 index 0000000..97d8154 --- /dev/null +++ b/internal/httpclient/token_scope_test.go @@ -0,0 +1,173 @@ +package httpclient + +import ( + "io" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "sync/atomic" + "testing" +) + +// tokenService issues "secret-token" only to requests carrying the configured +// credentials, and "anonymous-token" otherwise. +func tokenService(t *testing.T, issued *atomic.Int32) *httptest.Server { + t.Helper() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + issued.Add(1) + token := "anonymous-token" + if r.Header.Get("Authorization") == "Basic Y29ycDpwYXNz" { + token = "secret-token" + } + _, _ = io.WriteString(w, `{"token":"`+token+`","expires_in":3600}`) + })) + t.Cleanup(srv.Close) + return srv +} + +// challengingRegistry answers every request without a bearer token with a +// challenge naming realm, and records the Authorization headers it receives. +func challengingRegistry(t *testing.T, realm string, received *[]string) *httptest.Server { + t.Helper() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + auth := r.Header.Get("Authorization") + *received = append(*received, auth) + if !strings.HasPrefix(auth, "Bearer ") { + w.Header().Set("WWW-Authenticate", `Bearer realm="`+realm+`/token",service="registry.corp",scope="repository:corp/private:pull"`) + http.Error(w, "authentication required", http.StatusUnauthorized) + return + } + _, _ = io.WriteString(w, "blob") + })) + t.Cleanup(srv.Close) + return srv +} + +func credentialsFor(prefix string) AuthFunc { + return func(rawURL string) (string, string) { + if strings.HasPrefix(rawURL, prefix) { + return "Authorization", "Basic Y29ycDpwYXNz" + } + return "", "" + } +} + +func get(t *testing.T, client *http.Client, rawURL string) int { + t.Helper() + resp, err := client.Get(rawURL) + if err != nil { + t.Fatalf("GET %s: %v", rawURL, err) + } + _, _ = io.Copy(io.Discard, resp.Body) + _ = resp.Body.Close() + return resp.StatusCode +} + +// TestTransportDoesNotSendCredentialedTokenToAnotherSite is a regression test +// for a credential leak. Any host the proxy fetches from, such as one named in +// a Composer dist URL or a Helm index, could answer with a challenge naming a +// token service the proxy has credentials for. The proxy fetched a token with +// those credentials and sent it to that host. +func TestTransportDoesNotSendCredentialedTokenToAnotherSite(t *testing.T) { + var issued atomic.Int32 + realm := tokenService(t, &issued) + var received []string + evil := challengingRegistry(t, realm.URL, &received) + + client := &http.Client{Transport: NewTransport(http.DefaultTransport, credentialsFor(realm.URL))} + if status := get(t, client, evil.URL+"/v2/corp/private/blobs/sha256:abc"); status != http.StatusUnauthorized { + t.Errorf("status = %d, want the challenge passed back as 401", status) + } + for _, auth := range received { + if auth != "" { + t.Errorf("the challenging host received Authorization %q", auth) + } + } + if n := issued.Load(); n != 0 { + t.Errorf("token service was asked %d times on the challenging host's behalf, want 0", n) + } +} + +// TestTransportDoesNotSendCachedCredentialedTokenToAnotherSite checks the +// same for a token already cached from a legitimate pull: a challenge that +// names the same realm, service and scope must not be handed it. +func TestTransportDoesNotSendCachedCredentialedTokenToAnotherSite(t *testing.T) { + var issued atomic.Int32 + realm := tokenService(t, &issued) + var legitimate, leaked []string + registry := challengingRegistry(t, realm.URL, &legitimate) + evil := challengingRegistry(t, realm.URL, &leaked) + + // The registry itself has configured credentials, so it is trusted with + // tokens from its realm. + auth := func(rawURL string) (string, string) { + if strings.HasPrefix(rawURL, realm.URL) || strings.HasPrefix(rawURL, registry.URL) { + return "Authorization", "Basic Y29ycDpwYXNz" + } + return "", "" + } + client := &http.Client{Transport: NewTransport(http.DefaultTransport, auth)} + if status := get(t, client, registry.URL+"/v2/corp/private/blobs/sha256:abc"); status != http.StatusOK { + t.Fatalf("legitimate pull: status = %d, want 200", status) + } + if got := legitimate[len(legitimate)-1]; got != "Bearer secret-token" { + t.Fatalf("legitimate pull sent %q, want the credentialed token", got) + } + + get(t, client, evil.URL+"/v2/corp/private/blobs/sha256:abc") + for _, got := range leaked { + if strings.HasPrefix(got, "Bearer ") { + t.Errorf("the challenging host received %q", got) + } + } +} + +// TestTransportFollowsAnonymousTokenServiceOnAnotherHost keeps public pulls +// working where the token service is on another host and no credentials are +// configured for it: the token is anonymous, so any registry may have it. +func TestTransportFollowsAnonymousTokenServiceOnAnotherHost(t *testing.T) { + var issued atomic.Int32 + realm := tokenService(t, &issued) + var received []string + registry := challengingRegistry(t, realm.URL, &received) + + client := &http.Client{Transport: NewTransport(http.DefaultTransport, credentialsFor("https://elsewhere.example"))} + if status := get(t, client, registry.URL+"/v2/library/alpine/blobs/sha256:abc"); status != http.StatusOK { + t.Errorf("status = %d, want 200", status) + } + if got := received[len(received)-1]; got != "Bearer anonymous-token" { + t.Errorf("registry received %q, want the anonymous token", got) + } +} + +func TestSameSite(t *testing.T) { + for _, tt := range []struct { + registry, realm string + want bool + }{ + {"https://registry-1.docker.io", "https://auth.docker.io", true}, + {"https://registry.gitlab.com", "https://gitlab.com", true}, + {"https://ghcr.io", "https://ghcr.io", true}, + {"https://registry.corp.example", "https://auth.corp.example:8443", true}, + {"https://evil.example", "https://auth.docker.io", false}, + {"https://docker.io.evil.example", "https://auth.docker.io", false}, + {"https://a.github.io", "https://b.github.io", false}, // public suffix: separate sites + {"http://127.0.0.1:5000", "http://127.0.0.1:5000", true}, + {"http://127.0.0.1:5000", "http://127.0.0.1:5001", false}, + {"http://localhost:5000", "http://localhost:5001", false}, + } { + if got := sameSite(mustParse(t, tt.registry), mustParse(t, tt.realm)); got != tt.want { + t.Errorf("sameSite(%s, %s) = %v, want %v", tt.registry, tt.realm, got, tt.want) + } + } +} + +func mustParse(t *testing.T, rawURL string) *url.URL { + t.Helper() + u, err := url.Parse(rawURL) + if err != nil { + t.Fatal(err) + } + return u +} diff --git a/internal/httpclient/transport.go b/internal/httpclient/transport.go index 1ee9597..56c0f48 100644 --- a/internal/httpclient/transport.go +++ b/internal/httpclient/transport.go @@ -13,6 +13,8 @@ import ( "strings" "sync" "time" + + "golang.org/x/net/publicsuffix" ) const ( @@ -88,7 +90,7 @@ func (t *Transport) RoundTrip(req *http.Request) (*http.Response, error) { } challenge, ok := parseBearerChallenge(resp.Header.Values("WWW-Authenticate")) - if !ok || !canReplay(req) { + if !ok || !canReplay(req) || !t.trustedWithToken(req.URL, challenge) { return resp, nil } @@ -108,6 +110,44 @@ func (t *Transport) RoundTrip(req *http.Request) (*http.Response, error) { return t.base.RoundTrip(retry) } +// trustedWithToken reports whether a token from challenge's realm may be +// fetched for, and sent to, the registry at requestURL. The challenge comes +// from that registry, which can be any host the proxy fetches from, such as +// one named in a Composer dist URL or a Helm index. Credentials configured for +// the realm would otherwise be turned into a token handed to that host. A +// registry is trusted with such a token when it has configured credentials of +// its own, or is on the same site as the realm, as Docker Hub's +// registry-1.docker.io is with auth.docker.io. A token from a realm with no +// configured credentials is anonymous, and any registry may have it. +func (t *Transport) trustedWithToken(requestURL *url.URL, challenge bearerChallenge) bool { + if t.authForURL == nil { + return true + } + if name, value := t.authForURL(challenge.realm); name == "" || value == "" { + return true + } + if name, value := t.authForURL(requestURL.String()); name != "" && value != "" { + return true + } + realm, err := url.Parse(challenge.realm) + return err == nil && sameSite(requestURL, realm) +} + +// sameSite reports whether two URLs are on the same site: the same registrable +// domain (public suffix plus one label), or, for IP addresses and names +// without one, such as localhost, the same host and port. +func sameSite(a, b *url.URL) bool { + hostA, hostB := strings.ToLower(a.Hostname()), strings.ToLower(b.Hostname()) + if net.ParseIP(hostA) == nil && net.ParseIP(hostB) == nil { + siteA, errA := publicsuffix.EffectiveTLDPlusOne(hostA) + siteB, errB := publicsuffix.EffectiveTLDPlusOne(hostB) + if errA == nil && errB == nil { + return siteA == siteB + } + } + return strings.EqualFold(a.Host, b.Host) +} + func (t *Transport) applyAuthentication(req *http.Request, hasExplicitAuthorization bool) { t.applyConfiguredAuthentication(req) if hasExplicitAuthorization { From cbb1978026dd9a269509c70cb04aaca0aec58d3d Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Wed, 14 Oct 2026 16:43:11 +0200 Subject: [PATCH 293/633] Bump git-pkgs/integrity to v0.2.0 v0.2.0 adds verification-only SHA-1 and MD5, algorithm-prefixed checksum parsing and exact-digest verification (git-pkgs/integrity#3). --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 052604e..b76c338 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/git-pkgs/cooldown v0.2.0 github.com/git-pkgs/enrichment v0.7.2 github.com/git-pkgs/gcs v0.1.0 - github.com/git-pkgs/integrity v0.1.1 + github.com/git-pkgs/integrity v0.2.0 github.com/git-pkgs/magic v0.5.0 github.com/git-pkgs/purl v0.1.21 github.com/git-pkgs/registries v0.9.3 diff --git a/go.sum b/go.sum index c6bed9e..3533e64 100644 --- a/go.sum +++ b/go.sum @@ -135,8 +135,8 @@ github.com/git-pkgs/enrichment v0.7.2 h1:flD/M89HxkOegUi6tC8EOspFiIMsl7m0o576IQK github.com/git-pkgs/enrichment v0.7.2/go.mod h1:xGyc/iz7u1UnbrOHk8lIX5ZaNF3mKFDEZc3VqBJ/LOk= github.com/git-pkgs/gcs v0.1.0 h1:E3awGtsO0xZyHT9FUfEwMHjMkRxw41Bh+c7lgTmPvBo= github.com/git-pkgs/gcs v0.1.0/go.mod h1:bdkCFD66ryaWnU8MBhokVA3WkJfyAEchm9qec5woBpE= -github.com/git-pkgs/integrity v0.1.1 h1:nHQ7SktOiGM1dOb5BFnkdtttG/6FCgE6r5ru6QnsGts= -github.com/git-pkgs/integrity v0.1.1/go.mod h1:hxu24lcd230377hCF28JQW7sGcCbuNLqo/0ULeb+F1Q= +github.com/git-pkgs/integrity v0.2.0 h1:sxpAk7EUllZnTYttOoPTW/Yfe5J2imglpuJiuBUZOxM= +github.com/git-pkgs/integrity v0.2.0/go.mod h1:vPLb2RB+hR/iCMIgqph0PZbQ7iktXDTK+uupoHQikKk= github.com/git-pkgs/magic v0.5.0 h1:Pm+4fkHR+K6kg9pwJM4hnwWUuBlkYdLmF1yIAI8lU9Q= github.com/git-pkgs/magic v0.5.0/go.mod h1:SXOqcsNmbmpZjJZHEEWnwxprbsFvpwWgTAZrgXp4Jm4= github.com/git-pkgs/pom v0.1.8 h1:mXCuWw6XkrKKZb6IurRZyIznYLPhV5AYg5ZO38Uj/l8= From bf74d982d758c8c3e9bcef4612951f0f147aac8d Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Wed, 14 Oct 2026 15:43:39 +0200 Subject: [PATCH 294/633] Verify npm and Composer archives against their SHA-1 shasum npm versions published before dist.integrity existed carry only a SHA-1 dist.shasum, and Composer metadata lists only a SHA-1 dist.shasum, so neither could be checked. - Accept native integrity given as an algorithm-prefixed hex checksum, verified with integrity's exact-digest check, so a recorded "sha1-" is also checked on cache hits - npm: fall back to dist.shasum when dist.integrity is missing or cannot be parsed; a usable dist.integrity still takes precedence - Composer: verify an archive against its dist.shasum, except another commit of a branch fetched in place of the listed one, which the listed shasum does not describe - Fetch unverified as before when no shasum is listed, as Packagist does for GitHub archives, or it is not a SHA-1 digest --- internal/handler/composer.go | 23 +++++- internal/handler/composer_shasum_test.go | 96 ++++++++++++++++++++++++ internal/handler/handler.go | 9 ++- internal/handler/integrity.go | 32 +++++++- internal/handler/integrity_test.go | 33 ++++++++ internal/handler/npm.go | 22 +++--- internal/handler/npm_integrity_test.go | 6 +- 7 files changed, 202 insertions(+), 19 deletions(-) create mode 100644 internal/handler/composer_shasum_test.go diff --git a/internal/handler/composer.go b/internal/handler/composer.go index 9dac530..a4dab18 100644 --- a/internal/handler/composer.go +++ b/internal/handler/composer.go @@ -711,7 +711,8 @@ func (h *ComposerHandler) handleDownload(w http.ResponseWriter, r *http.Request) if commit := strings.TrimSuffix(filename, path.Ext(filename)); isDevVersion(version) && composerCommitPattern.MatchString(commit) { h.discardOtherCommit(packageName, version, cacheFilename, commit) } - result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "composer", packageName, version, cacheFilename, downloadURL) + result, err := h.proxy.GetOrFetchArtifactFromURLWithIntegrity(r.Context(), "composer", packageName, version, cacheFilename, downloadURL, + composerIntegrity(dist.shasum, downloadURL == listedURL)) if err != nil { h.proxy.serveArtifactError(w, err, "failed to fetch package") return @@ -776,10 +777,11 @@ func (h *ComposerHandler) metadataURLsForVersion(vendor, pkg, version string) [] } // composerDist is what a version's metadata says about its archive: the dist -// URL and type, and the raw publish time the cooldown is checked against. +// URL, type and shasum, and the raw publish time the cooldown is checked +// against. type composerDist struct { - url, distType string - published []byte + url, distType, shasum string + published []byte } // findDownloadURLFromMetadata fetches a metadata document and returns the dist @@ -862,6 +864,7 @@ func composerDistURL(body []byte, packageName, version string) (composerDist, er dist.url, _, _ = lookupJSONString(distRaw, "url") if dist.url != "" { dist.distType, _, _ = lookupJSONString(distRaw, "type") + dist.shasum, _, _ = lookupJSONString(distRaw, "shasum") dist.published = fields.get("time") return errStopScan } @@ -873,6 +876,18 @@ func composerDistURL(body []byte, packageName, version string) (composerDist, er return dist, err } +// composerIntegrity returns the integrity to verify an archive against: the +// SHA-1 its dist lists as shasum. That only describes the listed archive, not +// another commit of a branch fetched in its place. Packagist lists no shasum +// for GitHub archives, and an unusable one is ignored, so those are served +// unverified as before. +func composerIntegrity(shasum string, listed bool) string { + if !listed { + return "" + } + return sha1Checksum(shasum) +} + // proxyUpstream forwards a request to packagist.org without caching. func (h *ComposerHandler) proxyUpstream(w http.ResponseWriter, r *http.Request) { upstreamURL := h.upstreamURL + r.URL.Path diff --git a/internal/handler/composer_shasum_test.go b/internal/handler/composer_shasum_test.go new file mode 100644 index 0000000..a1cfeea --- /dev/null +++ b/internal/handler/composer_shasum_test.go @@ -0,0 +1,96 @@ +package handler + +import ( + "context" + "fmt" + "io" + "net/http" + "net/http/httptest" + "testing" + + "github.com/git-pkgs/registries/fetch" +) + +func TestComposerDownloadVerifiesDeclaredShasum(t *testing.T) { + const archive = "archive bytes" + + tests := []struct { + name string + shasum string + wantStatus int + }{ + {"matching shasum", sha1Hex(archive), http.StatusOK}, + {"mismatched shasum", sha1Hex("other bytes"), http.StatusBadGateway}, + {"empty shasum", "", http.StatusOK}, + {"unparseable shasum", "not-a-sha1", http.StatusOK}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + h, proxy := setupComposerShasum(t, "1.0.0", "/zip/p.zip", tt.shasum, map[string]string{"/zip/p.zip": archive}) + + w := httptest.NewRecorder() + h.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/files/v/p/1.0.0/p.zip", nil)) + if w.Code != tt.wantStatus { + t.Fatalf("status = %d, want %d; body: %s", w.Code, tt.wantStatus, w.Body.String()) + } + if wantCached := tt.wantStatus == http.StatusOK; isComposerArchiveCached(t, proxy, "1.0.0", "p.zip") != wantCached { + t.Errorf("cached = %v, want %v", !wantCached, wantCached) + } + }) + } +} + +// TestComposerDownloadPinnedCommitIgnoresListedShasum checks that the shasum +// listed for a branch's head is not held against an older commit a lock file +// pins: it describes another archive. +func TestComposerDownloadPinnedCommitIgnoresListedShasum(t *testing.T) { + h, _ := setupComposerShasum(t, "dev-master", "/zip/"+composerCommitB, sha1Hex("commit B"), map[string]string{ + "/zip/" + composerCommitA: "commit A", + "/zip/" + composerCommitB: "commit B", + }) + + w := httptest.NewRecorder() + h.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/files/v/p/dev-master/"+composerCommitA+".zip", nil)) + if w.Code != http.StatusOK || w.Body.String() != "commit A" { + t.Errorf("status %d, body %q, want 200 %q", w.Code, w.Body.String(), "commit A") + } +} + +func setupComposerShasum(t *testing.T, version, distPath, shasum string, archives map[string]string) (http.Handler, *Proxy) { + t.Helper() + var srv *httptest.Server + srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/p2/v/p.json" || r.URL.Path == "/p2/v/p~dev.json" { + _, _ = fmt.Fprintf(w, `{"packages":{"v/p":[{"version":%q,"dist":{"url":%q,"type":"zip","shasum":%q}}]}}`, + version, srv.URL+distPath, shasum) + return + } + body, ok := archives[r.URL.Path] + if !ok { + http.NotFound(w, r) + return + } + _, _ = io.WriteString(w, body) + })) + t.Cleanup(srv.Close) + + proxy, _, _, _ := setupTestProxy(t) + fetcher := fetch.NewFetcher(fetch.WithHTTPClient(srv.Client()), fetch.WithMaxRetries(0)) + proxy.Fetcher = fetcher + t.Cleanup(func() { _ = fetcher.Close() }) + return NewComposerHandlerWithUpstreams(proxy, "http://proxy", srv.URL, srv.URL).Routes(), proxy +} + +func isComposerArchiveCached(t *testing.T, proxy *Proxy, version, filename string) bool { + t.Helper() + cached, err := proxy.GetCachedArtifact(context.Background(), "composer", "v/p", version, filename) + if err != nil { + t.Fatalf("reading cache: %v", err) + } + if cached == nil { + return false + } + _ = cached.Reader.Close() + return true +} diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 5e0688d..c3a9ea0 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -1808,10 +1808,11 @@ func (p *Proxy) GetOrFetchArtifactFromURLWithDigest(ctx context.Context, ecosyst } // GetOrFetchArtifactFromURLWithIntegrity retrieves an artifact and verifies a -// newly fetched response against declared, the Subresource Integrity string -// the registry's metadata declares, before serving or caching it. declared is -// recorded with the cached version, so cache hits are verified against it as -// well. An empty declared fetches without verification. +// newly fetched response against declared, the integrity the registry's +// metadata declares, before serving or caching it: a Subresource Integrity +// string, or a single algorithm-prefixed hex checksum such as "sha1-". +// declared is recorded with the cached version, so cache hits are verified +// against it as well. An empty declared fetches without verification. func (p *Proxy) GetOrFetchArtifactFromURLWithIntegrity(ctx context.Context, ecosystem, name, version, filename, downloadURL, declared string) (*CacheResult, error) { return p.getOrFetchArtifactFromURL(ctx, ecosystem, name, version, filename, downloadURL, nil, "", declared) } diff --git a/internal/handler/integrity.go b/internal/handler/integrity.go index f78d231..7e800b8 100644 --- a/internal/handler/integrity.go +++ b/internal/handler/integrity.go @@ -10,6 +10,10 @@ import ( type integrityChecks struct { contentHash integrity.SRI native integrity.SRI + // checksum is native metadata given as a single algorithm-prefixed hex + // checksum, such as a registry's SHA-1 shasum, which is not SRI. + checksum integrity.Digest + hasChecksum bool algorithms []integrity.Algorithm } @@ -28,7 +32,13 @@ func newIntegrityChecks(contentHash, native string) (integrityChecks, error) { if native != "" { digests, err := integrity.ParseSRI(native) if err != nil { - return integrityChecks{}, fmt.Errorf("parse integrity: %w", err) + checksum, checksumErr := integrity.ParseChecksum(native) + if checksumErr != nil { + return integrityChecks{}, fmt.Errorf("parse integrity: %w", err) + } + checks.checksum, checks.hasChecksum = checksum, true + checks.algorithms = append(checks.algorithms, checksum.Algorithm()) + return checks, nil } checks.native = digests for _, digest := range digests { @@ -189,4 +199,24 @@ func (r *verifyingReader) verify() { r.onMismatch("integrity: " + err.Error()) } } + if r.checks.hasChecksum { + if err := result.VerifyDigest(r.checks.checksum); err != nil { + r.mismatched = true + r.onMismatch("integrity: " + err.Error()) + } + } +} + +// sha1Checksum returns a registry's hexadecimal SHA-1 shasum as the +// "sha1-" checksum newIntegrityChecks accepts, or "" when it is empty or +// not a SHA-1 digest. +func sha1Checksum(shasum string) string { + if shasum == "" { + return "" + } + digest, err := integrity.ParseHex(integrity.SHA1, shasum) + if err != nil { + return "" + } + return digest.Checksum() } diff --git a/internal/handler/integrity_test.go b/internal/handler/integrity_test.go index 9aa9958..16f47b1 100644 --- a/internal/handler/integrity_test.go +++ b/internal/handler/integrity_test.go @@ -8,6 +8,7 @@ import ( "encoding/hex" "errors" "io" + "slices" "strings" "testing" ) @@ -84,6 +85,38 @@ func TestNewIntegrityChecksRejectsMalformedMetadata(t *testing.T) { } } +// TestIntegrityChecksVerifyRegistryChecksum checks native metadata given as +// an algorithm-prefixed hex checksum, the form recorded for a registry's SHA-1 +// shasum, which is not SRI. +func TestIntegrityChecksVerifyRegistryChecksum(t *testing.T) { + const data = "hello world" + checksum := "sha1-" + sha1Hex(data) + checks, err := newIntegrityChecks(sha256Hex(data), checksum) + if err != nil { + t.Fatalf("newIntegrityChecks(%q): %v", checksum, err) + } + for _, tt := range []struct { + name, body string + wantMismatch bool + }{ + {"matching", data, false}, + {"mismatched", "other bytes", true}, + } { + t.Run(tt.name, func(t *testing.T) { + var reasons []string + reader, err := checks.wrapFailOnMismatch(io.NopCloser(strings.NewReader(tt.body)), func(reason string) { reasons = append(reasons, reason) }) + if err != nil { + t.Fatal(err) + } + _, _ = io.ReadAll(reader) + gotNative := slices.ContainsFunc(reasons, func(r string) bool { return strings.HasPrefix(r, "integrity: ") }) + if gotNative != tt.wantMismatch { + t.Errorf("checksum mismatch reported = %v, want %v (reasons %q)", gotNative, tt.wantMismatch, reasons) + } + }) + } +} + func TestVerifyingReader(t *testing.T) { const data = "hello world" goodSHA := sha256Hex(data) diff --git a/internal/handler/npm.go b/internal/handler/npm.go index 0e0d2b0..be42d4f 100644 --- a/internal/handler/npm.go +++ b/internal/handler/npm.go @@ -16,6 +16,7 @@ import ( "sync" "time" + "github.com/git-pkgs/integrity" "github.com/git-pkgs/proxy/internal/retention" ) @@ -878,18 +879,21 @@ func (h *NPMHandler) getTarball(r *http.Request, registryURL, packageName, versi return h.proxy.GetOrFetchArtifactFromURLWithIntegrity(r.Context(), "npm", packageName, version, cacheFilename, downloadURL, sri) } -// npmVersionIntegrity returns versions[version].dist.integrity from a -// packument when it is a Subresource Integrity string the proxy can verify, -// or "" otherwise, in which case the tarball is fetched unverified as before. +// npmVersionIntegrity returns the integrity a packument declares for a +// version's tarball: dist.integrity when it is a Subresource Integrity string +// the proxy can verify, or else the SHA-1 in dist.shasum, which is all +// versions published before integrity existed carry, as "sha1-". It +// returns "" when neither is usable, and the tarball is fetched unverified as +// before. func npmVersionIntegrity(body []byte, version string) string { sri, _, _ := lookupJSONString(body, "versions", version, "dist", "integrity") - if sri == "" { - return "" + if sri != "" { + if _, err := integrity.ParseSRI(sri); err == nil { + return sri + } } - if _, err := newIntegrityChecks("", sri); err != nil { - return "" - } - return sri + shasum, _, _ := lookupJSONString(body, "versions", version, "dist", "shasum") + return sha1Checksum(shasum) } // npmVersionTarball returns versions[version].dist.tarball from a packument, diff --git a/internal/handler/npm_integrity_test.go b/internal/handler/npm_integrity_test.go index 202a741..2c14614 100644 --- a/internal/handler/npm_integrity_test.go +++ b/internal/handler/npm_integrity_test.go @@ -27,7 +27,11 @@ var npmIntegrityCases = []struct { {"strongest algorithm decides", map[string]string{"integrity": sha256SRI("other bytes") + " " + sha512SRI(npmIntegrityTarball)}, true}, {"any digest of the strongest algorithm", map[string]string{"integrity": sha512SRI("other bytes") + " " + sha512SRI(npmIntegrityTarball)}, true}, {"unparseable integrity", map[string]string{"integrity": "sha512-nope"}, true}, - {"shasum alone is not checked yet", map[string]string{"shasum": sha1Hex("other bytes")}, true}, + {"matching shasum", map[string]string{"shasum": sha1Hex(npmIntegrityTarball)}, true}, + {"mismatched shasum", map[string]string{"shasum": sha1Hex("other bytes")}, false}, + {"integrity wins over shasum", map[string]string{"integrity": sha512SRI("other bytes"), "shasum": sha1Hex(npmIntegrityTarball)}, false}, + {"unparseable integrity falls back to shasum", map[string]string{"integrity": "sha512-nope", "shasum": sha1Hex("other bytes")}, false}, + {"unparseable shasum", map[string]string{"shasum": "not-a-sha1"}, true}, {"no checksum", map[string]string{}, true}, } From ba5e6e84c53ab9e7554e8244cddc576d282590ef Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Wed, 14 Oct 2026 18:06:01 +0200 Subject: [PATCH 295/633] Leave Composer dev branch archives unverified A version's integrity is a single value shared by every archive cached for it, but a dev branch caches one archive per commit. Recording the listed head's shasum made a cache hit for an older pinned commit fail its check, so its archive was discarded and fetched again, which cleared the head's recorded shasum in turn. The listed shasum only ever describes the current head, and Composer checks dist.shasum itself. - Skip the shasum for dev versions - Test a pinned commit, then the head, then both again from the cache --- internal/handler/composer.go | 20 ++++++---- internal/handler/composer_shasum_test.go | 50 +++++++++++++++++++++++- 2 files changed, 61 insertions(+), 9 deletions(-) diff --git a/internal/handler/composer.go b/internal/handler/composer.go index a4dab18..c89a18b 100644 --- a/internal/handler/composer.go +++ b/internal/handler/composer.go @@ -712,7 +712,7 @@ func (h *ComposerHandler) handleDownload(w http.ResponseWriter, r *http.Request) h.discardOtherCommit(packageName, version, cacheFilename, commit) } result, err := h.proxy.GetOrFetchArtifactFromURLWithIntegrity(r.Context(), "composer", packageName, version, cacheFilename, downloadURL, - composerIntegrity(dist.shasum, downloadURL == listedURL)) + composerIntegrity(dist.shasum, version, downloadURL == listedURL)) if err != nil { h.proxy.serveArtifactError(w, err, "failed to fetch package") return @@ -876,13 +876,17 @@ func composerDistURL(body []byte, packageName, version string) (composerDist, er return dist, err } -// composerIntegrity returns the integrity to verify an archive against: the -// SHA-1 its dist lists as shasum. That only describes the listed archive, not -// another commit of a branch fetched in its place. Packagist lists no shasum -// for GitHub archives, and an unusable one is ignored, so those are served -// unverified as before. -func composerIntegrity(shasum string, listed bool) string { - if !listed { +// composerIntegrity returns the integrity to verify an archive of version +// against: the SHA-1 its dist lists as shasum. A dev branch is left +// unverified: its listed shasum describes only the current head, while every +// commit of the branch is cached under the same version, which holds a single +// recorded integrity, so a cache hit for an older pinned commit would be +// checked against the head's shasum. Composer checks dist.shasum itself +// anyway. A shasum also only describes the listed archive, and Packagist +// lists none for GitHub archives; an unusable one is ignored. Those archives +// are served unverified as before. +func composerIntegrity(shasum, version string, listed bool) string { + if !listed || isDevVersion(version) { return "" } return sha1Checksum(shasum) diff --git a/internal/handler/composer_shasum_test.go b/internal/handler/composer_shasum_test.go index a1cfeea..b484795 100644 --- a/internal/handler/composer_shasum_test.go +++ b/internal/handler/composer_shasum_test.go @@ -6,6 +6,7 @@ import ( "io" "net/http" "net/http/httptest" + "sync" "testing" "github.com/git-pkgs/registries/fetch" @@ -59,6 +60,16 @@ func TestComposerDownloadPinnedCommitIgnoresListedShasum(t *testing.T) { func setupComposerShasum(t *testing.T, version, distPath, shasum string, archives map[string]string) (http.Handler, *Proxy) { t.Helper() + h, proxy, _ := setupComposerShasumCounting(t, version, distPath, shasum, archives) + return h, proxy +} + +// setupComposerShasumCounting is setupComposerShasum that also reports how +// many times each archive path was fetched from upstream. +func setupComposerShasumCounting(t *testing.T, version, distPath, shasum string, archives map[string]string) (http.Handler, *Proxy, func(string) int) { + t.Helper() + var mu sync.Mutex + fetched := map[string]int{} var srv *httptest.Server srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.URL.Path == "/p2/v/p.json" || r.URL.Path == "/p2/v/p~dev.json" { @@ -71,6 +82,9 @@ func setupComposerShasum(t *testing.T, version, distPath, shasum string, archive http.NotFound(w, r) return } + mu.Lock() + fetched[r.URL.Path]++ + mu.Unlock() _, _ = io.WriteString(w, body) })) t.Cleanup(srv.Close) @@ -79,7 +93,11 @@ func setupComposerShasum(t *testing.T, version, distPath, shasum string, archive fetcher := fetch.NewFetcher(fetch.WithHTTPClient(srv.Client()), fetch.WithMaxRetries(0)) proxy.Fetcher = fetcher t.Cleanup(func() { _ = fetcher.Close() }) - return NewComposerHandlerWithUpstreams(proxy, "http://proxy", srv.URL, srv.URL).Routes(), proxy + return NewComposerHandlerWithUpstreams(proxy, "http://proxy", srv.URL, srv.URL).Routes(), proxy, func(path string) int { + mu.Lock() + defer mu.Unlock() + return fetched[path] + } } func isComposerArchiveCached(t *testing.T, proxy *Proxy, version, filename string) bool { @@ -94,3 +112,33 @@ func isComposerArchiveCached(t *testing.T, proxy *Proxy, version, filename strin _ = cached.Reader.Close() return true } + +// TestComposerDevBranchArchivesStayServableFromCache is a regression test: a +// version's integrity is one value shared by all its cached archives, but a +// dev branch caches one archive per commit. Recording the head's shasum made +// a pinned older commit fail its check when served from the cache, so its +// archive was discarded and fetched again, which in turn cleared the head's +// recorded shasum. +func TestComposerDevBranchArchivesStayServableFromCache(t *testing.T) { + h, proxy, fetched := setupComposerShasumCounting(t, "dev-master", "/zip/"+composerCommitB, sha1Hex("commit B"), map[string]string{ + "/zip/" + composerCommitA: "commit A", + "/zip/" + composerCommitB: "commit B", + }) + + for i, commit := range []string{composerCommitA, composerCommitB, composerCommitA, composerCommitB} { + want := map[string]string{composerCommitA: "commit A", composerCommitB: "commit B"}[commit] + w := httptest.NewRecorder() + h.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/files/v/p/dev-master/"+commit+".zip", nil)) + if w.Code != http.StatusOK || w.Body.String() != want { + t.Errorf("request %d for %s: status %d, body %q; want 200 %q", i+1, commit[:4], w.Code, w.Body.String(), want) + } + } + for _, commit := range []string{composerCommitA, composerCommitB} { + if n := fetched("/zip/" + commit); n != 1 { + t.Errorf("commit %s fetched from upstream %d times, want once and then served from the cache", commit[:4], n) + } + if !isComposerArchiveCached(t, proxy, "dev-master", commit+".zip") { + t.Errorf("commit %s is no longer cached", commit[:4]) + } + } +} From 3bbb5aa7a37622635128e017e78174eca5c0c49b Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Wed, 14 Oct 2026 18:14:06 +0200 Subject: [PATCH 296/633] Give the detached fetch test time on slow runners TestServerKeepsFetchAfterClientHangsUp gave the upstream fetch 5s to start and the follow-up request 5s to be answered. On a Windows CI runner, where the whole test took 13s, the follow-up timed out awaiting headers with nothing wrong. The test checks the outcome, not the speed, so allow 30s for both. A real hang still fails. --- internal/server/detached_fetch_test.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/internal/server/detached_fetch_test.go b/internal/server/detached_fetch_test.go index 557de51..c18a4a6 100644 --- a/internal/server/detached_fetch_test.go +++ b/internal/server/detached_fetch_test.go @@ -36,6 +36,12 @@ func TestServerKeepsFetchAfterClientHangsUp(t *testing.T) { } } +// slowRunnerBudget bounds the steps that only need to finish eventually: the +// upstream fetch starting, and the follow-up request being answered. What the +// test checks is the outcome, not the speed, and on a slow CI runner a +// shorter budget made it fail with nothing wrong. +const slowRunnerBudget = 30 * time.Second + func testServerKeepsFetchAfterClientHangsUp(t *testing.T) { const pkgPath = "/core/os/x86_64/big-1.0-1-x86_64.pkg.tar.zst" release := make(chan struct{}) @@ -93,7 +99,7 @@ func testServerKeepsFetchAfterClientHangsUp(t *testing.T) { // The client gives up while the upstream is still sending nothing. impatient := &http.Client{Timeout: 300 * time.Millisecond} - deadline := time.Now().Add(5 * time.Second) + deadline := time.Now().Add(slowRunnerBudget) for hits.Load() == 0 { if _, err := impatient.Get(target); err == nil { t.Fatal("request succeeded while the upstream was held") @@ -106,7 +112,7 @@ func testServerKeepsFetchAfterClientHangsUp(t *testing.T) { time.Sleep(100 * time.Millisecond) releaseOnce.Do(func() { close(release) }) - client := &http.Client{Timeout: 5 * time.Second} + client := &http.Client{Timeout: slowRunnerBudget} resp, err := client.Get(target) if err != nil { t.Fatalf("follow-up request: %v", err) From 6051e20adfd8274fcd875840557d918547042efd Mon Sep 17 00:00:00 2001 From: Giles Westwood <201443+gilesw@users.noreply.github.com> Date: Wed, 14 Oct 2026 12:31:06 +0100 Subject: [PATCH 297/633] Cache Maven checksums and signatures as artifacts Checksum and .asc files beside a versioned artifact are as immutable as the artifact, but were routed through the metadata path: uncached until a first successful fetch, with no retry or circuit breaker. A build pulling hundreds of artifacts sent as many checksum requests straight to Maven Central, which rate-limited the proxy, after which every further checksum request still went upstream and failed. Only maven-metadata.xml and its own checksums stay on the metadata path, since the version listing changes with each release. --- internal/handler/download_test.go | 116 +++++++++++++++++------------- internal/handler/maven.go | 33 +++++---- internal/handler/maven_test.go | 83 +++++++++------------ 3 files changed, 121 insertions(+), 111 deletions(-) diff --git a/internal/handler/download_test.go b/internal/handler/download_test.go index 2797d42..2eece84 100644 --- a/internal/handler/download_test.go +++ b/internal/handler/download_test.go @@ -758,8 +758,8 @@ func TestMavenHandler_MetadataProxied(t *testing.T) { paths := []string{ "/com/google/guava/guava/maven-metadata.xml", - "/com/google/guava/guava/32.1.3-jre/guava-32.1.3-jre.jar.sha1", - "/com/google/guava/guava/32.1.3-jre/guava-32.1.3-jre.jar.md5", + "/com/google/guava/guava/maven-metadata.xml.sha1", + "/com/google/guava/guava/1.0-SNAPSHOT/maven-metadata.xml", } for _, path := range paths { @@ -829,6 +829,47 @@ func TestMavenHandler_ArtifactExtensions(t *testing.T) { } } +func TestMavenHandler_ChecksumsCachedAsArtifacts(t *testing.T) { + proxy, _, _, fetcher := setupTestProxy(t) + + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + t.Errorf("checksum %s proxied as metadata", r.URL.Path) + })) + defer upstream.Close() + + h := NewMavenHandler(proxy, "http://localhost", "", "") + h.upstreamURL = upstream.URL + proxy.HTTPClient = upstream.Client() + + srv := httptest.NewServer(h.Routes()) + defer srv.Close() + + for _, name := range []string{"lib-1.0.jar.sha1", "lib-1.0.pom.md5", "lib-1.0.jar.asc", "lib-1.0-bin.tar.gz.sha512"} { + path := "/com/example/lib/1.0/" + name + for i := range 2 { + fetcher.artifact = &fetch.Artifact{ + Body: io.NopCloser(strings.NewReader("checksum")), + ContentType: "text/plain", + } + fetcher.fetchCalled = false + + resp, err := http.Get(srv.URL + path) + if err != nil { + t.Fatalf("GET %s failed: %v", path, err) + } + body, _ := io.ReadAll(resp.Body) + _ = resp.Body.Close() + + if resp.StatusCode != http.StatusOK || string(body) != "checksum" { + t.Errorf("GET %s: status %d body %q", path, resp.StatusCode, body) + } + if fetcher.fetchCalled != (i == 0) { + t.Errorf("GET %s request %d: fetched = %v, want %v", path, i+1, fetcher.fetchCalled, i == 0) + } + } + } +} + func TestMavenHandler_CacheMiss(t *testing.T) { proxy, _, _, fetcher := setupTestProxy(t) fetcher.artifact = &fetch.Artifact{ @@ -929,10 +970,8 @@ func TestMavenHandler_GradlePluginMarkerFallbackAndCache(t *testing.T) { func TestMavenHandler_GradlePluginMarkerMetadataFallback(t *testing.T) { paths := map[string]string{ - "/com/diffplug/spotless/com.diffplug.spotless.gradle.plugin/8.4.0/com.diffplug.spotless.gradle.plugin-8.4.0.pom.sha1": "sha1", - "/com/diffplug/spotless/com.diffplug.spotless.gradle.plugin/8.4.0/com.diffplug.spotless.gradle.plugin-8.4.0.pom.sha256": "sha256", - "/com/diffplug/spotless/com.diffplug.spotless.gradle.plugin/8.4.0/com.diffplug.spotless.gradle.plugin-8.4.0.pom.md5": "md5", - "/com/diffplug/spotless/com.diffplug.spotless.gradle.plugin/maven-metadata.xml": "", + "/com/diffplug/spotless/com.diffplug.spotless.gradle.plugin/maven-metadata.xml": "", + "/com/diffplug/spotless/com.diffplug.spotless.gradle.plugin/maven-metadata.xml.sha1": "sha1", } primaryHits := map[string]int{} @@ -990,63 +1029,40 @@ func TestMavenHandler_GradlePluginMarkerMetadataFallback(t *testing.T) { } } -func TestMavenHandler_GradlePluginImplementationMetadataFallback(t *testing.T) { - paths := map[string]string{ - "/com/diffplug/spotless/spotless-plugin-gradle/8.4.0/spotless-plugin-gradle-8.4.0.jar.sha1": "impl-sha1", - "/com/diffplug/spotless/spotless-plugin-gradle/8.4.0/spotless-plugin-gradle-8.4.0.jar.sha256": "impl-sha256", +func TestMavenHandler_GradlePluginChecksumFallbackToPluginPortal(t *testing.T) { + primaryUpstream := "https://repo1.maven.org/maven2" + pluginPortalUpstream := "https://plugins.gradle.org/m2" + paths := []string{ + "/com/diffplug/spotless/com.diffplug.spotless.gradle.plugin/8.4.0/com.diffplug.spotless.gradle.plugin-8.4.0.pom.sha1", + "/com/diffplug/spotless/com.diffplug.spotless.gradle.plugin/8.4.0/com.diffplug.spotless.gradle.plugin-8.4.0.pom.md5", + "/com/diffplug/spotless/spotless-plugin-gradle/8.4.0/spotless-plugin-gradle-8.4.0.jar.sha1", + "/com/diffplug/spotless/spotless-plugin-gradle/8.4.0/spotless-plugin-gradle-8.4.0.jar.sha256", } - primaryHits := map[string]int{} - pluginHits := map[string]int{} - - primary := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - primaryHits[r.URL.Path]++ - if _, ok := paths[r.URL.Path]; ok { - http.NotFound(w, r) - return + for _, reqPath := range paths { + proxy, _, _, fetcher := setupTestProxy(t) + fetcher.fetchErrByURL = map[string]error{primaryUpstream + reqPath: ErrUpstreamNotFound} + fetcher.artifact = &fetch.Artifact{ + Body: io.NopCloser(strings.NewReader("checksum")), + ContentType: "text/plain", } - t.Fatalf("unexpected path to primary upstream: %s", r.URL.Path) - })) - defer primary.Close() - pluginPortal := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - pluginHits[r.URL.Path]++ - body, ok := paths[r.URL.Path] - if !ok { - http.NotFound(w, r) - return - } - _, _ = io.WriteString(w, body) - })) - defer pluginPortal.Close() + h := NewMavenHandler(proxy, "http://localhost", primaryUpstream, pluginPortalUpstream) + srv := httptest.NewServer(h.Routes()) - proxy, _, _, _ := setupTestProxy(t) - proxy.HTTPClient = primary.Client() - - h := NewMavenHandler(proxy, "http://localhost", primary.URL, pluginPortal.URL) - srv := httptest.NewServer(h.Routes()) - defer srv.Close() - - for reqPath, wantBody := range paths { resp, err := http.Get(srv.URL + reqPath) if err != nil { t.Fatalf("GET %s failed: %v", reqPath, err) } body, _ := io.ReadAll(resp.Body) _ = resp.Body.Close() + srv.Close() - if resp.StatusCode != http.StatusOK { - t.Fatalf("GET %s: status = %d, want %d", reqPath, resp.StatusCode, http.StatusOK) + if resp.StatusCode != http.StatusOK || string(body) != "checksum" { + t.Fatalf("GET %s: status %d body %q", reqPath, resp.StatusCode, body) } - if string(body) != wantBody { - t.Fatalf("GET %s: body = %q, want %q", reqPath, body, wantBody) - } - - if primaryHits[reqPath] == 0 { - t.Fatalf("GET %s did not hit primary upstream", reqPath) - } - if pluginHits[reqPath] == 0 { - t.Fatalf("GET %s did not hit plugin portal fallback", reqPath) + if want := pluginPortalUpstream + reqPath; fetcher.fetchedURL != want { + t.Fatalf("GET %s: fetched URL = %q, want %q", reqPath, fetcher.fetchedURL, want) } } } diff --git a/internal/handler/maven.go b/internal/handler/maven.go index 700c107..1922620 100644 --- a/internal/handler/maven.go +++ b/internal/handler/maven.go @@ -79,21 +79,19 @@ func (h *MavenHandler) handleRequest(w http.ResponseWriter, r *http.Request) { return } - // Check if this is an artifact file or metadata filename := path.Base(urlPath) - if h.isMetadataFile(filename) { + if h.isVersionListingFile(filename) { h.handleMetadata(w, r, urlPath) return } - if h.isArtifactFile(filename) { - // Cache artifact files + if h.isArtifactFile(filename) || h.isChecksumFile(filename) { h.handleDownload(w, r, urlPath) return } - // Proxy everything else (directory listings, checksums, etc.) + // Proxy everything else (directory listings etc.) h.proxyUpstream(w, r) } @@ -192,14 +190,23 @@ func (h *MavenHandler) isArtifactFile(filename string) bool { return false } -// isMetadataFile returns true if the filename is Maven metadata. -func (h *MavenHandler) isMetadataFile(filename string) bool { - return filename == "maven-metadata.xml" || - strings.HasSuffix(filename, ".sha1") || - strings.HasSuffix(filename, ".sha256") || - strings.HasSuffix(filename, ".sha512") || - strings.HasSuffix(filename, ".md5") || - strings.HasSuffix(filename, ".asc") +// isVersionListingFile matches maven-metadata.xml and its checksums. The +// listing changes with every release, so it is cached for the metadata TTL +// rather than kept like an artifact. +func (h *MavenHandler) isVersionListingFile(filename string) bool { + return strings.HasPrefix(filename, "maven-metadata.xml") +} + +// isChecksumFile matches the checksum and signature files published beside +// an artifact. They are as immutable as the artifact, so they are cached +// with it rather than refetched on every request. +func (h *MavenHandler) isChecksumFile(filename string) bool { + for _, ext := range []string{".sha1", ".sha256", ".sha512", ".md5", ".asc"} { + if strings.HasSuffix(filename, ext) { + return true + } + } + return false } // proxyUpstream forwards a request to Maven Central without caching. diff --git a/internal/handler/maven_test.go b/internal/handler/maven_test.go index 9331a00..beaa8f7 100644 --- a/internal/handler/maven_test.go +++ b/internal/handler/maven_test.go @@ -71,62 +71,49 @@ func TestMavenIsArtifactFile(t *testing.T) { } } -func TestMavenIsMetadataFile(t *testing.T) { +func TestMavenIsVersionListingFile(t *testing.T) { h := &MavenHandler{} tests := []struct { - name string filename string want bool }{ - { - name: "pom is artifact, not metadata", - filename: "com.diffplug.spotless.gradle.plugin-8.4.0.pom", - want: false, - }, - { - name: "pom checksum is metadata", - filename: "com.diffplug.spotless.gradle.plugin-8.4.0.pom.sha1", - want: true, - }, - { - name: "metadata file", - filename: "maven-metadata.xml", - want: true, - }, - { - name: "metadata checksum", - filename: "maven-metadata.xml.sha256", - want: true, - }, - { - name: "jar checksum is metadata", - filename: "guava-32.1.3-jre.jar.sha1", - want: true, - }, - { - name: "asc signature is metadata", - filename: "guava-32.1.3-jre.jar.asc", - want: true, - }, - { - name: "regular jar is not metadata", - filename: "guava-32.1.3-jre.jar", - want: false, - }, - { - name: "pom checksum is metadata", - filename: "guava-32.1.3-jre.pom.sha1", - want: true, - }, + {"maven-metadata.xml", true}, + {"maven-metadata.xml.sha1", true}, + {"maven-metadata.xml.sha256", true}, + {"guava-32.1.3-jre.jar", false}, + {"guava-32.1.3-jre.jar.sha1", false}, + {"com.diffplug.spotless.gradle.plugin-8.4.0.pom", false}, } for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got := h.isMetadataFile(tt.filename) - if got != tt.want { - t.Errorf("isMetadataFile(%q) = %v, want %v", tt.filename, got, tt.want) - } - }) + if got := h.isVersionListingFile(tt.filename); got != tt.want { + t.Errorf("isVersionListingFile(%q) = %v, want %v", tt.filename, got, tt.want) + } + } +} + +func TestMavenIsChecksumFile(t *testing.T) { + h := &MavenHandler{} + + tests := []struct { + filename string + want bool + }{ + {"guava-32.1.3-jre.jar.sha1", true}, + {"guava-32.1.3-jre.jar.md5", true}, + {"guava-32.1.3-jre.pom.sha1", true}, + {"guava-32.1.3-jre.jar.asc", true}, + {"apache-maven-3.9.9-bin.tar.gz.sha512", true}, + {"com.diffplug.spotless.gradle.plugin-8.4.0.pom.sha1", true}, + {"guava-32.1.3-jre.jar", false}, + {"guava-32.1.3-jre.pom", false}, + {"maven-metadata.xml", false}, + } + + for _, tt := range tests { + if got := h.isChecksumFile(tt.filename); got != tt.want { + t.Errorf("isChecksumFile(%q) = %v, want %v", tt.filename, got, tt.want) + } } } From e9ecde70edefdc34cca8ba2aaa589ab207b7986a Mon Sep 17 00:00:00 2001 From: Giles Westwood <201443+gilesw@users.noreply.github.com> Date: Wed, 14 Oct 2026 12:51:43 +0100 Subject: [PATCH 298/633] Hold requests to an upstream that rate limits the proxy A 429, or a 503 with Retry-After, from an upstream host now stops the proxy sending that host anything for as long as it asked (capped at an hour), or for a minute doubling up to fifteen when no Retry-After came. Held requests fail locally: cached metadata is served stale as before, and anything uncached answers 503 with the remaining Retry-After so the client paces itself too. The gate is a RoundTripper under the auth transport, so metadata, artifact and passthrough fetches all obey it. Metadata fetches had no retry or circuit breaker, so a rate-limited upstream kept receiving one request per cache miss for the whole block. /health lists held hosts under upstream_holds and proxy_upstream_rate_limit_holds_total counts holds. --- README.md | 6 + docs/swagger/docs.go | 7 + docs/swagger/swagger.json | 7 + internal/handler/download_test.go | 48 ++++++ internal/handler/handler.go | 15 ++ internal/handler/maven.go | 3 + internal/handler/upstream_held.go | 27 ++++ internal/httpclient/ratelimit.go | 204 +++++++++++++++++++++++++ internal/httpclient/ratelimit_test.go | 207 ++++++++++++++++++++++++++ internal/metrics/metrics.go | 14 ++ internal/metrics/metrics_test.go | 1 + internal/server/breakers_test.go | 42 ++++++ internal/server/health.go | 16 ++ internal/server/server.go | 10 +- 14 files changed, 606 insertions(+), 1 deletion(-) create mode 100644 internal/handler/upstream_held.go create mode 100644 internal/httpclient/ratelimit.go create mode 100644 internal/httpclient/ratelimit_test.go diff --git a/README.md b/README.md index 6fb3e8a..559f1a8 100644 --- a/README.md +++ b/README.md @@ -1364,6 +1364,7 @@ The proxy exposes Prometheus metrics at `GET /metrics`. All metric names are pre | `proxy_health_probe_failures_total` | counter | `step` | Storage health probe failures by failing step (`write`, `size`, `read`, `verify`, `delete`). | | `proxy_circuit_breaker_state` | gauge | `registry` | Artifact-fetch circuit breaker state per upstream registry (0 closed, 2 open). Published once that registry's breaker has tripped. | | `proxy_circuit_breaker_trips_total` | counter | `registry` | Circuit breaker trips per upstream registry. | +| `proxy_upstream_rate_limit_holds_total` | counter | `host` | Holds placed on an upstream host after it rate limited the proxy with HTTP 429, or 503 with a `Retry-After`. See `upstream_holds` under [Health Check](#health-check). | | `proxy_ecosystem_downloaded_bytes` | gauge | `ecosystem` | Accumulated bytes served from cache: cache hits multiplied by the artifact size they served. | | `proxy_ecosystem_artifact_downloads` | gauge | `ecosystem` | Accumulated artifact downloads served from cache. | | `proxy_ecosystem_cache_size_bytes` | gauge | `ecosystem` | Size of cached artifacts per ecosystem. | @@ -1468,6 +1469,9 @@ It carries three ecosystem filters rather than one, because `ecosystem` means th "circuit_breakers": { "registry.npmjs.org": "closed", "static.crates.io": "open" + }, + "upstream_holds": { + "repo1.maven.org": "2026-10-14T11:15:29Z" } } ``` @@ -1476,6 +1480,8 @@ Failing checks include an `"error"` field. Storage failures also include a `"ste `circuit_breakers` reports the state of each upstream's artifact-fetch circuit breaker (`"open"` or `"closed"`), keyed by upstream host — or by the `hostless-url-` placeholder described under [Monitoring](#monitoring) where the fetch URL has no host to read. The key is omitted until the proxy has fetched an artifact from at least one upstream, and a host appears only once a breaker has been created for it. Breakers trip after repeated upstream failures and retry the upstream after an exponential backoff. While one is open, artifact downloads for that host return HTTP 502 on a cache miss without contacting the upstream; already-cached artifacts are still served from storage, since the cache is checked before the fetcher. A breaker is reported as `"open"` throughout its backoff, including the half-open window in which it admits one probe request to test recovery. Breaker state is per process and in memory, so a restart clears it, but a restart is not needed for recovery: the backoff keeps retrying for as long as the breaker is open, so it closes on its own once the upstream serves again. +`upstream_holds` lists upstream hosts that answered the proxy with HTTP 429, or 503 with a `Retry-After`, and the time until which the proxy sends them nothing. Every request to a held host, metadata and artifacts alike, fails locally for the duration the upstream asked for (capped at one hour), or for one minute doubling up to fifteen when the upstream gave no `Retry-After`. Cached metadata is served stale during a hold, and anything uncached returns HTTP 503 with the remaining `Retry-After` so a client can pace itself too. Metadata requests do not go through the circuit breaker, so without the hold a rate-limited upstream would keep receiving one request per cache miss, which is exactly the traffic it is asking the proxy to stop sending. The first non-limited response after a hold ends clears it, and `proxy_upstream_rate_limit_holds_total` counts the holds placed. The key is omitted while no host is held. + An open breaker does **not** set `status` to `"error"` or change the HTTP status code: it reports a specific upstream refusing to serve, not this proxy being unfit to receive traffic, and failing the readiness probe over one unhealthy upstream would pull the pod out of rotation for every other ecosystem too. Use `proxy_circuit_breaker_state` for alerting on it. Storage probe results are cached for `health.storage_probe_interval` (default 30s) to bound the cost of probing remote backends. A probe holds an internal mutex for up to 10 seconds (the hardcoded per-probe timeout), so `/health` is intended as a Kubernetes **readiness** probe rather than a liveness probe — a slow S3 round-trip should pull the pod from rotation, not restart it. diff --git a/docs/swagger/docs.go b/docs/swagger/docs.go index 5e54d3c..7a8387d 100644 --- a/docs/swagger/docs.go +++ b/docs/swagger/docs.go @@ -825,6 +825,13 @@ const docTemplate = `{ }, "status": { "type": "string" + }, + "upstream_holds": { + "description": "UpstreamHolds lists upstream hosts that rate limited the proxy and\nthe RFC 3339 time until which it sends them nothing. A held host\nfails cache misses without contacting it and does not change Status.", + "type": "object", + "additionalProperties": { + "type": "string" + } } } }, diff --git a/docs/swagger/swagger.json b/docs/swagger/swagger.json index 65a271b..aad0cea 100644 --- a/docs/swagger/swagger.json +++ b/docs/swagger/swagger.json @@ -818,6 +818,13 @@ }, "status": { "type": "string" + }, + "upstream_holds": { + "description": "UpstreamHolds lists upstream hosts that rate limited the proxy and\nthe RFC 3339 time until which it sends them nothing. A held host\nfails cache misses without contacting it and does not change Status.", + "type": "object", + "additionalProperties": { + "type": "string" + } } } }, diff --git a/internal/handler/download_test.go b/internal/handler/download_test.go index 2797d42..d7a8e6b 100644 --- a/internal/handler/download_test.go +++ b/internal/handler/download_test.go @@ -6,11 +6,13 @@ import ( "io" "net/http" "net/http/httptest" + "net/url" "strings" "testing" "time" "github.com/git-pkgs/proxy/internal/database" + "github.com/git-pkgs/proxy/internal/httpclient" "github.com/git-pkgs/proxy/internal/metrics" "github.com/git-pkgs/proxy/internal/storage" "github.com/git-pkgs/purl" @@ -1258,3 +1260,49 @@ func TestRPMHandler_DownloadCacheMiss(t *testing.T) { t.Errorf("upstream URL = %q, want %q", fetcher.fetchedURL, want) } } + +func TestMavenHandler_UpstreamHeldReturns503(t *testing.T) { + held := &httpclient.HeldError{Host: "repo1.maven.org", Until: time.Now().Add(90 * time.Second), RetryAfter: 90 * time.Second} + + t.Run("artifact", func(t *testing.T) { + proxy, _, _, fetcher := setupTestProxy(t) + fetcher.fetchErr = fmt.Errorf("fetching artifact: %w", &url.Error{Op: "Get", URL: "https://repo1.maven.org/x", Err: held}) + + h := NewMavenHandler(proxy, "http://localhost", "", "") + srv := httptest.NewServer(h.Routes()) + defer srv.Close() + + resp, err := http.Get(srv.URL + "/com/example/lib/1.0/lib-1.0.jar") + if err != nil { + t.Fatalf("request failed: %v", err) + } + _ = resp.Body.Close() + if resp.StatusCode != http.StatusServiceUnavailable || resp.Header.Get("Retry-After") != "90" { + t.Errorf("status %d Retry-After %q, want 503 with Retry-After 90", resp.StatusCode, resp.Header.Get("Retry-After")) + } + }) + + t.Run("metadata", func(t *testing.T) { + proxy, _, _, _ := setupTestProxy(t) + proxy.HTTPClient = &http.Client{Transport: roundTripperFunc(func(*http.Request) (*http.Response, error) { + return nil, held + })} + + h := NewMavenHandler(proxy, "http://localhost", "", "") + srv := httptest.NewServer(h.Routes()) + defer srv.Close() + + resp, err := http.Get(srv.URL + "/com/example/lib/maven-metadata.xml") + if err != nil { + t.Fatalf("request failed: %v", err) + } + _ = resp.Body.Close() + if resp.StatusCode != http.StatusServiceUnavailable || resp.Header.Get("Retry-After") != "90" { + t.Errorf("status %d Retry-After %q, want 503 with Retry-After 90", resp.StatusCode, resp.Header.Get("Retry-After")) + } + }) +} + +type roundTripperFunc func(*http.Request) (*http.Response, error) + +func (fn roundTripperFunc) RoundTrip(req *http.Request) (*http.Response, error) { return fn(req) } diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 5e0688d..2d2e054 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -1121,6 +1121,9 @@ func (p *Proxy) ProxyUpstream(w http.ResponseWriter, r *http.Request, upstreamUR resp, err := p.HTTPClient.Do(req) if err != nil { + if writeUpstreamHeld(w, err) { + return + } p.Logger.Error("upstream request failed", "error", err) http.Error(w, "upstream request failed", http.StatusBadGateway) return @@ -1140,6 +1143,9 @@ func (p *Proxy) ProxyFile(w http.ResponseWriter, r *http.Request, upstreamURL st resp, err := p.HTTPClient.Do(req) if err != nil { + if writeUpstreamHeld(w, err) { + return + } http.Error(w, "failed to fetch from upstream", http.StatusBadGateway) return } @@ -1207,6 +1213,9 @@ func (p *Proxy) serveArtifactError(w http.ResponseWriter, err error, clientMsg s http.Error(w, "pushed package is temporarily unavailable", http.StatusServiceUnavailable) return } + if writeUpstreamHeld(w, err) { + return + } p.Logger.Error("failed to get artifact", "error", err) http.Error(w, clientMsg, http.StatusBadGateway) } @@ -1684,6 +1693,9 @@ func (p *Proxy) proxyCachedWithEncoding(w http.ResponseWriter, r *http.Request, http.Error(w, "not found", http.StatusNotFound) return } + if writeUpstreamHeld(w, err) { + return + } p.Logger.Error("metadata fetch failed", "error", err) http.Error(w, "failed to fetch from upstream", http.StatusBadGateway) return @@ -1770,6 +1782,9 @@ func (p *Proxy) proxyMetadataStream(w http.ResponseWriter, r *http.Request, upst resp, err := p.HTTPClient.Do(req) if err != nil { + if writeUpstreamHeld(w, err) { + return + } http.Error(w, "failed to fetch from upstream", http.StatusBadGateway) return } diff --git a/internal/handler/maven.go b/internal/handler/maven.go index 700c107..027a68d 100644 --- a/internal/handler/maven.go +++ b/internal/handler/maven.go @@ -115,6 +115,9 @@ func (h *MavenHandler) handleMetadata(w http.ResponseWriter, r *http.Request, ur http.Error(w, "not found", http.StatusNotFound) return } + if writeUpstreamHeld(w, err) { + return + } h.proxy.Logger.Error("metadata fetch failed", "error", err) http.Error(w, "failed to fetch from upstream", http.StatusBadGateway) return diff --git a/internal/handler/upstream_held.go b/internal/handler/upstream_held.go new file mode 100644 index 0000000..fa2ff96 --- /dev/null +++ b/internal/handler/upstream_held.go @@ -0,0 +1,27 @@ +package handler + +import ( + "errors" + "math" + "net/http" + "strconv" + + "github.com/git-pkgs/proxy/internal/httpclient" +) + +// writeUpstreamHeld answers a request the rate limit gate refused to send +// with 503 and the Retry-After the upstream asked for, and reports whether +// err was such a refusal. +func writeUpstreamHeld(w http.ResponseWriter, err error) bool { + var held *httpclient.HeldError + if !errors.As(err, &held) { + return false + } + secs := int(math.Ceil(held.RetryAfter.Seconds())) + if secs < 1 { + secs = 1 + } + w.Header().Set("Retry-After", strconv.Itoa(secs)) + http.Error(w, held.Error(), http.StatusServiceUnavailable) + return true +} diff --git a/internal/httpclient/ratelimit.go b/internal/httpclient/ratelimit.go new file mode 100644 index 0000000..b6f50e4 --- /dev/null +++ b/internal/httpclient/ratelimit.go @@ -0,0 +1,204 @@ +package httpclient + +import ( + "errors" + "fmt" + "log/slog" + "net/http" + "net/url" + "strconv" + "strings" + "sync" + "time" +) + +const ( + // holdWithoutHeader is the first hold for a 429 that carries no usable + // Retry-After; each further such 429 after a hold expires doubles it. + holdWithoutHeader = time.Minute + maxHoldWithoutHeader = 15 * time.Minute //nolint:mnd // doubling stops here + // maxHold caps what an upstream's Retry-After can ask for. + maxHold = time.Hour +) + +// ErrUpstreamHeld marks a request the gate did not send because the upstream +// is rate limiting this proxy. Match it with errors.Is; the *HeldError in +// the chain carries the host and how long to wait. +var ErrUpstreamHeld = errors.New("upstream is rate limiting this proxy") + +// HeldError is returned in place of a response while a host is held. +type HeldError struct { + Host string + Until time.Time + RetryAfter time.Duration +} + +func (e *HeldError) Error() string { + return fmt.Sprintf("%s is rate limiting this proxy, holding requests for %s", e.Host, e.RetryAfter.Round(time.Second)) +} + +func (e *HeldError) Is(target error) bool { return target == ErrUpstreamHeld } + +type hostHold struct { + until time.Time + strikes int +} + +// RateLimitGate stops sending to a host that answered 429, or 503 with a +// Retry-After, for as long as it asked, and fails those requests locally +// with a *HeldError instead. Clients and fetchers above it see an error +// rather than a slow upstream, so they fall back to cache or fail fast +// without adding to the traffic the upstream is already refusing. The first +// non-limited response from the host releases it. +type RateLimitGate struct { + base http.RoundTripper + Logger *slog.Logger + // OnHold is called with the host and hold length each time a hold starts. + OnHold func(host string, retryAfter time.Duration) + now func() time.Time + + mu sync.Mutex + holds map[string]*hostHold +} + +// NewRateLimitGate wraps base. A nil base uses http.DefaultTransport. +func NewRateLimitGate(base http.RoundTripper) *RateLimitGate { + if base == nil { + base = http.DefaultTransport + } + return &RateLimitGate{ + base: base, + now: time.Now, + holds: make(map[string]*hostHold), + } +} + +// RoundTrip implements http.RoundTripper. +func (g *RateLimitGate) RoundTrip(req *http.Request) (*http.Response, error) { + host := hostKey(req.URL) + if until, held := g.heldUntil(host); held { + now := g.now() + return nil, &HeldError{Host: host, Until: until, RetryAfter: until.Sub(now)} + } + + resp, err := g.base.RoundTrip(req) + if err != nil { + return nil, err + } + + retryAfter := resp.Header.Get("Retry-After") + switch { + case resp.StatusCode == http.StatusTooManyRequests, + resp.StatusCode == http.StatusServiceUnavailable && retryAfter != "": + g.hold(host, retryAfter) + default: + g.release(host) + } + return resp, nil +} + +// Holds returns the hosts currently held and when each hold ends. It is +// safe to call on a nil gate. +func (g *RateLimitGate) Holds() map[string]time.Time { + if g == nil { + return nil + } + now := g.now() + g.mu.Lock() + defer g.mu.Unlock() + held := make(map[string]time.Time) + for host, h := range g.holds { + if now.Before(h.until) { + held[host] = h.until + } + } + return held +} + +func (g *RateLimitGate) heldUntil(host string) (time.Time, bool) { + now := g.now() + g.mu.Lock() + defer g.mu.Unlock() + h, ok := g.holds[host] + if !ok || !now.Before(h.until) { + return time.Time{}, false + } + return h.until, true +} + +func (g *RateLimitGate) hold(host, header string) { + now := g.now() + + g.mu.Lock() + h, ok := g.holds[host] + if !ok { + h = &hostHold{} + g.holds[host] = h + } + // Responses already in flight when a hold starts all report the same + // rate limiting; only a 429 after the hold ran out is a further strike. + if !now.Before(h.until) { + h.strikes++ + } + d, fromHeader := parseRetryAfter(header, now) + if !fromHeader { + d = min(holdWithoutHeader<<(h.strikes-1), maxHoldWithoutHeader) + } + d = min(d, maxHold) + until := now.Add(d) + if until.After(h.until) { + h.until = until + } + g.mu.Unlock() + + if g.Logger != nil { + source := "default" + if fromHeader { + source = "retry-after" + } + g.Logger.Warn("upstream rate limited this proxy, holding its requests", + "host", host, "hold", d.Round(time.Second).String(), "source", source, + "until", until.UTC().Format(time.RFC3339)) + } + if g.OnHold != nil { + g.OnHold(host, d) + } +} + +func (g *RateLimitGate) release(host string) { + g.mu.Lock() + _, had := g.holds[host] + delete(g.holds, host) + g.mu.Unlock() + if had && g.Logger != nil { + g.Logger.Info("upstream rate limit cleared", "host", host) + } +} + +// parseRetryAfter reads a Retry-After value as delay seconds or an HTTP +// date. It reports false for anything unusable, including a zero or past +// value, so the caller applies its default instead. +func parseRetryAfter(value string, now time.Time) (time.Duration, bool) { + value = strings.TrimSpace(value) + if value == "" { + return 0, false + } + if secs, err := strconv.ParseInt(value, 10, 64); err == nil { + if secs <= 0 { + return 0, false + } + return time.Duration(secs) * time.Second, true + } + at, err := http.ParseTime(value) + if err != nil || !at.After(now) { + return 0, false + } + return at.Sub(now), true +} + +func hostKey(u *url.URL) string { + if u == nil { + return "" + } + return strings.ToLower(u.Host) +} diff --git a/internal/httpclient/ratelimit_test.go b/internal/httpclient/ratelimit_test.go new file mode 100644 index 0000000..2a5e3ec --- /dev/null +++ b/internal/httpclient/ratelimit_test.go @@ -0,0 +1,207 @@ +package httpclient + +import ( + "errors" + "io" + "net/http" + "strings" + "testing" + "time" +) + +type gateHarness struct { + gate *RateLimitGate + now time.Time + calls map[string]int + // respond returns the status and Retry-After the fake upstream answers with. + respond func(req *http.Request) (int, string) +} + +func newGateHarness() *gateHarness { + h := &gateHarness{ + now: time.Date(2026, 10, 14, 12, 0, 0, 0, time.UTC), + calls: map[string]int{}, + respond: func(*http.Request) (int, string) { + return http.StatusOK, "" + }, + } + h.gate = NewRateLimitGate(roundTripperFunc(func(req *http.Request) (*http.Response, error) { + h.calls[req.URL.Host]++ + status, retryAfter := h.respond(req) + resp := &http.Response{ + StatusCode: status, + Header: http.Header{}, + Body: io.NopCloser(strings.NewReader("")), + Request: req, + } + if retryAfter != "" { + resp.Header.Set("Retry-After", retryAfter) + } + return resp, nil + })) + h.gate.now = func() time.Time { return h.now } + return h +} + +func (h *gateHarness) get(t *testing.T, rawURL string) (*http.Response, error) { + t.Helper() + req, err := http.NewRequest(http.MethodGet, rawURL, nil) + if err != nil { + t.Fatalf("building request: %v", err) + } + resp, err := h.gate.RoundTrip(req) + if resp != nil { + _ = resp.Body.Close() + } + return resp, err +} + +func TestRateLimitGate_HoldsHostForRetryAfter(t *testing.T) { + h := newGateHarness() + h.respond = func(*http.Request) (int, string) { return http.StatusTooManyRequests, "785" } + + resp, err := h.get(t, "https://repo1.maven.org/maven2/a/b/maven-metadata.xml") + if err != nil || resp.StatusCode != http.StatusTooManyRequests { + t.Fatalf("first request: resp=%v err=%v, want the upstream 429 passed through", resp, err) + } + + h.respond = func(*http.Request) (int, string) { return http.StatusOK, "" } + _, err = h.get(t, "https://repo1.maven.org/maven2/a/b/b-1.0.jar") + var held *HeldError + if !errors.As(err, &held) || !errors.Is(err, ErrUpstreamHeld) { + t.Fatalf("second request error = %v, want *HeldError", err) + } + if held.Host != "repo1.maven.org" || held.RetryAfter != 785*time.Second { + t.Errorf("held = %+v, want repo1.maven.org for 785s", held) + } + if h.calls["repo1.maven.org"] != 1 { + t.Errorf("upstream calls = %d, want 1: held requests must not reach it", h.calls["repo1.maven.org"]) + } + + if _, err := h.get(t, "https://plugins.gradle.org/m2/x.pom"); err != nil { + t.Errorf("other host: %v, want unaffected", err) + } + + holds := h.gate.Holds() + if until, ok := holds["repo1.maven.org"]; !ok || !until.Equal(h.now.Add(785*time.Second)) { + t.Errorf("Holds() = %v, want repo1.maven.org until now+785s", holds) + } +} + +func TestRateLimitGate_ReleasesWhenHoldExpiresAndUpstreamServes(t *testing.T) { + h := newGateHarness() + h.respond = func(*http.Request) (int, string) { return http.StatusTooManyRequests, "60" } + _, _ = h.get(t, "https://repo1.maven.org/x") + + h.now = h.now.Add(59 * time.Second) + if _, err := h.get(t, "https://repo1.maven.org/x"); !errors.Is(err, ErrUpstreamHeld) { + t.Fatalf("at 59s: err = %v, want still held", err) + } + + h.now = h.now.Add(time.Second) + h.respond = func(*http.Request) (int, string) { return http.StatusOK, "" } + resp, err := h.get(t, "https://repo1.maven.org/x") + if err != nil || resp.StatusCode != http.StatusOK { + t.Fatalf("at 60s: resp=%v err=%v, want the request sent", resp, err) + } + if len(h.gate.Holds()) != 0 { + t.Errorf("Holds() = %v after a success, want empty", h.gate.Holds()) + } +} + +func TestRateLimitGate_DefaultHoldDoublesWithoutHeader(t *testing.T) { + h := newGateHarness() + h.respond = func(*http.Request) (int, string) { return http.StatusTooManyRequests, "" } + + wantHolds := []time.Duration{time.Minute, 2 * time.Minute, 4 * time.Minute} + for i, want := range wantHolds { + _, _ = h.get(t, "https://repo1.maven.org/x") + until := h.gate.Holds()["repo1.maven.org"] + if got := until.Sub(h.now); got != want { + t.Errorf("strike %d: hold = %s, want %s", i+1, got, want) + } + h.now = until + } +} + +func TestRateLimitGate_ConcurrentLimitedResponsesAreOneStrike(t *testing.T) { + h := newGateHarness() + h.respond = func(*http.Request) (int, string) { return http.StatusTooManyRequests, "" } + + // Two requests already in flight when the limit starts both see 429. + // The gate only consults the hold before sending, so drive the base + // directly for the second. + _, _ = h.get(t, "https://repo1.maven.org/x") + req, _ := http.NewRequest(http.MethodGet, "https://repo1.maven.org/y", nil) + h.gate.hold(hostKey(req.URL), "") + + if got := h.gate.Holds()["repo1.maven.org"].Sub(h.now); got != time.Minute { + t.Errorf("hold = %s after two concurrent 429s, want the single-strike %s", got, time.Minute) + } +} + +func TestRateLimitGate_CapsUpstreamRetryAfter(t *testing.T) { + h := newGateHarness() + h.respond = func(*http.Request) (int, string) { return http.StatusTooManyRequests, "86400" } + _, _ = h.get(t, "https://repo1.maven.org/x") + if got := h.gate.Holds()["repo1.maven.org"].Sub(h.now); got != maxHold { + t.Errorf("hold = %s, want capped at %s", got, maxHold) + } +} + +func TestRateLimitGate_ServiceUnavailableNeedsRetryAfter(t *testing.T) { + h := newGateHarness() + h.respond = func(*http.Request) (int, string) { return http.StatusServiceUnavailable, "" } + _, _ = h.get(t, "https://repo1.maven.org/x") + if len(h.gate.Holds()) != 0 { + t.Errorf("503 without Retry-After created hold %v, want none", h.gate.Holds()) + } + + h.respond = func(*http.Request) (int, string) { return http.StatusServiceUnavailable, "30" } + _, _ = h.get(t, "https://repo1.maven.org/x") + if got := h.gate.Holds()["repo1.maven.org"].Sub(h.now); got != 30*time.Second { + t.Errorf("hold = %s, want 30s from the 503's Retry-After", got) + } +} + +func TestRateLimitGate_ReportsHoldsAndNilSafe(t *testing.T) { + var nilGate *RateLimitGate + if holds := nilGate.Holds(); holds != nil { + t.Errorf("nil gate Holds() = %v, want nil", holds) + } + + h := newGateHarness() + var reported []string + h.gate.OnHold = func(host string, retryAfter time.Duration) { + reported = append(reported, host+"="+retryAfter.String()) + } + h.respond = func(*http.Request) (int, string) { return http.StatusTooManyRequests, "10" } + _, _ = h.get(t, "https://repo1.maven.org/x") + if len(reported) != 1 || reported[0] != "repo1.maven.org=10s" { + t.Errorf("OnHold reported %v, want [repo1.maven.org=10s]", reported) + } +} + +func TestParseRetryAfter(t *testing.T) { + now := time.Date(2026, 10, 14, 12, 0, 0, 0, time.UTC) + tests := []struct { + value string + want time.Duration + ok bool + }{ + {"785", 785 * time.Second, true}, + {" 30 ", 30 * time.Second, true}, + {"0", 0, false}, + {"-5", 0, false}, + {"", 0, false}, + {"soon", 0, false}, + {now.Add(90 * time.Second).Format(http.TimeFormat), 90 * time.Second, true}, + {now.Add(-time.Minute).Format(http.TimeFormat), 0, false}, + } + for _, tt := range tests { + got, ok := parseRetryAfter(tt.value, now) + if got != tt.want || ok != tt.ok { + t.Errorf("parseRetryAfter(%q) = (%s, %v), want (%s, %v)", tt.value, got, ok, tt.want, tt.ok) + } + } +} diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go index 7567ff8..36320b5 100644 --- a/internal/metrics/metrics.go +++ b/internal/metrics/metrics.go @@ -105,6 +105,14 @@ var ( []string{"registry"}, ) + UpstreamRateLimitHolds = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "proxy_upstream_rate_limit_holds_total", + Help: "Holds placed on an upstream host after it rate limited the proxy", + }, + []string{"host"}, + ) + // Storage metrics StorageOperationDuration = prometheus.NewHistogramVec( prometheus.HistogramOpts{ @@ -263,6 +271,7 @@ func init() { UpstreamErrors, CircuitBreakerState, CircuitBreakerTrips, + UpstreamRateLimitHolds, StorageOperationDuration, StorageErrors, ActiveRequests, @@ -482,6 +491,11 @@ func RecordCircuitBreakerTrip(registry string) { CircuitBreakerTrips.WithLabelValues(registry).Inc() } +// RecordUpstreamRateLimitHold counts a hold placed on host. +func RecordUpstreamRateLimitHold(host string, _ time.Duration) { + UpstreamRateLimitHolds.WithLabelValues(host).Inc() +} + // IncrementActiveRequests increments the active request counter. func IncrementActiveRequests() { ActiveRequests.Inc() diff --git a/internal/metrics/metrics_test.go b/internal/metrics/metrics_test.go index cca39c8..8259d3e 100644 --- a/internal/metrics/metrics_test.go +++ b/internal/metrics/metrics_test.go @@ -88,6 +88,7 @@ func TestMetricsAreRegistered(t *testing.T) { CachedArtifacts, CircuitBreakerState, CircuitBreakerTrips, + UpstreamRateLimitHolds, ActiveRequests, } diff --git a/internal/server/breakers_test.go b/internal/server/breakers_test.go index ae39759..bb5166e 100644 --- a/internal/server/breakers_test.go +++ b/internal/server/breakers_test.go @@ -12,6 +12,7 @@ import ( "testing" "time" + upstreamhttp "github.com/git-pkgs/proxy/internal/httpclient" "github.com/git-pkgs/proxy/internal/metrics" "github.com/git-pkgs/registries/fetch" "github.com/prometheus/client_golang/prometheus" @@ -346,3 +347,44 @@ func TestBreakerMonitor_SnapshotSerializesStateReads(t *testing.T) { t.Errorf("trips = %v, want 1", trips) } } + +func TestHealthEndpoint_ReportsUpstreamHolds(t *testing.T) { + ts := newTestServer(t) + defer ts.close() + + gate := upstreamhttp.NewRateLimitGate(roundTripperFunc(func(req *http.Request) (*http.Response, error) { + return &http.Response{ + StatusCode: http.StatusTooManyRequests, + Header: http.Header{"Retry-After": []string{"600"}}, + Body: http.NoBody, + Request: req, + }, nil + })) + req, _ := http.NewRequest(http.MethodGet, "https://repo1.maven.org/maven2/x", nil) + if resp, err := gate.RoundTrip(req); err == nil { + _ = resp.Body.Close() + } + ts.server.rateLimits = gate + + w := httptest.NewRecorder() + ts.handler.ServeHTTP(w, httptest.NewRequest("GET", "/health", nil)) + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want 200; body: %s", w.Code, w.Body.String()) + } + + var resp HealthResponse + if err := json.NewDecoder(w.Body).Decode(&resp); err != nil { + t.Fatalf("decoding response: %v", err) + } + until, err := time.Parse(time.RFC3339, resp.UpstreamHolds["repo1.maven.org"]) + if err != nil { + t.Fatalf("upstream_holds = %v, want repo1.maven.org with an RFC 3339 deadline", resp.UpstreamHolds) + } + if remaining := time.Until(until); remaining < 9*time.Minute || remaining > 10*time.Minute { + t.Errorf("hold ends in %s, want about 10m", remaining) + } +} + +type roundTripperFunc func(*http.Request) (*http.Response, error) + +func (fn roundTripperFunc) RoundTrip(req *http.Request) (*http.Response, error) { return fn(req) } diff --git a/internal/server/health.go b/internal/server/health.go index fd49cf1..cc1236d 100644 --- a/internal/server/health.go +++ b/internal/server/health.go @@ -38,6 +38,22 @@ type HealthResponse struct { // the upstream, but says nothing about this proxy's own health, so it does // not change Status. CircuitBreakers map[string]string `json:"circuit_breakers,omitempty"` + // UpstreamHolds lists upstream hosts that rate limited the proxy and + // the RFC 3339 time until which it sends them nothing. A held host + // fails cache misses without contacting it and does not change Status. + UpstreamHolds map[string]string `json:"upstream_holds,omitempty"` +} + +// formatHolds renders hold deadlines for the health report; nil when none. +func formatHolds(holds map[string]time.Time) map[string]string { + if len(holds) == 0 { + return nil + } + out := make(map[string]string, len(holds)) + for host, until := range holds { + out[host] = until.UTC().Format(time.RFC3339) + } + return out } // HealthCheck reports the status of a single subsystem check. diff --git a/internal/server/server.go b/internal/server/server.go index 5b35435..42dfad6 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -117,6 +117,7 @@ type Server struct { accessLog *accesslog.Logger ecr *ecrTokens breakers *breakerMonitor + rateLimits *upstreamhttp.RateLimitGate ecoStats ecosystemStatsCache sources sourceTracker @@ -493,7 +494,13 @@ func (s *Server) newUpstreamClients(artifactTimeout time.Duration) (metadataClie if s.accessLog != nil { baseTransport = upstreamhttp.NewAccessLogTransport(baseTransport, s.accessLog, s.logger) } - authTransport := upstreamhttp.NewTransport(baseTransport, upstreamhttp.AuthFunc(s.authForURL)) + // The gate sits under the auth transport so it sees every final upstream + // answer, including the retry after a bearer challenge. + gate := upstreamhttp.NewRateLimitGate(baseTransport) + gate.Logger = s.logger + gate.OnHold = metrics.RecordUpstreamRateLimitHold + s.rateLimits = gate + authTransport := upstreamhttp.NewTransport(gate, upstreamhttp.AuthFunc(s.authForURL)) metadata := *safeClient metadata.Timeout = s.cfg.ParseHTTPTimeout() metadata.Transport = authTransport @@ -1156,6 +1163,7 @@ func (s *Server) handleHealth(w http.ResponseWriter, r *http.Request) { Status: "ok", Checks: map[string]HealthCheck{}, CircuitBreakers: s.breakers.snapshot(), + UpstreamHolds: formatHolds(s.rateLimits.Holds()), } // Database check (short-circuit; do not waste a storage probe call when DB is down). From e74e5abc79523de0b1660fb434523fa690b91992 Mon Sep 17 00:00:00 2001 From: simon chrzanowski Date: Thu, 15 Oct 2026 12:46:55 +0200 Subject: [PATCH 299/633] Serve cached npm and Composer metadata gzip-compressed - Keep a gzip copy of each rewrite cache entry when it is smaller, made once per rewrite and counted against metadata_rewrite_cache_size - Serve it with Content-Encoding: gzip to clients that accept gzip, with an ETag of its own ("-gzip") and Vary: Accept-Encoding - Rewrites made outside the cache (cache disabled, cooldown on) stay uncompressed - rewriteCache.get and rewrite return rewrittenMetadata --- config.example.yaml | 4 +- docs/configuration.md | 2 + internal/handler/metadata_gzip_test.go | 227 +++++++++++++++++++ internal/handler/metadata_validators_test.go | 16 +- internal/handler/npm_rewrite_test.go | 2 +- internal/handler/rewrite_cache.go | 174 ++++++++++---- internal/handler/rewrite_cache_test.go | 40 ++-- 7 files changed, 384 insertions(+), 81 deletions(-) create mode 100644 internal/handler/metadata_gzip_test.go diff --git a/config.example.yaml b/config.example.yaml index c41f30f..accae89 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -19,8 +19,8 @@ base_url: "http://localhost:8080" # artifact_max_size: "" # Memory for rewritten npm and Composer metadata, so each upstream document is -# rewritten once rather than on every request. Set to "0" to rewrite on every -# request. Default: "256MB". +# rewritten and gzip-compressed once rather than on every request. Set to "0" +# to rewrite on every request. Default: "256MB". # metadata_rewrite_cache_size: "256MB" # Public URL where the web UI is reached. Defaults to base_url when unset. diff --git a/docs/configuration.md b/docs/configuration.md index c735c34..ffd57fe 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -966,6 +966,8 @@ Or via environment variable: `PROXY_ARTIFACT_MAX_SIZE=2GB`. The npm and Composer handlers rewrite every metadata document they serve, so that download URLs point at the proxy. That means decoding the whole document and encoding it again, and Composer's minified documents are also expanded, which together cost milliseconds per typical package and far more for very large ones. The proxy keeps rewritten documents in memory, so each distinct upstream document is rewritten once and requests arriving while it is being rewritten wait for that rewrite. New bytes from upstream are rewritten again. With version cooldown enabled the cache is bypassed, because cooldown filtering depends on the current time. +Each cached document is also kept gzip-compressed, when that makes it smaller, and served with `Content-Encoding: gzip` to clients that accept it. A large packument is compressed once rather than on every request, by the proxy or by a reverse proxy in front of it. The compressed copy counts against the cache size and typically adds about an eighth to each entry. Documents rewritten outside the cache are served uncompressed. + ```yaml metadata_rewrite_cache_size: "256MB" # default; "0" rewrites on every request ``` diff --git a/internal/handler/metadata_gzip_test.go b/internal/handler/metadata_gzip_test.go new file mode 100644 index 0000000..5826441 --- /dev/null +++ b/internal/handler/metadata_gzip_test.go @@ -0,0 +1,227 @@ +package handler + +import ( + "bytes" + "compress/gzip" + "fmt" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/git-pkgs/cooldown" +) + +// compressibleDocuments sets upstream documents with enough versions that +// gzip makes them smaller. +func compressibleDocuments(u *validatorUpstream) { + var npm, composer []string + for i := range 50 { + v := fmt.Sprintf("1.0.%d", i) + npm = append(npm, `"`+v+`":{"dist":{"tarball":"https://registry.npmjs.org/left-pad/-/left-pad-`+v+`.tgz"}}`) + composer = append(composer, `{"version":"`+v+`","dist":{"type":"zip","url":"https://example.com/pkg-`+v+`.zip"}}`) + } + u.set( + `{"name":"left-pad","versions":{`+strings.Join(npm, ",")+`}}`, + `{"packages":{"vendor/pkg":[`+strings.Join(composer, ",")+`]}}`, + ) +} + +func gunzip(t *testing.T, rec *httptest.ResponseRecorder) string { + t.Helper() + zr, err := gzip.NewReader(bytes.NewReader(rec.Body.Bytes())) + if err != nil { + t.Fatalf("body is not gzip: %v", err) + } + body, err := io.ReadAll(zr) + if err != nil { + t.Fatalf("reading gzip body: %v", err) + } + return string(body) +} + +var acceptGzip = http.Header{"Accept-Encoding": {"gzip, deflate, br"}} + +// TestRewrittenMetadataGzip checks that a client accepting gzip gets the +// cached compressed copy of rewritten metadata, on the first fetch and on +// the rewrite fast path, with an ETag of its own so it never matches the +// uncompressed representation. +func TestRewrittenMetadataGzip(t *testing.T) { + for _, c := range validatorCases { + for _, ttl := range []time.Duration{time.Hour, 0} { + t.Run(c.ecosystem+"/ttl="+ttl.String(), func(t *testing.T) { + u := newValidatorUpstream(t) + compressibleDocuments(u) + handler := c.handler(validatorProxy(t, u, ttl), u) + + identity := serve(handler, c.path, nil) + for range 2 { + expectGzipOf(t, serve(handler, c.path, acceptGzip), identity) + } + }) + } + } +} + +// expectGzipOf checks that rec is the gzip representation of the +// uncompressed response identity. +func expectGzipOf(t *testing.T, rec, identity *httptest.ResponseRecorder) { + t.Helper() + if rec.Code != http.StatusOK || rec.Header().Get("Content-Encoding") != "gzip" { + t.Fatalf("status %d, Content-Encoding %q; want 200 with gzip", rec.Code, rec.Header().Get("Content-Encoding")) + } + if rec.Body.Len() >= identity.Body.Len() { + t.Errorf("gzip body is %d bytes, uncompressed %d", rec.Body.Len(), identity.Body.Len()) + } + if got := gunzip(t, rec); got != identity.Body.String() { + t.Errorf("decompressed body differs from the uncompressed response") + } + if got := rec.Header().Get("Vary"); got != "Accept-Encoding" { + t.Errorf("Vary = %q, want Accept-Encoding", got) + } + if etag := rec.Header().Get("ETag"); etag == "" || etag == identity.Header().Get("ETag") { + t.Errorf("gzip ETag %q must differ from the uncompressed ETag %q", etag, identity.Header().Get("ETag")) + } + if rec.Header().Get("Last-Modified") != identity.Header().Get("Last-Modified") { + t.Errorf("Last-Modified %q, want %q", rec.Header().Get("Last-Modified"), identity.Header().Get("Last-Modified")) + } +} + +// TestRewrittenMetadataGzipRevalidation checks that each representation +// revalidates with its own ETag, also when a reverse proxy weakened it. +func TestRewrittenMetadataGzipRevalidation(t *testing.T) { + for _, c := range validatorCases { + t.Run(c.ecosystem, func(t *testing.T) { + u := newValidatorUpstream(t) + compressibleDocuments(u) + handler := c.handler(validatorProxy(t, u, time.Hour), u) + gzipETag := serve(handler, c.path, acceptGzip).Header().Get("ETag") + identityETag := serve(handler, c.path, nil).Header().Get("ETag") + + for _, ifNoneMatch := range []string{gzipETag, "W/" + gzipETag} { + rec := serve(handler, c.path, http.Header{"Accept-Encoding": {"gzip"}, "If-None-Match": {ifNoneMatch}}) + expectNotModified(t, rec, gzipETag) + if got := rec.Header().Get("Vary"); got != "Accept-Encoding" { + t.Errorf("304 Vary = %q, want Accept-Encoding", got) + } + } + if rec := serve(handler, c.path, http.Header{"Accept-Encoding": {"gzip"}, "If-None-Match": {identityETag}}); rec.Code != http.StatusOK { + t.Errorf("gzip request revalidating the uncompressed ETag: status %d, want 200", rec.Code) + } + if rec := serve(handler, c.path, http.Header{"If-None-Match": {gzipETag}}); rec.Code != http.StatusOK { + t.Errorf("uncompressed request revalidating the gzip ETag: status %d, want 200", rec.Code) + } + }) + } +} + +// TestRewrittenMetadataWithoutGzip checks that clients that do not accept +// gzip get the uncompressed document, which still varies on Accept-Encoding. +func TestRewrittenMetadataWithoutGzip(t *testing.T) { + for _, c := range validatorCases { + t.Run(c.ecosystem, func(t *testing.T) { + u := newValidatorUpstream(t) + compressibleDocuments(u) + handler := c.handler(validatorProxy(t, u, time.Hour), u) + + for _, acceptEncoding := range []string{"", "identity", "gzip;q=0", "br"} { + header := http.Header{} + if acceptEncoding != "" { + header.Set("Accept-Encoding", acceptEncoding) + } + rec := serve(handler, c.path, header) + if rec.Code != http.StatusOK || rec.Header().Get("Content-Encoding") != "" { + t.Errorf("Accept-Encoding %q: status %d, Content-Encoding %q; want 200 uncompressed", acceptEncoding, rec.Code, rec.Header().Get("Content-Encoding")) + } + if got := rec.Header().Get("Vary"); got != "Accept-Encoding" { + t.Errorf("Accept-Encoding %q: Vary = %q, want Accept-Encoding", acceptEncoding, got) + } + } + }) + } +} + +// TestRewrittenMetadataGzipOnlyFromRewriteCache checks that rewrites made +// outside the cache, with it disabled or with cooldown on, stay uncompressed: +// compressing them on every request is better left to a reverse proxy. +func TestRewrittenMetadataGzipOnlyFromRewriteCache(t *testing.T) { + setups := map[string]func(*Proxy){ + "cache disabled": func(p *Proxy) { p.SetMetadataRewriteCacheSize(0) }, + "cooldown": func(p *Proxy) { p.Cooldown = &cooldown.Config{Default: "3d"} }, + } + for _, c := range validatorCases { + for name, setup := range setups { + t.Run(c.ecosystem+"/"+name, func(t *testing.T) { + u := newValidatorUpstream(t) + compressibleDocuments(u) + proxy := validatorProxy(t, u, time.Hour) + setup(proxy) + handler := c.handler(proxy, u) + + for range 2 { + if rec := serve(handler, c.path, acceptGzip); rec.Code != http.StatusOK || rec.Header().Get("Content-Encoding") != "" { + t.Errorf("status %d, Content-Encoding %q; want 200 uncompressed", rec.Code, rec.Header().Get("Content-Encoding")) + } + } + }) + } + } +} + +// TestRewriteCache_KeepsGzipCopyOnlyWhenSmaller checks that the cache keeps a +// compressed copy of a rewrite, counted against its size, unless compressing +// does not make the rewrite smaller. +func TestRewriteCache_KeepsGzipCopyOnlyWhenSmaller(t *testing.T) { + c := newRewriteCache(1 << 20) + identity := func(b []byte) ([]byte, error) { return b, nil } + large := []byte(strings.Repeat(`{"version":"1.0.0"},`, 100)) + + for _, doc := range [][]byte{large, []byte("abc")} { + key := rewriteCacheKey("npm", "http://proxy", string(doc[:3]), doc) + if _, err := c.rewrite(t.Context(), key, doc, identity); err != nil { + t.Fatal(err) + } + } + + m, ok := c.get(rewriteCacheKey("npm", "http://proxy", string(large[:3]), large)) + if !ok || m.gzipped == nil { + t.Fatalf("large rewrite: cached %v, gzip copy %d bytes; want a gzip copy", ok, len(m.gzipped)) + } + if c.size != int64(len(large)+len(m.gzipped)+3) { + t.Errorf("cache size = %d, want both rewrites and the gzip copy (%d)", c.size, len(large)+len(m.gzipped)+3) + } + if small, ok := c.get(rewriteCacheKey("npm", "http://proxy", "abc", []byte("abc"))); !ok || small.gzipped != nil { + t.Errorf("small rewrite: cached %v, gzip copy %d bytes; want none", ok, len(small.gzipped)) + } +} + +func TestAcceptsGzip(t *testing.T) { + cases := map[string]bool{ + "": false, + "identity": false, + "br, deflate": false, + "gzip": true, + "GZIP": true, + "x-gzip": true, + "deflate, gzip;q=1.0": true, + "gzip; q=0.5": true, + "gzip;q=0": false, + "gzip;q=0.000": false, + "*": true, + "*;q=0": false, + "br;q=1, *;q=0.5": true, + "gzip;q=0, *": false, + "gzip;q=invalid, deflate": false, + } + for header, want := range cases { + r := httptest.NewRequest(http.MethodGet, "/", nil) + if header != "" { + r.Header.Set("Accept-Encoding", header) + } + if got := acceptsGzip(r); got != want { + t.Errorf("acceptsGzip(%q) = %v, want %v", header, got, want) + } + } +} diff --git a/internal/handler/metadata_validators_test.go b/internal/handler/metadata_validators_test.go index a057e5e..e34395d 100644 --- a/internal/handler/metadata_validators_test.go +++ b/internal/handler/metadata_validators_test.go @@ -481,17 +481,17 @@ func TestRewriteCacheETag(t *testing.T) { exclaim := func(b []byte) ([]byte, error) { return []byte(string(b) + "!"), nil } key := rewriteCacheKey("npm", "http://proxy", "demo", []byte("doc")) - out, etag, err := c.rewrite(t.Context(), key, []byte("doc"), exclaim) - if err != nil || string(out) != "doc!" { - t.Fatalf("rewrite = %q, %v", out, err) + m, err := c.rewrite(t.Context(), key, []byte("doc"), exclaim) + if err != nil || string(m.body) != "doc!" { + t.Fatalf("rewrite = %q, %v", m.body, err) } - if want := metadataETag([]byte("doc!")); etag.etag != want { - t.Errorf("rewrite ETag = %q, want %q", etag.etag, want) + if want := metadataETag([]byte("doc!")); m.etag != want { + t.Errorf("rewrite ETag = %q, want %q", m.etag, want) } - if _, cached, ok := c.get(key); !ok || cached != etag { - t.Errorf("get validators = %+v, %v; want %+v", cached, ok, etag) + if cached, ok := c.get(key); !ok || cached.metadataValidators != m.metadataValidators { + t.Errorf("get validators = %+v, %v; want %+v", cached.metadataValidators, ok, m.metadataValidators) } - if metadataETag([]byte("other")) == etag.etag { + if metadataETag([]byte("other")) == m.etag { t.Error("different output has the same ETag") } } diff --git a/internal/handler/npm_rewrite_test.go b/internal/handler/npm_rewrite_test.go index d9a787e..6503ae1 100644 --- a/internal/handler/npm_rewrite_test.go +++ b/internal/handler/npm_rewrite_test.go @@ -247,7 +247,7 @@ func TestNPMStoredRewriteSkippedWhenStale(t *testing.T) { body := []byte(`{}`) proxy.cacheMetadataBlob(t.Context(), "npm", "demo", metadataStoragePath("npm", "demo"), &upstreamMetadata{body: body}) key := rewriteCacheKey("npm", "http://proxy", "demo", body) - if _, _, err := proxy.rewrites.rewrite(t.Context(), key, body, func(b []byte) ([]byte, error) { return b, nil }); err != nil { + if _, err := proxy.rewrites.rewrite(t.Context(), key, body, func(b []byte) ([]byte, error) { return b, nil }); err != nil { t.Fatal(err) } if _, ok := proxy.storedRewrite("npm", "demo", "http://proxy", "demo"); !ok { diff --git a/internal/handler/rewrite_cache.go b/internal/handler/rewrite_cache.go index 42273c3..fb3114a 100644 --- a/internal/handler/rewrite_cache.go +++ b/internal/handler/rewrite_cache.go @@ -1,6 +1,8 @@ package handler import ( + "bytes" + "compress/gzip" "container/list" "context" "crypto/rand" @@ -11,6 +13,7 @@ import ( "io" "net/http" "slices" + "strconv" "strings" "sync" "time" @@ -36,6 +39,10 @@ import ( // copy without the output being hashed again on every request. Dates are not // the cache's to give: they must agree across replicas, so the database holds // them (see metadataDate). +// +// Each entry also keeps its output gzip-compressed, when that is smaller, so +// clients accepting gzip get it without the document being compressed again +// on every request, by the proxy or by a reverse proxy in front of it. type rewriteCache struct { maxBytes int64 @@ -47,18 +54,21 @@ type rewriteCache struct { } type rewriteEntry struct { - key string - out []byte - etag string + key string + rewrittenMetadata } -// inflightRewrite is one rewrite that concurrent callers share. out, etag -// and err are written before done closes and read only after, so the close is -// the handoff. +// size is how much of the cache's room the entry takes. +func (e *rewriteEntry) size() int64 { + return int64(len(e.body) + len(e.gzipped)) +} + +// inflightRewrite is one rewrite that concurrent callers share. m and err are +// written before done closes and read only after, so the close is the +// handoff. type inflightRewrite struct { done chan struct{} - out []byte - etag string + m rewrittenMetadata err error } @@ -70,9 +80,12 @@ type metadataValidators struct { } // rewrittenMetadata is a rewritten metadata document with its validators. +// gzipped is the document gzip-compressed, or nil when there is no +// compressed copy to serve. type rewrittenMetadata struct { body []byte metadataValidators + gzipped []byte } // metadataETag returns the strong ETag of a rewritten document. @@ -139,28 +152,27 @@ func rewriteCacheKeyForDigest(ecosystem, proxyURL, name, digest string) string { return strings.Join([]string{ecosystem, proxyURL, name, digest}, "\x00") } -// get returns the cached rewrite for key and its ETag, if there is one. -func (c *rewriteCache) get(key string) ([]byte, metadataValidators, bool) { +// get returns the cached rewrite for key, if there is one. +func (c *rewriteCache) get(key string) (rewrittenMetadata, bool) { if c == nil { - return nil, metadataValidators{}, false + return rewrittenMetadata{}, false } c.mu.Lock() defer c.mu.Unlock() el, ok := c.entries[key] if !ok { - return nil, metadataValidators{}, false + return rewrittenMetadata{}, false } c.order.MoveToFront(el) - e := el.Value.(*rewriteEntry) - return e.out, metadataValidators{etag: e.etag}, true + return el.Value.(*rewriteEntry).rewrittenMetadata, true } -// rewrite returns rewrite(in) and its ETag, from the cache when it can. -// Callers must treat the returned bytes as read-only: a cached result is -// shared. A caller waiting on another's rewrite leaves when ctx ends; the -// rewrite itself always runs to completion, so the result is cached for the -// next request. -func (c *rewriteCache) rewrite(ctx context.Context, key string, in []byte, rewrite func([]byte) ([]byte, error)) (out []byte, v metadataValidators, err error) { +// rewrite returns rewrite(in) with its ETag and gzip copy, from the cache +// when it can. Callers must treat the returned bytes as read-only: a cached +// result is shared. A caller waiting on another's rewrite leaves when ctx +// ends; the rewrite itself always runs to completion, so the result is cached +// for the next request. +func (c *rewriteCache) rewrite(ctx context.Context, key string, in []byte, rewrite func([]byte) ([]byte, error)) (rewrittenMetadata, error) { if c == nil { return rewriteWithETag(in, rewrite) } @@ -168,17 +180,16 @@ func (c *rewriteCache) rewrite(ctx context.Context, key string, in []byte, rewri c.mu.Lock() if el, ok := c.entries[key]; ok { c.order.MoveToFront(el) - e := el.Value.(*rewriteEntry) c.mu.Unlock() - return e.out, metadataValidators{etag: e.etag}, nil + return el.Value.(*rewriteEntry).rewrittenMetadata, nil } if f, ok := c.inFlight[key]; ok { c.mu.Unlock() select { case <-ctx.Done(): - return nil, metadataValidators{}, ctx.Err() + return rewrittenMetadata{}, ctx.Err() case <-f.done: - return f.out, metadataValidators{etag: f.etag}, f.err + return f.m, f.err } } f := &inflightRewrite{done: make(chan struct{}), err: errSharedRewriteAbandoned} @@ -189,31 +200,46 @@ func (c *rewriteCache) rewrite(ctx context.Context, key string, in []byte, rewri c.mu.Lock() delete(c.inFlight, key) if f.err == nil { - c.add(&rewriteEntry{key: key, out: f.out, etag: f.etag}) + c.add(&rewriteEntry{key: key, rewrittenMetadata: f.m}) } c.mu.Unlock() close(f.done) }() - var fv metadataValidators - f.out, fv, f.err = rewriteWithETag(in, rewrite) - f.etag = fv.etag - return f.out, fv, f.err + f.m, f.err = rewriteWithETag(in, rewrite) + if f.err == nil { + f.m.gzipped = gzipIfSmaller(f.m.body) + } + return f.m, f.err } // rewriteWithETag runs rewrite and computes the ETag of what it returns. -func rewriteWithETag(in []byte, rewrite func([]byte) ([]byte, error)) ([]byte, metadataValidators, error) { +func rewriteWithETag(in []byte, rewrite func([]byte) ([]byte, error)) (rewrittenMetadata, error) { out, err := rewrite(in) if err != nil { - return nil, metadataValidators{}, err + return rewrittenMetadata{}, err } - return out, metadataValidators{etag: metadataETag(out)}, nil + return rewrittenMetadata{body: out, metadataValidators: metadataValidators{etag: metadataETag(out)}}, nil +} + +// gzipIfSmaller returns body gzip-compressed, or nil when compressing does +// not make it smaller. +func gzipIfSmaller(body []byte) []byte { + var buf bytes.Buffer + zw := gzip.NewWriter(&buf) + if _, err := zw.Write(body); err != nil { + return nil + } + if err := zw.Close(); err != nil || buf.Len() >= len(body) { + return nil + } + return buf.Bytes() } // add stores e and evicts least recently used entries until the cache is -// back under its limit. Output larger than the whole cache is not stored. +// back under its limit. An entry larger than the whole cache is not stored. // Called with mu held. func (c *rewriteCache) add(e *rewriteEntry) { - n := int64(len(e.out)) + n := e.size() if n > c.maxBytes { return } @@ -224,7 +250,7 @@ func (c *rewriteCache) add(e *rewriteEntry) { e := oldest.Value.(*rewriteEntry) c.order.Remove(oldest) delete(c.entries, e.key) - c.size -= int64(len(e.out)) + c.size -= e.size() } } @@ -252,24 +278,23 @@ func (p *Proxy) cachedRewrite(ctx context.Context, ecosystem, proxyURL, name str if err != nil { return rewrittenMetadata{}, err } - return rewrittenMetadata{out, metadataValidators{etag: keptVersionsETag(digest, proxyURL, name, kept)}}, nil + return rewrittenMetadata{body: out, metadataValidators: metadataValidators{etag: keptVersionsETag(digest, proxyURL, name, kept)}}, nil } rewriteOnly := func(b []byte) ([]byte, error) { out, _, err := rewrite(b) return out, err } - var out []byte - var v metadataValidators + var m rewrittenMetadata var err error if p.rewrites == nil || p.cooldownEnabled() { - out, v, err = rewriteWithETag(in, rewriteOnly) + m, err = rewriteWithETag(in, rewriteOnly) } else { - out, v, err = p.rewrites.rewrite(ctx, rewriteCacheKey(ecosystem, proxyURL, name, in), in, rewriteOnly) + m, err = p.rewrites.rewrite(ctx, rewriteCacheKey(ecosystem, proxyURL, name, in), in, rewriteOnly) } if err == nil && !p.cooldownEnabled() { - v.lastModified = p.metadataDate(ecosystem, proxyURL, name, v.etag) + m.lastModified = p.metadataDate(ecosystem, proxyURL, name, m.etag) } - return rewrittenMetadata{out, v}, err + return m, err } // uncachedRewrite gives a rewrite made outside the cache its validators: an @@ -280,7 +305,7 @@ func (p *Proxy) uncachedRewrite(ecosystem, proxyURL, name string, out []byte) re if !p.cooldownEnabled() { v.lastModified = p.metadataDate(ecosystem, proxyURL, name, v.etag) } - return rewrittenMetadata{out, v} + return rewrittenMetadata{body: out, metadataValidators: v} } func (p *Proxy) cooldownEnabled() bool { @@ -307,29 +332,78 @@ func (p *Proxy) storedRewrite(ecosystem, cacheKey, proxyURL, name string) (rewri if !entry.FetchedAt.Valid || time.Since(entry.FetchedAt.Time) >= p.MetadataTTL { return rewrittenMetadata{}, false } - out, v, ok := p.rewrites.get(rewriteCacheKeyForDigest(ecosystem, proxyURL, name, entry.ContentDigest.String)) + m, ok := p.rewrites.get(rewriteCacheKeyForDigest(ecosystem, proxyURL, name, entry.ContentDigest.String)) if !ok { return rewrittenMetadata{}, false } - v.lastModified = p.metadataDate(ecosystem, proxyURL, name, v.etag) + m.lastModified = p.metadataDate(ecosystem, proxyURL, name, m.etag) metrics.RecordCacheHit(ecosystem) - return rewrittenMetadata{out, v}, true + return m, true } // serveRewrittenMetadata writes rewritten metadata with its validators, or -// 304 Not Modified when they show the client's copy is current. +// 304 Not Modified when they show the client's copy is current. A client +// accepting gzip gets the compressed copy when there is one. Being another +// representation, it has an ETag of its own. func serveRewrittenMetadata(w http.ResponseWriter, r *http.Request, m rewrittenMetadata) { - w.Header().Set(headerETag, m.etag) + body, etag := m.body, m.etag + compressed := m.gzipped != nil && acceptsGzip(r) + if compressed { + body, etag = m.gzipped, gzipETag(m.etag) + } + w.Header().Set("Vary", headerAcceptEncoding) + w.Header().Set(headerETag, etag) if !m.lastModified.IsZero() { w.Header().Set(headerLastModified, m.lastModified.UTC().Format(http.TimeFormat)) } - if notModified(r, m.etag, m.lastModified) { + if notModified(r, etag, m.lastModified) { w.WriteHeader(http.StatusNotModified) return } w.Header().Set(headerContentType, contentTypeJSON) + if compressed { + w.Header().Set(headerContentEncoding, "gzip") + } w.WriteHeader(http.StatusOK) - _, _ = w.Write(m.body) + _, _ = w.Write(body) +} + +// gzipETag returns the ETag of the gzip representation of the document +// whose ETag is etag. +func gzipETag(etag string) string { + return strings.TrimSuffix(etag, `"`) + `-gzip"` +} + +// acceptsGzip reports whether the request's Accept-Encoding allows a gzip +// response: gzip or x-gzip, or else "*", with a quality above zero. +func acceptsGzip(r *http.Request) bool { + wildcard := false + for _, value := range r.Header.Values(headerAcceptEncoding) { + for coding := range strings.SplitSeq(value, ",") { + name, params, _ := strings.Cut(coding, ";") + switch strings.ToLower(strings.TrimSpace(name)) { + case "gzip", "x-gzip": + return codingQualityPositive(params) + case "*": + wildcard = codingQualityPositive(params) + } + } + } + return wildcard +} + +// codingQualityPositive reports whether an Accept-Encoding entry's +// parameters give it a quality above zero. No q parameter means 1. +func codingQualityPositive(params string) bool { + for param := range strings.SplitSeq(params, ";") { + key, value, _ := strings.Cut(param, "=") + if strings.TrimSpace(key) != "q" { + continue + } + q, err := strconv.ParseFloat(strings.TrimSpace(value), 64) + return err == nil && q > 0 + } + return true } // notModified reports whether a conditional request's validators match. diff --git a/internal/handler/rewrite_cache_test.go b/internal/handler/rewrite_cache_test.go index 473f80a..8584312 100644 --- a/internal/handler/rewrite_cache_test.go +++ b/internal/handler/rewrite_cache_test.go @@ -42,9 +42,9 @@ func TestRewriteCache_RepeatServesCachedRewrite(t *testing.T) { key := rewriteCacheKey("npm", "http://proxy", "left-pad", []byte("doc")) for range 3 { - out, _, err := c.rewrite(context.Background(), key, []byte("doc"), r.fn) - if err != nil || string(out) != "DOC" { - t.Fatalf("rewrite = %q, %v", out, err) + m, err := c.rewrite(context.Background(), key, []byte("doc"), r.fn) + if err != nil || string(m.body) != "DOC" { + t.Fatalf("rewrite = %q, %v", m.body, err) } } if got := r.calls.Load(); got != 1 { @@ -58,9 +58,9 @@ func TestRewriteCache_NewUpstreamBytesRewriteAgain(t *testing.T) { for _, doc := range []string{"v1", "v2"} { key := rewriteCacheKey("npm", "http://proxy", "left-pad", []byte(doc)) - out, _, err := c.rewrite(context.Background(), key, []byte(doc), r.fn) - if err != nil || string(out) != strings.ToUpper(doc) { - t.Fatalf("rewrite(%s) = %q, %v", doc, out, err) + m, err := c.rewrite(context.Background(), key, []byte(doc), r.fn) + if err != nil || string(m.body) != strings.ToUpper(doc) { + t.Fatalf("rewrite(%s) = %q, %v", doc, m.body, err) } } if got := r.calls.Load(); got != 2 { @@ -80,11 +80,11 @@ func TestRewriteCache_ConcurrentRequestsShareRewrite(t *testing.T) { outs := make(chan string, n) run := func() { - out, _, err := c.rewrite(context.Background(), key, []byte("doc"), r.fn) + m, err := c.rewrite(context.Background(), key, []byte("doc"), r.fn) if err != nil { t.Error(err) } - outs <- string(out) + outs <- string(m.body) } go run() <-r.entered @@ -113,21 +113,21 @@ func TestRewriteCache_WaiterLeavingKeepsRewrite(t *testing.T) { first := make(chan error, 1) go func() { - _, _, err := c.rewrite(context.Background(), key, []byte("doc"), r.fn) + _, err := c.rewrite(context.Background(), key, []byte("doc"), r.fn) first <- err }() <-r.entered ctx, cancel := context.WithCancel(context.Background()) cancel() - if _, _, err := c.rewrite(ctx, key, []byte("doc"), r.fn); !errors.Is(err, context.Canceled) { + if _, err := c.rewrite(ctx, key, []byte("doc"), r.fn); !errors.Is(err, context.Canceled) { t.Errorf("waiter err = %v, want context.Canceled", err) } close(r.release) if err := <-first; err != nil { t.Fatal(err) } - if out, _, err := c.rewrite(context.Background(), key, []byte("doc"), r.fn); err != nil || string(out) != "DOC" { - t.Errorf("cached rewrite = %q, %v", out, err) + if m, err := c.rewrite(context.Background(), key, []byte("doc"), r.fn); err != nil || string(m.body) != "DOC" { + t.Errorf("cached rewrite = %q, %v", m.body, err) } if got := r.calls.Load(); got != 1 { t.Errorf("rewrites = %d, want 1", got) @@ -140,18 +140,18 @@ func TestRewriteCache_EvictsLeastRecentlyUsed(t *testing.T) { keyFor := func(doc string) string { return rewriteCacheKey("npm", "http://proxy", doc, []byte(doc)) } for _, doc := range []string{"aaa", "bbb"} { - _, _, _ = c.rewrite(context.Background(), keyFor(doc), []byte(doc), r.fn) + _, _ = c.rewrite(context.Background(), keyFor(doc), []byte(doc), r.fn) } - _, _, _ = c.rewrite(context.Background(), keyFor("aaa"), []byte("aaa"), r.fn) // aaa is now the most recent - _, _, _ = c.rewrite(context.Background(), keyFor("ccc"), []byte("ccc"), r.fn) // evicts bbb + _, _ = c.rewrite(context.Background(), keyFor("aaa"), []byte("aaa"), r.fn) // aaa is now the most recent + _, _ = c.rewrite(context.Background(), keyFor("ccc"), []byte("ccc"), r.fn) // evicts bbb before := r.calls.Load() - _, _, _ = c.rewrite(context.Background(), keyFor("aaa"), []byte("aaa"), r.fn) - _, _, _ = c.rewrite(context.Background(), keyFor("ccc"), []byte("ccc"), r.fn) + _, _ = c.rewrite(context.Background(), keyFor("aaa"), []byte("aaa"), r.fn) + _, _ = c.rewrite(context.Background(), keyFor("ccc"), []byte("ccc"), r.fn) if got := r.calls.Load() - before; got != 0 { t.Errorf("aaa and ccc rewritten %d times, want both still cached", got) } - _, _, _ = c.rewrite(context.Background(), keyFor("bbb"), []byte("bbb"), r.fn) + _, _ = c.rewrite(context.Background(), keyFor("bbb"), []byte("bbb"), r.fn) if got := r.calls.Load() - before; got != 1 { t.Errorf("bbb should have been evicted and rewritten") } @@ -162,7 +162,7 @@ func TestRewriteCache_OutputLargerThanCacheIsNotStored(t *testing.T) { var r countingRewrite key := rewriteCacheKey("npm", "http://proxy", "big", []byte("big")) - _, _, _ = c.rewrite(context.Background(), key, []byte("big"), r.fn) + _, _ = c.rewrite(context.Background(), key, []byte("big"), r.fn) if c.len() != 0 { t.Errorf("cached %d entries, want 0", c.len()) } @@ -175,7 +175,7 @@ func TestRewriteCache_ErrorsAreNotCached(t *testing.T) { key := rewriteCacheKey("npm", "http://proxy", "broken", []byte("doc")) for range 2 { - if _, _, err := c.rewrite(context.Background(), key, []byte("doc"), failing); err == nil { + if _, err := c.rewrite(context.Background(), key, []byte("doc"), failing); err == nil { t.Fatal("expected the rewrite error") } } From b2955ef4583e027094fb89fded1e0d65e0e18783 Mon Sep 17 00:00:00 2001 From: Giles Westwood <201443+gilesw@users.noreply.github.com> Date: Thu, 15 Oct 2026 13:35:12 +0100 Subject: [PATCH 300/633] Keep holds through late successes and away from the circuit breaker A response arriving during a hold was sent before it began, so a late success no longer releases the host; only one after the hold has ended does. Artifact fetches check RateLimitGate.Held before calling the fetcher, so held cache misses fail with the HeldError without counting as failures in the host's circuit breaker. Every handler that answers an upstream fetch error with 502 now answers a held request with 503 and Retry-After instead. OnHold and RecordUpstreamRateLimitHold drop the unused duration. --- internal/handler/arch.go | 3 + internal/handler/cargo.go | 3 + internal/handler/composer.go | 9 +++ internal/handler/composer_hosted.go | 3 + internal/handler/conan.go | 3 + internal/handler/conda.go | 3 + internal/handler/container.go | 6 ++ internal/handler/container_manifest.go | 3 + internal/handler/container_referrers.go | 3 + internal/handler/container_tags.go | 3 + internal/handler/download_test.go | 81 +++++++++++++++++++++++++ internal/handler/gem.go | 6 ++ internal/handler/go.go | 3 + internal/handler/go_sumdb.go | 3 + internal/handler/handler.go | 14 +++++ internal/handler/helm.go | 3 + internal/handler/hex.go | 3 + internal/handler/npm.go | 12 ++++ internal/handler/npm_hosted.go | 3 + internal/handler/nuget.go | 6 ++ internal/handler/nuget_cooldown.go | 3 + internal/handler/pub.go | 3 + internal/handler/pypi.go | 9 +++ internal/handler/swift.go | 9 +++ internal/httpclient/ratelimit.go | 38 +++++++++--- internal/httpclient/ratelimit_test.go | 58 ++++++++++++++++-- internal/metrics/metrics.go | 2 +- internal/server/server.go | 2 + 28 files changed, 285 insertions(+), 12 deletions(-) diff --git a/internal/handler/arch.go b/internal/handler/arch.go index 6706dae..f2abedf 100644 --- a/internal/handler/arch.go +++ b/internal/handler/arch.go @@ -477,6 +477,9 @@ func (h *ArchHandler) writeMetadata(w http.ResponseWriter, r *http.Request, cach http.Error(w, "not found", http.StatusNotFound) return } + if writeUpstreamHeld(w, res.err) { + return + } h.proxy.Logger.Error("metadata fetch failed", "error", res.err) http.Error(w, "failed to fetch from upstream", http.StatusBadGateway) return diff --git a/internal/handler/cargo.go b/internal/handler/cargo.go index 97d2f17..d30d2e6 100644 --- a/internal/handler/cargo.go +++ b/internal/handler/cargo.go @@ -113,6 +113,9 @@ func (h *CargoHandler) handleIndex(w http.ResponseWriter, r *http.Request) { http.Error(w, "not found", http.StatusNotFound) return } + if writeUpstreamHeld(w, err) { + return + } h.proxy.Logger.Error("failed to fetch upstream index", "error", err) http.Error(w, "failed to fetch from upstream", http.StatusBadGateway) return diff --git a/internal/handler/composer.go b/internal/handler/composer.go index 9dac530..adc4d5f 100644 --- a/internal/handler/composer.go +++ b/internal/handler/composer.go @@ -175,6 +175,9 @@ func (h *ComposerHandler) handlePackageMetadata(w http.ResponseWriter, r *http.R http.Error(w, "not found", http.StatusNotFound) return } + if writeUpstreamHeld(w, err) { + return + } h.proxy.Logger.Error("upstream request failed", "error", err) http.Error(w, "upstream request failed", http.StatusBadGateway) return @@ -652,6 +655,9 @@ func (h *ComposerHandler) handleDownload(w http.ResponseWriter, r *http.Request) for _, metaURL := range metaURLs { found, err := h.findDownloadURLFromMetadata(r.Context(), metaURL, packageName, version) if err != nil { + if writeUpstreamHeld(w, err) { + return + } h.proxy.Logger.Error("failed to fetch metadata", "error", err, "url", metaURL) http.Error(w, "failed to fetch metadata", http.StatusBadGateway) return @@ -890,6 +896,9 @@ func (h *ComposerHandler) proxyUpstream(w http.ResponseWriter, r *http.Request) resp, err := h.proxy.HTTPClient.Do(req) if err != nil { + if writeUpstreamHeld(w, err) { + return + } h.proxy.Logger.Error("upstream request failed", "error", err) http.Error(w, "upstream request failed", http.StatusBadGateway) return diff --git a/internal/handler/composer_hosted.go b/internal/handler/composer_hosted.go index a42a27a..9a27e6f 100644 --- a/internal/handler/composer_hosted.go +++ b/internal/handler/composer_hosted.go @@ -85,6 +85,9 @@ func (h *ComposerHandler) serveMetadataWithLocal(w http.ResponseWriter, r *http. case errors.Is(upstream.err, ErrUpstreamNotFound): out, err = localComposerDocument(packageName, local) case upstream.err != nil: + if writeUpstreamHeld(w, upstream.err) { + return + } h.proxy.Logger.Error("upstream request failed", "error", upstream.err) http.Error(w, "upstream request failed", http.StatusBadGateway) return diff --git a/internal/handler/conan.go b/internal/handler/conan.go index 6d88b43..9bb4401 100644 --- a/internal/handler/conan.go +++ b/internal/handler/conan.go @@ -196,6 +196,9 @@ func (h *ConanHandler) proxyUpstream(w http.ResponseWriter, r *http.Request) { resp, err := h.proxy.HTTPClient.Do(req) if err != nil { + if writeUpstreamHeld(w, err) { + return + } h.proxy.Logger.Error("upstream request failed", "error", err) http.Error(w, "upstream request failed", http.StatusBadGateway) return diff --git a/internal/handler/conda.go b/internal/handler/conda.go index 6e4be7e..d7b8b3f 100644 --- a/internal/handler/conda.go +++ b/internal/handler/conda.go @@ -183,6 +183,9 @@ func (h *CondaHandler) handleRepodata(w http.ResponseWriter, r *http.Request) { resp, err := h.proxy.HTTPClient.Do(req) if err != nil { + if writeUpstreamHeld(w, err) { + return + } h.proxy.Logger.Error("upstream request failed", "error", err) http.Error(w, "upstream request failed", http.StatusBadGateway) return diff --git a/internal/handler/container.go b/internal/handler/container.go index 49ef3b2..55f1715 100644 --- a/internal/handler/container.go +++ b/internal/handler/container.go @@ -450,6 +450,9 @@ func (h *ContainerHandler) handleBlobDownload(w http.ResponseWriter, r *http.Req h.containerError(w, http.StatusBadGateway, "DIGEST_INVALID", "blob digest verification failed") return } + if writeUpstreamHeld(w, err) { + return + } h.proxy.Logger.Error("failed to fetch blob", "error", err) h.containerError(w, http.StatusBadGateway, "INTERNAL_ERROR", "failed to fetch blob") return @@ -520,6 +523,9 @@ func (h *ContainerHandler) proxyBlobHead(w http.ResponseWriter, r *http.Request, resp, err := h.proxy.HTTPClient.Do(req) if err != nil { + if writeUpstreamHeld(w, err) { + return + } h.containerError(w, http.StatusBadGateway, "INTERNAL_ERROR", "failed to fetch from upstream") return } diff --git a/internal/handler/container_manifest.go b/internal/handler/container_manifest.go index 63b2d47..902ad3a 100644 --- a/internal/handler/container_manifest.go +++ b/internal/handler/container_manifest.go @@ -150,6 +150,9 @@ func (h *ContainerHandler) serveStaleManifestOrError(w http.ResponseWriter, r *h h.containerError(w, http.StatusNotFound, "MANIFEST_UNKNOWN", "manifest unknown") return } + if writeUpstreamHeld(w, err) { + return + } h.proxy.Logger.Error("failed to fetch manifest", "error", err) h.containerError(w, http.StatusBadGateway, "INTERNAL_ERROR", "failed to fetch from upstream") } diff --git a/internal/handler/container_referrers.go b/internal/handler/container_referrers.go index d03e859..37261c5 100644 --- a/internal/handler/container_referrers.go +++ b/internal/handler/container_referrers.go @@ -160,6 +160,9 @@ func (h *ContainerHandler) serveStaleReferrersOrError(w http.ResponseWriter, r * h.writeContainerReferrers(w, r, registryURL, cached, true) return } + if writeUpstreamHeld(w, err) { + return + } h.proxy.Logger.Error("failed to fetch container referrers", "error", err) h.containerError(w, http.StatusBadGateway, "INTERNAL_ERROR", "failed to fetch from upstream") } diff --git a/internal/handler/container_tags.go b/internal/handler/container_tags.go index a0ac9c1..6142590 100644 --- a/internal/handler/container_tags.go +++ b/internal/handler/container_tags.go @@ -117,6 +117,9 @@ func (h *ContainerHandler) serveStaleTagsOrError(w http.ResponseWriter, r *http. h.writeContainerTags(w, r, registryURL, cached, true) return } + if writeUpstreamHeld(w, err) { + return + } h.proxy.Logger.Error("failed to fetch container tag list", "error", err) h.containerError(w, http.StatusBadGateway, "INTERNAL_ERROR", "failed to fetch from upstream") } diff --git a/internal/handler/download_test.go b/internal/handler/download_test.go index d7a8e6b..5d70ce9 100644 --- a/internal/handler/download_test.go +++ b/internal/handler/download_test.go @@ -1306,3 +1306,84 @@ func TestMavenHandler_UpstreamHeldReturns503(t *testing.T) { type roundTripperFunc func(*http.Request) (*http.Response, error) func (fn roundTripperFunc) RoundTrip(req *http.Request) (*http.Response, error) { return fn(req) } + +func TestMavenHandler_HeldArtifactMissesSkipCircuitBreaker(t *testing.T) { + proxy, _, _, _ := setupTestProxy(t) + upstreamCalls := 0 + gate := httpclient.NewRateLimitGate(roundTripperFunc(func(req *http.Request) (*http.Response, error) { + upstreamCalls++ + return &http.Response{ + StatusCode: http.StatusTooManyRequests, + Header: http.Header{"Retry-After": []string{"600"}}, + Body: http.NoBody, + Request: req, + }, nil + })) + req, _ := http.NewRequest(http.MethodGet, "https://repo1.maven.org/maven2/x", nil) + if resp, err := gate.RoundTrip(req); err == nil { + _ = resp.Body.Close() + } + breaker := fetch.NewCircuitBreakerFetcher(fetch.NewFetcher(fetch.WithHTTPClient(&http.Client{Transport: gate}))) + proxy.Fetcher = breaker + proxy.RateLimits = gate + + h := NewMavenHandler(proxy, "http://localhost", "", "") + srv := httptest.NewServer(h.Routes()) + defer srv.Close() + + for i := range 8 { + resp, err := http.Get(fmt.Sprintf("%s/com/example/lib/1.%d/lib-1.%d.jar", srv.URL, i, i)) + if err != nil { + t.Fatalf("request %d failed: %v", i, err) + } + _ = resp.Body.Close() + if resp.StatusCode != http.StatusServiceUnavailable || resp.Header.Get("Retry-After") == "" { + t.Fatalf("request %d: status %d Retry-After %q, want 503 with Retry-After", i, resp.StatusCode, resp.Header.Get("Retry-After")) + } + } + if upstreamCalls != 1 { + t.Errorf("upstream calls = %d, want only the one that started the hold", upstreamCalls) + } + if state := breaker.GetBreakerState()["repo1.maven.org"]; state == "open" { + t.Errorf("breaker for repo1.maven.org is open after held misses, want closed") + } +} + +func TestHandlers_UpstreamHeldMetadataReturns503(t *testing.T) { + held := &httpclient.HeldError{Host: "upstream.example", Until: time.Now().Add(42 * time.Second), RetryAfter: 42 * time.Second} + tests := []struct { + name string + handler func(*Proxy) http.Handler + path string + }{ + {"npm metadata", func(p *Proxy) http.Handler { return NewNPMHandler(p, "http://localhost", "").Routes() }, "/lodash"}, + {"pypi simple", func(p *Proxy) http.Handler { return NewPyPIHandler(p, "http://localhost").Routes() }, "/simple/requests/"}, + {"pypi json", func(p *Proxy) http.Handler { return NewPyPIHandler(p, "http://localhost").Routes() }, "/pypi/requests/json"}, + {"cargo index", func(p *Proxy) http.Handler { + return NewCargoHandler(p, "http://localhost", "", "").Routes() + }, "/se/rd/serde"}, + {"pub metadata", func(p *Proxy) http.Handler { return NewPubHandler(p, "http://localhost").Routes() }, "/api/packages/http"}, + {"nuget service index", func(p *Proxy) http.Handler { return NewNuGetHandler(p, "http://localhost").Routes() }, "/v3/index.json"}, + {"nuget search", func(p *Proxy) http.Handler { return NewNuGetHandler(p, "http://localhost").Routes() }, "/query?q=json"}, + {"conda repodata", func(p *Proxy) http.Handler { return NewCondaHandler(p, "http://localhost").Routes() }, "/conda-forge/noarch/repodata.json"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + proxy, _, _, _ := setupTestProxy(t) + proxy.HTTPClient = &http.Client{Transport: roundTripperFunc(func(*http.Request) (*http.Response, error) { + return nil, held + })} + srv := httptest.NewServer(tt.handler(proxy)) + defer srv.Close() + + resp, err := http.Get(srv.URL + tt.path) + if err != nil { + t.Fatalf("request failed: %v", err) + } + _ = resp.Body.Close() + if resp.StatusCode != http.StatusServiceUnavailable || resp.Header.Get("Retry-After") != "42" { + t.Errorf("status %d Retry-After %q, want 503 with Retry-After 42", resp.StatusCode, resp.Header.Get("Retry-After")) + } + }) + } +} diff --git a/internal/handler/gem.go b/internal/handler/gem.go index 0657b6b..2d2136b 100644 --- a/internal/handler/gem.go +++ b/internal/handler/gem.go @@ -121,6 +121,9 @@ func (h *GemHandler) handleCompactIndex(w http.ResponseWriter, r *http.Request) indexResp, filteredVersions, err := h.fetchIndexAndVersions(r, name) if err != nil { + if writeUpstreamHeld(w, err) { + return + } h.proxy.Logger.Error("upstream compact index request failed", "error", err) http.Error(w, "upstream request failed", http.StatusBadGateway) return @@ -306,6 +309,9 @@ func (h *GemHandler) proxyUpstream(w http.ResponseWriter, r *http.Request) { resp, err := h.proxy.HTTPClient.Do(req) if err != nil { + if writeUpstreamHeld(w, err) { + return + } h.proxy.Logger.Error("upstream request failed", "error", err) http.Error(w, "upstream request failed", http.StatusBadGateway) return diff --git a/internal/handler/go.go b/internal/handler/go.go index a27c37b..c528472 100644 --- a/internal/handler/go.go +++ b/internal/handler/go.go @@ -137,6 +137,9 @@ func (h *GoHandler) handleDownload(w http.ResponseWriter, r *http.Request, modul http.Error(w, err.Error(), http.StatusForbidden) return } + if writeUpstreamHeld(w, err) { + return + } h.proxy.Logger.Error("failed to get artifact", "error", err) http.Error(w, "failed to fetch module", http.StatusBadGateway) return diff --git a/internal/handler/go_sumdb.go b/internal/handler/go_sumdb.go index 020a126..7de2513 100644 --- a/internal/handler/go_sumdb.go +++ b/internal/handler/go_sumdb.go @@ -407,6 +407,9 @@ func (h *GoHandler) proxySumDB(w http.ResponseWriter, r *http.Request, name, ups case errors.Is(err, ErrUpstreamNotFound): http.Error(w, "not found", http.StatusNotFound) default: + if writeUpstreamHeld(w, err) { + return + } p.Logger.Error("checksum database fetch failed", "url", upstreamURL, "error", err) http.Error(w, "failed to fetch from upstream", http.StatusBadGateway) } diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 2d2e054..560b8eb 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -20,6 +20,7 @@ import ( "github.com/git-pkgs/cooldown" "github.com/git-pkgs/proxy/internal/database" "github.com/git-pkgs/proxy/internal/denylist" + "github.com/git-pkgs/proxy/internal/httpclient" "github.com/git-pkgs/proxy/internal/metrics" "github.com/git-pkgs/proxy/internal/packageurl" "github.com/git-pkgs/proxy/internal/scanner" @@ -175,6 +176,10 @@ type Proxy struct { DirectServeBaseURL string HTTPClient *http.Client + // RateLimits is the gate under the upstream clients. Artifact fetches + // check it before calling Fetcher, so held hosts never reach its breaker. + RateLimits *httpclient.RateLimitGate + // HostedNPMScopes and HostedComposerVendors name namespaces served from // pushed packages alone, without asking the upstream. HostedNPMScopes []string @@ -510,6 +515,9 @@ func (p *Proxy) fetchAndCache(ctx context.Context, ecosystem, name, version, fil } } + if err := p.RateLimits.Held(info.URL); err != nil { + return artifacts.Artifact{}, "", err + } p.Logger.Info("fetching from upstream", "ecosystem", ecosystem, "name", name, "version", version, "url", info.URL) @@ -1917,6 +1925,9 @@ func (p *Proxy) discardStaleArtifact(pkgPURL, versionPURL, filename, upstreamHas } func (p *Proxy) fetchAndCacheFromURL(ctx context.Context, ecosystem, name, version, filename, pkgPURL, versionPURL, downloadURL string, headers http.Header, upstreamHash, declared string) (artifacts.Artifact, string, error) { + if err := p.RateLimits.Held(downloadURL); err != nil { + return artifacts.Artifact{}, "", err + } p.Logger.Info("fetching from upstream", "ecosystem", ecosystem, "name", name, "version", version, "url", downloadURL) @@ -1944,6 +1955,9 @@ func (p *Proxy) fetchAndCacheFromURL(ctx context.Context, ecosystem, name, versi // after the bytes have been sent, and an unterminated chunked response is the // only way left to make the client reject them (see serveArtifact). func (p *Proxy) streamFromUpstream(ctx context.Context, ecosystem, name, version, filename, versionPURL, upstreamURL, upstreamHash, declared string, fetchArtifact func(context.Context) (*fetch.Artifact, error)) (*CacheResult, error) { + if err := p.RateLimits.Held(upstreamURL); err != nil { + return nil, err + } p.Logger.Info("streaming from upstream", "ecosystem", ecosystem, "name", name, "version", version, "url", upstreamURL) diff --git a/internal/handler/helm.go b/internal/handler/helm.go index cddc943..6183d97 100644 --- a/internal/handler/helm.go +++ b/internal/handler/helm.go @@ -186,6 +186,9 @@ func (h *HelmHandler) serveIndexError(w http.ResponseWriter, err error) { http.Error(w, "Helm repository not found", http.StatusNotFound) return } + if writeUpstreamHeld(w, err) { + return + } h.proxy.Logger.Error("failed to fetch Helm index", "error", err) http.Error(w, "failed to fetch Helm repository index", http.StatusBadGateway) } diff --git a/internal/handler/hex.go b/internal/handler/hex.go index 79be510..847f955 100644 --- a/internal/handler/hex.go +++ b/internal/handler/hex.go @@ -118,6 +118,9 @@ func (h *HexHandler) handlePackages(w http.ResponseWriter, r *http.Request) { protoResp, filteredVersions, err := h.fetchPackageAndVersions(r, name) if err != nil { + if writeUpstreamHeld(w, err) { + return + } h.proxy.Logger.Error("upstream request failed", "error", err) http.Error(w, "upstream request failed", http.StatusBadGateway) return diff --git a/internal/handler/npm.go b/internal/handler/npm.go index 0e0d2b0..875dee7 100644 --- a/internal/handler/npm.go +++ b/internal/handler/npm.go @@ -163,6 +163,9 @@ func (h *NPMHandler) handleSecurity(w http.ResponseWriter, r *http.Request) { resp, err := h.proxy.HTTPClient.Do(req) if err != nil { + if writeUpstreamHeld(w, err) { + return + } h.proxy.Logger.Error("npm audit request failed", "path", r.URL.Path, "error", err) JSONError(w, http.StatusBadGateway, "failed to reach upstream registry") return @@ -240,6 +243,9 @@ func (h *NPMHandler) handlePackageMetadata(w http.ResponseWriter, r *http.Reques } return } + if writeUpstreamHeld(w, err) { + return + } h.proxy.Logger.Error("failed to fetch npm metadata", "error", err) JSONError(w, http.StatusBadGateway, "failed to fetch from upstream") return @@ -285,6 +291,9 @@ func (h *NPMHandler) handleFilteredNPMVersionMetadata(w http.ResponseWriter, r * JSONError(w, http.StatusNotFound, ErrPackageNotFound.Error()) return } + if writeUpstreamHeld(w, err) { + return + } h.proxy.Logger.Error("failed to fetch npm metadata", "error", err) JSONError(w, http.StatusBadGateway, "failed to fetch from upstream") return @@ -849,6 +858,9 @@ func (h *NPMHandler) handleDownload(w http.ResponseWriter, r *http.Request) { case errors.Is(err, ErrArtifactBlocked), errors.Is(err, ErrVersionDenied): JSONError(w, http.StatusForbidden, err.Error()) default: + if writeUpstreamHeld(w, err) { + return + } h.proxy.Logger.Error("failed to get artifact", "error", err) JSONError(w, http.StatusBadGateway, "failed to fetch package") } diff --git a/internal/handler/npm_hosted.go b/internal/handler/npm_hosted.go index e8998c9..65b59ba 100644 --- a/internal/handler/npm_hosted.go +++ b/internal/handler/npm_hosted.go @@ -105,6 +105,9 @@ func (h *NPMHandler) serveMetadataWithLocal(w http.ResponseWriter, r *http.Reque case errors.Is(upstream.err, ErrUpstreamNotFound): out, err = localNPMPackument(packageName, local) case upstream.err != nil: + if writeUpstreamHeld(w, upstream.err) { + return + } h.proxy.Logger.Error("failed to fetch npm metadata", "error", upstream.err) JSONError(w, http.StatusBadGateway, "failed to fetch from upstream") return diff --git a/internal/handler/nuget.go b/internal/handler/nuget.go index 1a3d0a1..3d9f727 100644 --- a/internal/handler/nuget.go +++ b/internal/handler/nuget.go @@ -96,6 +96,9 @@ func (h *NuGetHandler) handleServiceIndex(w http.ResponseWriter, r *http.Request http.Error(w, "not found", http.StatusNotFound) return } + if writeUpstreamHeld(w, err) { + return + } h.proxy.Logger.Error("upstream request failed", "error", err) http.Error(w, "upstream request failed", http.StatusBadGateway) return @@ -234,6 +237,9 @@ func (h *NuGetHandler) proxyUpstream(w http.ResponseWriter, r *http.Request) { resp, err := h.proxy.HTTPClient.Do(req) if err != nil { + if writeUpstreamHeld(w, err) { + return + } h.proxy.Logger.Error("upstream request failed", "error", err) http.Error(w, "upstream request failed", http.StatusBadGateway) return diff --git a/internal/handler/nuget_cooldown.go b/internal/handler/nuget_cooldown.go index 6b95ae9..afb147d 100644 --- a/internal/handler/nuget_cooldown.go +++ b/internal/handler/nuget_cooldown.go @@ -118,6 +118,9 @@ func (h *NuGetHandler) nugetMetadataError(w http.ResponseWriter, err error) { JSONError(w, http.StatusNotFound, "package metadata not found") return } + if writeUpstreamHeld(w, err) { + return + } h.proxy.Logger.Warn("failed to process NuGet metadata", "error", err) JSONError(w, http.StatusBadGateway, "failed to process package metadata") } diff --git a/internal/handler/pub.go b/internal/handler/pub.go index e907e73..12ff6d9 100644 --- a/internal/handler/pub.go +++ b/internal/handler/pub.go @@ -111,6 +111,9 @@ func (h *PubHandler) handlePackageMetadata(w http.ResponseWriter, r *http.Reques http.Error(w, "not found", http.StatusNotFound) return } + if writeUpstreamHeld(w, err) { + return + } h.proxy.Logger.Error("upstream request failed", "error", err) http.Error(w, "upstream request failed", http.StatusBadGateway) return diff --git a/internal/handler/pypi.go b/internal/handler/pypi.go index f4cef06..513f507 100644 --- a/internal/handler/pypi.go +++ b/internal/handler/pypi.go @@ -118,6 +118,9 @@ func (h *PyPIHandler) handleSimplePackage(w http.ResponseWriter, r *http.Request http.Error(w, "not found", http.StatusNotFound) return } + if writeUpstreamHeld(w, err) { + return + } h.proxy.Logger.Error("upstream request failed", "error", err) http.Error(w, "upstream request failed", http.StatusBadGateway) return @@ -418,6 +421,9 @@ func (h *PyPIHandler) proxyAndRewriteJSON(w http.ResponseWriter, r *http.Request http.Error(w, "not found", http.StatusNotFound) return } + if writeUpstreamHeld(w, err) { + return + } h.proxy.Logger.Error("upstream request failed", "error", err) http.Error(w, "upstream request failed", http.StatusBadGateway) return @@ -821,6 +827,9 @@ func (h *PyPIHandler) proxySimple(w http.ResponseWriter, r *http.Request, path s resp, err := h.proxy.HTTPClient.Do(req) if err != nil { + if writeUpstreamHeld(w, err) { + return + } h.proxy.Logger.Error("upstream request failed", "error", err) http.Error(w, "upstream request failed", http.StatusBadGateway) return diff --git a/internal/handler/swift.go b/internal/handler/swift.go index 243a37c..b506f1c 100644 --- a/internal/handler/swift.go +++ b/internal/handler/swift.go @@ -439,6 +439,9 @@ func (h *SwiftHandler) proxySwiftResource(w http.ResponseWriter, r *http.Request resp, err := h.proxy.HTTPClient.Do(req) if err != nil { + if writeUpstreamHeld(w, err) { + return + } writeSwiftProblem(w, http.StatusBadGateway, "upstream request failed") return } @@ -613,6 +616,9 @@ func (h *SwiftHandler) writeMetadataError(w http.ResponseWriter, err error) { writeSwiftProblem(w, http.StatusNotFound, "not found") return } + if writeUpstreamHeld(w, err) { + return + } h.proxy.Logger.Error("Swift metadata request failed", "error", err) writeSwiftProblem(w, http.StatusBadGateway, "upstream request failed") } @@ -622,6 +628,9 @@ func (h *SwiftHandler) writeArtifactError(w http.ResponseWriter, err error) { writeSwiftProblem(w, http.StatusNotFound, "release not found") return } + if writeUpstreamHeld(w, err) { + return + } h.proxy.Logger.Error("Swift archive request failed", "error", err) writeSwiftProblem(w, http.StatusBadGateway, "failed to fetch package") } diff --git a/internal/httpclient/ratelimit.go b/internal/httpclient/ratelimit.go index b6f50e4..53e3e8e 100644 --- a/internal/httpclient/ratelimit.go +++ b/internal/httpclient/ratelimit.go @@ -49,12 +49,12 @@ type hostHold struct { // with a *HeldError instead. Clients and fetchers above it see an error // rather than a slow upstream, so they fall back to cache or fail fast // without adding to the traffic the upstream is already refusing. The first -// non-limited response from the host releases it. +// non-limited response after the hold ends releases it. type RateLimitGate struct { base http.RoundTripper Logger *slog.Logger - // OnHold is called with the host and hold length each time a hold starts. - OnHold func(host string, retryAfter time.Duration) + // OnHold is called with the host each time a hold starts. + OnHold func(host string) now func() time.Time mu sync.Mutex @@ -77,8 +77,7 @@ func NewRateLimitGate(base http.RoundTripper) *RateLimitGate { func (g *RateLimitGate) RoundTrip(req *http.Request) (*http.Response, error) { host := hostKey(req.URL) if until, held := g.heldUntil(host); held { - now := g.now() - return nil, &HeldError{Host: host, Until: until, RetryAfter: until.Sub(now)} + return nil, &HeldError{Host: host, Until: until, RetryAfter: until.Sub(g.now())} } resp, err := g.base.RoundTrip(req) @@ -97,6 +96,24 @@ func (g *RateLimitGate) RoundTrip(req *http.Request) (*http.Response, error) { return resp, nil } +// Held returns a *HeldError when requests to rawURL's host are being held, +// and nil otherwise. It is safe to call on a nil gate. +func (g *RateLimitGate) Held(rawURL string) error { + if g == nil { + return nil + } + u, err := url.Parse(rawURL) + if err != nil { + return nil + } + host := hostKey(u) + until, held := g.heldUntil(host) + if !held { + return nil + } + return &HeldError{Host: host, Until: until, RetryAfter: until.Sub(g.now())} +} + // Holds returns the hosts currently held and when each hold ends. It is // safe to call on a nil gate. func (g *RateLimitGate) Holds() map[string]time.Time { @@ -161,13 +178,20 @@ func (g *RateLimitGate) hold(host, header string) { "until", until.UTC().Format(time.RFC3339)) } if g.OnHold != nil { - g.OnHold(host, d) + g.OnHold(host) } } +// release drops host's hold once it has ended. Nothing is sent during a +// hold, so a success arriving while one is active was sent before it began. func (g *RateLimitGate) release(host string) { + now := g.now() g.mu.Lock() - _, had := g.holds[host] + h, had := g.holds[host] + if had && now.Before(h.until) { + g.mu.Unlock() + return + } delete(g.holds, host) g.mu.Unlock() if had && g.Logger != nil { diff --git a/internal/httpclient/ratelimit_test.go b/internal/httpclient/ratelimit_test.go index 2a5e3ec..8ad7e9c 100644 --- a/internal/httpclient/ratelimit_test.go +++ b/internal/httpclient/ratelimit_test.go @@ -109,6 +109,56 @@ func TestRateLimitGate_ReleasesWhenHoldExpiresAndUpstreamServes(t *testing.T) { } } +func TestRateLimitGate_LateSuccessKeepsHold(t *testing.T) { + h := newGateHarness() + // The slow request is in flight when the limited one starts the hold, + // and its 200 arrives afterwards. + h.respond = func(req *http.Request) (int, string) { + if req.URL.Path == "/limited" { + return http.StatusTooManyRequests, "120" + } + _, _ = h.get(t, "https://repo1.maven.org/limited") + return http.StatusOK, "" + } + + resp, err := h.get(t, "https://repo1.maven.org/slow") + if err != nil || resp.StatusCode != http.StatusOK { + t.Fatalf("slow request: resp=%v err=%v, want its 200", resp, err) + } + if until, ok := h.gate.Holds()["repo1.maven.org"]; !ok || !until.Equal(h.now.Add(120*time.Second)) { + t.Fatalf("Holds() = %v after a late 200, want repo1.maven.org until now+120s", h.gate.Holds()) + } + if _, err := h.get(t, "https://repo1.maven.org/x"); !errors.Is(err, ErrUpstreamHeld) { + t.Errorf("next request err = %v, want held", err) + } +} + +func TestRateLimitGate_Held(t *testing.T) { + var nilGate *RateLimitGate + if err := nilGate.Held("https://repo1.maven.org/x"); err != nil { + t.Errorf("nil gate Held() = %v, want nil", err) + } + + h := newGateHarness() + if err := h.gate.Held("https://repo1.maven.org/x"); err != nil { + t.Errorf("before any limit: Held() = %v, want nil", err) + } + h.respond = func(*http.Request) (int, string) { return http.StatusTooManyRequests, "45" } + _, _ = h.get(t, "https://repo1.maven.org/x") + + var held *HeldError + if err := h.gate.Held("https://REPO1.maven.org/a/b.jar"); !errors.As(err, &held) || held.RetryAfter != 45*time.Second { + t.Errorf("Held() = %v, want *HeldError for 45s", err) + } + if err := h.gate.Held("https://plugins.gradle.org/m2/x.pom"); err != nil { + t.Errorf("other host: Held() = %v, want nil", err) + } + h.now = h.now.Add(45 * time.Second) + if err := h.gate.Held("https://repo1.maven.org/x"); err != nil { + t.Errorf("after the hold: Held() = %v, want nil", err) + } +} + func TestRateLimitGate_DefaultHoldDoublesWithoutHeader(t *testing.T) { h := newGateHarness() h.respond = func(*http.Request) (int, string) { return http.StatusTooManyRequests, "" } @@ -172,13 +222,13 @@ func TestRateLimitGate_ReportsHoldsAndNilSafe(t *testing.T) { h := newGateHarness() var reported []string - h.gate.OnHold = func(host string, retryAfter time.Duration) { - reported = append(reported, host+"="+retryAfter.String()) + h.gate.OnHold = func(host string) { + reported = append(reported, host) } h.respond = func(*http.Request) (int, string) { return http.StatusTooManyRequests, "10" } _, _ = h.get(t, "https://repo1.maven.org/x") - if len(reported) != 1 || reported[0] != "repo1.maven.org=10s" { - t.Errorf("OnHold reported %v, want [repo1.maven.org=10s]", reported) + if len(reported) != 1 || reported[0] != "repo1.maven.org" { + t.Errorf("OnHold reported %v, want [repo1.maven.org]", reported) } } diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go index 36320b5..5e9e1a6 100644 --- a/internal/metrics/metrics.go +++ b/internal/metrics/metrics.go @@ -492,7 +492,7 @@ func RecordCircuitBreakerTrip(registry string) { } // RecordUpstreamRateLimitHold counts a hold placed on host. -func RecordUpstreamRateLimitHold(host string, _ time.Duration) { +func RecordUpstreamRateLimitHold(host string) { UpstreamRateLimitHolds.WithLabelValues(host).Inc() } diff --git a/internal/server/server.go b/internal/server/server.go index 42dfad6..c503e28 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -238,6 +238,7 @@ func (s *Server) serve(listener net.Listener) error { } proxy := handler.NewProxy(s.db, s.storage, handler.LimitArtifactSize(fetcher, s.cfg.ParseArtifactMaxSize()), resolver, s.logger) proxy.HTTPClient = metadataClient + proxy.RateLimits = s.rateLimits cooldownPolicy, err := cooldownpolicy.New(cd, s.cfg.Cooldown.PackagePatterns) if err != nil { return fmt.Errorf("configuring cooldown policy: %w", err) @@ -519,6 +520,7 @@ func NewMirrorProxy(cfg *config.Config, db *database.DB, store storage.Storage, fetcher := handler.LimitArtifactSize(fetch.NewFetcher(fetch.WithHTTPClient(artifactClient)), cfg.ParseArtifactMaxSize()) proxy := handler.NewProxy(db, store, fetcher, fetch.NewResolver(), logger) proxy.HTTPClient = metadataClient + proxy.RateLimits = s.rateLimits return proxy, s.newContainerHandler(proxy) } From b884a5f63f64b0f344c9a6a261ff10b03198b021 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 15 Oct 2026 15:15:16 +0000 Subject: [PATCH 301/633] Bump golang.org/x/sync from 0.23.0 to 0.24.0 Bumps [golang.org/x/sync](https://github.com/golang/sync) from 0.23.0 to 0.24.0. - [Commits](https://github.com/golang/sync/compare/v0.23.0...v0.24.0) --- updated-dependencies: - dependency-name: golang.org/x/sync dependency-version: 0.24.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index b76c338..b760b01 100644 --- a/go.mod +++ b/go.mod @@ -31,7 +31,7 @@ require ( gocloud.dev v0.46.0 golang.org/x/mod v0.41.0 golang.org/x/net v0.59.0 - golang.org/x/sync v0.23.0 + golang.org/x/sync v0.24.0 google.golang.org/protobuf v1.36.12 gopkg.in/yaml.v3 v3.0.1 modernc.org/sqlite v1.60.1 diff --git a/go.sum b/go.sum index 3533e64..a6b42ee 100644 --- a/go.sum +++ b/go.sum @@ -331,8 +331,8 @@ golang.org/x/net v0.59.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg= golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.23.0 h1:KameEIfc1IkluZyXWLn39Wd4tURc6GbCiISGiZm2bQk= -golang.org/x/sync v0.23.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0= +golang.org/x/sync v0.24.0 h1:iqiDrGoDgyuvAKrhLQthUEzU/R378N78gXbBjxed7oA= +golang.org/x/sync v0.24.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210420072515-93ed5bcd2bfe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= From fc312a0c5936c38e2d6104c1ebf70c57c5076831 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 15 Oct 2026 15:16:38 +0000 Subject: [PATCH 302/633] Bump github.com/aws/aws-sdk-go-v2/config from 1.33.6 to 1.33.7 Bumps [github.com/aws/aws-sdk-go-v2/config](https://github.com/aws/aws-sdk-go-v2) from 1.33.6 to 1.33.7. - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/config/v1.33.6...config/v1.33.7) --- updated-dependencies: - dependency-name: github.com/aws/aws-sdk-go-v2/config dependency-version: 1.33.7 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 12 ++++++------ go.sum | 24 ++++++++++++------------ 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/go.mod b/go.mod index b76c338..5bc7999 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( github.com/BurntSushi/toml v1.6.0 github.com/CycloneDX/cyclonedx-go v0.12.0 github.com/Masterminds/semver/v3 v3.5.0 - github.com/aws/aws-sdk-go-v2/config v1.33.6 + github.com/aws/aws-sdk-go-v2/config v1.33.7 github.com/aws/aws-sdk-go-v2/service/ecr v1.66.1 github.com/git-pkgs/archives v0.8.1 github.com/git-pkgs/artifacts v0.2.1 @@ -53,7 +53,7 @@ require ( github.com/apapsch/go-jsonmerge/v2 v2.0.0 // indirect github.com/aws/aws-sdk-go-v2 v1.47.1 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.11 // indirect - github.com/aws/aws-sdk-go-v2/credentials v1.20.6 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.20.7 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.1 // indirect github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.2.3 // indirect github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.4 // indirect @@ -64,10 +64,10 @@ require ( github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.4 // indirect github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.25 // indirect github.com/aws/aws-sdk-go-v2/service/s3 v1.102.2 // indirect - github.com/aws/aws-sdk-go-v2/service/signin v1.10.1 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.38.1 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.1 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.51.1 // indirect + github.com/aws/aws-sdk-go-v2/service/signin v1.10.2 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.38.2 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.2 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.51.2 // indirect github.com/aws/smithy-go v1.28.1 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cenk/backoff v2.2.1+incompatible // indirect diff --git a/go.sum b/go.sum index 3533e64..4d69021 100644 --- a/go.sum +++ b/go.sum @@ -60,10 +60,10 @@ github.com/aws/aws-sdk-go-v2 v1.47.1 h1:uOIZnp4PK3ZhKI0dNrJrhTEsLxbpXHTAJlwoS1pv github.com/aws/aws-sdk-go-v2 v1.47.1/go.mod h1:bttEH6JqnUL8LepvDVfdrds/fZ5bCIxzpe3abyUrhDU= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.11 h1:h5+3VT69KUBK24grGuuA5saDJTj2IIjLb9au668Fo5I= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.11/go.mod h1:dnakxebH6UwFvcvujL0LVggYQ8nEvBGjU4G/V79Nv94= -github.com/aws/aws-sdk-go-v2/config v1.33.6 h1:MBjkSTLczek/UgiK+EYPIoRTqE7gP8vtW3OFbFo7Nug= -github.com/aws/aws-sdk-go-v2/config v1.33.6/go.mod h1:grRAFzdAZJrwcbasJRg2MPvIrVjtlfXllHssN6+E1JE= -github.com/aws/aws-sdk-go-v2/credentials v1.20.6 h1:NpAFXCU7NzXNkdGK3zQTtsRJ+3v9tZQV0xcdRw8uBdw= -github.com/aws/aws-sdk-go-v2/credentials v1.20.6/go.mod h1:mcZCoiPnyMvP8VMNbygNX5lLqSlkYJIMPODylQMurOk= +github.com/aws/aws-sdk-go-v2/config v1.33.7 h1:vgG8uuhm7vNsxr6BsXNG+Ciesh9iLdNPzarAAu/dB6A= +github.com/aws/aws-sdk-go-v2/config v1.33.7/go.mod h1:5KWNY38altxnd5cxcNoOTrFtJ7asFEiCiWfb2m+M2pE= +github.com/aws/aws-sdk-go-v2/credentials v1.20.7 h1:mit8rRVnmt+wvT4U6putiWSlwe3eKD2n6ElgyfcP/Mw= +github.com/aws/aws-sdk-go-v2/credentials v1.20.7/go.mod h1:/2pPi2qhPug21z6l3M0+AKUqAT1zn2MhRCtnQv8R5Zg= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.1 h1:8gALAAmacnIXh+z6VkdDanv4/IkG5APdg4DZLDTmLog= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.1/go.mod h1:Z7IJhJU+poOdJjUR2wpyY21ossQ1XS/R3Lk9Msq5kM4= github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.2.3 h1:w5OoDiMN6x53ROmiIImGzmVcxXv2q1GXY+aKV4WAJYM= @@ -86,14 +86,14 @@ github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.25 h1:2pQEbwf+/6EDb github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.25/go.mod h1:KvT6NCcQ0EZ+ZkVRrlBMt04Po3ok23YELEp7WimhLhM= github.com/aws/aws-sdk-go-v2/service/s3 v1.102.2 h1:ie4ElCmUKS26pzrZcIk/lmt4yWjAqLLcawstyQCh298= github.com/aws/aws-sdk-go-v2/service/s3 v1.102.2/go.mod h1:zjsomFeX5duj+4PlMB+o4JoWTIx+G0XMyzjYrUbQkN0= -github.com/aws/aws-sdk-go-v2/service/signin v1.10.1 h1:DzCCWLzcIRQ77F3DEUljud7bEjTgFOIKXP52NmVRyhU= -github.com/aws/aws-sdk-go-v2/service/signin v1.10.1/go.mod h1:xpo/geVldu8payT375WekctUzopG/hBU7miiqItMUlw= -github.com/aws/aws-sdk-go-v2/service/sso v1.38.1 h1:Umtl/0YZhng4xndfW3lKJrYYP7NLEjI6bGXVomwLcs0= -github.com/aws/aws-sdk-go-v2/service/sso v1.38.1/go.mod h1:rRD/dnm7q0HYE/I5TMaPgkWyyUGLcwuxHLABsLnQ3e0= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.1 h1:orIWdNiLgzrhu/11RcPPKO/SBzUUymbUQuZbSPImghg= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.1/go.mod h1:skwM/xsbR/1ReUTesv9BhpJp1VjajR7DWQnuVLwiXsQ= -github.com/aws/aws-sdk-go-v2/service/sts v1.51.1 h1:0HOqZXRvMytH6bFHVIc0oJX07sZjfhz0zXtjs6gdE8s= -github.com/aws/aws-sdk-go-v2/service/sts v1.51.1/go.mod h1:26zA0GhDrLo+yiLI2yXWxqB1PdsShfLikoI7GOEgugM= +github.com/aws/aws-sdk-go-v2/service/signin v1.10.2 h1:vrPP4O9VABnquYrDHfU+Fu0Fp6TuzaqPUu9Fh7VBBVY= +github.com/aws/aws-sdk-go-v2/service/signin v1.10.2/go.mod h1:xpo/geVldu8payT375WekctUzopG/hBU7miiqItMUlw= +github.com/aws/aws-sdk-go-v2/service/sso v1.38.2 h1:I8F/K5ahKJ4npqeGBBmMPftQy3/o1A55VRq2eUcP9Ts= +github.com/aws/aws-sdk-go-v2/service/sso v1.38.2/go.mod h1:rRD/dnm7q0HYE/I5TMaPgkWyyUGLcwuxHLABsLnQ3e0= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.2 h1:i5nn7Ri5CgFfNRzwlUiZOIIUA+5Vgg+unNEC0PzF1+s= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.2/go.mod h1:skwM/xsbR/1ReUTesv9BhpJp1VjajR7DWQnuVLwiXsQ= +github.com/aws/aws-sdk-go-v2/service/sts v1.51.2 h1:AWSPqynwEJV8J2pl/p7aizCf7a23/NGHR+rK/KLOxi4= +github.com/aws/aws-sdk-go-v2/service/sts v1.51.2/go.mod h1:26zA0GhDrLo+yiLI2yXWxqB1PdsShfLikoI7GOEgugM= github.com/aws/smithy-go v1.28.1 h1:R/nXH00c8qcfCzQVELtRw+eLQWtzv+VAIEFJ1/xxXlQ= github.com/aws/smithy-go v1.28.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= From ba61c450777e100a4d8343d80b86318f5c029f10 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 15 Oct 2026 15:16:44 +0000 Subject: [PATCH 303/633] Bump github.com/prometheus/client_golang from 1.24.1 to 1.25.0 Bumps [github.com/prometheus/client_golang](https://github.com/prometheus/client_golang) from 1.24.1 to 1.25.0. - [Release notes](https://github.com/prometheus/client_golang/releases) - [Changelog](https://github.com/prometheus/client_golang/blob/v1.25.0/CHANGELOG.md) - [Commits](https://github.com/prometheus/client_golang/compare/v1.24.1...v1.25.0) --- updated-dependencies: - dependency-name: github.com/prometheus/client_golang dependency-version: 1.25.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- go.mod | 10 +++++----- go.sum | 20 ++++++++++---------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/go.mod b/go.mod index b76c338..a438de9 100644 --- a/go.mod +++ b/go.mod @@ -24,7 +24,7 @@ require ( github.com/jmoiron/sqlx v1.4.0 github.com/lib/pq v1.12.3 github.com/opencontainers/go-digest v1.0.0 - github.com/prometheus/client_golang v1.24.1 + github.com/prometheus/client_golang v1.25.0 github.com/prometheus/client_model v0.6.3 github.com/spdx/tools-golang v0.5.7 github.com/swaggo/swag v1.16.6 @@ -91,7 +91,7 @@ require ( github.com/googleapis/enterprise-certificate-proxy v0.3.18 // indirect github.com/googleapis/gax-go/v2 v2.23.0 // indirect github.com/josharian/intern v1.0.0 // indirect - github.com/klauspost/compress v1.20.0 // indirect + github.com/klauspost/compress v1.20.1 // indirect github.com/kylelemons/godebug v1.1.0 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-isatty v0.0.24 // indirect @@ -102,8 +102,8 @@ require ( github.com/package-url/packageurl-go v0.1.7 // indirect github.com/pandatix/go-cvss v0.6.4 // indirect github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect - github.com/prometheus/common v0.70.1 // indirect - github.com/prometheus/procfs v0.21.1 // indirect + github.com/prometheus/common v0.72.0 // indirect + github.com/prometheus/procfs v0.22.0 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/rogpeppe/go-internal v1.16.0 // indirect github.com/rs/dnscache v0.0.0-20230804202142-fc85eb664529 // indirect @@ -118,7 +118,7 @@ require ( go.opentelemetry.io/otel/trace v1.45.0 // indirect go.yaml.in/yaml/v3 v3.0.5 // indirect golang.org/x/crypto v0.57.0 // indirect - golang.org/x/oauth2 v0.36.0 // indirect + golang.org/x/oauth2 v0.37.0 // indirect golang.org/x/sys v0.48.0 // indirect golang.org/x/text v0.42.0 // indirect golang.org/x/tools v0.50.0 // indirect diff --git a/go.sum b/go.sum index 3533e64..48b3ad8 100644 --- a/go.sum +++ b/go.sum @@ -208,8 +208,8 @@ github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFF github.com/juju/gnuflag v0.0.0-20171113085948-2ce1bb71843d/go.mod h1:2PavIy+JPciBPrBUjwbNvtwB6RQlve+hkpll6QSNmOE= github.com/keybase/go-keychain v0.0.1 h1:way+bWYa6lDppZoZcgMbYsvC7GxljxrskdNInRtuthU= github.com/keybase/go-keychain v0.0.1/go.mod h1:PdEILRW3i9D8JcdM+FmY6RwkHGnhHxXwkPPMeUgOK1k= -github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA= -github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= +github.com/klauspost/compress v1.20.1 h1:T7kKElXUMXrUJ2E9QhQhxFtcK5rPyLdsGZvdbLMPdiQ= +github.com/klauspost/compress v1.20.1/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= @@ -255,14 +255,14 @@ github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_golang v1.24.1 h1:JnJkREXzWxUdCuPFpIWZiPispT9xVV59uiuyR2bPlnU= -github.com/prometheus/client_golang v1.24.1/go.mod h1:F+oSRECHg4sse5ucfYpYDeIv/hu68Zo0uoHKetWnzcE= +github.com/prometheus/client_golang v1.25.0 h1:cDtW1TcFsMi1RJc8khwCKB4VhzwV04aKqbYtNOXuiK0= +github.com/prometheus/client_golang v1.25.0/go.mod h1:j2X/4/kaQHLPnmyEU8zjmkndo1T317VpCIQEml8LTKk= github.com/prometheus/client_model v0.6.3 h1:O0jaTVAYNxTHYInEPFJt5I3+sN8zqBtVMPTB1qyxiEo= github.com/prometheus/client_model v0.6.3/go.mod h1:gpN5P9S7Rr6Yr92PiQ+Ixvhf6JZEkF1dnxsYL2aPBEM= -github.com/prometheus/common v0.70.1 h1:1HvjP4D5oL3t8RsPlwxA9onvvStjtIHYE5XuuwOi/PY= -github.com/prometheus/common v0.70.1/go.mod h1:VdFUQDMZK3VLkurFUVhia6uys/0suUp86TJz5qbJRhc= -github.com/prometheus/procfs v0.21.1 h1:GljZCt+zSTS+NZq88cyQ1LjZ+RCHp3uVuabBWA5+OJI= -github.com/prometheus/procfs v0.21.1/go.mod h1:aB55Cww9pdSJVHk0hUf0inxWyyjPogFIjmHKYgMKmtY= +github.com/prometheus/common v0.72.0 h1:tAYsE+sPJxIncDAobm4H5aQjmox9ZxEIIqPbiffa8G4= +github.com/prometheus/common v0.72.0/go.mod h1:77NWqAQ2tXT7BIK40qjJdw5Acrsrg1TlHAnsQi3i6mk= +github.com/prometheus/procfs v0.22.0 h1:6q9+/JL9IKAPbCmBrv9n5O5Ty3NKnciV5X7YGw0oics= +github.com/prometheus/procfs v0.22.0/go.mod h1:CvmFr/GVhIjIvWJZW3tgkODBQMRIf0EyWMQLHCHab58= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rogpeppe/go-internal v1.16.0 h1:O9DK+vNMDVGLr2BeZqmpLeMjiMNkuXfcqntWbZV6S5g= @@ -328,8 +328,8 @@ golang.org/x/mod v0.41.0/go.mod h1:Ek9pY8RKWXwsWvd3rQiHYtMqkjSUV+s1Rj7j4H5Ur6o= golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1Kcs5dz7/ng1VjMUvfKvpfy+jM= golang.org/x/net v0.59.0 h1:5zfYln+w5XCxwrnMMJPufRgNoXEaGxl0wo5GqPXyues= golang.org/x/net v0.59.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg= -golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= -golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q= +golang.org/x/oauth2 v0.37.0 h1:JUlcxA8oAtauLfiH8FX2/FkAWHAdi0QtGCGc+hofE98= +golang.org/x/oauth2 v0.37.0/go.mod h1:IxwZNxUULJmpBFf9K/9NTMSIfZZuvuTy1gGxhigP/58= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.23.0 h1:KameEIfc1IkluZyXWLn39Wd4tURc6GbCiISGiZm2bQk= golang.org/x/sync v0.23.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0= From b8d5cf4f27c92f38425b6172b459ea25e168ac0d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 15 Oct 2026 15:33:50 +0000 Subject: [PATCH 304/633] Bump github.com/aws/aws-sdk-go-v2/service/ecr from 1.66.1 to 1.66.2 Bumps [github.com/aws/aws-sdk-go-v2/service/ecr](https://github.com/aws/aws-sdk-go-v2) from 1.66.1 to 1.66.2. - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/service/s3/v1.66.1...service/s3/v1.66.2) --- updated-dependencies: - dependency-name: github.com/aws/aws-sdk-go-v2/service/ecr dependency-version: 1.66.2 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index d807ae6..670f884 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ require ( github.com/CycloneDX/cyclonedx-go v0.12.0 github.com/Masterminds/semver/v3 v3.5.0 github.com/aws/aws-sdk-go-v2/config v1.33.7 - github.com/aws/aws-sdk-go-v2/service/ecr v1.66.1 + github.com/aws/aws-sdk-go-v2/service/ecr v1.66.2 github.com/git-pkgs/archives v0.8.1 github.com/git-pkgs/artifacts v0.2.1 github.com/git-pkgs/cooldown v0.2.0 diff --git a/go.sum b/go.sum index 18cc73f..d2ed244 100644 --- a/go.sum +++ b/go.sum @@ -74,8 +74,8 @@ github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.4 h1:dD4MR81I7YkpEBRk6UP github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.4/go.mod h1:EcXV1kAFd5XwSkDHlj94gnF3q5CkJyYiIJfH8N0VmrE= github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.4 h1:7Wo47d/xn/7KttCSBd8EGYeZ7ULRFRkUHr6vkZPBzVQ= github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.4/go.mod h1:tDB2IVC1xC3vX8o+6uRlzhTxP3g1b77CZXFX/oD2FnQ= -github.com/aws/aws-sdk-go-v2/service/ecr v1.66.1 h1:H63vyEXid/tHpv/UlvQUyM1c2QK5WgQRB3MK5gnAo8A= -github.com/aws/aws-sdk-go-v2/service/ecr v1.66.1/go.mod h1:WglfLchOYcHrYOwNV7jERuy0Xc+7jArLkEnQay93auY= +github.com/aws/aws-sdk-go-v2/service/ecr v1.66.2 h1:SNECRj8ZN0qPG1L33dLnqQ4EfmIQkGcFW7cv4Obw28M= +github.com/aws/aws-sdk-go-v2/service/ecr v1.66.2/go.mod h1:WglfLchOYcHrYOwNV7jERuy0Xc+7jArLkEnQay93auY= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.19 h1:bAdDl/HkGCcGPoe25ToSHEw23VIxt6CT5fLcg111BKg= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.19/go.mod h1:KaUzbLxv4CeSxh6ZCl9B4m7CuFenS8kUEaDs+f/DQr4= github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.18 h1:W/EyPFl9A5rXrtoilfwHYEvzHER+K4SpBPtMXi24Mos= From c0320c7c87a5be942bd8be893b73c3b07a2ea75f Mon Sep 17 00:00:00 2001 From: Christian Heim Date: Sat, 17 Oct 2026 23:20:45 +0200 Subject: [PATCH 305/633] Clearing a record and queueing its file for deletion now happen in one transaction, so a failed queue write keeps the record instead of leaving a file nothing points at. Eviction drops the queue entry again once it has deleted the file itself. --- internal/database/pending_deletes_test.go | 44 +++++++++++++- internal/database/queries.go | 73 ++++++++++++++++------- internal/server/eviction.go | 8 +-- internal/server/eviction_test.go | 55 +++++++++++++++++ 4 files changed, 151 insertions(+), 29 deletions(-) diff --git a/internal/database/pending_deletes_test.go b/internal/database/pending_deletes_test.go index 9af6c48..f124557 100644 --- a/internal/database/pending_deletes_test.go +++ b/internal/database/pending_deletes_test.go @@ -79,7 +79,7 @@ func TestUpsertArtifactSkipsRecordThatMoved(t *testing.T) { UpstreamURL: "https://example.com/" + pendingFilename, StoragePath: sql.NullString{String: "npm/pending/1.0.0/c/pending-1.0.0.tgz", Valid: true}, } - applied, err := db.upsertArtifactFrom(late, read) + applied, err := db.upsertArtifactFrom(db, late, read) if err != nil { t.Fatalf("upsertArtifactFrom: %v", err) } @@ -91,7 +91,7 @@ func TestUpsertArtifactSkipsRecordThatMoved(t *testing.T) { } missing := &Artifact{VersionPURL: "pkg:npm/pending@2.0.0", Filename: pendingFilename, UpstreamURL: "u"} - applied, err = db.upsertArtifactFrom(missing, sql.NullString{}) + applied, err = db.upsertArtifactFrom(db, missing, sql.NullString{}) if err != nil || !applied { t.Errorf("insert of a new record: applied=%v err=%v", applied, err) } @@ -173,6 +173,46 @@ func TestDiscardArtifactClearsAndQueues(t *testing.T) { }) } +// When the replaced path can't be queued, the write is rolled back, so the +// record keeps pointing at the old object instead of orphaning it. +func TestUpsertArtifactRollsBackWhenQueueingFails(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + upsertPendingArtifact(t, db, "npm/pending/1.0.0/a/pending-1.0.0.tgz") + if _, err := db.Exec(`DROP TABLE pending_deletes`); err != nil { + t.Fatalf("dropping pending_deletes: %v", err) + } + + err := db.UpsertArtifact(&Artifact{ + VersionPURL: pendingVersionPURL, + Filename: pendingFilename, + UpstreamURL: "https://example.com/" + pendingFilename, + StoragePath: sql.NullString{String: "npm/pending/1.0.0/b/pending-1.0.0.tgz", Valid: true}, + }) + if err == nil { + t.Fatal("UpsertArtifact succeeded without queueing the replaced path") + } + if got := recordedPath(t, db).String; got != "npm/pending/1.0.0/a/pending-1.0.0.tgz" { + t.Errorf("record points at %q, want the old path kept", got) + } + }) +} + +func TestDiscardArtifactRollsBackWhenQueueingFails(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + upsertPendingArtifact(t, db, "npm/pending/1.0.0/a/pending-1.0.0.tgz") + if _, err := db.Exec(`DROP TABLE pending_deletes`); err != nil { + t.Fatalf("dropping pending_deletes: %v", err) + } + + if err := db.DiscardArtifact(pendingVersionPURL, pendingFilename, "npm/pending/1.0.0/a/pending-1.0.0.tgz"); err == nil { + t.Fatal("DiscardArtifact succeeded without queueing the path") + } + if got := recordedPath(t, db).String; got != "npm/pending/1.0.0/a/pending-1.0.0.tgz" { + t.Errorf("record points at %q, want it kept", got) + } + }) +} + func TestGetDuePendingDeletes(t *testing.T) { runWithBothDatabases(t, func(t *testing.T, db *DB) { for _, path := range []string{"old", "referenced"} { diff --git a/internal/database/queries.go b/internal/database/queries.go index cd7c51c..ea63f15 100644 --- a/internal/database/queries.go +++ b/internal/database/queries.go @@ -9,6 +9,7 @@ import ( "github.com/git-pkgs/artifacts" "github.com/git-pkgs/purl" + "github.com/jmoiron/sqlx" "github.com/opencontainers/go-digest" ) @@ -340,26 +341,42 @@ func (db *DB) UpsertArtifact(a *Artifact) error { if err := db.Get(&previous, query, a.VersionPURL, a.Filename); err != nil && !errors.Is(err, sql.ErrNoRows) { return fmt.Errorf("reading artifact: %w", err) } - applied, err := db.upsertArtifactFrom(a, previous) + applied, err := db.upsertArtifactQueuing(a, previous) if err != nil { return err } - if !applied { - continue + if applied { + return nil } - if previous.Valid && previous.String != a.StoragePath.String { - if err := db.QueuePendingDelete(previous.String); err != nil { - return fmt.Errorf("queueing replaced artifact: %w", err) - } - } - return nil } return errors.New("upserting artifact: record kept changing") } +// upsertArtifactQueuing writes a as upsertArtifactFrom does and queues the +// path it replaced in the same transaction, so a failed queue leaves the +// record on the old path instead of orphaning it. +func (db *DB) upsertArtifactQueuing(a *Artifact, previous sql.NullString) (bool, error) { + tx, err := db.Beginx() + if err != nil { + return false, err + } + defer func() { _ = tx.Rollback() }() + + applied, err := db.upsertArtifactFrom(tx, a, previous) + if err != nil || !applied { + return false, err + } + if previous.Valid && previous.String != a.StoragePath.String { + if _, err := tx.Exec(db.Rebind(queuePendingDeleteQuery), previous.String, time.Now().UTC()); err != nil { + return false, fmt.Errorf("queueing replaced artifact: %w", err) + } + } + return true, tx.Commit() +} + // upsertArtifactFrom writes a if the record is absent or still holds // previous, and reports whether it did. -func (db *DB) upsertArtifactFrom(a *Artifact, previous sql.NullString) (bool, error) { +func (db *DB) upsertArtifactFrom(ex sqlx.Execer, a *Artifact, previous sql.NullString) (bool, error) { now := time.Now() var query string @@ -401,7 +418,7 @@ func (db *DB) upsertArtifactFrom(a *Artifact, previous sql.NullString) (bool, er ` } - res, err := db.Exec(query, + res, err := ex.Exec(query, a.VersionPURL, a.Filename, a.UpstreamURL, a.StoragePath, a.ContentHash, a.Size, a.ContentType, inUTC(a.FetchedAt), a.HitCount, inUTC(a.LastAccessedAt), now, now, a.sourceOrUpstream(), a.Manifest, previous, @@ -482,33 +499,43 @@ func (db *DB) GetCachedArtifactCount() (int64, error) { } // ClearArtifactCache marks an artifact uncached if its record still points at -// storagePath, and reports whether it did. A record a newer fetch has moved -// elsewhere is left alone, so a clear never orphans the object that fetch -// committed. +// storagePath, queues storagePath for deletion in the same transaction, and +// reports whether it did. A record a newer fetch has moved elsewhere is left +// alone, so a clear never orphans the object that fetch committed. If +// queueing fails, the record keeps its path. func (db *DB) ClearArtifactCache(versionPURL, filename, storagePath string) (bool, error) { + tx, err := db.Beginx() + if err != nil { + return false, err + } + defer func() { _ = tx.Rollback() }() + query := db.Rebind(` UPDATE artifacts SET storage_path = NULL, content_hash = NULL, size = NULL, content_type = NULL, fetched_at = NULL, updated_at = ? WHERE version_purl = ? AND filename = ? AND storage_path = ? `) - res, err := db.Exec(query, time.Now(), versionPURL, filename, storagePath) + res, err := tx.Exec(query, time.Now(), versionPURL, filename, storagePath) if err != nil { return false, err } n, err := res.RowsAffected() - return n > 0, err + if err != nil || n == 0 { + return false, err + } + if _, err := tx.Exec(db.Rebind(queuePendingDeleteQuery), storagePath, time.Now().UTC()); err != nil { + return false, fmt.Errorf("queueing %s for deletion: %w", storagePath, err) + } + return true, tx.Commit() } -// DiscardArtifact clears the record as ClearArtifactCache does and queues -// storagePath for deletion, for callers that must not delete an object another +// DiscardArtifact clears the record and queues storagePath as +// ClearArtifactCache does, for callers that must not delete an object another // request may be reading. func (db *DB) DiscardArtifact(versionPURL, filename, storagePath string) error { - cleared, err := db.ClearArtifactCache(versionPURL, filename, storagePath) - if err != nil || !cleared { - return err - } - return db.QueuePendingDelete(storagePath) + _, err := db.ClearArtifactCache(versionPURL, filename, storagePath) + return err } // QueuePendingDelete queues a storage path no record points at any more. diff --git a/internal/server/eviction.go b/internal/server/eviction.go index 7fe108a..fe69db2 100644 --- a/internal/server/eviction.go +++ b/internal/server/eviction.go @@ -120,6 +120,7 @@ func evictBatch(ctx context.Context, db *database.DB, store storage.Storage, log // Clear before deleting: a record a newer fetch moved has left this // path queued, and a request may still open it within the grace period. + // The clear queues the path too, so it is never left unrecorded. recordCleared, err := db.ClearArtifactCache(art.VersionPURL, art.Filename, path) if err != nil { logger.Warn("eviction: failed to clear artifact record", @@ -131,10 +132,9 @@ func evictBatch(ctx context.Context, db *database.DB, store storage.Storage, log } if err := store.Delete(ctx, path); err != nil { - logger.Warn("eviction: failed to delete from storage, queueing it", "path", path, "error", err) - if err := db.QueuePendingDelete(path); err != nil { - logger.Warn("eviction: failed to queue object for deletion", "path", path, "error", err) - } + logger.Warn("eviction: failed to delete from storage, leaving it queued", "path", path, "error", err) + } else if err := db.RemovePendingDelete(path); err != nil { + logger.Warn("eviction: failed to dequeue deleted object", "path", path, "error", err) } if art.Size.Valid { diff --git a/internal/server/eviction_test.go b/internal/server/eviction_test.go index 1535586..8730296 100644 --- a/internal/server/eviction_test.go +++ b/internal/server/eviction_test.go @@ -487,3 +487,58 @@ func TestEvictBatch_SkipsRecordThatMoved(t *testing.T) { t.Errorf("queue = %v, want the old path kept for reclaim", got) } } + +// TestEvictBatch_KeepsRecordWhenQueueingFails evicts while the queue refuses +// writes. The record must keep its path and the object stay, or a crash before +// the delete would leave a file nothing points at. +func TestEvictBatch_KeepsRecordWhenQueueingFails(t *testing.T) { + db, store := setupEvictionTest(t) + ctx := context.Background() + logger := slog.New(slog.NewTextHandler(io.Discard, nil)) + seedArtifact(t, ctx, db, store, "unqueued", 1000, time.Now().Add(-time.Hour)) + + rows, err := db.GetLeastRecentlyUsedArtifacts(evictionBatch) + if err != nil || len(rows) != 1 { + t.Fatalf("reading LRU rows: %v, %v", rows, err) + } + if _, err := db.Exec(`CREATE TRIGGER refuse_queue BEFORE INSERT ON pending_deletes + BEGIN SELECT RAISE(ABORT, 'queue refused'); END`); err != nil { + t.Fatalf("creating trigger: %v", err) + } + + cleared, freed := evictBatch(ctx, db, store, logger, rows, 0, 1000) + if cleared != 0 || freed != 0 { + t.Errorf("cleared %d records freeing %d bytes, want nothing", cleared, freed) + } + path := rows[0].StoragePath.String + record, err := db.GetArtifact(rows[0].VersionPURL, rows[0].Filename) + if err != nil || record == nil || record.StoragePath.String != path { + t.Fatalf("record = %+v (err %v), want it kept at %q", record, err, path) + } + if ok, _ := store.Exists(ctx, path); !ok { + t.Error("object deleted although its record was kept") + } +} + +// TestEvictBatch_DequeuesDeletedObjects checks that an object eviction deleted +// does not stay in the queue for reclaim to delete again. +func TestEvictBatch_DequeuesDeletedObjects(t *testing.T) { + db, store := setupEvictionTest(t) + ctx := context.Background() + logger := slog.New(slog.NewTextHandler(io.Discard, nil)) + seedArtifact(t, ctx, db, store, "gone", 1000, time.Now().Add(-time.Hour)) + + rows, err := db.GetLeastRecentlyUsedArtifacts(evictionBatch) + if err != nil || len(rows) != 1 { + t.Fatalf("reading LRU rows: %v, %v", rows, err) + } + if cleared, _ := evictBatch(ctx, db, store, logger, rows, 0, 1000); cleared != 1 { + t.Fatalf("cleared %d records, want 1", cleared) + } + if ok, _ := store.Exists(ctx, rows[0].StoragePath.String); ok { + t.Error("object still in storage") + } + if got := queuedPaths(t, db); len(got) != 0 { + t.Errorf("queue = %v, want it empty", got) + } +} From 83e2fb4bdaf69997c22841fb27beaf5facb47420 Mon Sep 17 00:00:00 2001 From: Christian Heim Date: Sun, 18 Oct 2026 19:31:46 +0200 Subject: [PATCH 306/633] Make the Gradle build cache read-only by default, so a client that can reach the proxy can no longer overwrite cache entries; CI jobs that upload now need gradle.build_cache.read_only: false (or PROXY_GRADLE_BUILD_CACHE_READ_ONLY=false). The environment variable is parsed strictly, so a value like "yes" keeps uploads off and logs a warning instead of opening them. Fixes #497. (#509) --- README.md | 6 +- cmd/proxy/main.go | 4 +- config.example.yaml | 5 +- docs/configuration.md | 6 +- internal/config/config.go | 30 ++++++++- internal/config/config_test.go | 67 +++++++++++++++++++- internal/server/dashboard.go | 7 +- internal/server/gradle_read_only_test.go | 81 ++++++++++++++++++++++++ internal/server/server.go | 3 + 9 files changed, 193 insertions(+), 16 deletions(-) create mode 100644 internal/server/gradle_read_only_test.go diff --git a/README.md b/README.md index 559f1a8..3ec9f3d 100644 --- a/README.md +++ b/README.md @@ -302,7 +302,7 @@ pluginManagement { ### Gradle HTTP Build Cache -Configure in `settings.gradle(.kts)`: +Configure in `settings.gradle.kts`: ```kotlin buildCache { @@ -311,11 +311,13 @@ buildCache { } remote { url = uri("http://localhost:8080/gradle/") - push = true + isPush = System.getenv("CI") != null } } ``` +The proxy rejects uploads unless `gradle.build_cache.read_only` is set to `false`, see [docs/configuration.md](docs/configuration.md#gradle-build-cache). + ### NuGet Configure in `nuget.config`: diff --git a/cmd/proxy/main.go b/cmd/proxy/main.go index 75f0097..f6e7f95 100644 --- a/cmd/proxy/main.go +++ b/cmd/proxy/main.go @@ -82,7 +82,7 @@ // PROXY_LOG_FORMAT - Log format // PROXY_ACCESS_LOG_PATH - JSONL access log path // PROXY_UPSTREAM_* - Upstream URLs and network access controls -// PROXY_GRADLE_BUILD_CACHE_READ_ONLY - Disable Gradle PUT uploads +// PROXY_GRADLE_BUILD_CACHE_READ_ONLY - Set false to allow Gradle PUT uploads (default true) // PROXY_GRADLE_BUILD_CACHE_MAX_UPLOAD_SIZE - Max Gradle PUT request body size // PROXY_GRADLE_BUILD_CACHE_MAX_AGE - Gradle cache max age eviction // PROXY_GRADLE_BUILD_CACHE_MAX_SIZE - Gradle cache max total size @@ -257,7 +257,7 @@ func runServe() { fmt.Fprintf(os.Stderr, " PROXY_UPSTREAM_OCI_DEFAULT Default OCI registry upstream URL\n") fmt.Fprintf(os.Stderr, " PROXY_UPSTREAM_DEBIAN Debian repository upstream URL\n") fmt.Fprintf(os.Stderr, " PROXY_UPSTREAM_RPM RPM repository upstream URL\n") - fmt.Fprintf(os.Stderr, " PROXY_GRADLE_BUILD_CACHE_READ_ONLY Disable Gradle PUT uploads\n") + fmt.Fprintf(os.Stderr, " PROXY_GRADLE_BUILD_CACHE_READ_ONLY Set false to allow Gradle PUT uploads (default true)\n") fmt.Fprintf(os.Stderr, " PROXY_GRADLE_BUILD_CACHE_MAX_UPLOAD_SIZE Max Gradle PUT request body size\n") fmt.Fprintf(os.Stderr, " PROXY_GRADLE_BUILD_CACHE_MAX_AGE Gradle cache max age eviction\n") fmt.Fprintf(os.Stderr, " PROXY_GRADLE_BUILD_CACHE_MAX_SIZE Gradle cache max total size\n") diff --git a/config.example.yaml b/config.example.yaml index accae89..326f918 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -347,8 +347,9 @@ upstream: # Gradle HttpBuildCache configuration gradle: build_cache: - # Set to true to disable PUT uploads (read-only cache mode) - read_only: false + # PUT uploads are rejected by default; set to false on the instance + # your CI jobs upload to + read_only: true # Maximum accepted Gradle cache upload body size # Required and must be > 0 diff --git a/docs/configuration.md b/docs/configuration.md index ffd57fe..53cffe7 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -674,7 +674,7 @@ The `/gradle` endpoint supports optional safeguards for upload control and cache ```yaml gradle: build_cache: - read_only: false + read_only: true max_upload_size: "100MB" max_age: "168h" max_size: "20GB" @@ -683,12 +683,14 @@ gradle: | Config | Environment | Description | |--------|-------------|-------------| -| `gradle.build_cache.read_only` | `PROXY_GRADLE_BUILD_CACHE_READ_ONLY` | Disable PUT uploads and keep GET/HEAD read-only | +| `gradle.build_cache.read_only` | `PROXY_GRADLE_BUILD_CACHE_READ_ONLY` | Reject PUT uploads and serve GET/HEAD only (default `true`; the variable takes `true`/`false` or `1`/`0`, any other value keeps uploads off and logs a warning) | | `gradle.build_cache.max_upload_size` | `PROXY_GRADLE_BUILD_CACHE_MAX_UPLOAD_SIZE` | Maximum accepted PUT body size (must be > 0) | | `gradle.build_cache.max_age` | `PROXY_GRADLE_BUILD_CACHE_MAX_AGE` | Delete entries older than this duration (default `168h`, also accepts days such as `7d`; set `0` to disable) | | `gradle.build_cache.max_size` | `PROXY_GRADLE_BUILD_CACHE_MAX_SIZE` | Total size cap for `_gradle/http-build-cache`, deleting oldest first (`0` disables) | | `gradle.build_cache.sweep_interval` | `PROXY_GRADLE_BUILD_CACHE_SWEEP_INTERVAL` | Frequency for background eviction sweeps | +Uploads are off by default, so a client that can reach the proxy cannot overwrite cache entries that end up in other people's builds. Set `read_only: false` (or `PROXY_GRADLE_BUILD_CACHE_READ_ONLY=false`) on the instance your CI jobs upload to, and let only CI push (`isPush = System.getenv("CI") != null`). Before this default changed, uploads were accepted without configuration; setups that upload need the setting now. + `max_age` and `max_size` are independent and can be combined. When both are set, age-based eviction runs first, then size-based eviction trims remaining entries oldest-first. ## Cooldown diff --git a/internal/config/config.go b/internal/config/config.go index a9d7c21..b4ad942 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -198,6 +198,9 @@ type Config struct { // Health configures the /health endpoint behavior. Health HealthConfig `json:"health" yaml:"health"` + + // envWarnings collects environment values LoadFromEnv ignored. + envWarnings []string } // CooldownConfig configures version cooldown periods. @@ -463,6 +466,7 @@ type GradleConfig struct { // GradleBuildCacheConfig configures Gradle HttpBuildCache safeguards. type GradleBuildCacheConfig struct { // ReadOnly disables PUT uploads and keeps cache reads (GET/HEAD) enabled. + // Default: true; set false on the instance that CI uploads to. ReadOnly bool `json:"read_only" yaml:"read_only"` // MaxUploadSize caps a single PUT body size (e.g., "100MB"). Must be > 0. @@ -1064,7 +1068,7 @@ func Default() *Config { }, Gradle: GradleConfig{ BuildCache: GradleBuildCacheConfig{ - ReadOnly: false, + ReadOnly: true, MaxUploadSize: defaultGradleMaxUploadSizeStr, MaxAge: "168h", MaxSize: "", @@ -1120,6 +1124,28 @@ func setEnvBool(dst *bool, key string) { } } +// setGradleReadOnlyFromEnv parses strictly: an unrecognised value keeps +// uploads off instead of opening them, and is reported via EnvWarnings. +func (c *Config) setGradleReadOnlyFromEnv() { + const key = "PROXY_GRADLE_BUILD_CACHE_READ_ONLY" + v := os.Getenv(key) + if v == "" { + return + } + readOnly, err := strconv.ParseBool(strings.TrimSpace(v)) + if err != nil { + c.Gradle.BuildCache.ReadOnly = true + c.envWarnings = append(c.envWarnings, fmt.Sprintf("ignoring %s=%q: not a boolean, keeping the Gradle build cache read-only", key, v)) + return + } + c.Gradle.BuildCache.ReadOnly = readOnly +} + +// EnvWarnings returns the environment values LoadFromEnv ignored. +func (c *Config) EnvWarnings() []string { + return c.envWarnings +} + func setEnvStringSlice(dst *[]string, key string) { value := os.Getenv(key) if value == "" { @@ -1222,7 +1248,7 @@ func (c *Config) LoadFromEnv() { setEnvString(&c.ArtifactMaxSize, "PROXY_ARTIFACT_MAX_SIZE") setEnvString(&c.MetadataRewriteCacheSize, "PROXY_METADATA_REWRITE_CACHE_SIZE") setEnvString(&c.HTTPTimeout, "PROXY_HTTP_TIMEOUT") - setEnvBool(&c.Gradle.BuildCache.ReadOnly, "PROXY_GRADLE_BUILD_CACHE_READ_ONLY") + c.setGradleReadOnlyFromEnv() setEnvString(&c.Gradle.BuildCache.MaxUploadSize, "PROXY_GRADLE_BUILD_CACHE_MAX_UPLOAD_SIZE") setEnvString(&c.Gradle.BuildCache.MaxAge, "PROXY_GRADLE_BUILD_CACHE_MAX_AGE") setEnvString(&c.Gradle.BuildCache.MaxSize, "PROXY_GRADLE_BUILD_CACHE_MAX_SIZE") diff --git a/internal/config/config_test.go b/internal/config/config_test.go index e3b0a2c..4750e5c 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -138,6 +138,9 @@ func TestDefault(t *testing.T) { if cfg.Gradle.BuildCache.MaxAge != "168h" { t.Errorf("Gradle.BuildCache.MaxAge = %q, want %q", cfg.Gradle.BuildCache.MaxAge, "168h") } + if !cfg.Gradle.BuildCache.ReadOnly { + t.Error("Gradle.BuildCache.ReadOnly = false, want true by default") + } if len(cfg.Upstream.AllowPrivateHosts) != 0 { t.Errorf("Upstream.AllowPrivateHosts = %v, want empty", cfg.Upstream.AllowPrivateHosts) } @@ -417,6 +420,64 @@ upstream: } } +func TestLoadGradleBuildCacheReadOnly(t *testing.T) { + tests := []struct { + name string + content string + want bool + }{ + {"block without read_only keeps default", "gradle:\n build_cache:\n max_age: \"12h\"\n", true}, + {"explicit false allows uploads", "gradle:\n build_cache:\n read_only: false\n", false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + path := filepath.Join(t.TempDir(), "config.yaml") + if err := os.WriteFile(path, []byte(tt.content), 0644); err != nil { + t.Fatalf("writing config file: %v", err) + } + cfg, err := Load(path) + if err != nil { + t.Fatalf("Load() error = %v", err) + } + if cfg.Gradle.BuildCache.ReadOnly != tt.want { + t.Errorf("Gradle.BuildCache.ReadOnly = %v, want %v", cfg.Gradle.BuildCache.ReadOnly, tt.want) + } + }) + } +} + +func TestLoadGradleBuildCacheReadOnlyFromEnv(t *testing.T) { + tests := []struct { + value string + fileRO bool + want bool + warnings int + }{ + {"false", true, false, 0}, + {"0", true, false, 0}, + {" FALSE ", true, false, 0}, + {"True", false, true, 0}, + {"1", false, true, 0}, + {"yes", false, true, 1}, + {"off", false, true, 1}, + {"", false, false, 0}, + } + for _, tt := range tests { + t.Run(tt.value, func(t *testing.T) { + t.Setenv("PROXY_GRADLE_BUILD_CACHE_READ_ONLY", tt.value) + cfg := Default() + cfg.Gradle.BuildCache.ReadOnly = tt.fileRO + cfg.LoadFromEnv() + if cfg.Gradle.BuildCache.ReadOnly != tt.want { + t.Errorf("Gradle.BuildCache.ReadOnly = %v, want %v", cfg.Gradle.BuildCache.ReadOnly, tt.want) + } + if got := len(cfg.EnvWarnings()); got != tt.warnings { + t.Errorf("len(EnvWarnings()) = %d, want %d: %v", got, tt.warnings, cfg.EnvWarnings()) + } + }) + } +} + func TestLoadJSON(t *testing.T) { dir := t.TempDir() path := filepath.Join(dir, "config.json") @@ -466,7 +527,7 @@ func TestLoadFromEnv(t *testing.T) { t.Setenv("PROXY_UI_FAILURE_DETAILS", "true") t.Setenv("PROXY_UPSTREAM_ALLOW_PRIVATE_HOSTS", "registry.internal, 10.0.0.12") t.Setenv("PROXY_UPSTREAM_ALLOW_LOOPBACK", "true") - t.Setenv("PROXY_GRADLE_BUILD_CACHE_READ_ONLY", "true") + t.Setenv("PROXY_GRADLE_BUILD_CACHE_READ_ONLY", "false") t.Setenv("PROXY_GRADLE_BUILD_CACHE_MAX_UPLOAD_SIZE", "32MB") t.Setenv("PROXY_GRADLE_BUILD_CACHE_MAX_AGE", "12h") t.Setenv("PROXY_GRADLE_BUILD_CACHE_MAX_SIZE", "10GB") @@ -517,8 +578,8 @@ func TestLoadFromEnv(t *testing.T) { t.Errorf("Upstream.AllowPrivateHosts after empty env = %q, want unchanged", got) } - if !cfg.Gradle.BuildCache.ReadOnly { - t.Error("Gradle.BuildCache.ReadOnly = false, want true") + if cfg.Gradle.BuildCache.ReadOnly { + t.Error("Gradle.BuildCache.ReadOnly = true, want false from the environment") } if cfg.Gradle.BuildCache.MaxUploadSize != "32MB" { t.Errorf("Gradle.BuildCache.MaxUploadSize = %q, want %q", cfg.Gradle.BuildCache.MaxUploadSize, "32MB") diff --git a/internal/server/dashboard.go b/internal/server/dashboard.go index dbc3a69..69f352a 100644 --- a/internal/server/dashboard.go +++ b/internal/server/dashboard.go @@ -359,13 +359,14 @@ index-url = ` + baseURL + `/pypi/simple/`), Language: "Java/Kotlin", Endpoint: "/gradle/", Instructions: template.HTML(`

Configure Gradle to use the proxy for HttpBuildCache:

-
// In settings.gradle(.kts)
+
// In settings.gradle.kts
 buildCache {
   remote<HttpBuildCache> {
     url = uri("` + baseURL + `/gradle/")
-    push = true
+    isPush = System.getenv("CI") != null
   }
-}
`),
+}
+

Uploads need gradle.build_cache.read_only: false on the proxy.

`), }, { ID: "nuget", diff --git a/internal/server/gradle_read_only_test.go b/internal/server/gradle_read_only_test.go new file mode 100644 index 0000000..0911bff --- /dev/null +++ b/internal/server/gradle_read_only_test.go @@ -0,0 +1,81 @@ +package server + +import ( + "context" + "io" + "log/slog" + "net" + "net/http" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/git-pkgs/proxy/internal/config" +) + +func startGradleServer(t *testing.T, readOnly bool) string { + t.Helper() + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("reserving proxy address: %v", err) + } + t.Cleanup(func() { _ = listener.Close() }) + + tempDir := t.TempDir() + cfg := config.Default() + cfg.Listen = listener.Addr().String() + cfg.BaseURL = "http://" + cfg.Listen + cfg.Database.Path = filepath.Join(tempDir, "proxy.db") + cfg.Storage.URL = "file://" + filepath.Join(tempDir, "artifacts") + cfg.Gradle.BuildCache.ReadOnly = readOnly + if err := cfg.Validate(); err != nil { + t.Fatalf("validating config: %v", err) + } + + proxyServer, err := New(cfg, slog.New(slog.NewTextHandler(io.Discard, nil)), BuildInfo{Version: "test", Commit: "test"}) + if err != nil { + t.Fatalf("creating server: %v", err) + } + startErr := make(chan error, 1) + go func() { startErr <- proxyServer.Start(listener) }() + t.Cleanup(func() { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + _ = proxyServer.Shutdown(ctx) + <-startErr + }) + return cfg.BaseURL +} + +func gradleRequest(t *testing.T, method, url, body string) int { + t.Helper() + req, err := http.NewRequest(method, url, strings.NewReader(body)) + if err != nil { + t.Fatalf("building request: %v", err) + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatalf("%s %s: %v", method, url, err) + } + _ = resp.Body.Close() + return resp.StatusCode +} + +func TestGradleBuildCacheReadOnlyFollowsConfig(t *testing.T) { + readOnly := startGradleServer(t, config.Default().Gradle.BuildCache.ReadOnly) + if got := gradleRequest(t, http.MethodPut, readOnly+"/gradle/abc123", "entry"); got != http.StatusMethodNotAllowed { + t.Errorf("PUT with default config = %d, want %d", got, http.StatusMethodNotAllowed) + } + if got := gradleRequest(t, http.MethodGet, readOnly+"/gradle/abc123", ""); got != http.StatusNotFound { + t.Errorf("GET after rejected PUT = %d, want %d", got, http.StatusNotFound) + } + + writable := startGradleServer(t, false) + if got := gradleRequest(t, http.MethodPut, writable+"/gradle/abc123", "entry"); got != http.StatusCreated { + t.Errorf("PUT with read_only false = %d, want %d", got, http.StatusCreated) + } + if got := gradleRequest(t, http.MethodGet, writable+"/gradle/abc123", ""); got != http.StatusOK { + t.Errorf("GET after PUT = %d, want %d", got, http.StatusOK) + } +} diff --git a/internal/server/server.go b/internal/server/server.go index d9699cd..f9bdf5d 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -379,6 +379,9 @@ func (s *Server) serve(listener net.Listener) error { "ui_url", s.cfg.UIBaseURL, "storage", s.storage.URL(), "database", s.cfg.Database.String()) + for _, warning := range s.cfg.EnvWarnings() { + s.logger.Warn(warning) + } go s.updateCacheStatsMetrics() go s.startEvictionLoop(bgCtx) go s.startRetentionLoop(bgCtx) From 8d96791f1a5083e754f341e521d0232401259582 Mon Sep 17 00:00:00 2001 From: Christian Heim Date: Sun, 18 Oct 2026 19:33:13 +0200 Subject: [PATCH 307/633] Re-mirror container images on a schedule (#508) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add mirror.schedules, a list of image tags the server re-mirrors at a fixed interval. Digests are rejected since they never move. * Run each schedule through the normal mirror, one run after the other. The first run starts at a random point within ten minutes and later runs get ±10 % jitter, so several instances don't hit the registry at once. * Start the schedules with the server and let shutdown wait for a running refresh before storage and database are closed. Docs included. * Check cached layers during a mirror without opening them, so a refresh no longer counts as a cache hit and doesn't keep scheduled images at the top of LRU eviction. --- config.example.yaml | 11 +- docs/configuration.md | 29 +++- internal/config/config.go | 9 ++ internal/config/mirror.go | 108 +++++++++++++ internal/config/mirror_test.go | 89 +++++++++++ internal/handler/container_mirror.go | 7 +- internal/handler/container_mirror_test.go | 53 +++++++ internal/handler/handler.go | 28 ++++ internal/mirror/schedule.go | 105 +++++++++++++ internal/mirror/schedule_test.go | 177 ++++++++++++++++++++++ internal/server/mirror_schedule.go | 47 ++++++ internal/server/mirror_schedule_test.go | 98 ++++++++++++ internal/server/server.go | 4 + 13 files changed, 758 insertions(+), 7 deletions(-) create mode 100644 internal/config/mirror.go create mode 100644 internal/config/mirror_test.go create mode 100644 internal/mirror/schedule.go create mode 100644 internal/mirror/schedule_test.go create mode 100644 internal/server/mirror_schedule.go create mode 100644 internal/server/mirror_schedule_test.go diff --git a/config.example.yaml b/config.example.yaml index 326f918..95b8002 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -103,7 +103,8 @@ storage: # Store fetched artifacts. Set to false to stream every download from # upstream without storing it; metadata filtering, cooldown and the # denylist still apply. Useful when another cache sits in front of the - # proxy. false is incompatible with direct_serve, scanning and mirror_api. + # proxy. false is incompatible with direct_serve, scanning, mirror_api and + # mirror.schedules. cache_artifacts: true # Redirect cached artifact downloads to presigned storage URLs (HTTP 302) @@ -377,6 +378,14 @@ gradle: # npm_scopes: ["@acme"] # composer_vendors: ["acme"] +# Re-mirror container image tags on a schedule, so a moved tag is picked up +# even when nobody pulls it. An unchanged tag costs one HEAD request. +# mirror: +# schedules: +# - images: ["pkg:docker/library/alpine@3.20", "pkg:docker/library/nginx@stable"] +# interval: "6h" +# platforms: ["linux/amd64"] + # Health endpoint configuration. health: # Minimum time between storage backend probes. diff --git a/docs/configuration.md b/docs/configuration.md index 53cffe7..06528ad 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -63,7 +63,7 @@ storage: Every download is a fresh upstream fetch, and concurrent requests for the same artifact are not combined. Artifacts with a digest known up front (OCI blobs, Swift archives, Helm charts) are verified while streaming: the response is sent chunked, and on a mismatch the connection is aborted before the response completes so the client never receives a tampered artifact as a good one. The same happens when the upstream connection fails mid-download. -`cache_artifacts: false` cannot be combined with `scanning.enabled`, `storage.direct_serve` or `mirror_api`, which all need stored artifacts, and the `mirror` command refuses to run with it. +`cache_artifacts: false` cannot be combined with `scanning.enabled`, `storage.direct_serve`, `mirror_api` or `mirror.schedules`, which all need stored artifacts, and the `mirror` command refuses to run with it. ### Retention @@ -1143,6 +1143,33 @@ proxy mirror 'pkg:oci/debian@sha256:...?repository_url=docker.io/library/debian& - Non-distributable (foreign) layers are skipped. - While `scanning.enabled` is set, the command refuses images, because scanners fetch staged layers from the running server. Use the mirror API instead. +### Scheduled refresh + +Tags such as `alpine:3.20` move. The server can re-mirror them on its own, so the cache follows the tag even when nobody pulls it or the proxy is offline later: + +```yaml +mirror: + schedules: + - images: [pkg:docker/library/alpine@3.20, pkg:docker/library/nginx@stable] + interval: 6h + platforms: [linux/amd64] +``` + +| Setting | Env | Description | +|---------|-----|-------------| +| `mirror.schedules[].images` | - | `pkg:docker` or `pkg:oci` PURLs with a tag, resolved like the `mirror` command | +| `mirror.schedules[].interval` | - | How often the images are mirrored, at least `1m` | +| `mirror.schedules[].platforms` | - | `os/arch[/variant]` entries limiting the platforms; empty mirrors all | + +Each run checks the tag with a HEAD request. If the tag still points to the cached digest, nothing else is fetched. If it moved, the new manifests and blobs are fetched first and the tag is switched only when all of them are cached, so a failed run leaves the previous image pullable. Images of one schedule are mirrored one after the other. + +- The first run starts within ten minutes of startup, later runs every `interval` ±10 %, so restarts and several instances don't hit the registry at once. Each instance runs the schedules on its own. +- Digest-pinned PURLs never move and are rejected. A `pkg:oci` digest with `tag=` is accepted; the tag is refreshed and a warning is logged when it no longer matches the digest. +- The previous digest stays cached, so pulls by digest keep working, until eviction or retention removes it. +- Docker Hub does not count HEAD requests against its pull limit, but every changed manifest is a pull. Set `platforms` and `upstream.auth` for `registry-1.docker.io` to stay under the anonymous limit. +- Mirrored layers are scanned when `scanning.enabled` is set. A blocked layer fails the run and keeps the previous image. +- Schedules are read at startup; changes need a restart. They do not need `mirror_api`. + ## Docker ### SQLite with Local Storage diff --git a/internal/config/config.go b/internal/config/config.go index b4ad942..3509d18 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -190,6 +190,9 @@ type Config struct { // Disabled by default to prevent unauthenticated users from triggering downloads. MirrorAPI bool `json:"mirror_api" yaml:"mirror_api"` + // Mirror configures scheduled mirroring. + Mirror MirrorConfig `json:"mirror" yaml:"mirror"` + // Hosted configures pushing packages to the proxy (/api/hosted). Hosted HostedConfig `json:"hosted" yaml:"hosted"` @@ -1369,6 +1372,8 @@ func (c *Config) validateCacheArtifacts() error { return fmt.Errorf("storage.cache_artifacts: false cannot be combined with storage.direct_serve: no artifacts are stored to redirect to") case c.MirrorAPI: return fmt.Errorf("storage.cache_artifacts: false cannot be combined with mirror_api: mirrored artifacts would never be served") + case len(c.Mirror.Schedules) > 0: + return fmt.Errorf("storage.cache_artifacts: false cannot be combined with mirror.schedules: mirrored artifacts would never be served") case c.Hosted.Enabled: return fmt.Errorf("storage.cache_artifacts: false cannot be combined with hosted.enabled: pushed packages would never be served") } @@ -1404,6 +1409,10 @@ func (c *Config) validateComponents() error { return err } + if err := c.Mirror.Validate(); err != nil { + return err + } + return c.Gradle.BuildCache.Validate() } diff --git a/internal/config/mirror.go b/internal/config/mirror.go new file mode 100644 index 0000000..7f6c668 --- /dev/null +++ b/internal/config/mirror.go @@ -0,0 +1,108 @@ +package config + +import ( + "fmt" + "strings" + "time" + + "github.com/git-pkgs/purl" +) + +const minMirrorScheduleInterval = time.Minute + +// MirrorConfig configures mirroring that the server runs on its own. +type MirrorConfig struct { + // Schedules re-mirror container image tags at a fixed interval, so a + // moved tag is picked up even when nobody pulls it. + Schedules []MirrorSchedule `json:"schedules" yaml:"schedules"` +} + +// MirrorSchedule is a set of image PURLs mirrored every Interval. +type MirrorSchedule struct { + // Images are pkg:docker or pkg:oci PURLs with a tag. + Images []string `json:"images" yaml:"images"` + + // Interval is how often the images are mirrored, at least one minute + // (e.g. "6h"). + Interval string `json:"interval" yaml:"interval"` + + // Platforms limit which platforms of an image index are mirrored, as + // os/architecture[/variant]. Empty mirrors every platform. + Platforms []string `json:"platforms" yaml:"platforms"` + + interval time.Duration +} + +// IntervalDuration returns the parsed Interval. Validate must run first. +func (s *MirrorSchedule) IntervalDuration() time.Duration { + return s.interval +} + +// Validate checks every schedule. +func (m *MirrorConfig) Validate() error { + for i := range m.Schedules { + if err := m.Schedules[i].validate(fmt.Sprintf("mirror.schedules[%d]", i)); err != nil { + return err + } + } + return nil +} + +func (s *MirrorSchedule) validate(field string) error { + d, err := time.ParseDuration(s.Interval) + if err != nil { + return fmt.Errorf("invalid %s.interval %q: %w", field, s.Interval, err) + } + if d < minMirrorScheduleInterval { + return fmt.Errorf("invalid %s.interval %q: must be at least %s", field, s.Interval, minMirrorScheduleInterval) + } + s.interval = d + + if len(s.Images) == 0 { + return fmt.Errorf("%s.images must not be empty", field) + } + for _, image := range s.Images { + if err := validateScheduledImage(image); err != nil { + return fmt.Errorf("invalid %s.images entry %q: %w", field, image, err) + } + } + for _, platform := range s.Platforms { + if err := validatePlatform(platform); err != nil { + return fmt.Errorf("invalid %s.platforms entry %q: %w", field, platform, err) + } + } + return nil +} + +func validateScheduledImage(image string) error { + p, err := purl.Parse(image) + if err != nil { + return err + } + if p.Type != "docker" && p.Type != "oci" { + return fmt.Errorf("only pkg:docker and pkg:oci images can be scheduled") + } + if p.Version == "" { + return fmt.Errorf("a tag is required") + } + repositoryURL := p.Qualifiers.Map()["repository_url"] + if p.Type == "oci" && repositoryURL == "" { + return fmt.Errorf("pkg:oci needs a repository_url qualifier") + } + // Only pkg:oci mirrors the tag qualifier next to the digest. + pinned := strings.HasPrefix(p.Version, "sha256:") + if pinned && (p.Type == "docker" || p.Qualifiers.Map()["tag"] == "") { + return fmt.Errorf("a digest never moves, so there is nothing to refresh; use a tag") + } + return nil +} + +// validatePlatform matches handler.ParsePlatform, which config cannot import. +func validatePlatform(value string) error { + os, rest, _ := strings.Cut(value, "/") + architecture, variant, _ := strings.Cut(rest, "/") + if os == "" || architecture == "" || strings.Contains(variant, "/") || strings.Contains(value, ",") { + return fmt.Errorf("want os/architecture[/variant]") + } + return nil +} diff --git a/internal/config/mirror_test.go b/internal/config/mirror_test.go new file mode 100644 index 0000000..ae1220d --- /dev/null +++ b/internal/config/mirror_test.go @@ -0,0 +1,89 @@ +package config + +import ( + "os" + "path/filepath" + "testing" + "time" +) + +func TestMirrorConfigValidate(t *testing.T) { + const digest = "sha256:0000000000000000000000000000000000000000000000000000000000000000" + tests := []struct { + name string + schedule MirrorSchedule + wantErr bool + }{ + {name: "docker tags", schedule: MirrorSchedule{Images: []string{"pkg:docker/library/alpine@3.20", "pkg:docker/nginx@stable"}, Interval: "6h"}}, + {name: "oci with tag", schedule: MirrorSchedule{Images: []string{"pkg:oci/app@v1?repository_url=ghcr.io/acme/app"}, Interval: "1m"}}, + {name: "oci digest with tag", schedule: MirrorSchedule{Images: []string{"pkg:oci/app@" + digest + "?repository_url=ghcr.io/acme/app&tag=v1"}, Interval: "1h"}}, + {name: "platforms", schedule: MirrorSchedule{Images: []string{"pkg:docker/alpine@3"}, Interval: "1h", Platforms: []string{"linux/amd64", "linux/arm64/v8"}}}, + {name: "missing interval", schedule: MirrorSchedule{Images: []string{"pkg:docker/alpine@3"}}, wantErr: true}, + {name: "bad interval", schedule: MirrorSchedule{Images: []string{"pkg:docker/alpine@3"}, Interval: "often"}, wantErr: true}, + {name: "interval under a minute", schedule: MirrorSchedule{Images: []string{"pkg:docker/alpine@3"}, Interval: "30s"}, wantErr: true}, + {name: "no images", schedule: MirrorSchedule{Interval: "1h"}, wantErr: true}, + {name: "unparsable purl", schedule: MirrorSchedule{Images: []string{"alpine:3"}, Interval: "1h"}, wantErr: true}, + {name: "not an image", schedule: MirrorSchedule{Images: []string{"pkg:npm/lodash@4.17.21"}, Interval: "1h"}, wantErr: true}, + {name: "no tag", schedule: MirrorSchedule{Images: []string{"pkg:docker/alpine"}, Interval: "1h"}, wantErr: true}, + {name: "docker digest", schedule: MirrorSchedule{Images: []string{"pkg:docker/alpine@" + digest}, Interval: "1h"}, wantErr: true}, + {name: "docker digest with tag qualifier", schedule: MirrorSchedule{Images: []string{"pkg:docker/alpine@" + digest + "?tag=3"}, Interval: "1h"}, wantErr: true}, + {name: "oci digest only", schedule: MirrorSchedule{Images: []string{"pkg:oci/app@" + digest + "?repository_url=ghcr.io/acme/app"}, Interval: "1h"}, wantErr: true}, + {name: "oci without repository_url", schedule: MirrorSchedule{Images: []string{"pkg:oci/app@v1"}, Interval: "1h"}, wantErr: true}, + {name: "platform without architecture", schedule: MirrorSchedule{Images: []string{"pkg:docker/alpine@3"}, Interval: "1h", Platforms: []string{"linux"}}, wantErr: true}, + {name: "platform list in one entry", schedule: MirrorSchedule{Images: []string{"pkg:docker/alpine@3"}, Interval: "1h", Platforms: []string{"linux/amd64,linux/arm64"}}, wantErr: true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + m := MirrorConfig{Schedules: []MirrorSchedule{tt.schedule}} + err := m.Validate() + if (err != nil) != tt.wantErr { + t.Errorf("Validate() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +func TestMirrorScheduleIntervalDuration(t *testing.T) { + m := MirrorConfig{Schedules: []MirrorSchedule{{Images: []string{"pkg:docker/alpine@3"}, Interval: "6h"}}} + if err := m.Validate(); err != nil { + t.Fatalf("Validate: %v", err) + } + if got := m.Schedules[0].IntervalDuration(); got != 6*time.Hour { + t.Errorf("IntervalDuration() = %v, want 6h", got) + } +} + +func TestValidateRejectsMirrorSchedulesWithoutCachedArtifacts(t *testing.T) { + cfg := Default() + cfg.Mirror.Schedules = []MirrorSchedule{{Images: []string{"pkg:docker/alpine@3"}, Interval: "1h"}} + cfg.Storage.CacheArtifacts = false + if err := cfg.Validate(); err == nil { + t.Error("Validate() accepted mirror.schedules with storage.cache_artifacts: false") + } +} + +func TestLoadYAMLMirrorSchedules(t *testing.T) { + path := filepath.Join(t.TempDir(), "config.yaml") + data := ` +mirror: + schedules: + - images: [pkg:docker/library/alpine@3.20] + interval: 6h + platforms: [linux/amd64] + - images: [pkg:docker/nginx@stable] + interval: 30s +` + if err := os.WriteFile(path, []byte(data), 0o600); err != nil { + t.Fatal(err) + } + cfg, err := Load(path) + if err != nil { + t.Fatalf("Load: %v", err) + } + if len(cfg.Mirror.Schedules) != 2 || cfg.Mirror.Schedules[0].Platforms[0] != "linux/amd64" { + t.Fatalf("schedules = %+v", cfg.Mirror.Schedules) + } + if err := cfg.Validate(); err == nil { + t.Error("Validate() accepted a 30s interval in the second schedule") + } +} diff --git a/internal/handler/container_mirror.go b/internal/handler/container_mirror.go index cb98a97..ddd3022 100644 --- a/internal/handler/container_mirror.go +++ b/internal/handler/container_mirror.go @@ -461,14 +461,11 @@ func (m *imageMirror) mirrorBlobs(ctx context.Context, manifest *ociManifest) er // mirrorBlob stores a blob under the same identity handleBlobDownload uses. func (m *imageMirror) mirrorBlob(ctx context.Context, digest string) error { proxy := m.h.proxy - cached, err := proxy.GetCachedArtifact(ctx, "oci", m.cacheName, digest, digest) + cached, err := proxy.HasCachedArtifact(ctx, "oci", m.cacheName, digest, digest) if err != nil { return fmt.Errorf("blob %s: %w", digest, err) } - if cached != nil { - if cached.Reader != nil { - _ = cached.Reader.Close() - } + if cached { return nil } result, err := proxy.GetOrFetchArtifactFromURLWithDigest(ctx, "oci", m.cacheName, digest, digest, diff --git a/internal/handler/container_mirror_test.go b/internal/handler/container_mirror_test.go index 332851a..149fb23 100644 --- a/internal/handler/container_mirror_test.go +++ b/internal/handler/container_mirror_test.go @@ -2,6 +2,7 @@ package handler import ( "context" + "database/sql" "encoding/json" "io" "net/http" @@ -450,3 +451,55 @@ func TestMirrorImageKeepsTagMovedByPullDuringWalk(t *testing.T) { t.Errorf("offline tag points to %s, want the newer target %s", sha256Digest(response.Body.Bytes()), image.amd64Digest) } } + +func TestMirrorImageCountsNoCacheHits(t *testing.T) { + registry, _ := newFakeImageRegistry(t) + proxy := newMirrorTestProxy(t, registry) + h := NewContainerHandlerWithRegistry(proxy, "http://proxy.example", registry.server.URL) + ref := ImageRef{Name: "team/app", References: []string{"multi"}} + + if _, err := h.MirrorImage(context.Background(), ref, nil); err != nil { + t.Fatalf("first MirrorImage: %v", err) + } + var hits int + var lastAccess sql.NullString + if err := proxy.DB.QueryRow("SELECT COALESCE(SUM(hit_count), 0), MAX(last_accessed_at) FROM artifacts").Scan(&hits, &lastAccess); err != nil { + t.Fatal(err) + } + if _, err := h.MirrorImage(context.Background(), ref, nil); err != nil { + t.Fatalf("second MirrorImage: %v", err) + } + var hitsAfter int + var lastAccessAfter sql.NullString + if err := proxy.DB.QueryRow("SELECT COALESCE(SUM(hit_count), 0), MAX(last_accessed_at) FROM artifacts").Scan(&hitsAfter, &lastAccessAfter); err != nil { + t.Fatal(err) + } + if hitsAfter != hits || lastAccessAfter != lastAccess { + t.Errorf("second run changed hits %d -> %d and last access %v -> %v, want no change", hits, hitsAfter, lastAccess, lastAccessAfter) + } +} + +func TestMirrorImageRefetchesBlobMissingFromStorage(t *testing.T) { + registry, image := newFakeImageRegistry(t) + proxy := newMirrorTestProxy(t, registry) + h := NewContainerHandlerWithRegistry(proxy, "http://proxy.example", registry.server.URL) + ref := ImageRef{Name: "team/app", References: []string{"multi"}} + + if _, err := h.MirrorImage(context.Background(), ref, nil); err != nil { + t.Fatalf("first MirrorImage: %v", err) + } + var path string + if err := proxy.DB.QueryRow("SELECT storage_path FROM artifacts WHERE filename = ?", image.amd64Layer).Scan(&path); err != nil { + t.Fatal(err) + } + if err := proxy.Storage.Delete(context.Background(), path); err != nil { + t.Fatal(err) + } + gets := registry.blobGets.Load() + if _, err := h.MirrorImage(context.Background(), ref, nil); err != nil { + t.Fatalf("second MirrorImage: %v", err) + } + if got := registry.blobGets.Load() - gets; got != 1 { + t.Errorf("second run fetched %d blobs, want the one missing from storage", got) + } +} diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 62d63ae..9262de0 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -329,6 +329,34 @@ func (p *Proxy) GetCachedArtifact(ctx context.Context, ecosystem, name, version, return p.checkCache(ctx, pkgPURL, versionPURL, filename) } +// HasCachedArtifact reports whether an artifact is cached and its object is +// stored, without opening it or counting a cache hit, so a mirror run does +// not look like client traffic. +func (p *Proxy) HasCachedArtifact(ctx context.Context, ecosystem, name, version, filename string) (bool, error) { + pkgPURL, versionPURL, err := packagePURLStrings(ecosystem, name, version) + if err != nil { + return false, err + } + if p.Denylist.Denied(versionPURL) { + return false, fmt.Errorf("%w: %s", ErrVersionDenied, versionPURL) + } + if p.StreamArtifacts { + return false, nil + } + artifact, err := p.DB.GetCachedArtifact(pkgPURL, versionPURL, filename) + if err != nil { + return false, fmt.Errorf("checking artifact cache: %w", err) + } + if artifact == nil { + return false, nil + } + // An unusable record is a miss; the fetch that follows clears it. + if _, err := newIntegrityChecks(artifact.Artifact.Digest.Encoded(), artifact.Integrity.String); err != nil { + return false, nil + } + return p.Storage.Exists(ctx, artifact.StoragePath) +} + // ClearCachedArtifact clears an artifact cache record after an external // integrity check fails, and queues its stored bytes for deletion. func (p *Proxy) ClearCachedArtifact(ecosystem, name, version, filename string) error { diff --git a/internal/mirror/schedule.go b/internal/mirror/schedule.go new file mode 100644 index 0000000..4ab0d2b --- /dev/null +++ b/internal/mirror/schedule.go @@ -0,0 +1,105 @@ +package mirror + +import ( + "context" + "log/slog" + "math/rand/v2" + "sync" + "time" +) + +const ( + maxFirstScheduleDelay = 10 * time.Minute + // scheduleJitter spreads later runs by ±interval/scheduleJitter. + scheduleJitter = 10 +) + +// Schedule mirrors a set of image PURLs every Interval. +type Schedule struct { + PURLs []string + Platforms []string + Interval time.Duration +} + +// Scheduler runs schedules in the background until its context ends. +type Scheduler struct { + mirror *Mirror + schedules []Schedule + logger *slog.Logger + wg sync.WaitGroup + // randN returns a value in [0, n); tests replace it. + randN func(n int64) int64 +} + +// NewScheduler creates a scheduler that runs the schedules through m. +func NewScheduler(m *Mirror, schedules []Schedule, logger *slog.Logger) *Scheduler { + return &Scheduler{ + mirror: m, + schedules: schedules, + logger: logger, + randN: rand.Int64N, //nolint:gosec // jitter, not security + } +} + +// Start starts one goroutine per schedule. +func (s *Scheduler) Start(ctx context.Context) { + for i, schedule := range s.schedules { + if schedule.Interval <= 0 { + s.logger.Error("mirror schedule has no interval, skipped", "schedule", i) + continue + } + s.wg.Add(1) + go s.loop(ctx, i, schedule) + } +} + +// Wait blocks until every schedule has stopped after its context ended. +func (s *Scheduler) Wait() { + s.wg.Wait() +} + +func (s *Scheduler) loop(ctx context.Context, index int, schedule Schedule) { + defer s.wg.Done() + timer := time.NewTimer(scheduleDelay(schedule.Interval, true, s.randN)) + defer timer.Stop() + for { + select { + case <-ctx.Done(): + return + case <-timer.C: + } + s.run(ctx, index, schedule) + // The next run is timed from the end of this one, so runs never overlap. + timer.Reset(scheduleDelay(schedule.Interval, false, s.randN)) + } +} + +func (s *Scheduler) run(ctx context.Context, index int, schedule Schedule) { + started := time.Now() + progress, err := s.mirror.Run(ctx, &PURLSource{PURLs: schedule.PURLs, Platforms: schedule.Platforms}) + if ctx.Err() != nil { + return + } + if err != nil { + s.logger.Error("mirror schedule failed", "schedule", index, "error", err) + return + } + level := slog.LevelInfo + if progress.Failed > 0 { + level = slog.LevelWarn + } + s.logger.Log(ctx, level, "mirror schedule run", + "schedule", index, "mirrored", progress.Completed, "unchanged", progress.Skipped, + "failed", progress.Failed, "bytes", progress.Bytes, "duration", time.Since(started).Round(time.Millisecond)) +} + +// scheduleDelay spreads runs so instances and schedules started together +// don't hit the registry at once. The first run comes within ten minutes, so +// a restart doesn't push a refresh back by a whole interval. +func scheduleDelay(interval time.Duration, first bool, randN func(int64) int64) time.Duration { + if first { + return time.Duration(randN(int64(min(interval, maxFirstScheduleDelay)))) + } + spread := interval / scheduleJitter + return interval - spread + time.Duration(randN(int64(2*spread)+1)) //nolint:mnd // spread on both sides +} diff --git a/internal/mirror/schedule_test.go b/internal/mirror/schedule_test.go new file mode 100644 index 0000000..c869c8a --- /dev/null +++ b/internal/mirror/schedule_test.go @@ -0,0 +1,177 @@ +package mirror + +import ( + "context" + "errors" + "io" + "log/slog" + "math/rand/v2" + "reflect" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/git-pkgs/proxy/internal/handler" +) + +func TestScheduleDelay(t *testing.T) { + lowest := func(int64) int64 { return 0 } + highest := func(n int64) int64 { return n - 1 } + tests := []struct { + name string + interval time.Duration + first bool + randN func(int64) int64 + want time.Duration + }{ + {name: "first run lowest", interval: 6 * time.Hour, first: true, randN: lowest, want: 0}, + {name: "first run capped at ten minutes", interval: 6 * time.Hour, first: true, randN: highest, want: 10*time.Minute - 1}, + {name: "first run within a short interval", interval: 2 * time.Minute, first: true, randN: highest, want: 2*time.Minute - 1}, + {name: "later run lowest", interval: 6 * time.Hour, randN: lowest, want: 6*time.Hour - 36*time.Minute}, + {name: "later run highest", interval: 6 * time.Hour, randN: highest, want: 6*time.Hour + 36*time.Minute}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := scheduleDelay(tt.interval, tt.first, tt.randN); got != tt.want { + t.Errorf("scheduleDelay() = %v, want %v", got, tt.want) + } + }) + } +} + +// scheduleImages records MirrorImage calls and can block or fail them. +type scheduleImages struct { + mu sync.Mutex + refs []handler.ImageRef + platforms [][]handler.Platform + calls atomic.Int32 + running atomic.Int32 + overlap atomic.Bool + block chan struct{} + err error +} + +func (f *scheduleImages) MirrorImage(ctx context.Context, ref handler.ImageRef, platforms []handler.Platform) (handler.ImageResult, error) { + if f.running.Add(1) > 1 { + f.overlap.Store(true) + } + defer f.running.Add(-1) + f.mu.Lock() + f.refs = append(f.refs, ref) + f.platforms = append(f.platforms, platforms) + f.mu.Unlock() + f.calls.Add(1) + if f.block != nil { + select { + case <-f.block: + case <-ctx.Done(): + return handler.ImageResult{}, ctx.Err() + } + } + return handler.ImageResult{}, f.err +} + +func newTestScheduler(images ImageMirrorer, schedules ...Schedule) *Scheduler { + logger := slog.New(slog.NewTextHandler(io.Discard, nil)) + m := New(nil, nil, nil, logger, 1) + m.Images = images + s := NewScheduler(m, schedules, logger) + s.randN = func(int64) int64 { return 0 } + return s +} + +func waitFor(t *testing.T, what string, cond func() bool) { + t.Helper() + deadline := time.Now().Add(5 * time.Second) + for !cond() { + if time.Now().After(deadline) { + t.Fatalf("timed out waiting for %s", what) + } + time.Sleep(time.Millisecond) + } +} + +func TestSchedulerRepeatsRunsAndStops(t *testing.T) { + images := &scheduleImages{err: errors.New("upstream down")} + s := newTestScheduler(images, Schedule{ + PURLs: []string{"pkg:docker/library/alpine@3.20"}, + Platforms: []string{"linux/arm64"}, + Interval: time.Millisecond, + }) + ctx, cancel := context.WithCancel(context.Background()) + s.Start(ctx) + // A failing run must not end the schedule. + waitFor(t, "three runs", func() bool { return images.calls.Load() >= 3 }) + cancel() + s.Wait() + + images.mu.Lock() + defer images.mu.Unlock() + wantRef := handler.ImageRef{Name: "library/alpine", References: []string{"3.20"}} + if !reflect.DeepEqual(images.refs[0], wantRef) { + t.Errorf("ref = %+v, want %+v", images.refs[0], wantRef) + } + wantPlatforms := []handler.Platform{{OS: "linux", Architecture: "arm64"}} + if !reflect.DeepEqual(images.platforms[0], wantPlatforms) { + t.Errorf("platforms = %+v, want %+v", images.platforms[0], wantPlatforms) + } +} + +func TestSchedulerRunsDoNotOverlap(t *testing.T) { + images := &scheduleImages{block: make(chan struct{})} + s := newTestScheduler(images, Schedule{PURLs: []string{"pkg:docker/alpine@3"}, Interval: time.Millisecond}) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + s.Start(ctx) + + waitFor(t, "first run", func() bool { return images.calls.Load() == 1 }) + // Many intervals pass while the run blocks; none may start a second run. + time.Sleep(50 * time.Millisecond) + if got := images.calls.Load(); got != 1 { + t.Fatalf("calls while the first run blocks = %d, want 1", got) + } + images.block <- struct{}{} + waitFor(t, "second run", func() bool { return images.calls.Load() == 2 }) + cancel() + s.Wait() + if images.overlap.Load() { + t.Error("two runs of one schedule overlapped") + } +} + +func TestSchedulerWaitReturnsAfterCancelDuringRun(t *testing.T) { + images := &scheduleImages{block: make(chan struct{})} + s := newTestScheduler(images, + Schedule{PURLs: []string{"pkg:docker/alpine@3"}, Interval: time.Hour}, + Schedule{PURLs: []string{"pkg:docker/nginx@stable"}, Interval: time.Hour}) + ctx, cancel := context.WithCancel(context.Background()) + s.Start(ctx) + waitFor(t, "both schedules running", func() bool { return images.calls.Load() == 2 }) + + cancel() + done := make(chan struct{}) + go func() { + s.Wait() + close(done) + }() + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatal("Wait did not return after cancel") + } +} + +func TestSchedulerSkipsScheduleWithoutInterval(t *testing.T) { + images := &scheduleImages{} + s := newTestScheduler(images, Schedule{PURLs: []string{"pkg:docker/alpine@3"}}) + // The real random source panics on a zero interval. + s.randN = rand.Int64N + ctx, cancel := context.WithCancel(context.Background()) + s.Start(ctx) + cancel() + s.Wait() + if got := images.calls.Load(); got != 0 { + t.Errorf("calls = %d, want 0", got) + } +} diff --git a/internal/server/mirror_schedule.go b/internal/server/mirror_schedule.go new file mode 100644 index 0000000..3f69e7f --- /dev/null +++ b/internal/server/mirror_schedule.go @@ -0,0 +1,47 @@ +package server + +import ( + "context" + "time" + + "github.com/git-pkgs/proxy/internal/handler" + "github.com/git-pkgs/proxy/internal/mirror" +) + +// mirrorScheduleShutdownWait bounds how long shutdown waits for a scheduled +// run to stop before storage and database are closed under it. +const mirrorScheduleShutdownWait = 30 * time.Second + +func (s *Server) startMirrorSchedules(ctx context.Context, proxy *handler.Proxy) { + if len(s.cfg.Mirror.Schedules) == 0 { + return + } + // One worker, so a schedule never fans out against the registry. + m := mirror.New(proxy, s.db, s.storage, s.logger, 1) + m.Images = s.newContainerHandler(proxy) + schedules := make([]mirror.Schedule, len(s.cfg.Mirror.Schedules)) + for i, schedule := range s.cfg.Mirror.Schedules { + schedules[i] = mirror.Schedule{PURLs: schedule.Images, Platforms: schedule.Platforms, Interval: schedule.IntervalDuration()} + } + s.scheduler = mirror.NewScheduler(m, schedules, s.logger) + s.scheduler.Start(ctx) + s.logger.Info("mirror schedules enabled", "schedules", len(schedules)) +} + +func (s *Server) waitForMirrorSchedules(ctx context.Context) { + if s.scheduler == nil { + return + } + ctx, cancel := context.WithTimeout(ctx, mirrorScheduleShutdownWait) + defer cancel() + done := make(chan struct{}) + go func() { + s.scheduler.Wait() + close(done) + }() + select { + case <-done: + case <-ctx.Done(): + s.logger.Warn("mirror schedule still running at shutdown") + } +} diff --git a/internal/server/mirror_schedule_test.go b/internal/server/mirror_schedule_test.go new file mode 100644 index 0000000..d2440e6 --- /dev/null +++ b/internal/server/mirror_schedule_test.go @@ -0,0 +1,98 @@ +package server + +import ( + "context" + "io" + "log/slog" + "sync/atomic" + "testing" + "time" + + "github.com/git-pkgs/proxy/internal/handler" + "github.com/git-pkgs/proxy/internal/mirror" +) + +// blockingImages blocks every mirror until release is closed, and with +// ignoreCancel even past the end of its context. +type blockingImages struct { + started atomic.Bool + release chan struct{} + ignoreCancel bool +} + +func (b *blockingImages) MirrorImage(ctx context.Context, _ handler.ImageRef, _ []handler.Platform) (handler.ImageResult, error) { + b.started.Store(true) + if b.ignoreCancel { + <-b.release + return handler.ImageResult{}, nil + } + select { + case <-b.release: + case <-ctx.Done(): + } + return handler.ImageResult{}, ctx.Err() +} + +func startBlockingSchedule(t *testing.T, images *blockingImages) (*Server, context.CancelFunc) { + t.Helper() + logger := slog.New(slog.NewTextHandler(io.Discard, nil)) + m := mirror.New(nil, nil, nil, logger, 1) + m.Images = images + // A 1ns interval starts the first run right away. + s := &Server{logger: logger, scheduler: mirror.NewScheduler(m, []mirror.Schedule{ + {PURLs: []string{"pkg:docker/alpine@3"}, Interval: time.Nanosecond}, + }, logger)} + ctx, cancel := context.WithCancel(context.Background()) + s.scheduler.Start(ctx) + deadline := time.Now().Add(5 * time.Second) + for !images.started.Load() { + if time.Now().After(deadline) { + t.Fatal("scheduled run did not start") + } + time.Sleep(time.Millisecond) + } + return s, cancel +} + +func TestWaitForMirrorSchedulesWaitsForRunningRun(t *testing.T) { + images := &blockingImages{release: make(chan struct{}), ignoreCancel: true} + s, cancel := startBlockingSchedule(t, images) + cancel() + + done := make(chan struct{}) + go func() { + s.waitForMirrorSchedules(context.Background()) + close(done) + }() + select { + case <-done: + t.Fatal("waitForMirrorSchedules returned while a run was still going") + case <-time.After(50 * time.Millisecond): + } + close(images.release) + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatal("waitForMirrorSchedules did not return after the run ended") + } +} + +func TestWaitForMirrorSchedulesGivesUpAtDeadline(t *testing.T) { + images := &blockingImages{release: make(chan struct{}), ignoreCancel: true} + s, cancel := startBlockingSchedule(t, images) + defer close(images.release) + cancel() + + ctx, stop := context.WithTimeout(context.Background(), 20*time.Millisecond) + defer stop() + started := time.Now() + s.waitForMirrorSchedules(ctx) + if elapsed := time.Since(started); elapsed > 5*time.Second { + t.Errorf("waitForMirrorSchedules took %v past its deadline", elapsed) + } +} + +func TestWaitForMirrorSchedulesWithoutSchedules(t *testing.T) { + s := &Server{} + s.waitForMirrorSchedules(context.Background()) +} diff --git a/internal/server/server.go b/internal/server/server.go index f9bdf5d..a6e121c 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -120,6 +120,7 @@ type Server struct { rateLimits *upstreamhttp.RateLimitGate ecoStats ecosystemStatsCache sources sourceTracker + scheduler *mirror.Scheduler // browseLimits overrides defaultBrowseLimits when set; tests use it. browseLimits archives.StreamOptions @@ -358,6 +359,7 @@ func (s *Server) serve(listener net.Listener) error { r.Delete("/api/mirror/{id}", mirrorAPI.HandleCancel) go jobStore.StartCleanup(bgCtx) } + s.startMirrorSchedules(bgCtx, proxy) // Hosted package endpoints (opt-in via hosted.enabled or PROXY_HOSTED_ENABLED) if s.cfg.Hosted.Enabled { @@ -635,6 +637,8 @@ func (s *Server) Shutdown(ctx context.Context) error { } } + s.waitForMirrorSchedules(ctx) + if s.storage != nil { if err := s.storage.Close(); err != nil { errs = append(errs, fmt.Errorf("storage close: %w", err)) From bdec446b69d42930e6cc1f8f0d812254f60fd830 Mon Sep 17 00:00:00 2001 From: Cestercian Date: Mon, 19 Oct 2026 00:55:13 +0530 Subject: [PATCH 308/633] Exit with status 1 when proxy mirror fails to mirror a package (#502) Fixes #500 --- README.md | 2 +- cmd/proxy/main.go | 16 +++++++++ cmd/proxy/mirror_test.go | 74 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 91 insertions(+), 1 deletion(-) create mode 100644 cmd/proxy/mirror_test.go diff --git a/README.md b/README.md index 3ec9f3d..f3508dd 100644 --- a/README.md +++ b/README.md @@ -1012,7 +1012,7 @@ proxy mirror --concurrency 8 pkg:npm/lodash@4.17.21 proxy mirror --platform linux/amd64 pkg:docker/library/alpine@3.20 ``` -The mirror command accepts the same storage and database flags as `serve`. Already-cached artifacts are skipped. +The mirror command accepts the same storage and database flags as `serve`. Already-cached artifacts are skipped. If any package version fails to mirror, the command lists the errors and exits with status 1, so a CI step that warms the cache fails on a partial mirror. A mirror API is also available when the server is running: diff --git a/cmd/proxy/main.go b/cmd/proxy/main.go index f6e7f95..0b140ea 100644 --- a/cmd/proxy/main.go +++ b/cmd/proxy/main.go @@ -599,6 +599,22 @@ func runMirror() { fmt.Fprintf(os.Stderr, " %s/%s@%s: %s\n", e.Ecosystem, e.Name, e.Version, e.Error) } } + + // A partial mirror has to be visible in the exit status, or a CI step + // that prewarms the cache passes even when every download failed. + if code := mirrorExitCode(progress); code != 0 { + _ = db.Close() + os.Exit(code) //nolint:gocritic // db closed above + } +} + +// mirrorExitCode returns the exit status for a finished mirror run: 1 when any +// package version failed to mirror, 0 otherwise. +func mirrorExitCode(progress *mirror.Progress) int { + if progress != nil && progress.Failed > 0 { + return 1 + } + return 0 } func printStats(db *database.DB, popular, recent int, asJSON bool) error { diff --git a/cmd/proxy/mirror_test.go b/cmd/proxy/mirror_test.go new file mode 100644 index 0000000..4db2a89 --- /dev/null +++ b/cmd/proxy/mirror_test.go @@ -0,0 +1,74 @@ +package main + +import ( + "errors" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/git-pkgs/proxy/internal/mirror" +) + +func TestMirrorExitCode(t *testing.T) { + tests := []struct { + name string + progress *mirror.Progress + want int + }{ + {"nil progress", nil, 0}, + {"everything mirrored", &mirror.Progress{Total: 2, Completed: 2}, 0}, + {"everything cached", &mirror.Progress{Total: 2, Skipped: 2}, 0}, + {"one failure", &mirror.Progress{Total: 2, Completed: 1, Failed: 1}, 1}, + {"all failed", &mirror.Progress{Total: 3, Failed: 3}, 1}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := mirrorExitCode(tt.progress); got != tt.want { + t.Errorf("mirrorExitCode() = %d, want %d", got, tt.want) + } + }) + } +} + +// proxy mirror used to exit 0 after printing "N failed", so a CI step that +// prewarms the cache passed even when every download failed (#500). An +// ecosystem the resolver rejects fails without touching the network. +func TestMirrorExitsNonZeroWhenDownloadsFail(t *testing.T) { + dir := t.TempDir() + cmd := exec.Command(os.Args[0], "-test.run=^TestMirrorFailureProcess$") + cmd.Env = append(os.Environ(), + "PROXY_TEST_MIRROR_FAILURE=1", + "PROXY_TEST_MIRROR_DIR="+dir, + ) + output, err := cmd.CombinedOutput() + + var exitErr *exec.ExitError + if !errors.As(err, &exitErr) { + t.Fatalf("mirror with a failed download exited with %v, want status 1\n%s", err, output) + } + if code := exitErr.ExitCode(); code != 1 { + t.Fatalf("exit status = %d, want 1\n%s", code, output) + } + if !strings.Contains(string(output), "1 failed") { + t.Errorf("output does not report the failure:\n%s", output) + } + if !strings.Contains(string(output), "bogus/foo@1.0.0") { + t.Errorf("output does not list the failed package:\n%s", output) + } +} + +func TestMirrorFailureProcess(*testing.T) { + if os.Getenv("PROXY_TEST_MIRROR_FAILURE") != "1" { + return + } + dir := os.Getenv("PROXY_TEST_MIRROR_DIR") + os.Args = []string{ + "proxy", "mirror", + "--storage-url", "file://" + filepath.Join(dir, "store"), + "--database-path", filepath.Join(dir, "cache.db"), + "pkg:bogus/foo@1.0.0", + } + main() +} From fa7ae81c764a7c0bc293e18ab0f71c49723349ce Mon Sep 17 00:00:00 2001 From: simon chrzanowski Date: Sun, 18 Oct 2026 21:27:18 +0200 Subject: [PATCH 309/633] Request npm packuments in one format for downloads too (#496) The metadata route caches a package's packument in the abbreviated install format, unless cooldown or npm_full_metadata needs the full one. Tarball downloads read the packument from the same cache row but asked for application/json. Once the row was due for revalidation, a tarball the proxy had not cached yet fetched the full packument, which did not match the abbreviated document's ETag, and stored it in its place. Until the next refresh every client was then served the full document, typically several times the size. Both now use packumentAccept. Downloads read only dist.tarball, dist.integrity and dist.shasum, which the abbreviated format carries; the publish times the cooldown needs come with the full format it selects. --- internal/handler/npm.go | 34 +++++--- internal/handler/npm_packument_accept_test.go | 78 +++++++++++++++++++ 2 files changed, 100 insertions(+), 12 deletions(-) create mode 100644 internal/handler/npm_packument_accept_test.go diff --git a/internal/handler/npm.go b/internal/handler/npm.go index 832a860..c92bf2a 100644 --- a/internal/handler/npm.go +++ b/internal/handler/npm.go @@ -207,17 +207,7 @@ func (h *NPMHandler) handlePackageMetadata(w http.ResponseWriter, r *http.Reques upstreamURL := registryURL + "/" + registryPath metadataCacheKey := npmMetadataCacheKey(cacheKey, packagePath, packageName) - // Prefer the smaller abbreviated packument format but include application/json - // as a fallback so upstreams that reject the abbreviated type (e.g. JFrog - // Artifactory, which returns 406) can still respond with full metadata. - // When cooldown is enabled we must use full metadata exclusively because the - // abbreviated format omits the "time" map required for version age filtering. - // Operators can also force full metadata so clients that gate on publish - // age (for example Yarn's npmMinimalAgeGate) keep working through the proxy. - accept := npmAcceptDefault - if h.proxy.NPMFullMetadata || (h.proxy.Cooldown != nil && h.proxy.Cooldown.Enabled()) { - accept = contentTypeJSON - } + accept := h.packumentAccept() specifier, versionLookup := npmVersionSpecifier(packagePath, packageName) if h.serveLocalMetadata(w, r, npmMetadataRequest{packageName, specifier, versionLookup, upstreamURL, accept, metadataCacheKey}) { @@ -282,6 +272,26 @@ func (h *NPMHandler) handlePackageMetadata(w http.ResponseWriter, r *http.Reques serveRewrittenMetadata(w, r, rewritten) } +// packumentAccept returns the Accept header packuments are requested with. +// It prefers the smaller abbreviated packument format but includes +// application/json as a fallback so upstreams that reject the abbreviated type +// (e.g. JFrog Artifactory, which returns 406) can still respond with full +// metadata. When cooldown is enabled we must use full metadata exclusively +// because the abbreviated format omits the "time" map required for version age +// filtering. Operators can also force full metadata so clients that gate on +// publish age (for example Yarn's npmMinimalAgeGate) keep working through the +// proxy. +// +// Every request for a packument uses it, since they share one cache row: a +// request with another Accept would replace the document the others are +// served with one in the other format. +func (h *NPMHandler) packumentAccept() string { + if h.proxy.NPMFullMetadata || h.proxy.cooldownEnabled() { + return contentTypeJSON + } + return npmAcceptDefault +} + // handleFilteredNPMVersionMetadata serves /pkg/ and /pkg/ from the // filtered packument so cooldown and dist-tags match the root metadata response. func (h *NPMHandler) handleFilteredNPMVersionMetadata(w http.ResponseWriter, r *http.Request, packageName, specifier, accept, registryURL, cacheKey, metadataCacheKey string) { @@ -841,7 +851,7 @@ func (h *NPMHandler) handleDownload(w http.ResponseWriter, r *http.Request) { registryURL, cacheKey := h.sourceFor(packageName) metadata := sync.OnceValues(func() ([]byte, error) { upstreamURL := registryURL + "/" + url.PathEscape(packageName) - body, _, err := h.proxy.FetchOrCacheMetadata(r.Context(), "npm", cacheKey, upstreamURL, contentTypeJSON) + body, _, err := h.proxy.FetchOrCacheMetadata(r.Context(), "npm", cacheKey, upstreamURL, h.packumentAccept()) return body, err }) if h.versionInCooldown(packageName, version, metadata) { diff --git a/internal/handler/npm_packument_accept_test.go b/internal/handler/npm_packument_accept_test.go new file mode 100644 index 0000000..7158cef --- /dev/null +++ b/internal/handler/npm_packument_accept_test.go @@ -0,0 +1,78 @@ +package handler + +import ( + "fmt" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + "time" + + "github.com/git-pkgs/registries/fetch" +) + +// TestNPMTarballKeepsCachedPackumentFormat checks that a tarball download +// revalidates the packument in the format the metadata route cached it in, +// rather than replacing the abbreviated document every client is served with +// the full one. +func TestNPMTarballKeepsCachedPackumentFormat(t *testing.T) { + const abbreviated = "application/vnd.npm.install-v1+json" + var ( + mu sync.Mutex + accepts []string + ) + var upstream *httptest.Server + upstream = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + accept := r.Header.Get("Accept") + mu.Lock() + accepts = append(accepts, accept) + mu.Unlock() + contentType, etag := contentTypeJSON, `"full"` + if strings.HasPrefix(accept, abbreviated) { + contentType, etag = abbreviated, `"abbreviated"` + } + w.Header().Set(headerETag, etag) + if r.Header.Get("If-None-Match") == etag { + w.WriteHeader(http.StatusNotModified) + return + } + w.Header().Set(headerContentType, contentType) + _, _ = fmt.Fprintf(w, `{"name":"demo","versions":{"1.0.0":{"name":"demo","version":"1.0.0","dist":{"tarball":%q}}}}`, + upstream.URL+"/demo/-/demo-1.0.0.tgz") + })) + t.Cleanup(upstream.Close) + + proxy, db, _, artifactFetcher := setupTestProxy(t) + proxy.CacheMetadata = true + proxy.MetadataTTL = time.Nanosecond // the tarball request revalidates + proxy.HTTPClient = upstream.Client() + artifactFetcher.artifact = &fetch.Artifact{Body: io.NopCloser(strings.NewReader("package")), ContentType: "application/gzip"} + h := NewNPMHandler(proxy, "http://proxy.test", upstream.URL) + + if w := serveNPM(t, h, http.MethodGet, "/demo", nil, nil); w.Code != http.StatusOK { + t.Fatalf("metadata: status %d: %s", w.Code, w.Body.String()) + } + if w := serveNPM(t, h, http.MethodGet, "/demo/-/demo-1.0.0.tgz", nil, nil); w.Code != http.StatusOK { + t.Fatalf("tarball: status %d: %s", w.Code, w.Body.String()) + } + + mu.Lock() + defer mu.Unlock() + if len(accepts) != 2 { + t.Fatalf("upstream packument requests = %d (%q), want 2", len(accepts), accepts) + } + for _, accept := range accepts { + if accept != npmAcceptDefault { + t.Errorf("packument requested with Accept %q, want %q", accept, npmAcceptDefault) + } + } + entry, err := db.GetMetadataCache("npm", "demo") + if err != nil || entry == nil { + t.Fatalf("cache row: %v, %v", entry, err) + } + if entry.ContentType.String != abbreviated { + t.Errorf("cached packument Content-Type = %q, want %q", entry.ContentType.String, abbreviated) + } +} From 85b674f2786c9614fd7bd9e289af3c17a5a6223b Mon Sep 17 00:00:00 2001 From: simon chrzanowski Date: Sun, 18 Oct 2026 21:29:43 +0200 Subject: [PATCH 310/633] Keep the stored metadata on an upstream 304 (#495) When upstream answered a revalidation with 304, the proxy read the cached document back from storage and then stored it again: a full upload and a SHA-256 over identical bytes, for every popular package once per TTL. A 304 now only moves the cache row's fetched_at on, keeping its digest. The update is conditional on the row still recording the digest of the document that was read, so a concurrent fetch that replaced it is not overwritten; in that case, and for rows written before digests were recorded, the document is stored as before. --- internal/database/metadata_cache_test.go | 47 ++++++++ internal/database/queries.go | 16 +++ internal/handler/handler.go | 32 ++++- internal/handler/handler_test.go | 2 + .../handler/metadata_not_modified_test.go | 112 ++++++++++++++++++ 5 files changed, 207 insertions(+), 2 deletions(-) create mode 100644 internal/handler/metadata_not_modified_test.go diff --git a/internal/database/metadata_cache_test.go b/internal/database/metadata_cache_test.go index 26e0a63..0e19f6c 100644 --- a/internal/database/metadata_cache_test.go +++ b/internal/database/metadata_cache_test.go @@ -401,3 +401,50 @@ func TestMetadataCacheTimesOffUTC(t *testing.T) { } }) } + +// TestRefreshMetadataCache checks that a refresh moves only fetched_at on, and +// only while the row still records the digest it was asked for. +func TestRefreshMetadataCache(t *testing.T) { + db := setupMetadataCacheDB(t) + fetched := time.Now().Add(-time.Hour).UTC().Truncate(time.Second) + entry := &MetadataCacheEntry{ + Ecosystem: testEcosystemNPM, + Name: "lodash", + StoragePath: "_metadata/npm/lodash/metadata", + ETag: sql.NullString{String: `"abc123"`, Valid: true}, + ContentDigest: sql.NullString{String: "sha256:aaaa", Valid: true}, + Size: sql.NullInt64{Int64: 1024, Valid: true}, + FetchedAt: sql.NullTime{Time: fetched, Valid: true}, + } + if err := db.UpsertMetadataCache(entry); err != nil { + t.Fatal(err) + } + + refreshed, err := db.RefreshMetadataCache(testEcosystemNPM, "lodash", "sha256:bbbb", time.Now()) + if err != nil || refreshed { + t.Fatalf("refresh with another digest = %v, %v, want false", refreshed, err) + } + if got, _ := db.GetMetadataCache(testEcosystemNPM, "lodash"); !got.FetchedAt.Time.Equal(fetched) { + t.Errorf("fetched_at = %v after a refresh with another digest, want %v", got.FetchedAt.Time, fetched) + } + + now := time.Now().Truncate(time.Second) + refreshed, err = db.RefreshMetadataCache(testEcosystemNPM, "lodash", "sha256:aaaa", now) + if err != nil || !refreshed { + t.Fatalf("refresh = %v, %v, want true", refreshed, err) + } + got, err := db.GetMetadataCache(testEcosystemNPM, "lodash") + if err != nil { + t.Fatal(err) + } + if !got.FetchedAt.Time.Equal(now) { + t.Errorf("fetched_at = %v, want %v", got.FetchedAt.Time, now) + } + if got.ETag.String != `"abc123"` || got.ContentDigest.String != "sha256:aaaa" || got.Size.Int64 != 1024 || got.StoragePath != entry.StoragePath { + t.Errorf("refresh changed the row: %+v", got) + } + + if refreshed, err := db.RefreshMetadataCache(testEcosystemNPM, "missing", "sha256:aaaa", now); err != nil || refreshed { + t.Errorf("refresh of a missing row = %v, %v, want false", refreshed, err) + } +} diff --git a/internal/database/queries.go b/internal/database/queries.go index cd7c51c..f1cdbe9 100644 --- a/internal/database/queries.go +++ b/internal/database/queries.go @@ -1140,6 +1140,22 @@ func (db *DB) GetMetadataCache(ecosystem, name string) (*MetadataCacheEntry, err return &entry, nil } +// RefreshMetadataCache sets fetched_at on the metadata cache row for ecosystem +// and name, provided the row still records digest, and reports whether it did. +// It is for an upstream 304, which leaves the stored document as it is. +func (db *DB) RefreshMetadataCache(ecosystem, name, digest string, fetchedAt time.Time) (bool, error) { + // fetched_at is stored in UTC, as in UpsertMetadataCache. + res, err := db.Exec(db.Rebind(` + UPDATE metadata_cache SET fetched_at = ?, updated_at = ? + WHERE ecosystem = ? AND name = ? AND content_digest = ? + `), fetchedAt.UTC(), time.Now(), ecosystem, name, digest) + if err != nil { + return false, err + } + n, err := res.RowsAffected() + return n > 0, err +} + func (db *DB) UpsertMetadataCache(entry *MetadataCacheEntry) error { now := time.Now() // fetched_at is a TIMESTAMP without zone on Postgres, and lib/pq reads it diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 9262de0..1af336c 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -1383,7 +1383,10 @@ func (p *Proxy) fetchMetadataFromUpstream(ctx context.Context, ecosystem, cacheK if err == nil { var digest string if p.CacheMetadata { - digest = p.cacheMetadataBlob(ctx, ecosystem, cacheKey, metadataStoragePath(ecosystem, cacheKey), meta) + digest = p.refreshMetadataBlob(ecosystem, cacheKey, meta) + if digest == "" { + digest = p.cacheMetadataBlob(ctx, ecosystem, cacheKey, metadataStoragePath(ecosystem, cacheKey), meta) + } } return metadataResult{body: meta.body, contentType: meta.contentType, contentEncoding: meta.contentEncoding, digest: digest} } @@ -1533,6 +1536,9 @@ type upstreamMetadata struct { contentEncoding string etag string lastModified time.Time + // notModified is the cache row upstream answered 304 for, whose stored + // bytes body holds. It is nil for a document upstream sent. + notModified *database.MetadataCacheEntry } // fetchUpstreamMetadata fetches metadata from upstream, using ETag for conditional revalidation. @@ -1574,7 +1580,7 @@ func (p *Proxy) fetchUpstreamMetadata(ctx context.Context, upstreamURL string, e if readErr != nil { return nil, errStale304 } - meta := &upstreamMetadata{body: data, contentType: contentTypeJSON, etag: entry.ETag.String} + meta := &upstreamMetadata{body: data, contentType: contentTypeJSON, etag: entry.ETag.String, notModified: entry} if entry.ContentType.Valid { meta.contentType = entry.ContentType.String } @@ -1615,6 +1621,28 @@ func (p *Proxy) fetchUpstreamMetadata(ctx context.Context, upstreamURL string, e return meta, nil } +// refreshMetadataBlob records that upstream answered 304 for the cached +// document meta was read from, by moving the row's fetched_at on. The bytes in +// storage are unchanged, so they are neither written nor hashed again. It +// returns the row's digest, or "" when the caller has to store meta as for a +// new document: when upstream sent one, the row records no digest, or a +// concurrent fetch has replaced the document since it was read. +func (p *Proxy) refreshMetadataBlob(ecosystem, cacheKey string, meta *upstreamMetadata) string { + entry := meta.notModified + if entry == nil || p.DB == nil || !entry.ContentDigest.Valid { + return "" + } + refreshed, err := p.DB.RefreshMetadataCache(ecosystem, cacheKey, entry.ContentDigest.String, time.Now()) + if err != nil { + p.Logger.Warn("failed to refresh cached metadata", "ecosystem", ecosystem, "key", cacheKey, "error", err) + return "" + } + if !refreshed { + return "" + } + return entry.ContentDigest.String +} + // cacheMetadataBlob stores metadata bytes in storage and updates the // database. It returns the digest it recorded for the bytes, or "" when they // were not cached. diff --git a/internal/handler/handler_test.go b/internal/handler/handler_test.go index ffc0d5f..1096cdf 100644 --- a/internal/handler/handler_test.go +++ b/internal/handler/handler_test.go @@ -38,6 +38,7 @@ type mockStorage struct { signedURL string signErr error seekable bool + stores int } func newMockStorage() *mockStorage { @@ -55,6 +56,7 @@ func (s *mockStorage) Store(_ context.Context, path string, r io.Reader) (int64, return 0, "", err } s.files[path] = data + s.stores++ return int64(len(data)), sha256Hex(string(data)), nil } diff --git a/internal/handler/metadata_not_modified_test.go b/internal/handler/metadata_not_modified_test.go new file mode 100644 index 0000000..a16c82c --- /dev/null +++ b/internal/handler/metadata_not_modified_test.go @@ -0,0 +1,112 @@ +package handler + +import ( + "context" + "database/sql" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + "time" + + "github.com/git-pkgs/proxy/internal/database" +) + +// TestMetadataNotModifiedKeepsStoredDocument checks that a 304 from upstream +// refreshes the cache row without writing the unchanged document to storage +// again, and still serves it. +func TestMetadataNotModifiedKeepsStoredDocument(t *testing.T) { + const body = `{"v":1}` + var notModified atomic.Int32 + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set(headerETag, `"v1"`) + if r.Header.Get("If-None-Match") == `"v1"` { + notModified.Add(1) + w.WriteHeader(http.StatusNotModified) + return + } + w.Header().Set(headerContentType, contentTypeJSON) + _, _ = w.Write([]byte(body)) + })) + t.Cleanup(upstream.Close) + + proxy, db, store, _ := setupTestProxy(t) + proxy.CacheMetadata = true + proxy.MetadataTTL = time.Nanosecond // revalidate on every request + proxy.HTTPClient = upstream.Client() + + ctx := context.Background() + first := proxy.fetchMetadataDocument(ctx, "test", "pkg", upstream.URL+"/pkg", contentTypeJSON) + if first.err != nil { + t.Fatal(first.err) + } + stored, err := db.GetMetadataCache("test", "pkg") + if err != nil || stored == nil { + t.Fatalf("cache row after first fetch: %v, %v", stored, err) + } + + for range 2 { + res := proxy.fetchMetadataDocument(ctx, "test", "pkg", upstream.URL+"/pkg", contentTypeJSON) + if res.err != nil || string(res.body) != body || res.digest != first.digest { + t.Fatalf("revalidated fetch = %q, digest %q, err %v; want %q, digest %q", res.body, res.digest, res.err, body, first.digest) + } + } + if n := notModified.Load(); n != 2 { + t.Fatalf("upstream answered 304 %d times, want 2", n) + } + if store.stores != 1 { + t.Errorf("document stored %d times, want once", store.stores) + } + refreshed, err := db.GetMetadataCache("test", "pkg") + if err != nil { + t.Fatal(err) + } + if !refreshed.FetchedAt.Time.After(stored.FetchedAt.Time) { + t.Errorf("fetched_at = %v, want later than %v", refreshed.FetchedAt.Time, stored.FetchedAt.Time) + } + if refreshed.ContentDigest != stored.ContentDigest || refreshed.ETag != stored.ETag { + t.Errorf("row validators changed: %+v, want %+v", refreshed, stored) + } +} + +// TestMetadataNotModifiedStoresRowWithoutDigest checks that a 304 for a row +// written before digests were recorded stores the document as before, so the +// row gains its digest. +func TestMetadataNotModifiedStoresRowWithoutDigest(t *testing.T) { + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("If-None-Match") == `"v1"` { + w.WriteHeader(http.StatusNotModified) + return + } + t.Errorf("unconditional upstream request") + })) + t.Cleanup(upstream.Close) + + proxy, db, store, _ := setupTestProxy(t) + proxy.CacheMetadata = true + proxy.MetadataTTL = time.Nanosecond + proxy.HTTPClient = upstream.Client() + + path := metadataStoragePath("test", "pkg") + store.files[path] = []byte(`{"v":1}`) + if err := db.UpsertMetadataCache(&database.MetadataCacheEntry{ + Ecosystem: "test", + Name: "pkg", + StoragePath: path, + ETag: sql.NullString{String: `"v1"`, Valid: true}, + FetchedAt: sql.NullTime{Time: time.Now().Add(-time.Hour), Valid: true}, + }); err != nil { + t.Fatal(err) + } + + res := proxy.fetchMetadataDocument(context.Background(), "test", "pkg", upstream.URL+"/pkg", contentTypeJSON) + if res.err != nil || string(res.body) != `{"v":1}` { + t.Fatalf("fetch = %q, %v", res.body, res.err) + } + if want := "sha256:" + sha256Hex(`{"v":1}`); res.digest != want { + t.Errorf("digest = %q, want %q", res.digest, want) + } + if store.stores != 1 { + t.Errorf("document stored %d times, want once", store.stores) + } +} From d4e89b08da6968a4c02dd999e2dd93a46fa8028e Mon Sep 17 00:00:00 2001 From: simon chrzanowski Date: Sun, 18 Oct 2026 21:32:02 +0200 Subject: [PATCH 311/633] Resolve Composer downloads from the cached metadata (#494) Every Composer archive download fetched the package's p2 metadata straight from upstream to find the archive's URL, before the archive cache was even checked. A cached archive cost one or two uncached Packagist requests per package and install, and failed with a 502 while Packagist was down. With cache_metadata on, downloads now look the dist up in the metadata the proxy already caches for Composer, under the keys handlePackageMetadata uses. Within the TTL that needs no upstream request, and when upstream fails the stale copy still resolves the archive. A version or file the cached documents do not list, such as one published since they were cached, is looked up upstream as before. --- docs/configuration.md | 2 + internal/handler/composer.go | 158 +++++++++++------- .../handler/composer_download_lookup_test.go | 128 ++++++++++++++ internal/handler/composer_test.go | 12 +- 4 files changed, 235 insertions(+), 65 deletions(-) create mode 100644 internal/handler/composer_download_lookup_test.go diff --git a/docs/configuration.md b/docs/configuration.md index 06528ad..5fcdc23 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -928,6 +928,8 @@ cache_metadata: true Or via environment variable: `PROXY_CACHE_METADATA=true`. +Composer archive downloads look up their upstream URL in the same cached metadata, so a cached archive is served without asking Packagist again and stays installable while it is down. Only a version or file the cached metadata does not list yet is looked up upstream. + The `proxy mirror` command always enables metadata caching regardless of this setting. ### Metadata TTL diff --git a/internal/handler/composer.go b/internal/handler/composer.go index 53a5a65..121f228 100644 --- a/internal/handler/composer.go +++ b/internal/handler/composer.go @@ -639,50 +639,6 @@ func (h *ComposerHandler) handleDownload(w http.ResponseWriter, r *http.Request) return } - // We need to fetch the metadata to get the actual download URL since - // Packagist URLs include a hash. Packagist serves dev versions (e.g. - // "3.x-dev", "dev-master") from a separate "~dev" metadata file, while - // tagged releases live in the regular file. Try the file most likely to - // contain this version first, then fall back to the other so that both - // stable and dev versions resolve correctly. - metaURLs := h.metadataURLsForVersion(vendor, pkg, version) - - h.proxy.Logger.Debug("resolving download URL", - "package", packageName, "version", version, - "metadata_urls", metaURLs) - - var dist composerDist - for _, metaURL := range metaURLs { - found, err := h.findDownloadURLFromMetadata(r.Context(), metaURL, packageName, version) - if err != nil { - if writeUpstreamHeld(w, err) { - return - } - h.proxy.Logger.Error("failed to fetch metadata", "error", err, "url", metaURL) - http.Error(w, "failed to fetch metadata", http.StatusBadGateway) - return - } - if found.url != "" { - dist = found - break - } - } - - if dist.url == "" { - h.proxy.Logger.Debug("version not found in any metadata source", - "package", packageName, "version", version, - "tried_urls", metaURLs) - http.Error(w, "version not found", http.StatusNotFound) - return - } - - // Lock files name archive URLs directly, so a version filtered out of the - // metadata still has to be withheld here. - if h.shouldFilterVersion(canonicalPackagePURL("composer", packageName), packageName, version, dist.published) { - http.Error(w, "version not found", http.StatusNotFound) - return - } - // The archive is cached under the file name from the path, so fetch the // one that name stands for. Compare it as sent, the way rewriteDist handed // it out, query included. @@ -690,13 +646,18 @@ func (h *ComposerHandler) handleDownload(w http.ResponseWriter, r *http.Request) if r.URL.RawQuery != "" { requested += "?" + r.URL.RawQuery } - listedURL := dist.url - downloadURL, ok := upstreamDistURL(listedURL, dist.distType, requested, isDevVersion(version)) + files := h.metadataFilesForVersion(vendor, pkg, version) + dist, downloadURL, ok := h.cachedDist(r.Context(), files, packageName, version, requested) if !ok { - h.proxy.Logger.Info("composer file not in upstream metadata", - "package", packageName, "version", version, - "filename", requested, "listed_url", listedURL) - http.Error(w, "file not found", http.StatusNotFound) + if dist, downloadURL, ok = h.upstreamDist(w, r, files, packageName, version, requested); !ok { + return + } + } + + // Lock files name archive URLs directly, so a version filtered out of the + // metadata still has to be withheld here. + if h.shouldFilterVersion(canonicalPackagePURL("composer", packageName), packageName, version, dist.published) { + http.Error(w, "version not found", http.StatusNotFound) return } @@ -718,7 +679,7 @@ func (h *ComposerHandler) handleDownload(w http.ResponseWriter, r *http.Request) h.discardOtherCommit(packageName, version, cacheFilename, commit) } result, err := h.proxy.GetOrFetchArtifactFromURLWithIntegrity(r.Context(), "composer", packageName, version, cacheFilename, downloadURL, - composerIntegrity(dist.shasum, version, downloadURL == listedURL)) + composerIntegrity(dist.shasum, version, downloadURL == dist.url)) if err != nil { h.proxy.serveArtifactError(w, err, "failed to fetch package") return @@ -767,19 +728,98 @@ func isDevVersion(version string) bool { return strings.HasPrefix(version, "dev-") || strings.HasSuffix(version, "-dev") } -// metadataURLsForVersion returns the upstream metadata URLs to consult for a +// composerMetadataFile is one of the metadata documents a version's dist can +// be listed in: its upstream URL and the key the proxy caches it under. +type composerMetadataFile struct { + url, cacheKey string +} + +// metadataFilesForVersion returns the metadata documents to consult for a // given version, in priority order. Dev versions are served from the "~dev" // file, tagged releases from the regular file; the other file is included as a -// fallback so an unexpected classification still resolves. -func (h *ComposerHandler) metadataURLsForVersion(vendor, pkg, version string) []string { - repoURL, _ := h.sourceFor(vendor + "/" + pkg) - stable := fmt.Sprintf("%s/p2/%s/%s.json", repoURL, vendor, pkg) - dev := fmt.Sprintf("%s/p2/%s/%s~dev.json", repoURL, vendor, pkg) +// fallback so an unexpected classification still resolves. The cache keys are +// the ones handlePackageMetadata stores the documents under. +func (h *ComposerHandler) metadataFilesForVersion(vendor, pkg, version string) []composerMetadataFile { + file := func(name string) composerMetadataFile { + repoURL, cacheKey := h.sourceFor(name) + return composerMetadataFile{url: fmt.Sprintf("%s/p2/%s.json", repoURL, name), cacheKey: cacheKey} + } + stable := file(vendor + "/" + pkg) + dev := file(vendor + "/" + pkg + composerDevFile) if isDevVersion(version) { - return []string{dev, stable} + return []composerMetadataFile{dev, stable} } - return []string{stable, dev} + return []composerMetadataFile{stable, dev} +} + +// cachedDist looks up the dist of version in the metadata documents the proxy +// caches for Composer, returning it with the URL to download the requested +// file from. While the documents are fresh this costs no upstream request, +// and when upstream fails their cached copies still resolve the archive. ok is +// false when neither document lists a dist the requested file stands for, as +// for a version published after the metadata was cached, so the caller asks +// upstream instead. Without cache_metadata there is nothing to look in. +func (h *ComposerHandler) cachedDist(ctx context.Context, files []composerMetadataFile, packageName, version, requested string) (composerDist, string, bool) { + if !h.proxy.CacheMetadata || h.proxy.DB == nil { + return composerDist{}, "", false + } + for _, f := range files { + doc := h.proxy.fetchMetadataDocument(ctx, "composer", f.cacheKey, f.url, contentTypeJSON) + if doc.err != nil { + continue + } + dist, err := composerDistURL(doc.body, packageName, version) + if err != nil || dist.url == "" { + continue + } + if downloadURL, ok := upstreamDistURL(dist.url, dist.distType, requested, isDevVersion(version)); ok { + return dist, downloadURL, true + } + } + return composerDist{}, "", false +} + +// upstreamDist looks up the dist of version in the metadata upstream serves +// now, returning it with the URL to download the requested file from. When it +// cannot, it writes the error response and returns false. +func (h *ComposerHandler) upstreamDist(w http.ResponseWriter, r *http.Request, files []composerMetadataFile, packageName, version, requested string) (composerDist, string, bool) { + h.proxy.Logger.Debug("resolving download URL", + "package", packageName, "version", version) + + var dist composerDist + for _, f := range files { + found, err := h.findDownloadURLFromMetadata(r.Context(), f.url, packageName, version) + if err != nil { + if writeUpstreamHeld(w, err) { + return composerDist{}, "", false + } + h.proxy.Logger.Error("failed to fetch metadata", "error", err, "url", f.url) + http.Error(w, "failed to fetch metadata", http.StatusBadGateway) + return composerDist{}, "", false + } + if found.url != "" { + dist = found + break + } + } + + if dist.url == "" { + h.proxy.Logger.Debug("version not found in any metadata source", + "package", packageName, "version", version) + http.Error(w, "version not found", http.StatusNotFound) + return composerDist{}, "", false + } + + downloadURL, ok := upstreamDistURL(dist.url, dist.distType, requested, isDevVersion(version)) + if !ok { + h.proxy.Logger.Info("composer file not in upstream metadata", + "package", packageName, "version", version, + "filename", requested, "listed_url", dist.url) + http.Error(w, "file not found", http.StatusNotFound) + return composerDist{}, "", false + } + return dist, downloadURL, true } // composerDist is what a version's metadata says about its archive: the dist diff --git a/internal/handler/composer_download_lookup_test.go b/internal/handler/composer_download_lookup_test.go new file mode 100644 index 0000000..5ac4c75 --- /dev/null +++ b/internal/handler/composer_download_lookup_test.go @@ -0,0 +1,128 @@ +package handler + +import ( + "fmt" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/git-pkgs/registries/fetch" +) + +// composerLookupUpstream serves v/p's metadata listing versions and the +// archives they point at, counts metadata and archive requests, and fails +// every request while down is set. +type composerLookupUpstream struct { + h http.Handler + versions atomic.Value // []string + down atomic.Bool + metadata atomic.Int32 + archives atomic.Int32 +} + +func setupComposerLookup(t *testing.T, cacheMetadata bool, versions ...string) *composerLookupUpstream { + t.Helper() + u := &composerLookupUpstream{} + u.versions.Store(versions) + var srv *httptest.Server + srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if u.down.Load() { + http.Error(w, "down", http.StatusServiceUnavailable) + return + } + switch { + case r.URL.Path == "/p2/v/p.json": + u.metadata.Add(1) + var entries []string + for _, v := range u.versions.Load().([]string) { + entries = append(entries, fmt.Sprintf(`{"version":%q,"dist":{"url":%q,"type":"zip"}}`, v, srv.URL+"/archives/"+v+".zip")) + } + _, _ = fmt.Fprintf(w, `{"packages":{"v/p":[%s]}}`, strings.Join(entries, ",")) + case r.URL.Path == "/p2/v/p~dev.json": + u.metadata.Add(1) + http.NotFound(w, r) + case strings.HasPrefix(r.URL.Path, "/archives/"): + u.archives.Add(1) + _, _ = io.WriteString(w, "archive "+r.URL.Path) + default: + http.NotFound(w, r) + } + })) + t.Cleanup(srv.Close) + + proxy, _, _, _ := setupTestProxy(t) + proxy.CacheMetadata = cacheMetadata + proxy.MetadataTTL = time.Hour + fetcher := fetch.NewFetcher(fetch.WithHTTPClient(srv.Client()), fetch.WithMaxRetries(0)) + proxy.Fetcher = fetcher + t.Cleanup(func() { _ = fetcher.Close() }) + u.h = NewComposerHandlerWithUpstreams(proxy, "http://proxy", srv.URL, srv.URL).Routes() + return u +} + +func (u *composerLookupUpstream) get(t *testing.T, target string) *httptest.ResponseRecorder { + t.Helper() + w := httptest.NewRecorder() + u.h.ServeHTTP(w, httptest.NewRequest(http.MethodGet, target, nil)) + return w +} + +func (u *composerLookupUpstream) download(t *testing.T, version string) { + t.Helper() + w := u.get(t, "/files/v/p/"+version+"/"+version+".zip") + if want := "archive /archives/" + version + ".zip"; w.Code != http.StatusOK || w.Body.String() != want { + t.Fatalf("download %s: status %d, body %q, want 200 %q", version, w.Code, w.Body.String(), want) + } +} + +// TestComposerDownloadResolvesFromCachedMetadata checks that downloads find +// their archive in the metadata Composer was just served, instead of fetching +// it from upstream again for every archive. +func TestComposerDownloadResolvesFromCachedMetadata(t *testing.T) { + u := setupComposerLookup(t, true, "1.0.0") + if w := u.get(t, "/p2/v/p.json"); w.Code != http.StatusOK { + t.Fatalf("metadata: status %d", w.Code) + } + u.download(t, "1.0.0") + u.download(t, "1.0.0") + if n := u.metadata.Load(); n != 1 { + t.Errorf("upstream metadata requests = %d, want 1 (the metadata request itself)", n) + } + if n := u.archives.Load(); n != 1 { + t.Errorf("upstream archive requests = %d, want 1", n) + } +} + +// TestComposerCachedArchiveServedWhileUpstreamDown checks that a cached +// archive stays installable when upstream fails, rather than failing on the +// metadata lookup that only names where to fetch it from. +func TestComposerCachedArchiveServedWhileUpstreamDown(t *testing.T) { + u := setupComposerLookup(t, true, "1.0.0") + u.download(t, "1.0.0") + u.down.Store(true) + u.download(t, "1.0.0") +} + +// TestComposerDownloadOfVersionNewerThanCachedMetadata checks that a version +// published after the metadata was cached, which a lock file can name before +// the cached copy expires, is still looked up upstream. +func TestComposerDownloadOfVersionNewerThanCachedMetadata(t *testing.T) { + u := setupComposerLookup(t, true, "1.0.0") + u.download(t, "1.0.0") + u.versions.Store([]string{"1.0.0", "1.1.0"}) + u.download(t, "1.1.0") +} + +// TestComposerDownloadWithoutMetadataCache checks that without cache_metadata +// a download asks upstream for the metadata once, as it always has. +func TestComposerDownloadWithoutMetadataCache(t *testing.T) { + u := setupComposerLookup(t, false, "1.0.0") + u.download(t, "1.0.0") + if n := u.metadata.Load(); n != 1 { + t.Errorf("upstream metadata requests = %d, want 1", n) + } +} diff --git a/internal/handler/composer_test.go b/internal/handler/composer_test.go index e6a9962..d2955eb 100644 --- a/internal/handler/composer_test.go +++ b/internal/handler/composer_test.go @@ -505,16 +505,16 @@ func TestComposerDownloadDevVersionUsesDevMetadata(t *testing.T) { // FIXED behavior: the handler consults the ~dev file (it is first in the // candidate list for dev versions) and resolves the dist URL. - urls := h.metadataURLsForVersion(vendor, name, version) - if len(urls) == 0 || !strings.HasSuffix(urls[0], "/p2/phpmd/phpmd~dev.json") { - t.Fatalf("dev version should consult the ~dev metadata file first, got %v", urls) + files := h.metadataFilesForVersion(vendor, name, version) + if len(files) == 0 || !strings.HasSuffix(files[0].url, "/p2/phpmd/phpmd~dev.json") || files[0].cacheKey != "phpmd/phpmd~dev" { + t.Fatalf("dev version should consult the ~dev metadata file first, got %v", files) } var resolved string - for _, u := range urls { - found, err := h.findDownloadURLFromMetadata(ctx, u, pkg, version) + for _, f := range files { + found, err := h.findDownloadURLFromMetadata(ctx, f.url, pkg, version) if err != nil { - t.Fatalf("unexpected error fetching metadata %q: %v", u, err) + t.Fatalf("unexpected error fetching metadata %q: %v", f.url, err) } resolved = found.url if resolved != "" { From 2e0f1322a2a99039c95f13bfb20da7b3e88b266f Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Mon, 19 Oct 2026 06:47:28 +0100 Subject: [PATCH 312/633] Check gofmt in CI and reformat (#528) Claude-Session: https://claude.ai/code/session_01NxsA75HBjdWQKcdx7FDTDc --- .golangci.yml | 4 ++++ internal/handler/npm_test.go | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.golangci.yml b/.golangci.yml index 9d4b957..57c0f69 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -26,3 +26,7 @@ linters: - goconst - dupl - mnd + +formatters: + enable: + - gofmt diff --git a/internal/handler/npm_test.go b/internal/handler/npm_test.go index a1ad53d..f71c59c 100644 --- a/internal/handler/npm_test.go +++ b/internal/handler/npm_test.go @@ -1065,7 +1065,7 @@ func TestNPMMetadataPath(t *testing.T) { func TestNPMVersionMetadataKeepsVersionSegment(t *testing.T) { tests := []struct { request, upstreamPath, wantTarball string - body string + body string }{ { request: "/lodash/4.17.21", From cae9c08dfe2e8d959ba247c9358e99c8e00b21d2 Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Mon, 19 Oct 2026 06:47:34 +0100 Subject: [PATCH 313/633] Stream conda repodata through the cooldown filter (#527) Repodata over metadata_max_size returned 502 with cooldown enabled, which covers conda-forge's noarch and linux-64 channels. Claude-Session: https://claude.ai/code/session_01NxsA75HBjdWQKcdx7FDTDc --- docs/configuration.md | 2 +- internal/handler/conda.go | 129 ++++++++++-------- internal/handler/conda_encoding_test.go | 102 ++++++++++++++- internal/handler/conda_test.go | 13 +- internal/jsonstream/jsonstream.go | 165 ++++++++++++++++++++++++ internal/jsonstream/jsonstream_test.go | 140 ++++++++++++++++++++ 6 files changed, 487 insertions(+), 64 deletions(-) create mode 100644 internal/jsonstream/jsonstream.go create mode 100644 internal/jsonstream/jsonstream_test.go diff --git a/docs/configuration.md b/docs/configuration.md index 5fcdc23..235c8dc 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -948,7 +948,7 @@ When upstream is unreachable and the cached entry is past its TTL, the proxy ser ### Metadata size limit -Upstream metadata responses are buffered in memory before being rewritten and served. `metadata_max_size` caps that buffer to protect against OOM from a misbehaving upstream. Some npm packages with thousands of versions (for example `renovate`) exceed the 100 MB default, so raise this if you see `metadata response exceeds size limit` in the logs. +Upstream metadata responses are buffered in memory before being rewritten and served. `metadata_max_size` caps that buffer to protect against OOM from a misbehaving upstream. Some npm packages with thousands of versions (for example `renovate`) exceed the 100 MB default, so raise this if you see `metadata response exceeds size limit` in the logs. Conda `repodata.json` with cooldown enabled is filtered as it streams and isn't held in memory, so this limit doesn't apply to it. ```yaml metadata_max_size: "100MB" # default diff --git a/internal/handler/conda.go b/internal/handler/conda.go index d7b8b3f..aecd01a 100644 --- a/internal/handler/conda.go +++ b/internal/handler/conda.go @@ -1,6 +1,7 @@ package handler import ( + "bufio" "compress/gzip" "encoding/json" "io" @@ -9,6 +10,7 @@ import ( "strings" "time" + "github.com/git-pkgs/proxy/internal/jsonstream" "github.com/git-pkgs/proxy/internal/retention" "github.com/git-pkgs/purl" ) @@ -209,75 +211,90 @@ func (h *CondaHandler) handleRepodata(w http.ResponseWriter, r *http.Request) { defer func() { _ = decoded.Close() }() reader = decoded } - body, err := h.proxy.ReadMetadata(reader) - if err != nil { - http.Error(w, "failed to read response", http.StatusBadGateway) - return - } - - filtered, err := h.applyCooldownFiltering(body) - if err != nil { - h.proxy.Logger.Warn("failed to filter repodata, proxying original", "error", err) - w.Header().Set(headerContentType, "application/json") - _, _ = w.Write(body) - return - } - + // conda-forge's linux-64 repodata.json is several hundred MB, so it is + // filtered as it streams rather than read through ReadMetadata. Once + // output has started, an error aborts the response so the client never + // takes a truncated or unfiltered index as complete. + out := &countingWriter{w: w} + buffered := bufio.NewWriterSize(out, repodataWriteBuffer) w.Header().Set(headerContentType, "application/json") - _, _ = w.Write(filtered) + err = h.filterRepodata(io.LimitReader(reader, maxRepodataSize), buffered) + if err == nil { + err = buffered.Flush() + } + if err != nil { + h.proxy.Logger.Error("failed to filter conda repodata", "url", upstreamURL, "error", err) + if out.n == 0 { + http.Error(w, "failed to read response", http.StatusBadGateway) + return + } + panic(http.ErrAbortHandler) + } +} + +// countingWriter records whether anything has reached the response yet. +type countingWriter struct { + w io.Writer + n int64 +} + +func (c *countingWriter) Write(p []byte) (int, error) { + n, err := c.w.Write(p) + c.n += int64(n) + return n, err } // condaTimestampDivisor converts Conda's millisecond timestamps to seconds. const condaTimestampDivisor = 1000 -// applyCooldownFiltering removes entries from repodata.json that were -// published too recently based on their timestamp field. -func (h *CondaHandler) applyCooldownFiltering(body []byte) ([]byte, error) { - if h.proxy.Cooldown == nil || !h.proxy.Cooldown.Enabled() { - return body, nil - } +const ( + repodataWriteBuffer = 64 << 10 + // maxRepodataSize stops a runaway upstream body. It is well above the + // largest conda-forge index, which is under 500 MB. + maxRepodataSize = 4 << 30 +) - var repodata map[string]any - if err := json.Unmarshal(body, &repodata); err != nil { - return nil, err - } - - for _, key := range []string{"packages", "packages.conda"} { - packages, ok := repodata[key].(map[string]any) - if !ok { - continue +// filterRepodata copies repodata.json from r to w, leaving out entries in +// packages and packages.conda that are still inside their cooldown. Kept +// entries and every other member are written with their upstream bytes. +func (h *CondaHandler) filterRepodata(r io.Reader, w io.Writer) error { + return jsonstream.CopyObject(json.NewDecoder(r), w, func(m *jsonstream.Member) error { + if m.Key != "packages" && m.Key != "packages.conda" { + return m.Copy() } - - for filename, entry := range packages { - entryMap, ok := entry.(map[string]any) - if !ok { - continue + return m.Object(func(entry *jsonstream.Member) error { + raw, err := entry.Raw() + if err != nil { + return err } - - ts, ok := entryMap["timestamp"].(float64) - if !ok || ts == 0 { - continue + if h.inCooldown(entry.Key, raw) { + return nil } + return entry.Write(raw) + }) + }) +} - publishedAt := time.Unix(int64(ts)/condaTimestampDivisor, 0) +type repodataEntry struct { + Name string `json:"name"` + Version string `json:"version"` + Timestamp float64 `json:"timestamp"` +} - name, _ := entryMap["name"].(string) - if name == "" { - continue - } - - packagePURL := canonicalPackagePURL("conda", name) - - if !h.proxy.Cooldown.IsAllowed("conda", packagePURL, publishedAt) { - version, _ := entryMap["version"].(string) - h.proxy.Logger.Info("cooldown: filtering conda package", - "name", name, "version", version, "filename", filename) - delete(packages, filename) - } - } +// inCooldown reports whether a repodata entry was published too recently. +// Entries without a name or timestamp are kept. +func (h *CondaHandler) inCooldown(filename string, raw []byte) bool { + var entry repodataEntry + if err := json.Unmarshal(raw, &entry); err != nil || entry.Name == "" || entry.Timestamp == 0 { + return false } - - return json.Marshal(repodata) + publishedAt := time.Unix(int64(entry.Timestamp)/condaTimestampDivisor, 0) + if h.proxy.Cooldown.IsAllowed("conda", canonicalPackagePURL("conda", entry.Name), publishedAt) { + return false + } + h.proxy.Logger.Info("cooldown: filtering conda package", + "name", entry.Name, "version", entry.Version, "filename", filename) + return true } // proxyCached forwards a metadata request with caching. diff --git a/internal/handler/conda_encoding_test.go b/internal/handler/conda_encoding_test.go index b453a00..4aa5951 100644 --- a/internal/handler/conda_encoding_test.go +++ b/internal/handler/conda_encoding_test.go @@ -3,6 +3,8 @@ package handler import ( "bytes" "encoding/json" + "fmt" + "io" "net/http" "net/http/httptest" "strings" @@ -194,7 +196,6 @@ func TestCondaRepodataCooldownRejectsInvalidGzip(t *testing.T) { }{ {"invalid header", []byte("not gzip")}, {"truncated body", compressed[:len(compressed)-4]}, - {"decoded size exceeds limit", compressed}, } { t.Run(tt.name, func(t *testing.T) { upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -205,9 +206,6 @@ func TestCondaRepodataCooldownRejectsInvalidGzip(t *testing.T) { proxy := testProxy() proxy.HTTPClient = upstream.Client() proxy.Cooldown = &cooldown.Config{Default: "3d"} - if tt.name == "decoded size exceeds limit" { - proxy.MetadataMaxSize = int64(len(compressed)) - } handler := NewCondaHandlerWithUpstream(proxy, "http://proxy.local", upstream.URL).Routes() w := httptest.NewRecorder() handler.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/conda-forge/linux-64/repodata.json", nil)) @@ -217,3 +215,99 @@ func TestCondaRepodataCooldownRejectsInvalidGzip(t *testing.T) { }) } } + +// largeRepodata builds repodata with n old and n new entries in each of +// packages and packages.conda. +func largeRepodata(n int) []byte { + old := time.Now().Add(-30 * 24 * time.Hour).UnixMilli() + recent := time.Now().UnixMilli() + var b strings.Builder + b.WriteString(`{"info":{"subdir":"linux-64"},"repodata_version":1`) + for _, key := range []string{"packages", "packages.conda"} { + fmt.Fprintf(&b, `,%q:{`, key) + for i := range 2 * n { + if i > 0 { + b.WriteByte(',') + } + ts, kind := old, "old" + if i%2 == 1 { + ts, kind = recent, "new" + } + fmt.Fprintf(&b, `"pkg-%d-%s.conda":{"name":"pkg%d","version":"1.0","timestamp":%d,"depends":["python >=3.10,<4","libgcc >=13"],"md5":"%032d"}`, + i, kind, i, ts, i) + } + b.WriteByte('}') + } + b.WriteString(`,"removed":[]}`) + return []byte(b.String()) +} + +func TestCondaRepodataCooldownStreamsPastMetadataLimit(t *testing.T) { + plain := largeRepodata(2000) + compressed := gzipPayload(t, plain) + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set(headerContentEncoding, "gzip") + _, _ = w.Write(compressed) + })) + defer upstream.Close() + + proxy := testProxy() + proxy.HTTPClient = upstream.Client() + proxy.Cooldown = &cooldown.Config{Default: "3d"} + proxy.MetadataMaxSize = int64(len(plain) / 10) + handler := NewCondaHandlerWithUpstream(proxy, "http://proxy.local", upstream.URL).Routes() + + w := httptest.NewRecorder() + handler.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/conda-forge/linux-64/repodata.json", nil)) + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want 200 for repodata %d times the metadata limit", w.Code, len(plain)/int(proxy.MetadataMaxSize)) + } + var result struct { + Info map[string]any `json:"info"` + Packages map[string]json.RawMessage `json:"packages"` + Conda map[string]json.RawMessage `json:"packages.conda"` + Removed []string `json:"removed"` + } + if err := json.Unmarshal(w.Body.Bytes(), &result); err != nil { + t.Fatalf("response is not valid JSON: %v", err) + } + if len(result.Packages) != 2000 || len(result.Conda) != 2000 || result.Info["subdir"] != "linux-64" || result.Removed == nil { + t.Fatalf("got %d packages, %d conda packages, info %v", len(result.Packages), len(result.Conda), result.Info) + } + for name := range result.Packages { + if strings.HasSuffix(name, "-new.conda") { + t.Fatalf("%s is inside the cooldown but was kept", name) + } + } + want := `"pkg-0-old.conda":{"name":"pkg0","version":"1.0","timestamp":` + if !strings.Contains(w.Body.String(), want) { + t.Errorf("kept entries should keep their upstream bytes") + } +} + +func TestCondaRepodataCooldownAbortsTruncatedStream(t *testing.T) { + compressed := gzipPayload(t, largeRepodata(2000)) + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set(headerContentEncoding, "gzip") + _, _ = w.Write(compressed[:len(compressed)*3/4]) + })) + defer upstream.Close() + + proxy := testProxy() + proxy.HTTPClient = upstream.Client() + proxy.Cooldown = &cooldown.Config{Default: "3d"} + server := httptest.NewServer(NewCondaHandlerWithUpstream(proxy, "http://proxy.local", upstream.URL).Routes()) + defer server.Close() + + resp, err := server.Client().Get(server.URL + "/conda-forge/linux-64/repodata.json") + if err != nil { + t.Fatal(err) + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode != http.StatusOK { + t.Fatalf("status = %d; want the stream to start before the upstream body breaks", resp.StatusCode) + } + if _, err := io.ReadAll(resp.Body); err == nil { + t.Fatal("read a truncated repodata response without an error") + } +} diff --git a/internal/handler/conda_test.go b/internal/handler/conda_test.go index 567bb6f..0c4c52a 100644 --- a/internal/handler/conda_test.go +++ b/internal/handler/conda_test.go @@ -1,6 +1,7 @@ package handler import ( + "bytes" "encoding/json" "log/slog" "net/http" @@ -105,7 +106,7 @@ func TestCondaCooldownFiltering(t *testing.T) { proxyURL: "http://localhost:8080", } - filtered, err := h.applyCooldownFiltering(body) + filtered, err := filterRepodataBytes(h, body) if err != nil { t.Fatal(err) } @@ -164,7 +165,7 @@ func TestCondaCooldownFilteringWithPackageOverride(t *testing.T) { proxyURL: "http://localhost:8080", } - filtered, err := h.applyCooldownFiltering(body) + filtered, err := filterRepodataBytes(h, body) if err != nil { t.Fatal(err) } @@ -208,7 +209,7 @@ func TestCondaCooldownFilteringNoTimestamp(t *testing.T) { proxyURL: "http://localhost:8080", } - filtered, err := h.applyCooldownFiltering(body) + filtered, err := filterRepodataBytes(h, body) if err != nil { t.Fatal(err) } @@ -336,3 +337,9 @@ func TestCondaRetentionKey(t *testing.T) { t.Error("condaRetentionKey(nil) accepted") } } + +func filterRepodataBytes(h *CondaHandler, body []byte) ([]byte, error) { + var out bytes.Buffer + err := h.filterRepodata(bytes.NewReader(body), &out) + return out.Bytes(), err +} diff --git a/internal/jsonstream/jsonstream.go b/internal/jsonstream/jsonstream.go new file mode 100644 index 0000000..d6e8de0 --- /dev/null +++ b/internal/jsonstream/jsonstream.go @@ -0,0 +1,165 @@ +// Package jsonstream copies JSON objects from a reader to a writer one member +// at a time, so a document can be filtered without holding it in memory. +// Values that are kept are written with their original bytes. +package jsonstream + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "io" +) + +// ErrNotObject is returned when a value expected to be an object is not. +var ErrNotObject = errors.New("JSON value is not an object") + +var ( + errConsumed = errors.New("jsonstream: member value already consumed") + errUnconsumed = errors.New("jsonstream: Write before the member value was read") +) + +// Member is one member of an object being copied. Its value must be consumed +// exactly once, with Raw, Skip or Object. +type Member struct { + Key string + dec *json.Decoder + out *objectWriter + consumed bool +} + +// Raw reads the member's value without writing it. Follow it with Write to +// keep the member. +func (m *Member) Raw() (json.RawMessage, error) { + if m.consumed { + return nil, errConsumed + } + m.consumed = true + var raw json.RawMessage + if err := m.dec.Decode(&raw); err != nil { + return nil, err + } + return raw, nil +} + +// Write writes the member with value as its raw JSON value. +func (m *Member) Write(value []byte) error { + if !m.consumed { + return errUnconsumed + } + if err := m.out.key(m.Key); err != nil { + return err + } + _, err := m.out.w.Write(value) + return err +} + +// Copy writes the member with its value unchanged. +func (m *Member) Copy() error { + raw, err := m.Raw() + if err != nil { + return err + } + return m.Write(raw) +} + +// Skip reads the member's value and leaves the member out. +func (m *Member) Skip() error { + _, err := m.Raw() + return err +} + +// Object writes the member and copies its value, which must be an object, +// calling fn for each of its members. A null value is written as null. +func (m *Member) Object(fn func(*Member) error) error { + if m.consumed { + return errConsumed + } + m.consumed = true + if err := m.out.key(m.Key); err != nil { + return err + } + return copyObject(m.dec, m.out.w, fn, true) +} + +// CopyObject copies the object read from dec to w, calling fn for each +// member. fn decides what is written by how it consumes the member's value; a +// member whose value fn leaves unconsumed is copied unchanged. +func CopyObject(dec *json.Decoder, w io.Writer, fn func(*Member) error) error { + if err := copyObject(dec, w, fn, false); err != nil { + return err + } + if _, err := dec.Token(); err != io.EOF { + if err == nil { + err = errors.New("unexpected data after the top-level object") + } + return err + } + return nil +} + +func copyObject(dec *json.Decoder, w io.Writer, fn func(*Member) error, allowNull bool) error { + tok, err := dec.Token() + if err != nil { + return err + } + if tok == nil && allowNull { + _, err := io.WriteString(w, "null") + return err + } + if tok != json.Delim('{') { + return ErrNotObject + } + if _, err := io.WriteString(w, "{"); err != nil { + return err + } + out := &objectWriter{w: w} + for dec.More() { + tok, err := dec.Token() + if err != nil { + return err + } + key, ok := tok.(string) + if !ok { + return fmt.Errorf("object key is %T, not a string", tok) + } + m := &Member{Key: key, dec: dec, out: out} + if err := fn(m); err != nil { + return err + } + if !m.consumed { + if err := m.Copy(); err != nil { + return err + } + } + } + if _, err := dec.Token(); err != nil { + return err + } + _, err = io.WriteString(w, "}") + return err +} + +type objectWriter struct { + w io.Writer + wrote bool +} + +// key writes the separator and key for the next member. Keys are re-encoded +// without HTML escaping, so plain keys keep their original bytes. +func (o *objectWriter) key(k string) error { + var buf bytes.Buffer + if o.wrote { + buf.WriteByte(',') + } + enc := json.NewEncoder(&buf) + enc.SetEscapeHTML(false) + if err := enc.Encode(k); err != nil { + return err + } + buf.Truncate(buf.Len() - 1) // Encode appends a newline + buf.WriteByte(':') + o.wrote = true + _, err := o.w.Write(buf.Bytes()) + return err +} diff --git a/internal/jsonstream/jsonstream_test.go b/internal/jsonstream/jsonstream_test.go new file mode 100644 index 0000000..1fc8ea8 --- /dev/null +++ b/internal/jsonstream/jsonstream_test.go @@ -0,0 +1,140 @@ +package jsonstream + +import ( + "bytes" + "encoding/json" + "errors" + "strings" + "testing" +) + +func copyString(t *testing.T, in string, fn func(*Member) error) (string, error) { + t.Helper() + var out bytes.Buffer + err := CopyObject(json.NewDecoder(strings.NewReader(in)), &out, fn) + return out.String(), err +} + +func TestCopyObjectKeepsValueBytes(t *testing.T) { + in := `{"a": 1.50, "b" : {"x": "<&>", "y":[1, 2]}, "c":null, "d":"é"}` + got, err := copyString(t, in, func(m *Member) error { return nil }) + if err != nil { + t.Fatal(err) + } + if want := `{"a":1.50,"b":{"x": "<&>", "y":[1, 2]},"c":null,"d":"é"}`; got != want { + t.Errorf("got %s\nwant %s", got, want) + } +} + +func TestCopyObjectFiltersNested(t *testing.T) { + in := `{"info":{"subdir":"noarch"},"packages":{"keep":{"n":1},"drop":{"n":2},"also":{"n":3}},"tail":[]}` + got, err := copyString(t, in, func(m *Member) error { + if m.Key != "packages" { + return m.Copy() + } + return m.Object(func(entry *Member) error { + raw, err := entry.Raw() + if err != nil { + return err + } + if entry.Key == "drop" { + return nil + } + return entry.Write(raw) + }) + }) + if err != nil { + t.Fatal(err) + } + if want := `{"info":{"subdir":"noarch"},"packages":{"keep":{"n":1},"also":{"n":3}},"tail":[]}`; got != want { + t.Errorf("got %s\nwant %s", got, want) + } + if !json.Valid([]byte(got)) { + t.Error("output is not valid JSON") + } +} + +func TestCopyObjectSkipReplaceAndEmpty(t *testing.T) { + got, err := copyString(t, `{"first":1,"secret":"x","n":2,"empty":{}}`, func(m *Member) error { + switch m.Key { + case "first", "secret": + return m.Skip() + case "n": + if _, err := m.Raw(); err != nil { + return err + } + return m.Write([]byte(`3`)) + case "empty": + return m.Object(func(*Member) error { return nil }) + } + return nil + }) + if err != nil { + t.Fatal(err) + } + if want := `{"n":3,"empty":{}}`; got != want { + t.Errorf("got %s, want %s", got, want) + } +} + +func TestCopyObjectKeys(t *testing.T) { + got, err := copyString(t, `{"a Date: Mon, 19 Oct 2026 08:58:13 +0200 Subject: [PATCH 314/633] Only cache real pub archives on download (#520) * Pub downloads without .tar.gz fetched the HTML version page from pub.dev and cached it as the archive, so one request broke that version for everyone after; they now get a 404, and the upstream URL is built from the escaped name and version so an encoded ? or # can't become a query or fragment. * Pub archives that older versions cached from a path without .tar.gz are now discarded and fetched again, so versions that were already broken heal on the next download. --- internal/handler/pub.go | 53 ++++++++++++++-- internal/handler/pub_test.go | 114 +++++++++++++++++++++++++++++++++++ 2 files changed, 161 insertions(+), 6 deletions(-) diff --git a/internal/handler/pub.go b/internal/handler/pub.go index 12ff6d9..1fc6997 100644 --- a/internal/handler/pub.go +++ b/internal/handler/pub.go @@ -5,17 +5,24 @@ import ( "errors" "fmt" "net/http" + "net/url" + "regexp" "strings" "time" + "github.com/git-pkgs/proxy/internal/database" "github.com/git-pkgs/proxy/internal/retention" ) const ( - pubUpstream = "https://pub.dev" - pubPathParts = 2 // name + version in path split by /versions/ + pubUpstream = "https://pub.dev" + pubPathParts = 2 // name + version in path split by /versions/ + pubArchiveExt = ".tar.gz" ) +// pubPackageName matches pub package names, which have to be Dart identifiers. +var pubPackageName = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`) + // Packages are stored under pkg:pub/{name} with the name from the download // path, and purl keeps pub names as they are, so package rules use the // default key mapping. @@ -69,19 +76,27 @@ func (h *PubHandler) handleDownload(w http.ResponseWriter, r *http.Request) { return } name := parts[0] - version := strings.TrimSuffix(parts[1], ".tar.gz") + // Only the archive is cached. Without the suffix the upstream path is the + // HTML version page on pub.dev, which must not land under the archive key. + version, ok := strings.CutSuffix(parts[1], pubArchiveExt) + if !ok { + http.Error(w, "not found", http.StatusNotFound) + return + } - if name == "" || version == "" { + if !pubPackageName.MatchString(name) || version == "" || strings.Contains(version, "/") { http.Error(w, "invalid request", http.StatusBadRequest) return } - filename := fmt.Sprintf("%s-%s.tar.gz", name, version) + filename := name + "-" + version + pubArchiveExt h.proxy.Logger.Info("pub download request", "name", name, "version", version) - downloadURL := h.upstreamURL + r.URL.Path + downloadURL := fmt.Sprintf("%s/packages/%s/versions/%s%s", + h.upstreamURL, url.PathEscape(name), url.PathEscape(version), pubArchiveExt) + h.discardVersionPage(name, version, filename) result, err := h.proxy.GetOrFetchArtifactFromURL( r.Context(), "pub", name, version, filename, downloadURL, ) @@ -93,6 +108,32 @@ func (h *PubHandler) handleDownload(w http.ResponseWriter, r *http.Request) { ServeArtifactRequest(w, r, result) } +// discardVersionPage clears a cached archive that was fetched from a path +// without the archive suffix. Proxy versions before the suffix check cached +// pub.dev's HTML version page under the archive's name that way. +func (h *PubHandler) discardVersionPage(name, version, filename string) { + _, versionPURL, err := packagePURLStrings("pub", name, version) + if err != nil { + return + } + record, err := h.proxy.DB.GetArtifact(versionPURL, filename) + if err != nil { + h.proxy.Logger.Warn("failed to read pub cache record", + "purl", versionPURL, "filename", filename, "error", err) + return + } + if record == nil || !record.StoragePath.Valid || record.Source == database.SourceLocal || + strings.HasSuffix(record.UpstreamURL, pubArchiveExt) { + return + } + h.proxy.Logger.Warn("cached pub archive was fetched without the archive suffix, discarding", + "purl", versionPURL, "filename", filename, "upstream_url", record.UpstreamURL) + if err := h.proxy.DB.DiscardArtifact(versionPURL, filename, record.StoragePath.String); err != nil { + h.proxy.Logger.Warn("failed to discard pub cache record", + "purl", versionPURL, "filename", filename, "error", err) + } +} + // handlePackageMetadata proxies package metadata and rewrites archive URLs. func (h *PubHandler) handlePackageMetadata(w http.ResponseWriter, r *http.Request) { name := r.PathValue("name") diff --git a/internal/handler/pub_test.go b/internal/handler/pub_test.go index 8a4c098..29526e0 100644 --- a/internal/handler/pub_test.go +++ b/internal/handler/pub_test.go @@ -2,11 +2,16 @@ package handler import ( "encoding/json" + "io" "log/slog" + "net/http" + "net/http/httptest" + "strings" "testing" "time" "github.com/git-pkgs/cooldown" + "github.com/git-pkgs/registries/fetch" ) func TestPubRewriteMetadata(t *testing.T) { @@ -92,3 +97,112 @@ func TestPubRewriteMetadataCooldown(t *testing.T) { t.Errorf("latest version = %v, want 1.0.0", latest["version"]) } } + +// Without the .tar.gz suffix pub.dev answers with the HTML version page, which +// would otherwise be cached under the archive's key. +func TestPubDownloadRequiresArchiveSuffix(t *testing.T) { + proxy, db, _, fetcher := setupTestProxy(t) + fetcher.artifact = &fetch.Artifact{ + Body: io.NopCloser(strings.NewReader("version page")), + ContentType: "text/html", + } + h := NewPubHandlerWithUpstream(proxy, "http://localhost", "https://pub.example.com") + + req := httptest.NewRequest(http.MethodGet, "/packages/http/versions/1.2.0", nil) + w := httptest.NewRecorder() + h.Routes().ServeHTTP(w, req) + + if w.Code != http.StatusNotFound { + t.Errorf("status = %d, want %d", w.Code, http.StatusNotFound) + } + if fetcher.fetchCalled { + t.Errorf("upstream fetched %q", fetcher.fetchedURL) + } + art, err := db.GetArtifact("pkg:pub/http@1.2.0", "http-1.2.0.tar.gz") + if err != nil { + t.Fatalf("GetArtifact: %v", err) + } + if art != nil { + t.Errorf("artifact cached from %q", art.UpstreamURL) + } +} + +func TestPubDownloadRejectsEncodedQueryInName(t *testing.T) { + proxy, _, _, fetcher := setupTestProxy(t) + fetcher.artifact = &fetch.Artifact{ + Body: io.NopCloser(strings.NewReader("archive")), + ContentType: "application/octet-stream", + } + h := NewPubHandlerWithUpstream(proxy, "http://localhost", "https://pub.example.com") + + for _, path := range []string{ + "/packages/http%3Fx/versions/1.2.0.tar.gz", + "/packages/http%23x/versions/1.2.0.tar.gz", + "/packages/http/versions/1.2.0%3Fx.tar.gz", + } { + fetcher.fetchCalled = false + fetcher.fetchedURL = "" + req := httptest.NewRequest(http.MethodGet, path, nil) + w := httptest.NewRecorder() + h.Routes().ServeHTTP(w, req) + + if strings.ContainsAny(fetcher.fetchedURL, "?#") { + t.Errorf("%s: upstream URL %q carries a query or fragment", path, fetcher.fetchedURL) + } + if strings.Contains(path, "/http%") && fetcher.fetchCalled { + t.Errorf("%s: invalid package name reached upstream as %q", path, fetcher.fetchedURL) + } + } +} + +// Older versions cached the HTML version page as the archive. Such an entry +// is fetched again instead of being served. +func TestPubDownloadRefetchesCachedVersionPage(t *testing.T) { + proxy, db, store, fetcher := setupTestProxy(t) + seedPackage(t, db, store, "pub", "http", "1.2.0", "http-1.2.0.tar.gz", "version page") + art, err := db.GetArtifact("pkg:pub/http@1.2.0", "http-1.2.0.tar.gz") + if err != nil || art == nil { + t.Fatalf("GetArtifact: %v", err) + } + art.UpstreamURL = "https://pub.dev/packages/http/versions/1.2.0" + if err := db.UpsertArtifact(art); err != nil { + t.Fatalf("UpsertArtifact: %v", err) + } + fetcher.artifact = &fetch.Artifact{ + Body: io.NopCloser(strings.NewReader("archive")), + ContentType: "application/octet-stream", + } + h := NewPubHandlerWithUpstream(proxy, "http://localhost", "https://pub.example.com") + + req := httptest.NewRequest(http.MethodGet, "/packages/http/versions/1.2.0.tar.gz", nil) + w := httptest.NewRecorder() + h.Routes().ServeHTTP(w, req) + + if !fetcher.fetchCalled { + t.Fatal("cached version page was served without fetching upstream") + } + if got := w.Body.String(); got != "archive" { + t.Errorf("body = %q, want %q", got, "archive") + } +} + +func TestPubDownloadServesCachedArchive(t *testing.T) { + proxy, db, store, fetcher := setupTestProxy(t) + seedPackage(t, db, store, "pub", "http", "1.2.0", "http-1.2.0.tar.gz", "cached archive") + fetcher.artifact = &fetch.Artifact{ + Body: io.NopCloser(strings.NewReader("refetched")), + ContentType: "application/octet-stream", + } + h := NewPubHandlerWithUpstream(proxy, "http://localhost", "https://pub.example.com") + + req := httptest.NewRequest(http.MethodGet, "/packages/http/versions/1.2.0.tar.gz", nil) + w := httptest.NewRecorder() + h.Routes().ServeHTTP(w, req) + + if fetcher.fetchCalled { + t.Errorf("cached archive fetched again from %q", fetcher.fetchedURL) + } + if got := w.Body.String(); got != "cached archive" { + t.Errorf("body = %q, want %q", got, "cached archive") + } +} From bf0c784feb24fba7148d41f92d13b7d41bf9cae4 Mon Sep 17 00:00:00 2001 From: Abhinav Gautam Date: Mon, 19 Oct 2026 12:29:39 +0530 Subject: [PATCH 315/633] Add Chef Supermarket proxy support for Berkshelf and Policyfiles (#518) * Add Chef Supermarket proxy support for Berkshelf and Policyfiles * Scope cached Chef cookbook archives to the upstream Supermarket --- README.md | 21 +- cmd/proxy/main.go | 1 + cmd/proxy/main_test.go | 1 + config.example.yaml | 4 + docs/configuration.md | 12 + internal/config/config.go | 7 + internal/config/config_test.go | 4 + internal/handler/chef.go | 354 ++++++++++++++ internal/handler/chef_test.go | 475 +++++++++++++++++++ internal/handler/notfound_ecosystems_test.go | 2 + internal/server/dashboard.go | 13 + internal/server/middleware.go | 2 +- internal/server/middleware_test.go | 2 +- internal/server/server.go | 3 + 14 files changed, 898 insertions(+), 3 deletions(-) create mode 100644 internal/handler/chef.go create mode 100644 internal/handler/chef_test.go diff --git a/README.md b/README.md index f3508dd..5b82620 100644 --- a/README.md +++ b/README.md @@ -66,7 +66,7 @@ The proxy never uploads artifact bytes to a scanner. Each scanner is notified wi | RPM | RHEL/Fedora | | ✓ | | Alpine | Alpine Linux | | ✓ | | Arch | Arch Linux | | ✓ | -| Chef | Chef | | ✗ | +| Chef Supermarket | Chef | | ✓ | | Generic | Any | | ✓ | | Helm | Kubernetes | Yes | ✓ | | Vagrant | Vagrant | | ✗ | @@ -371,6 +371,24 @@ conan remote disable conancenter Or configure in `~/.conan2/remotes.json`. +### Chef Supermarket (Berkshelf, Policyfiles) + +Point Berkshelf at the proxy in your `Berksfile`: + +```ruby +source "http://localhost:8080/chef" +``` + +Or set the default source of a Policyfile: + +```ruby +default_source :supermarket, "http://localhost:8080/chef" +``` + +The proxy serves the `/universe` dependency index and the cookbook API, rewriting every download location to itself while leaving dependency constraints unchanged. Cookbook archives are fetched through the Supermarket download redirect and cached, so they stay available when the Supermarket is down. With `cache_metadata` on, the universe and cookbook metadata follow the usual TTL and stale fallback. + +To proxy a private Supermarket instead of `supermarket.chef.io`, set `upstream.chef` (or `PROXY_UPSTREAM_CHEF`) and add credentials under `upstream.auth` if it needs them. See [docs/configuration.md](docs/configuration.md#upstream-registries). + ### Conda Configure in `~/.condarc`: @@ -1116,6 +1134,7 @@ Recently cached: | `GET /nuget/*` | NuGet V3 API | | `GET /composer/*` | Composer/Packagist protocol | | `GET /conan/*` | Conan C/C++ protocol | +| `GET /chef/*` | Chef Supermarket protocol (universe, cookbook API and downloads) | | `GET /conda/*` | Conda/Anaconda protocol | | `GET /cran/*` | CRAN (R) protocol | | `GET /julia/*` | Julia Pkg server protocol | diff --git a/cmd/proxy/main.go b/cmd/proxy/main.go index 0b140ea..7bc8eb4 100644 --- a/cmd/proxy/main.go +++ b/cmd/proxy/main.go @@ -250,6 +250,7 @@ func runServe() { fmt.Fprintf(os.Stderr, " PROXY_UPSTREAM_COMPOSER Packagist API upstream URL\n") fmt.Fprintf(os.Stderr, " PROXY_UPSTREAM_COMPOSER_REPOSITORY Packagist repository upstream URL\n") fmt.Fprintf(os.Stderr, " PROXY_UPSTREAM_CONAN Conan registry upstream URL\n") + fmt.Fprintf(os.Stderr, " PROXY_UPSTREAM_CHEF Chef Supermarket upstream URL\n") fmt.Fprintf(os.Stderr, " PROXY_UPSTREAM_CONDA Conda channel upstream URL\n") fmt.Fprintf(os.Stderr, " PROXY_UPSTREAM_CRAN CRAN mirror upstream URL\n") fmt.Fprintf(os.Stderr, " PROXY_UPSTREAM_JULIA Julia package server upstream URL\n") diff --git a/cmd/proxy/main_test.go b/cmd/proxy/main_test.go index 4393c44..3f17c27 100644 --- a/cmd/proxy/main_test.go +++ b/cmd/proxy/main_test.go @@ -35,6 +35,7 @@ func TestServeHelpListsUpstreamEnvironmentVariables(t *testing.T) { "PROXY_UPSTREAM_COMPOSER", "PROXY_UPSTREAM_COMPOSER_REPOSITORY", "PROXY_UPSTREAM_CONAN", + "PROXY_UPSTREAM_CHEF", "PROXY_UPSTREAM_CONDA", "PROXY_UPSTREAM_CRAN", "PROXY_UPSTREAM_JULIA", diff --git a/config.example.yaml b/config.example.yaml index 95b8002..504e6f4 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -225,6 +225,10 @@ upstream: # Conan registry URL conan: "https://center.conan.io" + # Chef Supermarket URL (used by /chef/). Point it at a private + # Supermarket to proxy that instead. + chef: "https://supermarket.chef.io" + # Conda channel base URL conda: "https://conda.anaconda.org" diff --git a/docs/configuration.md b/docs/configuration.md index 235c8dc..1466a24 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -250,6 +250,7 @@ Each upstream used by a built-in package route can be set in YAML or JSON under | `upstream.composer` | `PROXY_UPSTREAM_COMPOSER` | `https://packagist.org` | | `upstream.composer_repository` | `PROXY_UPSTREAM_COMPOSER_REPOSITORY` | `https://repo.packagist.org` | | `upstream.conan` | `PROXY_UPSTREAM_CONAN` | `https://center.conan.io` | +| `upstream.chef` | `PROXY_UPSTREAM_CHEF` | `https://supermarket.chef.io` | | `upstream.conda` | `PROXY_UPSTREAM_CONDA` | `https://conda.anaconda.org` | | `upstream.cran` | `PROXY_UPSTREAM_CRAN` | `https://cloud.r-project.org` | | `upstream.julia` | `PROXY_UPSTREAM_JULIA` | `https://pkg.julialang.org` | @@ -284,6 +285,17 @@ upstream: `upstream.hex_api` is used for cooldown timestamps and must expose Hex's `/api/packages/{name}` JSON endpoint. +`upstream.chef` is the Supermarket served at `/chef/`, for Berkshelf (`source "https://proxy.example.com/chef"`) and Policyfiles (`default_source :supermarket, "https://proxy.example.com/chef"`). The upstream has to serve the Supermarket API: `/universe`, `/api/v1/cookbooks/{name}`, its versions and their downloads. A private Supermarket that needs credentials takes them from `upstream.auth`, scoped to its URL. A download redirect to another host, such as the storage bucket behind Supermarket, does not receive them: + +```yaml +upstream: + chef: "https://supermarket.example.com" + auth: + "https://supermarket.example.com": + type: bearer + token: "${SUPERMARKET_TOKEN}" +``` + ### Private package routes Clients that use the proxy as their only registry can still install private packages. A package route sends every package whose name matches a pattern to a private registry, and never to the default upstream: diff --git a/internal/config/config.go b/internal/config/config.go index 3509d18..89a5c20 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -660,6 +660,11 @@ type UpstreamConfig struct { // Default: https://center.conan.io Conan string `json:"conan" yaml:"conan"` + // Chef is the upstream Chef Supermarket URL, served at /chef/. Point it + // at a private Supermarket to proxy that instead of the public one. + // Default: https://supermarket.chef.io + Chef string `json:"chef" yaml:"chef"` + // Conda is the upstream Conda channel base URL. // Default: https://conda.anaconda.org Conda string `json:"conda" yaml:"conda"` @@ -1059,6 +1064,7 @@ func Default() *Config { Composer: "https://packagist.org", ComposerRepository: "https://repo.packagist.org", Conan: "https://center.conan.io", + Chef: "https://supermarket.chef.io", Conda: "https://conda.anaconda.org", CRAN: "https://cloud.r-project.org", Julia: "https://pkg.julialang.org", @@ -1224,6 +1230,7 @@ func (c *Config) LoadFromEnv() { setEnvString(&c.Upstream.Composer, "PROXY_UPSTREAM_COMPOSER") setEnvString(&c.Upstream.ComposerRepository, "PROXY_UPSTREAM_COMPOSER_REPOSITORY") setEnvString(&c.Upstream.Conan, "PROXY_UPSTREAM_CONAN") + setEnvString(&c.Upstream.Chef, "PROXY_UPSTREAM_CHEF") setEnvString(&c.Upstream.Conda, "PROXY_UPSTREAM_CONDA") setEnvString(&c.Upstream.CRAN, "PROXY_UPSTREAM_CRAN") setEnvString(&c.Upstream.Julia, "PROXY_UPSTREAM_JULIA") diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 4750e5c..3622d63 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -33,6 +33,7 @@ func upstreamConfigValues(upstream UpstreamConfig) map[string]string { "composer": upstream.Composer, "composer_repository": upstream.ComposerRepository, "conan": upstream.Conan, + "chef": upstream.Chef, "conda": upstream.Conda, "cran": upstream.CRAN, "julia": upstream.Julia, @@ -64,6 +65,7 @@ func defaultUpstreamValues() map[string]string { "composer": "https://packagist.org", "composer_repository": "https://repo.packagist.org", "conan": "https://center.conan.io", + "chef": "https://supermarket.chef.io", "conda": "https://conda.anaconda.org", "cran": "https://cloud.r-project.org", "julia": "https://pkg.julialang.org", @@ -95,6 +97,7 @@ func upstreamEnvironmentVariables() map[string]string { "composer": "PROXY_UPSTREAM_COMPOSER", "composer_repository": "PROXY_UPSTREAM_COMPOSER_REPOSITORY", "conan": "PROXY_UPSTREAM_CONAN", + "chef": "PROXY_UPSTREAM_CHEF", "conda": "PROXY_UPSTREAM_CONDA", "cran": "PROXY_UPSTREAM_CRAN", "julia": "PROXY_UPSTREAM_JULIA", @@ -388,6 +391,7 @@ upstream: composer: "https://upstream.example.com/composer" composer_repository: "https://upstream.example.com/composer_repository" conan: "https://upstream.example.com/conan" + chef: "https://upstream.example.com/chef" conda: "https://upstream.example.com/conda" cran: "https://upstream.example.com/cran" julia: "https://upstream.example.com/julia" diff --git a/internal/handler/chef.go b/internal/handler/chef.go new file mode 100644 index 0000000..7283992 --- /dev/null +++ b/internal/handler/chef.go @@ -0,0 +1,354 @@ +package handler + +import ( + "bytes" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "net/http" + "net/url" + "path" + "regexp" + "strings" +) + +const ( + chefEcosystem = "chef" + chefMountPath = "/chef" + chefAPIPath = "/api/v1" + // chefUpstreamKeyLength is how much of the upstream URL's digest scopes + // metadata cache keys and archive filenames: enough to keep two upstreams + // apart. + chefUpstreamKeyLength = 16 +) + +var ( + // chefCookbookName matches Supermarket cookbook names: letters, digits, + // hyphens, underscores and dots, not starting with a dot. + chefCookbookName = regexp.MustCompile(`^[A-Za-z0-9_-][A-Za-z0-9_.-]*$`) + // chefVersion matches Chef cookbook versions, which have two or three + // numeric parts. + chefVersion = regexp.MustCompile(`^[0-9]+\.[0-9]+(\.[0-9]+)?$`) + + errChefDocument = errors.New("unexpected Chef Supermarket document") +) + +// ChefHandler serves a read-only Chef Supermarket at /chef. Berkshelf and +// Policyfiles resolve dependencies against /universe and then download each +// cookbook through the cookbook API. Every download location in those +// documents is rewritten to the proxy, so cookbooks are fetched through it and +// cached; dependency constraints and every other field pass through unchanged. +type ChefHandler struct { + proxy *Proxy + proxyURL string + upstreamURL string + // upstreamKey identifies the upstream in cache keys, so pointing + // upstream.chef at another Supermarket does not serve the previous one's + // documents or archives. + upstreamKey string +} + +// NewChefHandler creates a Chef Supermarket handler for the Supermarket at +// upstreamURL, such as https://supermarket.chef.io or a private Supermarket. +func NewChefHandler(proxy *Proxy, proxyURL, upstreamURL string) *ChefHandler { + upstreamURL = strings.TrimSuffix(upstreamURL, "/") + digest := sha256.Sum256([]byte(upstreamURL)) + return &ChefHandler{ + proxy: proxy, + proxyURL: strings.TrimSuffix(proxyURL, "/"), + upstreamURL: upstreamURL, + upstreamKey: hex.EncodeToString(digest[:])[:chefUpstreamKeyLength], + } +} + +// Routes returns the HTTP handler for Chef Supermarket requests. +func (h *ChefHandler) Routes() http.Handler { + mux := http.NewServeMux() + mux.HandleFunc("GET /universe", h.handleUniverse) + mux.HandleFunc("GET "+chefAPIPath+"/cookbooks/{name}", h.handleCookbook) + mux.HandleFunc("GET "+chefAPIPath+"/cookbooks/{name}/versions/{version}", h.handleVersion) + mux.HandleFunc("GET "+chefAPIPath+"/cookbooks/{name}/versions/{version}/download", h.handleDownload) + return mux +} + +func (h *ChefHandler) handleUniverse(w http.ResponseWriter, r *http.Request) { + h.serveDocument(w, r, "universe", h.upstreamURL+"/universe", h.rewriteUniverse) +} + +func (h *ChefHandler) handleCookbook(w http.ResponseWriter, r *http.Request) { + name, ok := chefCookbookFromRequest(r) + if !ok { + http.Error(w, "invalid cookbook name", http.StatusBadRequest) + return + } + h.serveDocument(w, r, "cookbooks/"+name, h.upstreamCookbookURL(name), func(body []byte) ([]byte, error) { + return h.rewriteCookbook(name, body) + }) +} + +func (h *ChefHandler) handleVersion(w http.ResponseWriter, r *http.Request) { + name, version, ok := chefVersionFromRequest(r) + if !ok { + http.Error(w, "invalid cookbook version", http.StatusBadRequest) + return + } + h.serveDocument(w, r, "cookbooks/"+name+"/versions/"+version, h.upstreamVersionURL(name, version), func(body []byte) ([]byte, error) { + return h.rewriteVersion(name, version, body) + }) +} + +// handleDownload serves a cookbook archive from the cache. An archive not +// cached yet is fetched through the Supermarket download endpoint, following +// the redirect it answers with. A cached archive is served without the +// upstream. Archives are cached per upstream, since two Supermarkets can +// publish different cookbooks under the same name and version. +func (h *ChefHandler) handleDownload(w http.ResponseWriter, r *http.Request) { + name, version, ok := chefVersionFromRequest(r) + if !ok { + http.Error(w, "invalid cookbook version", http.StatusBadRequest) + return + } + + result, err := h.proxy.GetOrFetchArtifactFromURL( + r.Context(), chefEcosystem, name, version, h.archiveFilename(name, version), h.upstreamVersionURL(name, version)+"/download") + if err != nil { + h.proxy.serveArtifactError(w, err, "failed to fetch cookbook") + return + } + if result.Artifact.MediaType == "" { + w.Header().Set(headerContentType, "application/gzip") + } + ServeArtifactRequest(w, r, result) +} + +// serveDocument serves a JSON document from the metadata cache, which applies +// the metadata TTL and falls back to the stored copy while the upstream is +// unavailable, rewritten by rewrite. +func (h *ChefHandler) serveDocument(w http.ResponseWriter, r *http.Request, documentKey, upstreamURL string, rewrite func([]byte) ([]byte, error)) { + cacheKey := h.cacheKey(documentKey) + if stored, ok := h.proxy.storedRewrite(chefEcosystem, cacheKey, h.proxyURL, cacheKey); ok { + serveRewrittenMetadata(w, r, stored) + return + } + + upstream := h.proxy.fetchMetadataDocument(r.Context(), chefEcosystem, cacheKey, upstreamURL, contentTypeJSON) + if err := upstream.err; err != nil { + if errors.Is(err, ErrUpstreamNotFound) { + http.Error(w, "not found", http.StatusNotFound) + return + } + if writeUpstreamHeld(w, err) { + return + } + h.proxy.Logger.Error("Chef Supermarket request failed", "url", upstreamURL, "error", err) + http.Error(w, "upstream request failed", http.StatusBadGateway) + return + } + + rewritten, err := h.proxy.cachedRewrite(r.Context(), chefEcosystem, h.proxyURL, cacheKey, upstream.body, upstream.digest, + func(body []byte) ([]byte, []string, error) { + out, err := rewrite(body) + return out, nil, err + }) + if err != nil { + if r.Context().Err() != nil { + return // the client left while waiting on a shared rewrite + } + // Serving the document unrewritten would send clients straight to + // the upstream, bypassing the cache. + h.proxy.Logger.Warn("failed to rewrite Chef Supermarket document", "url", upstreamURL, "error", err) + http.Error(w, "invalid Chef Supermarket response", http.StatusBadGateway) + return + } + serveRewrittenMetadata(w, r, rewritten) +} + +// rewriteUniverse points every version's location_path and download_url at +// the proxy. The universe maps cookbook names to versions to their location +// and dependencies; dependencies and any other field are copied as they are. +func (h *ChefHandler) rewriteUniverse(body []byte) ([]byte, error) { + var universe map[string]map[string]map[string]json.RawMessage + if err := json.Unmarshal(body, &universe); err != nil { + return nil, fmt.Errorf("%w: universe: %w", errChefDocument, err) + } + + locationPath, err := jsonString(h.apiURL()) + if err != nil { + return nil, err + } + for name, versions := range universe { + for version, entry := range versions { + if entry == nil { + continue + } + downloadURL, err := jsonString(h.downloadURL(name, version)) + if err != nil { + return nil, err + } + entry["location_path"] = locationPath + entry["download_url"] = downloadURL + } + } + return encodeChefJSON(universe) +} + +// rewriteCookbook points a cookbook's version URLs at the proxy. +func (h *ChefHandler) rewriteCookbook(name string, body []byte) ([]byte, error) { + var cookbook map[string]json.RawMessage + if err := json.Unmarshal(body, &cookbook); err != nil { + return nil, fmt.Errorf("%w: cookbook %s: %w", errChefDocument, name, err) + } + + if raw, ok := cookbook["latest_version"]; ok { + rewritten, err := h.rewriteVersionURL(name, raw) + if err != nil { + return nil, err + } + cookbook["latest_version"] = rewritten + } + if raw, ok := cookbook["versions"]; ok { + var versions []json.RawMessage + if err := json.Unmarshal(raw, &versions); err != nil { + return nil, fmt.Errorf("%w: cookbook %s versions: %w", errChefDocument, name, err) + } + for i, versionURL := range versions { + rewritten, err := h.rewriteVersionURL(name, versionURL) + if err != nil { + return nil, err + } + versions[i] = rewritten + } + encoded, err := encodeChefJSON(versions) + if err != nil { + return nil, err + } + cookbook["versions"] = encoded + } + return encodeChefJSON(cookbook) +} + +// rewriteVersion points a cookbook version's cookbook and file URLs at the +// proxy. +func (h *ChefHandler) rewriteVersion(name, version string, body []byte) ([]byte, error) { + var document map[string]json.RawMessage + if err := json.Unmarshal(body, &document); err != nil { + return nil, fmt.Errorf("%w: cookbook %s version %s: %w", errChefDocument, name, version, err) + } + + if _, ok := document["cookbook"]; ok { + cookbookURL, err := jsonString(h.cookbookURL(name)) + if err != nil { + return nil, err + } + document["cookbook"] = cookbookURL + } + if _, ok := document["file"]; ok { + fileURL, err := jsonString(h.downloadURL(name, version)) + if err != nil { + return nil, err + } + document["file"] = fileURL + } + return encodeChefJSON(document) +} + +// rewriteVersionURL rewrites an upstream cookbook version URL, a JSON string +// ending in /versions/{version}, to the proxy. Anything else, such as null, +// is returned unchanged. +func (h *ChefHandler) rewriteVersionURL(name string, raw json.RawMessage) (json.RawMessage, error) { + var versionURL string + if err := json.Unmarshal(raw, &versionURL); err != nil { + return raw, nil //nolint:nilerr // not a string, so not a URL to rewrite + } + parsed, err := url.Parse(versionURL) + if err != nil { + return raw, nil //nolint:nilerr // not a URL, so nothing to rewrite + } + version, ok := normalizeChefVersion(path.Base(parsed.Path)) + if !ok { + return raw, nil + } + return jsonString(h.versionURL(name, version)) +} + +func (h *ChefHandler) apiURL() string { + return h.proxyURL + chefMountPath + chefAPIPath +} + +func (h *ChefHandler) cookbookURL(name string) string { + return h.apiURL() + "/cookbooks/" + url.PathEscape(name) +} + +func (h *ChefHandler) versionURL(name, version string) string { + return h.cookbookURL(name) + "/versions/" + url.PathEscape(version) +} + +func (h *ChefHandler) downloadURL(name, version string) string { + return h.versionURL(name, version) + "/download" +} + +func (h *ChefHandler) upstreamCookbookURL(name string) string { + return h.upstreamURL + chefAPIPath + "/cookbooks/" + url.PathEscape(name) +} + +func (h *ChefHandler) upstreamVersionURL(name, version string) string { + return h.upstreamCookbookURL(name) + "/versions/" + url.PathEscape(version) +} + +// cacheKey prefixes a metadata cache key with the upstream's key. +func (h *ChefHandler) cacheKey(documentKey string) string { + return h.upstreamKey + "/" + documentKey +} + +// archiveFilename names a cookbook archive in the cache. The filename is part +// of the artifact's cache identity, so it carries the upstream's key; the +// .tgz extension is kept for browsing the archive. +func (h *ChefHandler) archiveFilename(name, version string) string { + return name + "-" + version + "-" + h.upstreamKey + ".tgz" +} + +func chefCookbookFromRequest(r *http.Request) (string, bool) { + name := r.PathValue("name") + if !chefCookbookName.MatchString(name) || containsPathTraversal(name) { + return "", false + } + return name, true +} + +func chefVersionFromRequest(r *http.Request) (name, version string, ok bool) { + name, ok = chefCookbookFromRequest(r) + if !ok { + return "", "", false + } + version, ok = normalizeChefVersion(r.PathValue("version")) + return name, version, ok +} + +// normalizeChefVersion accepts a version in either form Supermarket serves +// (1.2.3, or 1_2_3 as Berkshelf writes it in URLs) and returns the dotted +// form. +func normalizeChefVersion(version string) (string, bool) { + version = strings.ReplaceAll(version, "_", ".") + if !chefVersion.MatchString(version) { + return "", false + } + return version, true +} + +func jsonString(s string) (json.RawMessage, error) { + return encodeChefJSON(s) +} + +// encodeChefJSON encodes v without escaping HTML characters, so dependency +// constraints such as ">= 1.0" keep their upstream spelling. +func encodeChefJSON(v any) (json.RawMessage, error) { + var buf bytes.Buffer + enc := json.NewEncoder(&buf) + enc.SetEscapeHTML(false) + if err := enc.Encode(v); err != nil { + return nil, err + } + return bytes.TrimSuffix(buf.Bytes(), []byte("\n")), nil +} diff --git a/internal/handler/chef_test.go b/internal/handler/chef_test.go new file mode 100644 index 0000000..51290a5 --- /dev/null +++ b/internal/handler/chef_test.go @@ -0,0 +1,475 @@ +package handler + +import ( + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + upstreamhttp "github.com/git-pkgs/proxy/internal/httpclient" + "github.com/git-pkgs/registries/fetch" +) + +const ( + chefTestProxyURL = "http://proxy.example" + chefTestAPI = chefTestProxyURL + "/chef/api/v1" + chefTestTarball = "cookbook tarball bytes" +) + +// fakeSupermarket serves the parts of the Supermarket API Berkshelf uses, in +// the shapes supermarket.chef.io returns them. Downloads redirect to a second +// server standing in for the storage host, as Supermarket redirects to S3. +type fakeSupermarket struct { + api *httptest.Server + files *httptest.Server + // filesURL is files.URL on another hostname, so a redirect to it leaves + // the API host the way a redirect to S3 does. + filesURL string + + available atomic.Bool + // token, when set, is the bearer token the API requires. + token string + // tarball is the content of every cookbook archive it serves. + tarball string + + mu sync.Mutex + requests map[string]int + fileAuth []string +} + +func newFakeSupermarket(t *testing.T, token string) *fakeSupermarket { + t.Helper() + s := &fakeSupermarket{token: token, tarball: chefTestTarball, requests: map[string]int{}} + s.available.Store(true) + + s.files = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + s.mu.Lock() + s.requests["files "+r.URL.Path]++ + s.fileAuth = append(s.fileAuth, r.Header.Get("Authorization")) + s.mu.Unlock() + if !s.available.Load() { + http.Error(w, "unavailable", http.StatusServiceUnavailable) + return + } + if r.URL.Path != "/cookbook_versions/tarballs/34899/original/apt.tgz" { + http.NotFound(w, r) + return + } + w.Header().Set("Content-Type", "application/x-gzip") + _, _ = w.Write([]byte(s.tarball)) + })) + t.Cleanup(s.files.Close) + s.filesURL = strings.Replace(s.files.URL, "127.0.0.1", "localhost", 1) + + s.api = httptest.NewServer(http.HandlerFunc(s.serveAPI)) + t.Cleanup(s.api.Close) + return s +} + +func (s *fakeSupermarket) serveAPI(w http.ResponseWriter, r *http.Request) { + s.mu.Lock() + s.requests[r.URL.Path]++ + s.mu.Unlock() + if !s.available.Load() { + http.Error(w, "unavailable", http.StatusServiceUnavailable) + return + } + if s.token != "" && r.Header.Get("Authorization") != "Bearer "+s.token { + http.Error(w, "unauthorized", http.StatusUnauthorized) + return + } + + api := s.api.URL + "/api/v1" + w.Header().Set("Content-Type", "application/json") + switch r.URL.Path { + case "/universe": + // Like Supermarket's, these locations carry an explicit port. + located := api + _, _ = fmt.Fprintf(w, `{"apt":{"7.5.22":{"location_type":"opscode","location_path":%q,"download_url":%q,"dependencies":{"compat_resource":">= 0.0.0"}},`+ + `"7.6.0":{"location_type":"opscode","location_path":%q,"download_url":%q,"dependencies":{}}},`+ + `"compat_resource":{"12.19.1":{"location_type":"opscode","location_path":%q,"download_url":%q,"dependencies":{"apt":"~> 7.5"}}}}`, + located, api+"/cookbooks/apt/versions/7.5.22/download", + located, api+"/cookbooks/apt/versions/7.6.0/download", + located, api+"/cookbooks/compat_resource/versions/12.19.1/download") + case "/api/v1/cookbooks/apt": + _, _ = fmt.Fprintf(w, `{"name":"apt","maintainer":"sous-chefs","latest_version":%q,"external_url":"https://github.com/sous-chefs/apt","deprecated":false,"versions":[%q,%q]}`, + api+"/cookbooks/apt/versions/7.6.0", api+"/cookbooks/apt/versions/7.6.0", api+"/cookbooks/apt/versions/7.5.22") + case "/api/v1/cookbooks/apt/versions/7.5.22": + _, _ = fmt.Fprintf(w, `{"license":"Apache-2.0","tarball_file_size":%d,"version":"7.5.22","cookbook":%q,"file":%q,"dependencies":{"compat_resource":">= 0.0.0"}}`, + len(s.tarball), api+"/cookbooks/apt", api+"/cookbooks/apt/versions/7.5.22/download") + case "/api/v1/cookbooks/apt/versions/7.5.22/download": + http.Redirect(w, r, s.filesURL+"/cookbook_versions/tarballs/34899/original/apt.tgz?1714639378", http.StatusFound) + default: + http.NotFound(w, r) + } +} + +func (s *fakeSupermarket) count(key string) int { + s.mu.Lock() + defer s.mu.Unlock() + return s.requests[key] +} + +// newChefTestHandler returns a handler for s using client for every upstream +// request, with metadata caching on. +func newChefTestHandler(t *testing.T, s *fakeSupermarket, client *http.Client) (*ChefHandler, *Proxy) { + t.Helper() + proxy, _, _, _ := setupTestProxy(t) + proxy.CacheMetadata = true + proxy.MetadataTTL = time.Hour + proxy.HTTPClient = client + fetcher := fetch.NewFetcher(fetch.WithHTTPClient(client), fetch.WithMaxRetries(0)) + proxy.Fetcher = fetcher + t.Cleanup(func() { _ = fetcher.Close() }) + return NewChefHandler(proxy, chefTestProxyURL, s.api.URL), proxy +} + +func serveChefRequest(h *ChefHandler, target string) *httptest.ResponseRecorder { + w := httptest.NewRecorder() + h.Routes().ServeHTTP(w, httptest.NewRequest(http.MethodGet, target, nil)) + return w +} + +// proxyPath turns an absolute proxy URL from a rewritten document into the +// path the handler is mounted at. +func proxyPath(t *testing.T, rawURL string) string { + t.Helper() + parsed, err := url.Parse(rawURL) + if err != nil { + t.Fatalf("parse %q: %v", rawURL, err) + } + if parsed.Scheme+"://"+parsed.Host != chefTestProxyURL || !strings.HasPrefix(parsed.Path, "/chef/") { + t.Fatalf("URL %q does not point at the proxy", rawURL) + } + return strings.TrimPrefix(parsed.Path, "/chef") +} + +type chefUniverseEntry struct { + LocationType string `json:"location_type"` + LocationPath string `json:"location_path"` + DownloadURL string `json:"download_url"` + Dependencies json.RawMessage `json:"dependencies"` +} + +func decodeChefUniverse(t *testing.T, body []byte) map[string]map[string]chefUniverseEntry { + t.Helper() + var universe map[string]map[string]chefUniverseEntry + if err := json.Unmarshal(body, &universe); err != nil { + t.Fatalf("decode universe: %v\n%s", err, body) + } + return universe +} + +func TestChefUniverseRewritesLocations(t *testing.T) { + s := newFakeSupermarket(t, "") + h, _ := newChefTestHandler(t, s, s.api.Client()) + + w := serveChefRequest(h, "/universe") + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want 200: %s", w.Code, w.Body.String()) + } + if got := w.Header().Get("Content-Type"); got != contentTypeJSON { + t.Errorf("Content-Type = %q, want %q", got, contentTypeJSON) + } + body := w.Body.String() + if strings.Contains(body, s.api.URL) { + t.Errorf("rewritten universe still points at the upstream: %s", body) + } + + universe := decodeChefUniverse(t, w.Body.Bytes()) + wantDeps := map[string]map[string]string{ + "apt": {"7.5.22": `{"compat_resource":">= 0.0.0"}`, "7.6.0": `{}`}, + "compat_resource": {"12.19.1": `{"apt":"~> 7.5"}`}, + } + for name, versions := range wantDeps { + for version, deps := range versions { + entry, ok := universe[name][version] + if !ok { + t.Errorf("universe lost %s %s", name, version) + continue + } + if entry.LocationType != "opscode" { + t.Errorf("%s %s location_type = %q, want opscode", name, version, entry.LocationType) + } + if entry.LocationPath != chefTestAPI { + t.Errorf("%s %s location_path = %q, want %q", name, version, entry.LocationPath, chefTestAPI) + } + wantDownload := chefTestAPI + "/cookbooks/" + name + "/versions/" + version + "/download" + if entry.DownloadURL != wantDownload { + t.Errorf("%s %s download_url = %q, want %q", name, version, entry.DownloadURL, wantDownload) + } + if string(entry.Dependencies) != deps { + t.Errorf("%s %s dependencies = %s, want %s unchanged", name, version, entry.Dependencies, deps) + } + } + } + // Constraints keep their spelling rather than being HTML-escaped. + if !strings.Contains(body, `">= 0.0.0"`) || !strings.Contains(body, `"~> 7.5"`) { + t.Errorf("dependency constraints were re-escaped: %s", body) + } +} + +func TestChefCookbookMetadataRewritesVersionURLs(t *testing.T) { + s := newFakeSupermarket(t, "") + h, _ := newChefTestHandler(t, s, s.api.Client()) + + w := serveChefRequest(h, "/api/v1/cookbooks/apt") + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want 200: %s", w.Code, w.Body.String()) + } + var cookbook struct { + Name string `json:"name"` + LatestVersion string `json:"latest_version"` + Versions []string `json:"versions"` + ExternalURL string `json:"external_url"` + Deprecated bool `json:"deprecated"` + } + if err := json.Unmarshal(w.Body.Bytes(), &cookbook); err != nil { + t.Fatalf("decode: %v", err) + } + if cookbook.LatestVersion != chefTestAPI+"/cookbooks/apt/versions/7.6.0" { + t.Errorf("latest_version = %q", cookbook.LatestVersion) + } + want := []string{chefTestAPI + "/cookbooks/apt/versions/7.6.0", chefTestAPI + "/cookbooks/apt/versions/7.5.22"} + if strings.Join(cookbook.Versions, ",") != strings.Join(want, ",") { + t.Errorf("versions = %v, want %v", cookbook.Versions, want) + } + if cookbook.Name != "apt" || cookbook.ExternalURL != "https://github.com/sous-chefs/apt" || cookbook.Deprecated { + t.Errorf("unrelated fields changed: %+v", cookbook) + } +} + +func TestChefVersionMetadataAcceptsBerkshelfVersions(t *testing.T) { + s := newFakeSupermarket(t, "") + h, _ := newChefTestHandler(t, s, s.api.Client()) + + // Berkshelf writes versions with underscores in cookbook API paths. + for _, version := range []string{"7_5_22", "7.5.22"} { + w := serveChefRequest(h, "/api/v1/cookbooks/apt/versions/"+version) + if w.Code != http.StatusOK { + t.Fatalf("%s: status = %d, want 200: %s", version, w.Code, w.Body.String()) + } + var document struct { + Version string `json:"version"` + Cookbook string `json:"cookbook"` + File string `json:"file"` + Dependencies json.RawMessage `json:"dependencies"` + License string `json:"license"` + } + if err := json.Unmarshal(w.Body.Bytes(), &document); err != nil { + t.Fatalf("%s: decode: %v", version, err) + } + if document.Cookbook != chefTestAPI+"/cookbooks/apt" { + t.Errorf("%s: cookbook = %q", version, document.Cookbook) + } + if document.File != chefTestAPI+"/cookbooks/apt/versions/7.5.22/download" { + t.Errorf("%s: file = %q", version, document.File) + } + if string(document.Dependencies) != `{"compat_resource":">= 0.0.0"}` || document.License != "Apache-2.0" || document.Version != "7.5.22" { + t.Errorf("%s: unrelated fields changed: %+v", version, document) + } + } + // Both spellings are one document, fetched once. + if got := s.count("/api/v1/cookbooks/apt/versions/7.5.22"); got != 1 { + t.Errorf("upstream version requests = %d, want 1", got) + } +} + +// TestChefBerkshelfFlow follows the requests Berkshelf makes: the universe, +// then the cookbook version at the universe's location_path, then its file. +// The download redirects to the storage host and is cached. It is then served +// with the upstream down, as is the universe. +func TestChefBerkshelfFlow(t *testing.T) { + s := newFakeSupermarket(t, "") + h, proxy := newChefTestHandler(t, s, s.api.Client()) + + universe := decodeChefUniverse(t, serveChefRequest(h, "/universe").Body.Bytes()) + locationPath := universe["apt"]["7.5.22"].LocationPath + + versionResponse := serveChefRequest(h, proxyPath(t, locationPath+"/cookbooks/apt/versions/7_5_22")) + if versionResponse.Code != http.StatusOK { + t.Fatalf("version status = %d: %s", versionResponse.Code, versionResponse.Body.String()) + } + var version struct { + File string `json:"file"` + } + if err := json.Unmarshal(versionResponse.Body.Bytes(), &version); err != nil { + t.Fatalf("decode version: %v", err) + } + + download := serveChefRequest(h, proxyPath(t, version.File)) + if download.Code != http.StatusOK { + t.Fatalf("download status = %d: %s", download.Code, download.Body.String()) + } + if download.Body.String() != chefTestTarball { + t.Errorf("download body = %q, want the tarball", download.Body.String()) + } + if got := s.count("files /cookbook_versions/tarballs/34899/original/apt.tgz"); got != 1 { + t.Errorf("storage host requests = %d, want 1 after following the redirect", got) + } + + // A second download is a cache hit. + if again := serveChefRequest(h, proxyPath(t, version.File)); again.Code != http.StatusOK || again.Body.String() != chefTestTarball { + t.Fatalf("cached download = %d %q", again.Code, again.Body.String()) + } + if got := s.count("/api/v1/cookbooks/apt/versions/7.5.22/download"); got != 1 { + t.Errorf("upstream download requests = %d, want 1 (second served from cache)", got) + } + + // With the upstream down and the metadata TTL lapsed, the cached + // cookbook and the stored universe are still served. + s.available.Store(false) + proxy.MetadataTTL = 0 + if offline := serveChefRequest(h, proxyPath(t, version.File)); offline.Code != http.StatusOK || offline.Body.String() != chefTestTarball { + t.Errorf("offline download = %d %q, want the cached tarball", offline.Code, offline.Body.String()) + } + offlineUniverse := serveChefRequest(h, "/universe") + if offlineUniverse.Code != http.StatusOK { + t.Fatalf("offline universe status = %d: %s", offlineUniverse.Code, offlineUniverse.Body.String()) + } + if got := decodeChefUniverse(t, offlineUniverse.Body.Bytes())["apt"]["7.5.22"].DownloadURL; got != version.File { + t.Errorf("offline universe download_url = %q, want %q", got, version.File) + } +} + +func TestChefUpstreamAuthentication(t *testing.T) { + const token = "supermarket-token" + s := newFakeSupermarket(t, token) + // Credentials are scoped to the Supermarket, as upstream.auth scopes + // them by URL, so the storage host it redirects to never sees them. + client := &http.Client{Transport: upstreamhttp.NewTransport(http.DefaultTransport, + upstreamhttp.AuthFunc(func(requestURL string) (string, string) { + if strings.HasPrefix(requestURL, s.api.URL+"/") { + return "Authorization", "Bearer " + token + } + return "", "" + }))} + h, _ := newChefTestHandler(t, s, client) + + for _, target := range []string{"/universe", "/api/v1/cookbooks/apt", "/api/v1/cookbooks/apt/versions/7.5.22"} { + if w := serveChefRequest(h, target); w.Code != http.StatusOK { + t.Errorf("%s: status = %d, want 200: %s", target, w.Code, w.Body.String()) + } + } + download := serveChefRequest(h, "/api/v1/cookbooks/apt/versions/7.5.22/download") + if download.Code != http.StatusOK || download.Body.String() != chefTestTarball { + t.Fatalf("download = %d %q", download.Code, download.Body.String()) + } + + s.mu.Lock() + defer s.mu.Unlock() + for _, auth := range s.fileAuth { + if auth != "" { + t.Errorf("storage host received Authorization %q", auth) + } + } +} + +func TestChefRejectsInvalidRequests(t *testing.T) { + s := newFakeSupermarket(t, "") + h, _ := newChefTestHandler(t, s, s.api.Client()) + + tests := []struct { + target string + want int + }{ + {"/api/v1/cookbooks/.hidden", http.StatusBadRequest}, + {"/api/v1/cookbooks/apt/versions/latest", http.StatusBadRequest}, + {"/api/v1/cookbooks/apt/versions/1.2.3.4", http.StatusBadRequest}, + {"/api/v1/cookbooks/apt/versions/1.x/download", http.StatusBadRequest}, + {"/api/v1/cookbooks/missing", http.StatusNotFound}, + {"/api/v1/cookbooks/apt/versions/9.9.9", http.StatusNotFound}, + } + for _, tt := range tests { + if w := serveChefRequest(h, tt.target); w.Code != tt.want { + t.Errorf("%s: status = %d, want %d: %s", tt.target, w.Code, tt.want, w.Body.String()) + } + } +} + +// An upstream document that cannot be rewritten is refused rather than +// passed through, which would send clients around the proxy. +func TestChefUnreadableUniverseIsBadGateway(t *testing.T) { + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte(`["not","a","universe"]`)) + })) + defer upstream.Close() + + proxy, _, _, _ := setupTestProxy(t) + proxy.HTTPClient = upstream.Client() + h := NewChefHandler(proxy, chefTestProxyURL, upstream.URL) + + w := serveChefRequest(h, "/universe") + if w.Code != http.StatusBadGateway { + t.Fatalf("status = %d, want 502: %s", w.Code, w.Body.String()) + } + if strings.Contains(w.Body.String(), "not") { + t.Errorf("unreadable document was passed through: %s", w.Body.String()) + } +} + +// Switching upstream.chef to another Supermarket keeps the cache, but an +// archive cached from the previous one is not served for a cookbook the new +// one publishes under the same name and version. +func TestChefArchiveCacheDependsOnUpstream(t *testing.T) { + const target = "/api/v1/cookbooks/apt/versions/7.5.22/download" + public := newFakeSupermarket(t, "") + private := newFakeSupermarket(t, "") + private.tarball = "private cookbook tarball bytes" + + publicHandler, proxy := newChefTestHandler(t, public, public.api.Client()) + if w := serveChefRequest(publicHandler, target); w.Code != http.StatusOK || w.Body.String() != public.tarball { + t.Fatalf("public download = %d %q", w.Code, w.Body.String()) + } + + privateHandler := NewChefHandler(proxy, chefTestProxyURL, private.api.URL) + w := serveChefRequest(privateHandler, target) + if w.Code != http.StatusOK { + t.Fatalf("private download status = %d: %s", w.Code, w.Body.String()) + } + if w.Body.String() != private.tarball { + t.Errorf("private download = %q, want the private Supermarket's tarball", w.Body.String()) + } + if got := private.count(target); got != 1 { + t.Errorf("private download requests = %d, want 1", got) + } + + // Each upstream's archive stays cached under its own identity. + for _, tt := range []struct { + handler *ChefHandler + upstream *fakeSupermarket + }{{publicHandler, public}, {privateHandler, private}} { + if again := serveChefRequest(tt.handler, target); again.Code != http.StatusOK || again.Body.String() != tt.upstream.tarball { + t.Errorf("cached download from %s = %d %q, want %q", tt.upstream.api.URL, again.Code, again.Body.String(), tt.upstream.tarball) + } + if got := tt.upstream.count(target); got != 1 { + t.Errorf("download requests to %s = %d, want 1 (second served from cache)", tt.upstream.api.URL, got) + } + } +} + +func TestChefCacheKeyDependsOnUpstream(t *testing.T) { + a := NewChefHandler(nil, chefTestProxyURL, "https://supermarket.chef.io") + b := NewChefHandler(nil, chefTestProxyURL, "https://supermarket.example.com/") + if a.cacheKey("universe") == b.cacheKey("universe") { + t.Error("two Supermarkets share a universe cache key") + } + if containsPathTraversal(a.cacheKey("cookbooks/apt/versions/7.5.22")) { + t.Error("cache key is not a valid metadata key") + } + if a.archiveFilename("apt", "7.5.22") == b.archiveFilename("apt", "7.5.22") { + t.Error("two Supermarkets share an archive filename") + } + if got := a.archiveFilename("apt", "7.5.22"); !strings.HasPrefix(got, "apt-7.5.22-") || !strings.HasSuffix(got, ".tgz") { + t.Errorf("archive filename = %q, want apt-7.5.22-.tgz", got) + } + if NewChefHandler(nil, chefTestProxyURL, "https://supermarket.example.com").upstreamKey != b.upstreamKey { + t.Error("a trailing slash changes the upstream key") + } +} diff --git a/internal/handler/notfound_ecosystems_test.go b/internal/handler/notfound_ecosystems_test.go index e214124..e8b3b9b 100644 --- a/internal/handler/notfound_ecosystems_test.go +++ b/internal/handler/notfound_ecosystems_test.go @@ -39,6 +39,8 @@ func TestArtifactDownloadUpstreamNotFoundReturns404(t *testing.T) { func(p *Proxy) http.Handler { return NewGemHandler(p, "http://localhost").Routes() }}, {"hex", "/tarballs/phoenix-1.7.10.tar", func(p *Proxy) http.Handler { return NewHexHandler(p, "http://localhost").Routes() }}, + {"chef", "/api/v1/cookbooks/apt/versions/7.5.22/download", + func(p *Proxy) http.Handler { return NewChefHandler(p, "http://localhost", "http://localhost").Routes() }}, } for _, tt := range tests { diff --git a/internal/server/dashboard.go b/internal/server/dashboard.go index 69f352a..df48b22 100644 --- a/internal/server/dashboard.go +++ b/internal/server/dashboard.go @@ -136,6 +136,7 @@ func supportedEcosystems() []string { "alpine", "arch", "cargo", + "chef", "composer", "conan", "conda", @@ -189,6 +190,8 @@ func ecosystemColorClasses(ecosystem string) string { return "bg-violet-100 text-violet-700 dark:bg-violet-900/50 dark:text-violet-300" case "conan": return "bg-teal-100 text-teal-700 dark:bg-teal-900/50 dark:text-teal-300" + case "chef": + return "bg-rose-100 text-rose-700 dark:bg-rose-900/50 dark:text-rose-300" case "conda": return "bg-green-100 text-green-700 dark:bg-green-900/50 dark:text-green-300" case "cran": @@ -412,6 +415,16 @@ composer config -g repositories.proxy composer ` + baseURL + `/composerConfigure Conan to use the proxy:

conan remote add proxy ` + baseURL + `/conan
 conan remote disable conancenter
`), + }, + { + ID: "chef", + Name: "Chef Supermarket", + Language: "Chef", + Endpoint: "/chef/", + Instructions: template.HTML(`

Point Berkshelf at the proxy in your Berksfile:

+
source "` + baseURL + `/chef"
+

Or in a Policyfile:

+
default_source :supermarket, "` + baseURL + `/chef"
`), }, { ID: "conda", diff --git a/internal/server/middleware.go b/internal/server/middleware.go index f74590a..0de0d1b 100644 --- a/internal/server/middleware.go +++ b/internal/server/middleware.go @@ -106,7 +106,7 @@ func requestEcosystem(path string) string { segment, _, _ := strings.Cut(strings.TrimPrefix(path, "/"), "/") switch segment { case "npm", "cargo", "hex", "pub", "pypi", "maven", "gradle", "nuget", - "conan", "conda", "cran", "julia", "debian", "rpm", "arch", + "conan", "chef", "conda", "cran", "julia", "debian", "rpm", "arch", "helm", "homebrew", "generic", "swift": return segment case "apk": diff --git a/internal/server/middleware_test.go b/internal/server/middleware_test.go index 875b152..7c9eab9 100644 --- a/internal/server/middleware_test.go +++ b/internal/server/middleware_test.go @@ -311,7 +311,7 @@ func TestRequestEcosystemCoversEveryMountedRoute(t *testing.T) { "/npm/x": "npm", "/cargo/x": "cargo", "/gem/x": "rubygems", "/go/x": "golang", "/hex/x": "hex", "/pub/x": "pub", "/pypi/x": "pypi", "/maven/x": "maven", "/gradle/x": "gradle", "/nuget/x": "nuget", "/composer/x": "packagist", - "/conan/x": "conan", "/conda/x": "conda", "/cran/x": "cran", + "/conan/x": "conan", "/chef/x": "chef", "/conda/x": "conda", "/cran/x": "cran", "/julia/x": "julia", "/swift/x": "swift", "/v2/x": "oci", "/apk/x": "alpine", "/arch/x": "arch", "/debian/x": "debian", "/rpm/x": "rpm", "/helm/x": "helm", "/homebrew/x": "homebrew", "/generic/x": "generic", diff --git a/internal/server/server.go b/internal/server/server.go index a6e121c..321633b 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -13,6 +13,7 @@ // - /nuget/* - NuGet V3 API protocol // - /composer/* - Composer/Packagist protocol // - /conan/* - Conan C/C++ protocol +// - /chef/* - Chef Supermarket protocol // - /conda/* - Conda/Anaconda protocol // - /cran/* - CRAN (R) protocol // - /julia/* - Julia Pkg server protocol @@ -442,6 +443,7 @@ func (s *Server) mountProtocolHandlers(r chi.Router, proxy *handler.Proxy) { s.cfg.Upstream.ComposerRepository, ).WithPackageRoutes(s.cfg.Upstream.ComposerRoutes) conanHandler := handler.NewConanHandlerWithUpstream(proxy, s.cfg.BaseURL, s.cfg.Upstream.Conan) + chefHandler := handler.NewChefHandler(proxy, s.cfg.BaseURL, s.cfg.Upstream.Chef) condaHandler := handler.NewCondaHandlerWithUpstream(proxy, s.cfg.BaseURL, s.cfg.Upstream.Conda) cranHandler := handler.NewCRANHandlerWithUpstream(proxy, s.cfg.BaseURL, s.cfg.Upstream.CRAN) juliaHandler := handler.NewJuliaHandlerWithUpstream(proxy, s.cfg.Upstream.Julia) @@ -474,6 +476,7 @@ func (s *Server) mountProtocolHandlers(r chi.Router, proxy *handler.Proxy) { r.Mount("/nuget", http.StripPrefix("/nuget", nugetHandler.Routes())) r.Mount("/composer", http.StripPrefix("/composer", composerHandler.Routes())) r.Mount("/conan", http.StripPrefix("/conan", conanHandler.Routes())) + r.Mount("/chef", http.StripPrefix("/chef", chefHandler.Routes())) r.Mount("/conda", http.StripPrefix("/conda", condaHandler.Routes())) r.Mount("/cran", http.StripPrefix("/cran", cranHandler.Routes())) r.Mount("/julia", http.StripPrefix("/julia", juliaHandler.Routes())) From ad7f29919bb4ccd66bc0afa4084b007c46e19dec Mon Sep 17 00:00:00 2001 From: Abhinav Gautam Date: Mon, 19 Oct 2026 12:32:05 +0530 Subject: [PATCH 316/633] Refuse package mirroring from the CLI while scanning is enabled (#493) --- README.md | 2 + cmd/proxy/main.go | 21 +++++- cmd/proxy/mirror_test.go | 124 +++++++++++++++++++++++++++++++++ docs/configuration.md | 2 + internal/mirror/mirror.go | 11 +++ internal/mirror/mirror_test.go | 89 +++++++++++++++++++++++ 6 files changed, 246 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 5b82620..8611a37 100644 --- a/README.md +++ b/README.md @@ -1032,6 +1032,8 @@ proxy mirror --platform linux/amd64 pkg:docker/library/alpine@3.20 The mirror command accepts the same storage and database flags as `serve`. Already-cached artifacts are skipped. If any package version fails to mirror, the command lists the errors and exits with status 1, so a CI step that warms the cache fails on a partial mirror. +With `scanning.enabled`, the mirror command refuses every package and container image so nothing is cached unscanned. Each one is reported as failed, so the command exits with status 1. Scanners fetch the staged bytes from the running server, which this command does not start. Use the mirror API below instead, since its jobs run inside the server and go through the pre-cache scan. `--dry-run` still lists what would be mirrored. + A mirror API is also available when the server is running: ```bash diff --git a/cmd/proxy/main.go b/cmd/proxy/main.go index 7bc8eb4..e3505f2 100644 --- a/cmd/proxy/main.go +++ b/cmd/proxy/main.go @@ -559,9 +559,7 @@ func runMirror() { m := mirror.New(proxy, db, store, logger, *concurrency) m.Images = images if cfg.Scanning.Enabled { - // Scanners fetch staged layers from the running server, which this - // command does not start. - m.Images = mirror.UnavailableImages{Reason: "container images cannot be mirrored by this command while scanning is enabled; use the mirror API of the running server"} + refuseUnscannedMirroring(m) } ctx, cancel := context.WithCancel(context.Background()) @@ -618,6 +616,23 @@ func mirrorExitCode(progress *mirror.Progress) int { return 0 } +// Reasons the mirror command gives for refusing items while scanning is +// enabled. +const ( + mirrorScanningImagesReason = "container images cannot be mirrored by this command while scanning is enabled; use the mirror API of the running server" + mirrorScanningPackagesReason = "packages cannot be mirrored by this command while scanning is enabled; use the mirror API of the running server" +) + +// refuseUnscannedMirroring makes m fail every item instead of caching it. +// Scanners fetch the staged bytes from the running server, which this command +// does not start, so anything it cached would skip the pre-cache scan and +// later be served from the cache unscanned. The mirror API's jobs run inside +// the server with its scanners and are unaffected. +func refuseUnscannedMirroring(m *mirror.Mirror) { + m.Images = mirror.UnavailableImages{Reason: mirrorScanningImagesReason} + m.RefusePackages = mirrorScanningPackagesReason +} + func printStats(db *database.DB, popular, recent int, asJSON bool) error { defer func() { _ = db.Close() }() diff --git a/cmd/proxy/mirror_test.go b/cmd/proxy/mirror_test.go index 4db2a89..e0f7e6a 100644 --- a/cmd/proxy/mirror_test.go +++ b/cmd/proxy/mirror_test.go @@ -8,9 +8,133 @@ import ( "strings" "testing" + "github.com/git-pkgs/proxy/internal/database" "github.com/git-pkgs/proxy/internal/mirror" ) +// mirrorTestEnv carries the mirror command's arguments into the helper +// process, one per line. +const mirrorTestEnv = "PROXY_TEST_MIRROR_ARGS" + +// writeScanningMirrorConfig writes a config with scanning enabled and storage +// and database under dir. The scanner address is never reached. +func writeScanningMirrorConfig(t *testing.T, dir string) string { + t.Helper() + cfg := "storage:\n" + + " url: file://" + filepath.Join(dir, "artifacts") + "\n" + + "database:\n" + + " path: " + filepath.Join(dir, "proxy.db") + "\n" + + "scanning:\n" + + " enabled: true\n" + + " signing_key: test-signing-key\n" + + " scanners:\n" + + " - name: test\n" + + " url: http://127.0.0.1:1/scan\n" + path := filepath.Join(dir, "config.yaml") + if err := os.WriteFile(path, []byte(cfg), 0o600); err != nil { + t.Fatalf("writing config: %v", err) + } + return path +} + +// runMirrorProcess runs proxy mirror with args in a helper process and +// checks that it exits with wantCode. +func runMirrorProcess(t *testing.T, wantCode int, args ...string) string { + t.Helper() + cmd := exec.Command(os.Args[0], "-test.run=^TestMirrorHelperProcess$") + cmd.Env = append(os.Environ(), mirrorTestEnv+"="+strings.Join(args, "\n")) + output, err := cmd.CombinedOutput() + + code := 0 + var exitErr *exec.ExitError + if errors.As(err, &exitErr) { + code = exitErr.ExitCode() + } else if err != nil { + t.Fatalf("proxy mirror %v: %v\n%s", args, err, output) + } + if code != wantCode { + t.Fatalf("proxy mirror %v exited with status %d, want %d\n%s", args, code, wantCode, output) + } + return string(output) +} + +// With scanning enabled, proxy mirror refuses packages rather than caching +// them unscanned: scanners fetch staged bytes from the running server, which +// the command does not start. +// A refused package counts as failed, so the command exits 1. +func TestMirrorCommandRefusesPackagesWhileScanning(t *testing.T) { + dir := t.TempDir() + configPath := writeScanningMirrorConfig(t, dir) + + output := runMirrorProcess(t, 1, "-config", configPath, "pkg:npm/lodash@4.17.21") + + for _, want := range []string{mirrorScanningPackagesReason, "0 downloaded", "1 failed"} { + if !strings.Contains(output, want) { + t.Errorf("output missing %q:\n%s", want, output) + } + } + + db, err := database.Open(filepath.Join(dir, "proxy.db")) + if err != nil { + t.Fatalf("opening database: %v", err) + } + defer func() { _ = db.Close() }() + stats, err := db.GetCacheStats() + if err != nil { + t.Fatalf("GetCacheStats: %v", err) + } + if stats.TotalArtifacts != 0 { + t.Errorf("cached artifacts = %d, want 0", stats.TotalArtifacts) + } + + var stored []string + _ = filepath.WalkDir(filepath.Join(dir, "artifacts"), func(path string, d os.DirEntry, err error) error { + if err == nil && !d.IsDir() { + stored = append(stored, path) + } + return nil + }) + if len(stored) != 0 { + t.Errorf("files written to storage: %v", stored) + } +} + +// A dry run caches nothing, so it still lists what would be mirrored. +func TestMirrorCommandDryRunWhileScanning(t *testing.T) { + configPath := writeScanningMirrorConfig(t, t.TempDir()) + + output := runMirrorProcess(t, 0, "-config", configPath, "-dry-run", "pkg:npm/lodash@4.17.21") + + if !strings.Contains(output, "Would mirror 1 package versions") { + t.Errorf("dry run output:\n%s", output) + } + if strings.Contains(output, mirrorScanningPackagesReason) { + t.Errorf("dry run reported the refusal:\n%s", output) + } +} + +func TestRefuseUnscannedMirroring(t *testing.T) { + m := mirror.New(nil, nil, nil, nil, 1) + refuseUnscannedMirroring(m) + + if m.RefusePackages != mirrorScanningPackagesReason { + t.Errorf("RefusePackages = %q, want %q", m.RefusePackages, mirrorScanningPackagesReason) + } + images, ok := m.Images.(mirror.UnavailableImages) + if !ok || images.Reason != mirrorScanningImagesReason { + t.Errorf("Images = %#v, want UnavailableImages with the scanning reason", m.Images) + } +} + +func TestMirrorHelperProcess(*testing.T) { + args := os.Getenv(mirrorTestEnv) + if args == "" { + return + } + os.Args = append([]string{"proxy", "mirror"}, strings.Split(args, "\n")...) + main() +} + func TestMirrorExitCode(t *testing.T) { tests := []struct { name string diff --git a/docs/configuration.md b/docs/configuration.md index 1466a24..33eb2ed 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -874,6 +874,8 @@ Scanners configured for the same ecosystem all run concurrently, never sequentia A blocked download surfaces to the client as `403 Forbidden` with the scanner's reason, across every ecosystem handler. +Because scanners pull the staged bytes from the running server, the `proxy mirror` command cannot scan what it fetches. While scanning is enabled it refuses every package and container image instead of caching them unscanned. Mirror through the `/api/mirror` endpoints of the running server, whose jobs are scanned like any other download. + ### Scanner HTTP contract Any external service that implements this contract can act as a scanner — a trivy wrapper, a clamav-rest bridge, a Wiz connector, or an in-house service. The proxy POSTs a notify request to `scanning.scanners[].url` and waits for a JSON verdict. diff --git a/internal/mirror/mirror.go b/internal/mirror/mirror.go index fb98c11..e946825 100644 --- a/internal/mirror/mirror.go +++ b/internal/mirror/mirror.go @@ -26,6 +26,10 @@ type Mirror struct { // Images mirrors docker and oci items. Without it they fail like any // other unsupported ecosystem. Images ImageMirrorer + + // RefusePackages, when set, fails every item Images does not take with + // this reason instead of fetching it, so nothing is cached. + RefusePackages string } // New creates a new Mirror with the given dependencies. @@ -211,6 +215,13 @@ func (m *Mirror) mirrorOne(ctx context.Context, pv PackageVersion, tracker *prog m.mirrorImage(ctx, pv, tracker) return } + if m.RefusePackages != "" { + tracker.failed.Add(1) + tracker.addError(pv.Ecosystem, pv.Name, pv.Version, m.RefusePackages) + m.logger.Warn("mirror refused", + "ecosystem", pv.Ecosystem, "name", pv.Name, "version", pv.Version, "reason", m.RefusePackages) + return + } result, err := m.proxy.GetOrFetchArtifact(ctx, pv.Ecosystem, pv.Name, pv.Version, "") if err != nil { tracker.failed.Add(1) diff --git a/internal/mirror/mirror_test.go b/internal/mirror/mirror_test.go index 45dd6f3..6852687 100644 --- a/internal/mirror/mirror_test.go +++ b/internal/mirror/mirror_test.go @@ -3,10 +3,12 @@ package mirror import ( "context" "database/sql" + "errors" "log/slog" "net/http" "os" "strings" + "sync" "testing" "time" @@ -261,3 +263,90 @@ func TestNewMirrorDefaultWorkers(t *testing.T) { t.Errorf("workers = %d, want 1 (minimum)", m.workers) } } + +// recordingFetcher fails every fetch and records the URLs it was asked for, so +// a test can show nothing reached upstream. +type recordingFetcher struct { + mu sync.Mutex + urls []string +} + +func (f *recordingFetcher) record(url string) error { + f.mu.Lock() + defer f.mu.Unlock() + f.urls = append(f.urls, url) + return errors.New("recordingFetcher: no upstream in this test") +} + +func (f *recordingFetcher) Fetch(_ context.Context, url string) (*fetch.Artifact, error) { + return nil, f.record(url) +} + +func (f *recordingFetcher) FetchWithHeaders(_ context.Context, url string, _ http.Header) (*fetch.Artifact, error) { + return nil, f.record(url) +} + +func (f *recordingFetcher) Head(_ context.Context, url string) (int64, string, error) { + return 0, "", f.record(url) +} + +func (f *recordingFetcher) fetched() []string { + f.mu.Lock() + defer f.mu.Unlock() + return append([]string(nil), f.urls...) +} + +func TestMirrorRunRefusesPackages(t *testing.T) { + m := setupTestMirror(t, 2) + fetcher := &recordingFetcher{} + m.proxy.Fetcher = fetcher + const reason = "packages cannot be mirrored here" + m.RefusePackages = reason + + source := &PURLSource{PURLs: []string{"pkg:npm/lodash@4.17.21", "pkg:cargo/serde@1.0.0"}} + progress, err := m.Run(context.Background(), source) + if err != nil { + t.Fatalf("Run: %v", err) + } + + if progress.Failed != 2 || progress.Completed != 0 || progress.Skipped != 0 { + t.Errorf("progress = %d failed, %d completed, %d skipped; want 2 failed only", + progress.Failed, progress.Completed, progress.Skipped) + } + if len(progress.Errors) != 2 { + t.Fatalf("errors = %+v, want one per package", progress.Errors) + } + for _, e := range progress.Errors { + if e.Error != reason { + t.Errorf("%s/%s error = %q, want %q", e.Ecosystem, e.Name, e.Error, reason) + } + } + if urls := fetcher.fetched(); len(urls) != 0 { + t.Errorf("refused packages were fetched from upstream: %v", urls) + } + + stats, err := m.db.GetCacheStats() + if err != nil { + t.Fatalf("GetCacheStats: %v", err) + } + if stats.TotalArtifacts != 0 { + t.Errorf("cached artifacts = %d, want 0", stats.TotalArtifacts) + } +} + +// Images keep their own refusal, so a mirror refusing packages still hands +// docker and oci items to Images. +func TestMirrorRefusePackagesLeavesImagesToImages(t *testing.T) { + m := setupTestMirror(t, 1) + m.proxy.Fetcher = &recordingFetcher{} + m.RefusePackages = "packages refused" + m.Images = UnavailableImages{Reason: "images refused"} + + progress, err := m.Run(context.Background(), &PURLSource{PURLs: []string{"pkg:docker/library/alpine@3.20"}}) + if err != nil { + t.Fatalf("Run: %v", err) + } + if len(progress.Errors) != 1 || progress.Errors[0].Error != "images refused" { + t.Errorf("errors = %+v, want the image refusal", progress.Errors) + } +} From d58dd3dc3dfd2c36ab06da2f50c7ab0693d2633f Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Mon, 19 Oct 2026 08:35:46 +0100 Subject: [PATCH 317/633] Update x/net and remove unused production helpers --- go.mod | 2 +- go.sum | 4 +- internal/handler/composer.go | 18 +---- .../handler/composer_rewrite_bench_test.go | 2 +- internal/handler/composer_rewrite_test.go | 14 ++-- internal/handler/composer_test.go | 16 ++-- internal/handler/conan.go | 34 -------- internal/handler/conan_test.go | 81 ++++++++++--------- internal/handler/container.go | 8 -- internal/handler/container_referrers_test.go | 2 +- internal/handler/container_test.go | 6 +- internal/handler/denylist_test.go | 14 +++- internal/handler/download_test.go | 11 ++- internal/handler/handler_bench_test.go | 2 +- internal/handler/handler_test.go | 25 ++---- internal/handler/helm_test.go | 3 +- internal/handler/jsonscan_test.go | 2 +- internal/handler/npm.go | 10 +-- internal/handler/pypi.go | 5 -- internal/handler/rewrite_cache.go | 10 --- internal/handler/rewrite_cache_test.go | 10 +++ internal/handler/stream_artifacts_test.go | 2 +- internal/jsonstream/jsonstream.go | 8 +- internal/jsonstream/jsonstream_test.go | 10 ++- internal/metrics/failures.go | 9 --- internal/metrics/failures_test.go | 9 ++- internal/metrics/snapshot.go | 14 ---- internal/server/analytics_coverage_test.go | 7 +- internal/server/analytics_failures_test.go | 17 +--- internal/server/eviction_test.go | 12 +-- internal/server/retention_test.go | 2 +- internal/server/server_test.go | 2 +- internal/storage/blob_test.go | 2 +- internal/storage/storage.go | 12 --- internal/storage/storage_test.go | 25 ------ 35 files changed, 135 insertions(+), 275 deletions(-) diff --git a/go.mod b/go.mod index 670f884..0b791cd 100644 --- a/go.mod +++ b/go.mod @@ -30,7 +30,7 @@ require ( github.com/swaggo/swag v1.16.6 gocloud.dev v0.46.0 golang.org/x/mod v0.41.0 - golang.org/x/net v0.59.0 + golang.org/x/net v0.60.0 golang.org/x/sync v0.24.0 google.golang.org/protobuf v1.36.12 gopkg.in/yaml.v3 v3.0.1 diff --git a/go.sum b/go.sum index d2ed244..956fd28 100644 --- a/go.sum +++ b/go.sum @@ -326,8 +326,8 @@ golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZ golang.org/x/mod v0.41.0 h1:qJmnOUb4YB+FsEuM3HcWucdZASCPGhsX6uljO6pog0c= golang.org/x/mod v0.41.0/go.mod h1:Ek9pY8RKWXwsWvd3rQiHYtMqkjSUV+s1Rj7j4H5Ur6o= golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1Kcs5dz7/ng1VjMUvfKvpfy+jM= -golang.org/x/net v0.59.0 h1:5zfYln+w5XCxwrnMMJPufRgNoXEaGxl0wo5GqPXyues= -golang.org/x/net v0.59.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg= +golang.org/x/net v0.60.0 h1:79p50tfZlm0J9YfoDsSi639qSXNGVwEzOPLCxM2FsYU= +golang.org/x/net v0.60.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg= golang.org/x/oauth2 v0.37.0 h1:JUlcxA8oAtauLfiH8FX2/FkAWHAdi0QtGCGc+hofE98= golang.org/x/oauth2 v0.37.0/go.mod h1:IxwZNxUULJmpBFf9K/9NTMSIfZZuvuTy1gGxhigP/58= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= diff --git a/internal/handler/composer.go b/internal/handler/composer.go index 121f228..5bc2586 100644 --- a/internal/handler/composer.go +++ b/internal/handler/composer.go @@ -197,22 +197,8 @@ func (h *ComposerHandler) handlePackageMetadata(w http.ResponseWriter, r *http.R serveRewrittenMetadata(w, r, rewritten) } -// rewriteMetadata rewrites dist URLs in Composer metadata to point at this proxy. -// If cooldown is enabled, versions published too recently are filtered out. -// -// The document is never decoded as a whole. Packagist serves large packages -// as minified lists of thousands of versions, and expanding them into generic -// maps took many times the document's size for every rewrite. Instead each -// version list is walked in place and the response is assembled from slices -// of the original bytes: only dist values are written fresh, and minified -// metadata stays minified. -func (h *ComposerHandler) rewriteMetadata(body []byte) ([]byte, error) { - out, _, err := h.rewriteMetadataKeeping(body) - return out, err -} - -// rewriteMetadataKeeping is rewriteMetadata that also reports the versions -// it kept, each as its package name and version. +// rewriteMetadataKeeping rewrites dist URLs and filters versions in cooldown. +// It returns the kept package names and versions without expanding minified lists. func (h *ComposerHandler) rewriteMetadataKeeping(body []byte) ([]byte, []string, error) { return h.rewriteMetadataWithLocal(body, "", nil) } diff --git a/internal/handler/composer_rewrite_bench_test.go b/internal/handler/composer_rewrite_bench_test.go index 1fc8c5a..075b371 100644 --- a/internal/handler/composer_rewrite_bench_test.go +++ b/internal/handler/composer_rewrite_bench_test.go @@ -56,7 +56,7 @@ func BenchmarkComposerRewriteMetadata(b *testing.B) { b.SetBytes(int64(len(body))) b.ReportAllocs() for b.Loop() { - if _, err := h.rewriteMetadata(body); err != nil { + if _, _, err := h.rewriteMetadataKeeping(body); err != nil { b.Fatal(err) } } diff --git a/internal/handler/composer_rewrite_test.go b/internal/handler/composer_rewrite_test.go index 1eb6f7f..460e38e 100644 --- a/internal/handler/composer_rewrite_test.go +++ b/internal/handler/composer_rewrite_test.go @@ -86,7 +86,7 @@ func TestComposerRewriteMetadataKeepsUpstreamBytes(t *testing.T) { input := `{"packages":{"v/p":[{"version":"1.0.0","extra":{"big":12345678901234567890,"html":"&"},` + `"dist":{"type":"zip","url":"https://example.com/abc.zip","reference":"abc"}}]},"security-advisories":[]}` - out, err := h.rewriteMetadata([]byte(input)) + out, _, err := h.rewriteMetadataKeeping([]byte(input)) if err != nil { t.Fatal(err) } @@ -107,7 +107,7 @@ func TestComposerRewriteMetadataStaysMinified(t *testing.T) { `{"version":"1.0.0","require":"__unset"}` + `]}}` - out, err := h.rewriteMetadata([]byte(input)) + out, _, err := h.rewriteMetadataKeeping([]byte(input)) if err != nil { t.Fatal(err) } @@ -136,7 +136,7 @@ func TestComposerRewriteMetadataCooldownCarriesChanges(t *testing.T) { {"version":"5.4.8","time":"` + old + `"} ]}}` - out, err := cooldownComposerHandler().rewriteMetadata([]byte(input)) + out, _, err := cooldownComposerHandler().rewriteMetadataKeeping([]byte(input)) if err != nil { t.Fatal(err) } @@ -174,7 +174,7 @@ func TestComposerRewriteMetadataCooldownAfterDevReset(t *testing.T) { {"version":"1.x-dev","time":"` + old + `"} ]}}` - out, err := cooldownComposerHandler().rewriteMetadata([]byte(input)) + out, _, err := cooldownComposerHandler().rewriteMetadataKeeping([]byte(input)) if err != nil { t.Fatal(err) } @@ -196,7 +196,7 @@ func TestComposerRewriteMetadataCooldownAfterDevReset(t *testing.T) { func TestComposerRewriteMetadataInvalidJSON(t *testing.T) { h := &ComposerHandler{proxy: testProxy(), proxyURL: "http://proxy"} - if _, err := h.rewriteMetadata([]byte(`{"packages":{"v/p":[{"version":"1.0.0"}]}`)); err == nil { + if _, _, err := h.rewriteMetadataKeeping([]byte(`{"packages":{"v/p":[{"version":"1.0.0"}]}`)); err == nil { t.Error("expected an error for truncated JSON") } } @@ -204,7 +204,7 @@ func TestComposerRewriteMetadataInvalidJSON(t *testing.T) { func TestComposerRewriteMetadataWithoutPackages(t *testing.T) { h := &ComposerHandler{proxy: testProxy(), proxyURL: "http://proxy"} for _, input := range []string{`{"other":1}`, `{"packages":[]}`} { - out, err := h.rewriteMetadata([]byte(input)) + out, _, err := h.rewriteMetadataKeeping([]byte(input)) if err != nil { t.Fatalf("%s: %v", input, err) } @@ -266,7 +266,7 @@ func TestComposerRewriteMetadataRewritesEveryPackage(t *testing.T) { "v/b":[{"version":"2.0.0","dist":{"url":"https://example.com/b.zip","type":"zip"}}], "v/c":"not a list" }}` - out, err := h.rewriteMetadata([]byte(input)) + out, _, err := h.rewriteMetadataKeeping([]byte(input)) if err != nil { t.Fatal(err) } diff --git a/internal/handler/composer_test.go b/internal/handler/composer_test.go index d2955eb..a964dfc 100644 --- a/internal/handler/composer_test.go +++ b/internal/handler/composer_test.go @@ -38,7 +38,7 @@ func TestComposerRewriteMetadata(t *testing.T) { } }` - output, err := h.rewriteMetadata([]byte(input)) + output, _, err := h.rewriteMetadataKeeping([]byte(input)) if err != nil { t.Fatalf("rewriteMetadata failed: %v", err) } @@ -93,7 +93,7 @@ func TestComposerRewriteMetadataMinifiedInheritance(t *testing.T) { } }` - output, err := h.rewriteMetadata([]byte(input)) + output, _, err := h.rewriteMetadataKeeping([]byte(input)) if err != nil { t.Fatalf("rewriteMetadata failed: %v", err) } @@ -153,7 +153,7 @@ func TestComposerRewriteMetadataMinifiedDevReset(t *testing.T) { } }` - output, err := h.rewriteMetadata([]byte(input)) + output, _, err := h.rewriteMetadataKeeping([]byte(input)) if err != nil { t.Fatalf("rewriteMetadata failed: %v", err) } @@ -214,7 +214,7 @@ func TestComposerRewriteMetadataUnset(t *testing.T) { } }` - output, err := h.rewriteMetadata([]byte(input)) + output, _, err := h.rewriteMetadataKeeping([]byte(input)) if err != nil { t.Fatalf("rewriteMetadata failed: %v", err) } @@ -288,7 +288,7 @@ func TestComposerRewriteMetadataCooldownPreservesNames(t *testing.T) { } }` - output, err := h.rewriteMetadata([]byte(input)) + output, _, err := h.rewriteMetadataKeeping([]byte(input)) if err != nil { t.Fatalf("rewriteMetadata failed: %v", err) } @@ -352,7 +352,7 @@ func TestComposerRewriteMetadataGitHubZipballFilenames(t *testing.T) { } }` - output, err := h.rewriteMetadata([]byte(input)) + output, _, err := h.rewriteMetadataKeeping([]byte(input)) if err != nil { t.Fatalf("rewriteMetadata failed: %v", err) } @@ -405,7 +405,7 @@ func TestComposerExpandMinifiedSharedDistReferences(t *testing.T) { } }` - output, err := h.rewriteMetadata([]byte(input)) + output, _, err := h.rewriteMetadataKeeping([]byte(input)) if err != nil { t.Fatalf("rewriteMetadata failed: %v", err) } @@ -959,7 +959,7 @@ func TestComposerRewriteMetadataCooldown(t *testing.T) { } }` - output, err := h.rewriteMetadata([]byte(input)) + output, _, err := h.rewriteMetadataKeeping([]byte(input)) if err != nil { t.Fatalf("rewriteMetadata failed: %v", err) } diff --git a/internal/handler/conan.go b/internal/handler/conan.go index 9bb4401..4208be1 100644 --- a/internal/handler/conan.go +++ b/internal/handler/conan.go @@ -173,37 +173,3 @@ func (h *ConanHandler) proxyCached(w http.ResponseWriter, r *http.Request) { } h.proxy.ProxyCached(w, r, upstreamURL, "conan", cacheKey, "*/*") } - -// proxyUpstream forwards a request to conan center without caching. -func (h *ConanHandler) proxyUpstream(w http.ResponseWriter, r *http.Request) { - upstreamURL := h.upstreamURL + r.URL.Path - if r.URL.RawQuery != "" { - upstreamURL += "?" + r.URL.RawQuery - } - - h.proxy.Logger.Debug("proxying to upstream", "url", upstreamURL) - - req, err := http.NewRequestWithContext(r.Context(), http.MethodGet, upstreamURL, nil) - if err != nil { - http.Error(w, "failed to create request", http.StatusInternalServerError) - return - } - - // Copy authorization header for authenticated requests - if auth := r.Header.Get("Authorization"); auth != "" { - req.Header.Set("Authorization", auth) - } - - resp, err := h.proxy.HTTPClient.Do(req) - if err != nil { - if writeUpstreamHeld(w, err) { - return - } - h.proxy.Logger.Error("upstream request failed", "error", err) - http.Error(w, "upstream request failed", http.StatusBadGateway) - return - } - defer func() { _ = resp.Body.Close() }() - - h.proxy.relayResponse(w, r, resp, nil) -} diff --git a/internal/handler/conan_test.go b/internal/handler/conan_test.go index f2fb909..cf8038e 100644 --- a/internal/handler/conan_test.go +++ b/internal/handler/conan_test.go @@ -94,8 +94,10 @@ func TestConanPingV2(t *testing.T) { } } -func TestConanProxyUpstream(t *testing.T) { +func TestConanMetadata(t *testing.T) { + var requests atomic.Int32 upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests.Add(1) if r.URL.Path != "/v2/conans/search" { w.WriteHeader(http.StatusNotFound) return @@ -110,27 +112,26 @@ func TestConanProxyUpstream(t *testing.T) { })) defer upstream.Close() - h := &ConanHandler{ - proxy: conanTestProxy(), - upstreamURL: upstream.URL, - proxyURL: "http://proxy.local", + proxy, _, _, _ := setupTestProxy(t) + proxy.CacheMetadata = true + proxy.MetadataTTL = time.Hour + proxy.HTTPClient = upstream.Client() + h := NewConanHandlerWithUpstream(proxy, "http://proxy.local", upstream.URL) + routes := h.Routes() + for range 2 { + req := httptest.NewRequest(http.MethodGet, "/v2/conans/search?q=zlib", nil) + w := httptest.NewRecorder() + routes.ServeHTTP(w, req) + if w.Code != http.StatusOK || w.Body.String() != `{"results":["zlib/1.2.13"]}` { + t.Fatalf("response = %d %q", w.Code, w.Body.String()) + } } - - req := httptest.NewRequest(http.MethodGet, "/v2/conans/search?q=zlib", nil) - w := httptest.NewRecorder() - h.proxyUpstream(w, req) - - if w.Code != http.StatusOK { - t.Errorf("status = %d, want %d", w.Code, http.StatusOK) - } - - body := w.Body.String() - if !strings.Contains(body, "zlib/1.2.13") { - t.Errorf("response body does not contain expected result: %s", body) + if got := requests.Load(); got != 1 { + t.Errorf("upstream requests = %d, want 1", got) } } -func TestConanProxyUpstreamNotFound(t *testing.T) { +func TestConanMetadataNotFound(t *testing.T) { upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusNotFound) })) @@ -144,16 +145,16 @@ func TestConanProxyUpstreamNotFound(t *testing.T) { req := httptest.NewRequest(http.MethodGet, "/v2/conans/nonexistent", nil) w := httptest.NewRecorder() - h.proxyUpstream(w, req) + h.Routes().ServeHTTP(w, req) if w.Code != http.StatusNotFound { t.Errorf("status = %d, want %d", w.Code, http.StatusNotFound) } } -func TestConanProxyUpstreamCopiesHeaders(t *testing.T) { +func TestConanMetadataCopiesHeaders(t *testing.T) { upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("X-Custom-Header", "test-value") + w.Header().Set("ETag", `"metadata"`) w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusOK) _, _ = w.Write([]byte(`{}`)) @@ -168,17 +169,17 @@ func TestConanProxyUpstreamCopiesHeaders(t *testing.T) { req := httptest.NewRequest(http.MethodGet, "/v2/conans/test", nil) w := httptest.NewRecorder() - h.proxyUpstream(w, req) + h.Routes().ServeHTTP(w, req) - if w.Header().Get("X-Custom-Header") != "test-value" { - t.Errorf("X-Custom-Header = %q, want %q", w.Header().Get("X-Custom-Header"), "test-value") + if w.Header().Get("ETag") != `"metadata"` { + t.Errorf("ETag = %q, want %q", w.Header().Get("ETag"), `"metadata"`) } } -func TestConanProxyUpstreamForwardsAuthHeader(t *testing.T) { +func TestConanMetadataDoesNotForwardClientAuth(t *testing.T) { upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { auth := r.Header.Get("Authorization") - if auth != "Bearer mytoken" { + if auth != "" { w.WriteHeader(http.StatusUnauthorized) return } @@ -196,14 +197,14 @@ func TestConanProxyUpstreamForwardsAuthHeader(t *testing.T) { req := httptest.NewRequest(http.MethodGet, "/v2/conans/test", nil) req.Header.Set("Authorization", "Bearer mytoken") w := httptest.NewRecorder() - h.proxyUpstream(w, req) + h.Routes().ServeHTTP(w, req) if w.Code != http.StatusOK { t.Errorf("status = %d, want %d", w.Code, http.StatusOK) } } -func TestConanProxyUpstreamBadUpstream(t *testing.T) { +func TestConanMetadataBadUpstream(t *testing.T) { h := &ConanHandler{ proxy: conanTestProxy(), upstreamURL: "http://127.0.0.1:1", // unreachable @@ -212,7 +213,7 @@ func TestConanProxyUpstreamBadUpstream(t *testing.T) { req := httptest.NewRequest(http.MethodGet, "/v2/conans/test", nil) w := httptest.NewRecorder() - h.proxyUpstream(w, req) + h.Routes().ServeHTTP(w, req) if w.Code != http.StatusBadGateway { t.Errorf("status = %d, want %d", w.Code, http.StatusBadGateway) @@ -487,9 +488,9 @@ func TestConanRoutes(t *testing.T) { } } -func TestConanProxyUpstreamPreservesQueryString(t *testing.T) { +func TestConanMetadataPreservesQueryString(t *testing.T) { upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.URL.Query().Get("q") != "boost" && r.URL.Query().Get("page") != "2" { + if r.URL.Query().Get("q") != "boost" || r.URL.Query().Get("page") != "2" { w.WriteHeader(http.StatusBadRequest) return } @@ -506,14 +507,14 @@ func TestConanProxyUpstreamPreservesQueryString(t *testing.T) { req := httptest.NewRequest(http.MethodGet, "/v2/conans/search?q=boost&page=2", nil) w := httptest.NewRecorder() - h.proxyUpstream(w, req) + h.Routes().ServeHTTP(w, req) if w.Code != http.StatusOK { t.Errorf("status = %d, want %d", w.Code, http.StatusOK) } } -func TestConanProxyUpstreamLargeResponse(t *testing.T) { +func TestConanMetadataLargeResponse(t *testing.T) { largeBody := strings.Repeat("x", 1024*1024) upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) @@ -529,7 +530,7 @@ func TestConanProxyUpstreamLargeResponse(t *testing.T) { req := httptest.NewRequest(http.MethodGet, "/v2/conans/test", nil) w := httptest.NewRecorder() - h.proxyUpstream(w, req) + h.Routes().ServeHTTP(w, req) if w.Code != http.StatusOK { t.Errorf("status = %d, want %d", w.Code, http.StatusOK) @@ -564,7 +565,7 @@ func TestNewConanHandlerNoTrailingSlash(t *testing.T) { } } -func TestConanProxyUpstreamNoAuthHeaderWhenNotProvided(t *testing.T) { +func TestConanMetadataNoAuthHeaderWhenNotProvided(t *testing.T) { upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { auth := r.Header.Get("Authorization") if auth != "" { @@ -584,14 +585,14 @@ func TestConanProxyUpstreamNoAuthHeaderWhenNotProvided(t *testing.T) { req := httptest.NewRequest(http.MethodGet, "/v2/conans/test", nil) w := httptest.NewRecorder() - h.proxyUpstream(w, req) + h.Routes().ServeHTTP(w, req) if w.Code != http.StatusOK { t.Errorf("status = %d, want %d", w.Code, http.StatusOK) } } -func TestConanProxyUpstreamCopiesBody(t *testing.T) { +func TestConanMetadataCopiesBody(t *testing.T) { expected := `{"name":"zlib","version":"1.2.13","user":"_","channel":"_"}` upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") @@ -608,7 +609,7 @@ func TestConanProxyUpstreamCopiesBody(t *testing.T) { req := httptest.NewRequest(http.MethodGet, "/v2/conans/zlib/1.2.13/_/_/latest", nil) w := httptest.NewRecorder() - h.proxyUpstream(w, req) + h.Routes().ServeHTTP(w, req) if w.Code != http.StatusOK { t.Errorf("status = %d, want %d", w.Code, http.StatusOK) @@ -620,7 +621,7 @@ func TestConanProxyUpstreamCopiesBody(t *testing.T) { } } -func TestConanProxyUpstreamPreservesStatusCodes(t *testing.T) { +func TestConanMetadataPreservesStatusCodes(t *testing.T) { codes := []int{ http.StatusOK, http.StatusNotFound, @@ -641,7 +642,7 @@ func TestConanProxyUpstreamPreservesStatusCodes(t *testing.T) { req := httptest.NewRequest(http.MethodGet, "/v2/test", nil) w := httptest.NewRecorder() - h.proxyUpstream(w, req) + h.Routes().ServeHTTP(w, req) if w.Code != code { t.Errorf("status = %d, want %d", w.Code, code) diff --git a/internal/handler/container.go b/internal/handler/container.go index 55f1715..1d90605 100644 --- a/internal/handler/container.go +++ b/internal/handler/container.go @@ -121,14 +121,6 @@ type containerRegistry struct { registryURL string } -// NewContainerHandler creates a new container registry protocol handler. -// Named registries are selected with the repository prefix -// upstream/{name}/, leaving unprefixed requests compatible with the Docker Hub -// mirror behavior. -func NewContainerHandler(proxy *Proxy, proxyURL string, namedRegistries ...map[string]string) *ContainerHandler { - return newContainerHandler(proxy, proxyURL, dockerHubRegistry, namedRegistries...) -} - // NewContainerHandlerWithRegistry creates a container handler with a custom // default registry and optional named registries. func NewContainerHandlerWithRegistry( diff --git a/internal/handler/container_referrers_test.go b/internal/handler/container_referrers_test.go index 9201f62..77ef5c8 100644 --- a/internal/handler/container_referrers_test.go +++ b/internal/handler/container_referrers_test.go @@ -475,7 +475,7 @@ func TestContainerHandler_ReferrersFollowsNamespace(t *testing.T) { proxy, _, _, _ := setupTestProxy(t) proxy.HTTPClient = upstream.Client() proxy.MetadataTTL = time.Hour - h := NewContainerHandler(proxy, "http://proxy.example.test", map[string]string{"test": upstream.URL}) + h := NewContainerHandlerWithRegistry(proxy, "http://proxy.example.test", "", map[string]string{"test": upstream.URL}) routes := http.StripPrefix("/v2", h.Routes()) host := strings.TrimPrefix(upstream.URL, "http://") target := "/v2/owner/img/referrers/" + testReferrersSubject diff --git a/internal/handler/container_test.go b/internal/handler/container_test.go index f00a7b5..f7b7076 100644 --- a/internal/handler/container_test.go +++ b/internal/handler/container_test.go @@ -277,7 +277,7 @@ func TestContainerHandler_TagsListRewritesRelativePaginationLinkForNamedRegistry proxy, _, _, _ := setupTestProxy(t) proxy.HTTPClient = upstream.Client() - h := NewContainerHandler(proxy, "http://proxy.example.test", map[string]string{"test": upstream.URL}) + h := NewContainerHandlerWithRegistry(proxy, "http://proxy.example.test", "", map[string]string{"test": upstream.URL}) routes := http.StripPrefix("/v2", h.Routes()) first := httptest.NewRecorder() @@ -326,7 +326,7 @@ func TestContainerHandler_NamedOCIRegistryServesHelmArtifacts(t *testing.T) { fetcher := fetch.NewFetcher(fetch.WithHTTPClient(upstream.Client()), fetch.WithMaxRetries(0)) proxy.Fetcher = fetcher t.Cleanup(func() { _ = fetcher.Close() }) - h := NewContainerHandler(proxy, "http://proxy.example", map[string]string{"ghcr": upstream.URL}) + h := NewContainerHandlerWithRegistry(proxy, "http://proxy.example", "", map[string]string{"ghcr": upstream.URL}) manifestResponse := httptest.NewRecorder() h.Routes().ServeHTTP(manifestResponse, @@ -1317,7 +1317,7 @@ func TestContainerHandler_ManifestByTag_StaleHeadChecksUpstream(t *testing.T) { } func TestContainerHandler_Routes_VersionCheck(t *testing.T) { - h := NewContainerHandler(nil, "http://localhost:8080") + h := NewContainerHandlerWithRegistry(nil, "http://localhost:8080", "") handler := h.Routes() if handler == nil { diff --git a/internal/handler/denylist_test.go b/internal/handler/denylist_test.go index 383f95c..e08d662 100644 --- a/internal/handler/denylist_test.go +++ b/internal/handler/denylist_test.go @@ -144,7 +144,7 @@ func TestPyPIDenylistSimpleRepresentations(t *testing.T) { func TestPyPIDenylistJSONMetadata(t *testing.T) { p := testProxy() setTestDenylist(t, p, "pkg:pypi/demo@1.0.0") - h := NewPyPIHandler(p, "https://proxy.example") + h := NewPyPIHandlerWithUpstreams(p, "https://proxy.example", "", "") body := `{"info":{"name":"Demo","version":"1.0.0"},"releases":{"1.0.0":[{"url":"https://files.pythonhosted.org/packages/demo-1.0.0.tar.gz"}],"2.0.0":[]},"urls":[{"url":"https://files.pythonhosted.org/packages/demo-1.0.0.tar.gz"}]}` out, err := h.rewriteJSONMetadata([]byte(body)) if err != nil { @@ -170,7 +170,9 @@ func TestDenylistDownloadRoutes(t *testing.T) { handler func(*Proxy) http.Handler }{ {"npm", "/demo/-/demo-1.0.0.tgz", "demo-1.0.0.tgz", func(p *Proxy) http.Handler { return NewNPMHandler(p, "http://proxy.test", "").Routes() }}, - {"pypi", "/packages/a/b/demo-1.0.0.tar.gz", "demo-1.0.0.tar.gz", func(p *Proxy) http.Handler { return NewPyPIHandler(p, "http://proxy.test").Routes() }}, + {"pypi", "/packages/a/b/demo-1.0.0.tar.gz", "demo-1.0.0.tar.gz", func(p *Proxy) http.Handler { + return NewPyPIHandlerWithUpstreams(p, "http://proxy.test", "", "").Routes() + }}, {"cargo", "/crates/demo/1.0.0/download", "demo-1.0.0.crate", func(p *Proxy) http.Handler { return NewCargoHandler(p, "http://proxy.test", "", "").Routes() }}, {"deb", "/pool/main/d/demo/demo_1.0.0_amd64.deb", "demo_1.0.0_amd64.deb", func(p *Proxy) http.Handler { return NewDebianHandler(p, "http://proxy.test", "", nil).Routes() }}, } { @@ -229,8 +231,12 @@ func TestDenylistMalformedMetadataFailsClosed(t *testing.T) { handler func(*Proxy) http.Handler }{ {"npm", "/demo", `{"versions":`, contentTypeJSON, func(p *Proxy) http.Handler { return NewNPMHandler(p, "https://proxy.test", "").Routes() }}, - {"pypi-json", "/pypi/demo/json", `{"releases":`, contentTypeJSON, func(p *Proxy) http.Handler { return NewPyPIHandler(p, "https://proxy.test").Routes() }}, - {"pypi-simple", "/simple/demo/", `{"files":{}}`, pypiSimpleJSON, func(p *Proxy) http.Handler { return NewPyPIHandler(p, "https://proxy.test").Routes() }}, + {"pypi-json", "/pypi/demo/json", `{"releases":`, contentTypeJSON, func(p *Proxy) http.Handler { + return NewPyPIHandlerWithUpstreams(p, "https://proxy.test", "", "").Routes() + }}, + {"pypi-simple", "/simple/demo/", `{"files":{}}`, pypiSimpleJSON, func(p *Proxy) http.Handler { + return NewPyPIHandlerWithUpstreams(p, "https://proxy.test", "", "").Routes() + }}, } { t.Run(tc.name, func(t *testing.T) { p := testProxy() diff --git a/internal/handler/download_test.go b/internal/handler/download_test.go index 2ac8ace..eb332a9 100644 --- a/internal/handler/download_test.go +++ b/internal/handler/download_test.go @@ -14,7 +14,6 @@ import ( "github.com/git-pkgs/proxy/internal/database" "github.com/git-pkgs/proxy/internal/httpclient" "github.com/git-pkgs/proxy/internal/metrics" - "github.com/git-pkgs/proxy/internal/storage" "github.com/git-pkgs/purl" "github.com/git-pkgs/registries/fetch" ) @@ -44,7 +43,7 @@ func seedPackageWithPURL(t *testing.T, db *database.DB, store *mockStorage, ecos t.Fatalf("failed to upsert version: %v", err) } - storagePath := storage.ArtifactPath(ecosystem, "", name, version, filename) + storagePath := ecosystem + "/" + name + "/" + version + "/" + filename store.files[storagePath] = []byte(content) sharedArtifact := testArtifact(content, versionPURL, filename, "application/octet-stream") @@ -1373,8 +1372,12 @@ func TestHandlers_UpstreamHeldMetadataReturns503(t *testing.T) { path string }{ {"npm metadata", func(p *Proxy) http.Handler { return NewNPMHandler(p, "http://localhost", "").Routes() }, "/lodash"}, - {"pypi simple", func(p *Proxy) http.Handler { return NewPyPIHandler(p, "http://localhost").Routes() }, "/simple/requests/"}, - {"pypi json", func(p *Proxy) http.Handler { return NewPyPIHandler(p, "http://localhost").Routes() }, "/pypi/requests/json"}, + {"pypi simple", func(p *Proxy) http.Handler { + return NewPyPIHandlerWithUpstreams(p, "http://localhost", "", "").Routes() + }, "/simple/requests/"}, + {"pypi json", func(p *Proxy) http.Handler { + return NewPyPIHandlerWithUpstreams(p, "http://localhost", "", "").Routes() + }, "/pypi/requests/json"}, {"cargo index", func(p *Proxy) http.Handler { return NewCargoHandler(p, "http://localhost", "", "").Routes() }, "/se/rd/serde"}, diff --git a/internal/handler/handler_bench_test.go b/internal/handler/handler_bench_test.go index cdbb524..9dbdb74 100644 --- a/internal/handler/handler_bench_test.go +++ b/internal/handler/handler_bench_test.go @@ -186,7 +186,7 @@ func newBenchmarkNPMServer(b *testing.B) *benchmarkNPMServer { b.Cleanup(func() { _ = store.Close() }) content := bytes.Repeat([]byte("x"), benchmarkArtifactSize) - storagePath := storage.ArtifactPath("npm", "", "lodash", "4.17.21", "lodash-4.17.21.tgz") + storagePath := "npm/lodash/4.17.21/lodash-4.17.21.tgz" size, hash, err := store.Store(ctx, storagePath, bytes.NewReader(content)) if err != nil { b.Fatalf("store artifact: %v", err) diff --git a/internal/handler/handler_test.go b/internal/handler/handler_test.go index 1096cdf..3cc3c39 100644 --- a/internal/handler/handler_test.go +++ b/internal/handler/handler_test.go @@ -240,7 +240,7 @@ func seedPackage(t testing.TB, db *database.DB, store *mockStorage, ecosystem, n t.Fatalf("failed to upsert version: %v", err) } - storagePath := storage.ArtifactPath(ecosystem, "", name, version, filename) + storagePath := ecosystem + "/" + name + "/" + version + "/" + filename store.files[storagePath] = []byte(content) sharedArtifact := testArtifact(content, versionPURL, filename, "application/octet-stream") @@ -417,7 +417,7 @@ func assertMalformedCacheRejected(t *testing.T, malformedHash, malformedIntegrit if artifact.StoragePath.Valid { t.Error("unusable cache record retained its storage path") } - assertQueuedForDeletion(t, db, storage.ArtifactPath("npm", "", packageName, version, filename)) + assertQueuedForDeletion(t, db, "npm/"+packageName+"/"+version+"/"+filename) } // TestCorruptCachedArtifactIsQueuedForDeletion streams cached bytes that no @@ -427,7 +427,7 @@ func assertMalformedCacheRejected(t *testing.T, malformedHash, malformedIntegrit func TestCorruptCachedArtifactIsQueuedForDeletion(t *testing.T) { proxy, db, store, _ := setupTestProxy(t) seedPackage(t, db, store, "npm", "corrupt", "1.0.0", "corrupt-1.0.0.tgz", "cached content") - storagePath := storage.ArtifactPath("npm", "", "corrupt", "1.0.0", "corrupt-1.0.0.tgz") + storagePath := "npm/corrupt/1.0.0/corrupt-1.0.0.tgz" store.files[storagePath] = []byte("tampered bytes") result, err := proxy.GetCachedArtifact(context.Background(), "npm", "corrupt", "1.0.0", "corrupt-1.0.0.tgz") @@ -992,17 +992,7 @@ func TestGetOrFetchArtifactFromURL_CacheMiss(t *testing.T) { } } -// isolateRecentFailures empties the process-wide failure history for the -// test and again when it ends, so neither earlier tests nor repeated runs -// reach its assertions. -func isolateRecentFailures(t *testing.T) { - t.Helper() - metrics.ResetRecentFailures() - t.Cleanup(metrics.ResetRecentFailures) -} - func TestGetOrFetchArtifactFromURL_FetchError(t *testing.T) { - isolateRecentFailures(t) proxy, _, _, fetcher := setupTestProxy(t) fetcher.fetchErr = errors.New("connection refused") errorsBefore := testutil.ToFloat64(metrics.UpstreamErrors.WithLabelValues("pypi", "fetch_failed")) @@ -1018,14 +1008,13 @@ func TestGetOrFetchArtifactFromURL_FetchError(t *testing.T) { t.Errorf("upstream errors delta = %.0f, want 1", diff) } recent := metrics.RecentFailures("proxy_upstream_errors_total", map[string]string{"ecosystem": "pypi", "error_type": "fetch_failed"}) - if len(recent) != 1 || recent[0].PURL != "pkg:pypi/fail@1.0.0" || + if len(recent) == 0 || recent[0].PURL != "pkg:pypi/fail@1.0.0" || recent[0].URL != "https://pypi.org/files/fail-1.0.0.tar.gz" || recent[0].Error != "connection refused" { - t.Errorf("upstream failures = %+v, want one: the failed package, its URL and the error", recent) + t.Errorf("upstream failures = %+v, want the newest failure to contain the failed package, its URL and the error", recent) } } func TestGetOrFetchArtifactFromURL_StoreError(t *testing.T) { - isolateRecentFailures(t) proxy, _, store, fetcher := setupTestProxy(t) store.storeErr = errors.New("disk full") errorsBefore := testutil.ToFloat64(metrics.StorageErrors.WithLabelValues("write")) @@ -1045,9 +1034,9 @@ func TestGetOrFetchArtifactFromURL_StoreError(t *testing.T) { t.Errorf("storage errors delta = %.0f, want 1", diff) } recent := metrics.RecentFailures("proxy_storage_errors_total", map[string]string{"operation": "write"}) - if len(recent) != 1 || recent[0].PURL != "pkg:pypi/fail@1.0.0" || recent[0].Filename != "fail-1.0.0.tar.gz" || + if len(recent) == 0 || recent[0].PURL != "pkg:pypi/fail@1.0.0" || recent[0].Filename != "fail-1.0.0.tar.gz" || recent[0].Path == "" || recent[0].Error != "disk full" { - t.Errorf("storage failures = %+v, want one: the failed package, its storage path and the error", recent) + t.Errorf("storage failures = %+v, want the newest failure to contain the failed package, its storage path and the error", recent) } } diff --git a/internal/handler/helm_test.go b/internal/handler/helm_test.go index c4a60fe..c50533d 100644 --- a/internal/handler/helm_test.go +++ b/internal/handler/helm_test.go @@ -14,7 +14,6 @@ import ( "github.com/git-pkgs/cooldown" upstreamhttp "github.com/git-pkgs/proxy/internal/httpclient" - "github.com/git-pkgs/proxy/internal/storage" "github.com/git-pkgs/registries/fetch" "gopkg.in/yaml.v3" ) @@ -169,7 +168,7 @@ func TestHelmHandler_RejectsChartDigestMismatch(t *testing.T) { if err != nil { t.Fatalf("listing queued deletes: %v", err) } - chartDir := storage.ArtifactPath(helmMetadataEcosystem, "", "test", digest, "") + chartDir := helmMetadataEcosystem + "/test/" + digest + "/" stored := 0 for path := range store.files { if !strings.HasPrefix(path, chartDir) { diff --git a/internal/handler/jsonscan_test.go b/internal/handler/jsonscan_test.go index 9066a5f..0825000 100644 --- a/internal/handler/jsonscan_test.go +++ b/internal/handler/jsonscan_test.go @@ -91,7 +91,7 @@ func TestForEachJSONMemberOffsets(t *testing.T) { func TestWriteFilteredJSONObject(t *testing.T) { var out bytes.Buffer - err := writeFilteredJSONObject(&out, []byte(`{"a": 1, "b": {"x": 2}, "c": 3}`), func(k string) bool { return k != "b" }) + err := writeFilteredJSONObjectWith(&out, []byte(`{"a": 1, "b": {"x": 2}, "c": 3}`), func(k string) bool { return k != "b" }, nil) if err != nil || out.String() != `{"a": 1,"c": 3}` { t.Errorf("filtered = %s, %v", out.String(), err) } diff --git a/internal/handler/npm.go b/internal/handler/npm.go index c92bf2a..6530f5a 100644 --- a/internal/handler/npm.go +++ b/internal/handler/npm.go @@ -662,14 +662,8 @@ func (h *NPMHandler) writeNPMVersion(out *bytes.Buffer, packageName, version str "old", oldTarball, "new", newTarball) } -// writeFilteredJSONObject writes the object obj holds, keeping the members -// whose decoded key keep accepts. -func writeFilteredJSONObject(out *bytes.Buffer, obj []byte, keep func(string) bool) error { - return writeFilteredJSONObjectWith(out, obj, keep, nil) -} - -// writeFilteredJSONObjectWith is writeFilteredJSONObject followed by extra -// string members. +// writeFilteredJSONObjectWith keeps members whose decoded keys match keep, +// then appends extra string members. func writeFilteredJSONObjectWith(out *bytes.Buffer, obj []byte, keep func(string) bool, extra [][2]string) error { out.WriteByte('{') first := true diff --git a/internal/handler/pypi.go b/internal/handler/pypi.go index 513f507..e2ca4db 100644 --- a/internal/handler/pypi.go +++ b/internal/handler/pypi.go @@ -56,11 +56,6 @@ func init() { retention.Default.Register(retention.Spec{Key: "pypi", CanonicalPackage: retention.DefaultCanonical("pypi")}) } -// NewPyPIHandler creates a new PyPI protocol handler. -func NewPyPIHandler(proxy *Proxy, proxyURL string) *PyPIHandler { - return NewPyPIHandlerWithUpstreams(proxy, proxyURL, "", "") -} - // NewPyPIHandlerWithUpstreams creates a PyPI handler with custom API and // package download upstreams. func NewPyPIHandlerWithUpstreams(proxy *Proxy, proxyURL, upstreamURL, downloadURL string) *PyPIHandler { diff --git a/internal/handler/rewrite_cache.go b/internal/handler/rewrite_cache.go index fb3114a..23a996d 100644 --- a/internal/handler/rewrite_cache.go +++ b/internal/handler/rewrite_cache.go @@ -254,16 +254,6 @@ func (c *rewriteCache) add(e *rewriteEntry) { } } -// len reports how many rewrites are cached. -func (c *rewriteCache) len() int { - if c == nil { - return 0 - } - c.mu.Lock() - defer c.mu.Unlock() - return len(c.entries) -} - // cachedRewrite rewrites a metadata document through the proxy's rewrite // cache and returns it with its validators. digest identifies in when the // metadata cache stored it, and is empty otherwise. Cooldown filtering depends diff --git a/internal/handler/rewrite_cache_test.go b/internal/handler/rewrite_cache_test.go index 8584312..5c2b842 100644 --- a/internal/handler/rewrite_cache_test.go +++ b/internal/handler/rewrite_cache_test.go @@ -261,3 +261,13 @@ func TestMetadataHandlers_ServeCachedRewrite(t *testing.T) { } } } + +// len reports how many rewrites are cached. +func (c *rewriteCache) len() int { + if c == nil { + return 0 + } + c.mu.Lock() + defer c.mu.Unlock() + return len(c.entries) +} diff --git a/internal/handler/stream_artifacts_test.go b/internal/handler/stream_artifacts_test.go index 1c15a31..3a5929b 100644 --- a/internal/handler/stream_artifacts_test.go +++ b/internal/handler/stream_artifacts_test.go @@ -303,7 +303,7 @@ func TestStreamArtifactsOCIBlobShorterThanContentLengthIsIncomplete(t *testing.T proxy, _, store, _ := setupTestProxy(t) proxy.StreamArtifacts = true useRealFetcher(t, proxy, upstream) - h := NewContainerHandler(proxy, "http://proxy.example", map[string]string{"ghcr": upstream.URL}) + h := NewContainerHandlerWithRegistry(proxy, "http://proxy.example", "", map[string]string{"ghcr": upstream.URL}) srv := httptest.NewServer(h.Routes()) defer srv.Close() diff --git a/internal/jsonstream/jsonstream.go b/internal/jsonstream/jsonstream.go index d6e8de0..2d3d120 100644 --- a/internal/jsonstream/jsonstream.go +++ b/internal/jsonstream/jsonstream.go @@ -20,7 +20,7 @@ var ( ) // Member is one member of an object being copied. Its value must be consumed -// exactly once, with Raw, Skip or Object. +// exactly once, with Raw, Copy or Object. type Member struct { Key string dec *json.Decoder @@ -63,12 +63,6 @@ func (m *Member) Copy() error { return m.Write(raw) } -// Skip reads the member's value and leaves the member out. -func (m *Member) Skip() error { - _, err := m.Raw() - return err -} - // Object writes the member and copies its value, which must be an object, // calling fn for each of its members. A null value is written as null. func (m *Member) Object(fn func(*Member) error) error { diff --git a/internal/jsonstream/jsonstream_test.go b/internal/jsonstream/jsonstream_test.go index 1fc8ea8..44a877e 100644 --- a/internal/jsonstream/jsonstream_test.go +++ b/internal/jsonstream/jsonstream_test.go @@ -58,7 +58,8 @@ func TestCopyObjectSkipReplaceAndEmpty(t *testing.T) { got, err := copyString(t, `{"first":1,"secret":"x","n":2,"empty":{}}`, func(m *Member) error { switch m.Key { case "first", "secret": - return m.Skip() + _, err := m.Raw() + return err case "n": if _, err := m.Raw(); err != nil { return err @@ -93,13 +94,14 @@ func TestMemberMisuse(t *testing.T) { t.Errorf("Write before Raw: err = %v", err) } _, err = copyString(t, `{"a":1}`, func(m *Member) error { - if err := m.Skip(); err != nil { + if _, err := m.Raw(); err != nil { return err } - return m.Skip() + _, err := m.Raw() + return err }) if !errors.Is(err, errConsumed) { - t.Errorf("second Skip: err = %v", err) + t.Errorf("second Raw: err = %v", err) } } diff --git a/internal/metrics/failures.go b/internal/metrics/failures.go index 425ba8e..a5f276b 100644 --- a/internal/metrics/failures.go +++ b/internal/metrics/failures.go @@ -82,15 +82,6 @@ func recordFailure(name string, labels map[string]string, detail FailureDetail) recentFailures.series[key] = stored } -// ResetRecentFailures empties the recent failure history. The history is -// process-wide, so tests that record or read failures call it to start from a -// known state and to leave none behind. -func ResetRecentFailures() { - recentFailures.Lock() - defer recentFailures.Unlock() - recentFailures.series = make(map[string][]Failure) -} - func failureKey(name string, labels map[string]string) string { return name + "{" + labelKey(labels) + "}" } diff --git a/internal/metrics/failures_test.go b/internal/metrics/failures_test.go index 3d6ab88..2301f8b 100644 --- a/internal/metrics/failures_test.go +++ b/internal/metrics/failures_test.go @@ -14,8 +14,13 @@ import ( // reach its assertions. func isolateRecentFailures(t *testing.T) { t.Helper() - ResetRecentFailures() - t.Cleanup(ResetRecentFailures) + reset := func() { + recentFailures.Lock() + defer recentFailures.Unlock() + recentFailures.series = make(map[string][]Failure) + } + reset() + t.Cleanup(reset) } func TestRecentFailuresNewestFirst(t *testing.T) { diff --git a/internal/metrics/snapshot.go b/internal/metrics/snapshot.go index 6ac66fb..2c96dff 100644 --- a/internal/metrics/snapshot.go +++ b/internal/metrics/snapshot.go @@ -94,20 +94,6 @@ func sampleOf(m *dto.Metric) Sample { return s } -// Names returns the name of every metric family in the snapshot, sorted. -func (s *Snapshot) Names() []string { - if s == nil { - return nil - } - - names := make([]string, 0, len(s.families)) - for name := range s.families { - names = append(names, name) - } - sort.Strings(names) - return names -} - // Samples returns every series of a metric, or nil when it has never been // observed. Series are ordered by their label values so that rendering is // stable between scrapes. diff --git a/internal/server/analytics_coverage_test.go b/internal/server/analytics_coverage_test.go index 12cc895..f5f711e 100644 --- a/internal/server/analytics_coverage_test.go +++ b/internal/server/analytics_coverage_test.go @@ -5,6 +5,7 @@ import ( "testing" "github.com/git-pkgs/proxy/internal/metrics" + "github.com/prometheus/client_golang/prometheus" ) // metricSurface records where each registered metric is shown on /ui/analytics. @@ -54,7 +55,6 @@ var metricSurface = map[string]string{ // on the analytics page. The page is meant to be a complete view of what // /metrics exposes, so a silently unsurfaced metric is a gap, not a detail. func TestEveryMetricIsSurfaced(t *testing.T) { - isolateRecentFailures(t) // Touch every metric family so it is present in the registry output, since // a vector with no observed label values gathers as nothing at all. metrics.RecordRequest("npm", 200, 0) @@ -75,13 +75,14 @@ func TestEveryMetricIsSurfaced(t *testing.T) { metrics.RecordArtifactEvicted("retention", "npm") metrics.UpdateEcosystemStats([]metrics.EcosystemStats{{Ecosystem: "npm", DownloadedBytes: 1}}) - snap, err := metrics.Gather() + families, err := prometheus.DefaultGatherer.Gather() if err != nil { t.Fatalf("Gather: %v", err) } var registered []string - for _, name := range snap.Names() { + for _, family := range families { + name := family.GetName() if strings.HasPrefix(name, "proxy_") { registered = append(registered, name) } diff --git a/internal/server/analytics_failures_test.go b/internal/server/analytics_failures_test.go index 6f7856e..aa89cb5 100644 --- a/internal/server/analytics_failures_test.go +++ b/internal/server/analytics_failures_test.go @@ -9,15 +9,6 @@ import ( "github.com/prometheus/client_golang/prometheus" ) -// isolateRecentFailures empties the process-wide failure history for the -// test and again when it ends, so neither earlier tests nor repeated runs -// reach its assertions. -func isolateRecentFailures(t *testing.T) { - t.Helper() - metrics.ResetRecentFailures() - t.Cleanup(metrics.ResetRecentFailures) -} - func storageErrorSnapshot(t *testing.T, operation string) *metrics.Snapshot { t.Helper() reg := prometheus.NewRegistry() @@ -34,7 +25,6 @@ func storageErrorSnapshot(t *testing.T, operation string) *metrics.Snapshot { } func TestRuntimeViewAttachesRecentFailuresWhenEnabled(t *testing.T) { - isolateRecentFailures(t) const operation = "test-attach" metrics.RecordStorageError(operation, metrics.FailureDetail{ PURL: "pkg:npm/left-pad@1.0.0", Filename: "left-pad-1.0.0.tgz", Path: "npm/left-pad", Err: errors.New("disk full")}) @@ -44,8 +34,8 @@ func TestRuntimeViewAttachesRecentFailuresWhenEnabled(t *testing.T) { if !v.FailureDetailsOn { t.Error("FailureDetailsOn = false, want true") } - if len(v.StorageErrors) != 1 || len(v.StorageErrors[0].Recent) != 1 { - t.Fatalf("StorageErrors = %+v, want one row with one recent failure", v.StorageErrors) + if len(v.StorageErrors) != 1 || len(v.StorageErrors[0].Recent) == 0 { + t.Fatalf("StorageErrors = %+v, want one row with recent failures", v.StorageErrors) } got := v.StorageErrors[0].Recent[0] if got.PURL != "pkg:npm/left-pad@1.0.0" || got.Filename != "left-pad-1.0.0.tgz" || @@ -55,7 +45,6 @@ func TestRuntimeViewAttachesRecentFailuresWhenEnabled(t *testing.T) { } func TestRuntimeViewOmitsRecentFailuresWhenDisabled(t *testing.T) { - isolateRecentFailures(t) const operation = "test-omit" metrics.RecordStorageError(operation, metrics.FailureDetail{PURL: "pkg:npm/left-pad@1.0.0"}) @@ -72,7 +61,6 @@ func TestRuntimeViewOmitsRecentFailuresWhenDisabled(t *testing.T) { // Package names and upstream URLs say what the proxy's users depend on, so like // the caller table they are published only when asked for. func TestAnalyticsPageHidesFailureDetailsByDefault(t *testing.T) { - isolateRecentFailures(t) ts := newTestServer(t) defer ts.close() @@ -91,7 +79,6 @@ func TestAnalyticsPageHidesFailureDetailsByDefault(t *testing.T) { } func TestAnalyticsPageShowsFailureDetailsWhenEnabled(t *testing.T) { - isolateRecentFailures(t) ts := newTestServer(t) defer ts.close() diff --git a/internal/server/eviction_test.go b/internal/server/eviction_test.go index 1535586..2e61e71 100644 --- a/internal/server/eviction_test.go +++ b/internal/server/eviction_test.go @@ -70,7 +70,7 @@ func seedArtifact(t *testing.T, ctx context.Context, db *database.DB, store stor t.Fatalf("failed to upsert version: %v", err) } - storagePath := storage.ArtifactPath("npm", "", name, "1.0.0", filename) + storagePath := "npm/" + name + "/1.0.0/" + filename data := strings.NewReader(strings.Repeat("x", dataSize)) size, hash, err := store.Store(ctx, storagePath, data) if err != nil { @@ -150,7 +150,7 @@ func TestEvictLRU_EvictsOldestFirst(t *testing.T) { } // Storage file should be removed for old-pkg - storagePath := storage.ArtifactPath("npm", "", "old-pkg", "1.0.0", "old-pkg-1.0.0.tgz") + storagePath := "npm/old-pkg/1.0.0/old-pkg-1.0.0.tgz" exists, _ := store.Exists(ctx, storagePath) if exists { t.Error("expected old-pkg file to be deleted from storage") @@ -215,7 +215,7 @@ func TestEvictLRU_StorageFileDeleted(t *testing.T) { seedArtifact(t, ctx, db, store, "delete-me", 1000, time.Now().Add(-1*time.Hour)) - storagePath := storage.ArtifactPath("npm", "", "delete-me", "1.0.0", "delete-me-1.0.0.tgz") + storagePath := "npm/delete-me/1.0.0/delete-me-1.0.0.tgz" exists, _ := store.Exists(ctx, storagePath) if !exists { t.Fatal("expected artifact file to exist before eviction") @@ -352,8 +352,8 @@ func TestEvictLRU_QueuesObjectsStorageRefusesToDelete(t *testing.T) { t.Errorf("cached artifacts = %d, want 0", count) } want := []string{ - storage.ArtifactPath("npm", "", "new-pkg", "1.0.0", "new-pkg-1.0.0.tgz"), - storage.ArtifactPath("npm", "", "old-pkg", "1.0.0", "old-pkg-1.0.0.tgz"), + "npm/new-pkg/1.0.0/new-pkg-1.0.0.tgz", + "npm/old-pkg/1.0.0/old-pkg-1.0.0.tgz", } got := queuedPaths(t, db) slices.Sort(got) @@ -387,7 +387,7 @@ func TestEvictLRU_EndsPassWhenNothingCanBeCleared(t *testing.T) { t.Errorf("cached artifacts = %d, want 2", count) } for _, name := range []string{"old-pkg", "new-pkg"} { - if ok, _ := store.Exists(ctx, storage.ArtifactPath("npm", "", name, "1.0.0", name+"-1.0.0.tgz")); !ok { + if ok, _ := store.Exists(ctx, "npm/"+name+"/1.0.0/"+name+"-1.0.0.tgz"); !ok { t.Errorf("%s deleted although its record was not cleared", name) } } diff --git a/internal/server/retention_test.go b/internal/server/retention_test.go index 4ffd399..22fca0a 100644 --- a/internal/server/retention_test.go +++ b/internal/server/retention_test.go @@ -42,7 +42,7 @@ func seedAgedArtifact(t *testing.T, db *database.DB, store storage.Storage, dbEc if err := db.UpsertVersion(&database.Version{PURL: versionPURL, PackagePURL: pkgPURL}); err != nil { t.Fatalf("UpsertVersion: %v", err) } - path := storage.ArtifactPath(dbEcosystem, "", name, "1.0.0", filename) + path := dbEcosystem + "/" + name + "/1.0.0/" + filename size, hash, err := store.Store(context.Background(), path, strings.NewReader("artifact")) if err != nil { t.Fatalf("Store: %v", err) diff --git a/internal/server/server_test.go b/internal/server/server_test.go index 5109d23..611527d 100644 --- a/internal/server/server_test.go +++ b/internal/server/server_test.go @@ -87,7 +87,7 @@ func newTestServer(t *testing.T) *testServer { ) gemHandler := handler.NewGemHandler(proxy, cfg.BaseURL) goHandler := handler.NewGoHandler(proxy, cfg.BaseURL) - pypiHandler := handler.NewPyPIHandler(proxy, cfg.BaseURL) + pypiHandler := handler.NewPyPIHandlerWithUpstreams(proxy, cfg.BaseURL, "", "") gradleHandler := handler.NewGradleBuildCacheHandler(proxy) swiftHandler := handler.NewSwiftHandler(proxy, cfg.BaseURL, cfg.Upstream.Swift) diff --git a/internal/storage/blob_test.go b/internal/storage/blob_test.go index 5533174..37b9ac6 100644 --- a/internal/storage/blob_test.go +++ b/internal/storage/blob_test.go @@ -495,7 +495,7 @@ func seedLegacySidecar(t *testing.T, dir, key, payload string) string { // An interrupted setAttrs leaves a partial sidecar that fails every read of the // key, and nothing rewrites one now, so a store has to clear it. // -// One key per storage path the proxy builds: ArtifactPath across ecosystems, +// One key per storage path the proxy builds: legacy artifact keys across ecosystems, // metadata blobs, and the Gradle build cache. Scoped npm names, Go's "!" case // escaping and the ":" in OCI digests and Debian epochs are the characters // most likely to part fileblob's mapping from a plain path join. diff --git a/internal/storage/storage.go b/internal/storage/storage.go index 3b27ef4..dfce286 100644 --- a/internal/storage/storage.go +++ b/internal/storage/storage.go @@ -77,18 +77,6 @@ type Storage interface { Close() error } -// ArtifactPath builds the storage path artifacts were cached under before each -// fetch got its own; records from then still point at such paths. Only tests -// call it, to build such records. -// Format: {ecosystem}/{namespace}/{name}/{version}/{filename} -// For packages without namespace: {ecosystem}/{name}/{version}/{filename} -func ArtifactPath(ecosystem, namespace, name, version, filename string) string { - if namespace != "" { - return ecosystem + "/" + namespace + "/" + name + "/" + version + "/" + filename - } - return ecosystem + "/" + name + "/" + version + "/" + filename -} - // FetchPath builds the storage path for one fetch of an artifact: // {ecosystem}/{name}/{version}/{fetchID}/{filename}. Each fetch writes its own // object, so no fetch overwrites or deletes another's. diff --git a/internal/storage/storage_test.go b/internal/storage/storage_test.go index 67745d0..5d4f0b4 100644 --- a/internal/storage/storage_test.go +++ b/internal/storage/storage_test.go @@ -22,31 +22,6 @@ func TestIsFetchDir(t *testing.T) { } } -func TestArtifactPath(t *testing.T) { - tests := []struct { - ecosystem string - namespace string - name string - version string - filename string - want string - }{ - {"npm", "", "lodash", "4.17.21", "lodash-4.17.21.tgz", "npm/lodash/4.17.21/lodash-4.17.21.tgz"}, - {"npm", "babel", "core", "7.0.0", "core-7.0.0.tgz", "npm/babel/core/7.0.0/core-7.0.0.tgz"}, - {"cargo", "", "serde", "1.0.0", "serde-1.0.0.crate", "cargo/serde/1.0.0/serde-1.0.0.crate"}, - {"pypi", "", "requests", "2.28.0", "requests-2.28.0.tar.gz", "pypi/requests/2.28.0/requests-2.28.0.tar.gz"}, - {"maven", "org.apache", "commons-lang3", "3.12.0", "commons-lang3-3.12.0.jar", "maven/org.apache/commons-lang3/3.12.0/commons-lang3-3.12.0.jar"}, - } - - for _, tt := range tests { - got := ArtifactPath(tt.ecosystem, tt.namespace, tt.name, tt.version, tt.filename) - if got != tt.want { - t.Errorf("ArtifactPath(%q, %q, %q, %q, %q) = %q, want %q", - tt.ecosystem, tt.namespace, tt.name, tt.version, tt.filename, got, tt.want) - } - } -} - // assertLargeFileRoundTrip stores a 1MB file in the given storage, verifies size and // hash, then reads it back and confirms the content matches. func assertLargeFileRoundTrip(t *testing.T, s Storage) { From d643836649c9cbaec105807641943ab128d488db Mon Sep 17 00:00:00 2001 From: profiling <5w1h@outlook.com> Date: Mon, 19 Oct 2026 15:43:57 +0800 Subject: [PATCH 318/633] Add opt-in retention for generic artifacts (#513) * Add opt-in retention for generic artifacts * Remove constant parameter from generic retention test helper --------- Co-authored-by: ztzhou --- README.md | 2 +- config.example.yaml | 4 + docs/configuration.md | 15 ++- internal/config/config.go | 2 +- internal/config/retention_test.go | 13 +- internal/handler/generic.go | 5 + internal/handler/generic_test.go | 27 +++- internal/retention/retention.go | 11 +- internal/retention/retention_test.go | 33 ++++- internal/server/retention.go | 4 +- internal/server/retention_generic_test.go | 156 ++++++++++++++++++++++ internal/server/retention_test.go | 12 +- 12 files changed, 269 insertions(+), 15 deletions(-) create mode 100644 internal/server/retention_generic_test.go diff --git a/README.md b/README.md index 8611a37..d1a67e0 100644 --- a/README.md +++ b/README.md @@ -875,7 +875,7 @@ storage: url: "file:///var/cache/proxy/artifacts" max_size: "10GB" # Optional: evict LRU when exceeded retention: - default: "30d" # Optional: evict artifacts not downloaded for 30 days + default: "30d" # Optional: evict unused artifacts after 30 days; excludes generic database: driver: "sqlite" diff --git a/config.example.yaml b/config.example.yaml index 504e6f4..253ae7f 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -91,12 +91,16 @@ storage: # The default only applies to ecosystems that support retention; naming an # ecosystem or package that does not is a configuration error. See # docs/configuration.md for the supported ecosystems. + # Generic downloads require an explicit ecosystem or package rule; + # they never inherit default, since an upstream release may disappear. retention: default: "" # ecosystems: # npm: "30d" + # generic: "30d" # packages: # "pkg:npm/lodash": "0" + # "pkg:generic/owner%2Frepo": "90d" # How often to look for expired artifacts; at least "1m". sweep_interval: "10m" diff --git a/docs/configuration.md b/docs/configuration.md index 33eb2ed..987552d 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -85,7 +85,19 @@ storage: Each cached file ages on its own, counted from the last time the proxy served it, or from when it was fetched if it has not been served since. A newer release of the same package does not shorten the life of older versions. Durations take a `d` suffix for days, and `"0"` or an empty value means never evict by age. With no retention configured, artifacts stay until `max_size` evicts them. `sweep_interval` must be at least `1m`. -A package rule wins over an ecosystem rule, which wins over `default`. Package rules are keyed by package PURL without a version, written in one of the forms the table below lists for the ecosystem. The ecosystems and package rules can only name ecosystems that support retention in the running version; anything else, including a typo, stops the proxy at startup with an error naming the setting. `default` applies only to supported ecosystems. Support is added one ecosystem at a time, so a later release can make an existing `default` cover more ecosystems. On startup the proxy logs which ecosystems retention covers (`covered`) and which ones `default` does not reach yet (`default_not_applied`). +A package rule wins over an ecosystem rule, which wins over `default`. Package rules are keyed by package PURL without a version, written in one of the forms the table below lists for the ecosystem. The ecosystems and package rules can only name ecosystems that support retention in the running version; anything else, including a typo, stops the proxy at startup with an error naming the setting. `default` applies only to supported ecosystems, except `generic`, which requires an explicit rule. Support is added one ecosystem at a time, so a later release can make an existing `default` cover more ecosystems. On startup the proxy logs which ecosystems retention supports (`covered`) and which ones `default` does not reach (`default_not_applied`), including `generic`. + +Generic retention is opt-in because release assets may disappear upstream and cannot then be downloaded again after eviction. An existing `default` never starts evicting generic downloads. Set `ecosystems.generic: "30d"` to cover all generic artifacts, or configure only selected GitHub repositories: + +```yaml +storage: + retention: + default: "30d" # Other supported ecosystems; never generic. + packages: + "pkg:generic/jqlang%2Fjq": "30d" +``` + +The package rule covers every release and asset in that repository, but each cached file ages separately. Other generic repositories remain exempt from age-based eviction. A package rule of `"0"` also exempts that repository when `ecosystems.generic` is enabled. The independent `storage.max_size` limit still applies. | Ecosystem key | Retention supported | Package rule form | |---------------|---------------------|-------------------| @@ -97,6 +109,7 @@ A package rule wins over an ecosystem rule, which wins over `default`. Package r | `cran` | yes | `pkg:cran/ggplot2`, the package name as published, case included (`pkg:cran/Rcpp`). A rule covers source and binary builds alike. | | `deb` | yes | `pkg:deb/curl`; case doesn't matter. A vendor namespace such as `pkg:deb/debian/curl` is accepted and dropped, so a rule covers the package from every configured archive. Debian and Ubuntu archives only keep the current version per suite, so an evicted older one can't be fetched again. | | `gem` | yes | `pkg:gem/rails`, the gem name as published, case included. | +| `generic` | yes, explicit rules only | `pkg:generic/owner%2Frepo` for a GitHub Release repository such as `jqlang/jq`. The ecosystem rule covers all cached version-pinned generic downloads, including GitHub Release assets and configured `generic_artifacts`. The global `default` never applies; metadata/API responses are not artifact retention candidates. | | `golang` | yes | `pkg:golang/github.com/stretchr/testify`, the module path as in `go.mod`; case doesn't matter. A major version like `/v2` (or `.v3` on gopkg.in) is a module of its own and needs its own rule. The `!`-escaped form from proxy URLs and the module cache (`github.com/!azure/...`) is refused. | | `helm` | yes | No package rules: charts are cached per repository and digest, not under the chart name. The `helm` rule and `default` cover charts from HTTP repositories; charts pulled over OCI fall under `oci`. | | `hex` | yes | `pkg:hex/phoenix`, the package name as published. | diff --git a/internal/config/config.go b/internal/config/config.go index 89a5c20..2f78c3a 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -443,7 +443,7 @@ type StorageConfig struct { // for days; "0" or empty means never evict by age. type RetentionConfig struct { // Default applies to every ecosystem that supports retention and has no - // rule of its own. + // rule of its own, except generic, which requires an explicit rule. Default string `json:"default" yaml:"default"` // Ecosystems overrides the default per ecosystem key (e.g. "npm", "oci"). diff --git a/internal/config/retention_test.go b/internal/config/retention_test.go index e416301..eb614e8 100644 --- a/internal/config/retention_test.go +++ b/internal/config/retention_test.go @@ -12,6 +12,7 @@ const retentionDay = 24 * time.Hour func retentionTestRegistry() *retention.Registry { reg := retention.NewRegistry() + reg.Register(retention.Spec{Key: "generic", CanonicalPackage: retention.DefaultCanonical("generic")}) reg.Register(retention.Spec{Key: "npm", CanonicalPackage: retention.DefaultCanonical("npm")}) reg.Register(retention.Spec{Key: "julia"}) reg.Register(retention.Spec{Key: "helm"}) @@ -34,8 +35,9 @@ func TestRetentionRulesResolves(t *testing.T) { Default: "30d", Ecosystems: map[string]string{"npm": "14d"}, Packages: map[string]string{ - "pkg:npm/lodash": "0", - "pkg:npm/@babel/core": "90d", + "pkg:npm/lodash": "0", + "pkg:npm/@babel/core": "90d", + "pkg:generic/jqlang%2Fjq": "7d", }, SweepInterval: "5m", } @@ -56,6 +58,12 @@ func TestRetentionRulesResolves(t *testing.T) { if got := rules.For("npm", "pkg:npm/%40babel/core"); got != 90*retentionDay { t.Errorf("babel rule = %v, want 90d under its canonical key", got) } + if got := rules.For("generic", "pkg:generic/jqlang%2Fjq"); got != 7*retentionDay { + t.Errorf("generic package rule = %v, want 7d", got) + } + if got := rules.For("generic", "pkg:generic/other%2Frepo"); got != 0 { + t.Errorf("unconfigured generic package rule = %v, want 0 despite the global default", got) + } if got := cfg.ParseRetentionSweepInterval(); got != 5*time.Minute { t.Errorf("ParseRetentionSweepInterval() = %v, want 5m", got) } @@ -76,7 +84,6 @@ func TestRetentionRulesErrors(t *testing.T) { {"sweep interval below a minute", func(r *RetentionConfig) { r.SweepInterval = "30s" }, "must be at least 1m0s"}, {"bad sweep interval", func(r *RetentionConfig) { r.SweepInterval = "often" }, "sweep_interval"}, {"unknown ecosystem", func(r *RetentionConfig) { r.Ecosystems = map[string]string{"nmp": "1d"} }, "unknown ecosystem"}, - {"generic is not an ecosystem key", func(r *RetentionConfig) { r.Ecosystems = map[string]string{"generic": "1d"} }, "unknown ecosystem"}, {"ecosystem not registered", func(r *RetentionConfig) { r.Ecosystems = map[string]string{"oci": "14d"} }, `storage.retention.ecosystems.oci: retention is not supported for "oci"`}, {"bad ecosystem duration", func(r *RetentionConfig) { r.Ecosystems = map[string]string{"npm": "soon"} }, "storage.retention.ecosystems.npm"}, diff --git a/internal/handler/generic.go b/internal/handler/generic.go index bf7f088..33a6ed2 100644 --- a/internal/handler/generic.go +++ b/internal/handler/generic.go @@ -10,6 +10,7 @@ import ( "strings" "github.com/git-pkgs/proxy/internal/config" + "github.com/git-pkgs/proxy/internal/retention" ) const ( @@ -28,6 +29,10 @@ const ( // and served without revalidation once fetched. var githubReleaseAssetPattern = regexp.MustCompile(`^([^/]+)/([^/]+)/releases/download/([^/]+)/([^/]+)$`) +func init() { + retention.Default.Register(retention.Spec{Key: genericEcosystem, CanonicalPackage: retention.DefaultCanonical(genericEcosystem)}) +} + // builtinGenericArtifacts maps well-known download hosts to patterns for // their version-pinned paths. They apply to any generic upstream configured // with that base URL, after the upstream's own generic_artifacts. diff --git a/internal/handler/generic_test.go b/internal/handler/generic_test.go index 2165b1a..9fac379 100644 --- a/internal/handler/generic_test.go +++ b/internal/handler/generic_test.go @@ -12,11 +12,29 @@ import ( "github.com/git-pkgs/proxy/internal/config" upstreamhttp "github.com/git-pkgs/proxy/internal/httpclient" + "github.com/git-pkgs/proxy/internal/retention" "github.com/git-pkgs/registries/fetch" ) const testReleaseAssetPath = "/jqlang/jq/releases/download/jq-1.7.1/jq-linux-amd64" +func TestGenericRetentionConfig(t *testing.T) { + cfg := config.Default() + cfg.Storage.Retention.Ecosystems = map[string]string{"generic": "30d"} + cfg.Storage.Retention.Packages = map[string]string{"pkg:generic/jqlang%2Fjq": "90d"} + + rules, err := cfg.RetentionRules(retention.Default) + if err != nil { + t.Fatalf("RetentionRules() error = %v", err) + } + if got := rules.For(genericEcosystem, "pkg:generic/jqlang%2Fjq"); got != 90*24*time.Hour { + t.Errorf("jqlang/jq retention = %v, want 90d", got) + } + if got := rules.For(genericEcosystem, "pkg:generic/other%2Frepo"); got != 30*24*time.Hour { + t.Errorf("generic ecosystem retention = %v, want 30d", got) + } +} + func TestParseGitHubReleaseAsset(t *testing.T) { tests := []struct { path string @@ -328,7 +346,7 @@ func TestGenericHandler_ReleaseAssetIsCachedAndServedWhenUpstreamDown(t *testing })) defer upstream.Close() - proxy, _, store, _ := setupTestProxy(t) + proxy, db, store, _ := setupTestProxy(t) store.seekable = true fetcher := fetch.NewFetcher(fetch.WithHTTPClient(upstream.Client()), fetch.WithMaxRetries(0)) proxy.Fetcher = fetcher @@ -346,6 +364,13 @@ func TestGenericHandler_ReleaseAssetIsCachedAndServedWhenUpstreamDown(t *testing if got := upstreamRequests.Load(); got != 1 { t.Fatalf("upstream requests = %d, want 1", got) } + pkg, err := db.GetPackageByEcosystemName(genericEcosystem, "jqlang/jq") + if err != nil || pkg == nil { + t.Fatalf("release package = %v, %v; want a cached package", pkg, err) + } + if pkg.PURL != "pkg:generic/jqlang%2Fjq" { + t.Errorf("release package PURL = %q, want pkg:generic/jqlang%%2Fjq", pkg.PURL) + } // Second request must be served from cache, even with the upstream down. available.Store(false) diff --git a/internal/retention/retention.go b/internal/retention/retention.go index c26ef4a..465b220 100644 --- a/internal/retention/retention.go +++ b/internal/retention/retention.go @@ -23,7 +23,7 @@ import ( // yet is not supported by this version. var KnownKeys = []string{ "alpine", "cargo", "composer", "conan", "conda", "cran", "deb", "gem", - "golang", "helm", "hex", "julia", "maven", "npm", "nuget", "oci", "pub", + "generic", "golang", "helm", "hex", "julia", "maven", "npm", "nuget", "oci", "pub", "pypi", "rpm", "swift", } @@ -168,7 +168,8 @@ func (r *Registry) Canonical(p *purl.PURL) (string, bool) { // Rules are resolved retention durations. A duration of zero means an // artifact is never evicted by age. type Rules struct { - // Default applies to registered ecosystems without their own rule. + // Default applies to registered ecosystems without their own rule, + // except generic, which requires an explicit rule. Default time.Duration // Ecosystems holds rules keyed by registered ecosystem key. Ecosystems map[string]time.Duration @@ -178,7 +179,7 @@ type Rules struct { // For returns the retention for an artifact of the given ecosystem key and // stored package PURL: the package rule, else the ecosystem rule, else the -// default. +// default. Generic artifacts require an explicit package or ecosystem rule. func (r Rules) For(key, packagePURL string) time.Duration { if d, ok := r.Packages[packagePURL]; ok { return d @@ -186,6 +187,10 @@ func (r Rules) For(key, packagePURL string) time.Duration { if d, ok := r.Ecosystems[key]; ok { return d } + // Release assets may disappear upstream, so keep generic opt-in. + if key == "generic" { + return 0 + } return r.Default } diff --git a/internal/retention/retention_test.go b/internal/retention/retention_test.go index 9fa67b7..171fd70 100644 --- a/internal/retention/retention_test.go +++ b/internal/retention/retention_test.go @@ -56,7 +56,7 @@ func TestRegistryRegisterPanics(t *testing.T) { name string specs []Spec }{ - {"unknown key", []Spec{{Key: "generic"}}}, + {"unknown key", []Spec{{Key: "not-an-ecosystem"}}}, {"duplicate key", []Spec{{Key: "npm"}, {Key: "npm"}}}, {"overlapping ecosystem value", []Spec{{Key: "gem"}, {Key: "npm", ExtraDBEcosystems: []string{"rubygems"}}}}, } @@ -148,6 +148,37 @@ func TestRulesFor(t *testing.T) { } } +func TestGenericRulesRequireExplicitConfiguration(t *testing.T) { + day := 24 * time.Hour + const pkg = "pkg:generic/jqlang%2Fjq" + tests := []struct { + name string + ecosystems map[string]time.Duration + packages map[string]time.Duration + want time.Duration + }{ + {"global default ignored", nil, nil, 0}, + {"ecosystem rule", map[string]time.Duration{"generic": 7 * day}, nil, 7 * day}, + {"ecosystem disabled", map[string]time.Duration{"generic": 0}, nil, 0}, + {"package only", nil, map[string]time.Duration{pkg: 14 * day}, 14 * day}, + {"other package ignored", nil, map[string]time.Duration{"pkg:generic/other%2Frepo": day}, 0}, + {"package overrides ecosystem", map[string]time.Duration{"generic": 7 * day}, map[string]time.Duration{pkg: 14 * day}, 14 * day}, + {"package disables eviction", map[string]time.Duration{"generic": 7 * day}, map[string]time.Duration{pkg: 0}, 0}, + {"package enables eviction", map[string]time.Duration{"generic": 0}, map[string]time.Duration{pkg: day}, day}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + rules := Rules{Default: 30 * day, Ecosystems: tt.ecosystems, Packages: tt.packages} + if got := rules.For("generic", pkg); got != tt.want { + t.Errorf("generic retention = %v, want %v", got, tt.want) + } + if got := rules.For("npm", "pkg:npm/lodash"); got != 30*day { + t.Errorf("npm default retention = %v, want 30d", got) + } + }) + } +} + func TestRulesMinPositive(t *testing.T) { if got := (Rules{}).MinPositive(); got != 0 { t.Errorf("empty rules MinPositive() = %v, want 0", got) diff --git a/internal/server/retention.go b/internal/server/retention.go index 660716d..9a2e9fb 100644 --- a/internal/server/retention.go +++ b/internal/server/retention.go @@ -59,11 +59,11 @@ func (s *Server) startRetentionLoop(ctx context.Context) { } } -// notCovered returns the known ecosystem keys that are not in covered. +// notCovered returns the known ecosystem keys the global default cannot reach. func notCovered(covered []string) []string { var missing []string for _, key := range retention.KnownKeys { - if !slices.Contains(covered, key) { + if key == "generic" || !slices.Contains(covered, key) { missing = append(missing, key) } } diff --git a/internal/server/retention_generic_test.go b/internal/server/retention_generic_test.go new file mode 100644 index 0000000..8a74ccd --- /dev/null +++ b/internal/server/retention_generic_test.go @@ -0,0 +1,156 @@ +package server + +import ( + "context" + "io" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + "time" + + "github.com/git-pkgs/proxy/internal/config" + "github.com/git-pkgs/proxy/internal/database" + "github.com/git-pkgs/proxy/internal/handler" + "github.com/git-pkgs/proxy/internal/retention" + "github.com/git-pkgs/registries/fetch" +) + +const genericReleaseAssetPath = "/jqlang/jq/releases/download/jq-1.7.1/jq-linux-amd64" + +func genericRetentionUpstream(t *testing.T) (*httptest.Server, *atomic.Int64) { + t.Helper() + var downloads atomic.Int64 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != genericReleaseAssetPath { + http.NotFound(w, r) + return + } + downloads.Add(1) + _, _ = io.WriteString(w, "release asset") + })) + t.Cleanup(srv.Close) + return srv, &downloads +} + +func newGenericRetentionProxy(t *testing.T, upstream *httptest.Server) (*httptest.Server, *database.DB) { + t.Helper() + db, store := setupEvictionTest(t) + fetcher := fetch.NewFetcher(fetch.WithHTTPClient(upstream.Client()), fetch.WithMaxRetries(0)) + t.Cleanup(func() { _ = fetcher.Close() }) + proxy := handler.NewProxy(db, store, fetcher, fetch.NewResolver(), discardLogger()) + proxy.HTTPClient = upstream.Client() + routes := handler.NewGenericHandler(proxy, map[string]string{"github": upstream.URL}).Routes() + srv := httptest.NewServer(http.StripPrefix("/generic", routes)) + t.Cleanup(srv.Close) + return srv, db +} + +func genericReleaseArtifact(t *testing.T, db *database.DB) database.Artifact { + t.Helper() + const versionPURL = "pkg:generic/jqlang%2Fjq@jq-1.7.1" + var art database.Artifact + query := db.Rebind(`SELECT * FROM artifacts WHERE version_purl = ?`) + if err := db.Get(&art, query, versionPURL); err != nil { + t.Fatalf("reading release artifact %s: %v", versionPURL, err) + } + return art +} + +func TestGenericRetentionRequiresExplicitRule(t *testing.T) { + const packagePURL = "pkg:generic/jqlang%2Fjq" + tests := []struct { + name string + ecosystems map[string]string + packages map[string]string + wantCached bool + }{ + {"global default keeps generic", nil, nil, true}, + {"ecosystem opts in", map[string]string{"generic": "1d"}, nil, false}, + {"package opts in", nil, map[string]string{packagePURL: "1d"}, false}, + {"other package stays cached", nil, map[string]string{"pkg:generic/other%2Frepo": "1d"}, true}, + {"package disables ecosystem rule", map[string]string{"generic": "1d"}, map[string]string{packagePURL: "0"}, true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + upstream, _ := genericRetentionUpstream(t) + srv, db := newGenericRetentionProxy(t, upstream) + getOK(t, srv.URL+"/generic/github"+genericReleaseAssetPath) + versionPURL := packagePURL + "@jq-1.7.1" + now := time.Now() + old := now.Add(-30 * retentionDay) + backdate(t, db, "fetched_at", versionPURL, old) + backdate(t, db, "last_accessed_at", versionPURL, old) + + cfg := config.Default() + cfg.Storage.Retention.Default = "1d" + cfg.Storage.Retention.Ecosystems = tt.ecosystems + cfg.Storage.Retention.Packages = tt.packages + rules, err := cfg.RetentionRules(retention.Default) + if err != nil { + t.Fatalf("RetentionRules() error = %v", err) + } + sweepRetention(context.Background(), db, discardLogger(), retention.Default, rules, now) + if art := genericReleaseArtifact(t, db); art.StoragePath.Valid != tt.wantCached { + t.Errorf("release cached = %v, want %v", art.StoragePath.Valid, tt.wantCached) + } + }) + } +} + +func TestGenericRetentionEvictsAndRefetchesReleaseAsset(t *testing.T) { + upstream, downloads := genericRetentionUpstream(t) + srv, db := newGenericRetentionProxy(t, upstream) + const ( + packagePURL = "pkg:generic/jqlang%2Fjq" + versionPURL = packagePURL + "@jq-1.7.1" + ) + assetURL := srv.URL + "/generic/github" + genericReleaseAssetPath + + getOK(t, assetURL) + getOK(t, assetURL) + if got := downloads.Load(); got != 1 { + t.Fatalf("upstream downloads = %d, want 1 (second request from cache)", got) + } + art := genericReleaseArtifact(t, db) + if !art.LastAccessedAt.Valid { + t.Fatalf("cached release artifact = %+v; want a recorded cache hit", art) + } + + cfg := config.Default() + cfg.Storage.Retention.Packages = map[string]string{packagePURL: "1d"} + rules, err := cfg.RetentionRules(retention.Default) + if err != nil { + t.Fatalf("RetentionRules() error = %v", err) + } + now := time.Now() + old := now.Add(-30 * retentionDay) + backdate(t, db, "fetched_at", versionPURL, old) + backdate(t, db, "last_accessed_at", versionPURL, now) + sweepRetention(context.Background(), db, discardLogger(), retention.Default, rules, now) + art = genericReleaseArtifact(t, db) + if !art.StoragePath.Valid { + t.Fatalf("recently accessed release was evicted: %+v", art) + } + + backdate(t, db, "last_accessed_at", versionPURL, old) + sweepRetention(context.Background(), db, discardLogger(), retention.Default, rules, now) + art = genericReleaseArtifact(t, db) + if art.StoragePath.Valid { + t.Fatalf("unused release was not evicted: %+v", art) + } + + getOK(t, assetURL) + if got := downloads.Load(); got != 2 { + t.Errorf("upstream downloads = %d, want 2 after retention eviction", got) + } + art = genericReleaseArtifact(t, db) + if !art.StoragePath.Valid { + t.Fatalf("refetched release artifact = %+v; want cached artifact", art) + } + sweepRetention(context.Background(), db, discardLogger(), retention.Default, rules, time.Now()) + art = genericReleaseArtifact(t, db) + if !art.StoragePath.Valid { + t.Errorf("refetched release was evicted using its old access time: %+v", art) + } +} diff --git a/internal/server/retention_test.go b/internal/server/retention_test.go index 22fca0a..41acb92 100644 --- a/internal/server/retention_test.go +++ b/internal/server/retention_test.go @@ -18,10 +18,11 @@ import ( const retentionDay = 24 * time.Hour -// retentionRegistry registers npm and gem the way their handlers would, in a +// retentionRegistry registers generic, npm and gem the way their handlers would, in a // registry of the test's own so the handlers' registrations stay untouched. func retentionRegistry() *retention.Registry { reg := retention.NewRegistry() + reg.Register(retention.Spec{Key: "generic", CanonicalPackage: retention.DefaultCanonical("generic")}) reg.Register(retention.Spec{Key: "npm", CanonicalPackage: retention.DefaultCanonical("npm")}) reg.Register(retention.Spec{Key: "gem", CanonicalPackage: retention.DefaultCanonical("gem")}) return reg @@ -83,6 +84,7 @@ func TestSweepRetentionAppliesEachRule(t *testing.T) { Default: 30 * retentionDay, Ecosystems: map[string]time.Duration{"npm": 14 * retentionDay}, Packages: map[string]time.Duration{ + "pkg:generic/jqlang%2Fjq": 90 * retentionDay, "pkg:npm/pinned": 0, "pkg:npm/long-lived": 90 * retentionDay, "pkg:npm/long-lived-expired": 90 * retentionDay, @@ -104,6 +106,9 @@ func TestSweepRetentionAppliesEachRule(t *testing.T) { {"gem", "gem", "default-applies", ago(40), ago(40), false}, {"gem", "gem", "within-default", ago(40), ago(20), true}, {"rubygems", "gem", "mirrored", ago(40), time.Time{}, false}, + {"generic", "generic", "jqlang%2Fjq", ago(120), ago(100), false}, + {"generic", "generic", "frequently-used", ago(120), ago(5), true}, + {"generic", "generic", "default-does-not-apply", ago(120), ago(100), true}, {"gem", "gem", "short", ago(3), ago(3), false}, {"pypi", "pypi", "unregistered", ago(400), ago(400), true}, } @@ -250,13 +255,16 @@ func TestSweepRetentionWithoutRulesDoesNothing(t *testing.T) { } func TestNotCovered(t *testing.T) { - got := notCovered([]string{"npm", "pypi"}) + got := notCovered([]string{"generic", "npm", "pypi"}) if slices.Contains(got, "npm") || slices.Contains(got, "pypi") { t.Errorf("notCovered = %v, includes a covered key", got) } if len(got) != len(retention.KnownKeys)-2 { t.Errorf("notCovered returned %d keys, want %d", len(got), len(retention.KnownKeys)-2) } + if !slices.Contains(got, "generic") { + t.Error("notCovered must include generic even when its handler is registered") + } } func TestReclaimDueDrainsSeveralBatches(t *testing.T) { From 4de5419256a4c715f145f68bbafa299057c96dbc Mon Sep 17 00:00:00 2001 From: profiling <5w1h@outlook.com> Date: Mon, 19 Oct 2026 15:54:21 +0800 Subject: [PATCH 319/633] Make response write and artifact download timeouts configurable (#531) Add independent write_timeout and artifact_timeout settings with environment overrides and unchanged five-minute defaults. Keep mirror downloads unlimited and validate timeout values. Refs git-pkgs/proxy#530 Co-authored-by: ztzhou --- config.example.yaml | 8 ++ docs/configuration.md | 16 +++ internal/config/config.go | 62 +++++++++-- internal/config/request_timeouts_test.go | 76 +++++++++++++ internal/server/request_timeouts_test.go | 131 +++++++++++++++++++++++ internal/server/server.go | 13 ++- 6 files changed, 288 insertions(+), 18 deletions(-) create mode 100644 internal/config/request_timeouts_test.go create mode 100644 internal/server/request_timeouts_test.go diff --git a/config.example.yaml b/config.example.yaml index 253ae7f..ed168de 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -14,6 +14,14 @@ base_url: "http://localhost:8080" # Set to "0" to disable the timeout. Default: "30s". # http_timeout: "30s" +# Response write deadline, including download and scan time before sending data. +# Set to "0" to disable. Default: "5m". +# write_timeout: "5m" + +# Timeout for each upstream artifact download while serving requests; excludes +# scanning. Set to "0" to disable. Default: "5m". Does not limit proxy mirror. +# artifact_timeout: "5m" + # Largest upstream artifact to fetch (e.g. "2GB"). Larger ones are refused # and not cached. Default: unlimited. # artifact_max_size: "" diff --git a/docs/configuration.md b/docs/configuration.md index 987552d..08a9acd 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -22,6 +22,22 @@ See `config.example.yaml` in the repository root for a complete example. | `trust_forwarded_for` | `PROXY_TRUST_FORWARDED_FOR` | - | `false` | Attribute requests to the leftmost `X-Forwarded-For` entry instead of the TCP peer address, in the structured log, the access log and the request-source table on `/ui/analytics`. | | `ui_request_sources` | `PROXY_UI_REQUEST_SOURCES` | - | `false` | Show the request-source table on `/ui/analytics`, which reports caller addresses, the tool each ran and how much each pulled. | | `ui_failure_details` | `PROXY_UI_FAILURE_DETAILS` | - | `false` | List the latest failures behind each failure count on `/ui/analytics`: the package, its upstream URL or storage path, and the error. URLs are shown without credentials or query strings. | +| `http_timeout` | `PROXY_HTTP_TIMEOUT` | - | `30s` | Timeout for upstream metadata and pass-through requests. | +| `write_timeout` | `PROXY_WRITE_TIMEOUT` | - | `5m` | Response write deadline, including download and scanning time before the response starts. | +| `artifact_timeout` | `PROXY_ARTIFACT_TIMEOUT` | - | `5m` | Timeout for each upstream artifact download while serving requests; excludes subsequent scanning. Does not limit `proxy mirror`. | + +These timeouts accept Go durations such as `30s`, `5m`, or `1h`. An omitted value uses the default; `"0"` disables that timeout. Invalid or negative values are rejected at startup. Environment variables override file values. + +For long scans, increase `write_timeout` to cover the download, scan, and response transfer. `scanning.timeout` controls the scan separately; changing it alone does not extend the response deadline. `artifact_timeout` is independent of both. For example: + +```yaml +write_timeout: "18m" +artifact_timeout: "10m" +scanning: + timeout: "16m" +``` + +These are independent limits, not reserved time slices: a download approaching 10 minutes followed by a 16-minute scan would still exceed an 18-minute response deadline. Any reverse proxy in front of the server also needs sufficient timeout allowance. The upstream transport's existing response-header timeout remains unchanged. ## Storage diff --git a/internal/config/config.go b/internal/config/config.go index 2f78c3a..14c7462 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -186,6 +186,15 @@ type Config struct { // response headers within 60 seconds. HTTPTimeout string `json:"http_timeout" yaml:"http_timeout"` + // WriteTimeout bounds response writes, including time spent downloading and + // scanning before the response starts. Default: "5m"; "0" disables it. + WriteTimeout string `json:"write_timeout" yaml:"write_timeout"` + + // ArtifactTimeout bounds each upstream artifact download while serving + // requests, excluding scanning. Default: "5m"; "0" disables it. + // The mirror command keeps its unlimited artifact download timeout. + ArtifactTimeout string `json:"artifact_timeout" yaml:"artifact_timeout"` + // MirrorAPI enables the /api/mirror endpoints for starting mirror jobs via HTTP. // Disabled by default to prevent unauthenticated users from triggering downloads. MirrorAPI bool `json:"mirror_api" yaml:"mirror_api"` @@ -1258,6 +1267,8 @@ func (c *Config) LoadFromEnv() { setEnvString(&c.ArtifactMaxSize, "PROXY_ARTIFACT_MAX_SIZE") setEnvString(&c.MetadataRewriteCacheSize, "PROXY_METADATA_REWRITE_CACHE_SIZE") setEnvString(&c.HTTPTimeout, "PROXY_HTTP_TIMEOUT") + setEnvString(&c.WriteTimeout, "PROXY_WRITE_TIMEOUT") + setEnvString(&c.ArtifactTimeout, "PROXY_ARTIFACT_TIMEOUT") c.setGradleReadOnlyFromEnv() setEnvString(&c.Gradle.BuildCache.MaxUploadSize, "PROXY_GRADLE_BUILD_CACHE_MAX_UPLOAD_SIZE") setEnvString(&c.Gradle.BuildCache.MaxAge, "PROXY_GRADLE_BUILD_CACHE_MAX_AGE") @@ -1357,7 +1368,7 @@ func (c *Config) Validate() error { return err } - if err := validateHTTPTimeout(c.HTTPTimeout); err != nil { + if err := c.validateRequestTimeouts(); err != nil { return err } @@ -1481,6 +1492,8 @@ func (g *GradleBuildCacheConfig) Validate() error { const ( defaultMetadataTTL = 5 * time.Minute //nolint:mnd // sensible default defaultDirectServeTTL = 15 * time.Minute //nolint:mnd // sensible default + defaultWriteTimeout = 5 * time.Minute + defaultArtifactTimeout = 5 * time.Minute defaultHTTPTimeout = 30 * time.Second //nolint:mnd // sensible default defaultHitFlushInterval = time.Second defaultMetadataMaxSize = 100 << 20 @@ -1509,16 +1522,30 @@ func (c *Config) ParseMaxSize() int64 { return size } -func validateHTTPTimeout(s string) error { +func (c *Config) validateRequestTimeouts() error { + for _, timeout := range []struct{ field, value string }{ + {"http_timeout", c.HTTPTimeout}, + {"write_timeout", c.WriteTimeout}, + {"artifact_timeout", c.ArtifactTimeout}, + } { + if err := validateRequestTimeout(timeout.field, timeout.value); err != nil { + return err + } + } + + return nil +} + +func validateRequestTimeout(field, s string) error { if s == "" || s == "0" { return nil } d, err := time.ParseDuration(s) if err != nil { - return fmt.Errorf("invalid http_timeout %q: %w", s, err) + return fmt.Errorf("invalid %s %q: %w", field, s, err) } if d < 0 { - return fmt.Errorf("invalid http_timeout %q: must be non-negative", s) + return fmt.Errorf("invalid %s %q: must be non-negative", field, s) } return nil } @@ -1617,15 +1644,28 @@ func (c *Config) ParseArtifactMaxSize() int64 { // ParseHTTPTimeout returns the upstream HTTP client timeout. // Returns 30s if unset, 0 (no timeout) if explicitly set to "0". func (c *Config) ParseHTTPTimeout() time.Duration { - if c.HTTPTimeout == "" { - return defaultHTTPTimeout + return parseRequestTimeout(c.HTTPTimeout, defaultHTTPTimeout) +} + +// ParseWriteTimeout returns the response write timeout, defaulting to 5m. +// An explicit zero disables the timeout. +func (c *Config) ParseWriteTimeout() time.Duration { + return parseRequestTimeout(c.WriteTimeout, defaultWriteTimeout) +} + +// ParseArtifactTimeout returns the upstream artifact download timeout for +// serving requests, defaulting to 5m. An explicit zero disables the timeout. +func (c *Config) ParseArtifactTimeout() time.Duration { + return parseRequestTimeout(c.ArtifactTimeout, defaultArtifactTimeout) +} + +func parseRequestTimeout(value string, fallback time.Duration) time.Duration { + if value == "" { + return fallback } - if c.HTTPTimeout == "0" { - return 0 - } - d, err := time.ParseDuration(c.HTTPTimeout) + d, err := time.ParseDuration(value) if err != nil || d < 0 { - return defaultHTTPTimeout + return fallback } return d } diff --git a/internal/config/request_timeouts_test.go b/internal/config/request_timeouts_test.go new file mode 100644 index 0000000..01848c6 --- /dev/null +++ b/internal/config/request_timeouts_test.go @@ -0,0 +1,76 @@ +package config + +import ( + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +func TestRequestTimeouts(t *testing.T) { + for _, value := range []string{"", "0", "0s", "18m", "invalid", "-1s", "999999999999999999h"} { + t.Run(value, func(t *testing.T) { + cfg := Default() + cfg.WriteTimeout, cfg.ArtifactTimeout = value, value + valid := value == "" || value == "0" || value == "0s" || value == "18m" + want := 5 * time.Minute + if value == "0" || value == "0s" { + want = 0 + } + if value == "18m" { + want = 18 * time.Minute + } + if got := cfg.ParseWriteTimeout(); got != want { + t.Errorf("write timeout = %v, want %v", got, want) + } + if got := cfg.ParseArtifactTimeout(); got != want { + t.Errorf("artifact timeout = %v, want %v", got, want) + } + err := cfg.Validate() + if (err == nil) != valid { + t.Fatalf("Validate() = %v, valid = %v", err, valid) + } + if !valid { + if !strings.Contains(err.Error(), "write_timeout") { + t.Fatalf("error does not identify write_timeout: %v", err) + } + cfg.WriteTimeout = "" + if err := cfg.Validate(); err == nil || !strings.Contains(err.Error(), "artifact_timeout") { + t.Fatalf("artifact validation = %v", err) + } + } + }) + } +} + +func TestLoadRequestTimeouts(t *testing.T) { + for _, format := range []struct{ extension, data string }{ + {"yaml", "write_timeout: 18m\nartifact_timeout: 10m\nhttp_timeout: 90s\n"}, + {"json", `{"write_timeout":"18m","artifact_timeout":"10m","http_timeout":"90s"}`}, + } { + t.Run(format.extension, func(t *testing.T) { + path := filepath.Join(t.TempDir(), "config."+format.extension) + if err := os.WriteFile(path, []byte(format.data), 0600); err != nil { + t.Fatal(err) + } + cfg, err := Load(path) + if err != nil { + t.Fatal(err) + } + if cfg.ParseWriteTimeout() != 18*time.Minute || cfg.ParseArtifactTimeout() != 10*time.Minute || cfg.ParseHTTPTimeout() != 90*time.Second { + t.Fatalf("file timeouts = %v, %v, %v", cfg.WriteTimeout, cfg.ArtifactTimeout, cfg.HTTPTimeout) + } + t.Setenv("PROXY_WRITE_TIMEOUT", "20m") + t.Setenv("PROXY_ARTIFACT_TIMEOUT", "0") + t.Setenv("PROXY_HTTP_TIMEOUT", "15s") + cfg.LoadFromEnv() + if cfg.ParseWriteTimeout() != 20*time.Minute || cfg.ParseArtifactTimeout() != 0 || cfg.ParseHTTPTimeout() != 15*time.Second { + t.Fatalf("environment timeouts = %v, %v, %v", cfg.WriteTimeout, cfg.ArtifactTimeout, cfg.HTTPTimeout) + } + if err := cfg.Validate(); err != nil { + t.Fatal(err) + } + }) + } +} diff --git a/internal/server/request_timeouts_test.go b/internal/server/request_timeouts_test.go new file mode 100644 index 0000000..98b0a53 --- /dev/null +++ b/internal/server/request_timeouts_test.go @@ -0,0 +1,131 @@ +package server + +import ( + "context" + "errors" + "io" + "log/slog" + "net" + "net/http" + "net/http/httptest" + "path/filepath" + "testing" + "time" + + "github.com/git-pkgs/proxy/internal/config" +) + +// Exercise the real serve wiring: a slow artifact must use artifact_timeout, +// while waiting for it also consumes the independent response write deadline. +func TestServeRequestTimeouts(t *testing.T) { + for _, tc := range []struct { + name, write, artifact string + wantOK bool + wantConnectionError bool + }{ + {"independent from metadata timeout", "5s", "5s", true, false}, + {"write deadline expires while waiting", "1ms", "5s", false, true}, + {"artifact download times out", "5s", "1ms", false, false}, + {"both timeouts disabled", "0", "0", true, false}, + } { + t.Run(tc.name, func(t *testing.T) { + status, body, err := fetchWithRequestTimeouts(t, tc.write, tc.artifact) + if tc.wantConnectionError { + if err == nil { + t.Fatal("response succeeded after its write deadline") + } + return + } + if err != nil { + t.Fatal(err) + } + if tc.wantOK { + if status != http.StatusOK || body != "release asset" { + t.Fatalf("response = %d %q", status, body) + } + } else if status < 500 { + t.Fatalf("artifact timeout response = %d %q, want server error", status, body) + } + }) + } +} + +func fetchWithRequestTimeouts(t *testing.T, writeTimeout, artifactTimeout string) (int, string, error) { + t.Helper() + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + time.Sleep(100 * time.Millisecond) + _, _ = io.WriteString(w, "release asset") + })) + defer upstream.Close() + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer func() { _ = listener.Close() }() + dir := t.TempDir() + cfg := config.Default() + cfg.Listen = listener.Addr().String() + cfg.BaseURL = "http://" + cfg.Listen + cfg.Database.Path = filepath.Join(dir, "proxy.db") + cfg.Storage.URL = "file://" + filepath.Join(dir, "artifacts") + cfg.Upstream.Generic = map[string]string{"releases": upstream.URL} + cfg.Upstream.AllowLoopback = true + cfg.HTTPTimeout = "1ns" + cfg.WriteTimeout, cfg.ArtifactTimeout = writeTimeout, artifactTimeout + if err := cfg.Validate(); err != nil { + t.Fatal(err) + } + srv, err := New(cfg, slog.New(slog.NewTextHandler(io.Discard, nil)), BuildInfo{Version: "test"}) + if err != nil { + t.Fatal(err) + } + done := make(chan error, 1) + go func() { done <- srv.Start(listener) }() + defer func() { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if err := srv.Shutdown(ctx); err != nil { + t.Error(err) + } + if err := <-done; !errors.Is(err, http.ErrServerClosed) { + t.Errorf("Start() = %v", err) + } + }() + client := &http.Client{Timeout: 10 * time.Second} + defer client.CloseIdleConnections() + + resp, err := client.Get(cfg.BaseURL + "/generic/releases/owner/repo/releases/download/v1/asset.bin") + if err != nil { + return 0, "", err + } + defer func() { _ = resp.Body.Close() }() + body, err := io.ReadAll(resp.Body) + return resp.StatusCode, string(body), err +} + +func TestMirrorIgnoresServeArtifactTimeout(t *testing.T) { + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + time.Sleep(20 * time.Millisecond) + _, _ = io.WriteString(w, "mirrored artifact") + })) + defer upstream.Close() + db, store := setupEvictionTest(t) + defer func() { _ = store.Close() }() + cfg := config.Default() + cfg.Upstream.AllowLoopback = true + cfg.ArtifactTimeout = "1ns" + cfg.WriteTimeout = "1ns" + proxy, _ := NewMirrorProxy(cfg, db, store, slog.New(slog.NewTextHandler(io.Discard, nil))) + defer proxy.HTTPClient.CloseIdleConnections() + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + result, err := proxy.GetOrFetchArtifactFromURL(ctx, "generic", "mirror-timeout", "v1", "asset.bin", upstream.URL+"/asset.bin") + if err != nil { + t.Fatal(err) + } + defer func() { _ = result.Reader.Close() }() + body, err := io.ReadAll(result.Reader) + if err != nil || string(body) != "mirrored artifact" { + t.Fatalf("mirror result = %q, %v", body, err) + } +} diff --git a/internal/server/server.go b/internal/server/server.go index 321633b..5921cf0 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -90,11 +90,10 @@ import ( ) const ( - serverReadTimeout = 30 * time.Second - serverWriteTimeout = 5 * time.Minute - serverIdleTimeout = 60 * time.Second - dashboardTopN = 10 - hoursPerDay = 24 + serverReadTimeout = 30 * time.Second + serverIdleTimeout = 60 * time.Second + dashboardTopN = 10 + hoursPerDay = 24 // Upstream transport defaults, matching what fetch.NewFetcher would use // if we did not hand it our own client. Go's default transport keeps only @@ -226,7 +225,7 @@ func (s *Server) Start(listeners ...net.Listener) error { } func (s *Server) serve(listener net.Listener) error { - metadataClient, artifactClient := s.newUpstreamClients(serverWriteTimeout) + metadataClient, artifactClient := s.newUpstreamClients(s.cfg.ParseArtifactTimeout()) // Create shared components with circuit breaker. baseFetcher := fetch.NewFetcher(fetch.WithHTTPClient(artifactClient)) @@ -372,7 +371,7 @@ func (s *Server) serve(listener net.Listener) error { Addr: s.cfg.Listen, Handler: r, ReadTimeout: serverReadTimeout, - WriteTimeout: serverWriteTimeout, // Large artifacts need time + WriteTimeout: s.cfg.ParseWriteTimeout(), // Large artifacts need time IdleTimeout: serverIdleTimeout, } From 8e04713425bc74d78aafb4f44ca6bbb29e3928a5 Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Mon, 19 Oct 2026 12:28:55 +0100 Subject: [PATCH 320/633] Use shared registries size limits and rate-limit gate (#516) * Use shared registries size limits and rate-limit gate * Use registries v0.10.0 release --- go.mod | 2 +- go.sum | 4 +- internal/handler/artifact_size.go | 76 ------- internal/handler/artifact_size_test.go | 95 ++------ internal/handler/download_test.go | 8 +- internal/handler/handler.go | 4 +- internal/handler/upstream_held.go | 4 +- internal/httpclient/ratelimit.go | 228 ------------------- internal/httpclient/ratelimit_test.go | 257 ---------------------- internal/server/breakers_test.go | 4 +- internal/server/registry_policies_test.go | 79 +++++++ internal/server/server.go | 30 ++- 12 files changed, 139 insertions(+), 652 deletions(-) delete mode 100644 internal/handler/artifact_size.go delete mode 100644 internal/httpclient/ratelimit.go delete mode 100644 internal/httpclient/ratelimit_test.go create mode 100644 internal/server/registry_policies_test.go diff --git a/go.mod b/go.mod index 0b791cd..395a436 100644 --- a/go.mod +++ b/go.mod @@ -16,7 +16,7 @@ require ( github.com/git-pkgs/integrity v0.2.0 github.com/git-pkgs/magic v0.5.0 github.com/git-pkgs/purl v0.1.21 - github.com/git-pkgs/registries v0.9.3 + github.com/git-pkgs/registries v0.10.0 github.com/git-pkgs/spdx v0.3.3 github.com/git-pkgs/vers v0.7.2 github.com/git-pkgs/vulns v0.2.4 diff --git a/go.sum b/go.sum index 956fd28..4d4c37b 100644 --- a/go.sum +++ b/go.sum @@ -143,8 +143,8 @@ github.com/git-pkgs/pom v0.1.8 h1:mXCuWw6XkrKKZb6IurRZyIznYLPhV5AYg5ZO38Uj/l8= github.com/git-pkgs/pom v0.1.8/go.mod h1:sQBeYRzjCUn77fcABPn6I/a1aBHFxm55jTmkNxYtGtw= github.com/git-pkgs/purl v0.1.21 h1:PDZUX43lxIiXPn2eHcsNiiiwQo+HajntZ/4TwSvCGlw= github.com/git-pkgs/purl v0.1.21/go.mod h1:4dOmris48ehjiKmWBqSmMHguUQht48BLeXW+hQUnuj0= -github.com/git-pkgs/registries v0.9.3 h1:nXIP//6kVqLKqYXOWhEMoWUkJPDAZYVLRUYfHz3jM2Y= -github.com/git-pkgs/registries v0.9.3/go.mod h1:BpXlfRCfu4S5bTzm7u7igppVu94mXgll/PENe0AjKmI= +github.com/git-pkgs/registries v0.10.0 h1:0goaC5mxw9MglS/XOm2De7fkPBWCCerelm7RPrPh9hQ= +github.com/git-pkgs/registries v0.10.0/go.mod h1:bKxgdy/PwnzQXL2FnOktmSQHTKk6B85MA4VtvvYItPc= github.com/git-pkgs/spdx v0.3.3 h1:xXlUoZzjDSkwxC6mWHfaW+kc8a0+LH6d+gGHuZWftqI= github.com/git-pkgs/spdx v0.3.3/go.mod h1:n9rAicgw+Wqtfursju6oKaAL6cUCTiIS926mb1KWoKU= github.com/git-pkgs/vers v0.7.2 h1:Ctex1O+Ai/FONmyDaw37EolBXg0og6UdxbCjjYbeRHs= diff --git a/internal/handler/artifact_size.go b/internal/handler/artifact_size.go deleted file mode 100644 index d4c733b..0000000 --- a/internal/handler/artifact_size.go +++ /dev/null @@ -1,76 +0,0 @@ -package handler - -import ( - "context" - "errors" - "fmt" - "io" - "net/http" - - "github.com/git-pkgs/registries/fetch" -) - -// ErrArtifactTooLarge is returned for an upstream artifact larger than the -// configured artifact size limit. -var ErrArtifactTooLarge = errors.New("artifact exceeds size limit") - -// LimitArtifactSize wraps fetcher so no artifact over maxSize bytes is -// fetched. An artifact whose declared size is over the limit is refused -// before its body is read; one of unknown size fails once reading passes the -// limit, so the store or stream consuming it fails and nothing is cached. -// A maxSize of zero or less returns fetcher unchanged. -func LimitArtifactSize(fetcher fetch.FetcherInterface, maxSize int64) fetch.FetcherInterface { //nolint:ireturn // returns fetcher itself when there is no limit - if maxSize <= 0 { - return fetcher - } - return &sizeLimitedFetcher{FetcherInterface: fetcher, maxSize: maxSize} -} - -type sizeLimitedFetcher struct { - fetch.FetcherInterface - maxSize int64 -} - -func (f *sizeLimitedFetcher) Fetch(ctx context.Context, url string) (*fetch.Artifact, error) { - return f.limit(f.FetcherInterface.Fetch(ctx, url)) -} - -func (f *sizeLimitedFetcher) FetchWithHeaders(ctx context.Context, url string, headers http.Header) (*fetch.Artifact, error) { - return f.limit(f.FetcherInterface.FetchWithHeaders(ctx, url, headers)) -} - -func (f *sizeLimitedFetcher) limit(artifact *fetch.Artifact, err error) (*fetch.Artifact, error) { - if err != nil { - return nil, err - } - if artifact.Size > f.maxSize { - _ = artifact.Body.Close() - return nil, fmt.Errorf("%w: upstream declared %d bytes, limit is %d", ErrArtifactTooLarge, artifact.Size, f.maxSize) - } - limited := *artifact - limited.Body = &sizeLimitedBody{ReadCloser: artifact.Body, remaining: f.maxSize} - return &limited, nil -} - -// sizeLimitedBody fails with ErrArtifactTooLarge once more than the limit has -// been read. It reads one byte past the limit to tell an artifact of exactly -// the limit from a larger one. -type sizeLimitedBody struct { - io.ReadCloser - remaining int64 -} - -func (b *sizeLimitedBody) Read(p []byte) (int, error) { - if b.remaining < 0 { - return 0, ErrArtifactTooLarge - } - if int64(len(p)) > b.remaining+1 { - p = p[:b.remaining+1] - } - n, err := b.ReadCloser.Read(p) - b.remaining -= int64(n) - if b.remaining < 0 { - return n + int(b.remaining), ErrArtifactTooLarge - } - return n, err -} diff --git a/internal/handler/artifact_size_test.go b/internal/handler/artifact_size_test.go index f6fe9af..68dc25d 100644 --- a/internal/handler/artifact_size_test.go +++ b/internal/handler/artifact_size_test.go @@ -6,83 +6,36 @@ import ( "io" "net/http" "net/http/httptest" - "strings" "testing" "github.com/git-pkgs/registries/fetch" ) -type closeRecorder struct { - io.Reader - closed bool -} - -func (c *closeRecorder) Close() error { - c.closed = true - return nil -} - -func TestLimitArtifactSizeUnlimitedReturnsFetcher(t *testing.T) { - fetcher := &mockFetcher{} - if got := LimitArtifactSize(fetcher, 0); got != fetcher { - t.Errorf("LimitArtifactSize(f, 0) = %T, want the fetcher itself", got) - } -} - -func TestLimitArtifactSizeRefusesDeclaredOversize(t *testing.T) { - body := &closeRecorder{Reader: strings.NewReader("0123456789")} - limited := LimitArtifactSize(&mockFetcher{artifact: &fetch.Artifact{Body: body, Size: 10}}, 4) - - _, err := limited.Fetch(context.Background(), "https://registry.example/a.tgz") - if !errors.Is(err, ErrArtifactTooLarge) { - t.Fatalf("Fetch error = %v, want ErrArtifactTooLarge", err) - } - if !body.closed { - t.Error("refused body was not closed") - } -} - -func TestLimitArtifactSizeFailsUndeclaredOversizeWhileReading(t *testing.T) { - limited := LimitArtifactSize(&mockFetcher{artifact: &fetch.Artifact{ - Body: io.NopCloser(strings.NewReader("0123456789")), Size: -1, - }}, 4) - - artifact, err := limited.Fetch(context.Background(), "https://registry.example/a.tgz") - if err != nil { - t.Fatalf("Fetch failed: %v", err) - } - if _, err := io.ReadAll(artifact.Body); !errors.Is(err, ErrArtifactTooLarge) { - t.Errorf("reading body error = %v, want ErrArtifactTooLarge", err) - } -} - -func TestLimitArtifactSizeAllowsArtifactAtLimit(t *testing.T) { - limited := LimitArtifactSize(&mockFetcher{artifact: &fetch.Artifact{ - Body: io.NopCloser(strings.NewReader("0123")), Size: -1, - }}, 4) - - artifact, err := limited.FetchWithHeaders(context.Background(), "https://registry.example/a.tgz", nil) - if err != nil { - t.Fatalf("Fetch failed: %v", err) - } - data, err := io.ReadAll(artifact.Body) - if err != nil || string(data) != "0123" { - t.Errorf("body = %q, %v; want %q, nil", data, err, "0123") - } +func artifactSizeFetcher(t *testing.T) (*fetch.Fetcher, string) { + t.Helper() + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.(http.Flusher).Flush() + _, _ = io.WriteString(w, "0123456789") + })) + t.Cleanup(upstream.Close) + f := fetch.NewFetcher(fetch.WithHTTPClient(upstream.Client()), fetch.WithMaxArtifactSize(4)) + t.Cleanup(func() { _ = f.Close() }) + return f, upstream.URL } func TestOversizedArtifactIsNotCached(t *testing.T) { - proxy, db, store, fetcher := setupTestProxy(t) - fetcher.artifact = &fetch.Artifact{Body: io.NopCloser(strings.NewReader("0123456789")), Size: -1} - proxy.Fetcher = LimitArtifactSize(fetcher, 4) + proxy, db, store, _ := setupTestProxy(t) + limited, upstream := artifactSizeFetcher(t) + proxy.Fetcher = limited - _, err := proxy.GetOrFetchArtifactFromURL(context.Background(), "npm", "big", "1.0.0", "big-1.0.0.tgz", "https://registry.example/big-1.0.0.tgz") - if !errors.Is(err, ErrArtifactTooLarge) { - t.Fatalf("GetOrFetchArtifactFromURL error = %v, want ErrArtifactTooLarge", err) + _, err := proxy.GetOrFetchArtifactFromURL(context.Background(), "npm", "big", "1.0.0", "big-1.0.0.tgz", upstream+"/big-1.0.0.tgz") + if !errors.Is(err, fetch.ErrArtifactTooLarge) { + t.Fatalf("GetOrFetchArtifactFromURL error = %v, want fetch.ErrArtifactTooLarge", err) } if len(store.files) != 0 { t.Errorf("storage holds %d objects after an oversized fetch, want 0", len(store.files)) } + fetchErr := err record, err := db.GetArtifact("pkg:npm/big@1.0.0", "big-1.0.0.tgz") if err != nil { t.Fatalf("GetArtifact failed: %v", err) @@ -92,24 +45,24 @@ func TestOversizedArtifactIsNotCached(t *testing.T) { } rec := httptest.NewRecorder() - proxy.serveArtifactError(rec, err, "failed to fetch package") + proxy.serveArtifactError(rec, fetchErr, "failed to fetch package") if rec.Code != http.StatusBadGateway { t.Errorf("status = %d, want %d", rec.Code, http.StatusBadGateway) } } func TestOversizedStreamedArtifactFailsTheStream(t *testing.T) { - proxy, _, _, fetcher := setupTestProxy(t) + proxy, _, _, _ := setupTestProxy(t) proxy.StreamArtifacts = true - fetcher.artifact = &fetch.Artifact{Body: io.NopCloser(strings.NewReader("0123456789")), Size: -1} - proxy.Fetcher = LimitArtifactSize(fetcher, 4) + limited, upstream := artifactSizeFetcher(t) + proxy.Fetcher = limited - result, err := proxy.GetOrFetchArtifactFromURL(context.Background(), "npm", "big", "1.0.0", "big-1.0.0.tgz", "https://registry.example/big-1.0.0.tgz") + result, err := proxy.GetOrFetchArtifactFromURL(context.Background(), "npm", "big", "1.0.0", "big-1.0.0.tgz", upstream+"/big-1.0.0.tgz") if err != nil { t.Fatalf("GetOrFetchArtifactFromURL failed: %v", err) } defer func() { _ = result.Reader.Close() }() - if _, err := io.ReadAll(result.Reader); !errors.Is(err, ErrArtifactTooLarge) { - t.Errorf("streamed body error = %v, want ErrArtifactTooLarge", err) + if _, err := io.ReadAll(result.Reader); !errors.Is(err, fetch.ErrArtifactTooLarge) { + t.Errorf("streamed body error = %v, want fetch.ErrArtifactTooLarge", err) } } diff --git a/internal/handler/download_test.go b/internal/handler/download_test.go index eb332a9..ce44775 100644 --- a/internal/handler/download_test.go +++ b/internal/handler/download_test.go @@ -12,9 +12,9 @@ import ( "time" "github.com/git-pkgs/proxy/internal/database" - "github.com/git-pkgs/proxy/internal/httpclient" "github.com/git-pkgs/proxy/internal/metrics" "github.com/git-pkgs/purl" + registryclient "github.com/git-pkgs/registries/client" "github.com/git-pkgs/registries/fetch" ) @@ -1277,7 +1277,7 @@ func TestRPMHandler_DownloadCacheMiss(t *testing.T) { } func TestMavenHandler_UpstreamHeldReturns503(t *testing.T) { - held := &httpclient.HeldError{Host: "repo1.maven.org", Until: time.Now().Add(90 * time.Second), RetryAfter: 90 * time.Second} + held := ®istryclient.HeldError{Host: "repo1.maven.org", Until: time.Now().Add(90 * time.Second), RetryAfter: 90 * time.Second} t.Run("artifact", func(t *testing.T) { proxy, _, _, fetcher := setupTestProxy(t) @@ -1325,7 +1325,7 @@ func (fn roundTripperFunc) RoundTrip(req *http.Request) (*http.Response, error) func TestMavenHandler_HeldArtifactMissesSkipCircuitBreaker(t *testing.T) { proxy, _, _, _ := setupTestProxy(t) upstreamCalls := 0 - gate := httpclient.NewRateLimitGate(roundTripperFunc(func(req *http.Request) (*http.Response, error) { + gate := registryclient.NewRateLimitGate(roundTripperFunc(func(req *http.Request) (*http.Response, error) { upstreamCalls++ return &http.Response{ StatusCode: http.StatusTooManyRequests, @@ -1365,7 +1365,7 @@ func TestMavenHandler_HeldArtifactMissesSkipCircuitBreaker(t *testing.T) { } func TestHandlers_UpstreamHeldMetadataReturns503(t *testing.T) { - held := &httpclient.HeldError{Host: "upstream.example", Until: time.Now().Add(42 * time.Second), RetryAfter: 42 * time.Second} + held := ®istryclient.HeldError{Host: "upstream.example", Until: time.Now().Add(42 * time.Second), RetryAfter: 42 * time.Second} tests := []struct { name string handler func(*Proxy) http.Handler diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 1af336c..46b93ca 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -20,12 +20,12 @@ import ( "github.com/git-pkgs/cooldown" "github.com/git-pkgs/proxy/internal/database" "github.com/git-pkgs/proxy/internal/denylist" - "github.com/git-pkgs/proxy/internal/httpclient" "github.com/git-pkgs/proxy/internal/metrics" "github.com/git-pkgs/proxy/internal/packageurl" "github.com/git-pkgs/proxy/internal/scanner" "github.com/git-pkgs/proxy/internal/storage" "github.com/git-pkgs/purl" + registryclient "github.com/git-pkgs/registries/client" "github.com/git-pkgs/registries/fetch" "github.com/opencontainers/go-digest" ) @@ -178,7 +178,7 @@ type Proxy struct { // RateLimits is the gate under the upstream clients. Artifact fetches // check it before calling Fetcher, so held hosts never reach its breaker. - RateLimits *httpclient.RateLimitGate + RateLimits *registryclient.RateLimitGate // HostedNPMScopes and HostedComposerVendors name namespaces served from // pushed packages alone, without asking the upstream. diff --git a/internal/handler/upstream_held.go b/internal/handler/upstream_held.go index fa2ff96..1097e7e 100644 --- a/internal/handler/upstream_held.go +++ b/internal/handler/upstream_held.go @@ -6,14 +6,14 @@ import ( "net/http" "strconv" - "github.com/git-pkgs/proxy/internal/httpclient" + registryclient "github.com/git-pkgs/registries/client" ) // writeUpstreamHeld answers a request the rate limit gate refused to send // with 503 and the Retry-After the upstream asked for, and reports whether // err was such a refusal. func writeUpstreamHeld(w http.ResponseWriter, err error) bool { - var held *httpclient.HeldError + var held *registryclient.HeldError if !errors.As(err, &held) { return false } diff --git a/internal/httpclient/ratelimit.go b/internal/httpclient/ratelimit.go deleted file mode 100644 index 53e3e8e..0000000 --- a/internal/httpclient/ratelimit.go +++ /dev/null @@ -1,228 +0,0 @@ -package httpclient - -import ( - "errors" - "fmt" - "log/slog" - "net/http" - "net/url" - "strconv" - "strings" - "sync" - "time" -) - -const ( - // holdWithoutHeader is the first hold for a 429 that carries no usable - // Retry-After; each further such 429 after a hold expires doubles it. - holdWithoutHeader = time.Minute - maxHoldWithoutHeader = 15 * time.Minute //nolint:mnd // doubling stops here - // maxHold caps what an upstream's Retry-After can ask for. - maxHold = time.Hour -) - -// ErrUpstreamHeld marks a request the gate did not send because the upstream -// is rate limiting this proxy. Match it with errors.Is; the *HeldError in -// the chain carries the host and how long to wait. -var ErrUpstreamHeld = errors.New("upstream is rate limiting this proxy") - -// HeldError is returned in place of a response while a host is held. -type HeldError struct { - Host string - Until time.Time - RetryAfter time.Duration -} - -func (e *HeldError) Error() string { - return fmt.Sprintf("%s is rate limiting this proxy, holding requests for %s", e.Host, e.RetryAfter.Round(time.Second)) -} - -func (e *HeldError) Is(target error) bool { return target == ErrUpstreamHeld } - -type hostHold struct { - until time.Time - strikes int -} - -// RateLimitGate stops sending to a host that answered 429, or 503 with a -// Retry-After, for as long as it asked, and fails those requests locally -// with a *HeldError instead. Clients and fetchers above it see an error -// rather than a slow upstream, so they fall back to cache or fail fast -// without adding to the traffic the upstream is already refusing. The first -// non-limited response after the hold ends releases it. -type RateLimitGate struct { - base http.RoundTripper - Logger *slog.Logger - // OnHold is called with the host each time a hold starts. - OnHold func(host string) - now func() time.Time - - mu sync.Mutex - holds map[string]*hostHold -} - -// NewRateLimitGate wraps base. A nil base uses http.DefaultTransport. -func NewRateLimitGate(base http.RoundTripper) *RateLimitGate { - if base == nil { - base = http.DefaultTransport - } - return &RateLimitGate{ - base: base, - now: time.Now, - holds: make(map[string]*hostHold), - } -} - -// RoundTrip implements http.RoundTripper. -func (g *RateLimitGate) RoundTrip(req *http.Request) (*http.Response, error) { - host := hostKey(req.URL) - if until, held := g.heldUntil(host); held { - return nil, &HeldError{Host: host, Until: until, RetryAfter: until.Sub(g.now())} - } - - resp, err := g.base.RoundTrip(req) - if err != nil { - return nil, err - } - - retryAfter := resp.Header.Get("Retry-After") - switch { - case resp.StatusCode == http.StatusTooManyRequests, - resp.StatusCode == http.StatusServiceUnavailable && retryAfter != "": - g.hold(host, retryAfter) - default: - g.release(host) - } - return resp, nil -} - -// Held returns a *HeldError when requests to rawURL's host are being held, -// and nil otherwise. It is safe to call on a nil gate. -func (g *RateLimitGate) Held(rawURL string) error { - if g == nil { - return nil - } - u, err := url.Parse(rawURL) - if err != nil { - return nil - } - host := hostKey(u) - until, held := g.heldUntil(host) - if !held { - return nil - } - return &HeldError{Host: host, Until: until, RetryAfter: until.Sub(g.now())} -} - -// Holds returns the hosts currently held and when each hold ends. It is -// safe to call on a nil gate. -func (g *RateLimitGate) Holds() map[string]time.Time { - if g == nil { - return nil - } - now := g.now() - g.mu.Lock() - defer g.mu.Unlock() - held := make(map[string]time.Time) - for host, h := range g.holds { - if now.Before(h.until) { - held[host] = h.until - } - } - return held -} - -func (g *RateLimitGate) heldUntil(host string) (time.Time, bool) { - now := g.now() - g.mu.Lock() - defer g.mu.Unlock() - h, ok := g.holds[host] - if !ok || !now.Before(h.until) { - return time.Time{}, false - } - return h.until, true -} - -func (g *RateLimitGate) hold(host, header string) { - now := g.now() - - g.mu.Lock() - h, ok := g.holds[host] - if !ok { - h = &hostHold{} - g.holds[host] = h - } - // Responses already in flight when a hold starts all report the same - // rate limiting; only a 429 after the hold ran out is a further strike. - if !now.Before(h.until) { - h.strikes++ - } - d, fromHeader := parseRetryAfter(header, now) - if !fromHeader { - d = min(holdWithoutHeader<<(h.strikes-1), maxHoldWithoutHeader) - } - d = min(d, maxHold) - until := now.Add(d) - if until.After(h.until) { - h.until = until - } - g.mu.Unlock() - - if g.Logger != nil { - source := "default" - if fromHeader { - source = "retry-after" - } - g.Logger.Warn("upstream rate limited this proxy, holding its requests", - "host", host, "hold", d.Round(time.Second).String(), "source", source, - "until", until.UTC().Format(time.RFC3339)) - } - if g.OnHold != nil { - g.OnHold(host) - } -} - -// release drops host's hold once it has ended. Nothing is sent during a -// hold, so a success arriving while one is active was sent before it began. -func (g *RateLimitGate) release(host string) { - now := g.now() - g.mu.Lock() - h, had := g.holds[host] - if had && now.Before(h.until) { - g.mu.Unlock() - return - } - delete(g.holds, host) - g.mu.Unlock() - if had && g.Logger != nil { - g.Logger.Info("upstream rate limit cleared", "host", host) - } -} - -// parseRetryAfter reads a Retry-After value as delay seconds or an HTTP -// date. It reports false for anything unusable, including a zero or past -// value, so the caller applies its default instead. -func parseRetryAfter(value string, now time.Time) (time.Duration, bool) { - value = strings.TrimSpace(value) - if value == "" { - return 0, false - } - if secs, err := strconv.ParseInt(value, 10, 64); err == nil { - if secs <= 0 { - return 0, false - } - return time.Duration(secs) * time.Second, true - } - at, err := http.ParseTime(value) - if err != nil || !at.After(now) { - return 0, false - } - return at.Sub(now), true -} - -func hostKey(u *url.URL) string { - if u == nil { - return "" - } - return strings.ToLower(u.Host) -} diff --git a/internal/httpclient/ratelimit_test.go b/internal/httpclient/ratelimit_test.go deleted file mode 100644 index 8ad7e9c..0000000 --- a/internal/httpclient/ratelimit_test.go +++ /dev/null @@ -1,257 +0,0 @@ -package httpclient - -import ( - "errors" - "io" - "net/http" - "strings" - "testing" - "time" -) - -type gateHarness struct { - gate *RateLimitGate - now time.Time - calls map[string]int - // respond returns the status and Retry-After the fake upstream answers with. - respond func(req *http.Request) (int, string) -} - -func newGateHarness() *gateHarness { - h := &gateHarness{ - now: time.Date(2026, 10, 14, 12, 0, 0, 0, time.UTC), - calls: map[string]int{}, - respond: func(*http.Request) (int, string) { - return http.StatusOK, "" - }, - } - h.gate = NewRateLimitGate(roundTripperFunc(func(req *http.Request) (*http.Response, error) { - h.calls[req.URL.Host]++ - status, retryAfter := h.respond(req) - resp := &http.Response{ - StatusCode: status, - Header: http.Header{}, - Body: io.NopCloser(strings.NewReader("")), - Request: req, - } - if retryAfter != "" { - resp.Header.Set("Retry-After", retryAfter) - } - return resp, nil - })) - h.gate.now = func() time.Time { return h.now } - return h -} - -func (h *gateHarness) get(t *testing.T, rawURL string) (*http.Response, error) { - t.Helper() - req, err := http.NewRequest(http.MethodGet, rawURL, nil) - if err != nil { - t.Fatalf("building request: %v", err) - } - resp, err := h.gate.RoundTrip(req) - if resp != nil { - _ = resp.Body.Close() - } - return resp, err -} - -func TestRateLimitGate_HoldsHostForRetryAfter(t *testing.T) { - h := newGateHarness() - h.respond = func(*http.Request) (int, string) { return http.StatusTooManyRequests, "785" } - - resp, err := h.get(t, "https://repo1.maven.org/maven2/a/b/maven-metadata.xml") - if err != nil || resp.StatusCode != http.StatusTooManyRequests { - t.Fatalf("first request: resp=%v err=%v, want the upstream 429 passed through", resp, err) - } - - h.respond = func(*http.Request) (int, string) { return http.StatusOK, "" } - _, err = h.get(t, "https://repo1.maven.org/maven2/a/b/b-1.0.jar") - var held *HeldError - if !errors.As(err, &held) || !errors.Is(err, ErrUpstreamHeld) { - t.Fatalf("second request error = %v, want *HeldError", err) - } - if held.Host != "repo1.maven.org" || held.RetryAfter != 785*time.Second { - t.Errorf("held = %+v, want repo1.maven.org for 785s", held) - } - if h.calls["repo1.maven.org"] != 1 { - t.Errorf("upstream calls = %d, want 1: held requests must not reach it", h.calls["repo1.maven.org"]) - } - - if _, err := h.get(t, "https://plugins.gradle.org/m2/x.pom"); err != nil { - t.Errorf("other host: %v, want unaffected", err) - } - - holds := h.gate.Holds() - if until, ok := holds["repo1.maven.org"]; !ok || !until.Equal(h.now.Add(785*time.Second)) { - t.Errorf("Holds() = %v, want repo1.maven.org until now+785s", holds) - } -} - -func TestRateLimitGate_ReleasesWhenHoldExpiresAndUpstreamServes(t *testing.T) { - h := newGateHarness() - h.respond = func(*http.Request) (int, string) { return http.StatusTooManyRequests, "60" } - _, _ = h.get(t, "https://repo1.maven.org/x") - - h.now = h.now.Add(59 * time.Second) - if _, err := h.get(t, "https://repo1.maven.org/x"); !errors.Is(err, ErrUpstreamHeld) { - t.Fatalf("at 59s: err = %v, want still held", err) - } - - h.now = h.now.Add(time.Second) - h.respond = func(*http.Request) (int, string) { return http.StatusOK, "" } - resp, err := h.get(t, "https://repo1.maven.org/x") - if err != nil || resp.StatusCode != http.StatusOK { - t.Fatalf("at 60s: resp=%v err=%v, want the request sent", resp, err) - } - if len(h.gate.Holds()) != 0 { - t.Errorf("Holds() = %v after a success, want empty", h.gate.Holds()) - } -} - -func TestRateLimitGate_LateSuccessKeepsHold(t *testing.T) { - h := newGateHarness() - // The slow request is in flight when the limited one starts the hold, - // and its 200 arrives afterwards. - h.respond = func(req *http.Request) (int, string) { - if req.URL.Path == "/limited" { - return http.StatusTooManyRequests, "120" - } - _, _ = h.get(t, "https://repo1.maven.org/limited") - return http.StatusOK, "" - } - - resp, err := h.get(t, "https://repo1.maven.org/slow") - if err != nil || resp.StatusCode != http.StatusOK { - t.Fatalf("slow request: resp=%v err=%v, want its 200", resp, err) - } - if until, ok := h.gate.Holds()["repo1.maven.org"]; !ok || !until.Equal(h.now.Add(120*time.Second)) { - t.Fatalf("Holds() = %v after a late 200, want repo1.maven.org until now+120s", h.gate.Holds()) - } - if _, err := h.get(t, "https://repo1.maven.org/x"); !errors.Is(err, ErrUpstreamHeld) { - t.Errorf("next request err = %v, want held", err) - } -} - -func TestRateLimitGate_Held(t *testing.T) { - var nilGate *RateLimitGate - if err := nilGate.Held("https://repo1.maven.org/x"); err != nil { - t.Errorf("nil gate Held() = %v, want nil", err) - } - - h := newGateHarness() - if err := h.gate.Held("https://repo1.maven.org/x"); err != nil { - t.Errorf("before any limit: Held() = %v, want nil", err) - } - h.respond = func(*http.Request) (int, string) { return http.StatusTooManyRequests, "45" } - _, _ = h.get(t, "https://repo1.maven.org/x") - - var held *HeldError - if err := h.gate.Held("https://REPO1.maven.org/a/b.jar"); !errors.As(err, &held) || held.RetryAfter != 45*time.Second { - t.Errorf("Held() = %v, want *HeldError for 45s", err) - } - if err := h.gate.Held("https://plugins.gradle.org/m2/x.pom"); err != nil { - t.Errorf("other host: Held() = %v, want nil", err) - } - h.now = h.now.Add(45 * time.Second) - if err := h.gate.Held("https://repo1.maven.org/x"); err != nil { - t.Errorf("after the hold: Held() = %v, want nil", err) - } -} - -func TestRateLimitGate_DefaultHoldDoublesWithoutHeader(t *testing.T) { - h := newGateHarness() - h.respond = func(*http.Request) (int, string) { return http.StatusTooManyRequests, "" } - - wantHolds := []time.Duration{time.Minute, 2 * time.Minute, 4 * time.Minute} - for i, want := range wantHolds { - _, _ = h.get(t, "https://repo1.maven.org/x") - until := h.gate.Holds()["repo1.maven.org"] - if got := until.Sub(h.now); got != want { - t.Errorf("strike %d: hold = %s, want %s", i+1, got, want) - } - h.now = until - } -} - -func TestRateLimitGate_ConcurrentLimitedResponsesAreOneStrike(t *testing.T) { - h := newGateHarness() - h.respond = func(*http.Request) (int, string) { return http.StatusTooManyRequests, "" } - - // Two requests already in flight when the limit starts both see 429. - // The gate only consults the hold before sending, so drive the base - // directly for the second. - _, _ = h.get(t, "https://repo1.maven.org/x") - req, _ := http.NewRequest(http.MethodGet, "https://repo1.maven.org/y", nil) - h.gate.hold(hostKey(req.URL), "") - - if got := h.gate.Holds()["repo1.maven.org"].Sub(h.now); got != time.Minute { - t.Errorf("hold = %s after two concurrent 429s, want the single-strike %s", got, time.Minute) - } -} - -func TestRateLimitGate_CapsUpstreamRetryAfter(t *testing.T) { - h := newGateHarness() - h.respond = func(*http.Request) (int, string) { return http.StatusTooManyRequests, "86400" } - _, _ = h.get(t, "https://repo1.maven.org/x") - if got := h.gate.Holds()["repo1.maven.org"].Sub(h.now); got != maxHold { - t.Errorf("hold = %s, want capped at %s", got, maxHold) - } -} - -func TestRateLimitGate_ServiceUnavailableNeedsRetryAfter(t *testing.T) { - h := newGateHarness() - h.respond = func(*http.Request) (int, string) { return http.StatusServiceUnavailable, "" } - _, _ = h.get(t, "https://repo1.maven.org/x") - if len(h.gate.Holds()) != 0 { - t.Errorf("503 without Retry-After created hold %v, want none", h.gate.Holds()) - } - - h.respond = func(*http.Request) (int, string) { return http.StatusServiceUnavailable, "30" } - _, _ = h.get(t, "https://repo1.maven.org/x") - if got := h.gate.Holds()["repo1.maven.org"].Sub(h.now); got != 30*time.Second { - t.Errorf("hold = %s, want 30s from the 503's Retry-After", got) - } -} - -func TestRateLimitGate_ReportsHoldsAndNilSafe(t *testing.T) { - var nilGate *RateLimitGate - if holds := nilGate.Holds(); holds != nil { - t.Errorf("nil gate Holds() = %v, want nil", holds) - } - - h := newGateHarness() - var reported []string - h.gate.OnHold = func(host string) { - reported = append(reported, host) - } - h.respond = func(*http.Request) (int, string) { return http.StatusTooManyRequests, "10" } - _, _ = h.get(t, "https://repo1.maven.org/x") - if len(reported) != 1 || reported[0] != "repo1.maven.org" { - t.Errorf("OnHold reported %v, want [repo1.maven.org]", reported) - } -} - -func TestParseRetryAfter(t *testing.T) { - now := time.Date(2026, 10, 14, 12, 0, 0, 0, time.UTC) - tests := []struct { - value string - want time.Duration - ok bool - }{ - {"785", 785 * time.Second, true}, - {" 30 ", 30 * time.Second, true}, - {"0", 0, false}, - {"-5", 0, false}, - {"", 0, false}, - {"soon", 0, false}, - {now.Add(90 * time.Second).Format(http.TimeFormat), 90 * time.Second, true}, - {now.Add(-time.Minute).Format(http.TimeFormat), 0, false}, - } - for _, tt := range tests { - got, ok := parseRetryAfter(tt.value, now) - if got != tt.want || ok != tt.ok { - t.Errorf("parseRetryAfter(%q) = (%s, %v), want (%s, %v)", tt.value, got, ok, tt.want, tt.ok) - } - } -} diff --git a/internal/server/breakers_test.go b/internal/server/breakers_test.go index bb5166e..3a9e764 100644 --- a/internal/server/breakers_test.go +++ b/internal/server/breakers_test.go @@ -12,8 +12,8 @@ import ( "testing" "time" - upstreamhttp "github.com/git-pkgs/proxy/internal/httpclient" "github.com/git-pkgs/proxy/internal/metrics" + registryclient "github.com/git-pkgs/registries/client" "github.com/git-pkgs/registries/fetch" "github.com/prometheus/client_golang/prometheus" dto "github.com/prometheus/client_model/go" @@ -352,7 +352,7 @@ func TestHealthEndpoint_ReportsUpstreamHolds(t *testing.T) { ts := newTestServer(t) defer ts.close() - gate := upstreamhttp.NewRateLimitGate(roundTripperFunc(func(req *http.Request) (*http.Response, error) { + gate := registryclient.NewRateLimitGate(roundTripperFunc(func(req *http.Request) (*http.Response, error) { return &http.Response{ StatusCode: http.StatusTooManyRequests, Header: http.Header{"Retry-After": []string{"600"}}, diff --git a/internal/server/registry_policies_test.go b/internal/server/registry_policies_test.go new file mode 100644 index 0000000..881356a --- /dev/null +++ b/internal/server/registry_policies_test.go @@ -0,0 +1,79 @@ +package server + +import ( + "context" + "errors" + "fmt" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + "time" + + "github.com/git-pkgs/proxy/internal/config" + registryclient "github.com/git-pkgs/registries/client" + "github.com/git-pkgs/registries/fetch" +) + +func TestMirrorProxySharesHoldAfterBearerRetry(t *testing.T) { + var calls atomic.Int32 + var upstream *httptest.Server + upstream = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls.Add(1) + if r.URL.Path == "/token" { + _, _ = io.WriteString(w, `{"token":"fixture-token"}`) + return + } + if r.Header.Get("Authorization") != "Bearer fixture-token" { + w.Header().Set("WWW-Authenticate", fmt.Sprintf(`Bearer realm="%s/token",service="test",scope="repository:test:pull"`, upstream.URL)) + w.WriteHeader(http.StatusUnauthorized) + return + } + w.Header().Set("Retry-After", "9223372036854775807") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer upstream.Close() + cfg := config.Default() + cfg.Upstream.AllowLoopback = true + proxy, _ := NewMirrorProxy(cfg, nil, nil, slog.New(slog.NewTextHandler(io.Discard, nil))) + response, err := proxy.HTTPClient.Get(upstream.URL + "/v2/test/manifests/latest") + if err != nil { + t.Fatal(err) + } + _ = response.Body.Close() + if response.StatusCode != http.StatusTooManyRequests || calls.Load() != 3 { + t.Fatalf("bearer retry: status %d, calls %d", response.StatusCode, calls.Load()) + } + _, err = proxy.Fetcher.Fetch(context.Background(), upstream.URL+"/v2/test/blobs/content") + if !errors.Is(err, registryclient.ErrUpstreamHeld) { + t.Fatalf("artifact fetch = %v", err) + } + metadata := registryclient.NewClient(registryclient.WithHTTPClient(proxy.HTTPClient)) + metadata.BaseDelay = time.Hour + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + _, err = metadata.GetBody(ctx, upstream.URL+"/metadata") + if !errors.Is(err, registryclient.ErrUpstreamHeld) || calls.Load() != 3 { + t.Fatalf("metadata fetch = %v, upstream calls %d", err, calls.Load()) + } + var held *registryclient.HeldError + if !errors.As(proxy.RateLimits.Held(upstream.URL), &held) || held.RetryAfter < 59*time.Minute || len(proxy.RateLimits.Holds()) != 1 { + t.Fatalf("shared hold = %+v", held) + } +} + +func TestMirrorProxyAppliesArtifactSizeOption(t *testing.T) { + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = io.WriteString(w, "12345") + })) + defer upstream.Close() + cfg := config.Default() + cfg.Upstream.AllowLoopback = true + cfg.ArtifactMaxSize = "4" + proxy, _ := NewMirrorProxy(cfg, nil, nil, slog.New(slog.NewTextHandler(io.Discard, nil))) + if _, err := proxy.Fetcher.Fetch(context.Background(), upstream.URL+"/artifact"); !errors.Is(err, fetch.ErrArtifactTooLarge) { + t.Fatalf("Fetch = %v", err) + } +} diff --git a/internal/server/server.go b/internal/server/server.go index 5921cf0..f579002 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -82,6 +82,7 @@ import ( "github.com/git-pkgs/proxy/internal/packageurl" "github.com/git-pkgs/proxy/internal/scanner" "github.com/git-pkgs/proxy/internal/storage" + registryclient "github.com/git-pkgs/registries/client" "github.com/git-pkgs/registries/fetch" "github.com/git-pkgs/registries/safehttp" "github.com/git-pkgs/spdx" @@ -117,7 +118,7 @@ type Server struct { accessLog *accesslog.Logger ecr *ecrTokens breakers *breakerMonitor - rateLimits *upstreamhttp.RateLimitGate + rateLimits *registryclient.RateLimitGate ecoStats ecosystemStatsCache sources sourceTracker scheduler *mirror.Scheduler @@ -228,7 +229,7 @@ func (s *Server) serve(listener net.Listener) error { metadataClient, artifactClient := s.newUpstreamClients(s.cfg.ParseArtifactTimeout()) // Create shared components with circuit breaker. - baseFetcher := fetch.NewFetcher(fetch.WithHTTPClient(artifactClient)) + baseFetcher := fetch.NewFetcher(fetch.WithHTTPClient(artifactClient), fetch.WithMaxArtifactSize(s.cfg.ParseArtifactMaxSize())) fetcher := fetch.NewCircuitBreakerFetcher(baseFetcher) s.breakers = newBreakerMonitor(fetcher, s.logger) resolver := fetch.NewResolver() @@ -237,7 +238,7 @@ func (s *Server) serve(listener net.Listener) error { Ecosystems: s.cfg.Cooldown.Ecosystems, Packages: s.cfg.Cooldown.NormalizedPackages(), } - proxy := handler.NewProxy(s.db, s.storage, handler.LimitArtifactSize(fetcher, s.cfg.ParseArtifactMaxSize()), resolver, s.logger) + proxy := handler.NewProxy(s.db, s.storage, fetcher, resolver, s.logger) proxy.HTTPClient = metadataClient proxy.RateLimits = s.rateLimits cooldownPolicy, err := cooldownpolicy.New(cd, s.cfg.Cooldown.PackagePatterns) @@ -504,9 +505,24 @@ func (s *Server) newUpstreamClients(artifactTimeout time.Duration) (metadataClie } // The gate sits under the auth transport so it sees every final upstream // answer, including the retry after a bearer challenge. - gate := upstreamhttp.NewRateLimitGate(baseTransport) - gate.Logger = s.logger - gate.OnHold = metrics.RecordUpstreamRateLimitHold + gate := registryclient.NewRateLimitGate(baseTransport) + gate.OnHold = func(event registryclient.HoldEvent) { + metrics.RecordUpstreamRateLimitHold(event.Host) + if s.logger != nil { + source := "default" + if event.FromHeader { + source = "retry-after" + } + s.logger.Warn("upstream rate limited this proxy, holding its requests", + "host", event.Host, "hold", event.RetryAfter.Round(time.Second).String(), "source", source, + "until", event.Until.UTC().Format(time.RFC3339)) + } + } + gate.OnRelease = func(host string) { + if s.logger != nil { + s.logger.Info("upstream rate limit cleared", "host", host) + } + } s.rateLimits = gate authTransport := upstreamhttp.NewTransport(gate, upstreamhttp.AuthFunc(s.authForURL)) metadata := *safeClient @@ -524,7 +540,7 @@ func (s *Server) newUpstreamClients(artifactTimeout time.Duration) (metadataClie func NewMirrorProxy(cfg *config.Config, db *database.DB, store storage.Storage, logger *slog.Logger) (*handler.Proxy, *handler.ContainerHandler) { s := &Server{cfg: cfg, db: db, storage: store, logger: logger, ecr: newECRTokens(logger)} metadataClient, artifactClient := s.newUpstreamClients(0) - fetcher := handler.LimitArtifactSize(fetch.NewFetcher(fetch.WithHTTPClient(artifactClient)), cfg.ParseArtifactMaxSize()) + fetcher := fetch.NewFetcher(fetch.WithHTTPClient(artifactClient), fetch.WithMaxArtifactSize(cfg.ParseArtifactMaxSize())) proxy := handler.NewProxy(db, store, fetcher, fetch.NewResolver(), logger) proxy.HTTPClient = metadataClient proxy.RateLimits = s.rateLimits From 6d771a8d3aecfab94c60438ef609f38797aa48d0 Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Tue, 20 Oct 2026 08:53:22 +0100 Subject: [PATCH 321/633] Use shared cooldown package-pattern policy Use cooldown.NewPolicy for package-pattern overrides and remove the duplicate local implementation. --- go.mod | 2 +- go.sum | 4 +- internal/cooldownpolicy/policy.go | 114 ----------- internal/cooldownpolicy/policy_test.go | 223 --------------------- internal/handler/cooldown_patterns_test.go | 5 +- internal/server/server.go | 3 +- 6 files changed, 6 insertions(+), 345 deletions(-) delete mode 100644 internal/cooldownpolicy/policy.go delete mode 100644 internal/cooldownpolicy/policy_test.go diff --git a/go.mod b/go.mod index 395a436..d6d42bc 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/ecr v1.66.2 github.com/git-pkgs/archives v0.8.1 github.com/git-pkgs/artifacts v0.2.1 - github.com/git-pkgs/cooldown v0.2.0 + github.com/git-pkgs/cooldown v0.3.0 github.com/git-pkgs/enrichment v0.7.2 github.com/git-pkgs/gcs v0.1.0 github.com/git-pkgs/integrity v0.2.0 diff --git a/go.sum b/go.sum index 4d4c37b..cf4b97f 100644 --- a/go.sum +++ b/go.sum @@ -129,8 +129,8 @@ github.com/git-pkgs/archives v0.8.1 h1:IcT1bVsAec8SqJ4m9vTnD644KiadQZ4ERrWnm/30t github.com/git-pkgs/archives v0.8.1/go.mod h1:gIyPXPQr5e+YrMqA5KQ7x6pLrFXHNF1+CjdedNJdGUU= github.com/git-pkgs/artifacts v0.2.1 h1:VwdxR4yTDaqBZ34h0slxQBVyr2Xfa+QGOKCKviMx2xk= github.com/git-pkgs/artifacts v0.2.1/go.mod h1:Otosgq52pXT5UNN7lh6s/lszpWKVDO8XrOjN1M70/IA= -github.com/git-pkgs/cooldown v0.2.0 h1:0MWPHtkzZgvCR0wdiQeyvMea/dxgw9tParH1zzaFopc= -github.com/git-pkgs/cooldown v0.2.0/go.mod h1:v7APuK/UouTiu8mWQZbdDmj7DfxxkGUeuhjaRB5gv9E= +github.com/git-pkgs/cooldown v0.3.0 h1:eAl2CQFwqXxluoVP9hHmropAf1VoV3gSxLMkN+NEu8A= +github.com/git-pkgs/cooldown v0.3.0/go.mod h1:FnTg8wufEALATC9Sm/9AHjLdA+u9eVn96e9BqXXjGB0= github.com/git-pkgs/enrichment v0.7.2 h1:flD/M89HxkOegUi6tC8EOspFiIMsl7m0o576IQKOp48= github.com/git-pkgs/enrichment v0.7.2/go.mod h1:xGyc/iz7u1UnbrOHk8lIX5ZaNF3mKFDEZc3VqBJ/LOk= github.com/git-pkgs/gcs v0.1.0 h1:E3awGtsO0xZyHT9FUfEwMHjMkRxw41Bh+c7lgTmPvBo= diff --git a/internal/cooldownpolicy/policy.go b/internal/cooldownpolicy/policy.go deleted file mode 100644 index 096c4a4..0000000 --- a/internal/cooldownpolicy/policy.go +++ /dev/null @@ -1,114 +0,0 @@ -// Package cooldownpolicy applies package-pattern overrides to cooldown checks. -package cooldownpolicy - -import ( - "fmt" - "path" - "sort" - "strings" - "time" - - "github.com/git-pkgs/cooldown" -) - -// Policy applies exact PURL overrides before package-pattern overrides. -type Policy struct { - base *cooldown.Config - patterns []pattern - enabled bool -} - -type pattern struct { - glob string - duration time.Duration - config *cooldown.Config -} - -// New creates a Policy using the supplied exact and pattern overrides. -func New(base *cooldown.Config, packagePatterns map[string]string) (*Policy, error) { - if base == nil { - base = &cooldown.Config{} - } - - keys := make([]string, 0, len(packagePatterns)) - for glob := range packagePatterns { - keys = append(keys, glob) - } - sort.Strings(keys) - patterns := make([]pattern, 0, len(packagePatterns)) - seen := make(map[string]pattern) - enabled := base.Enabled() - for _, glob := range keys { - value := packagePatterns[glob] - if strings.ContainsAny(glob, "[\\") { - return nil, fmt.Errorf("invalid cooldown package pattern %q: character classes and escapes are not supported", glob) - } - canonicalGlob := strings.ReplaceAll(glob, "@", "%40") - if _, err := path.Match(canonicalGlob, ""); err != nil { - return nil, fmt.Errorf("invalid cooldown package pattern %q: %w", glob, err) - } - duration, err := cooldown.ParseDuration(value) - if err != nil { - return nil, fmt.Errorf("invalid cooldown duration for package pattern %q: %w", glob, err) - } - if previous, exists := seen[canonicalGlob]; exists { - if previous.duration != duration { - return nil, fmt.Errorf("conflicting cooldown package patterns %q and %q", previous.glob, glob) - } - continue - } - seen[canonicalGlob] = pattern{glob: glob, duration: duration} - config := &cooldown.Config{Default: value} - enabled = config.Enabled() || enabled - patterns = append(patterns, pattern{glob: canonicalGlob, duration: duration, config: config}) - } - sort.Slice(patterns, func(i, j int) bool { - left, right := literalLength(patterns[i].glob), literalLength(patterns[j].glob) - if left != right { - return left > right - } - return patterns[i].glob < patterns[j].glob - }) - - return &Policy{base: base, patterns: patterns, enabled: enabled}, nil -} - -func literalLength(glob string) int { - return len(glob) - strings.Count(glob, "*") - strings.Count(glob, "?") -} - -// For returns the duration, with exact overrides taking precedence over patterns. -func (p *Policy) For(ecosystem, packagePURL string) time.Duration { - return p.configFor(packagePURL).For(ecosystem, packagePURL) -} - -func (p *Policy) configFor(packagePURL string) *cooldown.Config { - if _, exact := p.base.Packages[packagePURL]; exact { - return p.base - } - - for _, candidate := range p.patterns { - matched, _ := path.Match(candidate.glob, packagePURL) - if !matched { - continue - } - return candidate.config - } - - return p.base -} - -// IsAllowed reports whether the package version has completed its cooldown. -func (p *Policy) IsAllowed(ecosystem, packagePURL string, publishedAt time.Time) bool { - return p.Evaluate(ecosystem, packagePURL, publishedAt, time.Now()).Allowed -} - -// Evaluate returns the cooldown decision at the supplied evaluation time. -func (p *Policy) Evaluate(ecosystem, packagePURL string, publishedAt, evaluatedAt time.Time) cooldown.Decision { - return p.configFor(packagePURL).Evaluate(ecosystem, packagePURL, publishedAt, evaluatedAt) -} - -// Enabled reports whether any configured cooldown can filter a package version. -func (p *Policy) Enabled() bool { - return p.enabled -} diff --git a/internal/cooldownpolicy/policy_test.go b/internal/cooldownpolicy/policy_test.go deleted file mode 100644 index 9e956c1..0000000 --- a/internal/cooldownpolicy/policy_test.go +++ /dev/null @@ -1,223 +0,0 @@ -package cooldownpolicy - -import ( - "strings" - "testing" - "time" - - "github.com/git-pkgs/cooldown" -) - -func TestPatternOverride(t *testing.T) { - policy, err := New(&cooldown.Config{ - Default: "7d", - Ecosystems: map[string]string{"npm": "7d"}, - }, map[string]string{ - "pkg:npm/@example/*": "0", - }) - if err != nil { - t.Fatalf("New returned error: %v", err) - } - - if !policy.IsAllowed("npm", "pkg:npm/%40example/widget", time.Now()) { - t.Fatal("matching package pattern should disable cooldown") - } - if policy.IsAllowed("npm", "pkg:npm/public-package", time.Now()) { - t.Fatal("non-matching package should use ecosystem cooldown") - } -} - -func TestExactOverrideTakesPrecedenceOverPattern(t *testing.T) { - purl := "pkg:npm/%40example/widget" - policy, err := New(&cooldown.Config{ - Default: "7d", - Packages: map[string]string{purl: "2d"}, - }, map[string]string{ - "pkg:npm/@example/*": "0", - }) - if err != nil { - t.Fatalf("New returned error: %v", err) - } - - if policy.IsAllowed("npm", purl, time.Now()) { - t.Fatal("exact package override should take precedence over pattern") - } -} - -func TestMoreSpecificPatternTakesPrecedence(t *testing.T) { - policy, err := New(&cooldown.Config{Default: "7d"}, map[string]string{ - "pkg:npm/@example/*": "0", - "pkg:npm/@example/critical": "2d", - }) - if err != nil { - t.Fatalf("New returned error: %v", err) - } - - if policy.IsAllowed("npm", "pkg:npm/%40example/critical", time.Now()) { - t.Fatal("more specific pattern should take precedence") - } -} - -func TestNewRejectsInvalidPattern(t *testing.T) { - for _, glob := range []string{"pkg:npm/[", "pkg:npm/[@a]*", "pkg:npm/[abcdef]*", `pkg:npm/\*`, `pkg:npm/\@example/*`} { - t.Run(glob, func(t *testing.T) { - if _, err := New(nil, map[string]string{glob: "0"}); err == nil || !strings.Contains(err.Error(), "character classes and escapes are not supported") { - t.Fatalf("unsupported pattern %q: error = %v", glob, err) - } - }) - } -} - -func TestQuestionMarkPattern(t *testing.T) { - policy, err := New(&cooldown.Config{Default: "7d"}, map[string]string{"pkg:npm/@example/widget-?": "0"}) - if err != nil { - t.Fatal(err) - } - for _, tc := range []struct { - purl string - want time.Duration - }{ - {"pkg:npm/%40example/widget-a", 0}, - {"pkg:npm/%40example/widget-ab", 7 * 24 * time.Hour}, - {"pkg:npm/%40example/widget-/", 7 * 24 * time.Hour}, - } { - if got := policy.For("npm", tc.purl); got != tc.want { - t.Errorf("For(%q) = %s, want %s", tc.purl, got, tc.want) - } - } -} - -func TestNormalizedPatternCollisions(t *testing.T) { - for _, duration := range []string{"0", "24h"} { - t.Run(duration, func(t *testing.T) { - for range 20 { - policy, err := New(nil, map[string]string{ - "pkg:npm/@example/*": "1d", - "pkg:npm/%40example/*": duration, - }) - if duration == "0" { - if err == nil || !strings.Contains(err.Error(), `conflicting cooldown package patterns "pkg:npm/%40example/*" and "pkg:npm/@example/*"`) { - t.Fatalf("conflict error = %v", err) - } - continue - } - if err != nil { - t.Fatal(err) - } - if got := policy.For("npm", "pkg:npm/%40example/widget"); got != 24*time.Hour { - t.Fatalf("duration = %s", got) - } - } - }) - } -} - -func TestForPrecedence(t *testing.T) { - policy, err := New(&cooldown.Config{ - Default: "1h", - Ecosystems: map[string]string{"npm": "2h"}, - Packages: map[string]string{"pkg:npm/%40example/exact": "0"}, - }, map[string]string{ - "pkg:npm/@example/*": "3h", - "pkg:npm/@example/specific*": "4h", - "pkg:npm/@example/ab*": "5h", - "pkg:npm/@example/a*c": "6h", - }) - if err != nil { - t.Fatal(err) - } - for _, tc := range []struct { - ecosystem, purl string - want time.Duration - }{ - {"npm", "pkg:npm/%40example/exact", 0}, - {"npm", "pkg:npm/%40example/widget", 3 * time.Hour}, - {"npm", "pkg:npm/%40example/specific-widget", 4 * time.Hour}, - {"npm", "pkg:npm/%40example/abc", 6 * time.Hour}, - {"npm", "pkg:npm/other", 2 * time.Hour}, - {"npm", "pkg:npm/%40example/nested/package", 2 * time.Hour}, - {"cargo", "pkg:cargo/serde", time.Hour}, - } { - t.Run(tc.purl, func(t *testing.T) { - if got := policy.For(tc.ecosystem, tc.purl); got != tc.want { - t.Errorf("For = %s, want %s", got, tc.want) - } - }) - } -} - -func TestPatternOnlyPolicy(t *testing.T) { - policy, err := New(nil, map[string]string{"pkg:nuget/example.*": "24h"}) - if err != nil { - t.Fatal(err) - } - if !policy.Enabled() { - t.Fatal("pattern-only policy should be enabled") - } - for _, tc := range []struct { - published time.Time - allowed bool - }{ - {time.Now().Add(-time.Hour), false}, - {time.Now().Add(-48 * time.Hour), true}, - {time.Time{}, true}, - } { - if got := policy.IsAllowed("nuget", "pkg:nuget/example.widget", tc.published); got != tc.allowed { - t.Errorf("published=%s: allowed=%t, want %t", tc.published, got, tc.allowed) - } - } - if !policy.IsAllowed("nuget", "pkg:nuget/other", time.Now()) { - t.Error("unmatched package should have no cooldown") - } -} - -func TestDisabledPolicy(t *testing.T) { - for _, patterns := range []map[string]string{nil, {"pkg:npm/@example/*": "0"}} { - policy, err := New(nil, patterns) - if err != nil { - t.Fatal(err) - } - if policy.Enabled() { - t.Error("zero cooldown should be disabled") - } - } -} - -func TestNewRejectsInvalidDuration(t *testing.T) { - if _, err := New(nil, map[string]string{"pkg:npm/*": "invalid"}); err == nil { - t.Fatal("invalid duration accepted") - } -} - -func TestEvaluate(t *testing.T) { - policy, err := New(&cooldown.Config{ - Default: "48h", - Ecosystems: map[string]string{"npm": "72h"}, - Packages: map[string]string{"pkg:npm/%40example/exact": "0"}, - }, map[string]string{"pkg:npm/@example/*": "7d"}) - if err != nil { - t.Fatal(err) - } - published := time.Date(2026, time.October, 1, 0, 0, 0, 0, time.UTC) - eligible := published.Add(7 * 24 * time.Hour) - for _, tc := range []struct { - name, ecosystem, purl string - published, at time.Time - want cooldown.Decision - }{ - {"waiting", "npm", "pkg:npm/%40example/widget", published, eligible.Add(-time.Nanosecond), cooldown.Decision{Cooldown: 7 * 24 * time.Hour, AvailableAt: eligible, Reason: cooldown.ReasonWaiting}}, - {"boundary", "npm", "pkg:npm/%40example/widget", published, eligible, cooldown.Decision{Allowed: true, Cooldown: 7 * 24 * time.Hour, AvailableAt: eligible, Reason: cooldown.ReasonElapsed}}, - {"elapsed", "npm", "pkg:npm/%40example/widget", published, eligible.Add(time.Hour), cooldown.Decision{Allowed: true, Cooldown: 7 * 24 * time.Hour, AvailableAt: eligible, Reason: cooldown.ReasonElapsed}}, - {"unknown publication", "npm", "pkg:npm/%40example/widget", time.Time{}, eligible, cooldown.Decision{Allowed: true, Cooldown: 7 * 24 * time.Hour, Reason: cooldown.ReasonUnknownPublicationTime}}, - {"exact exemption", "npm", "pkg:npm/%40example/exact", published, published, cooldown.Decision{Allowed: true, AvailableAt: published, Reason: cooldown.ReasonDisabled}}, - {"exemption without publication", "npm", "pkg:npm/%40example/exact", time.Time{}, eligible, cooldown.Decision{Allowed: true, Reason: cooldown.ReasonUnknownPublicationTime}}, - {"ecosystem fallback", "npm", "pkg:npm/other", published, published, cooldown.Decision{Cooldown: 72 * time.Hour, AvailableAt: published.Add(72 * time.Hour), Reason: cooldown.ReasonWaiting}}, - {"global fallback", "cargo", "pkg:cargo/serde", published, published, cooldown.Decision{Cooldown: 48 * time.Hour, AvailableAt: published.Add(48 * time.Hour), Reason: cooldown.ReasonWaiting}}, - } { - t.Run(tc.name, func(t *testing.T) { - if got := policy.Evaluate(tc.ecosystem, tc.purl, tc.published, tc.at); got != tc.want { - t.Errorf("Evaluate = %+v, want %+v", got, tc.want) - } - }) - } -} diff --git a/internal/handler/cooldown_patterns_test.go b/internal/handler/cooldown_patterns_test.go index 83034cf..71fcc23 100644 --- a/internal/handler/cooldown_patterns_test.go +++ b/internal/handler/cooldown_patterns_test.go @@ -10,7 +10,6 @@ import ( "time" "github.com/git-pkgs/cooldown" - "github.com/git-pkgs/proxy/internal/cooldownpolicy" ) func TestNuGetCooldownPackagePatterns(t *testing.T) { @@ -26,7 +25,7 @@ func TestNuGetCooldownPackagePatterns(t *testing.T) { } { t.Run(tc.name, func(t *testing.T) { p, db, store, fetcher := setupTestProxy(t) - policy, err := cooldownpolicy.New(&cooldown.Config{Default: tc.defaultDuration, Packages: tc.exact}, map[string]string{"pkg:nuget/test*": tc.pattern}) + policy, err := cooldown.NewPolicy(&cooldown.Config{Default: tc.defaultDuration, Packages: tc.exact}, map[string]string{"pkg:nuget/test*": tc.pattern}) if err != nil { t.Fatal(err) } @@ -56,7 +55,7 @@ func TestNuGetCooldownPackagePatterns(t *testing.T) { func TestNPMCooldownPatternExemptionRespectsDenylist(t *testing.T) { p, db, store, fetcher := setupTestProxy(t) var err error - p.Cooldown, err = cooldownpolicy.New(&cooldown.Config{Default: "7d"}, map[string]string{"pkg:npm/@example/*": "0"}) + p.Cooldown, err = cooldown.NewPolicy(&cooldown.Config{Default: "7d"}, map[string]string{"pkg:npm/@example/*": "0"}) if err != nil { t.Fatal(err) } diff --git a/internal/server/server.go b/internal/server/server.go index f579002..25da77f 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -71,7 +71,6 @@ import ( swaggerdoc "github.com/git-pkgs/proxy/docs/swagger" "github.com/git-pkgs/proxy/internal/accesslog" "github.com/git-pkgs/proxy/internal/config" - "github.com/git-pkgs/proxy/internal/cooldownpolicy" "github.com/git-pkgs/proxy/internal/database" "github.com/git-pkgs/proxy/internal/denylist" "github.com/git-pkgs/proxy/internal/enrichment" @@ -241,7 +240,7 @@ func (s *Server) serve(listener net.Listener) error { proxy := handler.NewProxy(s.db, s.storage, fetcher, resolver, s.logger) proxy.HTTPClient = metadataClient proxy.RateLimits = s.rateLimits - cooldownPolicy, err := cooldownpolicy.New(cd, s.cfg.Cooldown.PackagePatterns) + cooldownPolicy, err := cooldown.NewPolicy(cd, s.cfg.Cooldown.PackagePatterns) if err != nil { return fmt.Errorf("configuring cooldown policy: %w", err) } From 5dfd01ea74fc04bfcfa436ec97bd216795a0aa9b Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Tue, 20 Oct 2026 09:14:09 +0100 Subject: [PATCH 322/633] Use shared sbom parser for mirror sources (#510) * Use shared sbom parser for mirror sources * Use sbom v0.1.7 release --- go.mod | 4 +- go.sum | 22 +------- internal/mirror/source.go | 78 +++----------------------- internal/mirror/source_formats_test.go | 74 ++++++++++++++++++++++++ internal/mirror/source_test.go | 2 - 5 files changed, 86 insertions(+), 94 deletions(-) create mode 100644 internal/mirror/source_formats_test.go diff --git a/go.mod b/go.mod index d6d42bc..a760f5b 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,6 @@ go 1.26.7 require ( github.com/BurntSushi/toml v1.6.0 - github.com/CycloneDX/cyclonedx-go v0.12.0 github.com/Masterminds/semver/v3 v3.5.0 github.com/aws/aws-sdk-go-v2/config v1.33.7 github.com/aws/aws-sdk-go-v2/service/ecr v1.66.2 @@ -17,6 +16,7 @@ require ( github.com/git-pkgs/magic v0.5.0 github.com/git-pkgs/purl v0.1.21 github.com/git-pkgs/registries v0.10.0 + github.com/git-pkgs/sbom v0.1.7 github.com/git-pkgs/spdx v0.3.3 github.com/git-pkgs/vers v0.7.2 github.com/git-pkgs/vulns v0.2.4 @@ -26,7 +26,6 @@ require ( github.com/opencontainers/go-digest v1.0.0 github.com/prometheus/client_golang v1.25.0 github.com/prometheus/client_model v0.6.3 - github.com/spdx/tools-golang v0.5.7 github.com/swaggo/swag v1.16.6 gocloud.dev v0.46.0 golang.org/x/mod v0.41.0 @@ -49,7 +48,6 @@ require ( github.com/KyleBanks/depth v1.2.1 // indirect github.com/PuerkitoBio/purell v1.1.1 // indirect github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect - github.com/anchore/go-struct-converter v0.1.0 // indirect github.com/apapsch/go-jsonmerge/v2 v2.0.0 // indirect github.com/aws/aws-sdk-go-v2 v1.47.1 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.11 // indirect diff --git a/go.sum b/go.sum index cf4b97f..387062b 100644 --- a/go.sum +++ b/go.sum @@ -35,8 +35,6 @@ github.com/AzureAD/microsoft-authentication-library-for-go v1.7.0 h1:4iB+IesclUX github.com/AzureAD/microsoft-authentication-library-for-go v1.7.0/go.mod h1:HKpQxkWaGLJ+D/5H8QRpyQXA1eKjxkFlOMwck5+33Jk= github.com/BurntSushi/toml v1.6.0 h1:dRaEfpa2VI55EwlIW72hMRHdWouJeRF7TPYhI+AUQjk= github.com/BurntSushi/toml v1.6.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= -github.com/CycloneDX/cyclonedx-go v0.12.0 h1:/7Jum36UA6V043tQZ/fE3jf+Nf9gn/qxUFfd7QReMy8= -github.com/CycloneDX/cyclonedx-go v0.12.0/go.mod h1:V2577HhxDDCDLYfkm55WJrz16nHTfyQZwcWUBSG7Z28= github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.33.0 h1:l7+6kwRMJNwdCvYdDl7Eax+wzEYHSnNY7zrrfbhDdTA= github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.33.0/go.mod h1:pJTkW8hEUIIi3Pf65lPZOnn4Y81yCllX6IWk2jNXdkM= github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.55.0 h1:UnDZ/zFfG1JhH/DqxIZYU/1CUAlTUScoXD/LcM2Ykk8= @@ -52,8 +50,6 @@ github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbt github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/RaveNoX/go-jsoncommentstrip v1.0.0/go.mod h1:78ihd09MekBnJnxpICcwzCMzGrKSKYe4AqU6PDYYpjk= -github.com/anchore/go-struct-converter v0.1.0 h1:2rDRssAl6mgKBSLNiVCMADgZRhoqtw9dedlWa0OhD30= -github.com/anchore/go-struct-converter v0.1.0/go.mod h1:rYqSE9HbjzpHTI74vwPvae4ZVYZd1lue2ta6xHPdblA= github.com/apapsch/go-jsonmerge/v2 v2.0.0 h1:axGnT1gRIfimI7gJifB699GoE/oq+F2MU7Dml6nw9rQ= github.com/apapsch/go-jsonmerge/v2 v2.0.0/go.mod h1:lvDnEdqiQrp0O42VQGgmlKpxL1AP2+08jFMw88y4klk= github.com/aws/aws-sdk-go-v2 v1.47.1 h1:uOIZnp4PK3ZhKI0dNrJrhTEsLxbpXHTAJlwoS1pvAtw= @@ -99,8 +95,6 @@ github.com/aws/smithy-go v1.28.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqx github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bmatcuk/doublestar v1.1.1/go.mod h1:UD6OnuiIn0yFxxA2le/rnRU1G4RaI4UvFv1sNto9p6w= -github.com/bradleyjkemp/cupaloy/v2 v2.8.0 h1:any4BmKE+jGIaMpnU8YgH/I2LPiLBufr6oMMlVBbn9M= -github.com/bradleyjkemp/cupaloy/v2 v2.8.0/go.mod h1:bm7JXdkRd4BHJk9HpwqAI8BoAY1lps46Enkdqw6aRX0= github.com/cenk/backoff v2.2.1+incompatible h1:djdFT7f4gF2ttuzRKPbMOWgZajgesItGLwG5FTQKmmE= github.com/cenk/backoff v2.2.1+incompatible/go.mod h1:7FtoeaSnHoZnmZzz47cM35Y9nSW7tNyaidugnHTaFDE= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= @@ -110,8 +104,6 @@ github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2/go.mod h1:qwXFYgsP6T7X github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= -github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/ecosyste-ms/ecosystems-go v0.5.0 h1:qZqgar3Do9RrtW327OXlsrGKiRvNdeX8CmXb1aX4Njo= @@ -145,6 +137,8 @@ github.com/git-pkgs/purl v0.1.21 h1:PDZUX43lxIiXPn2eHcsNiiiwQo+HajntZ/4TwSvCGlw= github.com/git-pkgs/purl v0.1.21/go.mod h1:4dOmris48ehjiKmWBqSmMHguUQht48BLeXW+hQUnuj0= github.com/git-pkgs/registries v0.10.0 h1:0goaC5mxw9MglS/XOm2De7fkPBWCCerelm7RPrPh9hQ= github.com/git-pkgs/registries v0.10.0/go.mod h1:bKxgdy/PwnzQXL2FnOktmSQHTKk6B85MA4VtvvYItPc= +github.com/git-pkgs/sbom v0.1.7 h1:4eboy/Yed/SC9ZcC/xT50iywHPzWbmCU7azDb5T1RbA= +github.com/git-pkgs/sbom v0.1.7/go.mod h1:KIkZX0v31l62SeFsi2qsQXcZRVrAXvIdPQ4x4en+G2M= github.com/git-pkgs/spdx v0.3.3 h1:xXlUoZzjDSkwxC6mWHfaW+kc8a0+LH6d+gGHuZWftqI= github.com/git-pkgs/spdx v0.3.3/go.mod h1:n9rAicgw+Wqtfursju6oKaAL6cUCTiIS926mb1KWoKU= github.com/git-pkgs/vers v0.7.2 h1:Ctex1O+Ai/FONmyDaw37EolBXg0og6UdxbCjjYbeRHs= @@ -253,8 +247,6 @@ github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjL github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo= github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= -github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v1.25.0 h1:cDtW1TcFsMi1RJc8khwCKB4VhzwV04aKqbYtNOXuiK0= github.com/prometheus/client_golang v1.25.0/go.mod h1:j2X/4/kaQHLPnmyEU8zjmkndo1T317VpCIQEml8LTKk= github.com/prometheus/client_model v0.6.3 h1:O0jaTVAYNxTHYInEPFJt5I3+sN8zqBtVMPTB1qyxiEo= @@ -271,8 +263,6 @@ github.com/rs/dnscache v0.0.0-20230804202142-fc85eb664529 h1:18kd+8ZUlt/ARXhljq+ github.com/rs/dnscache v0.0.0-20230804202142-fc85eb664529/go.mod h1:qe5TWALJ8/a1Lqznoc5BDHpYX/8HU60Hm2AwRmqzxqA= github.com/rubyist/circuitbreaker v2.2.1+incompatible h1:KUKd/pV8Geg77+8LNDwdow6rVCAYOp8+kHUyFvL6Mhk= github.com/rubyist/circuitbreaker v2.2.1+incompatible/go.mod h1:Ycs3JgJADPuzJDwffe12k6BZT8hxVi6lFK+gWYJLN4A= -github.com/spdx/tools-golang v0.5.7 h1:+sWcKGnhwp3vLdMqPcLdA6QK679vd86cK9hQWH3AwCg= -github.com/spdx/tools-golang v0.5.7/go.mod h1:jg7w0LOpoNAw6OxKEzCoqPC2GCTj45LyTlVmXubDsYw= github.com/spiffe/go-spiffe/v2 v2.7.0 h1:uXe1MflJoHw58wAUvxVlcM7WpKtijWG7I1UidcGh6g4= github.com/spiffe/go-spiffe/v2 v2.7.0/go.mod h1:47Q0Q9/AqGha8QLHp+kxpH4Wca7X7EnOtlIJy3mxZ3U= github.com/spkg/bom v0.0.0-20160624110644-59b7046e48ad/go.mod h1:qLr4V1qq6nMqFKkMo8ZTx3f+BZEkzsRUY10Xsm2mwU0= @@ -283,16 +273,8 @@ github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWD github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg= github.com/swaggo/swag v1.16.6 h1:qBNcx53ZaX+M5dxVyTrgQ0PJ/ACK+NzhwcbieTt+9yI= github.com/swaggo/swag v1.16.6/go.mod h1:ngP2etMK5a0P3QBizic5MEwpRmluJZPHjXcMoj4Xesg= -github.com/terminalstatic/go-xsd-validate v0.1.8 h1:UVrTCy1j3DhwaYTTUF+QYO/Nan13S0tf+Jwi+p45Bf0= -github.com/terminalstatic/go-xsd-validate v0.1.8/go.mod h1:1kb47fi2c6onlf+B7UrrQ9VYraOhcYwFm3iG+J6F4Zo= github.com/ulikunitz/xz v0.5.16 h1:ld6NyySjx5lowVKwJvMRLnW5nxKX/xnpSiFYZ/Lxur0= github.com/ulikunitz/xz v0.5.16/go.mod h1:H9Rt/W6/Qj27PGauhQc6nfCDy7vHpzsOThBSaYDoEhw= -github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f h1:J9EGpcZtP0E/raorCMxlFGSTBrsSlaDGf3jU/qvAE2c= -github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= -github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0= -github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= -github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74= -github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= go.opentelemetry.io/contrib/detectors/gcp v1.44.0 h1:NmLfL734pJhM0JKaYd2Y28+nY9dPRWYAAbxhRCrKXPw= diff --git a/internal/mirror/source.go b/internal/mirror/source.go index 6d3d096..8fcf11c 100644 --- a/internal/mirror/source.go +++ b/internal/mirror/source.go @@ -1,18 +1,13 @@ package mirror import ( - "bytes" "context" - "encoding/json" "fmt" - cdx "github.com/CycloneDX/cyclonedx-go" "github.com/git-pkgs/purl" "github.com/git-pkgs/registries" _ "github.com/git-pkgs/registries/all" - spdxjson "github.com/spdx/tools-golang/json" - "github.com/spdx/tools-golang/spdx" - spdxtv "github.com/spdx/tools-golang/tagvalue" + "github.com/git-pkgs/sbom" ) // PackageVersion identifies a specific package version to mirror. @@ -130,75 +125,20 @@ func (s *SBOMSource) Enumerate(ctx context.Context, fn func(PackageVersion) erro } func (s *SBOMSource) extractPURLs() ([]string, error) { - // Try CycloneDX first - if purls, err := extractCycloneDXPURLs(s.Data); err == nil && len(purls) > 0 { - return purls, nil - } - - // Try SPDX JSON - if purls, err := extractSPDXJSONPURLs(s.Data); err == nil && len(purls) > 0 { - return purls, nil - } - - // Try SPDX tag-value - if purls, err := extractSPDXTVPURLs(s.Data); err == nil && len(purls) > 0 { - return purls, nil - } - - return nil, fmt.Errorf("could not parse SBOM as CycloneDX or SPDX") -} - -func extractCycloneDXPURLs(data []byte) ([]string, error) { - bom := new(cdx.BOM) - if err := json.Unmarshal(data, bom); err != nil { - // Try XML - decoder := cdx.NewBOMDecoder(bytes.NewReader(data), cdx.BOMFileFormatXML) - bom = new(cdx.BOM) - if err := decoder.Decode(bom); err != nil { - return nil, err - } - } - - if bom.Components == nil { - return nil, nil - } - - var purls []string - for _, c := range *bom.Components { - if c.PackageURL != "" { - purls = append(purls, c.PackageURL) - } - } - return purls, nil -} - -func extractSPDXJSONPURLs(data []byte) ([]string, error) { - doc, err := spdxjson.Read(bytes.NewReader(data)) + doc, err := sbom.Parse(s.Data) if err != nil { return nil, err } - return extractSPDXDocPURLs(doc), nil -} - -func extractSPDXTVPURLs(data []byte) ([]string, error) { - doc, err := spdxtv.Read(bytes.NewReader(data)) - if err != nil { - return nil, err - } - return extractSPDXDocPURLs(doc), nil -} - -func extractSPDXDocPURLs(doc *spdx.Document) []string { - if doc == nil { - return nil - } var purls []string - for _, pkg := range doc.Packages { - for _, ref := range pkg.PackageExternalReferences { - if ref.RefType == "purl" { + for _, p := range doc.Packages { + for _, ref := range p.ExternalRefs { + if ref.Type == "purl" && ref.Locator != "" { purls = append(purls, ref.Locator) } } } - return purls + if len(purls) == 0 { + return nil, fmt.Errorf("SBOM contains no package URLs") + } + return purls, nil } diff --git a/internal/mirror/source_formats_test.go b/internal/mirror/source_formats_test.go new file mode 100644 index 0000000..972c047 --- /dev/null +++ b/internal/mirror/source_formats_test.go @@ -0,0 +1,74 @@ +package mirror + +import ( + "context" + "reflect" + "testing" +) + +func TestSBOMSourcePreservesPackageIdentities(t *testing.T) { + for name, data := range map[string]string{ + "CycloneDX JSON": `{"components":[ + {"name":"shared","version":"1.0","purl":"pkg:npm/%40alpha/shared@1.0"}, + {"name":"shared","version":"1.0","purl":"pkg:npm/%40beta/shared@1.0"}]}`, + "CycloneDX XML": ` + shared1.0pkg:npm/%40alpha/shared@1.0 + shared1.0pkg:npm/%40beta/shared@1.0 + `, + "SPDX JSON": `{"spdxVersion":"SPDX-2.3","packages":[{"name":"shared","versionInfo":"1.0","externalRefs":[ + {"referenceType":"purl","referenceLocator":"pkg:npm/%40alpha/shared@1.0"}, + {"referenceType":"purl","referenceLocator":"pkg:npm/%40beta/shared@1.0"}]}]}`, + "SPDX tag-value": `DataLicense: CC0-1.0 +SPDXVersion: SPDX-2.3 +SPDXID: SPDXRef-DOCUMENT +DocumentName: example +DocumentNamespace: https://example.com/sbom +PackageName: shared +SPDXID: SPDXRef-shared +PackageVersion: 1.0 +ExternalRef: PACKAGE-MANAGER purl pkg:npm/%40alpha/shared@1.0 +ExternalRef: PACKAGE-MANAGER purl pkg:npm/%40beta/shared@1.0 +`, + } { + t.Run(name, func(t *testing.T) { + source := &SBOMSource{Data: []byte(data), Name: name} + var got []PackageVersion + err := source.Enumerate(context.Background(), func(pv PackageVersion) error { + got = append(got, pv) + return nil + }) + if err != nil { + t.Fatal(err) + } + want := []PackageVersion{ + {Ecosystem: "npm", Name: "@alpha/shared", Version: "1.0"}, + {Ecosystem: "npm", Name: "@beta/shared", Version: "1.0"}, + } + if !reflect.DeepEqual(got, want) { + t.Errorf("Enumerate = %+v, want %+v", got, want) + } + }) + } +} + +func TestSBOMSourcePreservesImageOptions(t *testing.T) { + source := &SBOMSource{ + Data: []byte(`{"bomFormat":"CycloneDX","components":[{"name":"alpine","purl":"pkg:docker/library/alpine@3.20?repository_url=ghcr.io&arch=arm64"}]}`), + Platforms: []string{"linux/amd64"}, + } + var got []PackageVersion + if err := source.Enumerate(context.Background(), func(pv PackageVersion) error { + got = append(got, pv) + return nil + }); err != nil { + t.Fatal(err) + } + want := []PackageVersion{{ + Ecosystem: "docker", Name: "library/alpine", Version: "3.20", + Qualifiers: map[string]string{"repository_url": "ghcr.io", "arch": "arm64"}, + Platforms: []string{"linux/amd64"}, + }} + if !reflect.DeepEqual(got, want) { + t.Errorf("Enumerate = %+v, want %+v", got, want) + } +} diff --git a/internal/mirror/source_test.go b/internal/mirror/source_test.go index fe56ca4..24fbbc0 100644 --- a/internal/mirror/source_test.go +++ b/internal/mirror/source_test.go @@ -211,8 +211,6 @@ func TestSBOMSourceEmptyCycloneDX(t *testing.T) { "bomFormat": "CycloneDX", "specVersion": "1.4", } - // This should fall through to SPDX parsing, which will also fail, - // resulting in an error about not being able to parse source := &SBOMSource{Data: marshalJSON(t, bom)} err := source.Enumerate(context.Background(), func(pv PackageVersion) error { return nil From d56d4367c968a8a6f2b2a1f652cb6f191ed24d6e Mon Sep 17 00:00:00 2001 From: Christian Heim Date: Tue, 20 Oct 2026 11:22:55 +0200 Subject: [PATCH 323/633] Count cached metadata against max_size and evict it LRU (#507) * max_size now counts cached metadata as well as artifacts, and eviction drops whichever was used least recently, so metadata no longer piles up for good (#461). Metadata rows record when they were last served, an artifact not yet served counts from its download, and Arch stores a database blob again if eviction removed it under a refreshed row. * Shorter comments on the metadata last-access migration. * Show evicted metadata on the analytics page next to evicted artifacts, and cover the new counter in the metric surface test. * Evicting metadata now drops the row and queues its path in one transaction, so a crash before the blob is deleted leaves it to reclaim instead of orphaning it. * The last-access backfill now runs in the background in batches of 1000 instead of inside the migration, resuming after a restart, and until a row is backfilled eviction orders it by its write time. --------- Co-authored-by: Andrew Nesbitt --- README.md | 3 +- docs/configuration.md | 2 +- internal/database/cache_eviction.go | 221 ++++++++++++ internal/database/cache_eviction_test.go | 325 ++++++++++++++++++ internal/database/hits.go | 72 +++- internal/database/queries.go | 21 +- internal/database/schema.go | 43 +++ internal/database/types.go | 1 + internal/handler/arch.go | 9 +- internal/handler/container_manifest.go | 1 + internal/handler/container_metadata.go | 1 + internal/handler/container_tags.go | 1 + internal/handler/handler.go | 10 + internal/handler/metadata_access_test.go | 181 ++++++++++ internal/handler/rewrite_cache.go | 1 + internal/metrics/metrics.go | 16 + internal/server/analytics_coverage_test.go | 2 + internal/server/analytics_runtime.go | 2 + internal/server/analytics_runtime_test.go | 18 + internal/server/eviction.go | 62 +++- internal/server/eviction_metadata_test.go | 194 +++++++++++ internal/server/eviction_test.go | 3 +- internal/server/metadata_backfill.go | 38 ++ internal/server/server.go | 1 + .../templates/components/runtime_metrics.html | 1 + 25 files changed, 1198 insertions(+), 31 deletions(-) create mode 100644 internal/database/cache_eviction.go create mode 100644 internal/database/cache_eviction_test.go create mode 100644 internal/handler/metadata_access_test.go create mode 100644 internal/server/eviction_metadata_test.go create mode 100644 internal/server/metadata_backfill.go diff --git a/README.md b/README.md index d1a67e0..49670c8 100644 --- a/README.md +++ b/README.md @@ -1379,6 +1379,7 @@ The proxy exposes Prometheus metrics at `GET /metrics`. All metric names are pre | `proxy_cache_size_bytes` | gauge | | Total size of cached artifacts | | `proxy_cached_artifacts_total` | gauge | | Number of cached artifacts | | `proxy_artifacts_evicted_total` | counter | `reason`, `ecosystem` | Artifacts evicted by the size limit (`lru`) or by age (`retention`) | +| `proxy_metadata_evicted_total` | counter | `ecosystem` | Cached metadata documents evicted by the size limit | | `proxy_upstream_fetch_duration_seconds` | histogram | `ecosystem` | Time spent fetching from upstream | | `proxy_upstream_errors_total` | counter | `ecosystem`, `error_type` | Upstream fetch failures | | `proxy_storage_operation_duration_seconds` | histogram | `operation` | Storage read/write latency | @@ -1442,7 +1443,7 @@ The same figures are available as JSON from `GET /stats`, which reports `downloa `ecosystem` means three slightly different things across `/metrics`, and queries that join across them need to know which. -**From the package record, normalized.** The six `proxy_ecosystem_*` gauges, `proxy_cache_hits_total`, `proxy_cache_misses_total`, `proxy_artifacts_evicted_total`, `proxy_integrity_failures_total` and the scan metrics. Aliases collapse here: `gem` reads as `rubygems`, `composer` as `packagist`, `go` as `golang`. +**From the package record, normalized.** The six `proxy_ecosystem_*` gauges, `proxy_cache_hits_total`, `proxy_cache_misses_total`, `proxy_artifacts_evicted_total`, `proxy_metadata_evicted_total`, `proxy_integrity_failures_total` and the scan metrics. Aliases collapse here: `gem` reads as `rubygems`, `composer` as `packagist`, `go` as `golang`. **From the request path.** `proxy_requests_total`, `proxy_request_duration_seconds` and `proxy_response_bytes_total`. The names mostly coincide with the normalized ones -- these also report `rubygems`, `packagist` and `golang` -- but the Debian route reports `debian` where the package record says `deb`, and any path that is not a package endpoint reports `other`, which corresponds to no ecosystem at all. diff --git a/docs/configuration.md b/docs/configuration.md index 08a9acd..ed7f13f 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -66,7 +66,7 @@ storage: | `storage.retention.default` | `PROXY_STORAGE_RETENTION_DEFAULT` | - | Evict artifacts not served for this long (e.g. "30d"); see [Retention](#retention) | | `storage.retention.sweep_interval` | `PROXY_STORAGE_RETENTION_SWEEP_INTERVAL` | - | How often expired artifacts are looked for (default: "10m", at least "1m") | -`storage.max_size` counts cached artifacts only. An artifact replaced by a refetch stays in storage for at least an hour, or `storage.direct_serve_ttl` if longer, so requests already reading it can finish, and storage use can exceed the limit by what was replaced in that time. +`storage.max_size` counts cached artifacts and cached metadata documents (`cache_metadata`). When the total goes over the limit, the least recently served entries of either kind are evicted first; an evicted metadata document is fetched from upstream again on the next request for it. Pushed artifacts are not counted or evicted. An artifact replaced by a refetch stays in storage for at least an hour, or `storage.direct_serve_ttl` if longer, so requests already reading it can finish, and storage use can exceed the limit by what was replaced in that time. ### Serving artifacts without storing them diff --git a/internal/database/cache_eviction.go b/internal/database/cache_eviction.go new file mode 100644 index 0000000..e14df8d --- /dev/null +++ b/internal/database/cache_eviction.go @@ -0,0 +1,221 @@ +package database + +import ( + "database/sql" + "slices" + "time" +) + +// Kinds of CacheEntry. +const ( + CacheEntryArtifact = "artifact" + CacheEntryMetadata = "metadata" +) + +// CacheEntry is a cached object size eviction can remove: an artifact fetched +// from upstream or a metadata document. For an artifact Key is its version +// PURL and Name its filename; for metadata Key is the ecosystem and Name the +// cache key. FetchedAt is only read for artifacts, UpdatedAt for metadata +// not backfilled yet. +type CacheEntry struct { + Kind string `db:"kind"` + Key string `db:"cache_key"` + Name string `db:"name"` + StoragePath sql.NullString `db:"storage_path"` + Size sql.NullInt64 `db:"size"` + LastAccessedAt sql.NullTime `db:"last_accessed_at"` + FetchedAt sql.NullTime `db:"fetched_at"` + UpdatedAt sql.NullTime `db:"updated_at"` +} + +// GetEvictableCacheSize returns the bytes max_size limits: cached upstream +// artifacts and cached metadata documents. +func (db *DB) GetEvictableCacheSize() (int64, error) { + var total int64 + err := db.Get(&total, ` + SELECT (SELECT COALESCE(SUM(size), 0) FROM artifacts WHERE storage_path IS NOT NULL AND source <> 'local') + + (SELECT COALESCE(SUM(size), 0) FROM metadata_cache)`) + return total, err +} + +// GetLeastRecentlyUsedCacheEntries returns up to limit artifacts and metadata +// documents, least recently used first. An artifact counts as used when it +// was fetched until it is first served, so a fresh download does not go +// before every metadata document. Each kind is read in order on its own and +// the two are merged here, which keeps the queries simple for both dialects. +func (db *DB) GetLeastRecentlyUsedCacheEntries(limit int) ([]CacheEntry, error) { + var artifacts, metadata []CacheEntry + if err := db.Select(&artifacts, db.Rebind(` + SELECT 'artifact' AS kind, version_purl AS cache_key, filename AS name, + storage_path, size, last_accessed_at, fetched_at + FROM artifacts + WHERE storage_path IS NOT NULL AND source <> 'local' + ORDER BY COALESCE(last_accessed_at, fetched_at) ASC NULLS FIRST + LIMIT ? + `), limit); err != nil { + return nil, err + } + if err := db.Select(&metadata, db.Rebind(` + SELECT 'metadata' AS kind, ecosystem AS cache_key, name, + storage_path, size, last_accessed_at, updated_at + FROM metadata_cache + ORDER BY COALESCE(last_accessed_at, updated_at) ASC NULLS FIRST + LIMIT ? + `), limit); err != nil { + return nil, err + } + // Rows not backfilled yet fall back to updated_at, which Postgres keeps + // in local time, so the order is fixed up once it is read as UTC. + for i := range metadata { + metadata[i].UpdatedAt = db.writtenAtUTC(metadata[i].UpdatedAt) + } + slices.SortStableFunc(metadata, func(a, b CacheEntry) int { + switch { + case a.usedAfter(b): + return 1 + case b.usedAfter(a): + return -1 + } + return 0 + }) + + entries := make([]CacheEntry, 0, min(limit, len(artifacts)+len(metadata))) + for len(entries) < limit && (len(artifacts) > 0 || len(metadata) > 0) { + if len(metadata) == 0 || (len(artifacts) > 0 && !artifacts[0].usedAfter(metadata[0])) { + entries, artifacts = append(entries, artifacts[0]), artifacts[1:] + } else { + entries, metadata = append(entries, metadata[0]), metadata[1:] + } + } + return entries, nil +} + +// lastUsed is when e was last served, or for an artifact not served since, +// when it was fetched. It is invalid when neither is known. +func (e CacheEntry) lastUsed() sql.NullTime { + if e.LastAccessedAt.Valid { + return e.LastAccessedAt + } + if e.FetchedAt.Valid { + return e.FetchedAt + } + return e.UpdatedAt +} + +// usedAfter reports whether e was used later than other. Entries with no +// time known count as the oldest, as NULLS FIRST orders them. +func (e CacheEntry) usedAfter(other CacheEntry) bool { + a, b := e.lastUsed(), other.lastUsed() + if !a.Valid { + return false + } + return !b.Valid || a.Time.After(b.Time) +} + +// EvictMetadataCache deletes a metadata cache row if it still points at +// storagePath and has not been used since lastAccessed, queueing the path in +// the same transaction so a crash before the blob is deleted cannot orphan it. +func (db *DB) EvictMetadataCache(ecosystem, name, storagePath string, lastAccessed sql.NullTime) (bool, error) { + query := ` + DELETE FROM metadata_cache + WHERE ecosystem = ? AND name = ? AND storage_path = ? AND last_accessed_at IS NULL` + args := []any{ecosystem, name, storagePath} + if lastAccessed.Valid { + query = ` + DELETE FROM metadata_cache + WHERE ecosystem = ? AND name = ? AND storage_path = ? + AND (last_accessed_at IS NULL OR last_accessed_at <= ?)` + args = append(args, lastAccessed.Time) + } + tx, err := db.Beginx() + if err != nil { + return false, err + } + defer func() { _ = tx.Rollback() }() + res, err := tx.Exec(db.Rebind(query), args...) + if err != nil { + return false, err + } + if n, err := res.RowsAffected(); err != nil || n == 0 { + return false, err + } + if _, err := tx.Exec(db.Rebind(queuePendingDeleteQuery), storagePath, time.Now().UTC()); err != nil { + return false, err + } + return true, tx.Commit() +} + +// RecordMetadataHit marks a cached metadata document as served now. Like +// RecordArtifactHit it is buffered when BatchHits is on. +func (db *DB) RecordMetadataHit(ecosystem, name string) error { + now := time.Now() + if db.hits != nil { + db.hits.addMetadata(metadataHitKey{ecosystem, name}, now) + return nil + } + _, err := db.Exec(db.Rebind(recordMetadataHitQuery), now.UTC(), now.UTC(), ecosystem, name) + return err +} + +// recordMetadataHitQuery takes the access time twice, then the ecosystem and +// key. The time only moves forward, as for artifacts. +const recordMetadataHitQuery = ` + UPDATE metadata_cache + SET last_accessed_at = CASE WHEN last_accessed_at IS NULL OR last_accessed_at < ? THEN ? ELSE last_accessed_at END + WHERE ecosystem = ? AND name = ?` + +// writtenAtUTC returns a metadata_cache updated_at or created_at as the +// instant it was written. Both were written in local time, and a Postgres +// TIMESTAMP keeps only the wall clock, which lib/pq reads back as UTC. +func (db *DB) writtenAtUTC(t sql.NullTime) sql.NullTime { + if t.Valid && db.dialect == DialectPostgres { + w := t.Time + t.Time = time.Date(w.Year(), w.Month(), w.Day(), w.Hour(), w.Minute(), w.Second(), w.Nanosecond(), time.Local) + } + if t.Valid { + t.Time = t.Time.UTC() + } + return t +} + +// BackfillMetadataLastAccessed sets last_accessed_at from the write time on +// up to limit rows that have none, in one transaction, and reports how many it +// set. Rows served in the meantime keep the time of that hit. +func (db *DB) BackfillMetadataLastAccessed(limit int) (int, error) { + var rows []struct { + ID int64 `db:"id"` + UpdatedAt sql.NullTime `db:"updated_at"` + CreatedAt sql.NullTime `db:"created_at"` + } + query := db.Rebind(`SELECT id, updated_at, created_at FROM metadata_cache WHERE last_accessed_at IS NULL LIMIT ?`) + if err := db.Select(&rows, query, limit); err != nil { + return 0, err + } + if len(rows) == 0 { + return 0, nil + } + tx, err := db.Beginx() + if err != nil { + return 0, err + } + defer func() { _ = tx.Rollback() }() + stmt, err := tx.Preparex(db.Rebind(`UPDATE metadata_cache SET last_accessed_at = ? WHERE id = ? AND last_accessed_at IS NULL`)) + if err != nil { + return 0, err + } + defer func() { _ = stmt.Close() }() + for _, r := range rows { + at := db.writtenAtUTC(r.UpdatedAt) + if !at.Valid { + at = db.writtenAtUTC(r.CreatedAt) + } + if !at.Valid { + // Nothing to go by; counting it as used now at least ends the loop. + at = sql.NullTime{Time: time.Now().UTC(), Valid: true} + } + if _, err := stmt.Exec(at.Time, r.ID); err != nil { + return 0, err + } + } + return len(rows), tx.Commit() +} diff --git a/internal/database/cache_eviction_test.go b/internal/database/cache_eviction_test.go new file mode 100644 index 0000000..4040316 --- /dev/null +++ b/internal/database/cache_eviction_test.go @@ -0,0 +1,325 @@ +package database + +import ( + "database/sql" + "strings" + "testing" + "time" +) + +func upsertTestMetadata(t *testing.T, db *DB, name string, size int64) { + t.Helper() + if err := db.UpsertMetadataCache(&MetadataCacheEntry{ + Ecosystem: testEcosystemNPM, + Name: name, + StoragePath: "_metadata/npm/" + name + "/metadata", + Size: sql.NullInt64{Int64: size, Valid: true}, + FetchedAt: sql.NullTime{Time: time.Now(), Valid: true}, + }); err != nil { + t.Fatalf("UpsertMetadataCache: %v", err) + } +} + +func setTestMetadataAccess(t *testing.T, db *DB, name string, at time.Time) { + t.Helper() + query := db.Rebind(`UPDATE metadata_cache SET last_accessed_at = ? WHERE ecosystem = ? AND name = ?`) + if _, err := db.Exec(query, at.UTC(), testEcosystemNPM, name); err != nil { + t.Fatalf("setting last access: %v", err) + } +} + +func TestUpsertMetadataCacheMarksAccess(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + before := time.Now().Add(-time.Second) + upsertTestMetadata(t, db, "lodash", 10) + + got, err := db.GetMetadataCache(testEcosystemNPM, "lodash") + if err != nil || got == nil { + t.Fatalf("GetMetadataCache: %v, %v", got, err) + } + if !got.LastAccessedAt.Valid || got.LastAccessedAt.Time.Before(before) { + t.Errorf("last_accessed_at = %v, want the time of the write", got.LastAccessedAt) + } + }) +} + +func TestGetEvictableCacheSizeCountsMetadata(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + upsertTestMetadata(t, db, "a", 300) + upsertTestMetadata(t, db, "b", 200) + + total, err := db.GetEvictableCacheSize() + if err != nil { + t.Fatalf("GetEvictableCacheSize: %v", err) + } + if total != 500 { + t.Errorf("evictable size = %d, want 500", total) + } + // The size shown in stats stays the artifacts' total. + artifacts, err := db.GetTotalCacheSize() + if err != nil || artifacts != 0 { + t.Errorf("GetTotalCacheSize = %d, %v, want 0", artifacts, err) + } + }) +} + +func TestGetLeastRecentlyUsedCacheEntriesOrdersMetadataAndArtifacts(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + now := time.Now() + upsertTestMetadata(t, db, "old", 1) + setTestMetadataAccess(t, db, "old", now.Add(-3*time.Hour)) + upsertTestMetadata(t, db, "new", 1) + setTestMetadataAccess(t, db, "new", now) + if err := db.UpsertArtifact(&Artifact{ + VersionPURL: "pkg:npm/mid@1.0.0", + Filename: "mid-1.0.0.tgz", + UpstreamURL: "https://example.test/mid-1.0.0.tgz", + StoragePath: sql.NullString{String: "npm/mid/1.0.0/mid-1.0.0.tgz", Valid: true}, + Size: sql.NullInt64{Int64: 1, Valid: true}, + LastAccessedAt: sql.NullTime{Time: now.Add(-time.Hour), Valid: true}, + }); err != nil { + t.Fatalf("UpsertArtifact: %v", err) + } + + entries, err := db.GetLeastRecentlyUsedCacheEntries(10) + if err != nil { + t.Fatalf("GetLeastRecentlyUsedCacheEntries: %v", err) + } + var got []string + for _, e := range entries { + got = append(got, e.Kind+":"+e.Name) + } + want := []string{"metadata:old", "artifact:mid-1.0.0.tgz", "metadata:new"} + if len(got) != len(want) { + t.Fatalf("entries = %v, want %v", got, want) + } + for i := range want { + if got[i] != want[i] { + t.Fatalf("entries = %v, want %v", got, want) + } + } + }) +} + +func TestEvictMetadataCacheKeepsRowUsedSinceRead(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + upsertTestMetadata(t, db, "lodash", 10) + setTestMetadataAccess(t, db, "lodash", time.Now().Add(-time.Hour)) + read, _ := db.GetMetadataCache(testEcosystemNPM, "lodash") + + if err := db.RecordMetadataHit(testEcosystemNPM, "lodash"); err != nil { + t.Fatalf("RecordMetadataHit: %v", err) + } + evicted, err := db.EvictMetadataCache(testEcosystemNPM, "lodash", read.StoragePath, read.LastAccessedAt) + if err != nil || evicted { + t.Fatalf("EvictMetadataCache = %v, %v, want the row kept", evicted, err) + } + + current, _ := db.GetMetadataCache(testEcosystemNPM, "lodash") + evicted, err = db.EvictMetadataCache(testEcosystemNPM, "lodash", current.StoragePath, current.LastAccessedAt) + if err != nil || !evicted { + t.Fatalf("EvictMetadataCache = %v, %v, want the row evicted", evicted, err) + } + if gone, _ := db.GetMetadataCache(testEcosystemNPM, "lodash"); gone != nil { + t.Error("row still present after eviction") + } + }) +} + +func TestRecordMetadataHitBatched(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + upsertTestMetadata(t, db, "lodash", 10) + old := time.Now().Add(-time.Hour) + setTestMetadataAccess(t, db, "lodash", old) + db.BatchHits(time.Hour, nil) + + if err := db.RecordMetadataHit(testEcosystemNPM, "lodash"); err != nil { + t.Fatalf("RecordMetadataHit: %v", err) + } + if got, _ := db.GetMetadataCache(testEcosystemNPM, "lodash"); got.LastAccessedAt.Time.After(old.Add(time.Minute)) { + t.Error("batched hit was written before the flush") + } + if err := db.flushHits(); err != nil { + t.Fatalf("flushHits: %v", err) + } + if got, _ := db.GetMetadataCache(testEcosystemNPM, "lodash"); !got.LastAccessedAt.Time.After(old.Add(time.Minute)) { + t.Errorf("last_accessed_at = %v after flush, want the hit's time", got.LastAccessedAt) + } + }) +} + +func TestPendingDeleteSkipsPathMetadataPointsAt(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + upsertTestMetadata(t, db, "lodash", 10) + path := "_metadata/npm/lodash/metadata" + if err := db.QueuePendingDelete(path); err != nil { + t.Fatalf("QueuePendingDelete: %v", err) + } + + due, err := db.GetDuePendingDeletes(time.Now().Add(time.Hour), 10) + if err != nil { + t.Fatalf("GetDuePendingDeletes: %v", err) + } + if len(due) != 0 { + t.Errorf("due = %v, want the path a metadata row uses skipped", due) + } + }) +} + +// TestGetLeastRecentlyUsedCacheEntriesCountsFetchAsUse checks that an +// artifact not served since its download ranks by when it was fetched, not +// ahead of every metadata document. +func TestGetLeastRecentlyUsedCacheEntriesCountsFetchAsUse(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + now := time.Now() + upsertTestMetadata(t, db, "old", 1) + setTestMetadataAccess(t, db, "old", now.Add(-time.Hour)) + if err := db.UpsertArtifact(&Artifact{ + VersionPURL: "pkg:npm/fresh@1.0.0", + Filename: "fresh-1.0.0.tgz", + UpstreamURL: "https://example.test/fresh-1.0.0.tgz", + StoragePath: sql.NullString{String: "npm/fresh/1.0.0/fresh-1.0.0.tgz", Valid: true}, + Size: sql.NullInt64{Int64: 1, Valid: true}, + FetchedAt: sql.NullTime{Time: now, Valid: true}, + }); err != nil { + t.Fatalf("UpsertArtifact: %v", err) + } + + entries, err := db.GetLeastRecentlyUsedCacheEntries(1) + if err != nil || len(entries) != 1 { + t.Fatalf("GetLeastRecentlyUsedCacheEntries: %v, %v", entries, err) + } + if entries[0].Kind != CacheEntryMetadata { + t.Errorf("least recently used = %+v, want the metadata used an hour ago", entries[0]) + } + }) +} + +// TestLeastRecentlyUsedQueriesUseIndexes checks on SQLite that eviction reads +// both tables off an index rather than sorting them whole. +func TestLeastRecentlyUsedQueriesUseIndexes(t *testing.T) { + db := setupMetadataCacheDB(t) + for _, q := range []string{ + `SELECT version_purl FROM artifacts WHERE storage_path IS NOT NULL AND source <> 'local' + ORDER BY COALESCE(last_accessed_at, fetched_at) ASC NULLS FIRST LIMIT 50`, + `SELECT name FROM metadata_cache ORDER BY COALESCE(last_accessed_at, updated_at) ASC NULLS FIRST LIMIT 50`, + } { + rows, err := db.Query("EXPLAIN QUERY PLAN " + q) + if err != nil { + t.Fatalf("explain: %v", err) + } + var plan []string + for rows.Next() { + var id, parent, unused int + var detail string + if err := rows.Scan(&id, &parent, &unused, &detail); err != nil { + t.Fatalf("scan: %v", err) + } + plan = append(plan, detail) + } + _ = rows.Close() + for _, step := range plan { + if strings.Contains(step, "TEMP B-TREE") { + t.Errorf("query sorts instead of using an index: %v\n%s", plan, q) + } + } + } +} + +// insertLegacyMetadata adds a row as the proxy wrote it before +// last_accessed_at existed: write times in local time, no access time. +func insertLegacyMetadata(t *testing.T, db *DB, name string, written time.Time) { + t.Helper() + if _, err := db.Exec(db.Rebind(` + INSERT INTO metadata_cache (ecosystem, name, storage_path, size, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?)`), testEcosystemNPM, name, "_metadata/npm/"+name+"/metadata", 1, written, written); err != nil { + t.Fatalf("inserting legacy row: %v", err) + } +} + +func TestMetadataLastAccessedMigrationLeavesBackfillToBackground(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + for _, q := range []string{ + "DROP INDEX idx_metadata_last_used", + "DROP INDEX idx_metadata_storage_path", + "DROP INDEX idx_artifacts_last_used", + "ALTER TABLE metadata_cache DROP COLUMN last_accessed_at", + "DELETE FROM migrations WHERE name = '014_add_metadata_last_accessed'", + } { + if _, err := db.Exec(q); err != nil { + t.Fatalf("%s: %v", q, err) + } + } + written := time.Date(2026, 1, 2, 3, 4, 5, 0, time.Local) + insertLegacyMetadata(t, db, "legacy", written) + + if err := db.MigrateSchema(); err != nil { + t.Fatalf("MigrateSchema: %v", err) + } + if got, _ := db.GetMetadataCache(testEcosystemNPM, "legacy"); got == nil || got.LastAccessedAt.Valid { + t.Fatalf("row after migration = %+v, want it kept with no access time", got) + } + if n, err := db.BackfillMetadataLastAccessed(10); err != nil || n != 1 { + t.Fatalf("BackfillMetadataLastAccessed = %d, %v, want 1", n, err) + } + got, _ := db.GetMetadataCache(testEcosystemNPM, "legacy") + if got == nil || !got.LastAccessedAt.Valid || !got.LastAccessedAt.Time.Equal(written) { + t.Errorf("backfilled last_accessed_at = %+v, want %v", got, written.UTC()) + } + }) +} + +// TestBackfillMetadataLastAccessedResumes runs the backfill in batches, as a +// restart after an interruption would, and leaves rows served meanwhile alone. +func TestBackfillMetadataLastAccessedResumes(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + written := time.Now().Add(-48 * time.Hour).Truncate(time.Second) + for _, name := range []string{"a", "b", "c"} { + insertLegacyMetadata(t, db, name, written) + } + if n, err := db.BackfillMetadataLastAccessed(2); err != nil || n != 2 { + t.Fatalf("first batch = %d, %v, want 2", n, err) + } + var left string + if err := db.Get(&left, `SELECT name FROM metadata_cache WHERE last_accessed_at IS NULL`); err != nil { + t.Fatalf("finding the row left: %v", err) + } + if err := db.RecordMetadataHit(testEcosystemNPM, left); err != nil { + t.Fatalf("RecordMetadataHit: %v", err) + } + if n, err := db.BackfillMetadataLastAccessed(2); err != nil || n != 0 { + t.Fatalf("second batch = %d, %v, want 0", n, err) + } + got, _ := db.GetMetadataCache(testEcosystemNPM, left) + if !got.LastAccessedAt.Time.After(written.Add(time.Hour)) { + t.Errorf("served row last_accessed_at = %v, want the hit kept", got.LastAccessedAt) + } + }) +} + +// TestLeastRecentlyUsedOrdersUnbackfilledMetadataByWriteTime checks that rows +// still waiting for the backfill are not all evicted first. +func TestLeastRecentlyUsedOrdersUnbackfilledMetadataByWriteTime(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + now := time.Now() + insertLegacyMetadata(t, db, "recent", now) + if err := db.UpsertArtifact(&Artifact{ + VersionPURL: "pkg:npm/old@1.0.0", + Filename: "old-1.0.0.tgz", + UpstreamURL: "https://example.test/old-1.0.0.tgz", + StoragePath: sql.NullString{String: "npm/old/1.0.0/old-1.0.0.tgz", Valid: true}, + Size: sql.NullInt64{Int64: 1, Valid: true}, + LastAccessedAt: sql.NullTime{Time: now.Add(-time.Hour), Valid: true}, + }); err != nil { + t.Fatalf("UpsertArtifact: %v", err) + } + + entries, err := db.GetLeastRecentlyUsedCacheEntries(1) + if err != nil || len(entries) != 1 { + t.Fatalf("GetLeastRecentlyUsedCacheEntries: %v, %v", entries, err) + } + if entries[0].Kind != CacheEntryArtifact { + t.Errorf("least recently used = %+v, want the artifact used an hour ago", entries[0]) + } + }) +} diff --git a/internal/database/hits.go b/internal/database/hits.go index 4b5de0b..3bb19c7 100644 --- a/internal/database/hits.go +++ b/internal/database/hits.go @@ -2,6 +2,7 @@ package database import ( "cmp" + "errors" "log/slog" "slices" "sync" @@ -20,9 +21,14 @@ type hitEntry struct { last time.Time } +// metadataHitKey identifies a metadata cache row; a hit only moves its +// last-access time. +type metadataHitKey struct{ ecosystem, name string } + type hitBatch struct { mu sync.Mutex pending map[hitKey]hitEntry + metadata map[metadataHitKey]time.Time stop chan struct{} stopOnce sync.Once done chan struct{} @@ -39,6 +45,22 @@ func (b *hitBatch) add(k hitKey, e hitEntry) { b.pending[k] = cur } +func (b *hitBatch) addMetadata(k metadataHitKey, last time.Time) { + b.mu.Lock() + defer b.mu.Unlock() + if last.After(b.metadata[k]) { + b.metadata[k] = last + } +} + +func (b *hitBatch) takeMetadata() map[metadataHitKey]time.Time { + b.mu.Lock() + defer b.mu.Unlock() + metadata := b.metadata + b.metadata = map[metadataHitKey]time.Time{} + return metadata +} + func (b *hitBatch) take() map[hitKey]hitEntry { b.mu.Lock() defer b.mu.Unlock() @@ -57,7 +79,7 @@ func (db *DB) BatchHits(interval time.Duration, logger *slog.Logger) { if logger == nil { logger = slog.New(slog.DiscardHandler) } - b := &hitBatch{pending: map[hitKey]hitEntry{}, stop: make(chan struct{}), done: make(chan struct{})} + b := &hitBatch{pending: map[hitKey]hitEntry{}, metadata: map[metadataHitKey]time.Time{}, stop: make(chan struct{}), done: make(chan struct{})} db.hits = b go func() { defer close(b.done) @@ -83,9 +105,10 @@ func (db *DB) BatchHits(interval time.Duration, logger *slog.Logger) { // fails they are put back for the next flush, so a failed write delays // counts rather than losing them. func (db *DB) flushHits() error { + metadataErr := db.flushMetadataHits() pending := db.hits.take() if len(pending) == 0 { - return nil + return metadataErr } err := db.writeHits(pending) @@ -94,9 +117,54 @@ func (db *DB) flushHits() error { db.hits.add(k, e) } } + return errors.Join(err, metadataErr) +} + +// flushMetadataHits writes the pending metadata hits like flushHits. +func (db *DB) flushMetadataHits() error { + metadata := db.hits.takeMetadata() + if len(metadata) == 0 { + return nil + } + err := db.writeMetadataHits(metadata) + if err != nil { + for k, last := range metadata { + db.hits.addMetadata(k, last) + } + } return err } +func (db *DB) writeMetadataHits(metadata map[metadataHitKey]time.Time) error { + tx, err := db.Beginx() + if err != nil { + return err + } + defer func() { _ = tx.Rollback() }() + + stmt, err := tx.Preparex(db.Rebind(recordMetadataHitQuery)) + if err != nil { + return err + } + defer func() { _ = stmt.Close() }() + + // Sorted for the same reason as the artifact hits. + keys := make([]metadataHitKey, 0, len(metadata)) + for k := range metadata { + keys = append(keys, k) + } + slices.SortFunc(keys, func(a, b metadataHitKey) int { + return cmp.Or(cmp.Compare(a.ecosystem, b.ecosystem), cmp.Compare(a.name, b.name)) + }) + for _, k := range keys { + last := metadata[k].UTC() + if _, err := stmt.Exec(last, last, k.ecosystem, k.name); err != nil { + return err + } + } + return tx.Commit() +} + func (db *DB) writeHits(pending map[hitKey]hitEntry) error { tx, err := db.Beginx() if err != nil { diff --git a/internal/database/queries.go b/internal/database/queries.go index f1cdbe9..49d4c9d 100644 --- a/internal/database/queries.go +++ b/internal/database/queries.go @@ -526,13 +526,16 @@ const queuePendingDeleteQuery = ` ON CONFLICT(path) DO UPDATE SET queued_at = excluded.queued_at` // GetDuePendingDeletes returns up to limit paths queued before cutoff, oldest -// first. A path a record points at again is skipped. +// first. A path a record points at again is skipped: a metadata document is +// stored at a path fixed by its key, so one fetched again after eviction +// reuses the path queued for it. func (db *DB) GetDuePendingDeletes(cutoff time.Time, limit int) ([]string, error) { var paths []string query := db.Rebind(` SELECT path FROM pending_deletes WHERE queued_at < ? AND NOT EXISTS (SELECT 1 FROM artifacts WHERE artifacts.storage_path = pending_deletes.path) + AND NOT EXISTS (SELECT 1 FROM metadata_cache WHERE metadata_cache.storage_path = pending_deletes.path) ORDER BY queued_at LIMIT ? `) @@ -1127,7 +1130,7 @@ func (db *DB) GetMetadataCache(ecosystem, name string) (*MetadataCacheEntry, err var entry MetadataCacheEntry query := db.Rebind(` SELECT id, ecosystem, name, storage_path, etag, link, content_type, content_encoding, - content_digest, size, last_modified, fetched_at, created_at, updated_at + content_digest, size, last_modified, fetched_at, last_accessed_at, created_at, updated_at FROM metadata_cache WHERE ecosystem = ? AND name = ? `) err := db.Get(&entry, query, ecosystem, name) @@ -1156,6 +1159,8 @@ func (db *DB) RefreshMetadataCache(ecosystem, name, digest string, fetchedAt tim return n > 0, err } +// UpsertMetadataCache writes entry and marks it accessed now, since a +// document is stored because a request asked for it. func (db *DB) UpsertMetadataCache(entry *MetadataCacheEntry) error { now := time.Now() // fetched_at is a TIMESTAMP without zone on Postgres, and lib/pq reads it @@ -1170,8 +1175,8 @@ func (db *DB) UpsertMetadataCache(entry *MetadataCacheEntry) error { if db.dialect == DialectPostgres { query = ` INSERT INTO metadata_cache (ecosystem, name, storage_path, etag, link, content_type, content_encoding, - content_digest, size, last_modified, fetched_at, created_at, updated_at) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13) + content_digest, size, last_modified, fetched_at, last_accessed_at, created_at, updated_at) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14) ON CONFLICT(ecosystem, name) DO UPDATE SET storage_path = EXCLUDED.storage_path, etag = EXCLUDED.etag, @@ -1182,13 +1187,14 @@ func (db *DB) UpsertMetadataCache(entry *MetadataCacheEntry) error { size = EXCLUDED.size, last_modified = EXCLUDED.last_modified, fetched_at = EXCLUDED.fetched_at, + last_accessed_at = EXCLUDED.last_accessed_at, updated_at = EXCLUDED.updated_at ` } else { query = ` INSERT INTO metadata_cache (ecosystem, name, storage_path, etag, link, content_type, content_encoding, - content_digest, size, last_modified, fetched_at, created_at, updated_at) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + content_digest, size, last_modified, fetched_at, last_accessed_at, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(ecosystem, name) DO UPDATE SET storage_path = excluded.storage_path, etag = excluded.etag, @@ -1199,13 +1205,14 @@ func (db *DB) UpsertMetadataCache(entry *MetadataCacheEntry) error { size = excluded.size, last_modified = excluded.last_modified, fetched_at = excluded.fetched_at, + last_accessed_at = excluded.last_accessed_at, updated_at = excluded.updated_at ` } _, err := db.Exec(query, entry.Ecosystem, entry.Name, entry.StoragePath, entry.ETag, entry.Link, - entry.ContentType, entry.ContentEncoding, entry.ContentDigest, entry.Size, entry.LastModified, fetchedAt, now, now, + entry.ContentType, entry.ContentEncoding, entry.ContentDigest, entry.Size, entry.LastModified, fetchedAt, now.UTC(), now, now, ) if err != nil { return fmt.Errorf("upserting metadata cache: %w", err) diff --git a/internal/database/schema.go b/internal/database/schema.go index ce12490..dc9fc80 100644 --- a/internal/database/schema.go +++ b/internal/database/schema.go @@ -116,10 +116,14 @@ CREATE TABLE IF NOT EXISTS metadata_cache ( size INTEGER, last_modified DATETIME, fetched_at DATETIME, + last_accessed_at DATETIME, created_at DATETIME, updated_at DATETIME ); CREATE UNIQUE INDEX IF NOT EXISTS idx_metadata_eco_name ON metadata_cache(ecosystem, name); +CREATE INDEX IF NOT EXISTS idx_metadata_storage_path ON metadata_cache(storage_path); +CREATE INDEX IF NOT EXISTS idx_metadata_last_used ON metadata_cache(COALESCE(last_accessed_at, updated_at)); +CREATE INDEX IF NOT EXISTS idx_artifacts_last_used ON artifacts(COALESCE(last_accessed_at, fetched_at)); CREATE TABLE IF NOT EXISTS metadata_dates ( ecosystem TEXT NOT NULL, @@ -236,10 +240,14 @@ CREATE TABLE IF NOT EXISTS metadata_cache ( size BIGINT, last_modified TIMESTAMP, fetched_at TIMESTAMP, + last_accessed_at TIMESTAMP, created_at TIMESTAMP, updated_at TIMESTAMP ); CREATE UNIQUE INDEX IF NOT EXISTS idx_metadata_eco_name ON metadata_cache(ecosystem, name); +CREATE INDEX IF NOT EXISTS idx_metadata_storage_path ON metadata_cache(storage_path); +CREATE INDEX IF NOT EXISTS idx_metadata_last_used ON metadata_cache((COALESCE(last_accessed_at, updated_at)) NULLS FIRST); +CREATE INDEX IF NOT EXISTS idx_artifacts_last_used ON artifacts((COALESCE(last_accessed_at, fetched_at)) NULLS FIRST); CREATE TABLE IF NOT EXISTS metadata_dates ( ecosystem TEXT NOT NULL, @@ -411,6 +419,7 @@ var migrations = []migration{ {"011_add_metadata_dates", migrateAddMetadataDates}, {"012_add_artifact_source", migrateAddArtifactSource}, {"013_add_artifact_manifest", migrateAddArtifactManifest}, + {"014_add_metadata_last_accessed", migrateAddMetadataLastAccessed}, } // isTableNotFound returns true if the error indicates a missing table. @@ -820,3 +829,37 @@ func migrateAddArtifactManifest(db *DB) error { } return nil } + +// migrateAddMetadataLastAccessed records when cached metadata was last served, +// for size eviction. Existing rows are backfilled in the background +// (BackfillMetadataLastAccessed), so startup does not wait for them. +func migrateAddMetadataLastAccessed(db *DB) error { + hasCol, err := db.HasColumn("metadata_cache", "last_accessed_at") + if err != nil { + return fmt.Errorf("checking column last_accessed_at: %w", err) + } + if !hasCol { + ts := sqliteDatetime + if db.dialect == DialectPostgres { + ts = postgresTimestamp + } + if _, err := db.Exec(fmt.Sprintf("ALTER TABLE metadata_cache ADD COLUMN last_accessed_at %s", ts)); err != nil { + return fmt.Errorf("adding column last_accessed_at to metadata_cache: %w", err) + } + } + indexes := []string{ + `CREATE INDEX IF NOT EXISTS idx_metadata_storage_path ON metadata_cache(storage_path)`, + `CREATE INDEX IF NOT EXISTS idx_metadata_last_used ON metadata_cache(COALESCE(last_accessed_at, updated_at))`, + `CREATE INDEX IF NOT EXISTS idx_artifacts_last_used ON artifacts(COALESCE(last_accessed_at, fetched_at))`, + } + if db.dialect == DialectPostgres { + indexes[1] = `CREATE INDEX IF NOT EXISTS idx_metadata_last_used ON metadata_cache((COALESCE(last_accessed_at, updated_at)) NULLS FIRST)` + indexes[2] = `CREATE INDEX IF NOT EXISTS idx_artifacts_last_used ON artifacts((COALESCE(last_accessed_at, fetched_at)) NULLS FIRST)` + } + for _, q := range indexes { + if _, err := db.Exec(q); err != nil { + return fmt.Errorf("indexing metadata_cache: %w", err) + } + } + return nil +} diff --git a/internal/database/types.go b/internal/database/types.go index e931f5b..fb49a1c 100644 --- a/internal/database/types.go +++ b/internal/database/types.go @@ -187,6 +187,7 @@ type MetadataCacheEntry struct { Size sql.NullInt64 `db:"size" json:"size,omitempty"` LastModified sql.NullTime `db:"last_modified" json:"last_modified,omitempty"` FetchedAt sql.NullTime `db:"fetched_at" json:"fetched_at,omitempty"` + LastAccessedAt sql.NullTime `db:"last_accessed_at" json:"last_accessed_at,omitempty"` CreatedAt time.Time `db:"created_at" json:"created_at"` UpdatedAt time.Time `db:"updated_at" json:"updated_at"` } diff --git a/internal/handler/arch.go b/internal/handler/arch.go index f2abedf..8b3961f 100644 --- a/internal/handler/arch.go +++ b/internal/handler/arch.go @@ -336,7 +336,7 @@ func (h *ArchHandler) storeDatabase(ctx context.Context, d archDatabase, meta *u defer h.lock(d.dbKey)() previous, _ := p.DB.GetMetadataCache(archEcosystem, d.dbKey) - if previous != nil && previous.StoragePath == path { + if previous != nil && previous.StoragePath == path && h.blobExists(ctx, path) { // Same bytes: a failed rewrite would delete the blob the row uses. h.touchDatabase(previous, meta) } else { @@ -356,6 +356,13 @@ func (h *ArchHandler) storeDatabase(ctx context.Context, d archDatabase, meta *u return metadataResult{body: meta.body, contentType: meta.contentType, contentEncoding: meta.contentEncoding}, row } +// blobExists reports whether path is stored. Size eviction can remove a blob +// while a refresh re-creates its row, and only a rewrite brings it back. +func (h *ArchHandler) blobExists(ctx context.Context, path string) bool { + ok, err := h.proxy.Storage.Exists(ctx, path) + return ok || err != nil +} + // touchDatabase marks an unchanged cached database as fetched now. func (h *ArchHandler) touchDatabase(row *database.MetadataCacheEntry, meta *upstreamMetadata) { updated := *row diff --git a/internal/handler/container_manifest.go b/internal/handler/container_manifest.go index 902ad3a..62609da 100644 --- a/internal/handler/container_manifest.go +++ b/internal/handler/container_manifest.go @@ -276,6 +276,7 @@ func (h *ContainerHandler) loadContainerManifest(ctx context.Context, cacheKey s if err != nil { return nil, err } + h.proxy.recordMetadataRead(entry) manifest := &cachedContainerManifest{body: body, size: int64(len(body))} if entry.ContentType.Valid { diff --git a/internal/handler/container_metadata.go b/internal/handler/container_metadata.go index 1168ebd..9e1b24a 100644 --- a/internal/handler/container_metadata.go +++ b/internal/handler/container_metadata.go @@ -57,5 +57,6 @@ func (h *ContainerHandler) loadContainerMetadata(ctx context.Context, ecosystem, if err != nil { return nil, nil, err } + h.proxy.recordMetadataRead(entry) return entry, body, nil } diff --git a/internal/handler/container_tags.go b/internal/handler/container_tags.go index 6142590..cca97b6 100644 --- a/internal/handler/container_tags.go +++ b/internal/handler/container_tags.go @@ -151,6 +151,7 @@ func (h *ContainerHandler) loadContainerTags(ctx context.Context, cacheKey strin if err != nil { return nil, err } + h.proxy.recordMetadataRead(entry) tags := &cachedContainerTags{body: body, contentType: contentTypeJSON, size: int64(len(body))} if entry.ContentType.Valid { diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 46b93ca..6b3a5c3 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -1516,6 +1516,7 @@ func (p *Proxy) readCachedMetadata(ctx context.Context, entry *database.Metadata if err != nil { return nil, "", err } + p.recordMetadataRead(entry) ct := contentTypeJSON if entry.ContentType.Valid { ct = entry.ContentType.String @@ -1523,6 +1524,14 @@ func (p *Proxy) readCachedMetadata(ctx context.Context, entry *database.Metadata return data, ct, nil } +// recordMetadataRead marks a cached metadata document as used, so size +// eviction drops the documents nobody asks for first. +func (p *Proxy) recordMetadataRead(entry *database.MetadataCacheEntry) { + if err := p.DB.RecordMetadataHit(entry.Ecosystem, entry.Name); err != nil { + p.Logger.Warn("failed to record metadata cache hit", "ecosystem", entry.Ecosystem, "key", entry.Name, "error", err) + } +} + func (p *Proxy) recordMetadataCacheMiss(ecosystem string) { if p.CacheMetadata { metrics.RecordCacheMiss(ecosystem) @@ -1640,6 +1649,7 @@ func (p *Proxy) refreshMetadataBlob(ecosystem, cacheKey string, meta *upstreamMe if !refreshed { return "" } + p.recordMetadataRead(entry) return entry.ContentDigest.String } diff --git a/internal/handler/metadata_access_test.go b/internal/handler/metadata_access_test.go new file mode 100644 index 0000000..eda97e4 --- /dev/null +++ b/internal/handler/metadata_access_test.go @@ -0,0 +1,181 @@ +package handler + +import ( + "context" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/git-pkgs/proxy/internal/database" +) + +// ageMetadataAccess moves a cache row's last access an hour back and returns +// that time. +func ageMetadataAccess(t *testing.T, db *database.DB, ecosystem, key string) time.Time { + t.Helper() + old := time.Now().Add(-time.Hour).UTC() + query := db.Rebind(`UPDATE metadata_cache SET last_accessed_at = ? WHERE ecosystem = ? AND name = ?`) + if _, err := db.Exec(query, old, ecosystem, key); err != nil { + t.Fatalf("ageing last access: %v", err) + } + return old +} + +func assertMetadataAccessedAfter(t *testing.T, db *database.DB, ecosystem, key string, old time.Time) { + t.Helper() + entry, err := db.GetMetadataCache(ecosystem, key) + if err != nil || entry == nil { + t.Fatalf("GetMetadataCache: %v, %v", entry, err) + } + if !entry.LastAccessedAt.Valid || !entry.LastAccessedAt.Time.After(old.Add(time.Minute)) { + t.Errorf("last_accessed_at = %v, want the time it was served from cache", entry.LastAccessedAt) + } +} + +func TestFetchOrCacheMetadata_CacheHitMarksAccess(t *testing.T) { + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"v":1}`)) + })) + t.Cleanup(upstream.Close) + + proxy, db, _, _ := setupTestProxy(t) + proxy.CacheMetadata = true + proxy.MetadataTTL = time.Hour + proxy.HTTPClient = upstream.Client() + ctx := context.Background() + + if _, _, err := proxy.FetchOrCacheMetadata(ctx, "test", "pkg", upstream.URL+"/pkg"); err != nil { + t.Fatalf("first fetch: %v", err) + } + old := ageMetadataAccess(t, db, "test", "pkg") + if _, _, err := proxy.FetchOrCacheMetadata(ctx, "test", "pkg", upstream.URL+"/pkg"); err != nil { + t.Fatalf("cached fetch: %v", err) + } + assertMetadataAccessedAfter(t, db, "test", "pkg", old) +} + +func TestProxyCached_NotModifiedMarksAccess(t *testing.T) { + const body = `{"v":1}` + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("ETag", `"v1"`) + if r.Header.Get("If-None-Match") == `"v1"` { + w.WriteHeader(http.StatusNotModified) + return + } + _, _ = w.Write([]byte(body)) + })) + t.Cleanup(upstream.Close) + + proxy, db, store, _ := setupTestProxy(t) + proxy.CacheMetadata = true + proxy.MetadataTTL = time.Nanosecond + proxy.HTTPClient = upstream.Client() + + for range 2 { + response := httptest.NewRecorder() + proxy.ProxyCached(response, httptest.NewRequest(http.MethodGet, "/pkg", nil), upstream.URL+"/pkg", "test", "pkg") + if response.Code != http.StatusOK || response.Body.String() != body { + t.Fatalf("response = %d %q, want 200 %q", response.Code, response.Body.String(), body) + } + if store.stores != 1 { + t.Fatalf("document stored %d times, want once", store.stores) + } + assertMetadataAccessedAfter(t, db, "test", "pkg", time.Now().Add(-time.Hour)) + ageMetadataAccess(t, db, "test", "pkg") + } +} + +func TestLoadContainerMetadata_MarksAccess(t *testing.T) { + proxy, db, _, _ := setupTestProxy(t) + h := NewContainerHandlerWithRegistry(proxy, "http://proxy.example.test", "") + ctx := context.Background() + + if _, err := h.storeContainerMetadata(ctx, "oci-test", "key", []byte("{}"), "", "", "application/json", "", time.Time{}, time.Now()); err != nil { + t.Fatalf("storing: %v", err) + } + old := ageMetadataAccess(t, db, "oci-test", "key") + if entry, _, err := h.loadContainerMetadata(ctx, "oci-test", "key"); err != nil || entry == nil { + t.Fatalf("loading: %v, %v", entry, err) + } + assertMetadataAccessedAfter(t, db, "oci-test", "key", old) +} + +func TestLoadContainerManifest_MarksAccess(t *testing.T) { + proxy, db, _, _ := setupTestProxy(t) + h := NewContainerHandlerWithRegistry(proxy, "http://proxy.example.test", "") + ctx := context.Background() + + manifest := &cachedContainerManifest{body: []byte("{}"), contentType: "application/json", fetchedAt: time.Now()} + if err := h.storeContainerManifest(ctx, "key", manifest); err != nil { + t.Fatalf("storing: %v", err) + } + old := ageMetadataAccess(t, db, containerManifestCacheEcosystem, "key") + if got, err := h.loadContainerManifest(ctx, "key"); err != nil || got == nil { + t.Fatalf("loading: %v, %v", got, err) + } + assertMetadataAccessedAfter(t, db, containerManifestCacheEcosystem, "key", old) +} + +func TestLoadContainerTags_MarksAccess(t *testing.T) { + proxy, db, _, _ := setupTestProxy(t) + h := NewContainerHandlerWithRegistry(proxy, "http://proxy.example.test", "") + ctx := context.Background() + + tags := &cachedContainerTags{body: []byte(`{"tags":[]}`), contentType: "application/json", fetchedAt: time.Now()} + if err := h.storeContainerTags(ctx, "key", tags); err != nil { + t.Fatalf("storing: %v", err) + } + old := ageMetadataAccess(t, db, containerTagsCacheEcosystem, "key") + if got, err := h.loadContainerTags(ctx, "key"); err != nil || got == nil { + t.Fatalf("loading: %v, %v", got, err) + } + assertMetadataAccessedAfter(t, db, containerTagsCacheEcosystem, "key", old) +} + +func TestStoredRewrite_MarksAccess(t *testing.T) { + proxy, db, _, _ := setupTestProxy(t) + proxy.CacheMetadata = true + proxy.MetadataTTL = time.Hour + proxy.SetMetadataRewriteCacheSize(1 << 20) + + body := []byte(`{}`) + proxy.cacheMetadataBlob(t.Context(), "npm", "demo", metadataStoragePath("npm", "demo"), &upstreamMetadata{body: body}) + key := rewriteCacheKey("npm", "http://proxy", "demo", body) + if _, err := proxy.rewrites.rewrite(t.Context(), key, body, func(b []byte) ([]byte, error) { return b, nil }); err != nil { + t.Fatal(err) + } + old := ageMetadataAccess(t, db, "npm", "demo") + if _, ok := proxy.storedRewrite("npm", "demo", "http://proxy", "demo"); !ok { + t.Fatal("expected a stored rewrite") + } + assertMetadataAccessedAfter(t, db, "npm", "demo", old) +} + +// TestArchHandler_RefreshRestoresEvictedBlob covers a database row whose blob +// size eviction removed while a refresh re-created the row: the next refresh +// with the same bytes stores the blob again, so the stale fallback works. +func TestArchHandler_RefreshRestoresEvictedBlob(t *testing.T) { + db := []byte("core-db-bytes") + upstream := newArchUpstream(t, map[string][]byte{"/core/os/x86_64/core.db": db}) + proxy := newArchTestProxy(t, upstream) + h := NewArchHandler(proxy, "http://proxy.example", map[string]string{"archlinux": upstream.URL}) + path := "/archlinux/core/os/x86_64/core.db" + + assertArchBody(t, serveArchRequest(h, http.MethodGet, path, nil), "seed", db) + entry, err := proxy.DB.GetMetadataCache(archEcosystem, h.metadataCacheKey("archlinux", upstream.URL, "core/os/x86_64/core.db")) + if err != nil || entry == nil { + t.Fatalf("metadata entry: %v, %v", entry, err) + } + if err := proxy.Storage.Delete(t.Context(), entry.StoragePath); err != nil { + t.Fatalf("deleting blob: %v", err) + } + + ageMetadata(t, proxy, h, "archlinux", upstream.URL, "core/os/x86_64/core.db") + assertArchBody(t, serveArchRequest(h, http.MethodGet, path, nil), "refresh", db) + + ageMetadata(t, proxy, h, "archlinux", upstream.URL, "core/os/x86_64/core.db") + upstream.available.Store(false) + assertArchBody(t, serveArchRequest(h, http.MethodGet, path, nil), "stale", db) +} diff --git a/internal/handler/rewrite_cache.go b/internal/handler/rewrite_cache.go index 23a996d..a96bfdc 100644 --- a/internal/handler/rewrite_cache.go +++ b/internal/handler/rewrite_cache.go @@ -328,6 +328,7 @@ func (p *Proxy) storedRewrite(ecosystem, cacheKey, proxyURL, name string) (rewri } m.lastModified = p.metadataDate(ecosystem, proxyURL, name, m.etag) metrics.RecordCacheHit(ecosystem) + p.recordMetadataRead(entry) return m, true } diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go index 5e9e1a6..da7e729 100644 --- a/internal/metrics/metrics.go +++ b/internal/metrics/metrics.go @@ -70,6 +70,14 @@ var ( []string{"reason", "ecosystem"}, ) + MetadataEvicted = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "proxy_metadata_evicted_total", + Help: "Total number of cached metadata documents evicted by the size limit, by ecosystem", + }, + []string{"ecosystem"}, + ) + // Upstream metrics UpstreamFetchDuration = prometheus.NewHistogramVec( prometheus.HistogramOpts{ @@ -267,6 +275,7 @@ func init() { CacheSize, CachedArtifacts, ArtifactsEvicted, + MetadataEvicted, UpstreamFetchDuration, UpstreamErrors, CircuitBreakerState, @@ -337,6 +346,13 @@ func RecordArtifactEvicted(reason, ecosystem string) { ArtifactsEvicted.WithLabelValues(reason, purl.NormalizeEcosystem(ecosystem)).Inc() } +// RecordMetadataEvicted counts one metadata document evicted by the size +// limit. ecosystem is the metadata cache's, normalized like the cache hit +// counter. +func RecordMetadataEvicted(ecosystem string) { + MetadataEvicted.WithLabelValues(purl.NormalizeEcosystem(ecosystem)).Inc() +} + // RecordCacheMiss increments cache miss counter. func RecordCacheMiss(ecosystem string) { CacheMisses.WithLabelValues(purl.NormalizeEcosystem(ecosystem)).Inc() diff --git a/internal/server/analytics_coverage_test.go b/internal/server/analytics_coverage_test.go index f5f711e..f97cf4b 100644 --- a/internal/server/analytics_coverage_test.go +++ b/internal/server/analytics_coverage_test.go @@ -46,6 +46,7 @@ var metricSurface = map[string]string{ "proxy_circuit_breaker_state": "Runtime: Circuit breakers", "proxy_circuit_breaker_trips_total": "Runtime: Circuit breaker trips", "proxy_artifacts_evicted_total": "Runtime: Evicted artifacts", + "proxy_metadata_evicted_total": "Runtime: Evicted metadata", "proxy_scan_duration_seconds": "Runtime: Pre-cache scanning -- Scans", "proxy_scan_blocked_total": "Runtime: Pre-cache scanning -- Artifacts blocked", "proxy_scan_errors_total": "Runtime: Pre-cache scanning -- Scan errors", @@ -73,6 +74,7 @@ func TestEveryMetricIsSurfaced(t *testing.T) { metrics.RecordScanError("npm", "clamav", "timeout", metrics.FailureDetail{}) metrics.UpdateCacheStats(1, 1) metrics.RecordArtifactEvicted("retention", "npm") + metrics.RecordMetadataEvicted("npm") metrics.UpdateEcosystemStats([]metrics.EcosystemStats{{Ecosystem: "npm", DownloadedBytes: 1}}) families, err := prometheus.DefaultGatherer.Gather() diff --git a/internal/server/analytics_runtime.go b/internal/server/analytics_runtime.go index 8e5d0d1..c99ec97 100644 --- a/internal/server/analytics_runtime.go +++ b/internal/server/analytics_runtime.go @@ -33,6 +33,7 @@ type RuntimeView struct { IntegrityFailures []LabelledCount ProbeFailures []LabelledCount Evictions []LabelledCount + MetadataEvictions []LabelledCount Breakers []BreakerRow BreakerTrips []LabelledCount @@ -139,6 +140,7 @@ func runtimeView(snap *metrics.Snapshot, failureDetails bool) RuntimeView { v.IntegrityFailures = failures("proxy_integrity_failures_total", "ecosystem") v.ProbeFailures = failures("proxy_health_probe_failures_total", "step") v.Evictions = countRows(snap, "proxy_artifacts_evicted_total", "reason", "ecosystem") + v.MetadataEvictions = countRows(snap, "proxy_metadata_evicted_total", "ecosystem") v.Breakers = breakerRows(snap) v.BreakerTrips = failures("proxy_circuit_breaker_trips_total", "registry") diff --git a/internal/server/analytics_runtime_test.go b/internal/server/analytics_runtime_test.go index 5bff0f3..3e476e0 100644 --- a/internal/server/analytics_runtime_test.go +++ b/internal/server/analytics_runtime_test.go @@ -152,6 +152,24 @@ func TestRuntimeViewShowsEvictions(t *testing.T) { } } +func TestRuntimeViewShowsMetadataEvictions(t *testing.T) { + reg := prometheus.NewRegistry() + evicted := prometheus.NewCounterVec( + prometheus.CounterOpts{Name: "proxy_metadata_evicted_total", Help: "t"}, []string{"ecosystem"}) + reg.MustRegister(evicted) + evicted.WithLabelValues("golang").Add(3) + + snap, err := metrics.GatherFrom(reg) + if err != nil { + t.Fatalf("GatherFrom: %v", err) + } + v := runtimeView(snap, false) + + if len(v.MetadataEvictions) != 1 || v.MetadataEvictions[0].Label != "golang" || v.MetadataEvictions[0].Count != "3" { + t.Errorf("MetadataEvictions = %+v, want golang = 3", v.MetadataEvictions) + } +} + // A counter sitting at zero carries no information and would bury the rows // that do, so it is left out. func TestRuntimeViewDropsZeroCounters(t *testing.T) { diff --git a/internal/server/eviction.go b/internal/server/eviction.go index 7fe108a..5bbae41 100644 --- a/internal/server/eviction.go +++ b/internal/server/eviction.go @@ -6,6 +6,7 @@ import ( "time" "github.com/git-pkgs/proxy/internal/database" + "github.com/git-pkgs/proxy/internal/metrics" "github.com/git-pkgs/proxy/internal/storage" ) @@ -41,8 +42,10 @@ func (s *Server) runEviction(ctx context.Context, maxSize int64) { evictLRU(ctx, s.db, s.storage, s.logger, maxSize) } +// evictLRU removes the least recently used cached artifacts and metadata +// documents until what max_size counts falls to maxSize. func evictLRU(ctx context.Context, db *database.DB, store storage.Storage, logger *slog.Logger, maxSize int64) { - totalSize, err := db.GetTotalCacheSize() + totalSize, err := db.GetEvictableCacheSize() if err != nil { logger.Warn("eviction: failed to get cache size", "error", err) return @@ -63,16 +66,16 @@ func evictLRU(ctx context.Context, db *database.DB, store storage.Storage, logge break } - artifacts, err := db.GetLeastRecentlyUsedArtifacts(evictionBatch) + entries, err := db.GetLeastRecentlyUsedCacheEntries(evictionBatch) if err != nil { - logger.Warn("eviction: failed to get LRU artifacts", "error", err) + logger.Warn("eviction: failed to get LRU entries", "error", err) break } - if len(artifacts) == 0 { + if len(entries) == 0 { break } - cleared, freed := evictBatch(ctx, db, store, logger, artifacts, maxSize, totalSize-freedBytes) + cleared, freed := evictBatch(ctx, db, store, logger, entries, maxSize, totalSize-freedBytes) evicted += cleared freedBytes += freed @@ -84,8 +87,8 @@ func evictLRU(ctx context.Context, db *database.DB, store storage.Storage, logge // stop rather than retry them without end. The next pass retries after // the interval. if cleared == 0 { - logger.Warn("eviction: no artifact in this batch could be evicted, ending the pass", - "batch", len(artifacts)) + logger.Warn("eviction: no entry in this batch could be evicted, ending the pass", + "batch", len(entries)) break } } @@ -96,53 +99,76 @@ func evictLRU(ctx context.Context, db *database.DB, store storage.Storage, logge } } -// evictBatch evicts from artifacts until the size still in use falls to +// evictBatch evicts from entries until the size still in use falls to // maxSize, reporting how many records it cleared and how many bytes that // freed. Progress is reported in records as well as bytes because a cached // record can have no size recorded, and clearing one is still progress. func evictBatch(ctx context.Context, db *database.DB, store storage.Storage, logger *slog.Logger, - artifacts []database.Artifact, maxSize, inUse int64) (int, int64) { + entries []database.CacheEntry, maxSize, inUse int64) (int, int64) { cleared := 0 freed := int64(0) - for _, art := range artifacts { + for _, entry := range entries { if inUse-freed <= maxSize { break } if ctx.Err() != nil { break } - if !art.StoragePath.Valid { + if !entry.StoragePath.Valid { continue } - path := art.StoragePath.String + path := entry.StoragePath.String // Clear before deleting: a record a newer fetch moved has left this // path queued, and a request may still open it within the grace period. - recordCleared, err := db.ClearArtifactCache(art.VersionPURL, art.Filename, path) + recordCleared, err := clearEntry(db, logger, entry) if err != nil { - logger.Warn("eviction: failed to clear artifact record", - "version_purl", art.VersionPURL, "filename", art.Filename, "error", err) + logger.Warn("eviction: failed to clear record", + "kind", entry.Kind, "key", entry.Key, "name", entry.Name, "error", err) continue } if !recordCleared { continue } + // Metadata goes now too: readers hold the whole document in memory, + // and a row refetched meanwhile refetches again on a missing blob. if err := store.Delete(ctx, path); err != nil { logger.Warn("eviction: failed to delete from storage, queueing it", "path", path, "error", err) if err := db.QueuePendingDelete(path); err != nil { logger.Warn("eviction: failed to queue object for deletion", "path", path, "error", err) } + } else if entry.Kind == database.CacheEntryMetadata { + // Queued with the row's delete in case the process stopped here. + if err := db.RemovePendingDelete(path); err != nil { + logger.Warn("eviction: failed to dequeue deleted object", "path", path, "error", err) + } } - if art.Size.Valid { - freed += art.Size.Int64 + if entry.Size.Valid { + freed += entry.Size.Int64 } cleared++ - recordLRUEviction(db, logger, art.VersionPURL) } return cleared, freed } + +// clearEntry drops the record of entry if it still points at the path read +// for it, and counts the eviction. +func clearEntry(db *database.DB, logger *slog.Logger, entry database.CacheEntry) (bool, error) { + if entry.Kind == database.CacheEntryMetadata { + cleared, err := db.EvictMetadataCache(entry.Key, entry.Name, entry.StoragePath.String, entry.LastAccessedAt) + if cleared { + metrics.RecordMetadataEvicted(entry.Key) + } + return cleared, err + } + cleared, err := db.ClearArtifactCache(entry.Key, entry.Name, entry.StoragePath.String) + if cleared { + recordLRUEviction(db, logger, entry.Key) + } + return cleared, err +} diff --git a/internal/server/eviction_metadata_test.go b/internal/server/eviction_metadata_test.go new file mode 100644 index 0000000..ffeb9c0 --- /dev/null +++ b/internal/server/eviction_metadata_test.go @@ -0,0 +1,194 @@ +package server + +import ( + "context" + "database/sql" + "io" + "log/slog" + "slices" + "strings" + "testing" + "time" + + "github.com/git-pkgs/proxy/internal/database" + "github.com/git-pkgs/proxy/internal/storage" +) + +// seedMetadata caches a 500-byte metadata document last used at +// accessedAt and returns its storage path. +func seedMetadata(t *testing.T, ctx context.Context, db *database.DB, store storage.Storage, key string, accessedAt time.Time) string { + t.Helper() + path := "_metadata/golang/" + key + "/metadata" + size, _, err := store.Store(ctx, path, strings.NewReader(strings.Repeat("m", 500))) + if err != nil { + t.Fatalf("storing metadata: %v", err) + } + if err := db.UpsertMetadataCache(&database.MetadataCacheEntry{ + Ecosystem: "golang", + Name: key, + StoragePath: path, + Size: sql.NullInt64{Int64: size, Valid: true}, + FetchedAt: sql.NullTime{Time: time.Now(), Valid: true}, + }); err != nil { + t.Fatalf("upserting metadata: %v", err) + } + query := db.Rebind(`UPDATE metadata_cache SET last_accessed_at = ? WHERE ecosystem = ? AND name = ?`) + if _, err := db.Exec(query, accessedAt.UTC(), "golang", key); err != nil { + t.Fatalf("setting last access: %v", err) + } + return path +} + +func TestEvictLRU_CountsAndEvictsMetadata(t *testing.T) { + db, store := setupEvictionTest(t) + ctx := context.Background() + logger := slog.New(slog.NewTextHandler(io.Discard, nil)) + now := time.Now() + + oldTile := seedMetadata(t, ctx, db, store, "tile-old", now.Add(-3*time.Hour)) + seedArtifact(t, ctx, db, store, "mid-pkg", 500, now.Add(-2*time.Hour)) + newTile := seedMetadata(t, ctx, db, store, "tile-new", now) + + // Artifacts alone are within the limit; with metadata the cache is + // 1500 bytes, so the least recently used entry has to go, and that is + // the old metadata document. + evictLRU(ctx, db, store, logger, 1100) + + if entry, _ := db.GetMetadataCache("golang", "tile-old"); entry != nil { + t.Error("least recently used metadata row was kept") + } + if ok, _ := store.Exists(ctx, oldTile); ok { + t.Error("least recently used metadata blob was kept") + } + if entry, _ := db.GetMetadataCache("golang", "tile-new"); entry == nil { + t.Error("recently used metadata row was evicted") + } + if ok, _ := store.Exists(ctx, newTile); !ok { + t.Error("recently used metadata blob was deleted") + } + art, err := db.GetArtifact("pkg:npm/mid-pkg@1.0.0", "mid-pkg-1.0.0.tgz") + if err != nil || art == nil || !art.StoragePath.Valid { + t.Errorf("artifact used after the old metadata was evicted: %+v, %v", art, err) + } +} + +func TestEvictLRU_EvictsArtifactBeforeNewerMetadata(t *testing.T) { + db, store := setupEvictionTest(t) + ctx := context.Background() + logger := slog.New(slog.NewTextHandler(io.Discard, nil)) + now := time.Now() + + seedArtifact(t, ctx, db, store, "old-pkg", 500, now.Add(-3*time.Hour)) + seedMetadata(t, ctx, db, store, "tile", now) + + evictLRU(ctx, db, store, logger, 600) + + art, err := db.GetArtifact("pkg:npm/old-pkg@1.0.0", "old-pkg-1.0.0.tgz") + if err != nil || art == nil || art.StoragePath.Valid { + t.Errorf("older artifact kept: %+v, %v", art, err) + } + if entry, _ := db.GetMetadataCache("golang", "tile"); entry == nil { + t.Error("newer metadata evicted although evicting the artifact was enough") + } +} + +// TestEvictBatch_KeepsMetadataUsedSinceRead evicts from a row read before a +// request served the document again. The row and its blob stay, and nothing +// is counted as freed. +func TestEvictBatch_KeepsMetadataUsedSinceRead(t *testing.T) { + db, store := setupEvictionTest(t) + ctx := context.Background() + logger := slog.New(slog.NewTextHandler(io.Discard, nil)) + path := seedMetadata(t, ctx, db, store, "tile", time.Now().Add(-time.Hour)) + + stale, err := db.GetLeastRecentlyUsedCacheEntries(evictionBatch) + if err != nil || len(stale) != 1 { + t.Fatalf("reading LRU rows: %v, %v", stale, err) + } + if err := db.RecordMetadataHit("golang", "tile"); err != nil { + t.Fatalf("recording hit: %v", err) + } + + cleared, freed := evictBatch(ctx, db, store, logger, stale, 0, 500) + if cleared != 0 || freed != 0 { + t.Errorf("cleared %d records freeing %d bytes, want nothing", cleared, freed) + } + if entry, _ := db.GetMetadataCache("golang", "tile"); entry == nil { + t.Error("metadata row served since it was read was evicted") + } + if ok, _ := store.Exists(ctx, path); !ok { + t.Error("metadata blob served since it was read was deleted") + } +} + +// TestEvictLRU_MetadataLeavesNothingQueued checks that a completed eviction +// takes the path it queued with the row's delete off the queue again. +func TestEvictLRU_MetadataLeavesNothingQueued(t *testing.T) { + db, store := setupEvictionTest(t) + ctx := context.Background() + logger := slog.New(slog.NewTextHandler(io.Discard, nil)) + path := seedMetadata(t, ctx, db, store, "tile", time.Now().Add(-time.Hour)) + + evictLRU(ctx, db, store, logger, 0) + + if ok, _ := store.Exists(ctx, path); ok { + t.Error("metadata blob kept") + } + if got := queuedPaths(t, db); len(got) != 0 { + t.Errorf("queue = %v, want it empty after the delete", got) + } +} + +// TestEvictMetadata_InterruptedBeforeDeleteIsReclaimed stops an eviction +// after the row is gone but before the blob is deleted, as a crash would. The +// path is queued, so reclaim still deletes the blob. +func TestEvictMetadata_InterruptedBeforeDeleteIsReclaimed(t *testing.T) { + db, store := setupEvictionTest(t) + ctx := context.Background() + logger := slog.New(slog.NewTextHandler(io.Discard, nil)) + path := seedMetadata(t, ctx, db, store, "tile", time.Now().Add(-time.Hour)) + + entries, err := db.GetLeastRecentlyUsedCacheEntries(evictionBatch) + if err != nil || len(entries) != 1 { + t.Fatalf("reading LRU rows: %v, %v", entries, err) + } + e := entries[0] + if ok, err := db.EvictMetadataCache(e.Key, e.Name, path, e.LastAccessedAt); err != nil || !ok { + t.Fatalf("EvictMetadataCache = %v, %v", ok, err) + } + if got := queuedPaths(t, db); !slices.Equal(got, []string{path}) { + t.Fatalf("queue = %v, want the evicted path", got) + } + + // Nothing left for a restarted eviction pass to retry; reclaim takes it. + if again, _ := db.GetLeastRecentlyUsedCacheEntries(evictionBatch); len(again) != 0 { + t.Errorf("LRU after interruption = %+v, want nothing", again) + } + reclaimDue(ctx, db, store, logger, time.Now().Add(time.Hour), time.Minute) + if objectExists(t, store, path) { + t.Error("blob still stored after reclaim") + } + if got := queuedPaths(t, db); len(got) != 0 { + t.Errorf("queue = %v, want it drained", got) + } +} + +func TestBackfillMetadataAccessDrainsInBatches(t *testing.T) { + db, _ := setupEvictionTest(t) + for _, name := range []string{"a", "b", "c", "d", "e"} { + query := db.Rebind(`INSERT INTO metadata_cache (ecosystem, name, storage_path, size, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?)`) + if _, err := db.Exec(query, "npm", name, "_metadata/npm/"+name+"/metadata", 1, time.Now(), time.Now()); err != nil { + t.Fatalf("inserting: %v", err) + } + } + + backfillMetadataAccess(context.Background(), db, slog.New(slog.NewTextHandler(io.Discard, nil)), 2) + + var left int + if err := db.Get(&left, `SELECT COUNT(*) FROM metadata_cache WHERE last_accessed_at IS NULL`); err != nil { + t.Fatalf("counting: %v", err) + } + if left != 0 { + t.Errorf("%d rows left without an access time", left) + } +} diff --git a/internal/server/eviction_test.go b/internal/server/eviction_test.go index 2e61e71..0567499 100644 --- a/internal/server/eviction_test.go +++ b/internal/server/eviction_test.go @@ -458,6 +458,7 @@ func TestEvictBatch_SkipsRecordThatMoved(t *testing.T) { if err != nil || len(stale) != 1 { t.Fatalf("reading LRU rows: %v, %v", stale, err) } + entries := []database.CacheEntry{{Kind: database.CacheEntryArtifact, Key: stale[0].VersionPURL, Name: stale[0].Filename, StoragePath: stale[0].StoragePath, Size: stale[0].Size}} newer := storage.FetchPath("npm", "moved", "1.0.0", storage.NewFetchID(), "moved-1.0.0.tgz") if _, _, err := store.Store(ctx, newer, strings.NewReader("refetched")); err != nil { t.Fatalf("storing refetch: %v", err) @@ -468,7 +469,7 @@ func TestEvictBatch_SkipsRecordThatMoved(t *testing.T) { t.Fatalf("moving record: %v", err) } - cleared, freed := evictBatch(ctx, db, store, logger, stale, 0, 1000) + cleared, freed := evictBatch(ctx, db, store, logger, entries, 0, 1000) if cleared != 0 || freed != 0 { t.Errorf("cleared %d records freeing %d bytes, want nothing counted", cleared, freed) } diff --git a/internal/server/metadata_backfill.go b/internal/server/metadata_backfill.go new file mode 100644 index 0000000..7444d66 --- /dev/null +++ b/internal/server/metadata_backfill.go @@ -0,0 +1,38 @@ +package server + +import ( + "context" + "log/slog" + "time" + + "github.com/git-pkgs/proxy/internal/database" +) + +// metadataBackfillBatch bounds each backfill transaction, so other writers +// get the database in between. +const metadataBackfillBatch = 1000 + +// backfillMetadataAccess gives metadata rows cached before last_accessed_at +// existed their write time, batch by batch. It is resumable: a restart picks +// up whatever rows are still empty. +func backfillMetadataAccess(ctx context.Context, db *database.DB, logger *slog.Logger, batch int) { + total := 0 + for ctx.Err() == nil { + n, err := db.BackfillMetadataLastAccessed(batch) + if err != nil { + logger.Warn("metadata backfill failed, retrying", "error", err) + select { + case <-ctx.Done(): + case <-time.After(time.Minute): + } + continue + } + total += n + if n < batch { + break + } + } + if total > 0 { + logger.Info("metadata backfill: completed", "rows", total) + } +} diff --git a/internal/server/server.go b/internal/server/server.go index 25da77f..c64b33c 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -385,6 +385,7 @@ func (s *Server) serve(listener net.Listener) error { s.logger.Warn(warning) } go s.updateCacheStatsMetrics() + go backfillMetadataAccess(bgCtx, s.db, s.logger, metadataBackfillBatch) go s.startEvictionLoop(bgCtx) go s.startRetentionLoop(bgCtx) go s.startReclaimLoop(bgCtx) diff --git a/internal/server/templates/components/runtime_metrics.html b/internal/server/templates/components/runtime_metrics.html index 9804b47..6eba427 100644 --- a/internal/server/templates/components/runtime_metrics.html +++ b/internal/server/templates/components/runtime_metrics.html @@ -61,6 +61,7 @@ {{template "runtime_counts" dict "Title" "Integrity failures" "Rows" .IntegrityFailures "Empty" "None"}} {{template "runtime_counts" dict "Title" "Health probe failures" "Rows" .ProbeFailures "Empty" "None"}} {{template "runtime_counts" dict "Title" "Evicted artifacts" "Rows" .Evictions "Empty" "None"}} + {{template "runtime_counts" dict "Title" "Evicted metadata" "Rows" .MetadataEvictions "Empty" "None"}} {{template "runtime_counts" dict "Title" "Circuit breaker trips" "Rows" .BreakerTrips "Empty" "None"}}
From 6857fad20ee0d0b0e55676a9d4af044517eaf8c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=91=D0=B0=D1=82-=D0=9E=D1=80=D1=88=D0=B8=D1=85?= <34618353+orshih6@users.noreply.github.com> Date: Tue, 20 Oct 2026 18:22:50 +0800 Subject: [PATCH 324/633] Abort the response when a cached artifact fails its integrity check (#504) * Abort the response when a cached artifact fails its integrity check * Read the seeded artifact's storage path from the database in the integrity test --- internal/handler/handler.go | 2 +- internal/handler/integrity.go | 84 ++++++++++++++++++++--- internal/handler/integrity_test.go | 103 +++++++++++++++++++++++++++++ 3 files changed, 177 insertions(+), 12 deletions(-) diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 6b3a5c3..7ffe718 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -465,7 +465,7 @@ func (p *Proxy) readStored(ctx context.Context, pkgPURL, versionPURL, filename s return nil, nil } - result.Reader, err = checks.wrap(reader, + result.Reader, err = checks.wrapFailOnMismatch(reader, func(reason string) { p.Logger.Error("cached artifact failed integrity check", "purl", versionPURL, "filename", filename, diff --git a/internal/handler/integrity.go b/internal/handler/integrity.go index 7e800b8..a04555f 100644 --- a/internal/handler/integrity.go +++ b/internal/handler/integrity.go @@ -79,10 +79,22 @@ func verifyIntegrity(body io.ReadCloser, declared string) (_ io.Reader, verified } func (c integrityChecks) wrap(source io.ReadCloser, onMismatch func(string)) (io.ReadCloser, error) { + return c.wrapSeekable(source, onMismatch, false) +} + +// wrapFailOnMismatch is wrap for bytes that are served to a client: the +// final chunk is held back until the digest is checked, and a mismatch drops +// it and is returned from Read as ErrArtifactDigestMismatch in place of +// io.EOF, so the caller can abort rather than complete the response. +func (c integrityChecks) wrapFailOnMismatch(source io.ReadCloser, onMismatch func(string)) (io.ReadCloser, error) { + return c.wrapSeekable(source, onMismatch, true) +} + +func (c integrityChecks) wrapSeekable(source io.ReadCloser, onMismatch func(string), failOnMismatch bool) (io.ReadCloser, error) { if len(c.algorithms) == 0 { return source, nil } - verified, err := c.newVerifyingReader(source, onMismatch, false) + verified, err := c.newVerifyingReader(source, onMismatch, failOnMismatch) if err != nil { return nil, err } @@ -97,13 +109,6 @@ func (c integrityChecks) wrap(source io.ReadCloser, onMismatch func(string)) (io }, nil } -// wrapFailOnMismatch is wrap for bytes that have not been checked anywhere -// else: a mismatch is also returned from Read as ErrArtifactDigestMismatch in -// place of io.EOF, so the caller can abort rather than complete the response. -func (c integrityChecks) wrapFailOnMismatch(source io.ReadCloser, onMismatch func(string)) (io.ReadCloser, error) { - return c.newVerifyingReader(source, onMismatch, true) -} - func (c integrityChecks) newVerifyingReader(source io.ReadCloser, onMismatch func(string), failOnMismatch bool) (io.ReadCloser, error) { if len(c.algorithms) == 0 { return source, nil @@ -132,6 +137,14 @@ type verifyingReader struct { failOnMismatch bool mismatched bool + + // With failOnMismatch, held (backed by buf) is read but not yet + // returned, and only its first releasable bytes are known not to end the + // body. The rest is the latest chunk, returned once the digest matches. + buf []byte + held []byte + releasable int + eof bool } // seekableVerifyingReader verifies ordinary reads until a successful seek, @@ -163,16 +176,65 @@ func (r *seekableVerifyingReader) Close() error { } func (r *verifyingReader) Read(p []byte) (int, error) { + if r.failOnMismatch { + return r.readHoldingBack(p) + } n, err := r.reader.Read(p) if err == io.EOF { r.verify() - if r.failOnMismatch && r.mismatched { - return n, ErrArtifactDigestMismatch - } } return n, err } +// readHoldingBack returns bytes one chunk behind the source, so the final +// chunk is released only after the digest over the whole body has matched. +func (r *verifyingReader) readHoldingBack(p []byte) (int, error) { + if len(p) == 0 { + return 0, nil + } + for r.releasable == 0 && !r.eof { + // Move the held chunk to the front of the buffer and read the next + // chunk after it; once that arrives, the held chunk is not the last. + var pending int + if cap(r.buf) < len(r.held)+len(p) { + buf := make([]byte, len(r.held)+len(p)) + pending = copy(buf, r.held) + r.buf = buf + } else { + pending = copy(r.buf[:cap(r.buf)], r.held) + } + n, err := r.reader.Read(r.buf[pending : pending+len(p)]) + r.held = r.buf[:pending+n] + if n > 0 { + r.releasable = pending + } + switch { + case err == io.EOF: + r.verify() + r.eof = true + case err != nil: + r.held, r.releasable = nil, 0 + return 0, err + case n == 0: + return 0, nil + } + } + if r.eof { + if r.mismatched { + r.held, r.releasable = nil, 0 + return 0, ErrArtifactDigestMismatch + } + r.releasable = len(r.held) + if r.releasable == 0 { + return 0, io.EOF + } + } + n := copy(p, r.held[:r.releasable]) + r.held = r.held[n:] + r.releasable -= n + return n, nil +} + func (r *verifyingReader) Close() error { return r.source.Close() } diff --git a/internal/handler/integrity_test.go b/internal/handler/integrity_test.go index 16f47b1..cb84b14 100644 --- a/internal/handler/integrity_test.go +++ b/internal/handler/integrity_test.go @@ -8,9 +8,12 @@ import ( "encoding/hex" "errors" "io" + "net/http" + "net/http/httptest" "slices" "strings" "testing" + "testing/iotest" ) func sha256Hex(data string) string { @@ -348,3 +351,103 @@ func TestVerifyingReaderVerifyOnce(t *testing.T) { t.Errorf("onMismatch called %d times, want 1", calls) } } + +// TestCorruptCachedArtifactAbortsResponse serves a cached artifact whose +// stored bytes no longer match its digest through a real server and client. +// The client must see a failed, short body rather than a complete response. +func TestCorruptCachedArtifactAbortsResponse(t *testing.T) { + proxy, db, store, _ := setupTestProxy(t) + content := strings.Repeat("cached content ", 10000) + seedPackage(t, db, store, "npm", "corrupt", "1.0.0", "corrupt-1.0.0.tgz", content) + seeded, err := db.GetArtifact("pkg:npm/corrupt@1.0.0", "corrupt-1.0.0.tgz") + if err != nil || seeded == nil || !seeded.StoragePath.Valid { + t.Fatalf("seeded record = %+v (err %v), want a storage path", seeded, err) + } + tampered := []byte(content) + tampered[len(tampered)-1] = '!' + store.files[seeded.StoragePath.String] = tampered + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + result, err := proxy.GetCachedArtifact(r.Context(), "npm", "corrupt", "1.0.0", "corrupt-1.0.0.tgz") + if err != nil || result == nil { + http.Error(w, "not cached", http.StatusInternalServerError) + return + } + ServeArtifactRequest(w, r, result) + })) + t.Cleanup(server.Close) + + resp, err := server.Client().Get(server.URL) + if err != nil { + t.Fatalf("GET: %v", err) + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode != http.StatusOK { + t.Fatalf("status = %d, want %d", resp.StatusCode, http.StatusOK) + } + if resp.ContentLength != int64(len(content)) { + t.Fatalf("Content-Length = %d, want %d", resp.ContentLength, len(content)) + } + body, err := io.ReadAll(resp.Body) + if err == nil { + t.Error("reading a corrupted cached artifact succeeded, want an error") + } + if int64(len(body)) >= resp.ContentLength { + t.Errorf("read %d bytes, want fewer than Content-Length %d", len(body), resp.ContentLength) + } + if !bytes.HasPrefix(tampered, body) { + t.Error("body is not a prefix of the stored bytes") + } + + artifact, err := db.GetArtifact("pkg:npm/corrupt@1.0.0", "corrupt-1.0.0.tgz") + if err != nil || artifact == nil || artifact.StoragePath.Valid { + t.Errorf("record = %+v (err %v), want it cleared", artifact, err) + } +} + +func TestFailOnMismatchHoldsBackFinalChunk(t *testing.T) { + data := strings.Repeat("0123456789", 1000) + for _, tt := range []struct { + name string + source func(string) io.Reader + }{ + {"single reads", func(s string) io.Reader { return strings.NewReader(s) }}, + {"one byte reads", func(s string) io.Reader { return iotest.OneByteReader(strings.NewReader(s)) }}, + {"data with EOF", func(s string) io.Reader { return iotest.DataErrReader(strings.NewReader(s)) }}, + } { + t.Run(tt.name, func(t *testing.T) { + checks, err := newIntegrityChecks(sha256Hex(data), "") + if err != nil { + t.Fatal(err) + } + + reader, err := checks.wrapFailOnMismatch(io.NopCloser(tt.source(data)), func(string) { + t.Error("onMismatch called for matching bytes") + }) + if err != nil { + t.Fatal(err) + } + got, err := io.ReadAll(reader) + if err != nil || string(got) != data { + t.Errorf("matching read = %d bytes, %v; want %d bytes, nil", len(got), err, len(data)) + } + + var calls int + tampered := data[:len(data)-1] + "!" + reader, err = checks.wrapFailOnMismatch(io.NopCloser(tt.source(tampered)), func(string) { calls++ }) + if err != nil { + t.Fatal(err) + } + got, err = io.ReadAll(reader) + if !errors.Is(err, ErrArtifactDigestMismatch) { + t.Errorf("mismatched read error = %v, want ErrArtifactDigestMismatch", err) + } + if len(got) >= len(tampered) || !strings.HasPrefix(tampered, string(got)) { + t.Errorf("mismatched read returned %d of %d bytes, want a strict prefix", len(got), len(tampered)) + } + if calls != 1 { + t.Errorf("onMismatch called %d times, want 1", calls) + } + }) + } +} From a97b6908582ca463f246000b94b63aec7b608176 Mon Sep 17 00:00:00 2001 From: Abhinav Gautam Date: Tue, 20 Oct 2026 15:55:33 +0530 Subject: [PATCH 325/633] Separate artifact and metadata cache metrics on /metrics and the analytics page (#463) * Separate artifact and metadata cache metrics on /metrics and the analytics page * Serve metadata cache stats from a shared snapshot and split Grafana cache panels by kind --- README.md | 15 ++- deploy/grafana/git-pkgs-proxy.json | 12 +- internal/database/metadata_stats_test.go | 60 +++++++++ internal/database/queries.go | 56 +++++++++ internal/handler/arch.go | 4 +- internal/handler/go_sumdb.go | 2 +- internal/handler/gradle.go | 8 +- internal/handler/gradle_test.go | 8 +- internal/handler/handler.go | 10 +- internal/handler/handler_test.go | 44 ++++--- internal/handler/npm_rewrite_test.go | 42 +++++++ internal/handler/rewrite_cache.go | 2 +- internal/metrics/metrics.go | 114 ++++++++++++++++-- internal/metrics/metrics_test.go | 109 ++++++++++++++--- internal/metrics/snapshot_test.go | 2 +- internal/server/analytics.go | 59 +++++++++ internal/server/analytics_coverage_test.go | 14 ++- internal/server/analytics_runtime.go | 52 +++++--- internal/server/analytics_runtime_test.go | 66 ++++++++-- internal/server/analytics_test.go | 93 ++++++++++++++ internal/server/ecosystem_cache.go | 56 ++++++--- internal/server/ecosystem_cache_test.go | 47 ++++++++ internal/server/server.go | 19 +++ .../templates/components/runtime_metrics.html | 38 ++++-- .../server/templates/pages/analytics.html | 52 ++++++++ internal/server/templates_test.go | 25 ++-- 26 files changed, 872 insertions(+), 137 deletions(-) create mode 100644 internal/database/metadata_stats_test.go diff --git a/README.md b/README.md index 49670c8..6fcab11 100644 --- a/README.md +++ b/README.md @@ -1374,10 +1374,13 @@ The proxy exposes Prometheus metrics at `GET /metrics`. All metric names are pre |--------|------|--------|-------------| | `proxy_requests_total` | counter | `ecosystem`, `status` | Proxy responses by package ecosystem and HTTP status | | `proxy_request_duration_seconds` | histogram | `ecosystem`, `status` | Proxy request duration | -| `proxy_cache_hits_total` | counter | `ecosystem` | Cache hits | -| `proxy_cache_misses_total` | counter | `ecosystem` | Cache misses | +| `proxy_cache_hits_total` | counter | `ecosystem`, `kind` | Cache hits. `kind` is `artifact` or `metadata`. | +| `proxy_cache_misses_total` | counter | `ecosystem`, `kind` | Cache misses. `kind` is `artifact` or `metadata`; metadata misses are counted only while `cache_metadata` is on. | | `proxy_cache_size_bytes` | gauge | | Total size of cached artifacts | | `proxy_cached_artifacts_total` | gauge | | Number of cached artifacts | +| `proxy_metadata_cache_served_bytes_total` | counter | `ecosystem` | Metadata bytes served from the metadata cache. | +| `proxy_metadata_cache_size_bytes` | gauge | `ecosystem` | Size of cached metadata documents per ecosystem. | +| `proxy_metadata_cache_entries` | gauge | `ecosystem` | Number of cached metadata documents per ecosystem. | | `proxy_artifacts_evicted_total` | counter | `reason`, `ecosystem` | Artifacts evicted by the size limit (`lru`) or by age (`retention`) | | `proxy_metadata_evicted_total` | counter | `ecosystem` | Cached metadata documents evicted by the size limit | | `proxy_upstream_fetch_duration_seconds` | histogram | `ecosystem` | Time spent fetching from upstream | @@ -1439,11 +1442,17 @@ For history, trends and alerting, scrape `/metrics` with Prometheus. That is the The same figures are available as JSON from `GET /stats`, which reports `downloaded_bytes`, `downloads` and an `ecosystems` array carrying the per-ecosystem breakdown, served from the same 60-second snapshot the page and the gauges read. When the aggregation fails with no snapshot to fall back on, the response carries `stats_unavailable: true` rather than passing zeros off as a count -- the endpoint keeps answering with the artifact count and cache size either way. +#### Artifact and metadata caches + +The proxy keeps two caches. Artifacts (tarballs, wheels, crates and so on) live in storage and are what `proxy_cache_size_bytes`, `proxy_ecosystem_downloaded_bytes` and the other artifact figures describe. Registry metadata is cached separately when `cache_metadata` is on. It is stored persistently in the `metadata_cache` table rather than held in memory. + +The `kind` label on `proxy_cache_hits_total` and `proxy_cache_misses_total` keeps the two apart, so an artifact hit rate is `sum(rate(proxy_cache_hits_total{kind="artifact"}[5m]))` over the same expression with misses added. Metadata has its own size and entry gauges plus `proxy_metadata_cache_served_bytes_total`. That one is a process counter rather than a database total, because metadata hits are not recorded per row. On `/ui/analytics` metadata appears in its own card and in the Runtime card's metadata lookups, never mixed into the artifact figures. The card and the two gauges read one 60-second snapshot, like the artifact figures. The bundled Grafana dashboard's hit ratio covers artifacts only, while its hits and misses panel shows each kind as its own series. + #### Three ecosystem label sets `ecosystem` means three slightly different things across `/metrics`, and queries that join across them need to know which. -**From the package record, normalized.** The six `proxy_ecosystem_*` gauges, `proxy_cache_hits_total`, `proxy_cache_misses_total`, `proxy_artifacts_evicted_total`, `proxy_metadata_evicted_total`, `proxy_integrity_failures_total` and the scan metrics. Aliases collapse here: `gem` reads as `rubygems`, `composer` as `packagist`, `go` as `golang`. +**From the package record, normalized.** The six `proxy_ecosystem_*` gauges, `proxy_cache_hits_total`, `proxy_cache_misses_total`, the three `proxy_metadata_cache_*` metrics, `proxy_artifacts_evicted_total`, `proxy_metadata_evicted_total`, `proxy_integrity_failures_total` and the scan metrics. Aliases collapse here: `gem` reads as `rubygems`, `composer` as `packagist`, `go` as `golang`. **From the request path.** `proxy_requests_total`, `proxy_request_duration_seconds` and `proxy_response_bytes_total`. The names mostly coincide with the normalized ones -- these also report `rubygems`, `packagist` and `golang` -- but the Debian route reports `debian` where the package record says `deb`, and any path that is not a package endpoint reports `other`, which corresponds to no ecosystem at all. diff --git a/deploy/grafana/git-pkgs-proxy.json b/deploy/grafana/git-pkgs-proxy.json index e875469..2592625 100644 --- a/deploy/grafana/git-pkgs-proxy.json +++ b/deploy/grafana/git-pkgs-proxy.json @@ -365,13 +365,13 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(increase(proxy_cache_hits_total{job=~\"$job\", ecosystem=~\"$ecosystem\"}[$__range]))\n/\n(\n sum(increase(proxy_cache_hits_total{job=~\"$job\", ecosystem=~\"$ecosystem\"}[$__range]))\n +\n sum(increase(proxy_cache_misses_total{job=~\"$job\", ecosystem=~\"$ecosystem\"}[$__range]))\n)", + "expr": "sum(increase(proxy_cache_hits_total{job=~\"$job\", ecosystem=~\"$ecosystem\", kind=\"artifact\"}[$__range]))\n/\n(\n sum(increase(proxy_cache_hits_total{job=~\"$job\", ecosystem=~\"$ecosystem\", kind=\"artifact\"}[$__range]))\n +\n sum(increase(proxy_cache_misses_total{job=~\"$job\", ecosystem=~\"$ecosystem\", kind=\"artifact\"}[$__range]))\n)", "range": true, "instant": false, "refId": "A" } ], - "title": "Cache hit ratio", + "title": "Artifact cache hit ratio", "type": "stat" }, { @@ -981,11 +981,11 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(rate(proxy_cache_hits_total{job=~\"$job\", ecosystem=~\"$ecosystem\"}[$__rate_interval]))", + "expr": "sum by (kind) (rate(proxy_cache_hits_total{job=~\"$job\", ecosystem=~\"$ecosystem\"}[$__rate_interval]))", "range": true, "instant": false, "refId": "A", - "legendFormat": "hits" + "legendFormat": "{{kind}} hits" }, { "datasource": { @@ -993,11 +993,11 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(rate(proxy_cache_misses_total{job=~\"$job\", ecosystem=~\"$ecosystem\"}[$__rate_interval]))", + "expr": "sum by (kind) (rate(proxy_cache_misses_total{job=~\"$job\", ecosystem=~\"$ecosystem\"}[$__rate_interval]))", "range": true, "instant": false, "refId": "B", - "legendFormat": "misses" + "legendFormat": "{{kind}} misses" } ], "title": "Cache hits and misses", diff --git a/internal/database/metadata_stats_test.go b/internal/database/metadata_stats_test.go new file mode 100644 index 0000000..e66e407 --- /dev/null +++ b/internal/database/metadata_stats_test.go @@ -0,0 +1,60 @@ +package database + +import ( + "database/sql" + "testing" +) + +func TestGetMetadataCacheStats(t *testing.T) { + db := newTestDB(t) + + stats, err := db.GetMetadataCacheStats() + if err != nil { + t.Fatalf("GetMetadataCacheStats on an empty cache: %v", err) + } + if len(stats) != 0 { + t.Fatalf("empty cache returned %+v, want no rows", stats) + } + + rows := []struct { + ecosystem, name string + size sql.NullInt64 + }{ + {"npm", "lodash", sql.NullInt64{Int64: 300, Valid: true}}, + {"npm", "react", sql.NullInt64{Int64: 200, Valid: true}}, + // gem and rubygems are one ecosystem and must share a row. + {"gem", "rails", sql.NullInt64{Int64: 1000, Valid: true}}, + {"rubygems", "rack", sql.NullInt64{Int64: 24, Valid: true}}, + // A row written before sizes were recorded still counts as an entry. + {"pypi", "requests", sql.NullInt64{}}, + } + for _, r := range rows { + if err := db.UpsertMetadataCache(&MetadataCacheEntry{ + Ecosystem: r.ecosystem, + Name: r.name, + StoragePath: "_metadata/" + r.ecosystem + "/" + r.name, + Size: r.size, + }); err != nil { + t.Fatalf("UpsertMetadataCache(%s/%s): %v", r.ecosystem, r.name, err) + } + } + + stats, err = db.GetMetadataCacheStats() + if err != nil { + t.Fatalf("GetMetadataCacheStats: %v", err) + } + + want := []MetadataCacheStats{ + {Ecosystem: "rubygems", Entries: 2, Size: 1024}, + {Ecosystem: "npm", Entries: 2, Size: 500}, + {Ecosystem: "pypi", Entries: 1, Size: 0}, + } + if len(stats) != len(want) { + t.Fatalf("got %+v, want %+v", stats, want) + } + for i := range want { + if stats[i] != want[i] { + t.Errorf("row %d = %+v, want %+v", i, stats[i], want[i]) + } + } +} diff --git a/internal/database/queries.go b/internal/database/queries.go index 49d4c9d..33ea93d 100644 --- a/internal/database/queries.go +++ b/internal/database/queries.go @@ -1126,6 +1126,62 @@ func (db *DB) CountCachedPackages(ecosystem string) (int64, error) { // Metadata cache queries +// MetadataCacheStats is one ecosystem's share of the metadata cache. +type MetadataCacheStats struct { + Ecosystem string + Entries int64 + Size int64 +} + +// GetMetadataCacheStats returns the number and total size of cached metadata +// documents per ecosystem, largest first. Ecosystem names are normalized so +// aliases such as "gem" and "rubygems" share one row. Rows written before +// sizes were recorded count as entries with no size. +func (db *DB) GetMetadataCacheStats() ([]MetadataCacheStats, error) { + hasTable, err := db.HasTable("metadata_cache") + if err != nil || !hasTable { + return nil, err + } + + rows, err := db.Query(`SELECT ecosystem, COUNT(*), COALESCE(SUM(size), 0) FROM metadata_cache GROUP BY ecosystem`) + if err != nil { + return nil, err + } + defer func() { _ = rows.Close() }() + + byEcosystem := make(map[string]*MetadataCacheStats) + for rows.Next() { + var ecosystem string + var entries, size int64 + if err := rows.Scan(&ecosystem, &entries, &size); err != nil { + return nil, err + } + key := purl.NormalizeEcosystem(ecosystem) + row, ok := byEcosystem[key] + if !ok { + row = &MetadataCacheStats{Ecosystem: key} + byEcosystem[key] = row + } + row.Entries += entries + row.Size += size + } + if err := rows.Err(); err != nil { + return nil, err + } + + stats := make([]MetadataCacheStats, 0, len(byEcosystem)) + for _, row := range byEcosystem { + stats = append(stats, *row) + } + sort.Slice(stats, func(i, j int) bool { + if stats[i].Size != stats[j].Size { + return stats[i].Size > stats[j].Size + } + return stats[i].Ecosystem < stats[j].Ecosystem + }) + return stats, nil +} + func (db *DB) GetMetadataCache(ecosystem, name string) (*MetadataCacheEntry, error) { var entry MetadataCacheEntry query := db.Rebind(` diff --git a/internal/handler/arch.go b/internal/handler/arch.go index 8b3961f..b53f25c 100644 --- a/internal/handler/arch.go +++ b/internal/handler/arch.go @@ -198,7 +198,7 @@ func (h *ArchHandler) handleDatabase(w http.ResponseWriter, r *http.Request, rep } d := h.database(repository, upstreamURL, path) if _, hit := p.cachedMetadataState(r.Context(), archEcosystem, d.dbKey, nil); hit != nil { - metrics.RecordCacheHit(archEcosystem) + metrics.RecordMetadataCacheHit(archEcosystem, len(hit.body)) h.writeMetadata(w, r, d.dbKey, *hit) return } @@ -217,7 +217,7 @@ func (h *ArchHandler) handleDatabaseSignature(w http.ResponseWriter, r *http.Req } d := h.database(repository, upstreamURL, strings.TrimSuffix(path, archSignatureSuffix)) if res, ok := h.cachedSignature(r.Context(), d); ok { - metrics.RecordCacheHit(archEcosystem) + metrics.RecordMetadataCacheHit(archEcosystem, len(res.body)) h.writeMetadata(w, r, d.sigKey, res) return } diff --git a/internal/handler/go_sumdb.go b/internal/handler/go_sumdb.go index 7de2513..8b38206 100644 --- a/internal/handler/go_sumdb.go +++ b/internal/handler/go_sumdb.go @@ -348,7 +348,7 @@ func (h *GoHandler) serveCachedSumDBTile(w http.ResponseWriter, r *http.Request, if check != nil && check(body) != nil { return false, true } - metrics.RecordCacheHit(goEcosystem) + metrics.RecordMetadataCacheHit(goEcosystem, len(body)) if entry.ETag.Valid { w.Header().Set(headerETag, entry.ETag.String) diff --git a/internal/handler/gradle.go b/internal/handler/gradle.go index e251209..cd3e78a 100644 --- a/internal/handler/gradle.go +++ b/internal/handler/gradle.go @@ -106,11 +106,11 @@ func (h *GradleBuildCacheHandler) handleGetOrHead(w http.ResponseWriter, r *http return } if !exists { - metrics.RecordCacheMiss("gradle") + metrics.RecordArtifactCacheMiss("gradle") http.NotFound(w, r) return } - metrics.RecordCacheHit("gradle") + metrics.RecordArtifactCacheHit("gradle") sizeStart := time.Now() size, err := h.proxy.Storage.Size(r.Context(), storagePath) @@ -130,7 +130,7 @@ func (h *GradleBuildCacheHandler) handleGetOrHead(w http.ResponseWriter, r *http metrics.RecordStorageOperation("read", time.Since(readStart)) if err != nil { if errors.Is(err, storage.ErrNotFound) { - metrics.RecordCacheMiss("gradle") + metrics.RecordArtifactCacheMiss("gradle") http.NotFound(w, r) return } @@ -140,7 +140,7 @@ func (h *GradleBuildCacheHandler) handleGetOrHead(w http.ResponseWriter, r *http return } defer func() { _ = reader.Close() }() - metrics.RecordCacheHit("gradle") + metrics.RecordArtifactCacheHit("gradle") w.WriteHeader(http.StatusOK) _, _ = io.Copy(w, reader) diff --git a/internal/handler/gradle_test.go b/internal/handler/gradle_test.go index a05d07e..a4b95c2 100644 --- a/internal/handler/gradle_test.go +++ b/internal/handler/gradle_test.go @@ -237,8 +237,8 @@ func TestGradleBuildCacheHandler_RecordsMetrics(t *testing.T) { srv := httptest.NewServer(h.Routes()) defer srv.Close() - hitsBefore := testutil.ToFloat64(metrics.CacheHits.WithLabelValues("gradle")) - missesBefore := testutil.ToFloat64(metrics.CacheMisses.WithLabelValues("gradle")) + hitsBefore := testutil.ToFloat64(metrics.CacheHits.WithLabelValues("gradle", metrics.CacheKindArtifact)) + missesBefore := testutil.ToFloat64(metrics.CacheMisses.WithLabelValues("gradle", metrics.CacheKindArtifact)) key := "metrics-key" putReq, err := http.NewRequest(http.MethodPut, srv.URL+"/"+key, strings.NewReader("payload")) @@ -273,8 +273,8 @@ func TestGradleBuildCacheHandler_RecordsMetrics(t *testing.T) { } _ = missResp.Body.Close() - hitsAfter := testutil.ToFloat64(metrics.CacheHits.WithLabelValues("gradle")) - missesAfter := testutil.ToFloat64(metrics.CacheMisses.WithLabelValues("gradle")) + hitsAfter := testutil.ToFloat64(metrics.CacheHits.WithLabelValues("gradle", metrics.CacheKindArtifact)) + missesAfter := testutil.ToFloat64(metrics.CacheMisses.WithLabelValues("gradle", metrics.CacheKindArtifact)) if diff := hitsAfter - hitsBefore; diff != 2 { t.Fatalf("cache hits delta = %.0f, want 2", diff) diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 7ffe718..d29508e 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -308,7 +308,7 @@ func (p *Proxy) GetOrFetchArtifact(ctx context.Context, ecosystem, name, version } else if cached != nil { return cached, nil } - metrics.RecordCacheMiss(ecosystem) + metrics.RecordArtifactCacheMiss(ecosystem) key := artifactCoalesceKey(versionPURL, filename, "", "", "") recheck := func() (artifacts.Artifact, string, bool) { @@ -507,7 +507,7 @@ func rewriteSignedURLHost(signed, baseURL string) string { func (p *Proxy) recordCacheHit(ecosystem, versionPURL, filename string) { _ = p.DB.RecordArtifactHit(versionPURL, filename) - metrics.RecordCacheHit(ecosystem) + metrics.RecordArtifactCacheHit(ecosystem) } func (p *Proxy) rejectUnusableCacheRecord(artifact *database.CachedArtifact, versionPURL, filename string, cause error) { @@ -1311,7 +1311,7 @@ func (p *Proxy) lookupMetadata(ctx context.Context, ecosystem, cacheKey, upstrea // Serve from cache if within TTL (skip upstream entirely) if _, hit := p.cachedMetadataState(ctx, ecosystem, cacheKey, validate); hit != nil { - metrics.RecordCacheHit(ecosystem) + metrics.RecordMetadataCacheHit(ecosystem, len(hit.body)) return *hit } p.recordMetadataCacheMiss(ecosystem) @@ -1534,7 +1534,7 @@ func (p *Proxy) recordMetadataRead(entry *database.MetadataCacheEntry) { func (p *Proxy) recordMetadataCacheMiss(ecosystem string) { if p.CacheMetadata { - metrics.RecordCacheMiss(ecosystem) + metrics.RecordMetadataCacheMiss(ecosystem) } } @@ -1932,7 +1932,7 @@ func (p *Proxy) getOrFetchArtifactFromURLWithCachePURLs(ctx context.Context, eco } else if cached != nil { return cached, nil } - metrics.RecordCacheMiss(ecosystem) + metrics.RecordArtifactCacheMiss(ecosystem) return p.coalescedFetchFromURL(ctx, ecosystem, name, version, filename, pkgPURL, versionPURL, downloadURL, headers, upstreamHash, declared) } diff --git a/internal/handler/handler_test.go b/internal/handler/handler_test.go index 3cc3c39..ff5a080 100644 --- a/internal/handler/handler_test.go +++ b/internal/handler/handler_test.go @@ -453,7 +453,7 @@ func assertQueuedForDeletion(t *testing.T, db *database.DB, path string) { func TestGetOrFetchArtifact_CacheMiss_NoPackage(t *testing.T) { proxy, _, _, fetcher := setupTestProxy(t) - missesBefore := testutil.ToFloat64(metrics.CacheMisses.WithLabelValues("npm")) + missesBefore := testutil.ToFloat64(metrics.CacheMisses.WithLabelValues("npm", metrics.CacheKindArtifact)) // The resolver will fail because "nonexistent" isn't a real package, // but we're testing that it tries to fetch (doesn't return from cache). @@ -463,7 +463,7 @@ func TestGetOrFetchArtifact_CacheMiss_NoPackage(t *testing.T) { if err == nil { t.Fatal("expected error for uncached package") } - missesAfter := testutil.ToFloat64(metrics.CacheMisses.WithLabelValues("npm")) + missesAfter := testutil.ToFloat64(metrics.CacheMisses.WithLabelValues("npm", metrics.CacheKindArtifact)) if diff := missesAfter - missesBefore; diff != 1 { t.Errorf("cache misses delta = %.0f, want 1", diff) } @@ -907,7 +907,7 @@ func newRangeTestResult(payload string) *CacheResult { func TestGetOrFetchArtifactFromURL_CacheHit(t *testing.T) { proxy, db, store, fetcher := setupTestProxy(t) seedPackage(t, db, store, "pypi", "requests", "2.28.0", "requests-2.28.0.tar.gz", "pypi content") - missesBefore := testutil.ToFloat64(metrics.CacheMisses.WithLabelValues("pypi")) + missesBefore := testutil.ToFloat64(metrics.CacheMisses.WithLabelValues("pypi", metrics.CacheKindArtifact)) result, err := proxy.GetOrFetchArtifactFromURL(context.Background(), "pypi", "requests", "2.28.0", "requests-2.28.0.tar.gz", "https://pypi.org/files/requests-2.28.0.tar.gz") if err != nil { @@ -921,7 +921,7 @@ func TestGetOrFetchArtifactFromURL_CacheHit(t *testing.T) { if fetcher.fetchCalled { t.Error("fetcher should not be called on cache hit") } - missesAfter := testutil.ToFloat64(metrics.CacheMisses.WithLabelValues("pypi")) + missesAfter := testutil.ToFloat64(metrics.CacheMisses.WithLabelValues("pypi", metrics.CacheKindArtifact)) if diff := missesAfter - missesBefore; diff != 0 { t.Errorf("cache misses delta = %.0f, want 0", diff) } @@ -929,7 +929,7 @@ func TestGetOrFetchArtifactFromURL_CacheHit(t *testing.T) { func TestGetOrFetchArtifactFromURL_CacheMiss(t *testing.T) { proxy, db, store, fetcher := setupTestProxy(t) - missesBefore := testutil.ToFloat64(metrics.CacheMisses.WithLabelValues("pypi")) + missesBefore := testutil.ToFloat64(metrics.CacheMisses.WithLabelValues("pypi", metrics.CacheKindArtifact)) fetchesBefore := histogramSampleCount(t, metrics.UpstreamFetchDuration.WithLabelValues("pypi")) writesBefore := histogramSampleCount(t, metrics.StorageOperationDuration.WithLabelValues("write")) readsBefore := histogramSampleCount(t, metrics.StorageOperationDuration.WithLabelValues("read")) @@ -977,7 +977,7 @@ func TestGetOrFetchArtifactFromURL_CacheMiss(t *testing.T) { if _, ok := store.files[storagePath]; !ok { t.Error("artifact was not stored in storage") } - missesAfter := testutil.ToFloat64(metrics.CacheMisses.WithLabelValues("pypi")) + missesAfter := testutil.ToFloat64(metrics.CacheMisses.WithLabelValues("pypi", metrics.CacheKindArtifact)) if diff := missesAfter - missesBefore; diff != 1 { t.Errorf("cache misses delta = %.0f, want 1", diff) } @@ -1296,8 +1296,10 @@ func TestProxyCached_NoValidators_OmitsHeaders(t *testing.T) { } func TestFetchOrCacheMetadata_TTL_ServesFreshFromCache(t *testing.T) { - hitsBefore := testutil.ToFloat64(metrics.CacheHits.WithLabelValues("test")) - missesBefore := testutil.ToFloat64(metrics.CacheMisses.WithLabelValues("test")) + hitsBefore := testutil.ToFloat64(metrics.CacheHits.WithLabelValues("test", metrics.CacheKindMetadata)) + missesBefore := testutil.ToFloat64(metrics.CacheMisses.WithLabelValues("test", metrics.CacheKindMetadata)) + artifactHitsBefore := testutil.ToFloat64(metrics.CacheHits.WithLabelValues("test", metrics.CacheKindArtifact)) + servedBefore := testutil.ToFloat64(metrics.MetadataServedBytes.WithLabelValues("test")) upstreamHits := 0 upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { upstreamHits++ @@ -1324,10 +1326,10 @@ func TestFetchOrCacheMetadata_TTL_ServesFreshFromCache(t *testing.T) { if upstreamHits != 1 { t.Fatalf("expected 1 upstream hit, got %d", upstreamHits) } - if diff := testutil.ToFloat64(metrics.CacheMisses.WithLabelValues("test")) - missesBefore; diff != 1 { + if diff := testutil.ToFloat64(metrics.CacheMisses.WithLabelValues("test", metrics.CacheKindMetadata)) - missesBefore; diff != 1 { t.Errorf("cache misses delta after first request = %.0f, want 1", diff) } - if diff := testutil.ToFloat64(metrics.CacheHits.WithLabelValues("test")) - hitsBefore; diff != 0 { + if diff := testutil.ToFloat64(metrics.CacheHits.WithLabelValues("test", metrics.CacheKindMetadata)) - hitsBefore; diff != 0 { t.Errorf("cache hits delta after first request = %.0f, want 0", diff) } @@ -1342,16 +1344,22 @@ func TestFetchOrCacheMetadata_TTL_ServesFreshFromCache(t *testing.T) { if upstreamHits != 1 { t.Errorf("expected upstream to still be hit only once, got %d", upstreamHits) } - if diff := testutil.ToFloat64(metrics.CacheHits.WithLabelValues("test")) - hitsBefore; diff != 1 { + if diff := testutil.ToFloat64(metrics.CacheHits.WithLabelValues("test", metrics.CacheKindMetadata)) - hitsBefore; diff != 1 { t.Errorf("cache hits delta after second request = %.0f, want 1", diff) } - if diff := testutil.ToFloat64(metrics.CacheMisses.WithLabelValues("test")) - missesBefore; diff != 1 { + if diff := testutil.ToFloat64(metrics.CacheMisses.WithLabelValues("test", metrics.CacheKindMetadata)) - missesBefore; diff != 1 { t.Errorf("cache misses delta after second request = %.0f, want 1", diff) } + if diff := testutil.ToFloat64(metrics.MetadataServedBytes.WithLabelValues("test")) - servedBefore; diff != float64(len(body)) { + t.Errorf("metadata served bytes delta = %.0f, want %d (only the cached response)", diff, len(body)) + } + if diff := testutil.ToFloat64(metrics.CacheHits.WithLabelValues("test", metrics.CacheKindArtifact)) - artifactHitsBefore; diff != 0 { + t.Errorf("artifact cache hits delta = %.0f, want 0 for a metadata lookup", diff) + } } func TestFetchOrCacheMetadata_TTL_Zero_AlwaysRevalidates(t *testing.T) { - missesBefore := testutil.ToFloat64(metrics.CacheMisses.WithLabelValues("test")) + missesBefore := testutil.ToFloat64(metrics.CacheMisses.WithLabelValues("test", metrics.CacheKindMetadata)) upstreamHits := 0 upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { upstreamHits++ @@ -1380,7 +1388,7 @@ func TestFetchOrCacheMetadata_TTL_Zero_AlwaysRevalidates(t *testing.T) { if upstreamHits != 2 { t.Errorf("expected 2 upstream hits with TTL=0, got %d", upstreamHits) } - missesAfter := testutil.ToFloat64(metrics.CacheMisses.WithLabelValues("test")) + missesAfter := testutil.ToFloat64(metrics.CacheMisses.WithLabelValues("test", metrics.CacheKindMetadata)) if diff := missesAfter - missesBefore; diff != 2 { t.Errorf("cache misses delta = %.0f, want 2", diff) } @@ -1398,16 +1406,16 @@ func TestFetchOrCacheMetadata_CacheDisabledDoesNotRecordMetrics(t *testing.T) { proxy, _, _, _ := setupTestProxy(t) proxy.HTTPClient = upstream.Client() - hitsBefore := testutil.ToFloat64(metrics.CacheHits.WithLabelValues(ecosystem)) - missesBefore := testutil.ToFloat64(metrics.CacheMisses.WithLabelValues(ecosystem)) + hitsBefore := testutil.ToFloat64(metrics.CacheHits.WithLabelValues(ecosystem, metrics.CacheKindMetadata)) + missesBefore := testutil.ToFloat64(metrics.CacheMisses.WithLabelValues(ecosystem, metrics.CacheKindMetadata)) _, _, err := proxy.FetchOrCacheMetadata(context.Background(), ecosystem, "pkg", upstream.URL+"/pkg") if err != nil { t.Fatalf("fetch metadata: %v", err) } - hitsAfter := testutil.ToFloat64(metrics.CacheHits.WithLabelValues(ecosystem)) - missesAfter := testutil.ToFloat64(metrics.CacheMisses.WithLabelValues(ecosystem)) + hitsAfter := testutil.ToFloat64(metrics.CacheHits.WithLabelValues(ecosystem, metrics.CacheKindMetadata)) + missesAfter := testutil.ToFloat64(metrics.CacheMisses.WithLabelValues(ecosystem, metrics.CacheKindMetadata)) if diff := hitsAfter - hitsBefore; diff != 0 { t.Errorf("cache hits delta = %.0f, want 0", diff) } diff --git a/internal/handler/npm_rewrite_test.go b/internal/handler/npm_rewrite_test.go index 6503ae1..60616d8 100644 --- a/internal/handler/npm_rewrite_test.go +++ b/internal/handler/npm_rewrite_test.go @@ -12,6 +12,8 @@ import ( "time" "github.com/git-pkgs/cooldown" + "github.com/git-pkgs/proxy/internal/metrics" + "github.com/prometheus/client_golang/prometheus/testutil" ) // TestNPMRewriteMetadataKeepsUpstreamBytes checks that only tarball URLs @@ -270,3 +272,43 @@ func TestNPMStoredRewriteSkippedWhenStale(t *testing.T) { t.Error("cooldown on: expected no stored rewrite") } } + +// TestNPMStoredRewriteRecordsMetadataHit checks the digest fast path counts as +// a metadata cache hit with the bytes it serves, never as an artifact hit. +func TestNPMStoredRewriteRecordsMetadataHit(t *testing.T) { + proxy, _, _, _ := setupTestProxy(t) + proxy.CacheMetadata = true + proxy.MetadataTTL = time.Hour + proxy.SetMetadataRewriteCacheSize(1 << 20) + + body := []byte(`{"name":"demo"}`) + proxy.cacheMetadataBlob(t.Context(), "npm", "hit-demo", metadataStoragePath("npm", "hit-demo"), &upstreamMetadata{body: body}) + key := rewriteCacheKey("npm", "http://proxy", "hit-demo", body) + if _, err := proxy.rewrites.rewrite(t.Context(), key, body, func(b []byte) ([]byte, error) { return b, nil }); err != nil { + t.Fatal(err) + } + + metadataHits := func() float64 { + return testutil.ToFloat64(metrics.CacheHits.WithLabelValues("npm", metrics.CacheKindMetadata)) + } + artifactHits := func() float64 { + return testutil.ToFloat64(metrics.CacheHits.WithLabelValues("npm", metrics.CacheKindArtifact)) + } + served := func() float64 { return testutil.ToFloat64(metrics.MetadataServedBytes.WithLabelValues("npm")) } + metadataBefore, artifactBefore, servedBefore := metadataHits(), artifactHits(), served() + + m, ok := proxy.storedRewrite("npm", "hit-demo", "http://proxy", "hit-demo") + if !ok { + t.Fatal("expected a stored rewrite") + } + + if diff := metadataHits() - metadataBefore; diff != 1 { + t.Errorf("metadata hits delta = %.0f, want 1", diff) + } + if diff := artifactHits() - artifactBefore; diff != 0 { + t.Errorf("artifact hits delta = %.0f, want 0", diff) + } + if diff := served() - servedBefore; diff != float64(len(m.body)) { + t.Errorf("metadata served bytes delta = %.0f, want %d", diff, len(m.body)) + } +} diff --git a/internal/handler/rewrite_cache.go b/internal/handler/rewrite_cache.go index a96bfdc..83ce2a1 100644 --- a/internal/handler/rewrite_cache.go +++ b/internal/handler/rewrite_cache.go @@ -327,7 +327,7 @@ func (p *Proxy) storedRewrite(ecosystem, cacheKey, proxyURL, name string) (rewri return rewrittenMetadata{}, false } m.lastModified = p.metadataDate(ecosystem, proxyURL, name, m.etag) - metrics.RecordCacheHit(ecosystem) + metrics.RecordMetadataCacheHit(ecosystem, len(m.body)) p.recordMetadataRead(entry) return m, true } diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go index da7e729..8fb857c 100644 --- a/internal/metrics/metrics.go +++ b/internal/metrics/metrics.go @@ -35,15 +35,23 @@ var ( CacheHits = prometheus.NewCounterVec( prometheus.CounterOpts{ Name: "proxy_cache_hits_total", - Help: "Total number of cache hits by ecosystem", + Help: "Total number of cache hits by ecosystem and kind (artifact or metadata)", }, - []string{"ecosystem"}, + []string{"ecosystem", "kind"}, ) CacheMisses = prometheus.NewCounterVec( prometheus.CounterOpts{ Name: "proxy_cache_misses_total", - Help: "Total number of cache misses by ecosystem", + Help: "Total number of cache misses by ecosystem and kind (artifact or metadata)", + }, + []string{"ecosystem", "kind"}, + ) + + MetadataServedBytes = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "proxy_metadata_cache_served_bytes_total", + Help: "Total metadata bytes served from the metadata cache, by ecosystem", }, []string{"ecosystem"}, ) @@ -214,6 +222,24 @@ var ( []string{"ecosystem"}, ) + // Metadata cache gauges, derived from the metadata_cache table on the + // same tick as the per-ecosystem gauges above. + MetadataCacheSize = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "proxy_metadata_cache_size_bytes", + Help: "Size of cached metadata documents per ecosystem in bytes", + }, + []string{"ecosystem"}, + ) + + MetadataCacheEntries = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "proxy_metadata_cache_entries", + Help: "Number of cached metadata documents per ecosystem", + }, + []string{"ecosystem"}, + ) + ResponseBytes = prometheus.NewCounterVec( prometheus.CounterOpts{ Name: "proxy_response_bytes_total", @@ -272,6 +298,7 @@ func init() { RequestDuration, CacheHits, CacheMisses, + MetadataServedBytes, CacheSize, CachedArtifacts, ArtifactsEvicted, @@ -292,6 +319,8 @@ func init() { EcosystemCachedArtifacts, EcosystemPackages, EcosystemVersions, + MetadataCacheSize, + MetadataCacheEntries, ResponseBytes, ClientRequests, ClientResponseBytes, @@ -331,9 +360,27 @@ func RecordResponse(ecosystem, client string, bytes int64) { ClientResponseBytes.WithLabelValues(client).Add(float64(bytes)) } -// RecordCacheHit increments cache hit counter. -func RecordCacheHit(ecosystem string) { - CacheHits.WithLabelValues(purl.NormalizeEcosystem(ecosystem)).Inc() +// Values of the kind label on proxy_cache_hits_total and +// proxy_cache_misses_total. Artifacts and metadata are separate caches with +// very different costs, so a hit on one says nothing about the other. +const ( + CacheKindArtifact = "artifact" + CacheKindMetadata = "metadata" +) + +// RecordArtifactCacheHit counts an artifact served from storage. +func RecordArtifactCacheHit(ecosystem string) { + CacheHits.WithLabelValues(purl.NormalizeEcosystem(ecosystem), CacheKindArtifact).Inc() +} + +// RecordMetadataCacheHit counts a metadata document served from the metadata +// cache, along with the bytes it served. +func RecordMetadataCacheHit(ecosystem string, bytes int) { + ecosystem = purl.NormalizeEcosystem(ecosystem) + CacheHits.WithLabelValues(ecosystem, CacheKindMetadata).Inc() + if bytes > 0 { + MetadataServedBytes.WithLabelValues(ecosystem).Add(float64(bytes)) + } } // RecordArtifactEvicted counts one evicted artifact. ecosystem is the @@ -353,9 +400,14 @@ func RecordMetadataEvicted(ecosystem string) { MetadataEvicted.WithLabelValues(purl.NormalizeEcosystem(ecosystem)).Inc() } -// RecordCacheMiss increments cache miss counter. -func RecordCacheMiss(ecosystem string) { - CacheMisses.WithLabelValues(purl.NormalizeEcosystem(ecosystem)).Inc() +// RecordArtifactCacheMiss counts an artifact that had to be fetched upstream. +func RecordArtifactCacheMiss(ecosystem string) { + CacheMisses.WithLabelValues(purl.NormalizeEcosystem(ecosystem), CacheKindArtifact).Inc() +} + +// RecordMetadataCacheMiss counts a metadata lookup the cache could not serve. +func RecordMetadataCacheMiss(ecosystem string) { + CacheMisses.WithLabelValues(purl.NormalizeEcosystem(ecosystem), CacheKindMetadata).Inc() } // RecordUpstreamFetch tracks upstream fetch duration. @@ -439,6 +491,8 @@ type EcosystemStats struct { var ( publishedMu sync.Mutex publishedEcosystems = map[string]bool{} + // publishedMetadataEcosystems does the same for the metadata cache gauges. + publishedMetadataEcosystems = map[string]bool{} ) // UpdateEcosystemStats republishes the per-ecosystem gauges from a fresh snapshot. @@ -496,6 +550,48 @@ func UpdateEcosystemStats(stats []EcosystemStats) { } } +// MetadataCacheStats is one ecosystem's share of the metadata cache. +type MetadataCacheStats struct { + Ecosystem string + Entries int64 + Size int64 +} + +// UpdateMetadataCacheStats republishes the metadata cache gauges from a fresh +// snapshot. Like UpdateEcosystemStats it sums rows by normalized label, sets +// each series in place and deletes only labels that have disappeared. +func UpdateMetadataCacheStats(stats []MetadataCacheStats) { + totals := make(map[string]MetadataCacheStats, len(stats)) + for _, s := range stats { + ecosystem := purl.NormalizeEcosystem(s.Ecosystem) + t := totals[ecosystem] + t.Entries += s.Entries + t.Size += s.Size + totals[ecosystem] = t + } + + for ecosystem, t := range totals { + MetadataCacheEntries.WithLabelValues(ecosystem).Set(float64(t.Entries)) + MetadataCacheSize.WithLabelValues(ecosystem).Set(float64(t.Size)) + } + + publishedMu.Lock() + defer publishedMu.Unlock() + + for ecosystem := range publishedMetadataEcosystems { + if _, still := totals[ecosystem]; still { + continue + } + MetadataCacheEntries.DeleteLabelValues(ecosystem) + MetadataCacheSize.DeleteLabelValues(ecosystem) + } + + publishedMetadataEcosystems = make(map[string]bool, len(totals)) + for ecosystem := range totals { + publishedMetadataEcosystems[ecosystem] = true + } +} + // UpdateCircuitBreakerState updates circuit breaker state gauge. // state: 0=closed, 1=half-open, 2=open func UpdateCircuitBreakerState(registry string, state int) { diff --git a/internal/metrics/metrics_test.go b/internal/metrics/metrics_test.go index 8259d3e..25ae1cd 100644 --- a/internal/metrics/metrics_test.go +++ b/internal/metrics/metrics_test.go @@ -21,10 +21,11 @@ func TestRecordRequest(t *testing.T) { } func TestRecordCacheOperations(t *testing.T) { - RecordCacheHit("npm") - RecordCacheHit("npm") - RecordCacheMiss("npm") - RecordCacheMiss("pypi") + RecordArtifactCacheHit("npm") + RecordArtifactCacheHit("npm") + RecordArtifactCacheMiss("npm") + RecordMetadataCacheHit("pypi", 128) + RecordMetadataCacheMiss("pypi") // No panics = success } @@ -114,9 +115,9 @@ func TestMetricsAreRegistered(t *testing.T) { func TestCacheHitRatio(t *testing.T) { // Reset by recording some hits and misses - RecordCacheHit("test") - RecordCacheHit("test") - RecordCacheMiss("test") + RecordArtifactCacheHit("test") + RecordArtifactCacheHit("test") + RecordArtifactCacheMiss("test") // Collect metrics hits := getMetricValue(t, CacheHits, "test") @@ -179,7 +180,7 @@ func getMetricValue(t *testing.T, collector prometheus.Collector, labelValue str func TestMetricsEndpointOutput(t *testing.T) { // Record some test metrics RecordRequest("npm", 200, 50*time.Millisecond) - RecordCacheHit("npm") + RecordArtifactCacheHit("npm") UpdateCacheStats(1024*1024, 10) // Get the handler and check it produces output @@ -207,7 +208,7 @@ func TestMetricsLabeling(t *testing.T) { for _, eco := range ecosystems { RecordRequest(eco.input, 200, 10*time.Millisecond) - RecordCacheHit(eco.input) + RecordArtifactCacheHit(eco.input) } // Verify each ecosystem has metrics @@ -220,18 +221,93 @@ func TestMetricsLabeling(t *testing.T) { } func TestCacheMetricLabelsAreNormalized(t *testing.T) { - rubyHitsBefore := testutil.ToFloat64(CacheHits.WithLabelValues("rubygems")) - composerMissesBefore := testutil.ToFloat64(CacheMisses.WithLabelValues("packagist")) + rubyHitsBefore := testutil.ToFloat64(CacheHits.WithLabelValues("rubygems", CacheKindArtifact)) + composerMissesBefore := testutil.ToFloat64(CacheMisses.WithLabelValues("packagist", CacheKindArtifact)) + goMetadataHitsBefore := testutil.ToFloat64(CacheHits.WithLabelValues("golang", CacheKindMetadata)) + goServedBefore := testutil.ToFloat64(MetadataServedBytes.WithLabelValues("golang")) - RecordCacheHit("gem") - RecordCacheMiss("composer") + RecordArtifactCacheHit("gem") + RecordArtifactCacheMiss("composer") + RecordMetadataCacheHit("go", 10) - if diff := testutil.ToFloat64(CacheHits.WithLabelValues("rubygems")) - rubyHitsBefore; diff != 1 { + if diff := testutil.ToFloat64(CacheHits.WithLabelValues("rubygems", CacheKindArtifact)) - rubyHitsBefore; diff != 1 { t.Errorf("rubygems cache hits delta = %.0f, want 1", diff) } - if diff := testutil.ToFloat64(CacheMisses.WithLabelValues("packagist")) - composerMissesBefore; diff != 1 { + if diff := testutil.ToFloat64(CacheMisses.WithLabelValues("packagist", CacheKindArtifact)) - composerMissesBefore; diff != 1 { t.Errorf("packagist cache misses delta = %.0f, want 1", diff) } + if diff := testutil.ToFloat64(CacheHits.WithLabelValues("golang", CacheKindMetadata)) - goMetadataHitsBefore; diff != 1 { + t.Errorf("golang metadata hits delta = %.0f, want 1", diff) + } + if diff := testutil.ToFloat64(MetadataServedBytes.WithLabelValues("golang")) - goServedBefore; diff != 10 { + t.Errorf("golang metadata served bytes delta = %.0f, want 10", diff) + } +} + +// TestCacheKindsAreCountedApart checks an artifact hit and a metadata hit land +// in different series. Only metadata hits add served bytes. +func TestCacheKindsAreCountedApart(t *testing.T) { + const eco = "kind-test" + artifactHits := func() float64 { return testutil.ToFloat64(CacheHits.WithLabelValues(eco, CacheKindArtifact)) } + metadataHits := func() float64 { return testutil.ToFloat64(CacheHits.WithLabelValues(eco, CacheKindMetadata)) } + artifactMisses := func() float64 { return testutil.ToFloat64(CacheMisses.WithLabelValues(eco, CacheKindArtifact)) } + metadataMisses := func() float64 { return testutil.ToFloat64(CacheMisses.WithLabelValues(eco, CacheKindMetadata)) } + served := func() float64 { return testutil.ToFloat64(MetadataServedBytes.WithLabelValues(eco)) } + + aHits, mHits, aMisses, mMisses, bytes := artifactHits(), metadataHits(), artifactMisses(), metadataMisses(), served() + + RecordArtifactCacheHit(eco) + RecordMetadataCacheHit(eco, 300) + RecordMetadataCacheHit(eco, 0) + RecordMetadataCacheMiss(eco) + + checks := []struct { + name string + got, want float64 + }{ + {"artifact hits", artifactHits() - aHits, 1}, + {"metadata hits", metadataHits() - mHits, 2}, + {"artifact misses", artifactMisses() - aMisses, 0}, + {"metadata misses", metadataMisses() - mMisses, 1}, + {"metadata served bytes", served() - bytes, 300}, + } + for _, c := range checks { + if c.got != c.want { + t.Errorf("%s delta = %.0f, want %.0f", c.name, c.got, c.want) + } + } +} + +func TestUpdateMetadataCacheStats(t *testing.T) { + UpdateMetadataCacheStats([]MetadataCacheStats{ + {Ecosystem: "gem", Entries: 2, Size: 100}, + {Ecosystem: "rubygems", Entries: 3, Size: 50}, + {Ecosystem: "npm", Entries: 1, Size: 10}, + }) + + if got := testutil.ToFloat64(MetadataCacheEntries.WithLabelValues("rubygems")); got != 5 { + t.Errorf("rubygems entries = %.0f, want 5 (aliases summed)", got) + } + if got := testutil.ToFloat64(MetadataCacheSize.WithLabelValues("rubygems")); got != 150 { + t.Errorf("rubygems size = %.0f, want 150 (aliases summed)", got) + } + if got := testutil.CollectAndCount(MetadataCacheSize); got != 2 { + t.Errorf("size series = %d, want 2", got) + } + + UpdateMetadataCacheStats([]MetadataCacheStats{{Ecosystem: "npm", Entries: 4, Size: 40}}) + + if got := testutil.CollectAndCount(MetadataCacheSize); got != 1 { + t.Errorf("size series after rubygems disappeared = %d, want 1", got) + } + if got := testutil.CollectAndCount(MetadataCacheEntries); got != 1 { + t.Errorf("entry series after rubygems disappeared = %d, want 1", got) + } + if got := testutil.ToFloat64(MetadataCacheSize.WithLabelValues("npm")); got != 40 { + t.Errorf("npm size = %.0f, want 40", got) + } + + UpdateMetadataCacheStats(nil) } func TestMetricNames(t *testing.T) { @@ -241,6 +317,9 @@ func TestMetricNames(t *testing.T) { "proxy_request_duration_seconds", "proxy_cache_hits_total", "proxy_cache_misses_total", + "proxy_metadata_cache_served_bytes_total", + "proxy_metadata_cache_size_bytes", + "proxy_metadata_cache_entries", "proxy_upstream_fetch_duration_seconds", "proxy_upstream_errors_total", "proxy_storage_operation_duration_seconds", diff --git a/internal/metrics/snapshot_test.go b/internal/metrics/snapshot_test.go index 1a20906..5b6c7fd 100644 --- a/internal/metrics/snapshot_test.go +++ b/internal/metrics/snapshot_test.go @@ -125,7 +125,7 @@ func TestSnapshotMissingMetric(t *testing.T) { // come back. This is the check that a newly added metric is reachable by the UI. func TestGatherSeesRegisteredMetrics(t *testing.T) { isolateRecentFailures(t) - RecordCacheHit("npm") + RecordArtifactCacheHit("npm") RecordScanError("npm", "clamav", "timeout", FailureDetail{}) snap, err := Gather() diff --git a/internal/server/analytics.go b/internal/server/analytics.go index b7b4a83..2e0e0ce 100644 --- a/internal/server/analytics.go +++ b/internal/server/analytics.go @@ -21,6 +21,7 @@ type AnalyticsData struct { Donut DonutView EnrichmentStats EnrichmentStatsView Ecosystems []EcosystemRow + Metadata MetadataCacheView Runtime RuntimeView // StatsFailed records that the per-ecosystem query errored, so the page // can say the figures are unavailable instead of claiming an empty cache. @@ -66,6 +67,54 @@ type EcosystemRow struct { SharePct string } +// MetadataCacheView summarises the metadata cache, which is stored separately +// from artifacts and so appears in none of the artifact figures. +type MetadataCacheView struct { + // Enabled reports whether cache_metadata is on. Entries can remain from + // an earlier run with it on, so both are shown independently. + Enabled bool + // Failed records that the query errored, so the card can say so rather + // than claim the cache is empty. + Failed bool + Entries string + Size string + Rows []MetadataCacheRow +} + +// MetadataCacheRow is one ecosystem's share of the metadata cache. +type MetadataCacheRow struct { + Ecosystem string + Entries string + Size string + SharePct string +} + +// metadataCacheView turns per-ecosystem metadata cache rows into the card's +// totals and table. +func metadataCacheView(stats []database.MetadataCacheStats, enabled bool) MetadataCacheView { + var entries, size int64 + for _, e := range stats { + entries += e.Entries + size += e.Size + } + + v := MetadataCacheView{ + Enabled: enabled, + Entries: formatCount(entries), + Size: formatSize(size), + Rows: make([]MetadataCacheRow, 0, len(stats)), + } + for _, e := range stats { + v.Rows = append(v.Rows, MetadataCacheRow{ + Ecosystem: e.Ecosystem, + Entries: formatCount(e.Entries), + Size: formatSize(e.Size), + SharePct: formatPercent(e.Size, size), + }) + } + return v +} + func formatCount(n int64) string { s := strconv.FormatInt(n, 10) neg := strings.HasPrefix(s, "-") @@ -165,6 +214,16 @@ func (s *Server) handleAnalytics(w http.ResponseWriter, r *http.Request) { data.Totals, data.Ecosystems = analyticsView(ecosystems) data.Donut = donutView(ecosystems, data.Totals.DownloadedBytes, data.Totals.Downloaded) + // The same 60-second snapshot the metadata cache gauges are published + // from, so a page load costs no query of its own. A failed read keeps the + // last good rows, which the card shows in preference to the error. + metadataStats, err := s.metaStats.Get(s.db) + if err != nil { + s.logger.Error("failed to get metadata cache stats", "error", err) + } + data.Metadata = metadataCacheView(metadataStats, s.cfg != nil && s.cfg.CacheMetadata) + data.Metadata.Failed = err != nil + // Process-lifetime counters come from the Prometheus registry rather than // the database; a failure here must not take the page down with it. snap, err := metrics.Gather() diff --git a/internal/server/analytics_coverage_test.go b/internal/server/analytics_coverage_test.go index f97cf4b..82e5ca2 100644 --- a/internal/server/analytics_coverage_test.go +++ b/internal/server/analytics_coverage_test.go @@ -25,6 +25,8 @@ var metricSurface = map[string]string{ "proxy_ecosystem_versions": "breakdown table", "proxy_cache_size_bytes": "Cache size tile", "proxy_cached_artifacts_total": "Cached artifacts tile", + "proxy_metadata_cache_size_bytes": "Metadata cache card", + "proxy_metadata_cache_entries": "Metadata cache card", // Request-path counters. "proxy_response_bytes_total": "Runtime: Served", @@ -35,8 +37,9 @@ var metricSurface = map[string]string{ "proxy_requests_total": "Runtime: Requests + Responses by status", "proxy_request_duration_seconds": "Runtime: Mean latency", "proxy_active_requests": "Runtime: In flight", - "proxy_cache_hits_total": "Runtime: Cache lookups + hit rate", - "proxy_cache_misses_total": "Runtime: Cache lookups + hit rate", + "proxy_cache_hits_total": "Runtime: Artifact and metadata cache lookups + artifact hit rate", + "proxy_cache_misses_total": "Runtime: Artifact and metadata cache lookups + artifact hit rate", + "proxy_metadata_cache_served_bytes_total": "Runtime: Metadata cache lookups -- served from cache", "proxy_upstream_fetch_duration_seconds": "Runtime: Upstream fetches + mean fetch", "proxy_upstream_errors_total": "Runtime: Upstream errors", "proxy_storage_operation_duration_seconds": "Runtime: Storage operations", @@ -60,8 +63,10 @@ func TestEveryMetricIsSurfaced(t *testing.T) { // a vector with no observed label values gathers as nothing at all. metrics.RecordRequest("npm", 200, 0) metrics.RecordResponse("npm", "npm", 1) - metrics.RecordCacheHit("npm") - metrics.RecordCacheMiss("npm") + metrics.RecordArtifactCacheHit("npm") + metrics.RecordArtifactCacheMiss("npm") + metrics.RecordMetadataCacheHit("npm", 1) + metrics.RecordMetadataCacheMiss("npm") metrics.RecordUpstreamFetch("npm", 0) metrics.RecordUpstreamError("npm", "fetch_failed", metrics.FailureDetail{}) metrics.RecordStorageOperation("get", 0) @@ -76,6 +81,7 @@ func TestEveryMetricIsSurfaced(t *testing.T) { metrics.RecordArtifactEvicted("retention", "npm") metrics.RecordMetadataEvicted("npm") metrics.UpdateEcosystemStats([]metrics.EcosystemStats{{Ecosystem: "npm", DownloadedBytes: 1}}) + metrics.UpdateMetadataCacheStats([]metrics.MetadataCacheStats{{Ecosystem: "npm", Entries: 1, Size: 1}}) families, err := prometheus.DefaultGatherer.Gather() if err != nil { diff --git a/internal/server/analytics_runtime.go b/internal/server/analytics_runtime.go index c99ec97..5773879 100644 --- a/internal/server/analytics_runtime.go +++ b/internal/server/analytics_runtime.go @@ -15,14 +15,15 @@ import ( type RuntimeView struct { Available bool - Requests string - ActiveRequests string - RequestMean string - StatusClasses []LabelledCount - CacheHits string - CacheMisses string - CacheHitRatio string - HasCacheTraffic bool + Requests string + ActiveRequests string + RequestMean string + StatusClasses []LabelledCount + // Artifact and metadata lookups are counted apart: they are separate + // caches. Only artifacts feed the database figures above. + ArtifactCache CacheLookups + MetadataCache CacheLookups + MetadataServed string UpstreamFetches string UpstreamFetchMean string @@ -59,6 +60,28 @@ type RuntimeView struct { HasFailures bool } +// CacheLookups is one cache's hits and misses since the process started. +type CacheLookups struct { + Hits string + Misses string + HitRatio string + HasTraffic bool +} + +// cacheLookups reads one kind's share of the hit and miss counters. +func cacheLookups(hits, misses map[string]float64, kind string) CacheLookups { + h, m := hits[kind], misses[kind] + c := CacheLookups{ + Hits: formatCount(int64(h)), + Misses: formatCount(int64(m)), + HasTraffic: h+m > 0, + } + if c.HasTraffic { + c.HitRatio = strconv.FormatFloat(h/(h+m)*100, 'f', 1, 64) //nolint:mnd // percent + } + return c +} + // ClientRow is one client tool's share of requests and bytes. type ClientRow struct { Client string @@ -122,14 +145,11 @@ func runtimeView(snap *metrics.Snapshot, failureDetails bool) RuntimeView { v.RequestMean = formatDuration(snap.Mean("proxy_request_duration_seconds")) v.StatusClasses = statusClasses(snap) - hits := snap.Sum("proxy_cache_hits_total") - misses := snap.Sum("proxy_cache_misses_total") - v.CacheHits = formatCount(int64(hits)) - v.CacheMisses = formatCount(int64(misses)) - v.HasCacheTraffic = hits+misses > 0 - if v.HasCacheTraffic { - v.CacheHitRatio = strconv.FormatFloat(hits/(hits+misses)*100, 'f', 1, 64) //nolint:mnd // percent - } + hits := snap.SumBy("proxy_cache_hits_total", "kind") + misses := snap.SumBy("proxy_cache_misses_total", "kind") + v.ArtifactCache = cacheLookups(hits, misses, metrics.CacheKindArtifact) + v.MetadataCache = cacheLookups(hits, misses, metrics.CacheKindMetadata) + v.MetadataServed = formatSize(int64(snap.Sum("proxy_metadata_cache_served_bytes_total"))) v.UpstreamFetches = formatCount(int64(snap.Count("proxy_upstream_fetch_duration_seconds"))) v.UpstreamFetchMean = formatDuration(snap.Mean("proxy_upstream_fetch_duration_seconds")) diff --git a/internal/server/analytics_runtime_test.go b/internal/server/analytics_runtime_test.go index 3e476e0..66c154e 100644 --- a/internal/server/analytics_runtime_test.go +++ b/internal/server/analytics_runtime_test.go @@ -40,9 +40,9 @@ func TestRuntimeViewShapesRegistry(t *testing.T) { scanErrors := prometheus.NewCounterVec( prometheus.CounterOpts{Name: "proxy_scan_errors_total", Help: "t"}, []string{"ecosystem", "scanner", "error_type"}) hits := prometheus.NewCounterVec( - prometheus.CounterOpts{Name: "proxy_cache_hits_total", Help: "t"}, []string{"ecosystem"}) + prometheus.CounterOpts{Name: "proxy_cache_hits_total", Help: "t"}, []string{"ecosystem", "kind"}) misses := prometheus.NewCounterVec( - prometheus.CounterOpts{Name: "proxy_cache_misses_total", Help: "t"}, []string{"ecosystem"}) + prometheus.CounterOpts{Name: "proxy_cache_misses_total", Help: "t"}, []string{"ecosystem", "kind"}) breaker := prometheus.NewGaugeVec( prometheus.GaugeOpts{Name: "proxy_circuit_breaker_state", Help: "t"}, []string{"registry"}) storage := prometheus.NewHistogramVec( @@ -53,8 +53,8 @@ func TestRuntimeViewShapesRegistry(t *testing.T) { requests.WithLabelValues("npm", "304").Add(5) requests.WithLabelValues("npm", "500").Add(5) scanErrors.WithLabelValues("npm", "clamav", "timeout").Add(3) - hits.WithLabelValues("npm").Add(80) - misses.WithLabelValues("npm").Add(20) + hits.WithLabelValues("npm", metrics.CacheKindArtifact).Add(80) + misses.WithLabelValues("npm", metrics.CacheKindArtifact).Add(20) breaker.WithLabelValues("registry.npmjs.org").Set(0) breaker.WithLabelValues("static.crates.io").Set(2) storage.WithLabelValues("get").Observe(0.002) @@ -71,8 +71,8 @@ func TestRuntimeViewShapesRegistry(t *testing.T) { if v.Requests != "100" { t.Errorf("Requests = %q, want %q", v.Requests, "100") } - if v.CacheHitRatio != "80.0" || !v.HasCacheTraffic { - t.Errorf("CacheHitRatio = %q (traffic=%v), want 80.0", v.CacheHitRatio, v.HasCacheTraffic) + if a := v.ArtifactCache; a.HitRatio != "80.0" || !a.HasTraffic { + t.Errorf("ArtifactCache = %+v, want an 80.0%% hit rate", a) } // Status codes collapse to classes; 5xx is flagged as a failure row. @@ -196,6 +196,48 @@ func TestRuntimeViewDropsZeroCounters(t *testing.T) { } } +// Artifact and metadata lookups are separate caches, so the view reports each +// on its own rather than pooling them into one hit rate. +func TestRuntimeViewSplitsCacheKinds(t *testing.T) { + reg := prometheus.NewRegistry() + hits := prometheus.NewCounterVec( + prometheus.CounterOpts{Name: "proxy_cache_hits_total", Help: "t"}, []string{"ecosystem", "kind"}) + misses := prometheus.NewCounterVec( + prometheus.CounterOpts{Name: "proxy_cache_misses_total", Help: "t"}, []string{"ecosystem", "kind"}) + served := prometheus.NewCounterVec( + prometheus.CounterOpts{Name: "proxy_metadata_cache_served_bytes_total", Help: "t"}, []string{"ecosystem"}) + reg.MustRegister(hits, misses, served) + + hits.WithLabelValues("npm", metrics.CacheKindArtifact).Add(80) + misses.WithLabelValues("npm", metrics.CacheKindArtifact).Add(20) + hits.WithLabelValues("pypi", metrics.CacheKindMetadata).Add(20) + hits.WithLabelValues("npm", metrics.CacheKindMetadata).Add(10) + misses.WithLabelValues("pypi", metrics.CacheKindMetadata).Add(10) + served.WithLabelValues("pypi").Add(2048) + + snap, err := metrics.GatherFrom(reg) + if err != nil { + t.Fatalf("GatherFrom: %v", err) + } + v := runtimeView(snap, false) + + want := map[string]struct { + got CacheLookups + want CacheLookups + }{ + "artifact": {v.ArtifactCache, CacheLookups{Hits: "80", Misses: "20", HitRatio: "80.0", HasTraffic: true}}, + "metadata": {v.MetadataCache, CacheLookups{Hits: "30", Misses: "10", HitRatio: "75.0", HasTraffic: true}}, + } + for kind, c := range want { + if c.got != c.want { + t.Errorf("%s lookups = %+v, want %+v", kind, c.got, c.want) + } + } + if v.MetadataServed != "2.0 KB" { + t.Errorf("MetadataServed = %q, want %q", v.MetadataServed, "2.0 KB") + } +} + // An empty registry must still render, reporting nothing rather than dividing // by zero or showing a hit rate of NaN. func TestRuntimeViewEmptyRegistry(t *testing.T) { @@ -208,11 +250,13 @@ func TestRuntimeViewEmptyRegistry(t *testing.T) { if !v.Available { t.Error("an empty registry is still a valid snapshot") } - if v.HasCacheTraffic { - t.Error("no traffic recorded, so HasCacheTraffic must be false") - } - if v.CacheHitRatio != "" { - t.Errorf("CacheHitRatio = %q, want empty when nothing was recorded", v.CacheHitRatio) + for name, c := range map[string]CacheLookups{"artifact": v.ArtifactCache, "metadata": v.MetadataCache} { + if c.HasTraffic { + t.Errorf("%s: no traffic recorded, so HasTraffic must be false", name) + } + if c.HitRatio != "" { + t.Errorf("%s: HitRatio = %q, want empty when nothing was recorded", name, c.HitRatio) + } } if v.Requests != "0" { t.Errorf("Requests = %q, want %q", v.Requests, "0") diff --git a/internal/server/analytics_test.go b/internal/server/analytics_test.go index 3ce5289..540408b 100644 --- a/internal/server/analytics_test.go +++ b/internal/server/analytics_test.go @@ -1,6 +1,7 @@ package server import ( + "database/sql" "net/http/httptest" "strings" "testing" @@ -275,3 +276,95 @@ func TestEcosystemMetricsMapping(t *testing.T) { t.Errorf("fields did not map across: %+v", m) } } + +// The metadata cache is stored apart from artifacts, so the page reports it in +// its own card rather than leaving it out of every figure. +func TestAnalyticsPageShowsMetadataCache(t *testing.T) { + ts := newTestServer(t) + defer ts.close() + ts.server.cfg.CacheMetadata = true + + for _, e := range []database.MetadataCacheEntry{ + {Ecosystem: "pypi", Name: "requests", StoragePath: "_metadata/pypi/requests", Size: sql.NullInt64{Int64: 3 << 10, Valid: true}}, + {Ecosystem: "pypi", Name: "urllib3", StoragePath: "_metadata/pypi/urllib3", Size: sql.NullInt64{Int64: 1 << 10, Valid: true}}, + } { + if err := ts.db.UpsertMetadataCache(&e); err != nil { + t.Fatalf("UpsertMetadataCache: %v", err) + } + } + + body := ts.getOK(t, "/ui/analytics") + for _, want := range []string{"Metadata cache", "Cached documents", "4.0 KB", "100.0%"} { + if !strings.Contains(body, want) { + t.Errorf("rendered page missing %q", want) + } + } + if strings.Contains(body, "Metadata caching is off") { + t.Error("the off-state note rendered with cache_metadata on") + } +} + +func TestAnalyticsPageNotesMetadataCachingOff(t *testing.T) { + ts := newTestServer(t) + defer ts.close() + + body := ts.getOK(t, "/ui/analytics") + for _, want := range []string{"Metadata cache", "Metadata caching is off", "No metadata cached yet"} { + if !strings.Contains(body, want) { + t.Errorf("rendered page missing %q", want) + } + } +} + +func TestMetadataCacheView(t *testing.T) { + v := metadataCacheView([]database.MetadataCacheStats{ + {Ecosystem: "npm", Entries: 3, Size: 3000}, + {Ecosystem: "pypi", Entries: 1, Size: 1000}, + }, true) + + if !v.Enabled || v.Entries != "4" || v.Size != "3.9 KB" { + t.Errorf("totals = %+v, want enabled, 4 entries, 3.9 KB", v) + } + if len(v.Rows) != 2 || v.Rows[0].SharePct != "75.0" || v.Rows[1].SharePct != "25.0" { + t.Errorf("rows = %+v, want npm 75.0%% then pypi 25.0%%", v.Rows) + } + + empty := metadataCacheView(nil, false) + if empty.Enabled || len(empty.Rows) != 0 || empty.Entries != "0" || empty.Size != "0 B" { + t.Errorf("empty view = %+v", empty) + } +} + +// The page reads the metadata card from the snapshot the gauges publish, so a +// page load issues no query of its own and the two always agree. +func TestAnalyticsPageMetadataCardUsesSnapshot(t *testing.T) { + ts := newTestServer(t) + defer ts.close() + ts.server.cfg.CacheMetadata = true + + seed := func(name string, size int64) { + t.Helper() + if err := ts.db.UpsertMetadataCache(&database.MetadataCacheEntry{ + Ecosystem: "pypi", Name: name, StoragePath: "_metadata/pypi/" + name, + Size: sql.NullInt64{Int64: size, Valid: true}, + }); err != nil { + t.Fatalf("UpsertMetadataCache: %v", err) + } + } + + seed("requests", 1<<10) + ts.server.updateMetadataCacheStats() + if !strings.Contains(ts.getOK(t, "/ui/analytics"), "1.0 KB") { + t.Fatal("page did not show the snapshot the metrics tick read") + } + + seed("urllib3", 2<<10) + if body := ts.getOK(t, "/ui/analytics"); !strings.Contains(body, "1.0 KB") || strings.Contains(body, "3.0 KB") { + t.Error("a page load re-queried the database instead of serving the snapshot") + } + + ts.server.updateMetadataCacheStats() + if !strings.Contains(ts.getOK(t, "/ui/analytics"), "3.0 KB") { + t.Error("page did not pick up the refreshed snapshot") + } +} diff --git a/internal/server/ecosystem_cache.go b/internal/server/ecosystem_cache.go index 83e8c0d..d12d3a2 100644 --- a/internal/server/ecosystem_cache.go +++ b/internal/server/ecosystem_cache.go @@ -17,20 +17,23 @@ const ecosystemStatsTTL = time.Minute // that is down is not queried once per request. const ecosystemStatsErrorTTL = 5 * time.Second -// ecosystemStatsCache memoizes GetEcosystemStats. +// statsCache memoizes one of the grouped statistics queries behind the +// analytics page and the metrics gauges. // -// The query is three grouped aggregations over the whole artifact table, which -// is fine once a minute for the metrics gauges but not once per page load: the -// UI is unauthenticated, so without this a client reloading /ui/analytics would -// keep the database busy for as long as it cared to. +// Each query is a grouped aggregation over a whole table, which is fine once a +// minute for the metrics gauges but not once per page load: the UI is +// unauthenticated, so without this a client reloading /ui/analytics would keep +// the database busy for as long as it cared to. The page and the gauges read +// the same snapshot, so they never disagree. // -// The zero value is usable, so a Server assembled as a struct literal -- as +// L selects the query. It is a zero-size type rather than a stored function so +// the zero value stays usable: a Server assembled as a struct literal -- as // tests do -- needs no constructor and cannot end up with a nil cache. -type ecosystemStatsCache struct { +type statsCache[E any, L statsLoader[E]] struct { ttl time.Duration mu sync.Mutex - stats []database.EcosystemStats + stats []E err error lastAt time.Time // goodAt is when stats was last read successfully, which is not lastAt @@ -38,15 +41,39 @@ type ecosystemStatsCache struct { goodAt time.Time } +// statsLoader runs the query a statsCache memoizes, returning one row per +// ecosystem. +type statsLoader[E any] interface { + load(db *database.DB) ([]E, error) +} + +type ecosystemStatsLoader struct{} + +func (ecosystemStatsLoader) load(db *database.DB) ([]database.EcosystemStats, error) { + return db.GetEcosystemStats() +} + +type metadataStatsLoader struct{} + +func (metadataStatsLoader) load(db *database.DB) ([]database.MetadataCacheStats, error) { + return db.GetMetadataCacheStats() +} + +// ecosystemStatsCache memoizes GetEcosystemStats. +type ecosystemStatsCache = statsCache[database.EcosystemStats, ecosystemStatsLoader] + +// metadataStatsCache memoizes GetMetadataCacheStats. +type metadataStatsCache = statsCache[database.MetadataCacheStats, metadataStatsLoader] + // SnapshotAt reports when the rows currently held were last read successfully. // The zero time means no read has ever succeeded. -func (c *ecosystemStatsCache) SnapshotAt() time.Time { +func (c *statsCache[E, L]) SnapshotAt() time.Time { c.mu.Lock() defer c.mu.Unlock() return c.goodAt } -func (c *ecosystemStatsCache) interval() time.Duration { +func (c *statsCache[E, L]) interval() time.Duration { if c.err != nil { return ecosystemStatsErrorTTL } @@ -60,7 +87,7 @@ func (c *ecosystemStatsCache) interval() time.Duration { // // A failed read returns the last good snapshot alongside the error, so a // caller can choose to serve stale figures rather than fail outright. -func (c *ecosystemStatsCache) Get(db *database.DB) ([]database.EcosystemStats, error) { +func (c *statsCache[E, L]) Get(db *database.DB) ([]E, error) { c.mu.Lock() defer c.mu.Unlock() @@ -72,14 +99,15 @@ func (c *ecosystemStatsCache) Get(db *database.DB) ([]database.EcosystemStats, e // Refresh forces a read, bypassing the TTL. The metrics loop uses it so its own // tick is never served a snapshot that is about to expire. -func (c *ecosystemStatsCache) Refresh(db *database.DB) ([]database.EcosystemStats, error) { +func (c *statsCache[E, L]) Refresh(db *database.DB) ([]E, error) { c.mu.Lock() defer c.mu.Unlock() return c.refreshLocked(db) } -func (c *ecosystemStatsCache) refreshLocked(db *database.DB) ([]database.EcosystemStats, error) { - stats, err := db.GetEcosystemStats() +func (c *statsCache[E, L]) refreshLocked(db *database.DB) ([]E, error) { + var loader L + stats, err := loader.load(db) c.lastAt = time.Now() c.err = err if err != nil { diff --git a/internal/server/ecosystem_cache_test.go b/internal/server/ecosystem_cache_test.go index 5323ad1..9356dc4 100644 --- a/internal/server/ecosystem_cache_test.go +++ b/internal/server/ecosystem_cache_test.go @@ -1,6 +1,7 @@ package server import ( + "database/sql" "path/filepath" "testing" "time" @@ -197,3 +198,49 @@ func TestEcosystemStatsCacheSnapshotAtTracksSuccessOnly(t *testing.T) { t.Errorf("SnapshotAt moved on a failed read: %v, want %v", got, at) } } + +// The metadata cache figures go through the same memo as the artifact ones. +// Rows added inside the TTL stay out of the snapshot until a Refresh, which is +// what the metrics tick calls. +func TestMetadataStatsCacheServesSnapshot(t *testing.T) { + db := cacheTestDB(t) + c := metadataStatsCache{ttl: time.Hour} + + seed := func(name string, size int64) { + t.Helper() + if err := db.UpsertMetadataCache(&database.MetadataCacheEntry{ + Ecosystem: "npm", + Name: name, + StoragePath: "_metadata/npm/" + name, + Size: sql.NullInt64{Int64: size, Valid: true}, + }); err != nil { + t.Fatalf("UpsertMetadataCache: %v", err) + } + } + entries := func(stats []database.MetadataCacheStats) int64 { + var n int64 + for _, s := range stats { + n += s.Entries + } + return n + } + + seed("lodash", 100) + stats, err := c.Get(db) + if err != nil || entries(stats) != 1 { + t.Fatalf("first Get = %+v, %v; want 1 entry", stats, err) + } + + seed("react", 50) + if stats, _ := c.Get(db); entries(stats) != 1 { + t.Errorf("Get inside the TTL = %+v; want the 1-entry snapshot", stats) + } + + stats, err = c.Refresh(db) + if err != nil || entries(stats) != 2 { + t.Fatalf("Refresh = %+v, %v; want 2 entries", stats, err) + } + if stats, _ := c.Get(db); entries(stats) != 2 { + t.Errorf("Get after Refresh = %+v; want the refreshed snapshot", stats) + } +} diff --git a/internal/server/server.go b/internal/server/server.go index c64b33c..42830be 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -119,6 +119,7 @@ type Server struct { breakers *breakerMonitor rateLimits *registryclient.RateLimitGate ecoStats ecosystemStatsCache + metaStats metadataStatsCache sources sourceTracker scheduler *mirror.Scheduler @@ -599,12 +600,30 @@ func (s *Server) updateCacheStatsMetrics() { // Update once immediately s.updateCacheStats() + s.updateMetadataCacheStats() for range ticker.C { s.updateCacheStats() + s.updateMetadataCacheStats() } } +// updateMetadataCacheStats republishes the metadata cache gauges from a fresh +// read of the snapshot the analytics page also serves. It runs separately +// from updateCacheStats so a failure in one leaves the other's gauges current. +func (s *Server) updateMetadataCacheStats() { + stats, err := s.metaStats.Refresh(s.db) + if err != nil { + s.logger.Warn("failed to get metadata cache stats for metrics", "error", err) + return + } + out := make([]metrics.MetadataCacheStats, 0, len(stats)) + for _, e := range stats { + out = append(out, metrics.MetadataCacheStats{Ecosystem: e.Ecosystem, Entries: e.Entries, Size: e.Size}) + } + metrics.UpdateMetadataCacheStats(out) +} + func (s *Server) updateCacheStats() { stats, err := s.db.GetCacheStats() if err != nil { diff --git a/internal/server/templates/components/runtime_metrics.html b/internal/server/templates/components/runtime_metrics.html index 6eba427..18a3140 100644 --- a/internal/server/templates/components/runtime_metrics.html +++ b/internal/server/templates/components/runtime_metrics.html @@ -25,8 +25,8 @@
{{.RequestMean}}
-
Cache hit rate
-
{{if .HasCacheTraffic}}{{.CacheHitRatio}}%{{else}}—{{end}}
+
Artifact hit rate
+
{{if .ArtifactCache.HasTraffic}}{{.ArtifactCache.HitRatio}}%{{else}}—{{end}}
Upstream fetches
@@ -41,16 +41,14 @@
{{template "runtime_counts" dict "Title" "Responses by status" "Rows" .StatusClasses "Empty" "No requests yet"}} + {{template "runtime_cache" dict "Title" "Artifact cache lookups" "Lookups" .ArtifactCache}} +
-

Cache lookups

-
+ {{template "runtime_cache" dict "Title" "Metadata cache lookups" "Lookups" .MetadataCache}} +
-
hits
-
{{.CacheHits}}
-
-
-
misses
-
{{.CacheMisses}}
+
served from cache
+
{{.MetadataServed}}
@@ -268,3 +266,23 @@ {{end}}
{{end}} + +{{define "runtime_cache"}} +
+

{{.Title}}

+
+
+
hits
+
{{.Lookups.Hits}}
+
+
+
misses
+
{{.Lookups.Misses}}
+
+
+
hit rate
+
{{if .Lookups.HasTraffic}}{{.Lookups.HitRatio}}%{{else}}—{{end}}
+
+
+
+{{end}} diff --git a/internal/server/templates/pages/analytics.html b/internal/server/templates/pages/analytics.html index da97a7c..cda81bb 100644 --- a/internal/server/templates/pages/analytics.html +++ b/internal/server/templates/pages/analytics.html @@ -215,6 +215,58 @@ {{end}}
+ +
+
+

Metadata cache

+

+ Cached registry metadata, kept apart from artifacts: the download, cache size and + artifact figures above do not include it. + {{if not .Metadata.Enabled}}Metadata caching is off; set cache_metadata to enable it.{{end}} +

+
+ {{if .Metadata.Rows}} +
+
+
Cached documents
+
{{.Metadata.Entries}}
+
+
+
Metadata size
+
{{.Metadata.Size}}
+
+
+
+ + + + + + + + + + + {{range .Metadata.Rows}} + + + + + + + {{end}} + +
EcosystemSizeShareDocuments
{{template "ecosystem_badge" .Ecosystem}}{{.Size}}{{.SharePct}}%{{.Entries}}
+
+ {{else if .Metadata.Failed}} +
+ The metadata cache query failed, so these figures are unavailable. Check the proxy logs. +
+ {{else}} +
No metadata cached yet
+ {{end}} +
+ {{if .EnrichmentStats.HasVulns}} {{template "security_overview" .EnrichmentStats}} {{end}} diff --git a/internal/server/templates_test.go b/internal/server/templates_test.go index 276af6e..708f02c 100644 --- a/internal/server/templates_test.go +++ b/internal/server/templates_test.go @@ -63,19 +63,18 @@ func TestTemplatesRenderAllPages(t *testing.T) { {Ecosystem: "cargo", CacheSize: 120, Downloads: 3004, DownloadedBytes: 500_000_000}, }, 1_500_000_000, "1.4 GB"), Runtime: RuntimeView{ - Available: true, - Requests: "12,004", - ActiveRequests: "2", - RequestMean: "8.1 ms", - CacheHits: "9,000", - CacheMisses: "1,000", - CacheHitRatio: "90.0", - HasCacheTraffic: true, - StatusClasses: []LabelledCount{{Label: "2xx", Count: "11,900"}, {Label: "5xx", Count: "104", Bad: true}}, - StorageOps: []LabelledStat{{Label: "get", Count: "9,000", Mean: "412 µs"}}, - ScanErrors: []LabelledCount{{Label: "npm · clamav · timeout", Count: "3", Bad: true}}, - ScanningOn: true, - Breakers: []BreakerRow{{Registry: "registry.npmjs.org", State: "closed"}}, + Available: true, + Requests: "12,004", + ActiveRequests: "2", + RequestMean: "8.1 ms", + ArtifactCache: CacheLookups{Hits: "9,000", Misses: "1,000", HitRatio: "90.0", HasTraffic: true}, + MetadataCache: CacheLookups{Hits: "4,000", Misses: "500", HitRatio: "88.9", HasTraffic: true}, + MetadataServed: "12.0 MB", + StatusClasses: []LabelledCount{{Label: "2xx", Count: "11,900"}, {Label: "5xx", Count: "104", Bad: true}}, + StorageOps: []LabelledStat{{Label: "get", Count: "9,000", Mean: "412 µs"}}, + ScanErrors: []LabelledCount{{Label: "npm · clamav · timeout", Count: "3", Bad: true}}, + ScanningOn: true, + Breakers: []BreakerRow{{Registry: "registry.npmjs.org", State: "closed"}}, }, }}, {"analytics", AnalyticsData{}}, From 8bebc560f7a11c5d1c6cc4bff56cf2a71dd58848 Mon Sep 17 00:00:00 2001 From: Giles Westwood Date: Tue, 20 Oct 2026 11:35:08 +0100 Subject: [PATCH 326/633] Delete cached artifacts through the API and the UI (#491) * Delete cached artifacts through the API and the UI cache_delete (PROXY_CACHE_DELETE) registers DELETE /api/cache/{ecosystem}/{name}/{version}, which removes a version's cached files, or only ?filename=, so the next request fetches them from the upstream again. Records are cleared and the stored objects queued for deletion like a replaced artifact. Pushed archives are left alone: a whole-version delete skips them and naming one answers 409. With it enabled, the version page shows a Delete button for each cached file and one for the whole version. The UI calls /ui/api/cache so the same ingress rules that guard /ui guard its deletes. Neither route has authentication of its own, so the option is off by default. * Load the cache delete script from a static file Move the version page's delete-button JavaScript into static/cache-delete.js and load it with a script tag, as packages-list.js does. --------- Co-authored-by: Giles Westwood <201443+gilesw@users.noreply.github.com> --- README.md | 14 ++ config.example.yaml | 5 + docs/configuration.md | 23 +++ docs/swagger/docs.go | 81 ++++++++ docs/swagger/swagger.json | 81 ++++++++ internal/config/config.go | 6 + internal/server/cache_api.go | 86 ++++++++ internal/server/cache_api_test.go | 183 ++++++++++++++++++ internal/server/dashboard.go | 3 + internal/server/errors.go | 1 + internal/server/layout.go | 3 + internal/server/server.go | 32 ++- internal/server/server_test.go | 1 + internal/server/static/cache-delete.js | 36 ++++ .../server/templates/pages/version_show.html | 14 +- 15 files changed, 559 insertions(+), 10 deletions(-) create mode 100644 internal/server/cache_api.go create mode 100644 internal/server/cache_api_test.go create mode 100644 internal/server/static/cache-delete.js diff --git a/README.md b/README.md index 6fcab11..6ab21cb 100644 --- a/README.md +++ b/README.md @@ -1170,6 +1170,14 @@ Registered when `hosted.enabled` is set. Pushing and deleting require `Authoriza | `GET /api/hosted/{ecosystem}/{name}` | List the archives pushed for a package | | `DELETE /api/hosted/{ecosystem}/{name}/{version}` | Delete a pushed version | +### Cache API + +Registered when `cache_delete` is set. It has no authentication of its own. + +| Endpoint | Description | +|----------|-------------| +| `DELETE /api/cache/{ecosystem}/{name}/{version}` | Delete a version's cached files, or only `?filename=` | + ### Enrichment API The proxy provides REST endpoints for package metadata enrichment, vulnerability scanning, and outdated detection. @@ -1655,6 +1663,12 @@ cache/artifacts/ └── nginx-1.24.0-1.fc39.x86_64.rpm ``` +To delete one version's cached files, so the next request fetches them again, enable `cache_delete` and use the Delete buttons on the version page in the web UI, or the [cache API](docs/configuration.md#cache-delete-api): + +```bash +curl -X DELETE 'http://localhost:8080/api/cache/maven/com.google.guava:guava/32.1.3-jre?filename=guava-32.1.3-jre.jar' +``` + Cache metadata is stored in SQLite (default) or PostgreSQL. To clear a local cache: ```bash diff --git a/config.example.yaml b/config.example.yaml index ed168de..277eb23 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -22,6 +22,11 @@ base_url: "http://localhost:8080" # scanning. Set to "0" to disable. Default: "5m". Does not limit proxy mirror. # artifact_timeout: "5m" +# Register DELETE /api/cache and show Delete buttons on the UI's version page, +# for removing cached artifacts so they are fetched again. Has no +# authentication: restrict /api/cache and /ui at the ingress first. +# cache_delete: false + # Largest upstream artifact to fetch (e.g. "2GB"). Larger ones are refused # and not cached. Default: unlimited. # artifact_max_size: "" diff --git a/docs/configuration.md b/docs/configuration.md index ed7f13f..dccdf26 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1148,6 +1148,29 @@ curl -X POST http://localhost:8080/api/mirror \ -d '{"purls":["pkg:docker/library/alpine@3.20"],"platforms":["linux/amd64"]}' ``` +## Cache Delete API + +Deleting cached artifacts is disabled by default. Enable it to register `DELETE /api/cache` and show Delete buttons on the version page of the web UI: + +```yaml +cache_delete: true +``` + +Or via environment variable: `PROXY_CACHE_DELETE=true`. + +The endpoint has no authentication. Before enabling it, restrict `/api/cache` and `/ui` to trusted users at your ingress or reverse proxy. The UI sends its deletes to `/ui/api/cache`, so whatever protects `/ui` protects those too. + +Delete every cached file of a version, or only the one named by `filename`: + +```bash +curl -X DELETE http://localhost:8080/api/cache/maven/com.google.guava:guava/32.1.3-jre +curl -X DELETE 'http://localhost:8080/api/cache/maven/com.google.guava:guava/32.1.3-jre?filename=guava-32.1.3-jre.jar' +``` + +The name is the package name the UI shows, for example `group:artifact` for Maven. The response lists the deleted files. The next request for a deleted file fetches it from the upstream again. The stored bytes are removed after the same delay as a replaced artifact, so downloads already in progress can finish. + +Pushed archives are not touched: a whole-version delete skips them, and naming one answers 409. Delete those through the [hosted API](#hosted-packages). + ## Mirror Command The `proxy mirror` command pre-populates the cache from various sources. It accepts the same storage and database flags as `serve`. diff --git a/docs/swagger/docs.go b/docs/swagger/docs.go index 7a8387d..7132ffb 100644 --- a/docs/swagger/docs.go +++ b/docs/swagger/docs.go @@ -115,6 +115,73 @@ const docTemplate = `{ } } }, + "/api/cache/{ecosystem}/{name}/{version}": { + "delete": { + "description": "Removes the cached files of name@version, or only the one named by filename, so the next request fetches them from the upstream again. Pushed archives are not touched; delete those through /api/hosted.", + "produces": [ + "application/json" + ], + "tags": [ + "cache" + ], + "summary": "Delete cached artifacts", + "parameters": [ + { + "type": "string", + "description": "Ecosystem", + "name": "ecosystem", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "Package name as shown in the UI, e.g. com.google.guava:guava for Maven", + "name": "name", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "Version", + "name": "version", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "Delete only this file", + "name": "filename", + "in": "query" + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/server.CacheDeleteResponse" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/server.ErrorResponse" + } + }, + "404": { + "description": "Not Found", + "schema": { + "$ref": "#/definitions/server.ErrorResponse" + } + }, + "409": { + "description": "Conflict", + "schema": { + "$ref": "#/definitions/server.ErrorResponse" + } + } + } + } + }, "/api/hosted/{ecosystem}/{name}": { "get": { "produces": [ @@ -753,6 +820,20 @@ const docTemplate = `{ } } }, + "server.CacheDeleteResponse": { + "type": "object", + "properties": { + "deleted": { + "type": "array", + "items": { + "type": "string" + } + }, + "purl": { + "type": "string" + } + } + }, "server.EcosystemStatsEntry": { "type": "object", "properties": { diff --git a/docs/swagger/swagger.json b/docs/swagger/swagger.json index aad0cea..fcb4123 100644 --- a/docs/swagger/swagger.json +++ b/docs/swagger/swagger.json @@ -108,6 +108,73 @@ } } }, + "/api/cache/{ecosystem}/{name}/{version}": { + "delete": { + "description": "Removes the cached files of name@version, or only the one named by filename, so the next request fetches them from the upstream again. Pushed archives are not touched; delete those through /api/hosted.", + "produces": [ + "application/json" + ], + "tags": [ + "cache" + ], + "summary": "Delete cached artifacts", + "parameters": [ + { + "type": "string", + "description": "Ecosystem", + "name": "ecosystem", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "Package name as shown in the UI, e.g. com.google.guava:guava for Maven", + "name": "name", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "Version", + "name": "version", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "Delete only this file", + "name": "filename", + "in": "query" + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/server.CacheDeleteResponse" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/server.ErrorResponse" + } + }, + "404": { + "description": "Not Found", + "schema": { + "$ref": "#/definitions/server.ErrorResponse" + } + }, + "409": { + "description": "Conflict", + "schema": { + "$ref": "#/definitions/server.ErrorResponse" + } + } + } + } + }, "/api/hosted/{ecosystem}/{name}": { "get": { "produces": [ @@ -746,6 +813,20 @@ } } }, + "server.CacheDeleteResponse": { + "type": "object", + "properties": { + "deleted": { + "type": "array", + "items": { + "type": "string" + } + }, + "purl": { + "type": "string" + } + } + }, "server.EcosystemStatsEntry": { "type": "object", "properties": { diff --git a/internal/config/config.go b/internal/config/config.go index 14c7462..e9976de 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -202,6 +202,11 @@ type Config struct { // Mirror configures scheduled mirroring. Mirror MirrorConfig `json:"mirror" yaml:"mirror"` + // CacheDelete enables DELETE /api/cache for removing cached artifacts, and + // the delete buttons on the UI's version page. It has no authentication of + // its own, so restrict /api/cache and /ui at the ingress when enabling it. + CacheDelete bool `json:"cache_delete" yaml:"cache_delete"` + // Hosted configures pushing packages to the proxy (/api/hosted). Hosted HostedConfig `json:"hosted" yaml:"hosted"` @@ -1257,6 +1262,7 @@ func (c *Config) LoadFromEnv() { setEnvString(&c.Scanning.FetchBaseURL, "PROXY_SCANNING_FETCH_BASE_URL") setEnvBool(&c.CacheMetadata, "PROXY_CACHE_METADATA") setEnvBool(&c.MirrorAPI, "PROXY_MIRROR_API") + setEnvBool(&c.CacheDelete, "PROXY_CACHE_DELETE") setEnvBool(&c.Hosted.Enabled, "PROXY_HOSTED_ENABLED") setEnvString(&c.Hosted.Token, "PROXY_HOSTED_TOKEN") setEnvString(&c.Hosted.MaxUploadSize, "PROXY_HOSTED_MAX_UPLOAD_SIZE") diff --git a/internal/server/cache_api.go b/internal/server/cache_api.go new file mode 100644 index 0000000..771d333 --- /dev/null +++ b/internal/server/cache_api.go @@ -0,0 +1,86 @@ +package server + +import ( + "net/http" + "strings" + + "github.com/git-pkgs/proxy/internal/database" + "github.com/go-chi/chi/v5" +) + +// CacheDeleteResponse lists the cached files removed for a version. +type CacheDeleteResponse struct { + PURL string `json:"purl"` + Deleted []string `json:"deleted"` +} + +// handleCacheDelete handles DELETE /api/cache/{ecosystem}/{name}/{version} +// @Summary Delete cached artifacts +// @Description Removes the cached files of name@version, or only the one named by filename, so the next request fetches them from the upstream again. Pushed archives are not touched; delete those through /api/hosted. +// @Tags cache +// @Produce json +// @Param ecosystem path string true "Ecosystem" +// @Param name path string true "Package name as shown in the UI, e.g. com.google.guava:guava for Maven" +// @Param version path string true "Version" +// @Param filename query string false "Delete only this file" +// @Success 200 {object} CacheDeleteResponse +// @Failure 400 {object} ErrorResponse +// @Failure 404 {object} ErrorResponse +// @Failure 409 {object} ErrorResponse +// @Router /api/cache/{ecosystem}/{name}/{version} [delete] +func (s *Server) handleCacheDelete(w http.ResponseWriter, r *http.Request) { + segments, err := packagePathSegments(r) + if err != nil { + badRequest(w, err.Error()) + return + } + if len(segments) < minVersionPathSegments { + badRequest(w, "path must be /api/cache/{ecosystem}/{name}/{version}") + return + } + last := len(segments) - 1 + ecosystem, name, version := chi.URLParam(r, "ecosystem"), strings.Join(segments[:last], "/"), segments[last] + filename := r.URL.Query().Get("filename") + + versionPURL := s.cachedVersionPURL(ecosystem, name, version) + if versionPURL == "" { + notFound(w, "package not found") + return + } + artifacts, err := s.db.GetArtifactsByVersionPURL(versionPURL) + if err != nil { + s.logger.Error("failed to list artifacts", "purl", versionPURL, "error", err) + internalError(w, "failed to list artifacts") + return + } + + deleted := []string{} + for _, art := range artifacts { + if filename != "" && art.Filename != filename { + continue + } + if !art.StoragePath.Valid { + continue + } + if art.Source == database.SourceLocal { + if filename != "" { + writeError(w, http.StatusConflict, ErrCodeConflict, "pushed archives are deleted through /api/hosted") + return + } + continue + } + if err := s.db.DiscardArtifact(versionPURL, art.Filename, art.StoragePath.String); err != nil { + s.logger.Error("failed to delete cached artifact", "purl", versionPURL, "filename", art.Filename, "error", err) + internalError(w, "failed to delete cached artifact") + return + } + deleted = append(deleted, art.Filename) + } + if len(deleted) == 0 { + notFound(w, "no cached artifact matched") + return + } + + s.logger.Info("deleted cached artifacts", "purl", versionPURL, "files", deleted) + writeJSON(w, CacheDeleteResponse{PURL: versionPURL, Deleted: deleted}) +} diff --git a/internal/server/cache_api_test.go b/internal/server/cache_api_test.go new file mode 100644 index 0000000..8e66755 --- /dev/null +++ b/internal/server/cache_api_test.go @@ -0,0 +1,183 @@ +package server + +import ( + "database/sql" + "encoding/json" + "net/http" + "net/http/httptest" + "slices" + "strings" + "testing" + "time" + + "github.com/git-pkgs/proxy/internal/database" + "github.com/go-chi/chi/v5" +) + +const ( + cacheTestPackage = "pkg:maven/com.example/lib" + cacheTestVersion = cacheTestPackage + "@1.0" +) + +// seedCacheDeleteVersion caches lib-1.0.jar and its .sha1 for +// com.example:lib 1.0, plus a pushed lib-1.0-local.jar when pushed is set. +func seedCacheDeleteVersion(t *testing.T, db *database.DB, pushed bool) { + t.Helper() + if err := db.UpsertPackage(&database.Package{PURL: cacheTestPackage, Ecosystem: "maven", Name: "com.example:lib"}); err != nil { + t.Fatalf("UpsertPackage: %v", err) + } + if err := db.UpsertVersion(&database.Version{PURL: cacheTestVersion, PackagePURL: cacheTestPackage}); err != nil { + t.Fatalf("UpsertVersion: %v", err) + } + files := map[string]string{"lib-1.0.jar": database.SourceUpstream, "lib-1.0.jar.sha1": database.SourceUpstream} + if pushed { + files["lib-1.0-local.jar"] = database.SourceLocal + } + for filename, source := range files { + if err := db.UpsertArtifact(&database.Artifact{ + VersionPURL: cacheTestVersion, + Filename: filename, + UpstreamURL: "https://repo1.maven.org/maven2/com/example/lib/1.0/" + filename, + StoragePath: sql.NullString{String: "objects/" + filename, Valid: true}, + FetchedAt: sql.NullTime{Time: time.Now(), Valid: true}, + Source: source, + }); err != nil { + t.Fatalf("UpsertArtifact %s: %v", filename, err) + } + } +} + +func cacheDelete(ts *testServer, target string) *httptest.ResponseRecorder { + r := chi.NewRouter() + r.Delete("/api/cache/{ecosystem}/*", ts.server.handleCacheDelete) + w := httptest.NewRecorder() + r.ServeHTTP(w, httptest.NewRequest(http.MethodDelete, target, nil)) + return w +} + +func cachedFiles(t *testing.T, db *database.DB) []string { + t.Helper() + artifacts, err := db.GetArtifactsByVersionPURL(cacheTestVersion) + if err != nil { + t.Fatalf("GetArtifactsByVersionPURL: %v", err) + } + var cached []string + for _, art := range artifacts { + if art.StoragePath.Valid { + cached = append(cached, art.Filename) + } + } + slices.Sort(cached) + return cached +} + +func queuedDeletes(t *testing.T, db *database.DB) []string { + t.Helper() + paths, err := db.GetDuePendingDeletes(time.Now().Add(time.Hour), 100) + if err != nil { + t.Fatalf("GetDuePendingDeletes: %v", err) + } + slices.Sort(paths) + return paths +} + +func TestCacheDelete_OneFile(t *testing.T) { + ts := newTestServer(t) + defer ts.close() + seedCacheDeleteVersion(t, ts.db, false) + + w := cacheDelete(ts, "/api/cache/maven/com.example:lib/1.0?filename=lib-1.0.jar") + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want 200; body: %s", w.Code, w.Body.String()) + } + var resp CacheDeleteResponse + if err := json.NewDecoder(w.Body).Decode(&resp); err != nil { + t.Fatalf("decoding response: %v", err) + } + if resp.PURL != cacheTestVersion || !slices.Equal(resp.Deleted, []string{"lib-1.0.jar"}) { + t.Errorf("response = %+v, want lib-1.0.jar deleted from %s", resp, cacheTestVersion) + } + if got := cachedFiles(t, ts.db); !slices.Equal(got, []string{"lib-1.0.jar.sha1"}) { + t.Errorf("still cached = %v, want only the .sha1", got) + } + if got := queuedDeletes(t, ts.db); !slices.Equal(got, []string{"objects/lib-1.0.jar"}) { + t.Errorf("queued for deletion = %v, want the jar's object", got) + } +} + +func TestCacheDelete_WholeVersionKeepsPushedFiles(t *testing.T) { + ts := newTestServer(t) + defer ts.close() + seedCacheDeleteVersion(t, ts.db, true) + + w := cacheDelete(ts, "/api/cache/maven/com.example:lib/1.0") + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want 200; body: %s", w.Code, w.Body.String()) + } + if got := cachedFiles(t, ts.db); !slices.Equal(got, []string{"lib-1.0-local.jar"}) { + t.Errorf("still cached = %v, want only the pushed jar", got) + } + if got := queuedDeletes(t, ts.db); !slices.Equal(got, []string{"objects/lib-1.0.jar", "objects/lib-1.0.jar.sha1"}) { + t.Errorf("queued for deletion = %v, want both upstream objects", got) + } +} + +func TestCacheDelete_Refusals(t *testing.T) { + tests := []struct { + name string + target string + want int + }{ + {"pushed file", "/api/cache/maven/com.example:lib/1.0?filename=lib-1.0-local.jar", http.StatusConflict}, + {"unknown file", "/api/cache/maven/com.example:lib/1.0?filename=other.jar", http.StatusNotFound}, + {"unknown version", "/api/cache/maven/com.example:lib/2.0", http.StatusNotFound}, + {"unknown package", "/api/cache/maven/com.example:other/1.0", http.StatusNotFound}, + {"no version", "/api/cache/maven/lib", http.StatusBadRequest}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ts := newTestServer(t) + defer ts.close() + seedCacheDeleteVersion(t, ts.db, true) + + w := cacheDelete(ts, tt.target) + if w.Code != tt.want { + t.Fatalf("status = %d, want %d; body: %s", w.Code, tt.want, w.Body.String()) + } + if got := cachedFiles(t, ts.db); len(got) != 3 { + t.Errorf("still cached = %v, want all three files untouched", got) + } + }) + } +} + +func TestVersionPage_CacheDeleteButtons(t *testing.T) { + for _, enabled := range []bool{false, true} { + ts := newTestServer(t) + seedCacheDeleteVersion(t, ts.db, true) + ts.server.cfg.CacheDelete = enabled + + w := httptest.NewRecorder() + ts.handler.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/ui/package/maven/com.example:lib/1.0", nil)) + ts.close() + if w.Code != http.StatusOK { + t.Fatalf("enabled=%v: status = %d, want 200", enabled, w.Code) + } + body := w.Body.String() + // One button per upstream file plus the version-wide one; the pushed + // jar gets none. + wantButtons := 0 + if enabled { + wantButtons = 3 + } + if got := strings.Count(body, `class="cache-delete `); got != wantButtons { + t.Errorf("enabled=%v: %d delete buttons, want %d", enabled, got, wantButtons) + } + if enabled && !strings.Contains(body, `data-cache-url="/ui/api/cache/maven/com.example:lib/1.0"`) { + t.Errorf("page has no data-cache-url for the version") + } + if got := strings.Contains(body, ` +{{end}} {{else}}
No artifacts cached yet From 03f0ebdd55207cdfad9b23bd7fe664edaec5d076 Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Tue, 20 Oct 2026 11:41:18 +0100 Subject: [PATCH 327/633] Use shared archives browser for cached artifacts (#512) * Use shared archives browser for cached artifacts * Pin archives to combined browser and Hex changes * Use archives v0.9.0 release * Return 404 when browsing archive directories as files --- go.mod | 4 +- go.sum | 8 +- internal/server/browse.go | 19 +- internal/server/browse_archive.go | 357 +------------------------ internal/server/browse_archive_test.go | 67 +++++ 5 files changed, 84 insertions(+), 371 deletions(-) diff --git a/go.mod b/go.mod index a760f5b..7878bee 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ require ( github.com/Masterminds/semver/v3 v3.5.0 github.com/aws/aws-sdk-go-v2/config v1.33.7 github.com/aws/aws-sdk-go-v2/service/ecr v1.66.2 - github.com/git-pkgs/archives v0.8.1 + github.com/git-pkgs/archives v0.9.0 github.com/git-pkgs/artifacts v0.2.1 github.com/git-pkgs/cooldown v0.3.0 github.com/git-pkgs/enrichment v0.7.2 @@ -106,7 +106,7 @@ require ( github.com/rogpeppe/go-internal v1.16.0 // indirect github.com/rs/dnscache v0.0.0-20230804202142-fc85eb664529 // indirect github.com/rubyist/circuitbreaker v2.2.1+incompatible // indirect - github.com/ulikunitz/xz v0.5.16 // indirect + github.com/ulikunitz/xz v0.5.17 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0 // indirect go.opentelemetry.io/otel v1.45.0 // indirect diff --git a/go.sum b/go.sum index 387062b..36686db 100644 --- a/go.sum +++ b/go.sum @@ -117,8 +117,8 @@ github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a h1:yDWHCSQ40h88yi github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a/go.mod h1:7Ga40egUymuWXxAe151lTNnCv97MddSOVsjpPPkityA= github.com/felixge/httpsnoop v1.1.0 h1:3YtUj32ZZkqZtt3sZZsClsymw/QDuVfpNhoA31zeORc= github.com/felixge/httpsnoop v1.1.0/go.mod h1:Zqxgdd+1Rkcz8euOqdr7lqgCRJztwr5hp9vDSi5UZCE= -github.com/git-pkgs/archives v0.8.1 h1:IcT1bVsAec8SqJ4m9vTnD644KiadQZ4ERrWnm/30t3o= -github.com/git-pkgs/archives v0.8.1/go.mod h1:gIyPXPQr5e+YrMqA5KQ7x6pLrFXHNF1+CjdedNJdGUU= +github.com/git-pkgs/archives v0.9.0 h1:j5+GtZLrcsefIG5QxAQ2dcdCFf57uizcn03PyGRFA3U= +github.com/git-pkgs/archives v0.9.0/go.mod h1:GJ/h7NPTUYiPinOUxpYNsHFCUikYZcJXvugBvP1VGNc= github.com/git-pkgs/artifacts v0.2.1 h1:VwdxR4yTDaqBZ34h0slxQBVyr2Xfa+QGOKCKviMx2xk= github.com/git-pkgs/artifacts v0.2.1/go.mod h1:Otosgq52pXT5UNN7lh6s/lszpWKVDO8XrOjN1M70/IA= github.com/git-pkgs/cooldown v0.3.0 h1:eAl2CQFwqXxluoVP9hHmropAf1VoV3gSxLMkN+NEu8A= @@ -273,8 +273,8 @@ github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWD github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg= github.com/swaggo/swag v1.16.6 h1:qBNcx53ZaX+M5dxVyTrgQ0PJ/ACK+NzhwcbieTt+9yI= github.com/swaggo/swag v1.16.6/go.mod h1:ngP2etMK5a0P3QBizic5MEwpRmluJZPHjXcMoj4Xesg= -github.com/ulikunitz/xz v0.5.16 h1:ld6NyySjx5lowVKwJvMRLnW5nxKX/xnpSiFYZ/Lxur0= -github.com/ulikunitz/xz v0.5.16/go.mod h1:H9Rt/W6/Qj27PGauhQc6nfCDy7vHpzsOThBSaYDoEhw= +github.com/ulikunitz/xz v0.5.17 h1:flR0y/x1hgM8EGV1AW3Xll6T413G0glV8UfBwR617V4= +github.com/ulikunitz/xz v0.5.17/go.mod h1:H9Rt/W6/Qj27PGauhQc6nfCDy7vHpzsOThBSaYDoEhw= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= go.opentelemetry.io/contrib/detectors/gcp v1.44.0 h1:NmLfL734pJhM0JKaYd2Y28+nY9dPRWYAAbxhRCrKXPw= diff --git a/internal/server/browse.go b/internal/server/browse.go index 43cb855..d172ace 100644 --- a/internal/server/browse.go +++ b/internal/server/browse.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "io" + "io/fs" "net/http" "path" "strings" @@ -98,20 +99,14 @@ func detectSingleRootDir(reader archives.Reader) string { return "" } - var root rootDetector - for _, f := range files { - if !root.add(f.Path) { - break - } - } - return root.prefix() + return archives.CommonRoot(files) } // openArchive opens a cached artifact as a random-access archive reader, // auto-detecting and stripping a single top-level directory prefix (like // GitHub zipballs). For npm, the hardcoded "package/" prefix takes precedence. // The whole artifact is buffered, so only the version diff uses it; listing -// and file reads stream through browseArchive instead. +// and file reads use the streaming browser instead. func openArchive(filename string, content io.Reader, ecosystem string) (archives.Reader, error) { //nolint:ireturn // wraps multiple archive implementations data, err := readBrowseInput(content, maxBrowseArchiveSize) if err != nil { @@ -142,7 +137,7 @@ func (s *Server) browseLimitsOrDefault() archives.StreamOptions { // openBrowseArchive prepares a cached artifact for streaming. content is the // already open storage reader; later passes reopen the artifact. -func (s *Server) openBrowseArchive(r *http.Request, artifact *database.Artifact, content io.Reader, ecosystem string) (*browseArchive, error) { +func (s *Server) openBrowseArchive(r *http.Request, artifact *database.Artifact, content io.Reader, ecosystem string) (*archives.Browser, error) { storagePath := artifact.StoragePath.String reopen := func() (io.ReadCloser, error) { return s.storage.Open(r.Context(), storagePath) @@ -398,13 +393,9 @@ func (s *Server) browseFile(w http.ResponseWriter, r *http.Request, ecosystem, n // Find the file and stream it straight from the archive fileReader, err := archive.Extract(filePath) switch { - case errors.Is(err, errBrowseNotFound): + case errors.Is(err, fs.ErrNotExist), errors.Is(err, archives.ErrIsDir): notFound(w, "file not found") return - case errors.Is(err, errBrowseIsDir): - s.logger.Error("failed to extract file", "error", err, "path", filePath) - internalError(w, "failed to extract file") - return case err != nil: s.browseArchiveError(w, err, cachedArtifact.Filename) return diff --git a/internal/server/browse_archive.go b/internal/server/browse_archive.go index 4af6afe..4545ed7 100644 --- a/internal/server/browse_archive.go +++ b/internal/server/browse_archive.go @@ -1,15 +1,11 @@ package server import ( - "bufio" "errors" "fmt" "io" - "path" - "strings" "github.com/git-pkgs/archives" - "github.com/git-pkgs/magic" ) // Limits applied when listing or reading a single file. Bodies are streamed @@ -31,14 +27,7 @@ var defaultBrowseLimits = archives.StreamOptions{ MaxEntries: maxBrowseEntries, } -var ( - errArtifactTooLarge = errors.New("artifact too large for browsing") - errBrowseNotFound = errors.New("file not found") - errBrowseIsDir = errors.New("path is a directory") - - // errStopWalk ends a walk early without reporting an error. - errStopWalk = errors.New("stop walk") -) +var errArtifactTooLarge = errors.New("artifact too large for browsing") // isBrowseLimitError reports whether err comes from one of the browse size or // entry-count limits rather than a malformed archive. @@ -50,44 +39,6 @@ func isBrowseLimitError(err error) bool { errors.Is(err, archives.ErrEntryLimit) } -// browseArchive reads a cached artifact sequentially for the browse endpoints -// without holding expanded file bodies in memory. TAR based formats stream from -// storage, reopening it when a second pass is needed. ZIP and conda need random -// access, so their input is read once and shared by every pass. -type browseArchive struct { - filename string - ecosystem string - limits archives.StreamOptions - reopen func() (io.ReadCloser, error) - - // first is the caller's reader, used for the first pass only. - first io.Reader - // data holds buffered ZIP or conda input. - data []byte -} - -// newBrowseArchive prepares content for streaming. The caller keeps ownership -// of content; reopen supplies a fresh reader for any later pass. -func newBrowseArchive(filename, ecosystem string, content io.Reader, reopen func() (io.ReadCloser, error), limits archives.StreamOptions) (*browseArchive, error) { - buffered := bufio.NewReaderSize(content, browseSniffSize) - sniff, err := buffered.Peek(browseSniffSize) - if err != nil && err != io.EOF { - return nil, fmt.Errorf("reading artifact: %w", err) - } - - a := &browseArchive{filename: filename, ecosystem: ecosystem, limits: limits, reopen: reopen} - if !needsRandomAccess(filename, sniff) { - a.first = buffered - return a, nil - } - - a.data, err = readBrowseInput(buffered, limits.MaxInputBytes) - if err != nil { - return nil, err - } - return a, nil -} - // readBrowseInput reads all of r, failing once it exceeds limit bytes. func readBrowseInput(r io.Reader, limit int64) ([]byte, error) { data, err := io.ReadAll(io.LimitReader(r, limit+1)) @@ -100,306 +51,10 @@ func readBrowseInput(r io.Reader, limit int64) ([]byte, error) { return data, nil } -// needsRandomAccess reports whether the archives library buffers this format -// when streaming it, following the library's own detection order: a known -// extension first, then the content's leading bytes. -func needsRandomAccess(filename string, sniff []byte) bool { - name := strings.ToLower(filename) - for _, suffix := range []string{".tar.gz", ".tar.bz2", ".tar.xz", ".tar.zst"} { - if strings.HasSuffix(name, suffix) { - return false - } +func newBrowseArchive(filename, ecosystem string, content io.Reader, reopen func() (io.ReadCloser, error), limits archives.StreamOptions) (*archives.Browser, error) { + options := archives.BrowseOptions{Limits: limits, DetectRoot: true} + if ecosystem == "npm" { + options.StripPrefix = npmPackagePrefix } - switch path.Ext(name) { - case ".zip", ".jar", ".whl", ".nupkg", ".egg", ".vsix", ".conda": - return true - case ".tar", ".tgz", ".crate", ".gem": - return false - } - return magic.DetectPrefix(sniff).Format == "zip" -} - -// openStream starts a pass over the archive. The returned function releases -// the stream and any reader opened for it. -func (a *browseArchive) openStream() (*archives.Stream, func(), error) { - if a.data != nil { - stream, err := archives.OpenStreamBytes(a.filename, a.data, a.limits) - if err != nil { - return nil, nil, err - } - return stream, func() { _ = stream.Close() }, nil - } - - if a.first != nil { - content := a.first - a.first = nil - stream, err := archives.OpenStream(a.filename, content, a.limits) - if err != nil { - return nil, nil, err - } - return stream, func() { _ = stream.Close() }, nil - } - - content, err := a.reopen() - if err != nil { - return nil, nil, fmt.Errorf("reopening artifact: %w", err) - } - stream, err := archives.OpenStream(a.filename, content, a.limits) - if err != nil { - _ = content.Close() - return nil, nil, err - } - return stream, func() { - _ = stream.Close() - _ = content.Close() - }, nil -} - -// walk calls fn for each entry in archive order until fn returns an error or -// the archive ends. fn may read the entry body from the stream. -func (a *browseArchive) walk(fn func(entry *archives.StreamEntry) error) error { - stream, release, err := a.openStream() - if err != nil { - return err - } - defer release() - - for { - entry, err := stream.Next() - if err == io.EOF { - return nil - } - if err != nil { - return err - } - if err := fn(entry); err != nil { - if errors.Is(err, errStopWalk) { - return nil - } - return err - } - } -} - -// prefix returns the directory stripped from every path: npm's "package/", or -// a single top-level directory shared by all entries (like GitHub zipballs). -// For other ecosystems this costs one pass, which stops early once two -// top-level names differ. -func (a *browseArchive) prefix() (string, error) { - if a.ecosystem == "npm" { - return npmPackagePrefix, nil - } - - var root rootDetector - err := a.walk(func(entry *archives.StreamEntry) error { - if !root.add(entry.Path) { - return errStopWalk - } - return nil - }) - if err != nil { - return "", err - } - return root.prefix(), nil -} - -// ListDir returns the entries directly inside dirPath, after prefix stripping, -// in a single pass that keeps only entry metadata. -func (a *browseArchive) ListDir(dirPath string) ([]archives.FileInfo, error) { - var entries []archives.FileInfo - var root rootDetector - err := a.walk(func(entry *archives.StreamEntry) error { - entries = append(entries, entry.FileInfo) - root.add(entry.Path) - return nil - }) - if err != nil { - return nil, err - } - - prefix := npmPackagePrefix - if a.ecosystem != "npm" { - prefix = root.prefix() - } - return listBrowseDir(entries, prefix, dirPath), nil -} - -// Extract returns the body of the first entry at filePath, streamed from the -// archive. The caller must close the returned reader. -func (a *browseArchive) Extract(filePath string) (io.ReadCloser, error) { - prefix, err := a.prefix() - if err != nil { - return nil, err - } - target := prefix + filePath - - stream, release, err := a.openStream() - if err != nil { - return nil, err - } - for { - entry, err := stream.Next() - if err == io.EOF { - release() - return nil, fmt.Errorf("%w: %s", errBrowseNotFound, filePath) - } - if err != nil { - release() - return nil, err - } - if entry.Path != target { - continue - } - if entry.IsDir { - release() - return nil, fmt.Errorf("%w: %s", errBrowseIsDir, filePath) - } - return &streamedFile{Reader: stream, release: release}, nil - } -} - -// streamedFile is an entry body that releases its archive stream on Close. -type streamedFile struct { - io.Reader - release func() -} - -func (f *streamedFile) Close() error { - if f.release != nil { - f.release() - f.release = nil - } - return nil -} - -// rootDetector tracks whether every path seen so far lives under one -// top-level directory. -type rootDetector struct { - root string - mixed bool - seen bool -} - -// add records p and reports whether a single root is still possible. -func (d *rootDetector) add(p string) bool { - if d.mixed { - return false - } - d.seen = true - dir, _, _ := strings.Cut(p, "/") - if d.root == "" { - d.root = dir - } else if dir != d.root { - d.mixed = true - } - return !d.mixed -} - -// prefix returns the shared root with a trailing slash, or "" if paths have -// more than one root or none were seen. -func (d *rootDetector) prefix() string { - if d.mixed || !d.seen || d.root == "" { - return "" - } - return d.root + "/" -} - -// listBrowseDir lists dirPath from entry metadata the same way a reader from -// archives.OpenWithPrefix does: the prefix is joined to dirPath before -// listing, then stripped from the results. -func listBrowseDir(entries []archives.FileInfo, prefix, dirPath string) []archives.FileInfo { - files := listDir(entries, prefix+dirPath) - if prefix == "" { - return files - } - return stripBrowsePrefix(files, prefix) -} - -// listDir mirrors archives.Reader.ListDir: entries directly in dirPath are -// returned in archive order. Deeper paths add one synthesized entry for each -// immediate subdirectory. -func listDir(entries []archives.FileInfo, dirPath string) []archives.FileInfo { - dirPath = normalizeBrowseDir(dirPath) - var files []archives.FileInfo - seenDirs := make(map[string]bool) - - for _, f := range entries { - p := f.Path - - if isInBrowseDir(p, dirPath) { - if f.IsDir { - name := strings.TrimSuffix(strings.TrimPrefix(p, dirPath), "/") - if seenDirs[name] { - continue - } - seenDirs[name] = true - } - files = append(files, f) - continue - } - - if dirPath != "" && !strings.HasPrefix(p, dirPath) { - continue - } - rel := strings.TrimSuffix(strings.TrimPrefix(p, dirPath), "/") - name, _, nested := strings.Cut(rel, "/") - if nested && !seenDirs[name] { - seenDirs[name] = true - files = append(files, archives.FileInfo{ - Path: dirPath + name + "/", - Name: name, - IsDir: true, - }) - } - } - - return files -} - -func normalizeBrowseDir(dirPath string) string { - dirPath = strings.Trim(strings.TrimSpace(dirPath), "/") - if dirPath == "" { - return "" - } - return dirPath + "/" -} - -// isInBrowseDir reports whether filePath is directly in the normalized -// dirPath. An entry naming the directory itself counts as inside it. -func isInBrowseDir(filePath, dirPath string) bool { - filePath = strings.TrimSuffix(filePath, "/") - if dirPath == "" { - return !strings.Contains(filePath, "/") - } - if filePath == dirPath[:len(dirPath)-1] { - return true - } - rest, ok := strings.CutPrefix(filePath, dirPath) - return ok && !strings.Contains(rest, "/") -} - -// stripBrowsePrefix removes prefix from each path, dropping entries outside -// it and the prefix directory itself. -func stripBrowsePrefix(files []archives.FileInfo, prefix string) []archives.FileInfo { - result := make([]archives.FileInfo, 0, len(files)) - for _, f := range files { - stripped, ok := strings.CutPrefix(f.Path, prefix) - if !ok || stripped == "" || stripped == "/" { - continue - } - f.Path = stripped - f.Name = entryName(stripped) - result = append(result, f) - } - return result -} - -// entryName returns the last element of an archive path, ignoring a trailing -// slash. -func entryName(p string) string { - p = strings.TrimSuffix(p, "/") - if i := strings.LastIndexByte(p, '/'); i >= 0 { - return p[i+1:] - } - return p + return archives.OpenBrowser(filename, content, reopen, options) } diff --git a/internal/server/browse_archive_test.go b/internal/server/browse_archive_test.go index c3c7796..838802e 100644 --- a/internal/server/browse_archive_test.go +++ b/internal/server/browse_archive_test.go @@ -9,6 +9,7 @@ import ( "encoding/json" "fmt" "io" + "log/slog" "net/http" "net/http/httptest" "net/url" @@ -332,6 +333,39 @@ func TestBrowseFileContents(t *testing.T) { } } +func TestBrowseFileDirectoryNotFound(t *testing.T) { + for _, format := range []string{"tar.gz", "zip"} { + for _, explicit := range []bool{false, true} { + t.Run(fmt.Sprintf("%s/explicit=%t", format, explicit), func(t *testing.T) { + ts := newTestServer(t) + defer ts.close() + var logs bytes.Buffer + ts.server.logger = slog.New(slog.NewTextHandler(&logs, &slog.HandlerOptions{Level: slog.LevelError})) + + entries := []archiveEntry{{name: "pkg/src/main.go", body: "package main\n"}} + if explicit { + entries = append([]archiveEntry{{name: "pkg/src/", dir: true}}, entries...) + } + data := buildTarGz(t, entries) + if format == "zip" { + data = buildZip(t, entries) + } + cacheBrowseArtifact(t, ts, "cargo", "demo", "demo."+format, data) + + for _, dir := range []string{"src", "src/"} { + response := browseGet(ts, browseFileURL("cargo", "demo", dir)) + if response.Code != http.StatusNotFound { + t.Errorf("directory %q returned %d, want 404: %s", dir, response.Code, response.Body.String()) + } + } + if logs.Len() != 0 { + t.Errorf("directory request logged an error: %s", logs.String()) + } + }) + } + } +} + func TestBrowseFileFirstDuplicateWins(t *testing.T) { ts := newTestServer(t) defer ts.close() @@ -347,6 +381,39 @@ func TestBrowseFileFirstDuplicateWins(t *testing.T) { } } +func TestBrowseSingleRootFile(t *testing.T) { + for _, format := range []string{"tar.gz", "zip"} { + t.Run(format, func(t *testing.T) { + ts := newTestServer(t) + defer ts.close() + entries := []archiveEntry{{name: "README.md", body: "single file\n"}} + data := buildTarGz(t, entries) + if format == "zip" { + data = buildZip(t, entries) + } + filename := "single." + format + cacheBrowseArtifact(t, ts, "cargo", "single", filename, data) + paths := listingPaths(decodeListing(t, browseGet(ts, browseListURL("cargo", "single", "")))) + if len(paths) != 1 || paths[0] != "README.md" { + t.Fatalf("listing = %v, want README.md", paths) + } + response := browseGet(ts, browseFileURL("cargo", "single", "README.md")) + if response.Code != http.StatusOK || response.Body.String() != "single file\n" { + t.Fatalf("file = %d %q", response.Code, response.Body.String()) + } + r, err := openArchive(filename, bytes.NewReader(data), "cargo") + if err != nil { + t.Fatal(err) + } + defer func() { _ = r.Close() }() + files, err := r.List() + if err != nil || len(files) != 1 || files[0].Path != "README.md" { + t.Fatalf("diff reader listing = %+v, %v", files, err) + } + }) + } +} + // TestBrowseFileEscapedPath requests files the way the source browser does, // with each path segment percent-encoded, so names holding URL syntax resolve. func TestBrowseFileEscapedPath(t *testing.T) { From 5b5a848c654010f45ea2d4cf8fa8b36b0763b1d2 Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Tue, 20 Oct 2026 12:29:22 +0100 Subject: [PATCH 328/633] Update Go dependencies (#534) --- go.mod | 94 ++++++++++----------- go.sum | 260 ++++++++++++++++++++++++++++----------------------------- 2 files changed, 176 insertions(+), 178 deletions(-) diff --git a/go.mod b/go.mod index 7878bee..3c1d4e2 100644 --- a/go.mod +++ b/go.mod @@ -5,31 +5,31 @@ go 1.26.7 require ( github.com/BurntSushi/toml v1.6.0 github.com/Masterminds/semver/v3 v3.5.0 - github.com/aws/aws-sdk-go-v2/config v1.33.7 - github.com/aws/aws-sdk-go-v2/service/ecr v1.66.2 + github.com/aws/aws-sdk-go-v2/config v1.33.9 + github.com/aws/aws-sdk-go-v2/service/ecr v1.66.5 github.com/git-pkgs/archives v0.9.0 github.com/git-pkgs/artifacts v0.2.1 github.com/git-pkgs/cooldown v0.3.0 - github.com/git-pkgs/enrichment v0.7.2 + github.com/git-pkgs/enrichment v0.7.3 github.com/git-pkgs/gcs v0.1.0 github.com/git-pkgs/integrity v0.2.0 github.com/git-pkgs/magic v0.5.0 - github.com/git-pkgs/purl v0.1.21 - github.com/git-pkgs/registries v0.10.0 + github.com/git-pkgs/purl v0.1.22 + github.com/git-pkgs/registries v0.11.0 github.com/git-pkgs/sbom v0.1.7 github.com/git-pkgs/spdx v0.3.3 github.com/git-pkgs/vers v0.7.2 github.com/git-pkgs/vulns v0.2.4 - github.com/go-chi/chi/v5 v5.3.2 + github.com/go-chi/chi/v5 v5.5.0 github.com/jmoiron/sqlx v1.4.0 github.com/lib/pq v1.12.3 github.com/opencontainers/go-digest v1.0.0 github.com/prometheus/client_golang v1.25.0 github.com/prometheus/client_model v0.6.3 github.com/swaggo/swag v1.16.6 - gocloud.dev v0.46.0 - golang.org/x/mod v0.41.0 - golang.org/x/net v0.60.0 + gocloud.dev v0.47.0 + golang.org/x/mod v0.42.0 + golang.org/x/net v0.61.0 golang.org/x/sync v0.24.0 google.golang.org/protobuf v1.36.12 gopkg.in/yaml.v3 v3.0.1 @@ -37,43 +37,42 @@ require ( ) require ( - cloud.google.com/go/auth v0.21.0 // indirect + cloud.google.com/go/auth v0.23.2 // indirect cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect cloud.google.com/go/compute/metadata v0.9.0 // indirect - github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.0 // indirect - github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1 // indirect - github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 // indirect - github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.4 // indirect - github.com/AzureAD/microsoft-authentication-library-for-go v1.7.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.23.1 // indirect + github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.14.1 // indirect + github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.8.0 // indirect + github.com/AzureAD/microsoft-authentication-library-for-go v1.9.0 // indirect github.com/KyleBanks/depth v1.2.1 // indirect github.com/PuerkitoBio/purell v1.1.1 // indirect github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect github.com/apapsch/go-jsonmerge/v2 v2.0.0 // indirect - github.com/aws/aws-sdk-go-v2 v1.47.1 // indirect - github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.11 // indirect - github.com/aws/aws-sdk-go-v2/credentials v1.20.7 // indirect - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.1 // indirect - github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.2.3 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.4 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.4 // indirect - github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.4 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.19 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.18 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.4 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.25 // indirect - github.com/aws/aws-sdk-go-v2/service/s3 v1.102.2 // indirect - github.com/aws/aws-sdk-go-v2/service/signin v1.10.2 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.38.2 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.2 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.51.2 // indirect - github.com/aws/smithy-go v1.28.1 // indirect + github.com/aws/aws-sdk-go-v2 v1.47.3 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.20 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.20.9 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.3 // indirect + github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.4.3 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.6 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.6 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.6 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.21 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.2 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.6 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.2 // indirect + github.com/aws/aws-sdk-go-v2/service/s3 v1.111.0 // indirect + github.com/aws/aws-sdk-go-v2/service/signin v1.10.4 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.38.4 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.4 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.51.4 // indirect + github.com/aws/smithy-go v1.28.5 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cenk/backoff v2.2.1+incompatible // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/ecosyste-ms/ecosystems-go v0.5.0 // indirect github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a // indirect - github.com/felixge/httpsnoop v1.1.0 // indirect github.com/git-pkgs/pom v0.1.8 // indirect github.com/github/go-spdx/v2 v2.7.0 // indirect github.com/go-logr/logr v1.4.4 // indirect @@ -86,8 +85,8 @@ require ( github.com/google/s2a-go v0.1.9 // indirect github.com/google/uuid v1.6.0 // indirect github.com/google/wire v0.7.0 // indirect - github.com/googleapis/enterprise-certificate-proxy v0.3.18 // indirect - github.com/googleapis/gax-go/v2 v2.23.0 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.21 // indirect + github.com/googleapis/gax-go/v2 v2.24.1 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/klauspost/compress v1.20.1 // indirect github.com/kylelemons/godebug v1.1.0 // indirect @@ -108,21 +107,20 @@ require ( github.com/rubyist/circuitbreaker v2.2.1+incompatible // indirect github.com/ulikunitz/xz v0.5.17 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect - go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0 // indirect - go.opentelemetry.io/otel v1.45.0 // indirect - go.opentelemetry.io/otel/metric v1.45.0 // indirect - go.opentelemetry.io/otel/sdk v1.45.0 // indirect - go.opentelemetry.io/otel/sdk/metric v1.45.0 // indirect - go.opentelemetry.io/otel/trace v1.45.0 // indirect + go.opentelemetry.io/otel v1.46.0 // indirect + go.opentelemetry.io/otel/metric v1.46.0 // indirect + go.opentelemetry.io/otel/sdk v1.46.0 // indirect + go.opentelemetry.io/otel/sdk/metric v1.46.0 // indirect + go.opentelemetry.io/otel/trace v1.46.0 // indirect go.yaml.in/yaml/v3 v3.0.5 // indirect - golang.org/x/crypto v0.57.0 // indirect + golang.org/x/crypto v0.58.0 // indirect golang.org/x/oauth2 v0.37.0 // indirect - golang.org/x/sys v0.48.0 // indirect - golang.org/x/text v0.42.0 // indirect - golang.org/x/tools v0.50.0 // indirect + golang.org/x/sys v0.49.0 // indirect + golang.org/x/text v0.43.0 // indirect + golang.org/x/tools v0.51.0 // indirect golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect - google.golang.org/api v0.288.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20260706201446-f0a921348800 // indirect + google.golang.org/api v0.297.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260904194346-d0f1323225a4 // indirect google.golang.org/grpc v1.83.2 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect modernc.org/libc v1.77.1 // indirect diff --git a/go.sum b/go.sum index 36686db..f486b50 100644 --- a/go.sum +++ b/go.sum @@ -1,46 +1,46 @@ -cel.dev/expr v0.25.2 h1:K6j46C81hXtZQfuX60cVWQFBJahKSE2gfRbNuvr5bFs= -cel.dev/expr v0.25.2/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4= +cel.dev/expr v0.25.3 h1:A2jO8jwOugrrovveCWfj0KEZOfqiLgAcwjpHPhzIGw0= +cel.dev/expr v0.25.3/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4= cloud.google.com/go v0.123.0 h1:2NAUJwPR47q+E35uaJeYoNhuNEM9kM8SjgRgdeOJUSE= cloud.google.com/go v0.123.0/go.mod h1:xBoMV08QcqUGuPW65Qfm1o9Y4zKZBpGS+7bImXLTAZU= -cloud.google.com/go/auth v0.21.0 h1:g/QwYfYb2Ai6HH8oomAOyBaIHLbscZ4+T/F/f5JZHkE= -cloud.google.com/go/auth v0.21.0/go.mod h1:M9o2Oz+YI2jAfxewJgb1vyI3vceHF+eohmxyzmrl+9s= +cloud.google.com/go/auth v0.23.2 h1:pxSCpfiji41hpzpPdMCftEUCezpgpqmmDdYiAjCKXxo= +cloud.google.com/go/auth v0.23.2/go.mod h1:4DhBRcqvtljQN3dJ57qtqbib5ZGCYE5f2crfiiC2EM0= cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIiLpZnkHRbnc= cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c= cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdBtwLoEkH9Zs= cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10= -cloud.google.com/go/iam v1.5.3 h1:+vMINPiDF2ognBJ97ABAYYwRgsaqxPbQDlMnbHMjolc= -cloud.google.com/go/iam v1.5.3/go.mod h1:MR3v9oLkZCTlaqljW6Eb2d3HGDGK5/bDv93jhfISFvU= -cloud.google.com/go/monitoring v1.24.3 h1:dde+gMNc0UhPZD1Azu6at2e79bfdztVDS5lvhOdsgaE= -cloud.google.com/go/monitoring v1.24.3/go.mod h1:nYP6W0tm3N9H/bOw8am7t62YTzZY+zUeQ+Bi6+2eonI= -cloud.google.com/go/storage v1.61.3 h1:VS//ZfBuPGDvakfD9xyPW1RGF1Vy3BWUoVZXgW1KMOg= -cloud.google.com/go/storage v1.61.3/go.mod h1:JtqK8BBB7TWv0HVGHubtUdzYYrakOQIsMLffZ2Z/HWk= +cloud.google.com/go/iam v1.13.0 h1:ufT3FPT5rFFXu6UtLkNoxaOaV5EuA1dsSkmemCSTo6U= +cloud.google.com/go/iam v1.13.0/go.mod h1:gHXdDEiPDvqd1q1KwBDGQlgZY/BwY760zU2LhOZS5w0= +cloud.google.com/go/monitoring v1.30.0 h1:r/d+JUbyKmJ8b07iznuKfzVzrIXTWxHQ3lBRm3x2LlY= +cloud.google.com/go/monitoring v1.30.0/go.mod h1:htlUR0QWVMrjFzZmN4LGnMAve9xB/eduwjmINxVZ8RM= +cloud.google.com/go/storage v1.67.0 h1:8xD3NvKuHE4c+b/kmSIVBdTc0BiOjplx+KecKGR9fr8= +cloud.google.com/go/storage v1.67.0/go.mod h1:UsS9OgFg/XHOSYakQ8ZtLWWeyGkk1WnmD/GsGfN0BHM= filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= filippo.io/edwards25519 v1.2.0 h1:crnVqOiS4jqYleHd9vaKZ+HKtHfllngJIiOpNpoJsjo= filippo.io/edwards25519 v1.2.0/go.mod h1:xzAOLCNug/yB62zG1bQ8uziwrIqIuxhctzJT18Q77mc= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.0 h1:fou+2+WFTib47nS+nz/ozhEBnvU96bKHy6LjRsY4E28= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.0/go.mod h1:t76Ruy8AHvUAC8GfMWJMa0ElSbuIcO03NLpynfbgsPA= -github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1 h1:Hk5QBxZQC1jb2Fwj6mpzme37xbCDdNTxU7O9eb5+LB4= -github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1/go.mod h1:IYus9qsFobWIc2YVwe/WPjcnyCkPKtnHAqUYeebc8z0= -github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.2 h1:yz1bePFlP5Vws5+8ez6T3HWXPmwOK7Yvq8QxDBD3SKY= -github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.2/go.mod h1:Pa9ZNPuoNu/GztvBSKk9J1cDJW6vk/n0zLtV4mgd8N8= -github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 h1:9iefClla7iYpfYWdzPCRDozdmndjTm8DXdpCzPajMgA= -github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2/go.mod h1:XtLgD3ZD34DAaVIIAyG3objl5DynM3CQ/vMcbBNJZGI= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.23.1 h1:zvXfGJCWvywnCA814d8ZiVyt+fm9nnTE8xSb99zRyfo= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.23.1/go.mod h1:iptorS+VYKFL2N6PnebpS91dubG35eAOEERnT4PJbQU= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.14.1 h1:u93s+zU2JD62im61Bm5CZIc1ZrOJaIAWEg0WOrMVkEo= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.14.1/go.mod h1:oXtinPO4OLj9d1DOTrqrL1oRwGhcqadvAmrl6wTeGlk= +github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.4.0 h1:xFaZZ+IubdftrDHnGGwZ6QvQ3KHTtWl2MCK+GMt2vxs= +github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.4.0/go.mod h1:mCBhUhlMjLLJKr5aqw2TNS/VqJOie8MzWq3DAMJeKso= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0 h1:fhqpLE3UEXi9lPaBRpQ6XuRW0nU7hgg4zlmZZa+a9q4= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0/go.mod h1:7dCRMLwisfRH3dBupKeNCioWYUZ4SS09Z14H+7i8ZoY= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1 h1:/Zt+cDPnpC3OVDm/JKLOs7M2DKmLRIIp3XIx9pHHiig= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1/go.mod h1:Ng3urmn6dYe8gnbCMoHHVl5APYz2txho3koEkV2o2HA= -github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.4 h1:jWQK1GI+LeGGUKBADtcH2rRqPxYB1Ljwms5gFA2LqrM= -github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.4/go.mod h1:8mwH4klAm9DUgR2EEHyEEAQlRDvLPyg5fQry3y+cDew= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.8.0 h1:irsmOWwkp0KCTTNS5e2hdFeIvSQClQo2No3IaNmL3Vw= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.8.0/go.mod h1:GWcBkQj3MqN7ozHKLaCCAuNLiXoIGv2RtanfAwSjY/Y= github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 h1:WJTmL004Abzc5wDB5VtZG2PJk5ndYDgVacGqfirKxjM= github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mod h1:tCcJZ0uHAmvjsVYzEFivsRTN00oz5BEsRgQHu5JZ9WE= -github.com/AzureAD/microsoft-authentication-library-for-go v1.7.0 h1:4iB+IesclUXdP0ICgAabvq2FYLXrJWKx1fJQ+GxSo3Y= -github.com/AzureAD/microsoft-authentication-library-for-go v1.7.0/go.mod h1:HKpQxkWaGLJ+D/5H8QRpyQXA1eKjxkFlOMwck5+33Jk= +github.com/AzureAD/microsoft-authentication-library-for-go v1.9.0 h1:MDT4FxAPve5FnYn6vOL1r7RCRDG+l9cI7a5LlCuHsqA= +github.com/AzureAD/microsoft-authentication-library-for-go v1.9.0/go.mod h1:Y33QHnf0FfdVewFFISOGe20mkZbxX4H839o955/PoeI= github.com/BurntSushi/toml v1.6.0 h1:dRaEfpa2VI55EwlIW72hMRHdWouJeRF7TPYhI+AUQjk= github.com/BurntSushi/toml v1.6.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.33.0 h1:l7+6kwRMJNwdCvYdDl7Eax+wzEYHSnNY7zrrfbhDdTA= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.33.0/go.mod h1:pJTkW8hEUIIi3Pf65lPZOnn4Y81yCllX6IWk2jNXdkM= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.55.0 h1:UnDZ/zFfG1JhH/DqxIZYU/1CUAlTUScoXD/LcM2Ykk8= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.55.0/go.mod h1:IA1C1U7jO/ENqm/vhi7V9YYpBsp+IMyqNrEN94N7tVc= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.55.0 h1:0s6TxfCu2KHkkZPnBfsQ2y5qia0jl3MMrmBhu3nCOYk= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.55.0/go.mod h1:Mf6O40IAyB9zR/1J8nGDDPirZQQPbYJni8Yisy7NTMc= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.37.0 h1:edOuLWehuDlQ68roBF900Y09jvBoJP5W5d/wusz9nWY= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.37.0/go.mod h1:iIyrdhveU5Ow84ipRlxtjCOQnIM0DiqSMxuxm50xwZY= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.61.0 h1:BeC1Bub7Ttk33TF8v3ZeQ2L8acvz4ZLqtQuLyJAZquc= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.61.0/go.mod h1:UgG/Xn9+NXb1/ueSczIFxyuya3hs3srFDiZG2JrpJwY= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.61.0 h1:KhpL26/GviYITHMCEGGqT/l50siFaBCBBKrjv6Z0G/k= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.61.0/go.mod h1:sg6Wqbl0BKKZ5zKnQj2q0qic/3ziF894SCx1LpbX37Y= github.com/KyleBanks/depth v1.2.1 h1:5h8fQADFrWtarTdtDudMmGsC7GPbOAu6RVB3ffsVFHc= github.com/KyleBanks/depth v1.2.1/go.mod h1:jzSb9d0L43HxTQfT+oSA1EEp2q+ne2uh6XgeJcm8brE= github.com/Masterminds/semver/v3 v3.5.0 h1:kQceYJfbupGfZOKZQg0kou0DgAKhzDg2NZPAwZ/2OOE= @@ -52,46 +52,46 @@ github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdko github.com/RaveNoX/go-jsoncommentstrip v1.0.0/go.mod h1:78ihd09MekBnJnxpICcwzCMzGrKSKYe4AqU6PDYYpjk= github.com/apapsch/go-jsonmerge/v2 v2.0.0 h1:axGnT1gRIfimI7gJifB699GoE/oq+F2MU7Dml6nw9rQ= github.com/apapsch/go-jsonmerge/v2 v2.0.0/go.mod h1:lvDnEdqiQrp0O42VQGgmlKpxL1AP2+08jFMw88y4klk= -github.com/aws/aws-sdk-go-v2 v1.47.1 h1:uOIZnp4PK3ZhKI0dNrJrhTEsLxbpXHTAJlwoS1pvAtw= -github.com/aws/aws-sdk-go-v2 v1.47.1/go.mod h1:bttEH6JqnUL8LepvDVfdrds/fZ5bCIxzpe3abyUrhDU= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.11 h1:h5+3VT69KUBK24grGuuA5saDJTj2IIjLb9au668Fo5I= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.11/go.mod h1:dnakxebH6UwFvcvujL0LVggYQ8nEvBGjU4G/V79Nv94= -github.com/aws/aws-sdk-go-v2/config v1.33.7 h1:vgG8uuhm7vNsxr6BsXNG+Ciesh9iLdNPzarAAu/dB6A= -github.com/aws/aws-sdk-go-v2/config v1.33.7/go.mod h1:5KWNY38altxnd5cxcNoOTrFtJ7asFEiCiWfb2m+M2pE= -github.com/aws/aws-sdk-go-v2/credentials v1.20.7 h1:mit8rRVnmt+wvT4U6putiWSlwe3eKD2n6ElgyfcP/Mw= -github.com/aws/aws-sdk-go-v2/credentials v1.20.7/go.mod h1:/2pPi2qhPug21z6l3M0+AKUqAT1zn2MhRCtnQv8R5Zg= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.1 h1:8gALAAmacnIXh+z6VkdDanv4/IkG5APdg4DZLDTmLog= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.1/go.mod h1:Z7IJhJU+poOdJjUR2wpyY21ossQ1XS/R3Lk9Msq5kM4= -github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.2.3 h1:w5OoDiMN6x53ROmiIImGzmVcxXv2q1GXY+aKV4WAJYM= -github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.2.3/go.mod h1:dAhgYp776bX3LuWvnSCFwQEjNs6fuFg7YXIy5PXcP3Q= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.4 h1:CLq4+8UHCI+ZZYl/EuJxXovaIVN2xeeT8JV+dsApQ5E= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.4/go.mod h1:Wv4q5sAM04xAMkoOedxLx2inVf6K5FdxYp+A61L+q/0= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.4 h1:dD4MR81I7YkpEBRk6UP9rocC2QnT3qVuXwzlYTtfGEs= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.4/go.mod h1:EcXV1kAFd5XwSkDHlj94gnF3q5CkJyYiIJfH8N0VmrE= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.4 h1:7Wo47d/xn/7KttCSBd8EGYeZ7ULRFRkUHr6vkZPBzVQ= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.4/go.mod h1:tDB2IVC1xC3vX8o+6uRlzhTxP3g1b77CZXFX/oD2FnQ= -github.com/aws/aws-sdk-go-v2/service/ecr v1.66.2 h1:SNECRj8ZN0qPG1L33dLnqQ4EfmIQkGcFW7cv4Obw28M= -github.com/aws/aws-sdk-go-v2/service/ecr v1.66.2/go.mod h1:WglfLchOYcHrYOwNV7jERuy0Xc+7jArLkEnQay93auY= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.19 h1:bAdDl/HkGCcGPoe25ToSHEw23VIxt6CT5fLcg111BKg= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.19/go.mod h1:KaUzbLxv4CeSxh6ZCl9B4m7CuFenS8kUEaDs+f/DQr4= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.18 h1:W/EyPFl9A5rXrtoilfwHYEvzHER+K4SpBPtMXi24Mos= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.18/go.mod h1:UG50K+pvd/uy6xExbobg0rjqFBFZe6I3l75EPDZw4tg= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.4 h1:29SvnfGhXjTl8ONxFwbj2rs6lbhiFXD2CgFQmbT/bXY= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.4/go.mod h1:wm04I5DMuNVvZHFe/dHnUxincvNbbK7AiNBbYsQivek= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.25 h1:2pQEbwf+/6EDbiit/GcBE2K4IUpMZymaA0kOz3xK978= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.25/go.mod h1:KvT6NCcQ0EZ+ZkVRrlBMt04Po3ok23YELEp7WimhLhM= -github.com/aws/aws-sdk-go-v2/service/s3 v1.102.2 h1:ie4ElCmUKS26pzrZcIk/lmt4yWjAqLLcawstyQCh298= -github.com/aws/aws-sdk-go-v2/service/s3 v1.102.2/go.mod h1:zjsomFeX5duj+4PlMB+o4JoWTIx+G0XMyzjYrUbQkN0= -github.com/aws/aws-sdk-go-v2/service/signin v1.10.2 h1:vrPP4O9VABnquYrDHfU+Fu0Fp6TuzaqPUu9Fh7VBBVY= -github.com/aws/aws-sdk-go-v2/service/signin v1.10.2/go.mod h1:xpo/geVldu8payT375WekctUzopG/hBU7miiqItMUlw= -github.com/aws/aws-sdk-go-v2/service/sso v1.38.2 h1:I8F/K5ahKJ4npqeGBBmMPftQy3/o1A55VRq2eUcP9Ts= -github.com/aws/aws-sdk-go-v2/service/sso v1.38.2/go.mod h1:rRD/dnm7q0HYE/I5TMaPgkWyyUGLcwuxHLABsLnQ3e0= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.2 h1:i5nn7Ri5CgFfNRzwlUiZOIIUA+5Vgg+unNEC0PzF1+s= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.2/go.mod h1:skwM/xsbR/1ReUTesv9BhpJp1VjajR7DWQnuVLwiXsQ= -github.com/aws/aws-sdk-go-v2/service/sts v1.51.2 h1:AWSPqynwEJV8J2pl/p7aizCf7a23/NGHR+rK/KLOxi4= -github.com/aws/aws-sdk-go-v2/service/sts v1.51.2/go.mod h1:26zA0GhDrLo+yiLI2yXWxqB1PdsShfLikoI7GOEgugM= -github.com/aws/smithy-go v1.28.1 h1:R/nXH00c8qcfCzQVELtRw+eLQWtzv+VAIEFJ1/xxXlQ= -github.com/aws/smithy-go v1.28.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= +github.com/aws/aws-sdk-go-v2 v1.47.3 h1:HRoT/MkNi3bjO5xF4dCzLzvVmkvRGHj3/cLINDnJejk= +github.com/aws/aws-sdk-go-v2 v1.47.3/go.mod h1:xSl+OSI5ZUw6bSMupWc218zE5Pl+65MCCu3ySWFY99A= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.20 h1:GPRlPwz40I2B2VrBEASOA3Bi77NyeqejNLkifosX0rs= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.20/go.mod h1:g7PNzKcsOKWb4fkSRBA7BZVAS6Y8IcxzN+nRohhQ1Q8= +github.com/aws/aws-sdk-go-v2/config v1.33.9 h1:Hv6bcQUJVnmuWaKjqD6Ot8sE75NrWsWV20oGT5aWle0= +github.com/aws/aws-sdk-go-v2/config v1.33.9/go.mod h1:ivowmrZNfBQLcg4QkaItgi5wae4Bzot9RzIdbY1hfTg= +github.com/aws/aws-sdk-go-v2/credentials v1.20.9 h1:SNO55PnWhaDafsOcv16lCA0kdoFMl/YlegWoVsU02s0= +github.com/aws/aws-sdk-go-v2/credentials v1.20.9/go.mod h1:+2reOTRtdD3VSBqpRDNwrZIYy8yMCYa7oK3WKTJ/I1I= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.3 h1:gr3b87OLB29jymWPzp/CAyXwQRJVE+sC4Wm0qBIKHsk= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.3/go.mod h1:BRjf4L/Yx2hY54o+ZibPbLJyMMeSwqYQXwcJfqW66a8= +github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.4.3 h1:vsFkmFrfzODkxVbyuPLscv5j5lC8KbG48LZwV+p56Y0= +github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.4.3/go.mod h1:pjQYVtvVBEtlsf2p6nXMsjPuDaWs9GM44FMdHbmnmYU= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.6 h1:CqL8NN3p4I0JTwwV5MEoatIl4Y2amZ2X9dRD+0hbRL0= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.6/go.mod h1:exdJ32hMqXl0Wyo6NwqEC5yIhtua3+PDaKE8P29m4qE= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.6 h1:FpBq24TG6WVSObTNhQOv7Q0088rounUUlwn3zlNItkM= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.6/go.mod h1:ONzdPVYC95oBOu0VWNEsKEntcoemP3xVKjT+qWml1s4= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.6 h1:Qmn60RWuk9+bd2ddvJJqwf3Cpac3VvGMOdbGemtUhX4= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.6/go.mod h1:h/YJhe4AVyQxEUMluph10iqV7mjajH7A9L+BHF5ppis= +github.com/aws/aws-sdk-go-v2/service/ecr v1.66.5 h1:163OpmhcdzTzgbXbLV+u2v1LY/QmGT148/eOwkczKCY= +github.com/aws/aws-sdk-go-v2/service/ecr v1.66.5/go.mod h1:uDza1Eipo/Olii3CCgLglHmAni24PScSAhpfIKbUl0c= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.21 h1:e6YW5uME30TkrElR+4/xcY7NB6CCj2dcv7vtmGXhfhQ= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.21/go.mod h1:obpHAMP693+f3/A8PkLhIqfK1C+NWWn8zXzqD3OH520= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.2 h1:CO485zrLOZBeq8mrdsudc8cjyhvC4LY4JFt8ZfHvXZ0= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.2/go.mod h1:cBAjVUKBr+5wnx1K3XWS8u5AZsqpA2S/0z7YtI6BGFc= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.6 h1:LHHdobjo5le1sjJdolitV7uVvYg3wkaeM1Ju2fDgeZQ= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.6/go.mod h1:CBGB0evAiaMD3Lk4wPO7LQCcCZLuhzL5Qf/wfEUHQ5w= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.2 h1:Enx5JcIy/o90kio1LFZ/HguxIsos3clbGuSpGdyshX8= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.2/go.mod h1:Ldmll8/2VqaVerWWDf8IGG+6pAvZY04gs3tevq3nIFs= +github.com/aws/aws-sdk-go-v2/service/s3 v1.111.0 h1:4PokON/ISKuD6Zl062n4WfQNnEFBMHvfeJy73Jp7K+g= +github.com/aws/aws-sdk-go-v2/service/s3 v1.111.0/go.mod h1:6ZFrmv/qIAYFmG+2WhA7mFU8+edldRlISwT+tNnyLuA= +github.com/aws/aws-sdk-go-v2/service/signin v1.10.4 h1:tvpDjuUvgKURf3Yk4hOaGbZ4QdY+070OWcA5d6LUv2A= +github.com/aws/aws-sdk-go-v2/service/signin v1.10.4/go.mod h1:3ibVKg5Ng82Zby8++jELTD2nHxiPk2DJRLEeW/LKp8M= +github.com/aws/aws-sdk-go-v2/service/sso v1.38.4 h1:ChCtagixSFOxltv0o6E3NT8F7ksvZtZGPwUBynO5Nmk= +github.com/aws/aws-sdk-go-v2/service/sso v1.38.4/go.mod h1:+FEO9jrVMpc9oN3CdAxJbtlQEu3z8UddDPSedpeM3PY= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.4 h1:j6c+HA4AoS0EBEjArPjavFEBrucJEryfrQU0TffmACk= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.4/go.mod h1:RhghaXUk9q66VtOQUX1FdPddQuHnqQZzbB0fEYXmPRA= +github.com/aws/aws-sdk-go-v2/service/sts v1.51.4 h1:OKEfbf2tcfsF2pBk/S9QqNpo6alaAtiBs4hysCsBjPw= +github.com/aws/aws-sdk-go-v2/service/sts v1.51.4/go.mod h1:mCd6qGo9b8hTtfjgsmNkFlzwjyce4owd7HDVFnCLXhM= +github.com/aws/smithy-go v1.28.5 h1:OqbqKtzY9NUnr2yAe9+hiTdFb20tNaK/lI0PMOGXX9k= +github.com/aws/smithy-go v1.28.5/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bmatcuk/doublestar v1.1.1/go.mod h1:UD6OnuiIn0yFxxA2le/rnRU1G4RaI4UvFv1sNto9p6w= @@ -109,8 +109,8 @@ github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+m github.com/ecosyste-ms/ecosystems-go v0.5.0 h1:qZqgar3Do9RrtW327OXlsrGKiRvNdeX8CmXb1aX4Njo= github.com/ecosyste-ms/ecosystems-go v0.5.0/go.mod h1:IhsYBZfxkofxya0O1O2DlhDt281utWJ6VQO0mPVb99I= github.com/envoyproxy/go-control-plane v0.14.0 h1:hbG2kr4RuFj222B6+7T83thSPqLjwBIfQawTkC++2HA= -github.com/envoyproxy/go-control-plane/envoy v1.37.0 h1:u3riX6BoYRfF4Dr7dwSOroNfdSbEPe9Yyl09/B6wBrQ= -github.com/envoyproxy/go-control-plane/envoy v1.37.0/go.mod h1:DReE9MMrmecPy+YvQOAOHNYMALuowAnbjjEMkkWOi6A= +github.com/envoyproxy/go-control-plane/envoy v1.39.0 h1:1uwRDYPYG8BIBU9Mj1sUAebNmlM6beu/ZKKweSLDxk8= +github.com/envoyproxy/go-control-plane/envoy v1.39.0/go.mod h1:5e4ylfTZO723MEEFsCpSW4ZEBWR8mwkEyXfwJBTCZ9c= github.com/envoyproxy/protoc-gen-validate v1.3.3 h1:MVQghNeW+LZcmXe7SY1V36Z+WFMDjpqGAGacLe2T0ds= github.com/envoyproxy/protoc-gen-validate v1.3.3/go.mod h1:TsndJ/ngyIdQRhMcVVGDDHINPLWB7C82oDArY51KfB0= github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a h1:yDWHCSQ40h88yih2JAcL6Ls/kVkSE8GFACTGVnMPruw= @@ -123,8 +123,8 @@ github.com/git-pkgs/artifacts v0.2.1 h1:VwdxR4yTDaqBZ34h0slxQBVyr2Xfa+QGOKCKviMx github.com/git-pkgs/artifacts v0.2.1/go.mod h1:Otosgq52pXT5UNN7lh6s/lszpWKVDO8XrOjN1M70/IA= github.com/git-pkgs/cooldown v0.3.0 h1:eAl2CQFwqXxluoVP9hHmropAf1VoV3gSxLMkN+NEu8A= github.com/git-pkgs/cooldown v0.3.0/go.mod h1:FnTg8wufEALATC9Sm/9AHjLdA+u9eVn96e9BqXXjGB0= -github.com/git-pkgs/enrichment v0.7.2 h1:flD/M89HxkOegUi6tC8EOspFiIMsl7m0o576IQKOp48= -github.com/git-pkgs/enrichment v0.7.2/go.mod h1:xGyc/iz7u1UnbrOHk8lIX5ZaNF3mKFDEZc3VqBJ/LOk= +github.com/git-pkgs/enrichment v0.7.3 h1:l8bGVUP2Ncg3uyoZz7Gve+pyojUVcIQ3oJOFbYhWSOM= +github.com/git-pkgs/enrichment v0.7.3/go.mod h1:ZN/cH4PnmzsapEVPvmAwREP7+ox+RX0yfR6KvoB8xKE= github.com/git-pkgs/gcs v0.1.0 h1:E3awGtsO0xZyHT9FUfEwMHjMkRxw41Bh+c7lgTmPvBo= github.com/git-pkgs/gcs v0.1.0/go.mod h1:bdkCFD66ryaWnU8MBhokVA3WkJfyAEchm9qec5woBpE= github.com/git-pkgs/integrity v0.2.0 h1:sxpAk7EUllZnTYttOoPTW/Yfe5J2imglpuJiuBUZOxM= @@ -133,10 +133,10 @@ github.com/git-pkgs/magic v0.5.0 h1:Pm+4fkHR+K6kg9pwJM4hnwWUuBlkYdLmF1yIAI8lU9Q= github.com/git-pkgs/magic v0.5.0/go.mod h1:SXOqcsNmbmpZjJZHEEWnwxprbsFvpwWgTAZrgXp4Jm4= github.com/git-pkgs/pom v0.1.8 h1:mXCuWw6XkrKKZb6IurRZyIznYLPhV5AYg5ZO38Uj/l8= github.com/git-pkgs/pom v0.1.8/go.mod h1:sQBeYRzjCUn77fcABPn6I/a1aBHFxm55jTmkNxYtGtw= -github.com/git-pkgs/purl v0.1.21 h1:PDZUX43lxIiXPn2eHcsNiiiwQo+HajntZ/4TwSvCGlw= -github.com/git-pkgs/purl v0.1.21/go.mod h1:4dOmris48ehjiKmWBqSmMHguUQht48BLeXW+hQUnuj0= -github.com/git-pkgs/registries v0.10.0 h1:0goaC5mxw9MglS/XOm2De7fkPBWCCerelm7RPrPh9hQ= -github.com/git-pkgs/registries v0.10.0/go.mod h1:bKxgdy/PwnzQXL2FnOktmSQHTKk6B85MA4VtvvYItPc= +github.com/git-pkgs/purl v0.1.22 h1:qT9FNXT/3/RbWnQCxMHn1CFeuruMsq0RBH154oLt1Xs= +github.com/git-pkgs/purl v0.1.22/go.mod h1:l8rCfvt5iXVrFIcoQ8+vCLOaHhrtEW2id0LENP+IzMw= +github.com/git-pkgs/registries v0.11.0 h1:kkB80NjO9UncyyxGfKmdPXWqUMNE/CkNw5Z/6gqW7Rs= +github.com/git-pkgs/registries v0.11.0/go.mod h1:bKxgdy/PwnzQXL2FnOktmSQHTKk6B85MA4VtvvYItPc= github.com/git-pkgs/sbom v0.1.7 h1:4eboy/Yed/SC9ZcC/xT50iywHPzWbmCU7azDb5T1RbA= github.com/git-pkgs/sbom v0.1.7/go.mod h1:KIkZX0v31l62SeFsi2qsQXcZRVrAXvIdPQ4x4en+G2M= github.com/git-pkgs/spdx v0.3.3 h1:xXlUoZzjDSkwxC6mWHfaW+kc8a0+LH6d+gGHuZWftqI= @@ -147,10 +147,10 @@ github.com/git-pkgs/vulns v0.2.4 h1:QnsQg9sWUfM8ZzA7A/1LSl8DJ/0/6Yrm7vfRwkpyXUY= github.com/git-pkgs/vulns v0.2.4/go.mod h1:csd+UWvmYWVVAkka7FNbyz840/1R3QXS7LwD2Irsf4Q= github.com/github/go-spdx/v2 v2.7.0 h1:GzfXx4wFdlilARxmFRXW/mgUy3A4vSqZocCMFV6XFdQ= github.com/github/go-spdx/v2 v2.7.0/go.mod h1:Ftc45YYG1WzpzwEPKRVm9Jv8vDqOrN4gWoCkK+bHer0= -github.com/go-chi/chi/v5 v5.3.2 h1:5YQkICvTCSZ25hoRsyJazN0scjzKGiu4VAUc7H1o1nY= -github.com/go-chi/chi/v5 v5.3.2/go.mod h1:R+tYY2hNuVUUjxoPtqUdgBqevM9s9njzkTLutVsOCto= -github.com/go-jose/go-jose/v4 v4.1.4 h1:moDMcTHmvE6Groj34emNPLs/qtYXRVcd6S7NHbHz3kA= -github.com/go-jose/go-jose/v4 v4.1.4/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08= +github.com/go-chi/chi/v5 v5.5.0 h1:xdKQ1dJmVJsmgahrXSKhUx8NyVPzpwdlyrrQFpMPJgg= +github.com/go-chi/chi/v5 v5.5.0/go.mod h1:d1IG7GtKPSjPyBP365PH8sAGVHk3qfB/IhjFNRH1wWM= +github.com/go-jose/go-jose/v4 v4.1.5 h1:RjgjO2LOtWOJKUC5wpwY9LR3B3vwVAz6JS2YHfYU6eA= +github.com/go-jose/go-jose/v4 v4.1.5/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.4 h1:tG4xh9yMsRCAiodLVTxyrkzSZ9+o0L1Kg/+cPVcbP/8= github.com/go-logr/logr v1.4.4/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= @@ -167,16 +167,16 @@ github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh github.com/go-openapi/swag v0.19.15 h1:D2NRCBzS9/pEY3gP9Nl8aDqGUcPFrwG2p+CNFrLyrCM= github.com/go-openapi/swag v0.19.15/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg= -github.com/go-sql-driver/mysql v1.9.3 h1:U/N249h2WzJ3Ukj8SowVFjdtZKfu9vlLZxjPXV1aweo= -github.com/go-sql-driver/mysql v1.9.3/go.mod h1:qn46aNg1333BRMNU69Lq93t8du/dwxI64Gl8i5p1WMU= +github.com/go-sql-driver/mysql v1.10.1 h1:arlSnNLq6a5yxGxV7qg9lF4j0C+KwD6NbQyKr9QL6ME= +github.com/go-sql-driver/mysql v1.10.1/go.mod h1:M+cqaI7+xxXGG9swrdeUIoPG3Y3KCkF0pZej+SK+nWk= github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY= github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= -github.com/google/go-replayers/grpcreplay v1.3.0 h1:1Keyy0m1sIpqstQmgz307zhiJ1pV4uIlFds5weTmxbo= -github.com/google/go-replayers/grpcreplay v1.3.0/go.mod h1:v6NgKtkijC0d3e3RW8il6Sy5sqRVUwoQa4mHOGEy8DI= +github.com/google/go-replayers/grpcreplay v1.5.1 h1:R90FMYaA6GQNhi/GG0D4zS14HEQ+wpO8bQAV0Qwyt08= +github.com/google/go-replayers/grpcreplay v1.5.1/go.mod h1:42YCe31fkUrhRo+haVAfLse+gNqGI1iO1wdNR+sWY9U= github.com/google/go-replayers/httpreplay v1.2.0 h1:VM1wEyyjaoU53BwrOnaf9VhAyQQEEioJvFYxYcLRKzk= github.com/google/go-replayers/httpreplay v1.2.0/go.mod h1:WahEFFZZ7a1P4VM1qEeHy+tME4bwyqPcwWbNlUI1Mcg= github.com/google/martian/v3 v3.3.3 h1:DIhPTQrbPkgs2yJYdXU/eNACCG5DVQjySNRNlflZ9Fc= @@ -189,10 +189,10 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/wire v0.7.0 h1:JxUKI6+CVBgCO2WToKy/nQk0sS+amI9z9EjVmdaocj4= github.com/google/wire v0.7.0/go.mod h1:n6YbUQD9cPKTnHXEBN2DXlOp/mVADhVErcMFb0v3J18= -github.com/googleapis/enterprise-certificate-proxy v0.3.18 h1:hvVi34VucdrV1IIsiWuqYM8kutw/92MxNEFxCJZEh0k= -github.com/googleapis/enterprise-certificate-proxy v0.3.18/go.mod h1:rSEsBUemEBZEexP2y6jPp16LUmUbjmSbcPMQizR0o4k= -github.com/googleapis/gax-go/v2 v2.23.0 h1:Tchl7qkvE7Ip3y+ztvNufYFvkfqTe7NfLTYGIdJRLuE= -github.com/googleapis/gax-go/v2 v2.23.0/go.mod h1:rBQKOVJCdb8IFEzg+FCwlt1LP/xMDGuqUXhUG+XMXEg= +github.com/googleapis/enterprise-certificate-proxy v0.3.21 h1:OFdQ3tnCX/zaQ0Cedur3D3z7kI6HiLX9g3TiAN4/DFU= +github.com/googleapis/enterprise-certificate-proxy v0.3.21/go.mod h1:L3D/IQExI6LqEjBdXcZQ1WluSgigQmSwBboFstVPM4w= +github.com/googleapis/gax-go/v2 v2.24.1 h1:AtqTN21IXMMWo99LiEVAiBfNNQmO40d8xUfZI640mc0= +github.com/googleapis/gax-go/v2 v2.24.1/go.mod h1:bWeBei0NVwaNZKb2y1HUBS7gLXIF3/Tu3pq7j8D2Tb0= github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/jmoiron/sqlx v1.4.0 h1:1PLqN7S1UYp5t4SrVVnt4nUVNemrDAtxlulVe+Qgm3o= @@ -263,8 +263,8 @@ github.com/rs/dnscache v0.0.0-20230804202142-fc85eb664529 h1:18kd+8ZUlt/ARXhljq+ github.com/rs/dnscache v0.0.0-20230804202142-fc85eb664529/go.mod h1:qe5TWALJ8/a1Lqznoc5BDHpYX/8HU60Hm2AwRmqzxqA= github.com/rubyist/circuitbreaker v2.2.1+incompatible h1:KUKd/pV8Geg77+8LNDwdow6rVCAYOp8+kHUyFvL6Mhk= github.com/rubyist/circuitbreaker v2.2.1+incompatible/go.mod h1:Ycs3JgJADPuzJDwffe12k6BZT8hxVi6lFK+gWYJLN4A= -github.com/spiffe/go-spiffe/v2 v2.7.0 h1:uXe1MflJoHw58wAUvxVlcM7WpKtijWG7I1UidcGh6g4= -github.com/spiffe/go-spiffe/v2 v2.7.0/go.mod h1:47Q0Q9/AqGha8QLHp+kxpH4Wca7X7EnOtlIJy3mxZ3U= +github.com/spiffe/go-spiffe/v2 v2.8.1 h1:eXZMLsu+3MLEPJyGJkolqtVrteZfQdUpOWj6LTiDl/E= +github.com/spiffe/go-spiffe/v2 v2.8.1/go.mod h1:47Q0Q9/AqGha8QLHp+kxpH4Wca7X7EnOtlIJy3mxZ3U= github.com/spkg/bom v0.0.0-20160624110644-59b7046e48ad/go.mod h1:qLr4V1qq6nMqFKkMo8ZTx3f+BZEkzsRUY10Xsm2mwU0= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= @@ -277,39 +277,39 @@ github.com/ulikunitz/xz v0.5.17 h1:flR0y/x1hgM8EGV1AW3Xll6T413G0glV8UfBwR617V4= github.com/ulikunitz/xz v0.5.17/go.mod h1:H9Rt/W6/Qj27PGauhQc6nfCDy7vHpzsOThBSaYDoEhw= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= -go.opentelemetry.io/contrib/detectors/gcp v1.44.0 h1:NmLfL734pJhM0JKaYd2Y28+nY9dPRWYAAbxhRCrKXPw= -go.opentelemetry.io/contrib/detectors/gcp v1.44.0/go.mod h1:tNAsgd8avTGke1+MndXlU5Cru4PQ9Ai/cCNWQv/ZJ/s= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0 h1:yI1/OhfEPy7J9eoa6Sj051C7n5dvpj0QX8g4sRchg04= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0/go.mod h1:NoUCKYWK+3ecatC4HjkRktREheMeEtrXoQxrqYFeHSc= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0 h1:8tvICD4vSTOOsNrsI4Ljf6C+6UKvpTEH5XY3JMoyPoo= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0/go.mod h1:z9+yiacE0IHRqM4qFfkbt/JYlmYXgss8GY/jXoNuPJI= -go.opentelemetry.io/otel v1.45.0 h1:pdrWmLHofpubmArBv1LgFSv1Z0Ie/ppdZzu+kUN5EeU= -go.opentelemetry.io/otel v1.45.0/go.mod h1:XZxIqPapzEYnhNSScF5DIqXhm/rYi0FzCe2XddAwZfQ= -go.opentelemetry.io/otel/metric v1.45.0 h1:7Eg1uH7CJ5cXv9is6tnBe1FI6rj1nwUdbFypRm3br/M= -go.opentelemetry.io/otel/metric v1.45.0/go.mod h1:HAPbm1nd3p1PmFH7v2dR+6BjXxw+Lq4a2+pndMAm08s= -go.opentelemetry.io/otel/metric/x v0.67.0 h1:PcicCNZFkZ4bXfSooXdo3WN7RBOVOtjVdo1wD358Uns= -go.opentelemetry.io/otel/metric/x v0.67.0/go.mod h1:FBjCWZe6wgcqxcMtjdGiClDKXb2YxxXii0CXftE4QtI= -go.opentelemetry.io/otel/sdk v1.45.0 h1:4VVSMgQ83dUgW2aoX5f6JgLvHwIvzcuLnF9lUdCSpCw= -go.opentelemetry.io/otel/sdk v1.45.0/go.mod h1:Sr40LgXV7DsKMMJMKOhUWOgMWTfAaqvm2kF0g7ilwuA= -go.opentelemetry.io/otel/sdk/metric v1.45.0 h1:oVFszMfyj1Am6s24Vtc7wBb8BKLcwepJjNEYILuiE3o= -go.opentelemetry.io/otel/sdk/metric v1.45.0/go.mod h1:vUWUxDZvu1WVRj8JA8S0AdhsPrZoDpA2DdZauIh4mDA= -go.opentelemetry.io/otel/trace v1.45.0 h1:l/mP6Uv7oNO7/TblbhpbgMidxhq1uO/rPsikOyVhxag= -go.opentelemetry.io/otel/trace v1.45.0/go.mod h1:qoJJA2xNMnxRrdISU/kLtfUH2wNeQbiv+jhs/CxI8bc= +go.opentelemetry.io/contrib/detectors/gcp v1.46.0 h1:PI8dGkqDaQkwJ8kOopqMhDTbrnK3UIeG/RCHH4HErbo= +go.opentelemetry.io/contrib/detectors/gcp v1.46.0/go.mod h1:nsrN5c/sOLoY2vsPxN/rQ0V0nvGrWJCqcW4UXLtqNG8= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.71.0 h1:B2h3uqicet1CT2N5TOFhS+Gq++9i0/CLmaxvhmhtP5s= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.71.0/go.mod h1:dylvB+ZiiwMvsDij9O84Uy7SijLgHMX4mbkncds+4Sw= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.71.0 h1:3g7B90UzBltIDKq1/5mrTGxTnOFDV0ICOhLoxiZ8jlg= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.71.0/go.mod h1:Ef8SuTh59BT7+ofpDxN9z+yOlc4t2GjLmKDgYNJL/NU= +go.opentelemetry.io/otel v1.46.0 h1:FHt5/CDyVxi/8IM1CH7VE/rRgq3kLHa2mSTVMO8AWyc= +go.opentelemetry.io/otel v1.46.0/go.mod h1:Gj3SEScelsNC45tp4nSxRYlS+f5iez7W8XPMCt905kE= +go.opentelemetry.io/otel/metric v1.46.0 h1:yBnkXvgV7AXFILZc5K6IZe/CBFF3OS7BJ8ov6/lj0K8= +go.opentelemetry.io/otel/metric v1.46.0/go.mod h1:iPmdWqifKUdzziPkvvzIJXITl56fQx2mGM/DHLB3/2o= +go.opentelemetry.io/otel/metric/x v0.68.0 h1:TA/cBT23D3MnxYPwHL7YFOdYGdx0A0v+s7Mzotpd1dU= +go.opentelemetry.io/otel/metric/x v0.68.0/go.mod h1:agudOmvWhwUTjgibWDzxD2PoWYnpw5Ht5jISYOD2Hd4= +go.opentelemetry.io/otel/sdk v1.46.0 h1:h5CNQQjEbuQXY/JfZtgt3i7HVFV3aHPO2OAwO2eTYPI= +go.opentelemetry.io/otel/sdk v1.46.0/go.mod h1:GAERFXFt5SYCEB+YiKUbMBeza6UaDH7GmGOZEfh2gSM= +go.opentelemetry.io/otel/sdk/metric v1.46.0 h1:0piZ26EG4RBfebb2jhDH6ERCYHoVWduc3kLgPCwSnSE= +go.opentelemetry.io/otel/sdk/metric v1.46.0/go.mod h1:I1PbKrdVc8Qu8HYVDNtqVIwLwjNrhsV/uFuxfwg8mO4= +go.opentelemetry.io/otel/trace v1.46.0 h1:OULy7ccdJnZtJ0UDYFOIGaCmiWzJ8Vi2G/Rsu60qs1c= +go.opentelemetry.io/otel/trace v1.46.0/go.mod h1:J7GAXweO77XSFkB/rmAqk9D6ihszhFjLU+d9WuUxDLI= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ= go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ= go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= -gocloud.dev v0.46.0 h1:niIuZwSjMtBx8K+ITB2s5kZullB13PGOS2ZoQPZxQ4Q= -gocloud.dev v0.46.0/go.mod h1:ACQe+2qO+hEO+pdcvvsM+RB63r8TyGD1W3ESCLFyzvM= -golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M= -golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA= -golang.org/x/mod v0.41.0 h1:qJmnOUb4YB+FsEuM3HcWucdZASCPGhsX6uljO6pog0c= -golang.org/x/mod v0.41.0/go.mod h1:Ek9pY8RKWXwsWvd3rQiHYtMqkjSUV+s1Rj7j4H5Ur6o= +gocloud.dev v0.47.0 h1:qN4/0vtutn97f1X5ziisf3WXuaFL9AR0FIRnyJRYzgA= +gocloud.dev v0.47.0/go.mod h1:OdMnyXQRhM17U0a0DwU57pe0c01qN3lJNmaeeRYW0D0= +golang.org/x/crypto v0.58.0 h1:COkYLr4k7nDI4r6QtMML2AfzeIswWHAJthLS6K/M/54= +golang.org/x/crypto v0.58.0/go.mod h1:Xh+kl5A+M0gMiWDytH2Hhr5Z7WbNCpv/eB/EFgfSk+w= +golang.org/x/mod v0.42.0 h1:ICeL3t2L30ljSPHHxDnR4fqVcBt/zR2zJAb9juavmQg= +golang.org/x/mod v0.42.0/go.mod h1:u5fzIrRInkL8MaO1cbRkUHT64v303YIT5KEqpNr7jRs= golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1Kcs5dz7/ng1VjMUvfKvpfy+jM= -golang.org/x/net v0.60.0 h1:79p50tfZlm0J9YfoDsSi639qSXNGVwEzOPLCxM2FsYU= -golang.org/x/net v0.60.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg= +golang.org/x/net v0.61.0 h1:RR+6j/BTBCrggOLugxa+66R8FvoN6MHlzDlZIbN9YFw= +golang.org/x/net v0.61.0/go.mod h1:WgDCOTH8iwtB66T4/GfI94AHo/hhdb7Hah9NDseQfiI= golang.org/x/oauth2 v0.37.0 h1:JUlcxA8oAtauLfiH8FX2/FkAWHAdi0QtGCGc+hofE98= golang.org/x/oauth2 v0.37.0/go.mod h1:IxwZNxUULJmpBFf9K/9NTMSIfZZuvuTy1gGxhigP/58= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -318,30 +318,30 @@ golang.org/x/sync v0.24.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210420072515-93ed5bcd2bfe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= -golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= +golang.org/x/sys v0.49.0 h1:XbzkgYJHdqh/8m2Uu0W/dQv8nktxx4BFHp1M0gROTXA= +golang.org/x/sys v0.49.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI= -golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E= +golang.org/x/text v0.43.0 h1:1QivrlhwAsnMOcqgOdl9my7s9OyZl8O5/UGUTJVXoWc= +golang.org/x/text v0.43.0/go.mod h1:hbSIYA/amXcRXTFSZqh8tWHEoqRvvbKPJ2uz/KU+a+s= golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.50.0 h1:c2ifzfcuY7L90lZ2aKd8S4K2NpASF08SZx9ZuJkHmSU= -golang.org/x/tools v0.50.0/go.mod h1:7ulVMw3831Mwi5EZD6RomGyffr4VFjuNYXf2BbCEAV0= +golang.org/x/tools v0.51.0 h1:k4Xc/1Om9jwkBJBo4NVLMSARBoWtK10mx+W5BnXCeAI= +golang.org/x/tools v0.51.0/go.mod h1:9eEncMayCV6zRMGhR5eZEC2iBx98qWcF1HZ9Z7wJOoA= golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da h1:noIWHXmPHxILtqtCOPIhSt0ABwskkZKjD3bXGnZGpNY= golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= -google.golang.org/api v0.288.0 h1:glhO/J88obKP5I269W3hB73dvBKrjU56ZfmNlNXpgTU= -google.golang.org/api v0.288.0/go.mod h1:lM2kYRzYUCBY91P9h6VF1PYmvhxii3O5hji37qRvIcY= -google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7 h1:XzmzkmB14QhVhgnawEVsOn6OFsnpyxNPRY9QV01dNB0= -google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7/go.mod h1:L43LFes82YgSonw6iTXTxXUX1OlULt4AQtkik4ULL/I= -google.golang.org/genproto/googleapis/api v0.0.0-20260630182238-925bb5da69e7 h1:jQ9p21COKWjP3VwuFrNRiiOTMh3mPpN45R7SLrH/HUU= -google.golang.org/genproto/googleapis/api v0.0.0-20260630182238-925bb5da69e7/go.mod h1:KqHwBx2upmfa1XSi1WuRvC+2VGCLtooKkfmyvRbUmqA= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260706201446-f0a921348800 h1:qEHAMpSaUhtD0p3NbEEI83HwNGFxEwaSJ1G9PLnCBZE= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260706201446-f0a921348800/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/api v0.297.0 h1:WktxTsnnx0yZNnsR6j0q6hR21RnnK81FHTOPy/ux4OE= +google.golang.org/api v0.297.0/go.mod h1:S4m8x0M6OkQpkOzGk1y9JG2sm4fFQrMh6dxzjCTszhE= +google.golang.org/genproto v0.0.0-20260904194346-d0f1323225a4 h1:6QRNo34FxJcE1LG6XfnmaQ17m6w21oXaKlKoBaF+UqE= +google.golang.org/genproto v0.0.0-20260904194346-d0f1323225a4/go.mod h1:eifJ91nefXnfP/bMkZsACVGm14ZtEL4eUXRLTryfbY4= +google.golang.org/genproto/googleapis/api v0.0.0-20260904194346-d0f1323225a4 h1:NCe/UiklGd/9xjT+ROBVhJ1kf6TRQaFedsR+z7u1gvo= +google.golang.org/genproto/googleapis/api v0.0.0-20260904194346-d0f1323225a4/go.mod h1:fJ2lYaWjqNknJyQBOCd0fA3HnEElJqGplH71a2txi+g= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260904194346-d0f1323225a4 h1:5t+ZydAFj5kGVLrgCvLmpmCf9ylGRd64hpEronfRaws= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260904194346-d0f1323225a4/go.mod h1:DjtHYE8FKJLivXcBEjGwndXfIC23G0VpXiXKqG179uA= google.golang.org/grpc v1.83.2 h1:EManeRomTObA0BU7I8vXgg/78uE5MJ9M8B39EX2WscU= google.golang.org/grpc v1.83.2/go.mod h1:YPI1hK3kDked6iHvgX3tR0y+nX/qpMFKhPgFsokw1S8= google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc= From f77d299b31922e974dfb9340807dd44f29c66b29 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Tue, 20 Oct 2026 21:33:08 +0200 Subject: [PATCH 329/633] Don't hide upstream 404 and 401/403 behind stale metadata With cache_metadata on, a failed upstream metadata fetch fell back to the cached copy for any error. A 404 (package removed) and a 401/403 (the proxy's credentials rejected) are definite answers, though, and serving the old version list in their place kept them from reaching the client. They now pass through: 404 as 404, 401/403 as 502 like other upstream failures. Timeouts, connection errors, 5xx and 429 still fall back. The npm, Composer and Chef handlers serve metadata through the rewrite cache, which dropped the stale flag, so a fallback looked like a fresh 200. They now set the same Warning: 110 header as other metadata. Fixes #499 --- docs/configuration.md | 2 +- internal/handler/chef.go | 1 + internal/handler/composer.go | 2 + internal/handler/composer_hosted.go | 1 + internal/handler/handler.go | 29 +++++- internal/handler/npm.go | 2 + internal/handler/npm_hosted.go | 1 + internal/handler/stale_metadata_test.go | 114 ++++++++++++++++++++++++ 8 files changed, 148 insertions(+), 4 deletions(-) create mode 100644 internal/handler/stale_metadata_test.go diff --git a/docs/configuration.md b/docs/configuration.md index dccdf26..3bffadb 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -987,7 +987,7 @@ Or via environment variable: `PROXY_METADATA_TTL=10m`. Set to `"0"` to always revalidate with upstream (ETag-based conditional requests still avoid re-downloading unchanged content). -When upstream is unreachable and the cached entry is past its TTL, the proxy serves the stale cached copy with a `Warning: 110 - "Response is Stale"` header so clients can tell the data may be outdated. +When upstream is unreachable and the cached entry is past its TTL, the proxy serves the stale cached copy with a `Warning: 110 - "Response is Stale"` header so clients can tell the data may be outdated. This covers timeouts, connection errors, 5xx and 429 responses. A 404 or a 401/403 from the upstream is an answer rather than an outage, so it is not covered: a removed package answers 404, and credentials the upstream rejects answer 502, instead of the client getting a cached version list that no longer exists or can no longer be read. ### Metadata size limit diff --git a/internal/handler/chef.go b/internal/handler/chef.go index 7283992..764d8a3 100644 --- a/internal/handler/chef.go +++ b/internal/handler/chef.go @@ -162,6 +162,7 @@ func (h *ChefHandler) serveDocument(w http.ResponseWriter, r *http.Request, docu http.Error(w, "invalid Chef Supermarket response", http.StatusBadGateway) return } + markStale(w, upstream.stale) serveRewrittenMetadata(w, r, rewritten) } diff --git a/internal/handler/composer.go b/internal/handler/composer.go index 5bc2586..643b287 100644 --- a/internal/handler/composer.go +++ b/internal/handler/composer.go @@ -189,11 +189,13 @@ func (h *ComposerHandler) handlePackageMetadata(w http.ResponseWriter, r *http.R return // the client left while waiting on a shared rewrite } h.proxy.Logger.Warn("failed to rewrite metadata, proxying original", "error", err) + markStale(w, upstream.stale) w.Header().Set(headerContentType, "application/json") _, _ = w.Write(upstream.body) return } + markStale(w, upstream.stale) serveRewrittenMetadata(w, r, rewritten) } diff --git a/internal/handler/composer_hosted.go b/internal/handler/composer_hosted.go index 9a27e6f..689cf8f 100644 --- a/internal/handler/composer_hosted.go +++ b/internal/handler/composer_hosted.go @@ -99,6 +99,7 @@ func (h *ComposerHandler) serveMetadataWithLocal(w http.ResponseWriter, r *http. http.Error(w, "failed to rewrite package metadata", http.StatusBadGateway) return } + markStale(w, upstream.stale) serveRewrittenMetadata(w, r, h.proxy.uncachedRewrite("composer", h.proxyURL, cacheKey, out)) } diff --git a/internal/handler/handler.go b/internal/handler/handler.go index d29508e..31451d3 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -1197,6 +1197,21 @@ func JSONError(w http.ResponseWriter, status int, message string) { _, _ = fmt.Fprintf(w, `{"error":%q}`, message) } +// ErrUpstreamRefused indicates the upstream answered 401 or 403: the +// credentials the proxy sends are missing, wrong or no longer allowed. +var ErrUpstreamRefused = errors.New("upstream refused the request") + +// staleWarning marks a response served from the metadata cache because the +// upstream could not be reached. +const staleWarning = `110 - "Response is Stale"` + +// markStale sets the stale warning on a metadata response when stale is set. +func markStale(w http.ResponseWriter, stale bool) { + if stale { + w.Header().Set("Warning", staleWarning) + } +} + // ErrUpstreamNotFound indicates the upstream returned 404. var ErrUpstreamNotFound = fmt.Errorf("upstream: %w", fetch.ErrNotFound) @@ -1391,6 +1406,13 @@ func (p *Proxy) fetchMetadataFromUpstream(ctx context.Context, ecosystem, cacheK return metadataResult{body: meta.body, contentType: meta.contentType, contentEncoding: meta.contentEncoding, digest: digest} } + // A 404 or 401/403 is the upstream's answer, not an outage: the package + // is gone, or the proxy may no longer read it. Serving the cached copy + // would hide that from the client, so only other failures fall back. + if errors.Is(err, ErrUpstreamNotFound) || errors.Is(err, ErrUpstreamRefused) { + return metadataResult{err: err} + } + // Upstream failed -- fall back to cache if available if !p.CacheMetadata || entry == nil { return metadataResult{err: fmt.Errorf("upstream failed and no cached metadata: %w", err)} @@ -1606,6 +1628,9 @@ func (p *Proxy) fetchUpstreamMetadata(ctx context.Context, upstreamURL string, e body, _ := io.ReadAll(io.LimitReader(resp.Body, upstreamNotFoundBodyLimit)) return nil, &upstreamNotFoundError{body: body} } + if resp.StatusCode == http.StatusUnauthorized || resp.StatusCode == http.StatusForbidden { + return nil, fmt.Errorf("%w: upstream returned %d", ErrUpstreamRefused, resp.StatusCode) + } if resp.StatusCode != http.StatusOK { return nil, fmt.Errorf("upstream returned %d", resp.StatusCode) } @@ -1816,9 +1841,7 @@ func (p *Proxy) writeMetadataCachedResponseWithEncoding(w http.ResponseWriter, r if contentEncoding != "" { w.Header().Set(headerContentEncoding, contentEncoding) } - if cm.stale { - w.Header().Set("Warning", `110 - "Response is Stale"`) - } + markStale(w, cm.stale) w.WriteHeader(http.StatusOK) if r.Method != http.MethodHead { _, _ = w.Write(body) diff --git a/internal/handler/npm.go b/internal/handler/npm.go index 6530f5a..4e00851 100644 --- a/internal/handler/npm.go +++ b/internal/handler/npm.go @@ -263,12 +263,14 @@ func (h *NPMHandler) handlePackageMetadata(w http.ResponseWriter, r *http.Reques } // If rewriting fails, just proxy the original h.proxy.Logger.Warn("failed to rewrite metadata, proxying original", "error", err) + markStale(w, upstream.stale) w.Header().Set(headerContentType, contentTypeJSON) w.WriteHeader(http.StatusOK) _, _ = w.Write(upstream.body) return } + markStale(w, upstream.stale) serveRewrittenMetadata(w, r, rewritten) } diff --git a/internal/handler/npm_hosted.go b/internal/handler/npm_hosted.go index 65b59ba..7532cbd 100644 --- a/internal/handler/npm_hosted.go +++ b/internal/handler/npm_hosted.go @@ -123,6 +123,7 @@ func (h *NPMHandler) serveMetadataWithLocal(w http.ResponseWriter, r *http.Reque JSONError(w, http.StatusBadGateway, "failed to rewrite package metadata") return } + markStale(w, upstream.stale) serveRewrittenMetadata(w, r, h.proxy.uncachedRewrite("npm", h.proxyURL, metadataCacheKey, out)) } diff --git a/internal/handler/stale_metadata_test.go b/internal/handler/stale_metadata_test.go new file mode 100644 index 0000000..a5c2aac --- /dev/null +++ b/internal/handler/stale_metadata_test.go @@ -0,0 +1,114 @@ +package handler + +import ( + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + "time" +) + +const stalePackument = `{"name":"left-pad","versions":{"1.0.0":{"name":"left-pad","version":"1.0.0",` + + `"dist":{"tarball":"https://registry.npmjs.org/left-pad/-/left-pad-1.0.0.tgz"}}}}` + +// npmUpstreamThen serves the packument once, with an ETag, and answers every +// later request with then. +func npmUpstreamThen(t *testing.T, then http.HandlerFunc) *httptest.Server { + t.Helper() + var requests atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if requests.Add(1) == 1 { + w.Header().Set("Content-Type", "application/json") + w.Header().Set("ETag", `"v1"`) + _, _ = w.Write([]byte(stalePackument)) + return + } + then(w, r) + })) + t.Cleanup(srv.Close) + return srv +} + +func TestNPMMetadataAfterUpstreamAnswer(t *testing.T) { + tests := []struct { + name string + then http.HandlerFunc + wantStatus int + wantStale bool + }{ + {name: "upstream error", then: func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusBadGateway) + }, wantStatus: http.StatusOK, wantStale: true}, + {name: "timeout", then: func(w http.ResponseWriter, r *http.Request) { + select { + case <-r.Context().Done(): + case <-time.After(2 * time.Second): + } + }, wantStatus: http.StatusOK, wantStale: true}, + {name: "not modified", then: func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusNotModified) + }, wantStatus: http.StatusOK}, + {name: "unpublished", then: func(w http.ResponseWriter, _ *http.Request) { + http.NotFound(w, nil) + }, wantStatus: http.StatusNotFound}, + {name: "credentials rejected", then: func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusUnauthorized) + }, wantStatus: http.StatusBadGateway}, + {name: "access forbidden", then: func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusForbidden) + }, wantStatus: http.StatusBadGateway}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + upstream := npmUpstreamThen(t, tt.then) + proxy, _, _, _ := setupTestProxy(t) + proxy.CacheMetadata = true + proxy.MetadataTTL = time.Millisecond + proxy.HTTPClient = &http.Client{Timeout: 200 * time.Millisecond} + h := NewNPMHandler(proxy, "http://proxy.test", upstream.URL+"/").Routes() + + first := httptest.NewRecorder() + h.ServeHTTP(first, httptest.NewRequest(http.MethodGet, "/left-pad", nil)) + if first.Code != http.StatusOK { + t.Fatalf("first request: status = %d", first.Code) + } + time.Sleep(5 * time.Millisecond) + + w := httptest.NewRecorder() + h.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/left-pad", nil)) + if w.Code != tt.wantStatus { + t.Fatalf("status = %d, want %d, body %s", w.Code, tt.wantStatus, w.Body) + } + if stale := w.Header().Get("Warning") == staleWarning; stale != tt.wantStale { + t.Errorf("Warning = %q, want stale %v", w.Header().Get("Warning"), tt.wantStale) + } + servedCopy := strings.Contains(w.Body.String(), "left-pad-1.0.0.tgz") + if wantCopy := tt.wantStatus == http.StatusOK; servedCopy != wantCopy { + t.Errorf("served the cached packument = %v, want %v", servedCopy, wantCopy) + } + }) + } +} + +func TestProxyCachedPassesDefiniteAnswersThrough(t *testing.T) { + for _, status := range []int{http.StatusNotFound, http.StatusUnauthorized, http.StatusForbidden} { + t.Run(http.StatusText(status), func(t *testing.T) { + upstream := npmUpstreamThen(t, func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(status) }) + proxy, _, _, _ := setupTestProxy(t) + proxy.CacheMetadata = true + proxy.MetadataTTL = time.Millisecond + proxy.HTTPClient = upstream.Client() + + w := httptest.NewRecorder() + proxy.ProxyCached(w, httptest.NewRequest(http.MethodGet, "/doc", nil), upstream.URL+"/doc", "test-eco", "doc") + time.Sleep(5 * time.Millisecond) + w = httptest.NewRecorder() + proxy.ProxyCached(w, httptest.NewRequest(http.MethodGet, "/doc", nil), upstream.URL+"/doc", "test-eco", "doc") + + if w.Code == http.StatusOK || strings.Contains(w.Body.String(), "left-pad") { + t.Errorf("status = %d, body %q, want the upstream's answer instead of the cached copy", w.Code, w.Body) + } + }) + } +} From 844e2ae4f34993de30361bc94fde50f7fb4becce Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Tue, 20 Oct 2026 21:34:07 +0200 Subject: [PATCH 330/633] Test that Composer metadata doesn't hide upstream 404 and 401/403 --- internal/handler/stale_metadata_test.go | 52 +++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/internal/handler/stale_metadata_test.go b/internal/handler/stale_metadata_test.go index a5c2aac..5cd8d6d 100644 --- a/internal/handler/stale_metadata_test.go +++ b/internal/handler/stale_metadata_test.go @@ -112,3 +112,55 @@ func TestProxyCachedPassesDefiniteAnswersThrough(t *testing.T) { }) } } + +const staleComposerDocument = `{"packages":{"acme/widget":[{"name":"acme/widget","version":"1.0.0",` + + `"dist":{"type":"zip","url":"https://repo.packagist.org/files/acme/widget/1.0.0/widget.zip"}}]},"minified":"composer/2.0"}` + +func TestComposerMetadataAfterUpstreamAnswer(t *testing.T) { + tests := []struct { + name string + status int + wantStatus int + wantStale bool + }{ + {name: "upstream error", status: http.StatusBadGateway, wantStatus: http.StatusOK, wantStale: true}, + {name: "removed", status: http.StatusNotFound, wantStatus: http.StatusNotFound}, + {name: "credentials rejected", status: http.StatusUnauthorized, wantStatus: http.StatusBadGateway}, + {name: "access forbidden", status: http.StatusForbidden, wantStatus: http.StatusBadGateway}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var requests atomic.Int32 + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + if requests.Add(1) == 1 { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(staleComposerDocument)) + return + } + w.WriteHeader(tt.status) + })) + t.Cleanup(upstream.Close) + proxy, _, _, _ := setupTestProxy(t) + proxy.CacheMetadata = true + proxy.MetadataTTL = time.Millisecond + proxy.HTTPClient = upstream.Client() + h := NewComposerHandlerWithUpstreams(proxy, "http://proxy.test", upstream.URL, upstream.URL).Routes() + + first := httptest.NewRecorder() + h.ServeHTTP(first, httptest.NewRequest(http.MethodGet, "/p2/acme/widget.json", nil)) + if first.Code != http.StatusOK { + t.Fatalf("first request: status = %d, body %s", first.Code, first.Body) + } + time.Sleep(5 * time.Millisecond) + + w := httptest.NewRecorder() + h.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/p2/acme/widget.json", nil)) + if w.Code != tt.wantStatus { + t.Fatalf("status = %d, want %d, body %s", w.Code, tt.wantStatus, w.Body) + } + if stale := w.Header().Get("Warning") == staleWarning; stale != tt.wantStale { + t.Errorf("Warning = %q, want stale %v", w.Header().Get("Warning"), tt.wantStale) + } + }) + } +} From 10fc27b2b8a5934ab8ebaeea9a97d33ac3aa0c68 Mon Sep 17 00:00:00 2001 From: simon chrzanowski Date: Wed, 21 Oct 2026 12:08:50 +0200 Subject: [PATCH 331/633] Cache Go modules that differ only in case apart (#540) Module zips are stored under the module's PURL, which lowercases golang names, and under the filename {last element}@{version}.zip. So github.com/Sirupsen/logrus and github.com/sirupsen/logrus shared one cache entry although upstream serves different zips, and the second one requested failed go's checksum check. A module path with capitals now gets a hash of its exact spelling in the filename. All-lowercase paths keep the plain name, so existing cache entries stay valid. A capitalised module cached before this change sits under its lowercase twin's plain filename. When a cached zip's upstream URL names a different module path, it is discarded and fetched again. Only the module path is compared, so a cache filled from another upstream.go host is kept. Fixes #466 --- internal/handler/go.go | 56 +++++++++- internal/handler/go_case_test.go | 135 +++++++++++++++++++++++ internal/server/retention_golang_test.go | 14 ++- 3 files changed, 200 insertions(+), 5 deletions(-) create mode 100644 internal/handler/go_case_test.go diff --git a/internal/handler/go.go b/internal/handler/go.go index c528472..7a1eb86 100644 --- a/internal/handler/go.go +++ b/internal/handler/go.go @@ -1,11 +1,13 @@ package handler import ( + "crypto/sha256" + "encoding/hex" "errors" - "fmt" "net/http" "strings" + "github.com/git-pkgs/proxy/internal/database" "github.com/git-pkgs/proxy/internal/retention" "github.com/git-pkgs/registries/fetch" ) @@ -119,12 +121,13 @@ func (h *GoHandler) handleRequest(w http.ResponseWriter, r *http.Request) { func (h *GoHandler) handleDownload(w http.ResponseWriter, r *http.Request, module, version string) { // Decode module path (! followed by lowercase = uppercase) decodedModule := decodeGoModule(module) - filename := fmt.Sprintf("%s@%s.zip", lastComponent(decodedModule), version) + filename := goZipFilename(decodedModule, version) h.proxy.Logger.Info("go module download request", "module", decodedModule, "version", version) downloadURL := h.upstreamURL + r.URL.Path + h.discardOtherModulesZip(decodedModule, module, version, filename) result, err := h.proxy.GetOrFetchArtifactFromURL( r.Context(), "golang", decodedModule, version, filename, downloadURL, ) @@ -148,6 +151,55 @@ func (h *GoHandler) handleDownload(w http.ResponseWriter, r *http.Request, modul ServeArtifactRequest(w, r, result) } +// goZipFilename is the name a module zip is cached under. Module paths are +// case-sensitive, but the zip is stored under the module's PURL, which +// lowercases golang names, so github.com/Sirupsen/logrus and +// github.com/sirupsen/logrus share one package. A path with uppercase +// letters therefore gets a hash of its exact spelling in the filename. An +// all-lowercase path keeps the plain name, so existing cache entries stay +// valid. +func goZipFilename(module, version string) string { + base := lastComponent(module) + "@" + version + if module == strings.ToLower(module) { + return base + ".zip" + } + digest := sha256.Sum256([]byte(module)) + return base + "~" + hex.EncodeToString(digest[:goModuleHashBytes]) + ".zip" +} + +// goModuleHashBytes is how much of a module path's hash a cached zip's +// filename carries: enough to tell apart paths that differ only in case. +const goModuleHashBytes = 8 + +// discardOtherModulesZip drops a cached zip that another module stored +// under filename. Before module paths kept their case, a module with +// uppercase letters was cached under the plain filename of its lowercase +// twin, and whichever came first was served for both. The upstream URL a +// zip was fetched from names its module in escaped form, so a record whose +// URL names a different module is discarded and fetched again. Only the +// module path is compared, so pointing upstream.go elsewhere keeps the cache. +func (h *GoHandler) discardOtherModulesZip(module, escapedModule, version, filename string) { + _, versionPURL, err := packagePURLStrings("golang", module, version) + if err != nil { + return + } + record, err := h.proxy.DB.GetArtifact(versionPURL, filename) + if err != nil || record == nil || !record.StoragePath.Valid || record.Source == database.SourceLocal { + return + } + // Only a URL that names another Go module proves the zip is someone + // else's; a record of unknown origin is kept. + if !strings.Contains(record.UpstreamURL, "/@v/") || + strings.HasSuffix(record.UpstreamURL, "/"+escapedModule+"/@v/"+version+".zip") { + return + } + h.proxy.Logger.Warn("cached zip belongs to a module that differs only in case, discarding", + "module", module, "version", version, "cached_from", record.UpstreamURL) + if err := h.proxy.DB.DiscardArtifact(versionPURL, filename, record.StoragePath.String); err != nil { + h.proxy.Logger.Warn("failed to discard cached zip", "module", module, "version", version, "error", err) + } +} + // proxyCached forwards a request with metadata caching. func (h *GoHandler) proxyCached(w http.ResponseWriter, r *http.Request, cacheKey string) { h.proxy.ProxyCached(w, r, h.upstreamURL+r.URL.Path, "golang", cacheKey, "*/*") diff --git a/internal/handler/go_case_test.go b/internal/handler/go_case_test.go new file mode 100644 index 0000000..9451caa --- /dev/null +++ b/internal/handler/go_case_test.go @@ -0,0 +1,135 @@ +package handler + +import ( + "context" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + + "github.com/git-pkgs/registries/fetch" +) + +const ( + upperLogrusPath = "/github.com/!sirupsen/logrus/@v/v1.0.6.zip" + lowerLogrusPath = "/github.com/sirupsen/logrus/@v/v1.0.6.zip" +) + +func TestGoZipFilenameKeepsCase(t *testing.T) { + lower := goZipFilename("github.com/sirupsen/logrus", "v1.0.6") + if lower != "logrus@v1.0.6.zip" { + t.Errorf("lowercase module filename = %q, want the plain name", lower) + } + upper := goZipFilename("github.com/Sirupsen/logrus", "v1.0.6") + if upper == lower || !strings.HasPrefix(upper, "logrus@v1.0.6~") || !strings.HasSuffix(upper, ".zip") { + t.Errorf("uppercase module filename = %q, want logrus@v1.0.6~{hash}.zip", upper) + } + if other := goZipFilename("github.com/SIRUPSEN/logrus", "v1.0.6"); other == upper { + t.Errorf("differently cased modules share filename %q", other) + } +} + +// goCaseUpstream serves different bytes for each module path and counts +// requests per path. +func goCaseUpstream(t *testing.T) (*httptest.Server, map[string]*atomic.Int32) { + t.Helper() + counts := map[string]*atomic.Int32{upperLogrusPath: {}, lowerLogrusPath: {}} + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + count, ok := counts[r.URL.Path] + if !ok { + http.NotFound(w, r) + return + } + count.Add(1) + _, _ = io.WriteString(w, "zip of "+r.URL.Path) + })) + t.Cleanup(srv.Close) + return srv, counts +} + +func newGoCaseHandler(t *testing.T, upstream *httptest.Server) (http.Handler, *Proxy) { + t.Helper() + proxy, _, store, _ := setupTestProxy(t) + store.seekable = true + proxy.HTTPClient = upstream.Client() + fetcher := fetch.NewFetcher(fetch.WithHTTPClient(upstream.Client()), fetch.WithMaxRetries(0)) + proxy.Fetcher = fetcher + t.Cleanup(func() { _ = fetcher.Close() }) + return NewGoHandlerWithUpstream(proxy, "http://localhost", upstream.URL).Routes(), proxy +} + +func getGo(h http.Handler, path string) *httptest.ResponseRecorder { + w := httptest.NewRecorder() + h.ServeHTTP(w, httptest.NewRequest(http.MethodGet, path, nil)) + return w +} + +func TestGoModulesDifferingInCaseAreCachedApart(t *testing.T) { + upstream, counts := goCaseUpstream(t) + h, _ := newGoCaseHandler(t, upstream) + + for range 2 { + for _, path := range []string{upperLogrusPath, lowerLogrusPath} { + if w := getGo(h, path); w.Code != http.StatusOK || w.Body.String() != "zip of "+path { + t.Fatalf("GET %s: status = %d, body = %q", path, w.Code, w.Body) + } + } + } + for path, count := range counts { + if n := count.Load(); n != 1 { + t.Errorf("%s fetched %d times, want once and then cached", path, n) + } + } +} + +// Before module paths kept their case, the uppercase module was cached under +// the lowercase module's plain filename. The lowercase module must not be +// served those bytes. +func TestGoZipCachedForAnotherModuleIsRefetched(t *testing.T) { + upstream, counts := goCaseUpstream(t) + h, proxy := newGoCaseHandler(t, upstream) + seedGoZip(t, proxy, "https://proxy.golang.org"+upperLogrusPath, "zip of "+upperLogrusPath) + + if w := getGo(h, lowerLogrusPath); w.Code != http.StatusOK || w.Body.String() != "zip of "+lowerLogrusPath { + t.Fatalf("status = %d, body = %q, want the lowercase module's zip", w.Code, w.Body) + } + if n := counts[lowerLogrusPath].Load(); n != 1 { + t.Errorf("lowercase module fetched %d times, want 1", n) + } +} + +// A zip cached from another upstream host for the same module stays valid: +// only the module path decides whether a cached zip belongs to the request. +func TestGoZipFromAnotherUpstreamHostIsKept(t *testing.T) { + upstream, counts := goCaseUpstream(t) + h, proxy := newGoCaseHandler(t, upstream) + seedGoZip(t, proxy, "https://old-proxy.example"+lowerLogrusPath, "zip of "+lowerLogrusPath) + + if w := getGo(h, lowerLogrusPath); w.Code != http.StatusOK || w.Body.String() != "zip of "+lowerLogrusPath { + t.Fatalf("status = %d, body = %q", w.Code, w.Body) + } + if n := counts[lowerLogrusPath].Load(); n != 0 { + t.Errorf("zip fetched %d times, want it served from the cache", n) + } +} + +// seedGoZip caches content as github.com/sirupsen/logrus v1.0.6 under the +// plain filename, recorded as fetched from upstreamURL. +func seedGoZip(t *testing.T, proxy *Proxy, upstreamURL, content string) { + t.Helper() + const storagePath = "golang/logrus/v1.0.6/seed/logrus@v1.0.6.zip" + store := proxy.Storage.(*mockStorage) + if _, _, err := store.Store(context.Background(), storagePath, strings.NewReader(content)); err != nil { + t.Fatalf("seeding storage: %v", err) + } + pkgPURL, versionPURL, err := packagePURLStrings("golang", "github.com/sirupsen/logrus", "v1.0.6") + if err != nil { + t.Fatalf("packagePURLStrings: %v", err) + } + artifact := testArtifact(content, versionPURL, "logrus@v1.0.6.zip", "application/zip") + if err := proxy.updateCacheDB("golang", "github.com/sirupsen/logrus", pkgPURL, upstreamURL, storagePath, artifact, ""); err != nil { + t.Fatalf("seeding cache record: %v", err) + } +} diff --git a/internal/server/retention_golang_test.go b/internal/server/retention_golang_test.go index 50c8dde..f74eaf4 100644 --- a/internal/server/retention_golang_test.go +++ b/internal/server/retention_golang_test.go @@ -74,10 +74,18 @@ func golangGet(t *testing.T, url string) { } // golangArtifact returns the record of the zip named filename. +// golangFilenameMatch matches a zip's cached filename: the plain name, or, +// for a module path with capitals, the plain name with a hash of the path. +const golangFilenameMatch = `(filename = ? OR filename LIKE ?)` + +func golangFilenameArgs(filename string) []any { + return []any{filename, strings.TrimSuffix(filename, ".zip") + "~%.zip"} +} + func golangArtifact(t *testing.T, db *database.DB, filename string) database.Artifact { t.Helper() var art database.Artifact - if err := db.Get(&art, db.Rebind(`SELECT * FROM artifacts WHERE filename = ?`), filename); err != nil { + if err := db.Get(&art, db.Rebind(`SELECT * FROM artifacts WHERE `+golangFilenameMatch), golangFilenameArgs(filename)...); err != nil { t.Fatalf("reading artifact %s: %v", filename, err) } return art @@ -85,8 +93,8 @@ func golangArtifact(t *testing.T, db *database.DB, filename string) database.Art func golangBackdate(t *testing.T, db *database.DB, filename string, fetched, accessed time.Time) { t.Helper() - query := db.Rebind(`UPDATE artifacts SET fetched_at = ?, last_accessed_at = ? WHERE filename = ?`) - if _, err := db.Exec(query, fetched, accessed, filename); err != nil { + query := db.Rebind(`UPDATE artifacts SET fetched_at = ?, last_accessed_at = ? WHERE ` + golangFilenameMatch) + if _, err := db.Exec(query, append([]any{fetched, accessed}, golangFilenameArgs(filename)...)...); err != nil { t.Fatalf("backdating %s: %v", filename, err) } } From cc6069d8a0d4cdb5c65b91eedb8dc3459c29add3 Mon Sep 17 00:00:00 2001 From: simon chrzanowski Date: Wed, 21 Oct 2026 12:10:49 +0200 Subject: [PATCH 332/633] Serve additional Maven repositories at /maven/{name}/ (#539) * Serve additional Maven repositories at /maven/{name}/ upstream.maven_repositories maps names to further Maven repositories, each served at /maven/{name}/ from its own upstream, like debian_repositories does for APT. upstream.maven stays the default at /maven/ with unchanged cache keys and its Gradle Plugin Portal fallback; named repositories don't fall back to the portal. Named repositories cache their files under {name}/{filename} and hash their metadata keys from name, upstream URL and path, so the same coordinates in two repositories, which may hold different bytes, never share a cache entry. A repository name is matched before the default repository and would hide group IDs starting with it, so names of two letters or less and common top-level domains (com, org, net, io, ...) are refused at config load. The documentation asks for names that can't start a group ID. Fixes #295 * Refuse every Maven Central group root as a repository name The hand-written list missed real roots such as tools (Jackson 3), software (AWS SDK), jakarta, javax and junit, so a repository named after one would have hidden those groups in the default repository. The name check now uses the first path segments of all group IDs on Maven Central, plus Google's androidx and android, embedded from the repository's directory listing. Names are compared case-sensitively, since routing is, so "ORG" is no longer refused for hiding "org". A named repository's files are now cached under its name and a hash of its upstream URL, so pointing it at a different URL stops serving what the old one sent. * Refuse Gradle Plugin Portal roots and unroutable repository names - The default repository falls back to the Gradle Plugin Portal, so its group roots (gradle, nebula, ...) would be hidden by a repository of the same name too. The embedded list now holds the roots of Maven Central and the Plugin Portal. - Repository names are limited to letters, digits, '.', '_' and '-', so a name can't contain characters such as '?', '#' or spaces that never reach the handler as configured. - Named repositories are built once in WithRepositories, with their cache prefix, instead of on every request. --- README.md | 2 + config.example.yaml | 6 + docs/configuration.md | 41 + internal/config/config.go | 72 ++ internal/config/config_test.go | 47 + internal/config/maven_group_roots.txt | 1228 +++++++++++++++++++ internal/handler/maven.go | 115 +- internal/handler/maven_repositories_test.go | 139 +++ internal/server/server.go | 2 +- 9 files changed, 1631 insertions(+), 21 deletions(-) create mode 100644 internal/config/maven_group_roots.txt create mode 100644 internal/handler/maven_repositories_test.go diff --git a/README.md b/README.md index 6ab21cb..d4881cc 100644 --- a/README.md +++ b/README.md @@ -290,6 +290,8 @@ Add to your `~/.m2/settings.xml`: The `/maven/` endpoint uses Maven Central as primary upstream and falls back to the Gradle Plugin Portal for Gradle plugin marker metadata and related artifacts when the primary upstream returns not found. +Further Maven repositories can be configured under `upstream.maven_repositories` and are served at `/maven/{name}/`, each from its own upstream (see [Maven repositories](docs/configuration.md#maven-repositories)). Point a second `` or Gradle `maven(url = ...)` entry at `http://localhost:8080/maven/{name}/`. + For Gradle plugin resolution via the same proxy endpoint: ```kotlin diff --git a/config.example.yaml b/config.example.yaml index 277eb23..f8bebe7 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -219,6 +219,12 @@ upstream: # Maven repository URL (used by /maven endpoint) maven: "https://repo1.maven.org/maven2" + # Additional Maven repositories, served at /maven/{name}/. A name must not + # be the start of a group ID (such as "org" or "tools"), since it would hide + # those groups in the default repository. + # maven_repositories: + # grails: "https://repo.grails.org/grails/restricted" + # Gradle Plugin Portal Maven URL (fallback for plugin marker artifacts) gradle_plugin_portal: "https://plugins.gradle.org/m2" diff --git a/docs/configuration.md b/docs/configuration.md index dccdf26..507cad5 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -623,6 +623,47 @@ those names unless the shadowing is intended. This field has no environment variable, as with the other named upstream maps. `PROXY_UPSTREAM_DEBIAN` still sets `upstream.debian`. +### Maven repositories + +```yaml +upstream: + + # The default repository, served at /maven/. + maven: "https://repo1.maven.org/maven2" + + # Additional Maven repositories, served at /maven/{name}/. + maven_repositories: + grails: "https://repo.grails.org/grails/restricted" +``` + +Builds often need more than one Maven repository. Each entry in +`upstream.maven_repositories` is served at `/maven/{name}/` with the upstream's +layout, e.g. `/maven/grails/org/grails/grails-core/6.0.0/grails-core-6.0.0.pom`, +and is fetched from that upstream alone: unlike the default repository, a named +one doesn't fall back to the Gradle Plugin Portal. Files are cached per +repository, so the same coordinates in two repositories never share a cache +entry. + +`upstream.maven_repositories` is additive. `/maven/…` keeps serving +`upstream.maven` with unchanged cache identities. A repository name is matched +before the default repository, so it hides every group ID whose first segment +is the same name there: a repository named `tools` would take +`/maven/tools/jackson/…` away from Maven Central. At config load the proxy +refuses names of two characters or less, common top-level domains such as `com` +and `io`, and every first segment of a group ID on Maven Central, the Gradle +Plugin Portal and Google's Maven repository (about 1,200 names, such as `javax`, +`junit`, `software` or `gradle`). Names are compared case-sensitively, as paths +are, and may only use letters, digits, `.`, `_` and `-`. Other repositories can +have group IDs the list doesn't know, so pick names that can't be the start of +a group ID, such as `grails` or `company-releases`. + +A repository's files are cached under its name and a hash of its upstream URL, +so pointing a repository at a different URL stops serving what the old one +sent. + +This field has no environment variable, as with the other named upstream maps. +`PROXY_UPSTREAM_MAVEN` still sets `upstream.maven`. + ## Authentication Configure authentication for private upstream registries. The same authentication-aware client is used for metadata and artifact downloads, and credentials can reference environment variables using `${VAR_NAME}` syntax. diff --git a/internal/config/config.go b/internal/config/config.go index e9976de..e7851c9 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -60,6 +60,7 @@ package config import ( + _ "embed" "encoding/base64" "encoding/json" "fmt" @@ -641,6 +642,14 @@ type UpstreamConfig struct { // Default: https://repo1.maven.org/maven2 Maven string `json:"maven" yaml:"maven"` + // MavenRepositories maps repository names to additional Maven repository + // base URLs, served at /maven/{name}/. upstream.maven stays the default + // at /maven/. A name must not be the first segment of a group ID, such + // as "org" or "tools", since it would hide those groups in the default + // repository; see validateMavenRepositoryName. + // Example: {"grails": "https://repo.grails.org/grails/restricted"}. + MavenRepositories map[string]string `json:"maven_repositories" yaml:"maven_repositories"` + // GradlePluginPortal is the upstream Gradle Plugin Portal Maven URL. // Used to resolve Gradle plugin marker artifacts. // Default: https://plugins.gradle.org/m2 @@ -837,6 +846,14 @@ func (u *UpstreamConfig) Validate() error { if err := u.validateGenericArtifacts(); err != nil { return err } + if err := validateNamedUpstreams("upstream.maven_repositories", u.MavenRepositories); err != nil { + return err + } + for name := range u.MavenRepositories { + if err := validateMavenRepositoryName(name); err != nil { + return err + } + } if err := validateNamedUpstreams("upstream.debian_repositories", u.DebianRepositories); err != nil { return err } @@ -893,6 +910,61 @@ func CompileGenericArtifactPattern(pattern string) (*regexp.Regexp, error) { return re, nil } +// mavenGroupRootList lists the first path segments of group IDs on Maven +// Central and Google's Maven repository, one per line. +// +//go:embed maven_group_roots.txt +var mavenGroupRootList string + +// mavenGroupRoots are the first segments of group IDs a repository name +// must not take: those in mavenGroupRootList, and generic top-level domains +// group IDs are commonly reversed from. +var mavenGroupRoots = func() map[string]bool { + roots := map[string]bool{} + for _, tld := range []string{"app", "biz", "cloud", "com", "dev", "edu", "gov", "info", "int", + "io", "mil", "name", "net", "org", "pro", "tech", "xyz"} { + roots[tld] = true + } + for line := range strings.Lines(mavenGroupRootList) { + if line = strings.TrimSpace(line); line != "" && !strings.HasPrefix(line, "#") { + roots[line] = true + } + } + return roots +}() + +// mavenRepositoryNamePattern is what a repository name may look like: the +// characters of a group ID segment, which clients send unescaped. +var mavenRepositoryNamePattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]*$`) + +// countryCodeLength is the length of a country-code top-level domain. +const countryCodeLength = 2 + +// validateMavenRepositoryName refuses names that are likely the first +// segment of a group ID. /maven/{name}/ is matched before the default +// repository, so such a name would hide every group under it there: +// a repository named "tools" would take /maven/tools/jackson/... away from +// Maven Central. Names are limited to characters that reach the handler +// unescaped. Paths are case-sensitive, so names are compared as they +// are. Names of two characters or less are refused as country codes. The +// list can't be complete, since group IDs are free-form, so the +// documentation also asks for names that can't be a group root. +func validateMavenRepositoryName(name string) error { + if !mavenRepositoryNamePattern.MatchString(name) { + return fmt.Errorf("invalid upstream.maven_repositories name %q: use letters, digits, '.', '_' and '-', "+ + "starting with a letter or digit, so the name works as a URL path segment", name) + } + if len(name) <= countryCodeLength { + return fmt.Errorf("invalid upstream.maven_repositories name %q: names of two characters or less "+ + "could hide country-code group IDs in the default repository", name) + } + if mavenGroupRoots[name] { + return fmt.Errorf("invalid upstream.maven_repositories name %q: it would hide group IDs starting with %q "+ + "in the default repository; use a name such as \"%s-repo\"", name, name, name) + } + return nil +} + // debianReservedRepositoryNames are the upstream.debian archive's own root // paths, which a repository of the same name would shadow. var debianReservedRepositoryNames = []string{"pool", "dists"} diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 3622d63..48ae1c7 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -1838,3 +1838,50 @@ func TestLoadArtifactMaxSizeFromEnv(t *testing.T) { t.Errorf("ArtifactMaxSize = %q, want %q", cfg.ArtifactMaxSize, "2GB") } } + +func TestMavenRepositoryNames(t *testing.T) { + for _, tt := range []struct { + name string + wantErr bool + }{ + {"grails", false}, + {"grails-restricted", false}, + {"company-releases", false}, + {"org", true}, + {"com", true}, + {"tools", true}, + {"software", true}, + {"jakarta", true}, + {"javax", true}, + {"junit", true}, + {"androidx", true}, + {"gradle", true}, + {"nebula", true}, + {"my repo", true}, + {"rel#1", true}, + {"rel?x", true}, + {"-releases", true}, + {"repo_2.x", false}, + {"ORG", false}, + {"Tools", false}, + {"io", true}, + {"IO", true}, + {"de", true}, + {"x", true}, + {"net", true}, + {"a/b", true}, + } { + cfg := Default() + cfg.Upstream.MavenRepositories = map[string]string{tt.name: "https://repo.example/maven"} + err := cfg.Validate() + if (err != nil) != tt.wantErr { + t.Errorf("maven_repositories name %q: error = %v, want error %v", tt.name, err, tt.wantErr) + } + } + + cfg := Default() + cfg.Upstream.MavenRepositories = map[string]string{"grails": "not a url"} + if err := cfg.Validate(); err == nil { + t.Error("maven_repositories with an invalid URL: want an error") + } +} diff --git a/internal/config/maven_group_roots.txt b/internal/config/maven_group_roots.txt new file mode 100644 index 0000000..b199222 --- /dev/null +++ b/internal/config/maven_group_roots.txt @@ -0,0 +1,1228 @@ +# First path segments of group IDs on Maven Central and the Gradle Plugin +# Portal, which the default repository falls back to, plus roots from +# Google's Maven repository. A maven_repositories name from this list would +# hide those groups in the default repository. Regenerate with: +# for u in https://repo1.maven.org/maven2/ https://plugins.gradle.org/m2/; do +# curl -s $u | grep -o 'href="[^"]*/"' | sed 's/href="//;s/\/"//' | grep -v '^\.\.$' +# done | sort -u # then add android and androidx +0x +abbot +ac +academy +acegisecurity +activation +activecluster +activeio +activemq +activemq-jaxb +activesoap +activespace +ad +adarwin +ae +aelfred +aero +africa +ag +agency +agorapulse +ai +aislib +ajk +al +all +alm +altrmi +am +amdonov +amnl +android +androidx +andromda +annogen +ant +ant-contrib +ant-doxygen +ant4eclipse +antlr +anttex +anverus +anvil +aopalliance +apache-jaxme +apl +app +aptconvert +ar +argelbargel +args4j +art +as +ashkay +ashkelon +asia +asm +aspectj +aspectwerkz +at +au +audio +avalon +avalon-activation +avalon-apps +avalon-composition +avalon-cornerstone +avalon-extension +avalon-framework +avalon-http +avalon-logging +avalon-logkit +avalon-meta +avalon-phoenix +avalon-repository +avalon-util +aws +ax +axion +axis +axis2 +az +azote +ba +backport-util-concurrent +backport175 +bairesqa +band +barsuift +baseline +batik +bayern +bcel +bd +be +beehive +beer +berkano +berkeleydb +berlin +best +bet +bg +bi +bike +binky +bio +biz +black +blissed +blog +blue +boo +bot +bouncycastle +box +boxstuff +br +brownshome +bsf +bsh +bue +build +builders +bump +burlap +buzz +by +bz +c10n +c3p0 +ca +cab +cactus +cafe +calces +cam +camp +capital +cards +care +cargo +cas +casa +cash +castor +cat +cc +ccd-sdk +cd +center +cern +cewolf +cf +cglib +cgoit +ch +charlotte +chat +cheap +checkstyle +church +ci +city +cj +cl +classworlds +click +clickstream +clinic +clirr +cloud +clover +club +cm +cmsdeploy +cn +co +coach +cobertura +coconut +cocoon +code +code316 +codehaus +CodeLineCounter +codes +coffee +colaba +colt +com +commons-attributes +commons-beanutils +commons-betwixt +commons-chain +commons-cli +commons-codec +commons-collections +commons-compress +commons-configuration +commons-daemon +commons-dbcp +commons-dbutils +commons-digester +commons-discovery +commons-el +commons-email +commons-fileupload +commons-grafolia +commons-grant +commons-graph +commons-http +commons-httpclient +commons-i18n +commons-io +commons-jdbc2pool +commons-jelly +commons-jexl +commons-jux +commons-jxpath +commons-lang +commons-latka +commons-launcher +commons-logging +commons-math +commons-messenger +commons-modeler +commons-naming +commons-net +commons-pool +commons-primitives +commons-resources +commons-scxml +commons-services +commons-sql +commons-test +commons-threadpool +commons-transaction +commons-util +commons-validator +commons-vfs +commons-xo +community +company +computer +concurrent +consulting +continuum +controlhaus +cool +coop +cornerstone-connection +cornerstone-datasources +cornerstone-scheduler +cornerstone-sockets +cornerstone-store +cornerstone-threads +cos +cover +craft +crimson +cryptix +cssparser +cup +cv +cwoongc +cx +cz +d-haven-event +d-haven-eventbus +d-haven-mpool +d3code +daemoniclabs +dalma +dalms +damagecontrol +dataforge +datasift +date +daveight +davethe8 +day +dbunit +de +decaf +decorutils +default +dentaku +dependency +dependency-maven-plugin +design +dev +digit +digital +dinistiq +directory +directory-asn1 +directory-authx +directory-clients +directory-naming +directory-network +directory-protocols +directory-shared +displaytag +ditchnet +dk +dm +dna +dnsjava +do +docbook +doccheck +docker +dom4j +domify +doxia +drone +drools +drools-examples +ds2 +dsh-vocabulary +dtdparser +dumbster +dwr +dynaop +earth +easyconf +easymock +ec +echo +echo3 +echo3extras +eclipse +eco +ecs +edenlib +edtftp +edu +education +ee +egor-n +ehcache +ejb +el-impl +elect86 +email +emberio +emetriq +emma +energy +engineer +engineering +enterprise +enterprises +ervacon +es +esdk +esper +eu +eus +events +excalibur +excalibur-altrmi +excalibur-cli +excalibur-collections +excalibur-component +excalibur-component-examples +excalibur-component-tests +excalibur-concurrent +excalibur-configuration +excalibur-containerkit +excalibur-datasource +excalibur-event +excalibur-extension +excalibur-fortress +excalibur-i18n +excalibur-instrument +excalibur-instrument-manager +excalibur-instrument-manager-interfaces +excalibur-io +excalibur-lifecycle +excalibur-logger +excalibur-monitor +excalibur-mpool +excalibur-naming +excalibur-pool +excalibur-sourceresolve +excalibur-store +excalibur-thread +excalibur-util +excalibur-xmlutil +exchange +exist +exml +exo +exolabcore +expert +express +exteca +external +ezy +family +fan +fans +farm +fashion +fastutil +fesi +fi +film +finance +financial +findbugs +firebase +firedancer +fish +fit +flavor +flox +flux +fm +fo +foo +fop +forehead +formproc +foundation +foxtrot +fr +framework +freebxml +freemarker +frl +fulcrum +fun +fwrap +fyi +ga +gabriel +gae +games +garden +gay +gbean +gdn +ge +generama +genjar +genjava +gent +geronimo +geronimo-spec +getui +gg +github +glass +glassfish +gliby +global +gmbh +gnu +gnu-regexp +gov +gq +gr +gradle +gradle-autoconf +gradle-clojure +gradle-jenkins-job-dsl-plugin +graphics +graphlayout +green +greendao +gregdev +grizzly +grizzly-cachetest +groovy +groovy-xmlrpc +groovycalamari +groovyx +group +gryphon +gs +gsbase +guru +haibison +hamburg +health +help +hessian +hibernate +hihex +hivemind +hk +hm +hmcts +hnct +Hoop +host +house +howl +hr +hsqldb +htmlunit +HTTPClient +httpcomponents-httpcore +httpunit +hu +hudson +huqi +i +ical4j +icu +icu4j +id +idb +ie +iirekm +ikakara +il +im +immo +in +inc +industries +info +informa +ink +innig +int +io +iq +ir +is +isorelax +it +itext +ivory +izpack +jaas +jaci +jackcess +jackson +jacl +jaf +jaimbot +jakarta +jakarta-regexp +jalopy +james +janino +jardiff +jarjar +jarsync +jasper-jsr199 +jasperreports +java_cup +java2html +java3d +javacc +javaconfig +javadb +javadoc +javaee +javagroups +javainetlocator +javamail +javamuc +javancss +javanettasks +javassist +javatar +javax +javazoom +javolution +jawin +jaxb +jaxen +jaxme +jaxr-ra +jblanket +jboss +jca +jcache +jcharts +jcifs +jcollardbovy +jcom +jcommon +jcoverage +jcs +jcs-javagroups +jcvsii +jdbc +jdbm +jdepend +jdiff +jdo +jdom +jdring +jdsl +je +jen +jencks +jep +jepi +jersey +jetty +jexcelapi +jface +jfree +jfreechart +jgen +jgoodies +jgraph +jgrapht +jgroups +jhunlang +jimmy +jini +jintention +jisp +jivesoftware +jlibdiff +jline +jmagick +jmaki +jmdns +jmfayard +jmimemagic +jmml +jmock +jms +jmscts +jmsn +joda-time +john-test +jotm +jp +jparsec +jpl +jpox +jpox-dbcp +jpox-enhancer +jpox-java5 +jrexx +jrms +jrobin +jruby +jsch +jsf-extensions +jspapi +jsptags +jstl +jstyle +jta +jtds +jtidy +juddi +judy +jug +jung +junit +junit-addons +junit-doclet +junitperf +juno +JuspayAssets +jwebunit +jxta +jython +k8s +kante +kawa +ke +keith +kg +kh +kim +king +kirin +kiwi +kkdt +kliche +koeln +kohsuke +kotlin2ts +kr +krd +kred +kruse +kt-logging +ktorm-tools +kxml +kxml2 +ky +kz +la +land +lat +lazy +lc +ldapd-common +ldapsdk +lenala +lgbt +li +libgdx-oboe +life +lingo +link +live +lk +llc +locc +log4j +log4unit +loggersoft +logkit +lol +loom +love +lpg +lt +ltd +lu +lucene +luisrjaeger +lv +ly +ma +magicGball +maps +market +markets +marmalade +math +maven +maven-integration-test-helper +maven-javanet-plugin +maven-new +maven-plugins +maven-proxy +maven-taglib +maven-torque-plugin +maven-validator +maven-xdoclet2-plugin +maxq +mckoi +md +me +media +merlin +merlin-developer +merlin-tutorial +messenger +metaclass +mevenide +micro +microcontainer +middlegen +mil +milyn +mk +ml +mm +mm-mysql +mn +mnm +mobi +mockcreator +mockit +mockmaker +mockobjects +mockrunner +modello +moe +monetdb +money +monster +mp +mpern +mrj +ms +mstor +msv +mt +mu +mule +muse +mx +mx4j +my +myfaces +mysql +mz +name +nanning +nanocontainer +nebula +nekohtml +neo +net +netbeans +network +neva +new +nexus +nf +ng +ninja +nl +no +norbert +nova +np +nrlssc +nsuml +nu +null +nz +oauth +odmg +oe +ognl +ojb +ojdbc +old +one +oness +onl +online +open +open-esb +opencypher +openejb +openim +openjms +opennms +opensymphony +openxp +oracle +org +org.elasticsearch +org.elasticsearch.client +org.elasticsearch.distribution.integ-test-zip +org.elasticsearch.distribution.zip +org.elasticsearch.gradle +org.elasticsearch.plugin +org.elasticsearch.test +oro +oscube +ovh +p2psockets +p6spy +page +partners +party +pathwaycommons +patterntesting +payload +pcj +pdfbox +pe +penguin +pet +petridish +ph +phd +photos +piccolo +picocontainer +picounit +pink +piotrmadry +pircbot +pitt +pk +pl +plexus +plj +plugin +plus +pluto-container +pm +pmd +pnuts +poi +poolman +portlet-api +postgresql +prevayler +pro +proctor +productions +profiler +progress +projektor +protop +proxool +proxytoys +pt +pub +pubscribe +pull-parser +pw +py +qa +qdox +qfork +quartz +quest +quilt +radeox +re +readline +red +redhill +redis +redmine +regexp +relaxngDatatype +ren +report +reportrunner +rest +rgoldberg +rhino +ricosys +rip +rkennel +ro +robo-guice +roboguice +rocks +rodeo +roller +rome +rpost +rs +rss4j +rsslibj +ru +rubygems +run +sa +sablecc +sagex +sale +sap +sax +saxon +saxpath +sbnarra +sbs +sc +scabbard +schibsted +schoenja +school +schwarz +science +scot +scout +scraping-engine +se +sebersole +security +securityfilter +seek +serg +servicemix +servicemix-ws +services +servletapi +servlets +setpoint +sfx4j +sg +sh +shellix +shocks +shop +show +si +si6 +sillyexceptions +simple +simple-jms +simple-jndi +site +sk +skaringa +ski +skinlf +slide +smartrics +sn +so +soap +social +software +soimp +solar +solarisrealm +solutions +source +space +speexx +spice +spring +springframework +springmodules +sqlline +srl +sshtools +sslext +st +stapler +statcvs +stax +stax-utils +store +stratum +stream +struts +struts-menu +strutstestcase +studio +stxx +su +subpersistence +subshell +substance +suiterunner +surefire +swanysimon +swarmcache +swehacker +swiss +swt +synapticloop +systems +sysunit +tablelayout +tagalog +tagishauth +taglibrarydoc +taglibs +tagsoup +tambora +tane +tangram +tanukisoft +tanvd +tapestry +tclib +team +tec +tech +technology +tekgenesis +tel +temportalist +textarea +tf +th +thad +thaiopensource +thefireplace +tiffrenderer +tjdo +tk +tl +tmporb +to +today +tokyo +tomcat +tomcat-util +tonic +tools +top +toplink +torque +torque-gen +touchstone +town +toys +tqrg +tr +trade +traer +trail-taglib +tranql +travel +trove +trust +tui +turbine +tv +tw +tylerthrailkill +tyrex +tz +ua +uaihebert +ubique +ud +ug +uispec4j +uk +umple +uno +uno-jar +urbanophile +urlrewrite +us +utry +uy +uz +vast +vc +vdoclet +velocity +velocity-anakia +velocity-dvsl +velocity-tools +ventures +vet +vicJuspay +video +view_inspector +village +vin +vinccool96 +vip +vision +vn +vogle +vontikov +vu +wadi +waena +wang +web +webmacro +website +webtest +weixinkeji +werken-xpath +werkflow +werkz +westhawk +wf +wg +which +wicket +wiki +win +wind57 +woodstox +work +works +world +wrapper +ws +ws-commons +ws-commons-java5 +ws-commons-util +ws-scout +wsdl4j +wsrf +wss4j +wstx +wtf +wurfl +wutka +xajile +xalan +xbean +xdoclet +xdoclet-plugins +xerces +xercesjarv +xfire +xfire-root +xin +xjavadoc +xml-apis +xml-resolver +xml-security +xmlbeans +xmldb +xmlenc +xmlmind +xmlpull +xmlrpc +xmlrpc-helma +xmlunit +xmlwise +xmlwriter +xom +xpp3 +xsddoc +xsdlib +xstream +xtc +xtiff-jai +xxl +xyz +yakworks +yan +yanjun +ymsg +yom +za +zip +zm +zone +zw diff --git a/internal/handler/maven.go b/internal/handler/maven.go index 1e4e2eb..8fa170f 100644 --- a/internal/handler/maven.go +++ b/internal/handler/maven.go @@ -1,6 +1,8 @@ package handler import ( + "crypto/sha256" + "encoding/hex" "errors" "fmt" "net/http" @@ -35,12 +37,44 @@ func mavenRetentionKey(p *purl.PURL) (string, bool) { return retention.DefaultCanonical("maven")(p) } -// MavenHandler handles Maven repository protocol requests. +// MavenHandler handles Maven repository protocol requests. The default +// repository is served at /maven/ and falls back to the Gradle Plugin Portal; +// each named repository is served at /maven/{name}/ from its own upstream +// alone. type MavenHandler struct { proxy *Proxy upstreamURL string pluginPortalUpstreamURL string proxyURL string + repositories map[string]mavenSource +} + +// mavenSource is the repository a request is served from. +type mavenSource struct { + // name is the repository name, or "" for the default repository. + name string + upstreamURL string + // fallbackURL is tried when upstreamURL doesn't have a file. Only the + // default repository has one, the Gradle Plugin Portal. + fallbackURL string + // cachePrefix is prepended to cached filenames: empty for the default + // repository, {name}/{hash of upstreamURL}/ for a named one. + cachePrefix string +} + +// WithRepositories adds named repositories, served at /maven/{name}/. +func (h *MavenHandler) WithRepositories(repositories map[string]string) *MavenHandler { + h.repositories = make(map[string]mavenSource, len(repositories)) + for name, upstreamURL := range repositories { + upstreamURL = strings.TrimSuffix(upstreamURL, "/") + digest := sha256.Sum256([]byte(upstreamURL)) + h.repositories[name] = mavenSource{ + name: name, + upstreamURL: upstreamURL, + cachePrefix: name + "/" + hex.EncodeToString(digest[:mavenUpstreamHashBytes]) + "/", + } + } + return h } // NewMavenHandler creates a new Maven repository handler. @@ -79,30 +113,70 @@ func (h *MavenHandler) handleRequest(w http.ResponseWriter, r *http.Request) { return } + source := mavenSource{upstreamURL: h.upstreamURL, fallbackURL: h.pluginPortalUpstreamURL} + if name, rest, ok := strings.Cut(urlPath, "/"); ok { + if named, found := h.repositories[name]; found { + source = named + urlPath = rest + } + } else if _, found := h.repositories[urlPath]; found { + http.NotFound(w, r) + return + } + if urlPath == "" { + http.NotFound(w, r) + return + } + filename := path.Base(urlPath) if h.isVersionListingFile(filename) { - h.handleMetadata(w, r, urlPath) + h.handleMetadata(w, r, source, urlPath) return } if h.isArtifactFile(filename) || h.isChecksumFile(filename) { - h.handleDownload(w, r, urlPath) + h.handleDownload(w, r, source, urlPath) return } // Proxy everything else (directory listings etc.) - h.proxyUpstream(w, r) + h.proxyUpstream(w, r, source, urlPath) } -func (h *MavenHandler) handleMetadata(w http.ResponseWriter, r *http.Request, urlPath string) { - cacheKey := strings.ReplaceAll(urlPath, "/", "_") - upstreamURL := fmt.Sprintf("%s/%s", h.upstreamURL, urlPath) +// metadataCacheKey is the metadata cache key of urlPath. The default +// repository keeps its original keys. A named repository's keys are hashed +// from its name, upstream URL and the path, so repositories never share +// entries and repointing one drops what it cached. +func (s mavenSource) metadataCacheKey(urlPath string) string { + if s.name == "" { + return strings.ReplaceAll(urlPath, "/", "_") + } + digest := sha256.Sum256([]byte(s.name + "\x00" + s.upstreamURL + "\x00" + urlPath)) + return "repo_" + hex.EncodeToString(digest[:]) +} + +// cacheFilename is the name a file is cached under. A named repository's +// files are prefixed with its name and a hash of its upstream URL, so the +// same coordinates in two repositories, which may hold different bytes, are +// cached apart, and repointing a repository stops serving what the old +// upstream sent. +func (s mavenSource) cacheFilename(filename string) string { + return s.cachePrefix + filename +} + +// mavenUpstreamHashBytes is how much of the upstream URL's hash a named +// repository's cache filenames carry: enough to tell its upstreams apart. +const mavenUpstreamHashBytes = 8 + +func (h *MavenHandler) handleMetadata(w http.ResponseWriter, r *http.Request, source mavenSource, urlPath string) { + cacheKey := source.metadataCacheKey(urlPath) + upstreamURL := fmt.Sprintf("%s/%s", source.upstreamURL, urlPath) body, contentType, err := h.proxy.FetchOrCacheMetadata(r.Context(), "maven", cacheKey, upstreamURL, "*/*") - if err != nil { + if err != nil && source.fallbackURL != "" { if errors.Is(err, ErrUpstreamNotFound) { - pluginPortalURL := fmt.Sprintf("%s/%s", h.pluginPortalUpstreamURL, urlPath) + pluginPortalURL := fmt.Sprintf("%s/%s", source.fallbackURL, urlPath) h.proxy.Logger.Info("maven metadata unavailable in primary upstream, trying Gradle Plugin Portal", "path", urlPath) body, contentType, err = h.proxy.FetchOrCacheMetadata(r.Context(), "maven", cacheKey, pluginPortalURL, "*/*") @@ -125,30 +199,31 @@ func (h *MavenHandler) handleMetadata(w http.ResponseWriter, r *http.Request, ur } // handleDownload serves an artifact file, fetching and caching from upstream if needed. -func (h *MavenHandler) handleDownload(w http.ResponseWriter, r *http.Request, urlPath string) { +func (h *MavenHandler) handleDownload(w http.ResponseWriter, r *http.Request, source mavenSource, urlPath string) { // Parse Maven path: group/artifact/version/filename // e.g., com/google/guava/guava/32.1.3-jre/guava-32.1.3-jre.jar group, artifact, version, filename := h.parsePath(urlPath) if artifact == "" { - h.proxyUpstream(w, r) + h.proxyUpstream(w, r, source, urlPath) return } // Maven uses group:artifact as the package name name := fmt.Sprintf("%s:%s", group, artifact) - h.proxy.Logger.Info("maven download request", + h.proxy.Logger.Info("maven download request", "repository", source.name, "group", group, "artifact", artifact, "version", version, "filename", filename) - upstreamURL := fmt.Sprintf("%s/%s", h.upstreamURL, urlPath) + upstreamURL := fmt.Sprintf("%s/%s", source.upstreamURL, urlPath) + cacheFilename := source.cacheFilename(filename) - result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "maven", name, version, filename, upstreamURL) - if err != nil { + result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "maven", name, version, cacheFilename, upstreamURL) + if err != nil && source.fallbackURL != "" { if errors.Is(err, ErrUpstreamNotFound) { - pluginPortalURL := fmt.Sprintf("%s/%s", h.pluginPortalUpstreamURL, urlPath) + pluginPortalURL := fmt.Sprintf("%s/%s", source.fallbackURL, urlPath) h.proxy.Logger.Info("maven artifact not found in primary upstream, trying Gradle Plugin Portal", "group", group, "artifact", artifact, "version", version, "filename", filename) - result, err = h.proxy.GetOrFetchArtifactFromURL(r.Context(), "maven", name, version, filename, pluginPortalURL) + result, err = h.proxy.GetOrFetchArtifactFromURL(r.Context(), "maven", name, version, cacheFilename, pluginPortalURL) } } if err != nil { @@ -212,7 +287,7 @@ func (h *MavenHandler) isChecksumFile(filename string) bool { return false } -// proxyUpstream forwards a request to Maven Central without caching. -func (h *MavenHandler) proxyUpstream(w http.ResponseWriter, r *http.Request) { - h.proxy.ProxyUpstream(w, r, h.upstreamURL+r.URL.Path, nil) +// proxyUpstream forwards a request to the source's upstream without caching. +func (h *MavenHandler) proxyUpstream(w http.ResponseWriter, r *http.Request, source mavenSource, urlPath string) { + h.proxy.ProxyUpstream(w, r, source.upstreamURL+"/"+urlPath, nil) } diff --git a/internal/handler/maven_repositories_test.go b/internal/handler/maven_repositories_test.go new file mode 100644 index 0000000..8272f37 --- /dev/null +++ b/internal/handler/maven_repositories_test.go @@ -0,0 +1,139 @@ +package handler + +import ( + "io" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + + "github.com/git-pkgs/registries/fetch" +) + +// mavenRepoUpstream serves label-specific bytes for every path it knows and +// counts its requests. +func mavenRepoUpstream(t *testing.T, label string, known ...string) (*httptest.Server, *atomic.Int32) { + t.Helper() + var requests atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests.Add(1) + for _, p := range known { + if r.URL.Path == p { + _, _ = io.WriteString(w, label+" "+r.URL.Path) + return + } + } + http.NotFound(w, r) + })) + t.Cleanup(srv.Close) + return srv, &requests +} + +func newMavenRepositoriesHandler(t *testing.T, central, portal, grails *httptest.Server) http.Handler { + t.Helper() + proxy, _, store, _ := setupTestProxy(t) + store.seekable = true + proxy.HTTPClient = central.Client() + fetcher := fetch.NewFetcher(fetch.WithHTTPClient(central.Client()), fetch.WithMaxRetries(0)) + proxy.Fetcher = fetcher + t.Cleanup(func() { _ = fetcher.Close() }) + return NewMavenHandler(proxy, "http://localhost", central.URL, portal.URL). + WithRepositories(map[string]string{"grails": grails.URL + "/restricted/"}).Routes() +} + +func getMaven(h http.Handler, path string) *httptest.ResponseRecorder { + w := httptest.NewRecorder() + h.ServeHTTP(w, httptest.NewRequest(http.MethodGet, path, nil)) + return w +} + +const ( + mavenJarPath = "/org/grails/grails-core/6.0.0/grails-core-6.0.0.jar" + mavenMetadataPath = "/org/grails/grails-core/maven-metadata.xml" +) + +func TestMavenRepositoriesAreServedFromTheirOwnUpstream(t *testing.T) { + central, _ := mavenRepoUpstream(t, "central", mavenJarPath, mavenMetadataPath) + portal, _ := mavenRepoUpstream(t, "portal") + grails, _ := mavenRepoUpstream(t, "grails", "/restricted"+mavenJarPath, "/restricted"+mavenMetadataPath) + h := newMavenRepositoriesHandler(t, central, portal, grails) + + // The same coordinates in both repositories are cached apart. + for range 2 { + if w := getMaven(h, "/grails"+mavenJarPath); w.Code != http.StatusOK || w.Body.String() != "grails /restricted"+mavenJarPath { + t.Fatalf("named jar: status = %d, body = %q", w.Code, w.Body) + } + if w := getMaven(h, mavenJarPath); w.Code != http.StatusOK || w.Body.String() != "central "+mavenJarPath { + t.Fatalf("default jar: status = %d, body = %q", w.Code, w.Body) + } + } + + if w := getMaven(h, "/grails"+mavenMetadataPath); w.Code != http.StatusOK || !strings.HasPrefix(w.Body.String(), "grails ") { + t.Errorf("named metadata: status = %d, body = %q", w.Code, w.Body) + } + if w := getMaven(h, mavenMetadataPath); w.Code != http.StatusOK || !strings.HasPrefix(w.Body.String(), "central ") { + t.Errorf("default metadata: status = %d, body = %q", w.Code, w.Body) + } +} + +func TestMavenRepositoriesDoNotFallBackToThePluginPortal(t *testing.T) { + const pluginPath = "/org/example/plugin/1.0/plugin-1.0.jar" + central, _ := mavenRepoUpstream(t, "central") + portal, portalRequests := mavenRepoUpstream(t, "portal", pluginPath, "/org/example/plugin/maven-metadata.xml") + grails, _ := mavenRepoUpstream(t, "grails") + h := newMavenRepositoriesHandler(t, central, portal, grails) + + for _, path := range []string{"/grails" + pluginPath, "/grails/org/example/plugin/maven-metadata.xml"} { + if w := getMaven(h, path); w.Code != http.StatusNotFound { + t.Errorf("GET %s status = %d, want 404", path, w.Code) + } + } + if n := portalRequests.Load(); n != 0 { + t.Errorf("named repository asked the plugin portal %d times", n) + } + if w := getMaven(h, pluginPath); w.Code != http.StatusOK || !strings.HasPrefix(w.Body.String(), "portal ") { + t.Errorf("default repository: status = %d, body = %q, want the plugin portal's copy", w.Code, w.Body) + } +} + +func TestMavenRepositoryRootIsNotFound(t *testing.T) { + central, centralRequests := mavenRepoUpstream(t, "central") + portal, _ := mavenRepoUpstream(t, "portal") + grails, _ := mavenRepoUpstream(t, "grails") + h := newMavenRepositoriesHandler(t, central, portal, grails) + + for _, path := range []string{"/grails", "/grails/"} { + if w := getMaven(h, path); w.Code != http.StatusNotFound { + t.Errorf("GET %s status = %d, want 404", path, w.Code) + } + } + if n := centralRequests.Load(); n != 0 { + t.Errorf("repository root went to the default upstream %d times", n) + } +} + +func TestMavenRepositoryRepointedServesTheNewUpstream(t *testing.T) { + central, _ := mavenRepoUpstream(t, "central") + portal, _ := mavenRepoUpstream(t, "portal") + staging, _ := mavenRepoUpstream(t, "staging", mavenJarPath) + production, _ := mavenRepoUpstream(t, "production", mavenJarPath) + + proxy, _, store, _ := setupTestProxy(t) + store.seekable = true + proxy.HTTPClient = central.Client() + fetcher := fetch.NewFetcher(fetch.WithHTTPClient(central.Client()), fetch.WithMaxRetries(0)) + proxy.Fetcher = fetcher + t.Cleanup(func() { _ = fetcher.Close() }) + + for _, upstream := range []struct { + srv *httptest.Server + label string + }{{staging, "staging"}, {production, "production"}} { + h := NewMavenHandler(proxy, "http://localhost", central.URL, portal.URL). + WithRepositories(map[string]string{"grails": upstream.srv.URL}).Routes() + if w := getMaven(h, "/grails"+mavenJarPath); w.Code != http.StatusOK || w.Body.String() != upstream.label+" "+mavenJarPath { + t.Errorf("%s: status = %d, body = %q", upstream.label, w.Code, w.Body) + } + } +} diff --git a/internal/server/server.go b/internal/server/server.go index 36ece1c..b8a7a8b 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -435,7 +435,7 @@ func (s *Server) mountProtocolHandlers(r chi.Router, proxy *handler.Proxy) { s.cfg.BaseURL, s.cfg.Upstream.Maven, s.cfg.Upstream.GradlePluginPortal, - ) + ).WithRepositories(s.cfg.Upstream.MavenRepositories) gradleHandler := handler.NewGradleBuildCacheHandler(proxy) nugetHandler := handler.NewNuGetHandlerWithUpstreams( proxy, From ed77e40816af2e4274186b0b2ac4039cc64cd08d Mon Sep 17 00:00:00 2001 From: simon chrzanowski Date: Wed, 21 Oct 2026 12:12:05 +0200 Subject: [PATCH 333/633] Log rewritten npm tarball URLs once per document (#538) The npm handler logged a debug line for every version whose tarball URL it rewrote. Packages with hundreds of versions produced hundreds of lines per packument, and with cooldown enabled, where the rewrite cache is bypassed, again on every request. It now logs one line per document with the number of rewritten URLs. --- internal/handler/npm.go | 36 +++++++++++++++++++++++++----------- internal/handler/npm_test.go | 20 ++++++++++++++++++++ 2 files changed, 45 insertions(+), 11 deletions(-) diff --git a/internal/handler/npm.go b/internal/handler/npm.go index 6530f5a..a874c61 100644 --- a/internal/handler/npm.go +++ b/internal/handler/npm.go @@ -444,7 +444,9 @@ func (h *NPMHandler) rewriteMetadataWithLocal(packageName string, body []byte, l return nil, nil, ErrVersionDenied } var out bytes.Buffer - h.writeNPMVersion(&out, packageName, version, body) + if h.writeNPMVersion(&out, packageName, version, body) { + h.logTarballRewrites(packageName, 1) + } return out.Bytes(), []string{version}, nil } if len(h.proxy.Denylist.Versions(canonicalPackagePURL("npm", packageName))) != 0 { @@ -596,6 +598,7 @@ func (d *npmPackument) write(writeValue func(out *bytes.Buffer, i int) error) ([ func (h *NPMHandler) writeNPMVersions(out *bytes.Buffer, packageName string, versionsRaw []byte, keep map[string]any, local []npmLocalEntry) error { out.WriteByte('{') first := true + rewritten := 0 err := forEachJSONMember(versionsRaw, func(m jsonMember) error { version, err := jsonKey(m.key(versionsRaw)) if err != nil { @@ -610,9 +613,12 @@ func (h *NPMHandler) writeNPMVersions(out *bytes.Buffer, packageName string, ver first = false out.Write(m.key(versionsRaw)) out.WriteByte(':') - h.writeNPMVersion(out, packageName, version, m.value(versionsRaw)) + if h.writeNPMVersion(out, packageName, version, m.value(versionsRaw)) { + rewritten++ + } return nil }) + h.logTarballRewrites(packageName, rewritten) for _, e := range local { if !first { out.WriteByte(',') @@ -628,38 +634,46 @@ func (h *NPMHandler) writeNPMVersions(out *bytes.Buffer, packageName string, ver } // writeNPMVersion writes one version entry with its dist.tarball pointing at -// this proxy. An entry without a string tarball is written unchanged. -func (h *NPMHandler) writeNPMVersion(out *bytes.Buffer, packageName, version string, raw []byte) { +// this proxy, and reports whether it rewrote the tarball. An entry without a +// string tarball is written unchanged. +func (h *NPMHandler) writeNPMVersion(out *bytes.Buffer, packageName, version string, raw []byte) bool { dist, ok, err := findJSONMember(raw, "dist") if err != nil || !ok { out.Write(raw) - return + return false } distRaw := dist.value(raw) tarball, ok, err := findJSONMember(distRaw, "tarball") if err != nil || !ok || distRaw[tarball.valStart] != '"' { out.Write(raw) - return + return false } var oldTarball string if err := json.Unmarshal(tarball.value(distRaw), &oldTarball); err != nil { out.Write(raw) - return + return false } newTarball := h.proxyTarballURL(packageName, version, oldTarball) encoded, err := json.Marshal(newTarball) if err != nil { out.Write(raw) - return + return false } out.Write(raw[:dist.valStart+tarball.valStart]) out.Write(encoded) out.Write(raw[dist.valStart+tarball.valEnd:]) + return true +} - h.proxy.Logger.Debug("rewrote tarball URL", - "package", packageName, "version", version, - "old", oldTarball, "new", newTarball) +// logTarballRewrites logs once per document how many tarball URLs it +// rewrote. A line per version flooded debug logs: packages such as eslint +// list hundreds of versions, and with cooldown on every request rewrites +// the document again. +func (h *NPMHandler) logTarballRewrites(packageName string, count int) { + if count > 0 { + h.proxy.Logger.Debug("rewrote tarball URLs", "package", packageName, "count", count) + } } // writeFilteredJSONObjectWith keeps members whose decoded keys match keep, diff --git a/internal/handler/npm_test.go b/internal/handler/npm_test.go index f71c59c..c8a4eb3 100644 --- a/internal/handler/npm_test.go +++ b/internal/handler/npm_test.go @@ -174,6 +174,26 @@ func TestNPMRewriteMetadata(t *testing.T) { } } +func TestNPMRewriteMetadataLogsOncePerDocument(t *testing.T) { + var logs bytes.Buffer + proxy := testProxy() + proxy.Logger = slog.New(slog.NewTextHandler(&logs, &slog.HandlerOptions{Level: slog.LevelDebug})) + h := &NPMHandler{proxy: proxy, proxyURL: "http://localhost:8080"} + + input := `{"name":"lodash","versions":{ + "4.17.19":{"dist":{"tarball":"https://registry.npmjs.org/lodash/-/lodash-4.17.19.tgz"}}, + "4.17.20":{"dist":{"tarball":"https://registry.npmjs.org/lodash/-/lodash-4.17.20.tgz"}}, + "4.17.21":{"dist":{"tarball":"https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz"}}}}` + if _, err := h.rewriteMetadata("lodash", []byte(input)); err != nil { + t.Fatalf("rewriteMetadata failed: %v", err) + } + + lines := strings.Split(strings.TrimSpace(logs.String()), "\n") + if len(lines) != 1 || !strings.Contains(lines[0], `msg="rewrote tarball URLs"`) || !strings.Contains(lines[0], "count=3") { + t.Errorf("logs = %q, want one line counting 3 rewrites", logs.String()) + } +} + func TestNPMRewriteMetadataScopedPackage(t *testing.T) { h := &NPMHandler{ proxy: testProxy(), From 994a9368d1086283314fbd09f653f26774de62dd Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Wed, 21 Oct 2026 12:23:33 +0200 Subject: [PATCH 334/633] Keep the stale warning on filtered npm version metadata With cooldown or a version denylist, /pkg/ reads the filtered packument through handleFilteredNPMVersionMetadata, which dropped the stale flag. --- internal/handler/npm.go | 7 +++-- internal/handler/stale_metadata_test.go | 42 +++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 3 deletions(-) diff --git a/internal/handler/npm.go b/internal/handler/npm.go index 4e00851..af7bc2e 100644 --- a/internal/handler/npm.go +++ b/internal/handler/npm.go @@ -298,8 +298,8 @@ func (h *NPMHandler) packumentAccept() string { // filtered packument so cooldown and dist-tags match the root metadata response. func (h *NPMHandler) handleFilteredNPMVersionMetadata(w http.ResponseWriter, r *http.Request, packageName, specifier, accept, registryURL, cacheKey, metadataCacheKey string) { _, packumentPath := npmMetadataPath(packageName) - packument, _, err := h.proxy.FetchOrCacheMetadata(r.Context(), "npm", cacheKey, registryURL+"/"+packumentPath, accept) - if err != nil { + upstream := h.proxy.fetchMetadataDocument(r.Context(), "npm", cacheKey, registryURL+"/"+packumentPath, accept) + if err := upstream.err; err != nil { if errors.Is(err, ErrUpstreamNotFound) { JSONError(w, http.StatusNotFound, ErrPackageNotFound.Error()) return @@ -312,7 +312,7 @@ func (h *NPMHandler) handleFilteredNPMVersionMetadata(w http.ResponseWriter, r * return } - rewritten, err := h.proxy.cachedRewrite(r.Context(), "npm", h.proxyURL, metadataCacheKey, packument, "", func(b []byte) ([]byte, []string, error) { + rewritten, err := h.proxy.cachedRewrite(r.Context(), "npm", h.proxyURL, metadataCacheKey, upstream.body, "", func(b []byte) ([]byte, []string, error) { out, err := h.rewriteNPMVersionFromPackument(packageName, specifier, b) return out, nil, err }) @@ -337,6 +337,7 @@ func (h *NPMHandler) handleFilteredNPMVersionMetadata(w http.ResponseWriter, r * return } + markStale(w, upstream.stale) w.Header().Set(headerContentType, contentTypeJSON) w.WriteHeader(http.StatusOK) _, _ = w.Write(rewritten.body) diff --git a/internal/handler/stale_metadata_test.go b/internal/handler/stale_metadata_test.go index 5cd8d6d..1fd8238 100644 --- a/internal/handler/stale_metadata_test.go +++ b/internal/handler/stale_metadata_test.go @@ -91,6 +91,48 @@ func TestNPMMetadataAfterUpstreamAnswer(t *testing.T) { } } +// With a denylist entry for the package, version requests read the filtered +// packument through handleFilteredNPMVersionMetadata. +func TestNPMFilteredVersionMetadataAfterUpstreamAnswer(t *testing.T) { + tests := []struct { + name string + status int + wantStatus int + wantStale bool + }{ + {name: "upstream error", status: http.StatusServiceUnavailable, wantStatus: http.StatusOK, wantStale: true}, + {name: "unpublished", status: http.StatusNotFound, wantStatus: http.StatusNotFound}, + {name: "credentials rejected", status: http.StatusUnauthorized, wantStatus: http.StatusBadGateway}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + upstream := npmUpstreamThen(t, func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(tt.status) }) + proxy, _, _, _ := setupTestProxy(t) + proxy.CacheMetadata = true + proxy.MetadataTTL = time.Millisecond + proxy.HTTPClient = upstream.Client() + setTestDenylist(t, proxy, "pkg:npm/left-pad@2.0.0") + h := NewNPMHandler(proxy, "http://proxy.test", upstream.URL+"/").Routes() + + first := httptest.NewRecorder() + h.ServeHTTP(first, httptest.NewRequest(http.MethodGet, "/left-pad", nil)) + if first.Code != http.StatusOK { + t.Fatalf("first request: status = %d", first.Code) + } + time.Sleep(5 * time.Millisecond) + + w := httptest.NewRecorder() + h.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/left-pad/1.0.0", nil)) + if w.Code != tt.wantStatus { + t.Fatalf("status = %d, want %d, body %s", w.Code, tt.wantStatus, w.Body) + } + if stale := w.Header().Get("Warning") == staleWarning; stale != tt.wantStale { + t.Errorf("Warning = %q, want stale %v", w.Header().Get("Warning"), tt.wantStale) + } + }) + } +} + func TestProxyCachedPassesDefiniteAnswersThrough(t *testing.T) { for _, status := range []int{http.StatusNotFound, http.StatusUnauthorized, http.StatusForbidden} { t.Run(http.StatusText(status), func(t *testing.T) { From d13b8be0ae9a583c7a037fc866622d3f1f69a3f0 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Tue, 20 Oct 2026 19:39:14 +0200 Subject: [PATCH 335/633] Host generic files and share one push path across hosted ecosystems Hosted ecosystems are now entries in one table: each defines its package names, how the stored filename is chosen, how the manifest is read and which namespaces are served from pushes alone. npm and Composer move there unchanged, and generic is added as a third entry. Generic files are pushed to /api/hosted/generic/{name}/{version}/{file} and served without a token from /generic/hosted/{name}/{version}/{file}, with the sha256 as ETag. They have no manifest, a version can hold several files, and they never come from an upstream. "hosted" is reserved as an upstream.generic name. DELETE with a file removes only that file. For all hosted ecosystems: - A second push of the same file is refused with 409. Clients pin pushed files by version and often by hash, so a published file is no longer silently replaced. Delete it first to push it again. - PUT takes an optional ?sha256= and refuses bytes that don't match. proxy hosted push always sends it. Versions of generic files carry no version-level integrity, since it could only describe one of their files; each file is checked against its own sha256. Refs #133, #134, #535 --- cmd/proxy/hosted.go | 47 +++++-- cmd/proxy/hosted_test.go | 36 ++++- docs/configuration.md | 33 ++++- docs/swagger/docs.go | 18 ++- docs/swagger/swagger.json | 18 ++- internal/config/config.go | 7 + internal/config/config_test.go | 7 + internal/database/local_artifacts.go | 27 ++-- internal/database/local_artifacts_test.go | 4 +- internal/handler/composer_hosted_test.go | 4 +- internal/handler/generic.go | 41 ++++++ internal/handler/generic_test.go | 1 + internal/handler/hosted.go | 164 +++++++++++++++++----- internal/handler/hosted_manifest.go | 13 +- internal/handler/hosted_test.go | 33 ++++- internal/handler/npm_hosted_test.go | 4 +- internal/server/hosted_api.go | 95 ++++++++++--- internal/server/hosted_api_test.go | 122 ++++++++++++++++ 18 files changed, 569 insertions(+), 105 deletions(-) diff --git a/cmd/proxy/hosted.go b/cmd/proxy/hosted.go index bbc947d..ec61b3a 100644 --- a/cmd/proxy/hosted.go +++ b/cmd/proxy/hosted.go @@ -1,6 +1,8 @@ package main import ( + "crypto/sha256" + "encoding/hex" "encoding/json" "errors" "flag" @@ -9,9 +11,12 @@ import ( "net/http" "net/url" "os" + "path/filepath" "strings" "text/tabwriter" "time" + + "github.com/git-pkgs/proxy/internal/handler" ) const ( @@ -21,7 +26,7 @@ const ( pushArgCount = 4 // ecosystem, name, version, file listArgCount = 2 // ecosystem, name - deleteArgCount = 3 // ecosystem, name, version + deleteArgCount = 3 // ecosystem, name, version[, file] tablePadding = 2 ) @@ -69,15 +74,17 @@ func newHostedCommand(name, usage, summary string, getenv func(string) string) h return cmd } -// parse parses flags and checks for exactly want positional arguments. -func (c hostedCommand) parse(args []string, want int) ([]string, error) { +// parse parses flags and checks for exactly want positional arguments, or +// one more when optional is set. +func (c hostedCommand) parse(args []string, want int, optional ...bool) ([]string, error) { if err := c.flags.Parse(args); err != nil { if errors.Is(err, flag.ErrHelp) { c.flags.Usage() } return nil, fmt.Errorf("%w\nusage: %s", err, c.usage) } - if c.flags.NArg() != want { + n := c.flags.NArg() + if n != want && (len(optional) == 0 || !optional[0] || n != want+1) { return nil, fmt.Errorf("usage: %s", c.usage) } return c.flags.Args(), nil @@ -121,7 +128,22 @@ func runPush(args []string, getenv func(string) string, stdout io.Writer, client } defer func() { _ = file.Close() }() - req, err := http.NewRequest(http.MethodPut, cmd.endpoint(ecosystem, name, version), file) + // The proxy checks the upload against this, so a truncated or corrupted + // transfer is refused rather than stored. + digest := sha256.New() + if _, err := io.Copy(digest, file); err != nil { + return err + } + if _, err := file.Seek(0, io.SeekStart); err != nil { + return err + } + + segments := []string{version} + if handler.HostedFileInPath(ecosystem) { + segments = append(segments, filepath.Base(path)) + } + endpoint := cmd.endpoint(ecosystem, name, segments...) + "?sha256=" + hex.EncodeToString(digest.Sum(nil)) + req, err := http.NewRequest(http.MethodPut, endpoint, file) if err != nil { return err } @@ -161,8 +183,9 @@ func runList(args []string, getenv func(string) string, stdout io.Writer, client } func runDelete(args []string, getenv func(string) string, stdout io.Writer, client *http.Client) error { - cmd := newHostedCommand("delete", "proxy hosted delete [-url URL] ", "Delete a pushed version", getenv) - positional, err := cmd.parse(args, deleteArgCount) + cmd := newHostedCommand("delete", "proxy hosted delete [-url URL] [file]", + "Delete a pushed version, or one generic file of it", getenv) + positional, err := cmd.parse(args, deleteArgCount, true) if err != nil { return err } @@ -170,7 +193,7 @@ func runDelete(args []string, getenv func(string) string, stdout io.Writer, clie if err != nil { return err } - req, err := http.NewRequest(http.MethodDelete, cmd.endpoint(positional[0], positional[1], positional[2]), nil) + req, err := http.NewRequest(http.MethodDelete, cmd.endpoint(positional[0], positional[1], positional[2:]...), nil) if err != nil { return err } @@ -179,7 +202,11 @@ func runDelete(args []string, getenv func(string) string, stdout io.Writer, clie if err := doHosted(client, req, http.StatusNoContent, nil); err != nil { return err } - _, err = fmt.Fprintf(stdout, "deleted %s %s@%s\n", positional[0], positional[1], positional[2]) + deleted := positional[1] + "@" + positional[2] + if len(positional) > deleteArgCount { + deleted += " " + positional[deleteArgCount] + } + _, err = fmt.Fprintf(stdout, "deleted %s %s\n", positional[0], deleted) return err } @@ -210,7 +237,7 @@ func doHosted(client *http.Client, req *http.Request, wantStatus int, out any) e const hostedUsage = "usage: proxy hosted [flags] ...\n\n" + " push Push a package archive to the proxy\n" + " list List pushed package archives\n" + - " delete Delete a pushed version" + " delete Delete a pushed version, or one generic file of it" type hostedSubcommand func([]string, func(string) string, io.Writer, *http.Client) error diff --git a/cmd/proxy/hosted_test.go b/cmd/proxy/hosted_test.go index fef7160..34f9a7b 100644 --- a/cmd/proxy/hosted_test.go +++ b/cmd/proxy/hosted_test.go @@ -62,7 +62,9 @@ func TestRunPush(t *testing.T) { if recorded.method != http.MethodPut || recorded.path != "/api/hosted/npm/@types/node/20.1.0" { t.Errorf("request = %s %s", recorded.method, recorded.path) } - if recorded.authorization != "Bearer tok" || recorded.body != "tarball" || recorded.query != "" { + // sha256 of "tarball", so the proxy can check the upload. + const tarballSHA256 = "db4b4d0d1cb480bf9aeea253771c00febe627f236765fa37d6a5614f079a3aa0" + if recorded.authorization != "Bearer tok" || recorded.body != "tarball" || recorded.query != "sha256="+tarballSHA256 { t.Errorf("authorization/body/query = %q/%q/%q", recorded.authorization, recorded.body, recorded.query) } if out := stdout.String(); !strings.Contains(out, "node-20.1.0.tgz") || !strings.Contains(out, "abc123") { @@ -70,6 +72,21 @@ func TestRunPush(t *testing.T) { } } +func TestRunPushGenericNamesTheFile(t *testing.T) { + srv, recorded := hostedTestServer(t, http.StatusCreated, + `{"ecosystem":"generic","name":"tools","version":"1.2.0","filename":"tool-linux-arm64.tar.gz","sha256":"abc123","size":7}`) + archive := writeArchive(t, "tool-linux-arm64.tar.gz", "tarball") + + err := runPush([]string{"-url", srv.URL, "generic", "tools", "1.2.0", archive}, + hostedEnv(map[string]string{"PROXY_HOSTED_TOKEN": "tok"}), io.Discard, srv.Client()) + if err != nil { + t.Fatalf("runPush: %v", err) + } + if recorded.path != "/api/hosted/generic/tools/1.2.0/tool-linux-arm64.tar.gz" { + t.Errorf("path = %q, want the file name appended", recorded.path) + } +} + func TestRunPushErrors(t *testing.T) { srv, _ := hostedTestServer(t, http.StatusBadRequest, `{"code":"BAD_REQUEST","message":"pushing packages is not supported for this ecosystem: cargo"}`) archive := writeArchive(t, "serde-1.0.200.crate", "crate") @@ -141,6 +158,23 @@ func TestRunDelete(t *testing.T) { } } +func TestRunDeleteOneFile(t *testing.T) { + srv, recorded := hostedTestServer(t, http.StatusNoContent, "") + var stdout bytes.Buffer + + err := runDelete([]string{"-url", srv.URL, "generic", "tools", "1.2.0", "tool-linux-arm64.tar.gz"}, + hostedEnv(map[string]string{"PROXY_HOSTED_TOKEN": "tok"}), &stdout, srv.Client()) + if err != nil { + t.Fatalf("runDelete: %v", err) + } + if recorded.path != "/api/hosted/generic/tools/1.2.0/tool-linux-arm64.tar.gz" { + t.Errorf("path = %q", recorded.path) + } + if out := stdout.String(); out != "deleted generic tools@1.2.0 tool-linux-arm64.tar.gz\n" { + t.Errorf("output = %q", out) + } +} + func TestRunHostedDispatchesSubcommands(t *testing.T) { srv, recorded := hostedTestServer(t, http.StatusOK, `{"ecosystem":"npm","name":"request","artifacts":[]}`) diff --git a/docs/configuration.md b/docs/configuration.md index 507cad5..a08ec4c 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1076,7 +1076,7 @@ Set to `"0"` to disable the timeout entirely (requests then rely only on the ser ## Hosted Packages -Operators can push a package archive into the proxy under a name and version. The proxy serves it in place of the upstream archive with the same version, or in addition to the upstream's versions. Use it to ship a patched build of a dependency without running a separate registry. +Operators can push a package archive into the proxy under a name and version. The proxy serves it in place of the upstream archive with the same version, or in addition to the upstream's versions. Use it to ship a patched build of a dependency without running a separate registry. Files that aren't packages, such as prebuilt binaries other builds download, can be pushed as [generic files](#generic-files). The `/api/hosted` endpoints are disabled by default. Because a pushed version replaces the upstream one for every client, enabling them requires a token: @@ -1093,7 +1093,7 @@ hosted: |--------|-------------|-------------| | `hosted.enabled` | `PROXY_HOSTED_ENABLED` | Register the `/api/hosted` endpoints (default `false`) | | `hosted.token` | `PROXY_HOSTED_TOKEN` | Bearer token required to push and delete; required when enabled, supports `${VAR}` | -| `hosted.max_upload_size` | `PROXY_HOSTED_MAX_UPLOAD_SIZE` | Maximum size of one pushed archive (default `100MB`) | +| `hosted.max_upload_size` | `PROXY_HOSTED_MAX_UPLOAD_SIZE` | Maximum size of one pushed archive or file (default `100MB`) | | `hosted.npm_scopes` | `PROXY_HOSTED_NPM_SCOPES` | npm scopes served from pushed packages only (comma-separated in the environment) | | `hosted.composer_vendors` | `PROXY_HOSTED_COMPOSER_VENDORS` | Composer vendors served from pushed packages only | @@ -1114,13 +1114,17 @@ hosted: ```bash curl -X PUT --data-binary @request-2.88.3.tgz \ -H "Authorization: Bearer $PROXY_HOSTED_TOKEN" \ - https://proxy.example.com/api/hosted/npm/request/2.88.3 + "https://proxy.example.com/api/hosted/npm/request/2.88.3?sha256=$(sha256sum request-2.88.3.tgz | cut -d' ' -f1)" ``` + The `sha256` query parameter is optional. When it is set, a push whose bytes don't match is refused with 400. `proxy hosted push` always sends it. + 3. Install it as usual, e.g. `npm install request@2.88.3`. The version is listed in the package metadata, so ranges like `^2.88.0` resolve to it too. 4. Check what is pushed: `proxy hosted list npm request` or `GET /api/hosted/npm/request`. 5. Remove it again: `proxy hosted delete npm request 2.88.3` or `DELETE /api/hosted/npm/request/2.88.3`. If the upstream has the version, clients get the upstream archive again. +A pushed file is never replaced: pushing the same version again answers 409. Clients that pinned the file by its hash would otherwise get different bytes under the same version. To push it again, delete it first. + ### Supported ecosystems Download handlers look archives up by the filename clients request, so a push is stored under that name: @@ -1129,10 +1133,11 @@ Download handlers look archives up by the filename clients request, so a push is |-----------|------|----------| | `npm` | `{name}` or `@{scope}/{name}` | `{name}-{version}.tgz`, without the scope | | `composer` | `{vendor}/{package}` | `{package}-{version}.zip` | +| `generic` | `{name}`, one path segment | named in the push path, see [Generic files](#generic-files) | Other ecosystems are not supported yet. -The archive must contain the package manifest: `package.json` in the top directory of an npm tarball, or `composer.json` at the root or in the single top directory of a Composer zip. Its name, and its version when it has one, must match the push. Otherwise the push is refused with 400. +npm and Composer archives must contain the package manifest: `package.json` in the top directory of an npm tarball, or `composer.json` at the root or in the single top directory of a Composer zip. Its name, and its version when it has one, must match the push. Otherwise the push is refused with 400. ### Package metadata @@ -1157,6 +1162,26 @@ For private packages: Other packages, including ones in other scopes and vendors, are proxied as usual. +### Generic files + +Files that aren't packages, such as a prebuilt binary that other builds download and check against a pinned sha256, are pushed to the `generic` ecosystem. They have no manifest; the name, version and filename come from the path, and a version can hold several files: + +```bash +proxy hosted push generic tools 1.2.0 ./tool-linux-arm64.tar.gz +# or +curl -X PUT --data-binary @tool-linux-arm64.tar.gz \ + -H "Authorization: Bearer $PROXY_HOSTED_TOKEN" \ + https://proxy.example.com/api/hosted/generic/tools/1.2.0/tool-linux-arm64.tar.gz +``` + +They are downloaded without a token from `/generic/hosted/{name}/{version}/{file}`, with the file's sha256 as `ETag`: + +```bash +curl -O https://proxy.example.com/generic/hosted/tools/1.2.0/tool-linux-arm64.tar.gz +``` + +Generic files never come from an upstream: anything not pushed answers 404, and `hosted` can't be used as an `upstream.generic` name. `proxy hosted list generic tools` lists all versions and files. `proxy hosted delete generic tools 1.2.0` removes a version with all its files, and `proxy hosted delete generic tools 1.2.0 tool-linux-arm64.tar.gz` removes only that file. + ### Storage and eviction Pushed archives are stored like cached ones but marked as local. The cache size limit and storage retention never remove them; only a delete does. Scanning, when enabled, checks pushed archives like fetched ones. diff --git a/docs/swagger/docs.go b/docs/swagger/docs.go index 7132ffb..e35a86f 100644 --- a/docs/swagger/docs.go +++ b/docs/swagger/docs.go @@ -230,7 +230,7 @@ const docTemplate = `{ "BearerAuth": [] } ], - "description": "Stores the request body as the archive for name@version. It is served in place of the upstream archive with the same version and is never evicted.", + "description": "Stores the request body as the archive for name@version. It is served in place of the upstream archive with the same version and is never evicted. Generic files are pushed to /api/hosted/generic/{name}/{version}/{file} and served at /generic/hosted/{name}/{version}/{file}. A file that was already pushed is refused with 409; delete it first to push it again.", "consumes": [ "application/octet-stream" ], @@ -244,7 +244,7 @@ const docTemplate = `{ "parameters": [ { "type": "string", - "description": "Ecosystem (npm or composer)", + "description": "Ecosystem (npm, composer or generic)", "name": "ecosystem", "in": "path", "required": true @@ -262,6 +262,12 @@ const docTemplate = `{ "name": "version", "in": "path", "required": true + }, + { + "type": "string", + "description": "Expected sha256 of the body, hex-encoded; a mismatch is refused", + "name": "sha256", + "in": "query" } ], "responses": { @@ -289,6 +295,12 @@ const docTemplate = `{ "$ref": "#/definitions/server.ErrorResponse" } }, + "409": { + "description": "Conflict", + "schema": { + "$ref": "#/definitions/server.ErrorResponse" + } + }, "413": { "description": "Request Entity Too Large", "schema": { @@ -303,7 +315,7 @@ const docTemplate = `{ "BearerAuth": [] } ], - "description": "Removes the archives pushed for name@version. If the upstream has the version, clients get the upstream archive again.", + "description": "Removes the archives pushed for name@version. If the upstream has the version, clients get the upstream archive again. For generic files, /api/hosted/generic/{name}/{version}/{file} removes only that file.", "tags": [ "hosted" ], diff --git a/docs/swagger/swagger.json b/docs/swagger/swagger.json index fcb4123..09e4f11 100644 --- a/docs/swagger/swagger.json +++ b/docs/swagger/swagger.json @@ -223,7 +223,7 @@ "BearerAuth": [] } ], - "description": "Stores the request body as the archive for name@version. It is served in place of the upstream archive with the same version and is never evicted.", + "description": "Stores the request body as the archive for name@version. It is served in place of the upstream archive with the same version and is never evicted. Generic files are pushed to /api/hosted/generic/{name}/{version}/{file} and served at /generic/hosted/{name}/{version}/{file}. A file that was already pushed is refused with 409; delete it first to push it again.", "consumes": [ "application/octet-stream" ], @@ -237,7 +237,7 @@ "parameters": [ { "type": "string", - "description": "Ecosystem (npm or composer)", + "description": "Ecosystem (npm, composer or generic)", "name": "ecosystem", "in": "path", "required": true @@ -255,6 +255,12 @@ "name": "version", "in": "path", "required": true + }, + { + "type": "string", + "description": "Expected sha256 of the body, hex-encoded; a mismatch is refused", + "name": "sha256", + "in": "query" } ], "responses": { @@ -282,6 +288,12 @@ "$ref": "#/definitions/server.ErrorResponse" } }, + "409": { + "description": "Conflict", + "schema": { + "$ref": "#/definitions/server.ErrorResponse" + } + }, "413": { "description": "Request Entity Too Large", "schema": { @@ -296,7 +308,7 @@ "BearerAuth": [] } ], - "description": "Removes the archives pushed for name@version. If the upstream has the version, clients get the upstream archive again.", + "description": "Removes the archives pushed for name@version. If the upstream has the version, clients get the upstream archive again. For generic files, /api/hosted/generic/{name}/{version}/{file} removes only that file.", "tags": [ "hosted" ], diff --git a/internal/config/config.go b/internal/config/config.go index e7851c9..e788f8d 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -840,6 +840,9 @@ func (u *UpstreamConfig) Validate() error { if err := validateOCIMirrors(u.OCIMirrors, u.OCI); err != nil { return err } + if _, reserved := u.Generic[HostedGenericRepository]; reserved { + return fmt.Errorf("invalid upstream.generic name %q: reserved for pushed files at /generic/%s/", HostedGenericRepository, HostedGenericRepository) + } if err := validateNamedUpstreams("upstream.generic", u.Generic); err != nil { return err } @@ -894,6 +897,10 @@ func (c *Config) validateUpstream() error { return nil } +// HostedGenericRepository is the /generic/{name}/ path that serves files +// pushed through /api/hosted/generic. No upstream.generic entry may use it. +const HostedGenericRepository = "hosted" + // CompileGenericArtifactPattern compiles an upstream.generic_artifacts pattern // so that it only matches a whole path, and checks it has the `version` and // `file` groups. diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 48ae1c7..b21c348 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -1455,6 +1455,13 @@ func TestValidateNamedUpstreams(t *testing.T) { }, wantErr: true, }, + { + name: "generic upstream name reserved for pushed files", + modify: func(cfg *Config) { + cfg.Upstream.Generic = map[string]string{"hosted": "https://example.com"} + }, + wantErr: true, + }, { name: "generic artifact patterns", modify: func(cfg *Config) { diff --git a/internal/database/local_artifacts.go b/internal/database/local_artifacts.go index 8319776..1bb391f 100644 --- a/internal/database/local_artifacts.go +++ b/internal/database/local_artifacts.go @@ -31,27 +31,32 @@ func (db *DB) GetLocalArtifacts(packagePURL string) ([]LocalArtifact, error) { return artifacts, nil } -// DeleteLocalArtifacts removes the pushed artifacts of a version and queues -// their stored objects for deletion in the same transaction, so a failed -// queue keeps the records instead of orphaning the objects. Artifacts cached -// from an upstream are left alone. It returns how many records it removed. -func (db *DB) DeleteLocalArtifacts(versionPURL string) (int, error) { +// DeleteLocalArtifacts removes the pushed artifacts of a version, or only the +// one named filename when it is set, and queues their stored objects for +// deletion in the same transaction, so a failed queue keeps the records +// instead of orphaning the objects. Artifacts cached from an upstream are left +// alone. It returns how many records it removed. +func (db *DB) DeleteLocalArtifacts(versionPURL, filename string) (int, error) { tx, err := db.Beginx() if err != nil { return 0, err } defer func() { _ = tx.Rollback() }() + where := "version_purl = ? AND source = 'local'" + args := []any{versionPURL} + if filename != "" { + where += " AND filename = ?" + args = append(args, filename) + } + var paths []string - selectQuery := db.Rebind(` - SELECT storage_path FROM artifacts - WHERE version_purl = ? AND source = 'local' AND storage_path IS NOT NULL - `) - if err := tx.Select(&paths, selectQuery, versionPURL); err != nil { + selectQuery := db.Rebind(`SELECT storage_path FROM artifacts WHERE ` + where + ` AND storage_path IS NOT NULL`) + if err := tx.Select(&paths, selectQuery, args...); err != nil { return 0, fmt.Errorf("reading local artifacts: %w", err) } - res, err := tx.Exec(db.Rebind(`DELETE FROM artifacts WHERE version_purl = ? AND source = 'local'`), versionPURL) + res, err := tx.Exec(db.Rebind(`DELETE FROM artifacts WHERE `+where), args...) if err != nil { return 0, fmt.Errorf("deleting local artifacts: %w", err) } diff --git a/internal/database/local_artifacts_test.go b/internal/database/local_artifacts_test.go index 81d5ec8..02335da 100644 --- a/internal/database/local_artifacts_test.go +++ b/internal/database/local_artifacts_test.go @@ -183,7 +183,7 @@ func TestDeleteLocalArtifacts(t *testing.T) { seedSourcedArtifact(t, db, "express", SourceUpstream, 100, time.Now()) const pushedPath = "npm/request/1.0.0/request-1.0.0.tgz" - deleted, err := db.DeleteLocalArtifacts("pkg:npm/request@1.0.0") + deleted, err := db.DeleteLocalArtifacts("pkg:npm/request@1.0.0", "") if err != nil || deleted != 1 { t.Fatalf("DeleteLocalArtifacts(request) = %d, %v, want 1, nil", deleted, err) } @@ -196,7 +196,7 @@ func TestDeleteLocalArtifacts(t *testing.T) { t.Errorf("queued deletes for %s = %d, %v, want 1", pushedPath, queued, err) } - deleted, err = db.DeleteLocalArtifacts("pkg:npm/express@1.0.0") + deleted, err = db.DeleteLocalArtifacts("pkg:npm/express@1.0.0", "") if err != nil || deleted != 0 { t.Errorf("DeleteLocalArtifacts(upstream) = %d, %v, want 0, nil", deleted, err) } diff --git a/internal/handler/composer_hosted_test.go b/internal/handler/composer_hosted_test.go index 048c291..539e999 100644 --- a/internal/handler/composer_hosted_test.go +++ b/internal/handler/composer_hosted_test.go @@ -58,11 +58,11 @@ func newComposerHostedFixture(t *testing.T, upstream map[string]string) composer func (f composerHostedFixture) push(t *testing.T, name, version, manifest string) HostedArtifact { t.Helper() - target, err := HostedTargetFor("composer", name, version) + target, err := HostedTargetFor("composer", name, version, "") if err != nil { t.Fatalf("HostedTargetFor: %v", err) } - pushed, err := f.proxy.PushLocalArtifact(context.Background(), "composer", target, bytes.NewReader(composerZip("", manifest)), "") + pushed, err := f.proxy.PushLocalArtifact(context.Background(), "composer", target, bytes.NewReader(composerZip("", manifest)), "", "") if err != nil { t.Fatalf("PushLocalArtifact: %v", err) } diff --git a/internal/handler/generic.go b/internal/handler/generic.go index 33a6ed2..f411430 100644 --- a/internal/handler/generic.go +++ b/internal/handler/generic.go @@ -126,6 +126,10 @@ func (h *GenericHandler) Routes() http.Handler { } repository, rest, ok := strings.Cut(path, "/") + if repository == HostedGenericRepository { + h.handleHosted(w, r, rest) + return + } upstreamURL, found := h.repositories[repository] if !ok || rest == "" || !found { http.NotFound(w, r) @@ -141,6 +145,38 @@ func (h *GenericHandler) Routes() http.Handler { }) } +// HostedGenericRepository is the /generic/{name}/ path that serves pushed +// generic files instead of an upstream. upstream.generic can't use the name. +const HostedGenericRepository = config.HostedGenericRepository + +// handleHosted serves a pushed file at {name}/{version}/{file}. Pushed files +// never come from an upstream, so anything not pushed is a 404. +func (h *GenericHandler) handleHosted(w http.ResponseWriter, r *http.Request, path string) { + segments := strings.Split(path, "/") + if !h.proxy.HostedEnabled || len(segments) != hostedGenericPathSegments { + http.NotFound(w, r) + return + } + target, err := HostedTargetFor(genericEcosystem, segments[0], segments[1], segments[2]) + if err != nil { + http.NotFound(w, r) + return + } + result, err := h.proxy.GetLocalArtifact(r.Context(), genericEcosystem, target.Name, target.Version, target.Filename) + if err != nil { + h.proxy.serveArtifactError(w, err, "failed to read pushed file") + return + } + if result == nil { + http.NotFound(w, r) + return + } + serveGenericArtifact(w, r, result) +} + +// hostedGenericPathSegments is {name}/{version}/{file}. +const hostedGenericPathSegments = 3 + // genericArtifact is the identity of a version-pinned download. type genericArtifact struct { name string @@ -234,7 +270,12 @@ func (h *GenericHandler) handleArtifact(w http.ResponseWriter, r *http.Request, h.proxy.serveArtifactError(w, err, "failed to fetch artifact") return } + serveGenericArtifact(w, r, result) +} +// serveGenericArtifact serves a cached or pushed file, as a plain download +// when its media type is unknown. +func serveGenericArtifact(w http.ResponseWriter, r *http.Request, result *CacheResult) { if result.Artifact.MediaType == "" { result.Artifact.MediaType = "application/octet-stream" } diff --git a/internal/handler/generic_test.go b/internal/handler/generic_test.go index 9fac379..6948b86 100644 --- a/internal/handler/generic_test.go +++ b/internal/handler/generic_test.go @@ -312,6 +312,7 @@ func TestGenericHandler_RejectsUnknownUpstreamAndBadPaths(t *testing.T) { {"traversal", http.MethodGet, "/github/../etc/passwd", http.StatusBadRequest}, {"encoded traversal", http.MethodGet, "/github/%2e%2e/etc/passwd", http.StatusBadRequest}, {"post", http.MethodPost, "/github/owner/repo/releases/download/v1/asset", http.StatusMethodNotAllowed}, + {"hosted files with hosting off", http.MethodGet, "/hosted/tools/1.2.0/tool.tar.gz", http.StatusNotFound}, } for _, tt := range tests { diff --git a/internal/handler/hosted.go b/internal/handler/hosted.go index 41ccb93..3e15868 100644 --- a/internal/handler/hosted.go +++ b/internal/handler/hosted.go @@ -23,6 +23,16 @@ var ( ErrHostedUnsupported = errors.New("pushing packages is not supported for this ecosystem") ErrHostedInvalidName = errors.New("invalid package name or version") ErrHostedVersionNotFound = errors.New("no pushed artifacts for this version") + ErrHostedFileNotFound = errors.New("no pushed file of this name for this version") + + // ErrHostedExists refuses a second push of the same file. Clients pin + // pushed files by version and often by hash, so a published file is never + // silently replaced; delete it first to push it again. + ErrHostedExists = errors.New("this file was already pushed; delete it first to push it again") + + // ErrHostedChecksumMismatch reports a push whose bytes don't match the + // sha256 the client sent along. + ErrHostedChecksumMismatch = errors.New("pushed bytes don't match the expected sha256") // ErrHostedArtifactUnavailable and ErrHostedArtifactMissing report a // pushed artifact the cache can't serve. They stop the caller from @@ -50,53 +60,105 @@ type HostedArtifact struct { PushedAt time.Time `json:"pushed_at"` } -// hostedNaming describes how an ecosystem names pushed archives: which -// package names it has, split on "/", and the filename its clients request. -type hostedNaming struct { +// hostedEcosystem is what an ecosystem defines for pushed packages. Storing, +// listing, deleting and the push rules are shared; adding an ecosystem means +// adding an entry here and, if its clients read version lists, listing pushed +// versions in its metadata. +type hostedEcosystem struct { + // validName reports whether a package name, split on "/", is one of the + // ecosystem's. validName func(segments []string) bool - filename func(name, version string) string + + // filename is the name clients request the archive by, derived from the + // package name and version. When nil, the filename is the last segment of + // the push path, and the package name must be a single segment so the + // path is unambiguous. + filename func(name, version string) string + + // readManifest reads the package manifest out of a pushed archive and + // checks that it describes target. When nil, pushes carry no manifest. + readManifest func(archive io.Reader, target HostedTarget) (json.RawMessage, error) + + // hostedOnly reports whether a namespace (an npm scope, a Composer + // vendor) is served from pushed packages alone. When nil, the ecosystem + // has no such namespaces. + hostedOnly func(p *Proxy, namespace string) bool } -var hostedNamings = map[string]hostedNaming{ +var hostedEcosystems = map[string]hostedEcosystem{ "npm": { validName: func(segments []string) bool { return len(segments) == 1 || (len(segments) == 2 && strings.HasPrefix(segments[0], "@")) }, - filename: func(name, version string) string { return lastSegment(name) + "-" + version + ".tgz" }, + filename: func(name, version string) string { return lastSegment(name) + "-" + version + ".tgz" }, + readManifest: readNPMManifest, + hostedOnly: func(p *Proxy, scope string) bool { + return slices.Contains(p.HostedNPMScopes, scope) + }, }, "composer": { - validName: func(segments []string) bool { return len(segments) == vendorPackageParts }, - filename: func(name, version string) string { return lastSegment(name) + "-" + version + ".zip" }, + validName: func(segments []string) bool { return len(segments) == vendorPackageParts }, + filename: func(name, version string) string { return lastSegment(name) + "-" + version + ".zip" }, + readManifest: readComposerManifest, + hostedOnly: func(p *Proxy, vendor string) bool { + return slices.ContainsFunc(p.HostedComposerVendors, func(v string) bool { + return strings.EqualFold(v, vendor) + }) + }, + }, + // generic holds files that aren't packages, such as prebuilt binaries. + // They are served at /generic/hosted/{name}/{version}/{file} and never + // come from an upstream. + genericEcosystem: { + validName: func(segments []string) bool { return len(segments) == 1 }, }, } // HostedSupported reports whether packages can be pushed for ecosystem. func HostedSupported(ecosystem string) bool { - _, ok := hostedNamings[ecosystem] + _, ok := hostedEcosystems[ecosystem] return ok } +// HostedFileInPath reports whether pushes for ecosystem name the file in the +// path, {name}/{version}/{file}, rather than deriving it from the name and +// version. +func HostedFileInPath(ecosystem string) bool { + eco, ok := hostedEcosystems[ecosystem] + return ok && eco.filename == nil +} + func lastSegment(name string) string { return name[strings.LastIndex(name, "/")+1:] } // HostedTargetFor validates a push of name@version to ecosystem and returns -// where it is stored. -func HostedTargetFor(ecosystem, name, version string) (HostedTarget, error) { - naming, ok := hostedNamings[ecosystem] +// where it is stored. filename is required for ecosystems whose pushes name +// the file (see HostedFileInPath) and must be empty for the others. +func HostedTargetFor(ecosystem, name, version, filename string) (HostedTarget, error) { + eco, ok := hostedEcosystems[ecosystem] if !ok { return HostedTarget{}, fmt.Errorf("%w: %s", ErrHostedUnsupported, ecosystem) } segments := strings.Split(name, "/") - if !naming.validName(segments) { + if !eco.validName(segments) { return HostedTarget{}, ErrHostedInvalidName } - for _, segment := range append(segments, version) { + checked := append(slices.Clone(segments), version) + if eco.filename == nil { + checked = append(checked, filename) + } else if filename != "" { + return HostedTarget{}, ErrHostedInvalidName + } + for _, segment := range checked { if !validHostedSegment(segment) { return HostedTarget{}, ErrHostedInvalidName } } - return HostedTarget{Name: name, Version: version, Filename: naming.filename(name, version)}, nil + if eco.filename != nil { + filename = eco.filename(name, version) + } + return HostedTarget{Name: name, Version: version, Filename: filename}, nil } func validHostedSegment(s string) bool { @@ -114,9 +176,11 @@ func validHostedSegment(s string) bool { // PushLocalArtifact stores body as the artifact for target and records it as // local, replacing any cached upstream copy. Download handlers serve it from // the cache without contacting the upstream, and eviction never removes it. -// The archive's manifest must name target; it is kept so the ecosystem's -// metadata can list the version with its dependencies. -func (p *Proxy) PushLocalArtifact(ctx context.Context, ecosystem string, target HostedTarget, body io.Reader, contentType string) (HostedArtifact, error) { +// A file that was already pushed is refused with ErrHostedExists. When +// wantSHA256 is set, the pushed bytes must hash to it. If the ecosystem has +// manifests, the archive's manifest must name target; it is kept so the +// ecosystem's metadata can list the version with its dependencies. +func (p *Proxy) PushLocalArtifact(ctx context.Context, ecosystem string, target HostedTarget, body io.Reader, contentType, wantSHA256 string) (HostedArtifact, error) { pkgPURL, versionPURL, err := packagePURLStrings(ecosystem, target.Name, target.Version) if err != nil { return HostedArtifact{}, err @@ -124,6 +188,9 @@ func (p *Proxy) PushLocalArtifact(ctx context.Context, ecosystem string, target if p.versionDenied(ecosystem, target.Name, target.Version) { return HostedArtifact{}, fmt.Errorf("%w: %s", ErrVersionDenied, versionPURL) } + if err := p.checkNotPushed(versionPURL, target.Filename); err != nil { + return HostedArtifact{}, err + } push := storedPush{ storagePath: storage.FetchPath(ecosystem, target.Name, target.Version, storage.NewFetchID(), target.Filename), @@ -140,11 +207,23 @@ func (p *Proxy) PushLocalArtifact(ctx context.Context, ecosystem string, target } push.integrity = "sha512-" + base64.StdEncoding.EncodeToString(sha512Hash.Sum(nil)) + if wantSHA256 != "" && !strings.EqualFold(wantSHA256, push.sha256) { + p.discardStored(ctx, push.storagePath) + return HostedArtifact{}, fmt.Errorf("%w: got %s", ErrHostedChecksumMismatch, push.sha256) + } + if err := p.checkPush(ctx, ecosystem, target, versionPURL, &push); err != nil { p.discardStored(ctx, push.storagePath) return HostedArtifact{}, err } + // A concurrent push of the same file may have finished while this one + // was being stored and checked. + if err := p.checkNotPushed(versionPURL, target.Filename); err != nil { + p.discardStored(ctx, push.storagePath) + return HostedArtifact{}, err + } + push.pushedAt = time.Now() if err := p.recordLocalArtifact(ecosystem, target, pkgPURL, versionPURL, push); err != nil { p.discardStored(ctx, push.storagePath) @@ -160,6 +239,19 @@ func (p *Proxy) PushLocalArtifact(ctx context.Context, ecosystem string, target }, nil } +// checkNotPushed refuses a push of a file that was already pushed. A cached +// upstream copy under the same name is fine to replace. +func (p *Proxy) checkNotPushed(versionPURL, filename string) error { + record, err := p.DB.GetArtifact(versionPURL, filename) + if err != nil { + return fmt.Errorf("checking existing artifact: %w", err) + } + if record != nil && record.Source == database.SourceLocal { + return ErrHostedExists + } + return nil +} + // storedPush is a pushed archive once it is in storage. type storedPush struct { storagePath string @@ -210,10 +302,14 @@ func (p *Proxy) recordLocalArtifact(ecosystem string, target HostedTarget, pkgPU } } + // Ecosystems that name the file in the push path hold several files per + // version, and the version's integrity can only describe one of them. + // Their files are checked against their own sha256 instead. + oneFilePerVersion := hostedEcosystems[ecosystem].filename != nil if err := p.DB.UpsertVersion(&database.Version{ PURL: versionPURL, PackagePURL: pkgPURL, - Integrity: sql.NullString{String: push.integrity, Valid: true}, + Integrity: sql.NullString{String: push.integrity, Valid: oneFilePerVersion}, }); err != nil { return fmt.Errorf("upserting version: %w", err) } @@ -227,7 +323,7 @@ func (p *Proxy) recordLocalArtifact(ecosystem string, target HostedTarget, pkgPU ContentType: sql.NullString{String: push.contentType, Valid: push.contentType != ""}, FetchedAt: sql.NullTime{Time: push.pushedAt, Valid: true}, Source: database.SourceLocal, - Manifest: sql.NullString{String: string(push.manifest), Valid: true}, + Manifest: sql.NullString{String: string(push.manifest), Valid: push.manifest != nil}, }); err != nil { return fmt.Errorf("upserting artifact: %w", err) } @@ -266,19 +362,8 @@ func (p *Proxy) hostedOnly(ecosystem, name string) bool { return false } namespace, _, namespaced := strings.Cut(name, "/") - if !namespaced { - return false - } - switch ecosystem { - case "npm": - return slices.Contains(p.HostedNPMScopes, namespace) - case "composer": - return slices.ContainsFunc(p.HostedComposerVendors, func(vendor string) bool { - return strings.EqualFold(vendor, namespace) - }) - default: - return false - } + eco := hostedEcosystems[ecosystem] + return namespaced && eco.hostedOnly != nil && eco.hostedOnly(p, namespace) } // localVersion is a pushed version as ecosystem metadata lists it. @@ -328,16 +413,25 @@ func (p *Proxy) localVersions(ecosystem, name string) ([]localVersion, error) { // DeleteLocalVersion removes the archives pushed for name@version. If the // upstream has the version, clients get the upstream archive again. func (p *Proxy) DeleteLocalVersion(ecosystem, name, version string) error { + return p.deleteLocal(ecosystem, name, version, "", ErrHostedVersionNotFound) +} + +// DeleteLocalFile removes one pushed file of name@version. +func (p *Proxy) DeleteLocalFile(ecosystem, name, version, filename string) error { + return p.deleteLocal(ecosystem, name, version, filename, ErrHostedFileNotFound) +} + +func (p *Proxy) deleteLocal(ecosystem, name, version, filename string, notFound error) error { _, versionPURL, err := packagePURLStrings(ecosystem, name, version) if err != nil { return err } - deleted, err := p.DB.DeleteLocalArtifacts(versionPURL) + deleted, err := p.DB.DeleteLocalArtifacts(versionPURL, filename) if err != nil { return fmt.Errorf("deleting local artifacts: %w", err) } if deleted == 0 { - return ErrHostedVersionNotFound + return notFound } return nil } diff --git a/internal/handler/hosted_manifest.go b/internal/handler/hosted_manifest.go index 3bd9b38..cf90dcb 100644 --- a/internal/handler/hosted_manifest.go +++ b/internal/handler/hosted_manifest.go @@ -26,15 +26,16 @@ const maxManifestSize = 1 << 20 // readHostedManifest reads the package manifest out of a pushed archive and // checks that it describes target. The manifest is what the ecosystem's // metadata lists the version with, so its dependencies reach the client. +// Ecosystems without manifests return nil. func readHostedManifest(ecosystem string, archive io.Reader, target HostedTarget) (json.RawMessage, error) { - switch ecosystem { - case "npm": - return readNPMManifest(archive, target) - case "composer": - return readComposerManifest(archive, target) - default: + eco, ok := hostedEcosystems[ecosystem] + if !ok { return nil, fmt.Errorf("%w: %s", ErrHostedUnsupported, ecosystem) } + if eco.readManifest == nil { + return nil, nil + } + return eco.readManifest(archive, target) } // readNPMManifest reads package.json from the top directory of an npm diff --git a/internal/handler/hosted_test.go b/internal/handler/hosted_test.go index a4e7145..252d94b 100644 --- a/internal/handler/hosted_test.go +++ b/internal/handler/hosted_test.go @@ -15,6 +15,27 @@ import ( "github.com/git-pkgs/registries/fetch" ) +func TestHostedTargetForFileInPath(t *testing.T) { + got, err := HostedTargetFor("generic", "tools", "1.2.0", "tool.tar.gz") + if err != nil || got != (HostedTarget{Name: "tools", Version: "1.2.0", Filename: "tool.tar.gz"}) { + t.Errorf("generic target = %+v, %v", got, err) + } + for _, tt := range []struct{ ecosystem, name, version, file string }{ + {"generic", "tools", "1.2.0", ""}, + {"generic", "tools", "1.2.0", ".."}, + {"generic", "acme/tools", "1.2.0", "tool.tar.gz"}, + {"npm", "request", "2.88.3", "request-2.88.3.tgz"}, + } { + if _, err := HostedTargetFor(tt.ecosystem, tt.name, tt.version, tt.file); !errors.Is(err, ErrHostedInvalidName) { + t.Errorf("HostedTargetFor(%q, %q, %q, %q) error = %v, want ErrHostedInvalidName", + tt.ecosystem, tt.name, tt.version, tt.file, err) + } + } + if !HostedFileInPath("generic") || HostedFileInPath("npm") || HostedFileInPath("cargo") { + t.Error("HostedFileInPath should hold for generic only") + } +} + func TestHostedTargetFor(t *testing.T) { tests := []struct { name string @@ -49,7 +70,7 @@ func TestHostedTargetFor(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, err := HostedTargetFor(tt.ecosystem, tt.pkg, tt.version) + got, err := HostedTargetFor(tt.ecosystem, tt.pkg, tt.version, "") if tt.wantErr != nil { if !errors.Is(err, tt.wantErr) { t.Fatalf("HostedTargetFor() error = %v, want %v", err, tt.wantErr) @@ -114,11 +135,11 @@ var pushedTarball = string(npmTarball(requestManifest)) func pushRequest(t *testing.T, proxy *Proxy) HostedArtifact { t.Helper() - target, err := HostedTargetFor("npm", "request", "2.88.3") + target, err := HostedTargetFor("npm", "request", "2.88.3", "") if err != nil { t.Fatalf("HostedTargetFor: %v", err) } - pushed, err := proxy.PushLocalArtifact(context.Background(), "npm", target, strings.NewReader(pushedTarball), "application/gzip") + pushed, err := proxy.PushLocalArtifact(context.Background(), "npm", target, strings.NewReader(pushedTarball), "application/gzip", "") if err != nil { t.Fatalf("PushLocalArtifact: %v", err) } @@ -290,11 +311,11 @@ func TestPushLocalArtifactChecksManifest(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { proxy, _, store, _ := setupTestProxy(t) - target, err := HostedTargetFor(tt.ecosystem, tt.pkg, tt.version) + target, err := HostedTargetFor(tt.ecosystem, tt.pkg, tt.version, "") if err != nil { t.Fatalf("HostedTargetFor: %v", err) } - _, err = proxy.PushLocalArtifact(context.Background(), tt.ecosystem, target, bytes.NewReader(tt.archive), "") + _, err = proxy.PushLocalArtifact(context.Background(), tt.ecosystem, target, bytes.NewReader(tt.archive), "", "") if tt.wantErr == nil { if err != nil { t.Fatalf("PushLocalArtifact: %v", err) @@ -401,7 +422,7 @@ func TestComposerManifestLookup(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - target, err := HostedTargetFor("composer", "monolog/monolog", "3.5.0") + target, err := HostedTargetFor("composer", "monolog/monolog", "3.5.0", "") if err != nil { t.Fatalf("HostedTargetFor: %v", err) } diff --git a/internal/handler/npm_hosted_test.go b/internal/handler/npm_hosted_test.go index 7102046..75c82a1 100644 --- a/internal/handler/npm_hosted_test.go +++ b/internal/handler/npm_hosted_test.go @@ -60,11 +60,11 @@ func newNPMHostedFixture(t *testing.T, upstream map[string]string) npmHostedFixt func (f npmHostedFixture) push(t *testing.T, name, version, manifest string) HostedArtifact { t.Helper() - target, err := HostedTargetFor("npm", name, version) + target, err := HostedTargetFor("npm", name, version, "") if err != nil { t.Fatalf("HostedTargetFor: %v", err) } - pushed, err := f.proxy.PushLocalArtifact(context.Background(), "npm", target, bytes.NewReader(npmTarball(manifest)), "") + pushed, err := f.proxy.PushLocalArtifact(context.Background(), "npm", target, bytes.NewReader(npmTarball(manifest)), "", "") if err != nil { t.Fatalf("PushLocalArtifact: %v", err) } diff --git a/internal/server/hosted_api.go b/internal/server/hosted_api.go index e9033d6..1e4e1a4 100644 --- a/internal/server/hosted_api.go +++ b/internal/server/hosted_api.go @@ -2,6 +2,8 @@ package server import ( "bufio" + "crypto/sha256" + "encoding/hex" "errors" "io" "log/slog" @@ -50,32 +52,41 @@ func registerHostedRoutes(r chi.Router, h *HostedAPIHandler, token string) { }) } -// HandlePush handles PUT /api/hosted/{ecosystem}/{name}/{version} +// HandlePush handles PUT /api/hosted/{ecosystem}/{name}/{version}, and +// PUT /api/hosted/generic/{name}/{version}/{file} for generic files. // @Summary Push a package archive -// @Description Stores the request body as the archive for name@version. It is served in place of the upstream archive with the same version and is never evicted. +// @Description Stores the request body as the archive for name@version. It is served in place of the upstream archive with the same version and is never evicted. Generic files are pushed to /api/hosted/generic/{name}/{version}/{file} and served at /generic/hosted/{name}/{version}/{file}. A file that was already pushed is refused with 409; delete it first to push it again. // @Tags hosted // @Accept octet-stream // @Produce json // @Security BearerAuth -// @Param ecosystem path string true "Ecosystem (npm or composer)" +// @Param ecosystem path string true "Ecosystem (npm, composer or generic)" // @Param name path string true "Package name, including an npm scope or the Composer vendor" // @Param version path string true "Version" +// @Param sha256 query string false "Expected sha256 of the body, hex-encoded; a mismatch is refused" // @Success 201 {object} HostedPushResponse // @Failure 400 {object} ErrorResponse // @Failure 401 {object} ErrorResponse // @Failure 403 {object} ErrorResponse +// @Failure 409 {object} ErrorResponse // @Failure 413 {object} ErrorResponse // @Router /api/hosted/{ecosystem}/{name}/{version} [put] func (h *HostedAPIHandler) HandlePush(w http.ResponseWriter, r *http.Request) { - ecosystem, name, version, ok := hostedVersionPath(w, r) + path, ok := hostedPath(w, r, true) if !ok { return } - target, err := handler.HostedTargetFor(ecosystem, name, version) + target, err := handler.HostedTargetFor(path.ecosystem, path.name, path.version, path.filename) if err != nil { badRequest(w, err.Error()) return } + wantSHA256 := r.URL.Query().Get("sha256") + if wantSHA256 != "" && !validSHA256(wantSHA256) { + badRequest(w, "sha256 must be 64 hex characters") + return + } + ecosystem := path.ecosystem body := bufio.NewReader(http.MaxBytesReader(w, r.Body, h.maxUploadBytes)) if _, err := body.Peek(1); errors.Is(err, io.EOF) { @@ -83,7 +94,7 @@ func (h *HostedAPIHandler) HandlePush(w http.ResponseWriter, r *http.Request) { return } - pushed, err := h.proxy.PushLocalArtifact(r.Context(), ecosystem, target, body, r.Header.Get("Content-Type")) + pushed, err := h.proxy.PushLocalArtifact(r.Context(), ecosystem, target, body, r.Header.Get("Content-Type"), wantSHA256) if err != nil { h.pushError(w, err) return @@ -102,8 +113,11 @@ func (h *HostedAPIHandler) pushError(w http.ResponseWriter, err error) { switch { case errors.As(err, &tooLarge): writeError(w, http.StatusRequestEntityTooLarge, ErrCodeBadRequest, "archive exceeds hosted.max_upload_size") - case errors.Is(err, handler.ErrHostedInvalidArchive), errors.Is(err, handler.ErrHostedManifestMismatch): + case errors.Is(err, handler.ErrHostedInvalidArchive), errors.Is(err, handler.ErrHostedManifestMismatch), + errors.Is(err, handler.ErrHostedChecksumMismatch): badRequest(w, err.Error()) + case errors.Is(err, handler.ErrHostedExists): + writeError(w, http.StatusConflict, ErrCodeConflict, err.Error()) case errors.Is(err, handler.ErrVersionDenied), errors.Is(err, handler.ErrArtifactBlocked): writeError(w, http.StatusForbidden, ErrCodeBadRequest, err.Error()) default: @@ -143,9 +157,10 @@ func (h *HostedAPIHandler) HandleList(w http.ResponseWriter, r *http.Request) { writeJSON(w, HostedListResponse{Ecosystem: ecosystem, Name: name, Artifacts: listed}) } -// HandleDelete handles DELETE /api/hosted/{ecosystem}/{name}/{version} +// HandleDelete handles DELETE /api/hosted/{ecosystem}/{name}/{version}, and +// DELETE /api/hosted/generic/{name}/{version}/{file} for one generic file. // @Summary Delete a pushed version -// @Description Removes the archives pushed for name@version. If the upstream has the version, clients get the upstream archive again. +// @Description Removes the archives pushed for name@version. If the upstream has the version, clients get the upstream archive again. For generic files, /api/hosted/generic/{name}/{version}/{file} removes only that file. // @Tags hosted // @Security BearerAuth // @Param ecosystem path string true "Ecosystem" @@ -156,19 +171,25 @@ func (h *HostedAPIHandler) HandleList(w http.ResponseWriter, r *http.Request) { // @Failure 404 {object} ErrorResponse // @Router /api/hosted/{ecosystem}/{name}/{version} [delete] func (h *HostedAPIHandler) HandleDelete(w http.ResponseWriter, r *http.Request) { - ecosystem, name, version, ok := hostedVersionPath(w, r) + path, ok := hostedPath(w, r, false) if !ok { return } - err := h.proxy.DeleteLocalVersion(ecosystem, name, version) + var err error + if path.filename != "" { + err = h.proxy.DeleteLocalFile(path.ecosystem, path.name, path.version, path.filename) + } else { + err = h.proxy.DeleteLocalVersion(path.ecosystem, path.name, path.version) + } switch { - case errors.Is(err, handler.ErrHostedVersionNotFound): + case errors.Is(err, handler.ErrHostedVersionNotFound), errors.Is(err, handler.ErrHostedFileNotFound): notFound(w, err.Error()) case err != nil: h.logger.Error("failed to delete pushed version", "error", err) internalError(w, "failed to delete pushed version") default: - h.logger.Info("deleted pushed version", "ecosystem", ecosystem, "name", name, "version", version) + h.logger.Info("deleted pushed version", "ecosystem", path.ecosystem, "name", path.name, + "version", path.version, "filename", path.filename) w.WriteHeader(http.StatusNoContent) } } @@ -176,19 +197,53 @@ func (h *HostedAPIHandler) HandleDelete(w http.ResponseWriter, r *http.Request) // minVersionPathSegments is one name segment plus the version. const minVersionPathSegments = 2 -// hostedVersionPath splits {name}/{version} from the wildcard. The version is -// always the last segment; the name may span several (npm scopes, Composer -// vendors). -func hostedVersionPath(w http.ResponseWriter, r *http.Request) (ecosystem, name, version string, ok bool) { +// hostedPushPath is a parsed /api/hosted/{ecosystem}/... path. +type hostedPushPath struct { + ecosystem string + name string + version string + filename string +} + +// hostedPath splits the wildcard into name, version and, for ecosystems +// whose pushes name the file, the filename. The version is the last segment +// otherwise, and the name may span several (npm scopes, Composer vendors). +// Ecosystems that name the file have single-segment names, so their path is +// {name}/{version}/{file}; requireFile is false for deletes, where +// {name}/{version} removes the whole version. +func hostedPath(w http.ResponseWriter, r *http.Request, requireFile bool) (hostedPushPath, bool) { segments, err := packagePathSegments(r) if err != nil { badRequest(w, err.Error()) - return "", "", "", false + return hostedPushPath{}, false + } + path := hostedPushPath{ecosystem: chi.URLParam(r, "ecosystem")} + if handler.HostedFileInPath(path.ecosystem) { + switch { + case len(segments) == minVersionPathSegments+1: + path.name, path.version, path.filename = segments[0], segments[1], segments[2] + case len(segments) == minVersionPathSegments && !requireFile: + path.name, path.version = segments[0], segments[1] + default: + badRequest(w, "path must be /api/hosted/"+path.ecosystem+"/{name}/{version}/{file}") + return hostedPushPath{}, false + } + return path, true } if len(segments) < minVersionPathSegments { badRequest(w, "path must be /api/hosted/{ecosystem}/{name}/{version}") - return "", "", "", false + return hostedPushPath{}, false } last := len(segments) - 1 - return chi.URLParam(r, "ecosystem"), strings.Join(segments[:last], "/"), segments[last], true + path.name, path.version = strings.Join(segments[:last], "/"), segments[last] + return path, true +} + +// validSHA256 reports whether s is a hex-encoded sha256 digest. +func validSHA256(s string) bool { + if len(s) != sha256.Size*2 { + return false + } + _, err := hex.DecodeString(s) + return err == nil } diff --git a/internal/server/hosted_api_test.go b/internal/server/hosted_api_test.go index 101ff31..2185f3a 100644 --- a/internal/server/hosted_api_test.go +++ b/internal/server/hosted_api_test.go @@ -2,6 +2,8 @@ package server import ( "context" + "crypto/sha256" + "encoding/hex" "encoding/json" "io" "log/slog" @@ -80,6 +82,7 @@ func newHostedTestServer(t *testing.T) hostedTestServer { r.Mount("/npm", http.StripPrefix("/npm", handler.NewNPMHandler(proxy, "http://proxy.test", upstream.URL).Routes())) composer := handler.NewComposerHandlerWithUpstreams(proxy, "http://proxy.test", upstream.URL, upstream.URL) r.Mount("/composer", http.StripPrefix("/composer", composer.Routes())) + r.Mount("/generic", http.StripPrefix("/generic", handler.NewGenericHandler(proxy, nil).Routes())) return hostedTestServer{router: r, fetches: &calls, upstreamRequests: &upstreamRequests} } @@ -214,6 +217,10 @@ func TestHostedPushRejections(t *testing.T) { {name: "manifest of another version", target: "/api/hosted/npm/request/2.88.4", token: testHostedToken, body: requestTarball, wantStatus: http.StatusBadRequest}, {name: "empty body", target: "/api/hosted/npm/request/2.88.3", token: testHostedToken, wantStatus: http.StatusBadRequest}, {name: "too large", target: "/api/hosted/npm/request/2.88.3", token: testHostedToken, body: strings.Repeat("x", testHostedMaxUpload+1), wantStatus: http.StatusRequestEntityTooLarge}, + {name: "sha256 mismatch", target: "/api/hosted/npm/request/2.88.3?sha256=" + strings.Repeat("0", 64), token: testHostedToken, body: requestTarball, wantStatus: http.StatusBadRequest}, + {name: "malformed sha256", target: "/api/hosted/npm/request/2.88.3?sha256=abc", token: testHostedToken, body: requestTarball, wantStatus: http.StatusBadRequest}, + {name: "generic without file", target: "/api/hosted/generic/request/2.88.3", token: testHostedToken, body: "binary", wantStatus: http.StatusBadRequest}, + {name: "generic name with two segments", target: "/api/hosted/generic/acme/tools/1.0.0/tool.bin", token: testHostedToken, body: "binary", wantStatus: http.StatusBadRequest}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -360,3 +367,118 @@ func TestHostedComposerVersionIsInstallableFromMetadata(t *testing.T) { t.Errorf("download of %s status = %d", distPath, rec.Code) } } + +func TestHostedRepushIsRefused(t *testing.T) { + s := newHostedTestServer(t) + const target = "/api/hosted/npm/request/2.88.3" + + if rec := s.do(t, http.MethodPut, target, testHostedToken, requestTarball); rec.Code != http.StatusCreated { + t.Fatalf("first PUT status = %d, body %s", rec.Code, rec.Body) + } + if rec := s.do(t, http.MethodPut, target, testHostedToken, requestTarball); rec.Code != http.StatusConflict { + t.Fatalf("second PUT status = %d, want 409, body %s", rec.Code, rec.Body) + } + if rec := s.do(t, http.MethodDelete, target, testHostedToken, ""); rec.Code != http.StatusNoContent { + t.Fatalf("DELETE status = %d", rec.Code) + } + if rec := s.do(t, http.MethodPut, target, testHostedToken, requestTarball); rec.Code != http.StatusCreated { + t.Fatalf("PUT after DELETE status = %d, want 201, body %s", rec.Code, rec.Body) + } +} + +func TestHostedPushWithMatchingSHA256(t *testing.T) { + s := newHostedTestServer(t) + digest := sha256.Sum256([]byte(requestTarball)) + target := "/api/hosted/npm/request/2.88.3?sha256=" + strings.ToUpper(hex.EncodeToString(digest[:])) + + if rec := s.do(t, http.MethodPut, target, testHostedToken, requestTarball); rec.Code != http.StatusCreated { + t.Fatalf("PUT status = %d, body %s", rec.Code, rec.Body) + } +} + +func TestHostedGenericFiles(t *testing.T) { + s := newHostedTestServer(t) + const ( + arm64 = "tool-linux-arm64.tar.gz" + amd64 = "tool-linux-amd64.tar.gz" + ) + + for _, file := range []string{arm64, amd64} { + rec := s.do(t, http.MethodPut, "/api/hosted/generic/tools/1.2.0/"+file, testHostedToken, "bytes of "+file) + if rec.Code != http.StatusCreated { + t.Fatalf("PUT %s status = %d, body %s", file, rec.Code, rec.Body) + } + var pushed HostedPushResponse + if err := json.NewDecoder(rec.Body).Decode(&pushed); err != nil { + t.Fatalf("decoding PUT response: %v", err) + } + digest := sha256.Sum256([]byte("bytes of " + file)) + if pushed.Filename != file || pushed.Version != "1.2.0" || pushed.SHA256 != hex.EncodeToString(digest[:]) { + t.Errorf("PUT response = %+v", pushed) + } + } + + rec := s.do(t, http.MethodGet, "/generic/hosted/tools/1.2.0/"+arm64, "", "") + if rec.Code != http.StatusOK || rec.Body.String() != "bytes of "+arm64 { + t.Fatalf("download status = %d, body %q", rec.Code, rec.Body) + } + digest := sha256.Sum256([]byte("bytes of " + arm64)) + if etag := rec.Header().Get("ETag"); etag != `"`+hex.EncodeToString(digest[:])+`"` { + t.Errorf("ETag = %q, want the sha256", etag) + } + if rec := s.do(t, http.MethodHead, "/generic/hosted/tools/1.2.0/"+arm64, "", ""); rec.Code != http.StatusOK { + t.Errorf("HEAD status = %d", rec.Code) + } + + listed := s.do(t, http.MethodGet, "/api/hosted/generic/tools", "", "") + var list HostedListResponse + if err := json.NewDecoder(listed.Body).Decode(&list); err != nil { + t.Fatalf("decoding list: %v", err) + } + if len(list.Artifacts) != 2 { + t.Fatalf("listed %d files, want 2: %+v", len(list.Artifacts), list.Artifacts) + } + + if rec := s.do(t, http.MethodDelete, "/api/hosted/generic/tools/1.2.0/"+arm64, testHostedToken, ""); rec.Code != http.StatusNoContent { + t.Fatalf("DELETE file status = %d, body %s", rec.Code, rec.Body) + } + if rec := s.do(t, http.MethodGet, "/generic/hosted/tools/1.2.0/"+arm64, "", ""); rec.Code != http.StatusNotFound { + t.Errorf("download of deleted file status = %d, want 404", rec.Code) + } + if rec := s.do(t, http.MethodGet, "/generic/hosted/tools/1.2.0/"+amd64, "", ""); rec.Code != http.StatusOK { + t.Errorf("download of the other file status = %d, want 200", rec.Code) + } + if rec := s.do(t, http.MethodDelete, "/api/hosted/generic/tools/1.2.0/"+arm64, testHostedToken, ""); rec.Code != http.StatusNotFound { + t.Errorf("second DELETE status = %d, want 404", rec.Code) + } + + for _, path := range []string{ + "/generic/hosted/tools/1.2.0/missing.bin", + "/generic/hosted/tools/1.2.0", + "/generic/hosted/tools/1.2.0/a/b", + } { + if rec := s.do(t, http.MethodGet, path, "", ""); rec.Code != http.StatusNotFound { + t.Errorf("GET %s status = %d, want 404", path, rec.Code) + } + } + if *s.fetches != 0 || *s.upstreamRequests != 0 { + t.Errorf("hosted generic files went upstream: %d fetches, %d requests", *s.fetches, *s.upstreamRequests) + } +} + +func TestHostedGenericFileIsServedByTheRunningServer(t *testing.T) { + base := startHostedServer(t, config.HostedConfig{Enabled: true, Token: testHostedToken}) + if status := putBody(t, base+"/api/hosted/generic/tools/1.2.0/tool.tar.gz", testHostedToken, "tool bytes"); status != http.StatusCreated { + t.Fatalf("push status = %d, want 201", status) + } + + resp, err := http.Get(base + "/generic/hosted/tools/1.2.0/tool.tar.gz") + if err != nil { + t.Fatalf("GET file: %v", err) + } + defer func() { _ = resp.Body.Close() }() + body, _ := io.ReadAll(resp.Body) + if resp.StatusCode != http.StatusOK || string(body) != "tool bytes" { + t.Errorf("download status = %d, body %q", resp.StatusCode, body) + } +} From 5c0120f3050ad7df5f9eaeeab351123bf0e39d65 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Tue, 20 Oct 2026 20:03:06 +0200 Subject: [PATCH 336/633] Record pushes atomically and address review findings - Record the package, version and artifact of a push in one transaction that only inserts the artifact if no pushed copy exists. Of two concurrent pushes of the same file exactly one wins, and the loser's version integrity is rolled back with it. - Check for an existing file before reading the request body, and send Expect: 100-continue from proxy hosted push, so a refused re-push isn't uploaded first. - Leave an existing version record alone for multi-file ecosystems, so a generic push doesn't reset a version cached from upstream.generic. - Reserve the upstream.generic name "hosted" only while hosting is enabled, and only intercept /generic/hosted/ then. - Accept a file argument in proxy hosted delete for generic only. - Mark multi-file ecosystems explicitly, and give parse a min/max range. --- cmd/proxy/hosted.go | 34 ++++--- cmd/proxy/hosted_test.go | 21 ++++- docs/configuration.md | 4 +- internal/config/config.go | 10 ++- internal/config/config_test.go | 23 +++-- internal/database/local_artifacts.go | 85 ++++++++++++++++++ internal/database/local_artifacts_test.go | 89 ++++++++++++++++++ internal/handler/generic.go | 6 +- internal/handler/generic_test.go | 17 ++++ internal/handler/hosted.go | 105 +++++++++++----------- internal/handler/hosted_test.go | 53 +++++++++++ internal/server/hosted_api.go | 7 ++ internal/server/hosted_api_test.go | 24 +++++ 13 files changed, 394 insertions(+), 84 deletions(-) diff --git a/cmd/proxy/hosted.go b/cmd/proxy/hosted.go index ec61b3a..36a798d 100644 --- a/cmd/proxy/hosted.go +++ b/cmd/proxy/hosted.go @@ -24,10 +24,11 @@ const ( hostedTokenEnv = "PROXY_HOSTED_TOKEN" hostedClientTimeout = 10 * time.Minute - pushArgCount = 4 // ecosystem, name, version, file - listArgCount = 2 // ecosystem, name - deleteArgCount = 3 // ecosystem, name, version[, file] - tablePadding = 2 + pushArgCount = 4 // ecosystem, name, version, file + listArgCount = 2 // ecosystem, name + deleteArgCount = 3 // ecosystem, name, version + deleteMaxArgCount = 4 // ecosystem, name, version, file for generic files + tablePadding = 2 ) // hostedArtifact mirrors the JSON the /api/hosted endpoints return. @@ -74,17 +75,16 @@ func newHostedCommand(name, usage, summary string, getenv func(string) string) h return cmd } -// parse parses flags and checks for exactly want positional arguments, or -// one more when optional is set. -func (c hostedCommand) parse(args []string, want int, optional ...bool) ([]string, error) { +// parse parses flags and checks for between minArgs and maxArgs positional +// arguments. +func (c hostedCommand) parse(args []string, minArgs, maxArgs int) ([]string, error) { if err := c.flags.Parse(args); err != nil { if errors.Is(err, flag.ErrHelp) { c.flags.Usage() } return nil, fmt.Errorf("%w\nusage: %s", err, c.usage) } - n := c.flags.NArg() - if n != want && (len(optional) == 0 || !optional[0] || n != want+1) { + if n := c.flags.NArg(); n < minArgs || n > maxArgs { return nil, fmt.Errorf("usage: %s", c.usage) } return c.flags.Args(), nil @@ -112,7 +112,7 @@ func hostedToken(getenv func(string) string) (string, error) { func runPush(args []string, getenv func(string) string, stdout io.Writer, client *http.Client) error { cmd := newHostedCommand("push", "proxy hosted push [-url URL] ", "Push a package archive to the proxy", getenv) - positional, err := cmd.parse(args, pushArgCount) + positional, err := cmd.parse(args, pushArgCount, pushArgCount) if err != nil { return err } @@ -131,7 +131,8 @@ func runPush(args []string, getenv func(string) string, stdout io.Writer, client // The proxy checks the upload against this, so a truncated or corrupted // transfer is refused rather than stored. digest := sha256.New() - if _, err := io.Copy(digest, file); err != nil { + size, err := io.Copy(digest, file) + if err != nil { return err } if _, err := file.Seek(0, io.SeekStart); err != nil { @@ -147,8 +148,12 @@ func runPush(args []string, getenv func(string) string, stdout io.Writer, client if err != nil { return err } + req.ContentLength = size req.Header.Set("Authorization", "Bearer "+token) req.Header.Set("Content-Type", "application/octet-stream") + // The proxy refuses an existing file or a bad token before reading the + // body, so the file is only uploaded once it would be accepted. + req.Header.Set("Expect", "100-continue") var pushed hostedPushResponse if err := doHosted(client, req, http.StatusCreated, &pushed); err != nil { @@ -161,7 +166,7 @@ func runPush(args []string, getenv func(string) string, stdout io.Writer, client func runList(args []string, getenv func(string) string, stdout io.Writer, client *http.Client) error { cmd := newHostedCommand("list", "proxy hosted list [-url URL] ", "List pushed package archives", getenv) - positional, err := cmd.parse(args, listArgCount) + positional, err := cmd.parse(args, listArgCount, listArgCount) if err != nil { return err } @@ -185,10 +190,13 @@ func runList(args []string, getenv func(string) string, stdout io.Writer, client func runDelete(args []string, getenv func(string) string, stdout io.Writer, client *http.Client) error { cmd := newHostedCommand("delete", "proxy hosted delete [-url URL] [file]", "Delete a pushed version, or one generic file of it", getenv) - positional, err := cmd.parse(args, deleteArgCount, true) + positional, err := cmd.parse(args, deleteArgCount, deleteMaxArgCount) if err != nil { return err } + if len(positional) > deleteArgCount && !handler.HostedFileInPath(positional[0]) { + return fmt.Errorf("only generic files can be deleted one at a time\nusage: %s", cmd.usage) + } token, err := hostedToken(getenv) if err != nil { return err diff --git a/cmd/proxy/hosted_test.go b/cmd/proxy/hosted_test.go index 34f9a7b..33fe226 100644 --- a/cmd/proxy/hosted_test.go +++ b/cmd/proxy/hosted_test.go @@ -12,7 +12,8 @@ import ( ) type recordedRequest struct { - method, path, query, authorization, body string + method, path, query, authorization, body, expect string + contentLength int64 } func hostedTestServer(t *testing.T, status int, response string) (*httptest.Server, *recordedRequest) { @@ -26,6 +27,8 @@ func hostedTestServer(t *testing.T, status int, response string) (*httptest.Serv query: r.URL.RawQuery, authorization: r.Header.Get("Authorization"), body: string(body), + expect: r.Header.Get("Expect"), + contentLength: r.ContentLength, } w.Header().Set("Content-Type", "application/json") w.WriteHeader(status) @@ -67,6 +70,9 @@ func TestRunPush(t *testing.T) { if recorded.authorization != "Bearer tok" || recorded.body != "tarball" || recorded.query != "sha256="+tarballSHA256 { t.Errorf("authorization/body/query = %q/%q/%q", recorded.authorization, recorded.body, recorded.query) } + if recorded.expect != "100-continue" || recorded.contentLength != int64(len("tarball")) { + t.Errorf("expect/content length = %q/%d, want 100-continue and the file size", recorded.expect, recorded.contentLength) + } if out := stdout.String(); !strings.Contains(out, "node-20.1.0.tgz") || !strings.Contains(out, "abc123") { t.Errorf("output = %q, want filename and sha256", out) } @@ -175,6 +181,19 @@ func TestRunDeleteOneFile(t *testing.T) { } } +func TestRunDeleteOneFileOnlyForGeneric(t *testing.T) { + srv, recorded := hostedTestServer(t, http.StatusNoContent, "") + + err := runDelete([]string{"-url", srv.URL, "npm", "@acme", "widget", "1.0.0"}, + hostedEnv(map[string]string{"PROXY_HOSTED_TOKEN": "tok"}), io.Discard, srv.Client()) + if err == nil || !strings.Contains(err.Error(), "only generic files") { + t.Errorf("npm delete with a fourth argument error = %v, want a refusal", err) + } + if recorded.method != "" { + t.Errorf("refused delete sent %s %s", recorded.method, recorded.path) + } +} + func TestRunHostedDispatchesSubcommands(t *testing.T) { srv, recorded := hostedTestServer(t, http.StatusOK, `{"ecosystem":"npm","name":"request","artifacts":[]}`) diff --git a/docs/configuration.md b/docs/configuration.md index a08ec4c..f805387 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1123,7 +1123,7 @@ hosted: 4. Check what is pushed: `proxy hosted list npm request` or `GET /api/hosted/npm/request`. 5. Remove it again: `proxy hosted delete npm request 2.88.3` or `DELETE /api/hosted/npm/request/2.88.3`. If the upstream has the version, clients get the upstream archive again. -A pushed file is never replaced: pushing the same version again answers 409. Clients that pinned the file by its hash would otherwise get different bytes under the same version. To push it again, delete it first. +A pushed file is never replaced: pushing the same version again answers 409, before the body is uploaded when the client sends `Expect: 100-continue` (as `proxy hosted push` does). Clients that pinned the file by its hash would otherwise get different bytes under the same version. To push it again, delete it first. ### Supported ecosystems @@ -1180,7 +1180,7 @@ They are downloaded without a token from `/generic/hosted/{name}/{version}/{file curl -O https://proxy.example.com/generic/hosted/tools/1.2.0/tool-linux-arm64.tar.gz ``` -Generic files never come from an upstream: anything not pushed answers 404, and `hosted` can't be used as an `upstream.generic` name. `proxy hosted list generic tools` lists all versions and files. `proxy hosted delete generic tools 1.2.0` removes a version with all its files, and `proxy hosted delete generic tools 1.2.0 tool-linux-arm64.tar.gz` removes only that file. +Generic files never come from an upstream: anything not pushed answers 404. While hosting is enabled, `hosted` can't be used as an `upstream.generic` name. `proxy hosted list generic tools` lists all versions and files. `proxy hosted delete generic tools 1.2.0` removes a version with all its files, and `proxy hosted delete generic tools 1.2.0 tool-linux-arm64.tar.gz` removes only that file. ### Storage and eviction diff --git a/internal/config/config.go b/internal/config/config.go index e788f8d..b06122f 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -840,9 +840,6 @@ func (u *UpstreamConfig) Validate() error { if err := validateOCIMirrors(u.OCIMirrors, u.OCI); err != nil { return err } - if _, reserved := u.Generic[HostedGenericRepository]; reserved { - return fmt.Errorf("invalid upstream.generic name %q: reserved for pushed files at /generic/%s/", HostedGenericRepository, HostedGenericRepository) - } if err := validateNamedUpstreams("upstream.generic", u.Generic); err != nil { return err } @@ -898,7 +895,8 @@ func (c *Config) validateUpstream() error { } // HostedGenericRepository is the /generic/{name}/ path that serves files -// pushed through /api/hosted/generic. No upstream.generic entry may use it. +// pushed through /api/hosted/generic. While hosted packages are enabled, no +// upstream.generic entry may use it. const HostedGenericRepository = "hosted" // CompileGenericArtifactPattern compiles an upstream.generic_artifacts pattern @@ -1511,6 +1509,10 @@ func (c *Config) validateComponents() error { if err := c.Hosted.Validate(); err != nil { return err } + if _, reserved := c.Upstream.Generic[HostedGenericRepository]; reserved && c.Hosted.Enabled { + return fmt.Errorf("invalid upstream.generic name %q: reserved for pushed files at /generic/%s/ when hosted.enabled is set", + HostedGenericRepository, HostedGenericRepository) + } if err := c.Mirror.Validate(); err != nil { return err diff --git a/internal/config/config_test.go b/internal/config/config_test.go index b21c348..224c651 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -1455,13 +1455,6 @@ func TestValidateNamedUpstreams(t *testing.T) { }, wantErr: true, }, - { - name: "generic upstream name reserved for pushed files", - modify: func(cfg *Config) { - cfg.Upstream.Generic = map[string]string{"hosted": "https://example.com"} - }, - wantErr: true, - }, { name: "generic artifact patterns", modify: func(cfg *Config) { @@ -1892,3 +1885,19 @@ func TestMavenRepositoryNames(t *testing.T) { t.Error("maven_repositories with an invalid URL: want an error") } } + +func TestGenericUpstreamNamedHosted(t *testing.T) { + cfg := Default() + cfg.Upstream.Generic = map[string]string{"hosted": "https://example.com"} + if err := cfg.Validate(); err != nil { + t.Errorf("upstream.generic named hosted with hosting off: %v, want it accepted", err) + } + + cfg = Default() + cfg.Upstream.Generic = map[string]string{"hosted": "https://example.com"} + cfg.Hosted.Enabled = true + cfg.Hosted.Token = "secret" + if err := cfg.Validate(); err == nil || !strings.Contains(err.Error(), "reserved") { + t.Errorf("upstream.generic named hosted with hosting on: %v, want it refused", err) + } +} diff --git a/internal/database/local_artifacts.go b/internal/database/local_artifacts.go index 1bb391f..1bd8555 100644 --- a/internal/database/local_artifacts.go +++ b/internal/database/local_artifacts.go @@ -2,6 +2,7 @@ package database import ( "database/sql" + "errors" "fmt" "time" ) @@ -73,3 +74,87 @@ func (db *DB) DeleteLocalArtifacts(versionPURL, filename string) (int, error) { } return int(deleted), tx.Commit() } + +// ErrLocalArtifactExists is returned by RecordLocalArtifact when the file +// was already pushed. +var ErrLocalArtifactExists = errors.New("local artifact already exists") + +// RecordLocalArtifact records a pushed artifact in one transaction: the +// package if it is missing, the version, and the artifact. An artifact cached +// from an upstream under the same name is replaced and its stored object +// queued for deletion; a pushed one is never replaced, and the whole record +// is refused with ErrLocalArtifactExists, so of two concurrent pushes of the +// same file exactly one wins. +// +// When setIntegrity is false, an existing version record is left as it is. +// Otherwise its integrity is set to v.Integrity. +func (db *DB) RecordLocalArtifact(pkg *Package, v *Version, setIntegrity bool, a *Artifact) error { + tx, err := db.Beginx() + if err != nil { + return err + } + defer func() { _ = tx.Rollback() }() + now := time.Now() + + if _, err := tx.Exec(db.Rebind(` + INSERT INTO packages (purl, ecosystem, name, created_at, updated_at) + VALUES (?, ?, ?, ?, ?) + ON CONFLICT(purl) DO NOTHING + `), pkg.PURL, pkg.Ecosystem, pkg.Name, now, now); err != nil { + return fmt.Errorf("inserting package: %w", err) + } + + versionConflict := `DO NOTHING` + if setIntegrity { + versionConflict = `DO UPDATE SET integrity = excluded.integrity, updated_at = excluded.updated_at` + } + if _, err := tx.Exec(db.Rebind(` + INSERT INTO versions (purl, package_purl, integrity, created_at, updated_at) + VALUES (?, ?, ?, ?, ?) + ON CONFLICT(purl) `+versionConflict, + ), v.PURL, v.PackagePURL, v.Integrity, now, now); err != nil { + return fmt.Errorf("recording version: %w", err) + } + + var previous sql.NullString + if err := tx.Get(&previous, db.Rebind(`SELECT storage_path FROM artifacts WHERE version_purl = ? AND filename = ?`), + a.VersionPURL, a.Filename); err != nil && !errors.Is(err, sql.ErrNoRows) { + return fmt.Errorf("reading artifact: %w", err) + } + + res, err := tx.Exec(db.Rebind(` + INSERT INTO artifacts (version_purl, filename, upstream_url, storage_path, content_hash, + size, content_type, fetched_at, hit_count, created_at, updated_at, + source, manifest) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, 0, ?, ?, ?, ?) + ON CONFLICT(version_purl, filename) DO UPDATE SET + upstream_url = excluded.upstream_url, + storage_path = excluded.storage_path, + content_hash = excluded.content_hash, + size = excluded.size, + content_type = excluded.content_type, + fetched_at = excluded.fetched_at, + updated_at = excluded.updated_at, + source = excluded.source, + manifest = excluded.manifest + WHERE artifacts.source IS NULL OR artifacts.source <> 'local' + `), a.VersionPURL, a.Filename, a.UpstreamURL, a.StoragePath, a.ContentHash, + a.Size, a.ContentType, inUTC(a.FetchedAt), now, now, SourceLocal, a.Manifest) + if err != nil { + return fmt.Errorf("recording artifact: %w", err) + } + n, err := res.RowsAffected() + if err != nil { + return fmt.Errorf("recording artifact: %w", err) + } + if n == 0 { + return ErrLocalArtifactExists + } + + if previous.Valid && previous.String != a.StoragePath.String { + if _, err := tx.Exec(db.Rebind(queuePendingDeleteQuery), previous.String, now.UTC()); err != nil { + return fmt.Errorf("queueing replaced artifact: %w", err) + } + } + return tx.Commit() +} diff --git a/internal/database/local_artifacts_test.go b/internal/database/local_artifacts_test.go index 02335da..08cacea 100644 --- a/internal/database/local_artifacts_test.go +++ b/internal/database/local_artifacts_test.go @@ -2,6 +2,7 @@ package database import ( "database/sql" + "errors" "path/filepath" "testing" "time" @@ -295,3 +296,91 @@ func TestUpsertPackageKeepsSource(t *testing.T) { } }) } + +func localPush(versionPURL, filename, storagePath string) *Artifact { + return &Artifact{ + VersionPURL: versionPURL, + Filename: filename, + StoragePath: sql.NullString{String: storagePath, Valid: true}, + ContentHash: sql.NullString{String: storagePath, Valid: true}, + Size: sql.NullInt64{Int64: 1, Valid: true}, + FetchedAt: sql.NullTime{Time: time.Now(), Valid: true}, + } +} + +func TestRecordLocalArtifactNeverReplacesAPush(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + pkg := &Package{PURL: "pkg:npm/request", Ecosystem: "npm", Name: "request"} + version := func(integrity string) *Version { + return &Version{PURL: "pkg:npm/request@1.0.0", PackagePURL: pkg.PURL, + Integrity: sql.NullString{String: integrity, Valid: true}} + } + + first := localPush("pkg:npm/request@1.0.0", "request-1.0.0.tgz", "npm/request/first") + if err := db.RecordLocalArtifact(pkg, version("sha512-first"), true, first); err != nil { + t.Fatalf("first RecordLocalArtifact: %v", err) + } + second := localPush("pkg:npm/request@1.0.0", "request-1.0.0.tgz", "npm/request/second") + if err := db.RecordLocalArtifact(pkg, version("sha512-second"), true, second); !errors.Is(err, ErrLocalArtifactExists) { + t.Fatalf("second RecordLocalArtifact error = %v, want ErrLocalArtifactExists", err) + } + + art, err := db.GetArtifact("pkg:npm/request@1.0.0", "request-1.0.0.tgz") + if err != nil || art == nil || art.StoragePath.String != "npm/request/first" || art.Source != SourceLocal { + t.Fatalf("artifact after refused push = %+v, %v, want the first push", art, err) + } + v, err := db.GetVersionByPURL("pkg:npm/request@1.0.0") + if err != nil || v == nil || v.Integrity.String != "sha512-first" { + t.Errorf("version integrity after refused push = %+v, %v, want the first push's", v, err) + } + var queued int + if err := db.Get(&queued, db.Rebind(`SELECT COUNT(*) FROM pending_deletes`)); err != nil || queued != 0 { + t.Errorf("queued deletes = %d, %v, want none", queued, err) + } + }) +} + +func TestRecordLocalArtifactReplacesUpstreamCopy(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + seedSourcedArtifact(t, db, "request", SourceUpstream, 100, time.Now()) + pkg := &Package{PURL: "pkg:npm/request", Ecosystem: "npm", Name: "request"} + v := &Version{PURL: "pkg:npm/request@1.0.0", PackagePURL: pkg.PURL} + + pushed := localPush("pkg:npm/request@1.0.0", "request-1.0.0.tgz", "npm/request/pushed") + if err := db.RecordLocalArtifact(pkg, v, true, pushed); err != nil { + t.Fatalf("RecordLocalArtifact: %v", err) + } + art, err := db.GetArtifact("pkg:npm/request@1.0.0", "request-1.0.0.tgz") + if err != nil || art == nil || art.Source != SourceLocal || art.StoragePath.String != "npm/request/pushed" { + t.Fatalf("artifact = %+v, %v, want the push", art, err) + } + const upstreamPath = "npm/request/1.0.0/request-1.0.0.tgz" + var queued int + if err := db.Get(&queued, db.Rebind(`SELECT COUNT(*) FROM pending_deletes WHERE path = ?`), upstreamPath); err != nil || queued != 1 { + t.Errorf("queued deletes for the upstream copy = %d, %v, want 1", queued, err) + } + }) +} + +func TestRecordLocalArtifactCanKeepTheVersionRecord(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + pkg := &Package{PURL: "pkg:generic/tools", Ecosystem: "generic", Name: "tools"} + if err := db.UpsertPackage(pkg); err != nil { + t.Fatalf("UpsertPackage: %v", err) + } + if err := db.UpsertVersion(&Version{PURL: "pkg:generic/tools@1.2.0", PackagePURL: pkg.PURL, + License: sql.NullString{String: "MIT", Valid: true}, + Integrity: sql.NullString{String: "sha256-upstream", Valid: true}}); err != nil { + t.Fatalf("UpsertVersion: %v", err) + } + + pushed := localPush("pkg:generic/tools@1.2.0", "tool.tar.gz", "generic/tools/pushed") + if err := db.RecordLocalArtifact(pkg, &Version{PURL: "pkg:generic/tools@1.2.0", PackagePURL: pkg.PURL}, false, pushed); err != nil { + t.Fatalf("RecordLocalArtifact: %v", err) + } + v, err := db.GetVersionByPURL("pkg:generic/tools@1.2.0") + if err != nil || v == nil || v.Integrity.String != "sha256-upstream" || v.License.String != "MIT" { + t.Errorf("version after push = %+v, %v, want it unchanged", v, err) + } + }) +} diff --git a/internal/handler/generic.go b/internal/handler/generic.go index f411430..700e53e 100644 --- a/internal/handler/generic.go +++ b/internal/handler/generic.go @@ -126,7 +126,7 @@ func (h *GenericHandler) Routes() http.Handler { } repository, rest, ok := strings.Cut(path, "/") - if repository == HostedGenericRepository { + if repository == HostedGenericRepository && h.proxy.HostedEnabled { h.handleHosted(w, r, rest) return } @@ -146,14 +146,14 @@ func (h *GenericHandler) Routes() http.Handler { } // HostedGenericRepository is the /generic/{name}/ path that serves pushed -// generic files instead of an upstream. upstream.generic can't use the name. +// generic files instead of an upstream while hosted packages are enabled. const HostedGenericRepository = config.HostedGenericRepository // handleHosted serves a pushed file at {name}/{version}/{file}. Pushed files // never come from an upstream, so anything not pushed is a 404. func (h *GenericHandler) handleHosted(w http.ResponseWriter, r *http.Request, path string) { segments := strings.Split(path, "/") - if !h.proxy.HostedEnabled || len(segments) != hostedGenericPathSegments { + if len(segments) != hostedGenericPathSegments { http.NotFound(w, r) return } diff --git a/internal/handler/generic_test.go b/internal/handler/generic_test.go index 6948b86..424eafb 100644 --- a/internal/handler/generic_test.go +++ b/internal/handler/generic_test.go @@ -2,6 +2,7 @@ package handler import ( "context" + "io" "net/http" "net/http/httptest" "regexp" @@ -629,3 +630,19 @@ func serveGenericRequest(h *GenericHandler, target string) *httptest.ResponseRec h.Routes().ServeHTTP(w, httptest.NewRequest(http.MethodGet, target, nil)) return w } + +func TestGenericHandler_UpstreamNamedHostedWhileHostingIsOff(t *testing.T) { + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = io.WriteString(w, "from upstream "+r.URL.Path) + })) + t.Cleanup(upstream.Close) + proxy, _, _, _ := setupTestProxy(t) + proxy.HTTPClient = upstream.Client() + h := NewGenericHandler(proxy, map[string]string{"hosted": upstream.URL}) + + w := httptest.NewRecorder() + h.Routes().ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/hosted/index.json", nil)) + if w.Code != http.StatusOK || w.Body.String() != "from upstream /index.json" { + t.Errorf("status = %d, body %q, want the upstream response", w.Code, w.Body) + } +} diff --git a/internal/handler/hosted.go b/internal/handler/hosted.go index 3e15868..72b3929 100644 --- a/internal/handler/hosted.go +++ b/internal/handler/hosted.go @@ -75,6 +75,10 @@ type hostedEcosystem struct { // path is unambiguous. filename func(name, version string) string + // multiFile is set when a version can hold several files. Such versions + // get no version-level integrity, which could only describe one of them. + multiFile bool + // readManifest reads the package manifest out of a pushed archive and // checks that it describes target. When nil, pushes carry no manifest. readManifest func(archive io.Reader, target HostedTarget) (json.RawMessage, error) @@ -111,6 +115,7 @@ var hostedEcosystems = map[string]hostedEcosystem{ // come from an upstream. genericEcosystem: { validName: func(segments []string) bool { return len(segments) == 1 }, + multiFile: true, }, } @@ -181,14 +186,11 @@ func validHostedSegment(s string) bool { // manifests, the archive's manifest must name target; it is kept so the // ecosystem's metadata can list the version with its dependencies. func (p *Proxy) PushLocalArtifact(ctx context.Context, ecosystem string, target HostedTarget, body io.Reader, contentType, wantSHA256 string) (HostedArtifact, error) { - pkgPURL, versionPURL, err := packagePURLStrings(ecosystem, target.Name, target.Version) - if err != nil { + if err := p.CheckPush(ecosystem, target); err != nil { return HostedArtifact{}, err } - if p.versionDenied(ecosystem, target.Name, target.Version) { - return HostedArtifact{}, fmt.Errorf("%w: %s", ErrVersionDenied, versionPURL) - } - if err := p.checkNotPushed(versionPURL, target.Filename); err != nil { + pkgPURL, versionPURL, err := packagePURLStrings(ecosystem, target.Name, target.Version) + if err != nil { return HostedArtifact{}, err } @@ -217,13 +219,6 @@ func (p *Proxy) PushLocalArtifact(ctx context.Context, ecosystem string, target return HostedArtifact{}, err } - // A concurrent push of the same file may have finished while this one - // was being stored and checked. - if err := p.checkNotPushed(versionPURL, target.Filename); err != nil { - p.discardStored(ctx, push.storagePath) - return HostedArtifact{}, err - } - push.pushedAt = time.Now() if err := p.recordLocalArtifact(ecosystem, target, pkgPURL, versionPURL, push); err != nil { p.discardStored(ctx, push.storagePath) @@ -239,10 +234,19 @@ func (p *Proxy) PushLocalArtifact(ctx context.Context, ecosystem string, target }, nil } -// checkNotPushed refuses a push of a file that was already pushed. A cached -// upstream copy under the same name is fine to replace. -func (p *Proxy) checkNotPushed(versionPURL, filename string) error { - record, err := p.DB.GetArtifact(versionPURL, filename) +// CheckPush refuses a push of a denied version or of a file that was already +// pushed, before any bytes are read, so a client can be told without +// uploading the file. A cached upstream copy under the same name is fine to +// replace. Recording the push checks again atomically, for concurrent pushes. +func (p *Proxy) CheckPush(ecosystem string, target HostedTarget) error { + _, versionPURL, err := packagePURLStrings(ecosystem, target.Name, target.Version) + if err != nil { + return err + } + if p.versionDenied(ecosystem, target.Name, target.Version) { + return fmt.Errorf("%w: %s", ErrVersionDenied, versionPURL) + } + record, err := p.DB.GetArtifact(versionPURL, target.Filename) if err != nil { return fmt.Errorf("checking existing artifact: %w", err) } @@ -287,45 +291,38 @@ func (p *Proxy) discardStored(ctx context.Context, storagePath string) { } } -// recordLocalArtifact writes the package, version and artifact records. The -// version's integrity is set to the pushed bytes, so the cache's integrity -// check verifies them rather than an upstream archive's digest. An existing -// package record is kept as it is, since it may describe the upstream package. +// recordLocalArtifact writes the package, version and artifact records in +// one transaction. An existing package record is kept as it is, since it may +// describe the upstream package. For ecosystems with one file per version, +// the version's integrity is set to the pushed bytes, so the cache's +// integrity check verifies them rather than an upstream archive's digest. +// Multi-file versions can't have one integrity, so an existing version record +// is left alone and each file is checked against its own sha256. func (p *Proxy) recordLocalArtifact(ecosystem string, target HostedTarget, pkgPURL, versionPURL string, push storedPush) error { - existing, err := p.DB.GetPackageByPURL(pkgPURL) + oneFilePerVersion := !hostedEcosystems[ecosystem].multiFile + err := p.DB.RecordLocalArtifact( + &database.Package{PURL: pkgPURL, Ecosystem: ecosystem, Name: target.Name}, + &database.Version{ + PURL: versionPURL, + PackagePURL: pkgPURL, + Integrity: sql.NullString{String: push.integrity, Valid: oneFilePerVersion}, + }, + oneFilePerVersion, + &database.Artifact{ + VersionPURL: versionPURL, + Filename: target.Filename, + StoragePath: sql.NullString{String: push.storagePath, Valid: true}, + ContentHash: sql.NullString{String: push.sha256, Valid: true}, + Size: sql.NullInt64{Int64: push.size, Valid: true}, + ContentType: sql.NullString{String: push.contentType, Valid: push.contentType != ""}, + FetchedAt: sql.NullTime{Time: push.pushedAt, Valid: true}, + Manifest: sql.NullString{String: string(push.manifest), Valid: push.manifest != nil}, + }) + if errors.Is(err, database.ErrLocalArtifactExists) { + return ErrHostedExists + } if err != nil { - return fmt.Errorf("reading package: %w", err) - } - if existing == nil { - if err := p.DB.UpsertPackage(&database.Package{PURL: pkgPURL, Ecosystem: ecosystem, Name: target.Name}); err != nil { - return fmt.Errorf("upserting package: %w", err) - } - } - - // Ecosystems that name the file in the push path hold several files per - // version, and the version's integrity can only describe one of them. - // Their files are checked against their own sha256 instead. - oneFilePerVersion := hostedEcosystems[ecosystem].filename != nil - if err := p.DB.UpsertVersion(&database.Version{ - PURL: versionPURL, - PackagePURL: pkgPURL, - Integrity: sql.NullString{String: push.integrity, Valid: oneFilePerVersion}, - }); err != nil { - return fmt.Errorf("upserting version: %w", err) - } - - if err := p.DB.UpsertArtifact(&database.Artifact{ - VersionPURL: versionPURL, - Filename: target.Filename, - StoragePath: sql.NullString{String: push.storagePath, Valid: true}, - ContentHash: sql.NullString{String: push.sha256, Valid: true}, - Size: sql.NullInt64{Int64: push.size, Valid: true}, - ContentType: sql.NullString{String: push.contentType, Valid: push.contentType != ""}, - FetchedAt: sql.NullTime{Time: push.pushedAt, Valid: true}, - Source: database.SourceLocal, - Manifest: sql.NullString{String: string(push.manifest), Valid: push.manifest != nil}, - }); err != nil { - return fmt.Errorf("upserting artifact: %w", err) + return fmt.Errorf("recording pushed artifact: %w", err) } return nil } diff --git a/internal/handler/hosted_test.go b/internal/handler/hosted_test.go index 252d94b..01993de 100644 --- a/internal/handler/hosted_test.go +++ b/internal/handler/hosted_test.go @@ -7,8 +7,10 @@ import ( "compress/gzip" "context" "errors" + "fmt" "io" "strings" + "sync" "testing" "github.com/git-pkgs/proxy/internal/database" @@ -436,3 +438,54 @@ func TestComposerManifestLookup(t *testing.T) { }) } } + +func TestConcurrentPushesOfTheSameFileHaveOneWinner(t *testing.T) { + proxy, _, _, _ := setupTestProxy(t) + proxy.HostedEnabled = true + target, err := HostedTargetFor("generic", "tools", "1.2.0", "tool.tar.gz") + if err != nil { + t.Fatalf("HostedTargetFor: %v", err) + } + + const pushes = 8 + start := make(chan struct{}) + results := make([]error, pushes) + pushed := make([]HostedArtifact, pushes) + var wg sync.WaitGroup + for i := range pushes { + wg.Add(1) + go func() { + defer wg.Done() + <-start + body := strings.NewReader(fmt.Sprintf("build %d", i)) + pushed[i], results[i] = proxy.PushLocalArtifact(context.Background(), "generic", target, body, "", "") + }() + } + close(start) + wg.Wait() + + winner := -1 + for i, err := range results { + switch { + case err == nil && winner >= 0: + t.Fatalf("pushes %d and %d both succeeded", winner, i) + case err == nil: + winner = i + case !errors.Is(err, ErrHostedExists): + t.Fatalf("push %d error = %v, want ErrHostedExists", i, err) + } + } + if winner < 0 { + t.Fatal("no push succeeded") + } + + result, err := proxy.GetLocalArtifact(context.Background(), "generic", "tools", "1.2.0", "tool.tar.gz") + if err != nil || result == nil { + t.Fatalf("GetLocalArtifact = %v, %v", result, err) + } + served, _ := io.ReadAll(result.Reader) + _ = result.Reader.Close() + if string(served) != fmt.Sprintf("build %d", winner) { + t.Errorf("served %q, want the winning push %q", served, fmt.Sprintf("build %d", winner)) + } +} diff --git a/internal/server/hosted_api.go b/internal/server/hosted_api.go index 1e4e1a4..ff4efeb 100644 --- a/internal/server/hosted_api.go +++ b/internal/server/hosted_api.go @@ -88,6 +88,13 @@ func (h *HostedAPIHandler) HandlePush(w http.ResponseWriter, r *http.Request) { } ecosystem := path.ecosystem + // Refuse before reading the body, so a client sending + // "Expect: 100-continue" isn't made to upload a file that is refused. + if err := h.proxy.CheckPush(ecosystem, target); err != nil { + h.pushError(w, err) + return + } + body := bufio.NewReader(http.MaxBytesReader(w, r.Body, h.maxUploadBytes)) if _, err := body.Peek(1); errors.Is(err, io.EOF) { badRequest(w, "request body is empty") diff --git a/internal/server/hosted_api_test.go b/internal/server/hosted_api_test.go index 2185f3a..eaa5a56 100644 --- a/internal/server/hosted_api_test.go +++ b/internal/server/hosted_api_test.go @@ -482,3 +482,27 @@ func TestHostedGenericFileIsServedByTheRunningServer(t *testing.T) { t.Errorf("download status = %d, body %q", resp.StatusCode, body) } } + +// unreadBody fails the test if the handler reads it. +type unreadBody struct{ t *testing.T } + +func (b unreadBody) Read([]byte) (int, error) { + b.t.Error("body of a refused push was read") + return 0, io.EOF +} + +func TestHostedRepushIsRefusedBeforeTheBodyIsRead(t *testing.T) { + s := newHostedTestServer(t) + const target = "/api/hosted/generic/tools/1.2.0/tool.tar.gz" + if rec := s.do(t, http.MethodPut, target, testHostedToken, "first"); rec.Code != http.StatusCreated { + t.Fatalf("first PUT status = %d, body %s", rec.Code, rec.Body) + } + + req := httptest.NewRequest(http.MethodPut, target, unreadBody{t}) + req.Header.Set("Authorization", "Bearer "+testHostedToken) + rec := httptest.NewRecorder() + s.router.ServeHTTP(rec, req) + if rec.Code != http.StatusConflict { + t.Fatalf("second PUT status = %d, want 409, body %s", rec.Code, rec.Body) + } +} From 5f431a3d892efa6db480820ba5818d68db71928e Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Tue, 20 Oct 2026 20:07:04 +0200 Subject: [PATCH 337/633] Test concurrent pushes of one file at the database level The handler-level test runs on SQLite only. This one also runs against Postgres when PROXY_DATABASE_URL is set. --- internal/database/local_artifacts_test.go | 35 +++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/internal/database/local_artifacts_test.go b/internal/database/local_artifacts_test.go index 08cacea..7930675 100644 --- a/internal/database/local_artifacts_test.go +++ b/internal/database/local_artifacts_test.go @@ -3,7 +3,9 @@ package database import ( "database/sql" "errors" + "fmt" "path/filepath" + "sync" "testing" "time" ) @@ -384,3 +386,36 @@ func TestRecordLocalArtifactCanKeepTheVersionRecord(t *testing.T) { } }) } + +func TestRecordLocalArtifactConcurrentPushesHaveOneWinner(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + pkg := &Package{PURL: "pkg:generic/tools", Ecosystem: "generic", Name: "tools"} + const pushes = 8 + start := make(chan struct{}) + errs := make([]error, pushes) + var wg sync.WaitGroup + for i := range pushes { + wg.Go(func() { + <-start + v := &Version{PURL: "pkg:generic/tools@1.2.0", PackagePURL: pkg.PURL} + a := localPush("pkg:generic/tools@1.2.0", "tool.tar.gz", fmt.Sprintf("generic/tools/push-%d", i)) + errs[i] = db.RecordLocalArtifact(pkg, v, false, a) + }) + } + close(start) + wg.Wait() + + winners := 0 + for i, err := range errs { + switch { + case err == nil: + winners++ + case !errors.Is(err, ErrLocalArtifactExists): + t.Errorf("push %d error = %v, want ErrLocalArtifactExists", i, err) + } + } + if winners != 1 { + t.Errorf("%d pushes succeeded, want exactly 1", winners) + } + }) +} From 48916bcb09591eae000f57c6220dca6107ebe2e5 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Tue, 20 Oct 2026 20:21:01 +0200 Subject: [PATCH 338/633] Never let a cache fill replace a pushed artifact - Record a fetched artifact with its package and version in one transaction that refuses to replace a pushed artifact, so a fetch that finishes after a push keeps both the pushed file and its version's integrity. Pushes and cache fills share this write, including the compare-and-swap on the storage path that queues every replaced object. - Don't open the stored file for ecosystems without a manifest. - Derive multi-file versions from the file being named in the path. - Validate DELETE paths like PUT paths, answering 400 for malformed ones. - Parse ?sha256= with integrity.ParseHex. --- internal/database/local_artifacts.go | 95 ++++------------ internal/database/local_artifacts_test.go | 35 ++++++ internal/database/pending_deletes_test.go | 4 +- internal/database/queries.go | 125 +++++++++++++++++----- internal/handler/handler.go | 22 ++-- internal/handler/hosted.go | 73 ++++++++----- internal/handler/hosted_test.go | 42 ++++++++ internal/server/hosted_api.go | 12 +-- internal/server/hosted_api_test.go | 15 +++ 9 files changed, 277 insertions(+), 146 deletions(-) diff --git a/internal/database/local_artifacts.go b/internal/database/local_artifacts.go index 1bd8555..26aea31 100644 --- a/internal/database/local_artifacts.go +++ b/internal/database/local_artifacts.go @@ -2,7 +2,6 @@ package database import ( "database/sql" - "errors" "fmt" "time" ) @@ -75,10 +74,6 @@ func (db *DB) DeleteLocalArtifacts(versionPURL, filename string) (int, error) { return int(deleted), tx.Commit() } -// ErrLocalArtifactExists is returned by RecordLocalArtifact when the file -// was already pushed. -var ErrLocalArtifactExists = errors.New("local artifact already exists") - // RecordLocalArtifact records a pushed artifact in one transaction: the // package if it is missing, the version, and the artifact. An artifact cached // from an upstream under the same name is replaced and its stored object @@ -89,72 +84,28 @@ var ErrLocalArtifactExists = errors.New("local artifact already exists") // When setIntegrity is false, an existing version record is left as it is. // Otherwise its integrity is set to v.Integrity. func (db *DB) RecordLocalArtifact(pkg *Package, v *Version, setIntegrity bool, a *Artifact) error { - tx, err := db.Beginx() - if err != nil { - return err - } - defer func() { _ = tx.Rollback() }() - now := time.Now() - - if _, err := tx.Exec(db.Rebind(` - INSERT INTO packages (purl, ecosystem, name, created_at, updated_at) - VALUES (?, ?, ?, ?, ?) - ON CONFLICT(purl) DO NOTHING - `), pkg.PURL, pkg.Ecosystem, pkg.Name, now, now); err != nil { - return fmt.Errorf("inserting package: %w", err) - } - - versionConflict := `DO NOTHING` - if setIntegrity { - versionConflict = `DO UPDATE SET integrity = excluded.integrity, updated_at = excluded.updated_at` - } - if _, err := tx.Exec(db.Rebind(` - INSERT INTO versions (purl, package_purl, integrity, created_at, updated_at) - VALUES (?, ?, ?, ?, ?) - ON CONFLICT(purl) `+versionConflict, - ), v.PURL, v.PackagePURL, v.Integrity, now, now); err != nil { - return fmt.Errorf("recording version: %w", err) - } - - var previous sql.NullString - if err := tx.Get(&previous, db.Rebind(`SELECT storage_path FROM artifacts WHERE version_purl = ? AND filename = ?`), - a.VersionPURL, a.Filename); err != nil && !errors.Is(err, sql.ErrNoRows) { - return fmt.Errorf("reading artifact: %w", err) - } - - res, err := tx.Exec(db.Rebind(` - INSERT INTO artifacts (version_purl, filename, upstream_url, storage_path, content_hash, - size, content_type, fetched_at, hit_count, created_at, updated_at, - source, manifest) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, 0, ?, ?, ?, ?) - ON CONFLICT(version_purl, filename) DO UPDATE SET - upstream_url = excluded.upstream_url, - storage_path = excluded.storage_path, - content_hash = excluded.content_hash, - size = excluded.size, - content_type = excluded.content_type, - fetched_at = excluded.fetched_at, - updated_at = excluded.updated_at, - source = excluded.source, - manifest = excluded.manifest - WHERE artifacts.source IS NULL OR artifacts.source <> 'local' - `), a.VersionPURL, a.Filename, a.UpstreamURL, a.StoragePath, a.ContentHash, - a.Size, a.ContentType, inUTC(a.FetchedAt), now, now, SourceLocal, a.Manifest) - if err != nil { - return fmt.Errorf("recording artifact: %w", err) - } - n, err := res.RowsAffected() - if err != nil { - return fmt.Errorf("recording artifact: %w", err) - } - if n == 0 { - return ErrLocalArtifactExists - } - - if previous.Valid && previous.String != a.StoragePath.String { - if _, err := tx.Exec(db.Rebind(queuePendingDeleteQuery), previous.String, now.UTC()); err != nil { - return fmt.Errorf("queueing replaced artifact: %w", err) + local := *a + local.Source = SourceLocal + return db.recordArtifact(&local, func(tx execer) error { + now := time.Now() + if _, err := tx.Exec(db.Rebind(` + INSERT INTO packages (purl, ecosystem, name, created_at, updated_at) + VALUES (?, ?, ?, ?, ?) + ON CONFLICT(purl) DO NOTHING + `), pkg.PURL, pkg.Ecosystem, pkg.Name, now, now); err != nil { + return fmt.Errorf("inserting package: %w", err) } - } - return tx.Commit() + versionConflict := `DO NOTHING` + if setIntegrity { + versionConflict = `DO UPDATE SET integrity = excluded.integrity, updated_at = excluded.updated_at` + } + if _, err := tx.Exec(db.Rebind(` + INSERT INTO versions (purl, package_purl, integrity, created_at, updated_at) + VALUES (?, ?, ?, ?, ?) + ON CONFLICT(purl) `+versionConflict, + ), v.PURL, v.PackagePURL, v.Integrity, now, now); err != nil { + return fmt.Errorf("recording version: %w", err) + } + return nil + }, ErrLocalArtifactExists) } diff --git a/internal/database/local_artifacts_test.go b/internal/database/local_artifacts_test.go index 7930675..b476d8d 100644 --- a/internal/database/local_artifacts_test.go +++ b/internal/database/local_artifacts_test.go @@ -419,3 +419,38 @@ func TestRecordLocalArtifactConcurrentPushesHaveOneWinner(t *testing.T) { } }) } + +func TestRecordCachedArtifactNeverReplacesAPush(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + pkg := &Package{PURL: "pkg:npm/request", Ecosystem: "npm", Name: "request"} + pushed := localPush("pkg:npm/request@1.0.0", "request-1.0.0.tgz", "npm/request/pushed") + if err := db.RecordLocalArtifact(pkg, &Version{PURL: "pkg:npm/request@1.0.0", PackagePURL: pkg.PURL, + Integrity: sql.NullString{String: "sha512-pushed", Valid: true}}, true, pushed); err != nil { + t.Fatalf("RecordLocalArtifact: %v", err) + } + + fetched := &Artifact{ + VersionPURL: "pkg:npm/request@1.0.0", + Filename: "request-1.0.0.tgz", + UpstreamURL: "https://registry.example/request-1.0.0.tgz", + StoragePath: sql.NullString{String: "npm/request/fetched", Valid: true}, + } + err := db.RecordCachedArtifact(pkg, &Version{PURL: "pkg:npm/request@1.0.0", PackagePURL: pkg.PURL, + Integrity: sql.NullString{String: "sha512-upstream", Valid: true}}, fetched) + if !errors.Is(err, ErrLocalArtifactExists) { + t.Fatalf("RecordCachedArtifact error = %v, want ErrLocalArtifactExists", err) + } + if err := db.UpsertArtifact(fetched); !errors.Is(err, ErrLocalArtifactExists) { + t.Errorf("UpsertArtifact error = %v, want ErrLocalArtifactExists", err) + } + + art, err := db.GetArtifact("pkg:npm/request@1.0.0", "request-1.0.0.tgz") + if err != nil || art == nil || art.Source != SourceLocal || art.StoragePath.String != "npm/request/pushed" { + t.Errorf("artifact = %+v, %v, want the push", art, err) + } + v, err := db.GetVersionByPURL("pkg:npm/request@1.0.0") + if err != nil || v == nil || v.Integrity.String != "sha512-pushed" { + t.Errorf("version = %+v, %v, want the push's integrity", v, err) + } + }) +} diff --git a/internal/database/pending_deletes_test.go b/internal/database/pending_deletes_test.go index 9af6c48..82bdaca 100644 --- a/internal/database/pending_deletes_test.go +++ b/internal/database/pending_deletes_test.go @@ -79,7 +79,7 @@ func TestUpsertArtifactSkipsRecordThatMoved(t *testing.T) { UpstreamURL: "https://example.com/" + pendingFilename, StoragePath: sql.NullString{String: "npm/pending/1.0.0/c/pending-1.0.0.tgz", Valid: true}, } - applied, err := db.upsertArtifactFrom(late, read) + applied, err := db.upsertArtifactFrom(db, late, read) if err != nil { t.Fatalf("upsertArtifactFrom: %v", err) } @@ -91,7 +91,7 @@ func TestUpsertArtifactSkipsRecordThatMoved(t *testing.T) { } missing := &Artifact{VersionPURL: "pkg:npm/pending@2.0.0", Filename: pendingFilename, UpstreamURL: "u"} - applied, err = db.upsertArtifactFrom(missing, sql.NullString{}) + applied, err = db.upsertArtifactFrom(db, missing, sql.NullString{}) if err != nil || !applied { t.Errorf("insert of a new record: applied=%v err=%v", applied, err) } diff --git a/internal/database/queries.go b/internal/database/queries.go index 33ea93d..9560d88 100644 --- a/internal/database/queries.go +++ b/internal/database/queries.go @@ -52,7 +52,17 @@ func (db *DB) GetPackageByEcosystemName(ecosystem, name string) (*Package, error return &pkg, nil } +// execer runs statements on the database or inside a transaction. +type execer interface { + Exec(query string, args ...any) (sql.Result, error) + Get(dest any, query string, args ...any) error +} + func (db *DB) UpsertPackage(pkg *Package) error { + return db.upsertPackage(db, pkg) +} + +func (db *DB) upsertPackage(exec execer, pkg *Package) error { now := time.Now() var query string @@ -90,7 +100,7 @@ func (db *DB) UpsertPackage(pkg *Package) error { ` } - _, err := db.Exec(query, + _, err := exec.Exec(query, pkg.PURL, pkg.Ecosystem, pkg.Name, pkg.LatestVersion, pkg.License, pkg.Description, pkg.Homepage, pkg.RepositoryURL, pkg.RegistryURL, pkg.EnrichedAt, now, now, @@ -136,6 +146,10 @@ func (db *DB) GetVersionsByPackagePURL(packagePURL string) ([]Version, error) { } func (db *DB) UpsertVersion(v *Version) error { + return db.upsertVersion(db, v) +} + +func (db *DB) upsertVersion(exec execer, v *Version) error { now := time.Now() var query string @@ -167,7 +181,7 @@ func (db *DB) UpsertVersion(v *Version) error { ` } - _, err := db.Exec(query, + _, err := exec.Exec(query, v.PURL, v.PackagePURL, v.License, v.Integrity, v.PublishedAt, v.Yanked, v.EnrichedAt, now, now, ) @@ -334,32 +348,87 @@ func inUTC(t sql.NullTime) sql.NullTime { } func (db *DB) UpsertArtifact(a *Artifact) error { - for range upsertArtifactAttempts { - var previous sql.NullString - query := db.Rebind(`SELECT storage_path FROM artifacts WHERE version_purl = ? AND filename = ?`) - if err := db.Get(&previous, query, a.VersionPURL, a.Filename); err != nil && !errors.Is(err, sql.ErrNoRows) { - return fmt.Errorf("reading artifact: %w", err) - } - applied, err := db.upsertArtifactFrom(a, previous) - if err != nil { - return err - } - if !applied { - continue - } - if previous.Valid && previous.String != a.StoragePath.String { - if err := db.QueuePendingDelete(previous.String); err != nil { - return fmt.Errorf("queueing replaced artifact: %w", err) - } - } - return nil - } - return errors.New("upserting artifact: record kept changing") + return db.recordArtifact(a, nil, ErrLocalArtifactExists) } -// upsertArtifactFrom writes a if the record is absent or still holds -// previous, and reports whether it did. -func (db *DB) upsertArtifactFrom(a *Artifact, previous sql.NullString) (bool, error) { +// ErrLocalArtifactExists is returned when an artifact would replace one that +// was pushed to the proxy. Pushed artifacts are only removed by deleting them. +var ErrLocalArtifactExists = errors.New("a pushed artifact already exists") + +// RecordCachedArtifact records an artifact fetched from an upstream together +// with its package and version, in one transaction. If the artifact was +// pushed in the meantime, nothing is written and ErrLocalArtifactExists is +// returned, so the push's version integrity is kept as well. +func (db *DB) RecordCachedArtifact(pkg *Package, v *Version, a *Artifact) error { + return db.recordArtifact(a, func(tx execer) error { + if err := db.upsertPackage(tx, pkg); err != nil { + return err + } + return db.upsertVersion(tx, v) + }, ErrLocalArtifactExists) +} + +// artifactSlot is what an artifact record holds before it is replaced. +type artifactSlot struct { + StoragePath sql.NullString `db:"storage_path"` + Source sql.NullString `db:"source"` +} + +// recordArtifact writes before's records and a in one transaction. A pushed +// artifact is never replaced: the transaction is rolled back and localErr +// returned. A storage path the record stops pointing at is queued for +// deletion in the same transaction, since each fetch or push stores its own +// object and nothing else would remove it. The write applies only if the +// record still holds the path read before it, so commits racing on one +// artifact each queue the path they replaced. +func (db *DB) recordArtifact(a *Artifact, before func(tx execer) error, localErr error) error { + for range upsertArtifactAttempts { + done, err := db.tryRecordArtifact(a, before, localErr) + if err != nil || done { + return err + } + } + return errors.New("recording artifact: record kept changing") +} + +func (db *DB) tryRecordArtifact(a *Artifact, before func(tx execer) error, localErr error) (bool, error) { + tx, err := db.Beginx() + if err != nil { + return false, err + } + defer func() { _ = tx.Rollback() }() + + if before != nil { + if err := before(tx); err != nil { + return false, err + } + } + + var current artifactSlot + query := db.Rebind(`SELECT storage_path, source FROM artifacts WHERE version_purl = ? AND filename = ?`) + if err := tx.Get(¤t, query, a.VersionPURL, a.Filename); err != nil && !errors.Is(err, sql.ErrNoRows) { + return false, fmt.Errorf("reading artifact: %w", err) + } + if current.Source.String == SourceLocal { + return false, localErr + } + + applied, err := db.upsertArtifactFrom(tx, a, current.StoragePath) + if err != nil || !applied { + return false, err + } + previous := current.StoragePath + if previous.Valid && previous.String != a.StoragePath.String { + if _, err := tx.Exec(db.Rebind(queuePendingDeleteQuery), previous.String, time.Now().UTC()); err != nil { + return false, fmt.Errorf("queueing replaced artifact: %w", err) + } + } + return true, tx.Commit() +} + +// upsertArtifactFrom writes a if the record is absent, or still holds +// previous and wasn't pushed, and reports whether it did. +func (db *DB) upsertArtifactFrom(exec execer, a *Artifact, previous sql.NullString) (bool, error) { now := time.Now() var query string @@ -380,6 +449,7 @@ func (db *DB) upsertArtifactFrom(a *Artifact, previous sql.NullString) (bool, er source = EXCLUDED.source, manifest = EXCLUDED.manifest WHERE artifacts.storage_path IS NOT DISTINCT FROM $15 + AND (artifacts.source IS NULL OR artifacts.source <> 'local') ` } else { query = ` @@ -398,10 +468,11 @@ func (db *DB) upsertArtifactFrom(a *Artifact, previous sql.NullString) (bool, er source = excluded.source, manifest = excluded.manifest WHERE artifacts.storage_path IS ? + AND (artifacts.source IS NULL OR artifacts.source <> 'local') ` } - res, err := db.Exec(query, + res, err := exec.Exec(query, a.VersionPURL, a.Filename, a.UpstreamURL, a.StoragePath, a.ContentHash, a.Size, a.ContentType, inUTC(a.FetchedAt), a.HitCount, inUTC(a.LastAccessedAt), now, now, a.sourceOrUpstream(), a.Manifest, previous, diff --git a/internal/handler/handler.go b/internal/handler/handler.go index d29508e..9280a04 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -642,7 +642,12 @@ func (p *Proxy) storeArtifact(ctx context.Context, ecosystem, name, version, fil // Update database if err := p.updateCacheDB(ecosystem, name, pkgPURL, upstreamURL, storagePath, sharedArtifact, declared); err != nil { - p.Logger.Warn("failed to update cache database", "error", err) + if errors.Is(err, database.ErrLocalArtifactExists) { + p.Logger.Info("artifact was pushed while it was fetched; keeping the pushed copy", + "purl", versionPURL, "filename", filename) + } else { + p.Logger.Warn("failed to update cache database", "error", err) + } // Continue anyway - we have the file. Queue it for deletion in case // no record points at it; the queue skips it while one does. if qErr := p.DB.QueuePendingDelete(storagePath); qErr != nil { @@ -903,10 +908,6 @@ func (p *Proxy) updateCacheDB(ecosystem, name, pkgPURL, upstreamURL, storagePath RegistryURL: sql.NullString{String: upstreamURL, Valid: true}, EnrichedAt: sql.NullTime{Time: now, Valid: true}, } - if err := p.DB.UpsertPackage(pkg); err != nil { - return fmt.Errorf("upserting package: %w", err) - } - // Upsert version ver := &database.Version{ PURL: artifact.PURL, @@ -914,10 +915,6 @@ func (p *Proxy) updateCacheDB(ecosystem, name, pkgPURL, upstreamURL, storagePath Integrity: sql.NullString{String: declared, Valid: declared != ""}, EnrichedAt: sql.NullTime{Time: now, Valid: true}, } - if err := p.DB.UpsertVersion(ver); err != nil { - return fmt.Errorf("upserting version: %w", err) - } - // Upsert artifact art := &database.Artifact{ VersionPURL: artifact.PURL, @@ -929,10 +926,11 @@ func (p *Proxy) updateCacheDB(ecosystem, name, pkgPURL, upstreamURL, storagePath ContentType: sql.NullString{String: artifact.MediaType, Valid: true}, FetchedAt: sql.NullTime{Time: now, Valid: true}, } - if err := p.DB.UpsertArtifact(art); err != nil { - return fmt.Errorf("upserting artifact: %w", err) + // One transaction, so a push that landed while this was fetched keeps + // both its artifact and its version's integrity. + if err := p.DB.RecordCachedArtifact(pkg, ver, art); err != nil { + return fmt.Errorf("recording artifact: %w", err) } - return nil } diff --git a/internal/handler/hosted.go b/internal/handler/hosted.go index 72b3929..6342a54 100644 --- a/internal/handler/hosted.go +++ b/internal/handler/hosted.go @@ -71,14 +71,10 @@ type hostedEcosystem struct { // filename is the name clients request the archive by, derived from the // package name and version. When nil, the filename is the last segment of - // the push path, and the package name must be a single segment so the - // path is unambiguous. + // the push path, the package name must be a single segment so the path is + // unambiguous, and a version can hold several files (see multiFile). filename func(name, version string) string - // multiFile is set when a version can hold several files. Such versions - // get no version-level integrity, which could only describe one of them. - multiFile bool - // readManifest reads the package manifest out of a pushed archive and // checks that it describes target. When nil, pushes carry no manifest. readManifest func(archive io.Reader, target HostedTarget) (json.RawMessage, error) @@ -115,10 +111,16 @@ var hostedEcosystems = map[string]hostedEcosystem{ // come from an upstream. genericEcosystem: { validName: func(segments []string) bool { return len(segments) == 1 }, - multiFile: true, }, } +// multiFile reports whether a version can hold several files: those named in +// the push path. Such versions get no version-level integrity, which could +// only describe one of them. +func (e hostedEcosystem) multiFile() bool { + return e.filename == nil +} + // HostedSupported reports whether packages can be pushed for ecosystem. func HostedSupported(ecosystem string) bool { _, ok := hostedEcosystems[ecosystem] @@ -130,7 +132,7 @@ func HostedSupported(ecosystem string) bool { // version. func HostedFileInPath(ecosystem string) bool { eco, ok := hostedEcosystems[ecosystem] - return ok && eco.filename == nil + return ok && eco.multiFile() } func lastSegment(name string) string { @@ -141,29 +143,44 @@ func lastSegment(name string) string { // where it is stored. filename is required for ecosystems whose pushes name // the file (see HostedFileInPath) and must be empty for the others. func HostedTargetFor(ecosystem, name, version, filename string) (HostedTarget, error) { + if err := ValidateHostedPath(ecosystem, name, version, filename); err != nil { + return HostedTarget{}, err + } + eco := hostedEcosystems[ecosystem] + if eco.filename != nil { + filename = eco.filename(name, version) + } else if filename == "" { + return HostedTarget{}, ErrHostedInvalidName + } + return HostedTarget{Name: name, Version: version, Filename: filename}, nil +} + +// ValidateHostedPath checks the parts of a hosted API path: a supported +// ecosystem, one of its package names, and path segments that are safe to +// store. filename may be empty, as in a delete of a whole version, but is +// only allowed for ecosystems whose pushes name the file. +func ValidateHostedPath(ecosystem, name, version, filename string) error { eco, ok := hostedEcosystems[ecosystem] if !ok { - return HostedTarget{}, fmt.Errorf("%w: %s", ErrHostedUnsupported, ecosystem) + return fmt.Errorf("%w: %s", ErrHostedUnsupported, ecosystem) } segments := strings.Split(name, "/") if !eco.validName(segments) { - return HostedTarget{}, ErrHostedInvalidName + return ErrHostedInvalidName } checked := append(slices.Clone(segments), version) - if eco.filename == nil { + if filename != "" { + if !eco.multiFile() { + return ErrHostedInvalidName + } checked = append(checked, filename) - } else if filename != "" { - return HostedTarget{}, ErrHostedInvalidName } for _, segment := range checked { if !validHostedSegment(segment) { - return HostedTarget{}, ErrHostedInvalidName + return ErrHostedInvalidName } } - if eco.filename != nil { - filename = eco.filename(name, version) - } - return HostedTarget{Name: name, Version: version, Filename: filename}, nil + return nil } func validHostedSegment(s string) bool { @@ -269,14 +286,16 @@ type storedPush struct { // checkPush reads the manifest of a stored push and runs the scanners on it. func (p *Proxy) checkPush(ctx context.Context, ecosystem string, target HostedTarget, versionPURL string, push *storedPush) error { - stored, err := p.Storage.Open(ctx, push.storagePath) - if err != nil { - return fmt.Errorf("reading pushed archive: %w", err) - } - push.manifest, err = readHostedManifest(ecosystem, stored, target) - _ = stored.Close() - if err != nil { - return err + if hostedEcosystems[ecosystem].readManifest != nil { + stored, err := p.Storage.Open(ctx, push.storagePath) + if err != nil { + return fmt.Errorf("reading pushed archive: %w", err) + } + push.manifest, err = readHostedManifest(ecosystem, stored, target) + _ = stored.Close() + if err != nil { + return err + } } if p.Scanners != nil && p.Scanners.Enabled() { @@ -299,7 +318,7 @@ func (p *Proxy) discardStored(ctx context.Context, storagePath string) { // Multi-file versions can't have one integrity, so an existing version record // is left alone and each file is checked against its own sha256. func (p *Proxy) recordLocalArtifact(ecosystem string, target HostedTarget, pkgPURL, versionPURL string, push storedPush) error { - oneFilePerVersion := !hostedEcosystems[ecosystem].multiFile + oneFilePerVersion := !hostedEcosystems[ecosystem].multiFile() err := p.DB.RecordLocalArtifact( &database.Package{PURL: pkgPURL, Ecosystem: ecosystem, Name: target.Name}, &database.Version{ diff --git a/internal/handler/hosted_test.go b/internal/handler/hosted_test.go index 01993de..fbe00a5 100644 --- a/internal/handler/hosted_test.go +++ b/internal/handler/hosted_test.go @@ -489,3 +489,45 @@ func TestConcurrentPushesOfTheSameFileHaveOneWinner(t *testing.T) { t.Errorf("served %q, want the winning push %q", served, fmt.Sprintf("build %d", winner)) } } + +// A fetch that started before a push and records after it must not replace +// the pushed artifact or its version's integrity. +func TestCacheFillAfterPushKeepsThePush(t *testing.T) { + proxy, _, store, _ := setupTestProxy(t) + proxy.HostedEnabled = true + pushRequest(t, proxy) + + const fetchedPath = "npm/request/2.88.3/fetched/request-2.88.3.tgz" + if _, _, err := store.Store(context.Background(), fetchedPath, strings.NewReader("upstream bytes")); err != nil { + t.Fatalf("storing fetched copy: %v", err) + } + fetched := testArtifact("upstream bytes", "pkg:npm/request@2.88.3", "request-2.88.3.tgz", "application/gzip") + err := proxy.updateCacheDB("npm", "request", "pkg:npm/request", "https://registry.npmjs.org/request/-/request-2.88.3.tgz", + fetchedPath, fetched, "sha512-upstream") + if !errors.Is(err, database.ErrLocalArtifactExists) { + t.Fatalf("updateCacheDB over a push error = %v, want ErrLocalArtifactExists", err) + } + + result, err := proxy.GetLocalArtifact(context.Background(), "npm", "request", "2.88.3", "request-2.88.3.tgz") + if err != nil || result == nil { + t.Fatalf("GetLocalArtifact = %v, %v, want the pushed copy", result, err) + } + served, err := io.ReadAll(result.Reader) + _ = result.Reader.Close() + if err != nil || string(served) != pushedTarball { + t.Errorf("served %d bytes, %v, want the pushed tarball, integrity intact", len(served), err) + } +} + +func TestGenericPushDoesNotReadTheStoredFile(t *testing.T) { + proxy, _, store, _ := setupTestProxy(t) + proxy.HostedEnabled = true + target, err := HostedTargetFor("generic", "tools", "1.2.0", "tool.tar.gz") + if err != nil { + t.Fatalf("HostedTargetFor: %v", err) + } + store.openErr = errors.New("storage opened") + if _, err := proxy.PushLocalArtifact(context.Background(), "generic", target, strings.NewReader("bytes"), "", ""); err != nil { + t.Fatalf("PushLocalArtifact: %v", err) + } +} diff --git a/internal/server/hosted_api.go b/internal/server/hosted_api.go index ff4efeb..6940156 100644 --- a/internal/server/hosted_api.go +++ b/internal/server/hosted_api.go @@ -2,14 +2,13 @@ package server import ( "bufio" - "crypto/sha256" - "encoding/hex" "errors" "io" "log/slog" "net/http" "strings" + "github.com/git-pkgs/integrity" "github.com/git-pkgs/proxy/internal/handler" "github.com/go-chi/chi/v5" ) @@ -182,6 +181,10 @@ func (h *HostedAPIHandler) HandleDelete(w http.ResponseWriter, r *http.Request) if !ok { return } + if err := handler.ValidateHostedPath(path.ecosystem, path.name, path.version, path.filename); err != nil { + badRequest(w, err.Error()) + return + } var err error if path.filename != "" { err = h.proxy.DeleteLocalFile(path.ecosystem, path.name, path.version, path.filename) @@ -248,9 +251,6 @@ func hostedPath(w http.ResponseWriter, r *http.Request, requireFile bool) (hoste // validSHA256 reports whether s is a hex-encoded sha256 digest. func validSHA256(s string) bool { - if len(s) != sha256.Size*2 { - return false - } - _, err := hex.DecodeString(s) + _, err := integrity.ParseHex(integrity.SHA256, s) return err == nil } diff --git a/internal/server/hosted_api_test.go b/internal/server/hosted_api_test.go index eaa5a56..3a24aab 100644 --- a/internal/server/hosted_api_test.go +++ b/internal/server/hosted_api_test.go @@ -506,3 +506,18 @@ func TestHostedRepushIsRefusedBeforeTheBodyIsRead(t *testing.T) { t.Fatalf("second PUT status = %d, want 409, body %s", rec.Code, rec.Body) } } + +func TestHostedDeleteRejectsMalformedPaths(t *testing.T) { + s := newHostedTestServer(t) + for _, target := range []string{ + "/api/hosted/generic/tools/1.2.0/%2e%2e", + "/api/hosted/generic/tools/1.2.0/tool%0a", + "/api/hosted/npm/request/2.88.3/request-2.88.3.tgz/extra", + "/api/hosted/composer/monolog/3.5.0", + "/api/hosted/cargo/serde/1.0.200", + } { + if rec := s.do(t, http.MethodDelete, target, testHostedToken, ""); rec.Code != http.StatusBadRequest { + t.Errorf("DELETE %s status = %d, want 400, body %s", target, rec.Code, rec.Body) + } + } +} From 0832c6142be4ddbb3936298eab1366b3383c4b80 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Wed, 21 Oct 2026 12:33:43 +0200 Subject: [PATCH 339/633] Refuse pushes of a version that holds a pushed file under another name Composer names are case-insensitive, so acme/private and acme/Private share the version record and its integrity but get different filenames. A second push under the other case was accepted and broke the integrity check of the first archive. For ecosystems with one file per version, any pushed file of the version now refuses the push, both before the upload and atomically when recording it. --- docs/configuration.md | 2 +- internal/database/local_artifacts.go | 23 ++++++++-- internal/database/local_artifacts_test.go | 51 +++++++++++++++++++++++ internal/handler/hosted.go | 26 ++++++++---- internal/server/hosted_api_test.go | 16 +++++++ 5 files changed, 106 insertions(+), 12 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index f805387..57cc423 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1123,7 +1123,7 @@ hosted: 4. Check what is pushed: `proxy hosted list npm request` or `GET /api/hosted/npm/request`. 5. Remove it again: `proxy hosted delete npm request 2.88.3` or `DELETE /api/hosted/npm/request/2.88.3`. If the upstream has the version, clients get the upstream archive again. -A pushed file is never replaced: pushing the same version again answers 409, before the body is uploaded when the client sends `Expect: 100-continue` (as `proxy hosted push` does). Clients that pinned the file by its hash would otherwise get different bytes under the same version. To push it again, delete it first. +A pushed file is never replaced: pushing the same version again answers 409, also under a differently cased Composer name, before the body is uploaded when the client sends `Expect: 100-continue` (as `proxy hosted push` does). Clients that pinned the file by its hash would otherwise get different bytes under the same version. To push it again, delete it first. ### Supported ecosystems diff --git a/internal/database/local_artifacts.go b/internal/database/local_artifacts.go index 26aea31..162b471 100644 --- a/internal/database/local_artifacts.go +++ b/internal/database/local_artifacts.go @@ -81,9 +81,13 @@ func (db *DB) DeleteLocalArtifacts(versionPURL, filename string) (int, error) { // is refused with ErrLocalArtifactExists, so of two concurrent pushes of the // same file exactly one wins. // -// When setIntegrity is false, an existing version record is left as it is. -// Otherwise its integrity is set to v.Integrity. -func (db *DB) RecordLocalArtifact(pkg *Package, v *Version, setIntegrity bool, a *Artifact) error { +// When oneFilePerVersion is false, an existing version record is left as it +// is. Otherwise its integrity is set to v.Integrity, and the push is refused +// if the version already holds a pushed artifact under any filename, since +// aliases of one name (Composer's are case-insensitive) share the version +// record and its integrity. The version upsert locks that record first, so +// concurrent pushes of aliases see each other. +func (db *DB) RecordLocalArtifact(pkg *Package, v *Version, oneFilePerVersion bool, a *Artifact) error { local := *a local.Source = SourceLocal return db.recordArtifact(&local, func(tx execer) error { @@ -96,7 +100,7 @@ func (db *DB) RecordLocalArtifact(pkg *Package, v *Version, setIntegrity bool, a return fmt.Errorf("inserting package: %w", err) } versionConflict := `DO NOTHING` - if setIntegrity { + if oneFilePerVersion { versionConflict = `DO UPDATE SET integrity = excluded.integrity, updated_at = excluded.updated_at` } if _, err := tx.Exec(db.Rebind(` @@ -106,6 +110,17 @@ func (db *DB) RecordLocalArtifact(pkg *Package, v *Version, setIntegrity bool, a ), v.PURL, v.PackagePURL, v.Integrity, now, now); err != nil { return fmt.Errorf("recording version: %w", err) } + if oneFilePerVersion { + var pushed int + if err := tx.Get(&pushed, db.Rebind(` + SELECT COUNT(*) FROM artifacts WHERE version_purl = ? AND source = ? + `), v.PURL, SourceLocal); err != nil { + return fmt.Errorf("checking pushed artifacts: %w", err) + } + if pushed > 0 { + return ErrLocalArtifactExists + } + } return nil }, ErrLocalArtifactExists) } diff --git a/internal/database/local_artifacts_test.go b/internal/database/local_artifacts_test.go index b476d8d..a6d0ed5 100644 --- a/internal/database/local_artifacts_test.go +++ b/internal/database/local_artifacts_test.go @@ -420,6 +420,57 @@ func TestRecordLocalArtifactConcurrentPushesHaveOneWinner(t *testing.T) { }) } +// Aliases of one name, such as Composer names differing in case, share the +// version record and its integrity, so only one of them may be pushed. +func TestRecordLocalArtifactOneFilePerVersionRefusesAliases(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + pkg := &Package{PURL: "pkg:composer/acme/private", Ecosystem: "composer", Name: "acme/private"} + const pushes = 8 + start := make(chan struct{}) + errs := make([]error, pushes) + var wg sync.WaitGroup + for i := range pushes { + wg.Go(func() { + <-start + v := &Version{PURL: "pkg:composer/acme/private@1.0.0", PackagePURL: pkg.PURL, + Integrity: sql.NullString{String: fmt.Sprintf("sha512-push-%d", i), Valid: true}} + a := localPush("pkg:composer/acme/private@1.0.0", fmt.Sprintf("alias-%d-1.0.0.zip", i), + fmt.Sprintf("composer/acme/private/push-%d", i)) + errs[i] = db.RecordLocalArtifact(pkg, v, true, a) + }) + } + close(start) + wg.Wait() + + winner := -1 + for i, err := range errs { + switch { + case err == nil: + if winner >= 0 { + t.Errorf("pushes %d and %d both succeeded, want exactly 1", winner, i) + } + winner = i + case !errors.Is(err, ErrLocalArtifactExists): + t.Errorf("push %d error = %v, want ErrLocalArtifactExists", i, err) + } + } + if winner < 0 { + t.Fatal("no push succeeded") + } + artifacts, err := db.GetArtifactsByVersionPURL("pkg:composer/acme/private@1.0.0") + if err != nil { + t.Fatal(err) + } + if len(artifacts) != 1 || artifacts[0].Filename != fmt.Sprintf("alias-%d-1.0.0.zip", winner) { + t.Errorf("artifacts = %+v, want only push %d", artifacts, winner) + } + v, err := db.GetVersionByPURL("pkg:composer/acme/private@1.0.0") + if err != nil || v == nil || v.Integrity.String != fmt.Sprintf("sha512-push-%d", winner) { + t.Errorf("version = %+v, %v, want the integrity of push %d", v, err, winner) + } + }) +} + func TestRecordCachedArtifactNeverReplacesAPush(t *testing.T) { runWithBothDatabases(t, func(t *testing.T, db *DB) { pkg := &Package{PURL: "pkg:npm/request", Ecosystem: "npm", Name: "request"} diff --git a/internal/handler/hosted.go b/internal/handler/hosted.go index 6342a54..5ff43ac 100644 --- a/internal/handler/hosted.go +++ b/internal/handler/hosted.go @@ -253,8 +253,10 @@ func (p *Proxy) PushLocalArtifact(ctx context.Context, ecosystem string, target // CheckPush refuses a push of a denied version or of a file that was already // pushed, before any bytes are read, so a client can be told without -// uploading the file. A cached upstream copy under the same name is fine to -// replace. Recording the push checks again atomically, for concurrent pushes. +// uploading the file. For ecosystems with one file per version, any pushed +// file of the version counts, so a differently cased alias of the name is +// refused too. A cached upstream copy is fine to replace. Recording the push +// checks again atomically, for concurrent pushes. func (p *Proxy) CheckPush(ecosystem string, target HostedTarget) error { _, versionPURL, err := packagePURLStrings(ecosystem, target.Name, target.Version) if err != nil { @@ -263,12 +265,22 @@ func (p *Proxy) CheckPush(ecosystem string, target HostedTarget) error { if p.versionDenied(ecosystem, target.Name, target.Version) { return fmt.Errorf("%w: %s", ErrVersionDenied, versionPURL) } - record, err := p.DB.GetArtifact(versionPURL, target.Filename) - if err != nil { - return fmt.Errorf("checking existing artifact: %w", err) + var existing []database.Artifact + if hostedEcosystems[ecosystem].multiFile() { + record, err := p.DB.GetArtifact(versionPURL, target.Filename) + if err != nil { + return fmt.Errorf("checking existing artifact: %w", err) + } + if record != nil { + existing = append(existing, *record) + } + } else if existing, err = p.DB.GetArtifactsByVersionPURL(versionPURL); err != nil { + return fmt.Errorf("checking existing artifacts: %w", err) } - if record != nil && record.Source == database.SourceLocal { - return ErrHostedExists + for _, a := range existing { + if a.Source == database.SourceLocal { + return ErrHostedExists + } } return nil } diff --git a/internal/server/hosted_api_test.go b/internal/server/hosted_api_test.go index 3a24aab..38b34b3 100644 --- a/internal/server/hosted_api_test.go +++ b/internal/server/hosted_api_test.go @@ -386,6 +386,22 @@ func TestHostedRepushIsRefused(t *testing.T) { } } +func TestHostedComposerRepushUnderAnotherCaseIsRefused(t *testing.T) { + s := newHostedTestServer(t) + original := composerZip(`{"name":"acme/private","description":"first"}`) + if rec := s.do(t, http.MethodPut, "/api/hosted/composer/acme/private/1.0.0", testHostedToken, original); rec.Code != http.StatusCreated { + t.Fatalf("first PUT status = %d, body %s", rec.Code, rec.Body) + } + if rec := s.do(t, http.MethodPut, "/api/hosted/composer/acme/Private/1.0.0", testHostedToken, + composerZip(`{"name":"acme/private","description":"second"}`)); rec.Code != http.StatusConflict { + t.Fatalf("PUT of acme/Private status = %d, want 409, body %s", rec.Code, rec.Body) + } + rec := s.do(t, http.MethodGet, "/composer/files/acme/private/1.0.0/private-1.0.0.zip", "", "") + if rec.Code != http.StatusOK || rec.Body.String() != original { + t.Fatalf("download status = %d, want 200 and the first archive", rec.Code) + } +} + func TestHostedPushWithMatchingSHA256(t *testing.T) { s := newHostedTestServer(t) digest := sha256.Sum256([]byte(requestTarball)) From 626a8e80ec73ad49e322c7d161930ad2e15b6913 Mon Sep 17 00:00:00 2001 From: montehurd Date: Wed, 21 Oct 2026 03:47:16 -0700 Subject: [PATCH 340/633] Serve Git repositories from a local cache (#492) * Serve Git repositories from a local cache Named upstream.git hosts are served read-only over Git's smart HTTP protocol at /git/{name}/{repo}, so git clone and git fetch work against the proxy through an insteadOf rewrite. Each repository is a bare clone at git.cache_dir/{name}-{hash of the upstream URL}/{repo}.git, served by git upload-pack. Every request for refs checks the upstream's branches, tags and HEAD with a protocol v2 ls-refs limited to those prefixes, and fetches only when they changed, atomically and with --prune. Concurrent requests share one refresh. Only branches and tags are cached, and HEAD follows the upstream's default branch. A commit a client asks for by ID that the cache lacks, such as a pull request head, is fetched and kept under a hidden ref until a branch or tag includes it or for 7 days. One the upstream cannot provide is not asked for again for a minute, or an hour when it is larger than git.max_repo_size leaves. This is off for upstreams upstream.auth has credentials for, since GitHub serves commits from any repository in a fork network. git never reaches an upstream itself. Its fetches go through a loopback relay that forwards with the proxy's upstream client, so address checks, upstream.auth and the access log apply, and git never sees a credential. Redirects are not followed, and git runs with an environment of its own. The cached copy is served while the upstream fails or is slower than git.refresh_wait, until git.stale_max_age has passed since the last successful check. Refusals and missing repositories are never answered from the cache. git.max_repo_size cuts a transfer off as it streams, and every fetched pack is kept as a pack, so a cut-off leaves nothing behind. A repository that can no longer be refreshed within the limit is cloned again with only its branches and tags. git.max_size evicts the least recently used repositories, never one in use. At startup only the directory of a repointed upstream's old URL is removed, and Close waits for refreshes in progress. Hosts with upstream.auth credentials need a repos allowlist, and a repository outside it answers 404. Refs #403 * Document setting the Git rewrite through environment variables The Git section of the configuration docs now explains what insteadOf does, with the URL a clone fetches from, and shows the same rewrites set through GIT_CONFIG_COUNT, GIT_CONFIG_KEY_n and GIT_CONFIG_VALUE_n for CI. Package managers that run git for Git dependencies inherit those variables, so their fetches are cached too; npm's GitHub tarball downloads do not go through git and are not. Refs #403 * Note that upstream.auth credentials reach every route to a host Credentials are matched by URL, so a token configured for a Git upstream is also sent by a generic upstream at the same URL, which forwards any path and checks no repos allowlist. With a token for a private GitHub org, /generic/github/acme/private.git/info/refs would list that repository's refs to anyone who can reach the proxy. * Cap concurrent upload-pack processes and request body size Every POST git-upload-pack started a pack-objects with git's defaults, a delta-search thread per CPU and unbounded window memory, and nothing limited how many ran at once, so a burst of CI clones of one large repository could run the proxy out of memory. At most two upload-pack processes per CPU (GOMAXPROCS, so a container's limit) now build packs at once, and further requests wait for a slot while the client is connected. ls-refs builds no pack and does not wait. Serving runs with pack.threads=1 and pack.windowMemory=64m, and every git process with core.bigFileThreshold=8m, so index-pack streams large files instead of holding them in memory. A request body was streamed to upload-pack unbounded, so a small gzip body could feed it an arbitrarily long negotiation for as long as the request may run. Bodies are now limited to 100MB, both as received and decoded, and one past the limit fails with an error instead of being cut short quietly. * Apply refresh_wait to the ref check only refresh_wait cut off the whole refresh, fetch included, so after a push with a large pack a client got the old refs once it passed, though the upstream was fine and the docs say a clone right after a push sees the new commit. It now bounds only the check of the upstream's branches and tags. Once the check finds they changed, requests wait for the fetch; one that fails, or stalls and is abandoned by http.lowSpeedTime, still falls back to the cached copy within stale_max_age. * Lock cache_dir against a second process The cache clears its temporary directory at startup, so a second process given the same cache_dir, such as a second pod on a shared volume, would delete the first one's clones in progress, and each process's eviction knows only its own readers. New now locks {cache_dir}/.gitcache-lock before touching anything and holds it until Close: flock on Unix, a handle opened without sharing on Windows. The OS releases it when the process exits, so a crashed process can restart. A second process fails to start with an error naming the directory. * Evict again once repositories in use are released Eviction skips repositories a request or refresh is using, and only the release of one too large to refresh ran it again, so a cache filled past max_size while its repositories were in use stayed over it once they went idle, until the next fetch. Eviction now records when it is left over max_size, and the next release runs it again. * Cap git.max_repo_size at git.max_size A repository larger than max_size could still be cloned, up to max_repo_size, which defaults to 5GB whatever max_size is. Once released it was evicted to bring the cache back within max_size, and the next request cloned it again. max_size now caps max_repo_size, so such a repository is refused for an hour like any other too large one, with an error that names both limits. * Count fetches in progress toward git.max_size Fetches enforced only max_repo_size, and eviction ran after each one, so up to 16 concurrent fetches could take the cache past max_size by as much as each may download. Each fetch now reserves space within max_size as the relay receives its bytes, a chunk at a time, evicting least recently used idle repositories to make room. When the fetch ends, the repository's measured size replaces the reservation under the cache lock, so its bytes are neither counted twice nor missed. When evicting every idle repository would not make room, none is evicted for the fetch, and it is cut off with ErrCacheFull rather than ErrTooLarge, so the repository is not refused for an hour: a clone answers 507, and a refresh serves the cached copy as when the upstream fails. A fetch that runs out of room while another has just been cut off waits, up to a minute, for that one's space, so two fetches that do not fit together do not both fail. A fetch that was cut off never waits, so no two wait on each other. --------- Co-authored-by: Andrew Nesbitt --- Dockerfile | 2 +- README.md | 23 + config.example.yaml | 17 + deploy/charts/proxy/values.yaml | 1 + docs/architecture.md | 15 + docs/configuration.md | 144 ++++++ internal/config/config.go | 20 + internal/config/git.go | 224 +++++++++ internal/config/git_test.go | 172 +++++++ internal/gitcache/budget.go | 126 +++++ internal/gitcache/budget_test.go | 218 ++++++++ internal/gitcache/cache.go | 499 +++++++++++++++++++ internal/gitcache/cache_test.go | 765 +++++++++++++++++++++++++++++ internal/gitcache/git.go | 104 ++++ internal/gitcache/helpers_test.go | 354 +++++++++++++ internal/gitcache/lock_unix.go | 24 + internal/gitcache/lock_windows.go | 27 + internal/gitcache/pktline.go | 65 +++ internal/gitcache/refresh.go | 376 ++++++++++++++ internal/gitcache/relay.go | 189 +++++++ internal/gitcache/relay_test.go | 96 ++++ internal/gitcache/serve.go | 227 +++++++++ internal/gitcache/serve_test.go | 327 ++++++++++++ internal/gitcache/upstream.go | 214 ++++++++ internal/gitcache/wants.go | 265 ++++++++++ internal/gitcache/wants_test.go | 319 ++++++++++++ internal/handler/git.go | 215 ++++++++ internal/handler/git_test.go | 247 ++++++++++ internal/server/git_test.go | 160 ++++++ internal/server/middleware.go | 2 +- internal/server/middleware_test.go | 2 +- internal/server/server.go | 33 ++ 32 files changed, 5469 insertions(+), 3 deletions(-) create mode 100644 internal/config/git.go create mode 100644 internal/config/git_test.go create mode 100644 internal/gitcache/budget.go create mode 100644 internal/gitcache/budget_test.go create mode 100644 internal/gitcache/cache.go create mode 100644 internal/gitcache/cache_test.go create mode 100644 internal/gitcache/git.go create mode 100644 internal/gitcache/helpers_test.go create mode 100644 internal/gitcache/lock_unix.go create mode 100644 internal/gitcache/lock_windows.go create mode 100644 internal/gitcache/pktline.go create mode 100644 internal/gitcache/refresh.go create mode 100644 internal/gitcache/relay.go create mode 100644 internal/gitcache/relay_test.go create mode 100644 internal/gitcache/serve.go create mode 100644 internal/gitcache/serve_test.go create mode 100644 internal/gitcache/upstream.go create mode 100644 internal/gitcache/wants.go create mode 100644 internal/gitcache/wants_test.go create mode 100644 internal/handler/git.go create mode 100644 internal/handler/git_test.go create mode 100644 internal/server/git_test.go diff --git a/Dockerfile b/Dockerfile index de784e8..4113999 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,7 +18,7 @@ RUN CGO_ENABLED=0 GOOS=linux GOARCH=${TARGETARCH} go build -ldflags="-s -w" -o / FROM alpine:3.24.2 -RUN apk add --no-cache ca-certificates +RUN apk add --no-cache ca-certificates git COPY --from=builder /proxy /usr/local/bin/proxy diff --git a/README.md b/README.md index d4881cc..2c538ad 100644 --- a/README.md +++ b/README.md @@ -68,6 +68,7 @@ The proxy never uploads artifact bytes to a scanner. Each scanner is notified wi | Arch | Arch Linux | | ✓ | | Chef Supermarket | Chef | | ✓ | | Generic | Any | | ✓ | +| Git | Any | | ✓ | | Helm | Kubernetes | Yes | ✓ | | Vagrant | Vagrant | | ✗ | @@ -826,6 +827,27 @@ go.download_mirror = "http://localhost:8080/generic/golang" Other download hosts need their version-pinned paths declared in `upstream.generic_artifacts`. +### Git repositories + +Configure named Git servers: + +```yaml +upstream: + git: + github: "https://github.com" +``` + +Then point clones at the proxy: + +```bash +git config --global url."http://localhost:8080/git/github/".insteadOf https://github.com/ +``` + +Each repository is cached as a bare clone and refreshed whenever its branches +or tags change upstream. See [configuration](docs/configuration.md#git-repositories) +for setting the rewrite through environment variables, size limits, private +repositories and outage behavior. + ## Configuration The proxy can be configured via: @@ -1150,6 +1172,7 @@ Recently cached: | `GET /apk/{repository}/*` | Alpine APK repository protocol | | `GET /arch/{repository}/*` | Arch Linux pacman repository protocol | | `GET /generic/{name}/*` | Generic HTTP download proxy (GitHub release assets, mise/aqua) | +| `GET /git/{name}/{repo}/info/refs`, `POST /git/{name}/{repo}/git-upload-pack` | Git smart HTTP protocol, read-only | | `GET /debian/*` | Debian/APT repository protocol (main archive) | | `GET /debian/{repository}/*` | Debian/APT repository protocol (named archive, e.g. security) | | `GET /rpm/*` | RPM/Yum repository protocol | diff --git a/config.example.yaml b/config.example.yaml index f8bebe7..7534858 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -332,6 +332,15 @@ upstream: # apache: # - 'dist/(?Pmaven)/maven-3/(?P[^/]+)/binaries/(?P[^/]+)' + # Named Git servers (used by /git/{name}/{repo}), served read-only from + # cached bare repositories. repos limits which repositories may be served + # and is required when upstream.auth has credentials for the server. + # git: + # github: "https://github.com" + # wikimedia: + # url: "https://gerrit.wikimedia.org/r" + # repos: ["mediawiki/**"] + # Authentication for upstream registries # Keys are absolute URL scopes. Scheme, host, effective port, and path # segment boundaries must match; the longest matching scope wins. @@ -372,6 +381,14 @@ upstream: # "https://123456789012.dkr.ecr.eu-west-1.amazonaws.com": # type: ecr +# Git repository cache, used when upstream.git is set +# git: +# cache_dir: ./cache/git # local disk, used by this process only +# max_size: "" # evict least recently used repositories past this +# max_repo_size: 5GB # refuse a repository that grows past this +# refresh_wait: 5s # then serve the cached copy if the upstream check is slow +# stale_max_age: 24h # never serve a copy not checked for this long + # Gradle HttpBuildCache configuration gradle: build_cache: diff --git a/deploy/charts/proxy/values.yaml b/deploy/charts/proxy/values.yaml index 3861784..9a74e47 100644 --- a/deploy/charts/proxy/values.yaml +++ b/deploy/charts/proxy/values.yaml @@ -1,6 +1,7 @@ # More than one replica requires config.data.database on Postgres, # config.data.storage on object storage, deploymentStrategy: RollingUpdate, # and a PVC access mode other than ReadWriteOnce (or persistence disabled). +# With upstream.git, each replica also needs its own git.cache_dir. replicaCount: 1 nameOverride: "" diff --git a/docs/architecture.md b/docs/architecture.md index a259e91..f9325b2 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -301,6 +301,21 @@ HTTP protocol handlers for each registry type. - Proxies the Swift Package Registry v1 read endpoints - Rewrites release URLs and caches source archives +**GitHandler:** +- Serves `/git/{name}/{repo}` from `internal/gitcache`, read-only over smart HTTP +- Validates repository paths and the `repos` allowlist, and maps cache errors to statuses + +### `internal/gitcache` + +Local bare clones of upstream Git repositories, outside the artifact cache and the database, at `{git.cache_dir}/{name}-{URL hash}/{repo}.git`. The hash of the upstream's URL means repointing an upstream starts a fresh cache; at startup only the configured upstreams' directories are loaded, and those of their old URLs removed. + +- Each request for refs checks the upstream's branches and tags with protocol v2 `ls-refs` and fetches only when they changed; concurrent requests share one refresh +- Clients are served by `git upload-pack --stateless-rpc` +- Spawned `git` reaches the upstream only through a loopback relay that forwards with the proxy's upstream client, so address checks, `upstream.auth` and the access log apply +- Commits a client asks for by ID that are on no cached branch or tag are fetched and kept under hidden `refs/proxy/wanted/{time}/{id}` refs until a branch or tag includes them or for 7 days; off for upstreams with credentials +- Evicts least recently used repositories after each fetch once `git.max_size` is passed, and drops a repository that can no longer be refreshed within `git.max_repo_size` so it is cloned again; never one in use. A commit fetched by ID that does not fit is refused instead +- Every fetched pack is kept as a pack (`transfer.unpackLimit=1`), so a transfer cut off at the size limit leaves nothing behind + ### `internal/server` HTTP server setup, web UI, and API handlers. diff --git a/docs/configuration.md b/docs/configuration.md index 507cad5..19ad938 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -623,6 +623,150 @@ those names unless the shadowing is intended. This field has no environment variable, as with the other named upstream maps. `PROXY_UPSTREAM_DEBIAN` still sets `upstream.debian`. +### Git repositories + +Git servers configured under `upstream.git` are served read-only at +`/git/{name}/{repo}` over Git's smart HTTP protocol, from bare repositories +cached on local disk: + +```yaml +upstream: + git: + github: "https://github.com" + wikimedia: + url: "https://gerrit.wikimedia.org/r" + repos: ["mediawiki/core", "mediawiki/extensions/**"] # optional allowlist + +git: + cache_dir: /data/git # local disk, used by this process only + max_size: 50GB # evict least recently used repositories past this + max_repo_size: 5GB # refuse a repository that grows past this + refresh_wait: 5s # then serve the cached copy if the upstream check is slow + stale_max_age: 24h # never serve a copy not checked for this long +``` + +Clients reach the proxy through Git's `insteadOf` setting, which rewrites the +start of a URL before Git connects. After + +```bash +git config --global url."http://localhost:8080/git/github/".insteadOf https://github.com/ +``` + +`git clone https://github.com/git-pkgs/purl` fetches from +`http://localhost:8080/git/github/git-pkgs/purl`, while the clone's remote +stays `https://github.com/git-pkgs/purl`. + +In CI, the same setting can come from environment variables instead of a +config file (Git 2.31 or newer), next to the variables that point package +managers at the proxy. Each `GIT_CONFIG_KEY_n` and `GIT_CONFIG_VALUE_n` pair is +one `git config` setting, and `GIT_CONFIG_COUNT` says how many pairs there are. +This sends both upstreams from the example above through the proxy: + +```bash +export GIT_CONFIG_COUNT=2 +export GIT_CONFIG_KEY_0=url.http://localhost:8080/git/github/.insteadOf +export GIT_CONFIG_VALUE_0=https://github.com/ +export GIT_CONFIG_KEY_1=url.http://localhost:8080/git/wikimedia/.insteadOf +export GIT_CONFIG_VALUE_1=https://gerrit.wikimedia.org/r/ +``` + +Every process the job starts inherits these variables, so the rewrite also +applies when a package manager runs `git` for a Git dependency, as pip, +Bundler, Composer, SwiftPM, and Go for modules it fetches directly do. Those +fetches are cached too. npm downloads GitHub dependencies as tarballs over +HTTPS where it can, without `git`, so the rewrite does not reach them. + +`repos` patterns use `path.Match` syntax, where `*` does not cross a `/`; a +pattern ending in `/**` also matches every repository nested below its +prefix, so `mediawiki/**` matches `mediawiki/skins/Vector`. An upstream +without `repos` serves every repository on it, and a repository outside +`repos` answers 404. When `upstream.auth` has credentials for an upstream, +`repos` is required: everyone who can reach the proxy can read every +repository it serves, so list only those they may read. Per-client +permissions are not supported. `upstream.auth` matches credentials by URL, so +they are sent by every route to that host, not only `/git`. With a token for +`https://github.com`, a `generic` upstream at the same URL, such as the one +the mise setup in the README uses, sends it for any path and checks no +`repos`, so `/generic/github/acme/private.git/info/refs?service=git-upload-pack` +would list a private repository's refs to anyone. + +Every request for a repository's refs checks the upstream's branches and tags, +and fetches only when they changed, so a clone right after a push sees the new +commit. Requests that arrive while a check or fetch is running share it. Only +branches and tags are cached and served; other refs, such as Gerrit's +`refs/changes/*`, are left out, and protocol v2 lets the server filter them +before sending. + +A commit a client asks for by object ID that is on no cached branch or tag, +such as a pull request head fetched by `actions/checkout`, is fetched from the +upstream when the client asks for it, and kept until a cached branch or tag +includes it or for 7 days. A commit the upstream could not provide, or could +not be reached for, is not asked for again for a minute, or for an hour when +it is larger than what `max_repo_size` leaves. This is off for upstreams that `upstream.auth` has +credentials for: GitHub serves a commit from any repository in a fork network +through any other, so a fetch by ID could reach repositories outside `repos`. +Any commit the cache holds can be fetched by ID, including one a force-push +left behind, as from the upstream. + +If the upstream fails, or its check of branches and tags takes longer than +`refresh_wait`, a repository checked successfully within `stale_max_age` is +served from the cache while the refresh continues. Once the check finds new +commits, the request waits for their fetch, so a clone right after a push gets +them; if the fetch fails or stalls, the cached copy is served. A +`stale_max_age` of `0` never serves a copy the upstream has not just +confirmed. A repository the upstream reports missing (404), refuses (401, 403) +or redirects is not served from the cache. Redirects are never followed, so +point the upstream at a repository's current location. + +The proxy runs `git` 2.31 or newer, which must be installed (the Docker image includes it). +The `git` processes never reach the network themselves: their fetches go +through a loopback endpoint in the proxy that forwards them with the same +upstream client as every other request, so the address checks +(`allow_loopback`, `allow_private_hosts`), `upstream.auth` and the access log +apply, and credentials are never passed to `git`. + +`max_repo_size` defaults to 5GB, and `max_size` to no limit. `max_size` also +caps `max_repo_size`, since a repository larger than the whole cache would be +evicted as soon as no request was reading it. A clone or fetch that would +pass `max_repo_size` is stopped, and nothing of it is kept. A new repository +is then refused for an hour. When a refresh of one already cached does not +fit, it is served as it is and dropped once no request is reading it, so the +next request clones it again with only its branches and tags; if that is +still too large, it is refused for an hour. A commit fetched by ID that does +not fit is refused, and the repository kept. + +`max_size` counts fetches in progress as well as cached repositories, so the +cache stays within it however many fetches run at once. As a fetch receives +data, the least recently used repositories are evicted to make room, never +one a request is reading; those are evicted once released if the cache is +still over. When evicting every idle repository would not make room, none is +evicted for the fetch, and it is stopped and nothing of it is kept, but the +repository is not refused: a clone answers 507, and a refresh serves the +cached copy, as when the upstream fails. The count is of bytes received, so +index files and garbage collection can take the cache a little past +`max_size` until the fetch ends and the repository is measured. +A transfer that receives nothing for 10 minutes is abandoned. Git's automatic +garbage collection runs as part of fetches, and files over 8MB are kept +without delta compression, so one large file does not take a fetch's memory. + +Packs for clients are built by at most two `upload-pack` processes per CPU +the proxy may use, each with one delta-search thread and 64MB of delta +window, and further clones wait for one to finish; listing refs does not +wait. An `upload-pack` request is limited to 100MB, before and after gzip. + +Repositories are kept under `{cache_dir}/{name}-{hash of the URL}/`, so +repointing an upstream starts a fresh cache, and the old URL's directory is +removed at startup. Nothing else in `cache_dir` is touched, and the +directories of upstreams no longer configured are only logged, so remove +those yourself. The proxy locks `cache_dir` while it runs, through a +`.gitcache-lock` file in it, so a second process given the same directory, +such as another replica on a shared volume, fails to start; give each its own. + +Pushing, the dumb HTTP protocol and Git LFS are not supported. The settings +under `git` can also be set with `PROXY_GIT_CACHE_DIR`, `PROXY_GIT_MAX_SIZE`, +`PROXY_GIT_MAX_REPO_SIZE`, `PROXY_GIT_REFRESH_WAIT` and +`PROXY_GIT_STALE_MAX_AGE`. + ### Maven repositories ```yaml diff --git a/internal/config/config.go b/internal/config/config.go index e7851c9..11af301 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -217,6 +217,9 @@ type Config struct { // Health configures the /health endpoint behavior. Health HealthConfig `json:"health" yaml:"health"` + // Git configures the Git repository cache served at /git/{name}/. + Git GitConfig `json:"git" yaml:"git"` + // envWarnings collects environment values LoadFromEnv ignored. envWarnings []string } @@ -776,6 +779,11 @@ type UpstreamConfig struct { // Example: {"apache": ["dist/(?Pmaven)/maven-3/(?P[^/]+)/binaries/(?P[^/]+)"]}. GenericArtifacts map[string][]string `json:"generic_artifacts" yaml:"generic_artifacts"` + // Git maps names to Git servers, served read-only at /git/{name}/{repo} + // from a local cache of bare repositories. See GitUpstream. + // Example: {"github": {"url": "https://github.com"}}. + Git map[string]GitUpstream `json:"git" yaml:"git"` + // Auth configures authentication for upstream registries. // Keys are absolute URL scopes matched by scheme, host, effective port, // and path-segment prefix. @@ -846,6 +854,9 @@ func (u *UpstreamConfig) Validate() error { if err := u.validateGenericArtifacts(); err != nil { return err } + if err := u.validateGitUpstreams(); err != nil { + return err + } if err := validateNamedUpstreams("upstream.maven_repositories", u.MavenRepositories); err != nil { return err } @@ -1353,6 +1364,11 @@ func (c *Config) LoadFromEnv() { setEnvString(&c.Gradle.BuildCache.MaxSize, "PROXY_GRADLE_BUILD_CACHE_MAX_SIZE") setEnvString(&c.Gradle.BuildCache.SweepInterval, "PROXY_GRADLE_BUILD_CACHE_SWEEP_INTERVAL") setEnvString(&c.Health.StorageProbeInterval, "PROXY_HEALTH_STORAGE_PROBE_INTERVAL") + setEnvString(&c.Git.CacheDir, "PROXY_GIT_CACHE_DIR") + setEnvString(&c.Git.MaxSize, "PROXY_GIT_MAX_SIZE") + setEnvString(&c.Git.MaxRepoSize, "PROXY_GIT_MAX_REPO_SIZE") + setEnvString(&c.Git.RefreshWait, "PROXY_GIT_REFRESH_WAIT") + setEnvString(&c.Git.StaleMaxAge, "PROXY_GIT_STALE_MAX_AGE") } // validateAbsoluteURL returns an error if value is not a parseable URL with @@ -1509,6 +1525,10 @@ func (c *Config) validateComponents() error { return err } + if err := c.Git.Validate(); err != nil { + return err + } + return c.Gradle.BuildCache.Validate() } diff --git a/internal/config/git.go b/internal/config/git.go new file mode 100644 index 0000000..860adb7 --- /dev/null +++ b/internal/config/git.go @@ -0,0 +1,224 @@ +package config + +import ( + "encoding/json" + "fmt" + "path" + "strings" + "time" + + "gopkg.in/yaml.v3" +) + +const ( + defaultGitCacheDir = "./cache/git" + defaultGitMaxRepoSize = "5GB" + defaultGitRefreshWait = "5s" + defaultGitStaleMaxAge = "24h" +) + +// GitUpstream is an upstream.git entry: a Git server's base URL and, +// optionally, the repositories under it that may be served. In YAML or JSON +// it is either the URL alone or an object with url and repos. +type GitUpstream struct { + URL string `json:"url" yaml:"url"` + + // Repos lists path.Match patterns for repository paths under URL, such + // as "mediawiki/*". A pattern ending in "/**" also matches every + // repository nested below its prefix: "mediawiki/**" matches + // mediawiki/skins/Vector. Empty allows every repository. Required when + // upstream.auth has credentials for the URL, since everyone who can + // reach the proxy can read what it serves. + Repos []string `json:"repos" yaml:"repos"` +} + +// UnmarshalYAML accepts a bare URL as well as the object form. +func (g *GitUpstream) UnmarshalYAML(node *yaml.Node) error { + if node.Kind == yaml.ScalarNode { + return node.Decode(&g.URL) + } + type plain GitUpstream + return node.Decode((*plain)(g)) +} + +// UnmarshalJSON accepts a bare URL as well as the object form. +func (g *GitUpstream) UnmarshalJSON(data []byte) error { + if err := json.Unmarshal(data, &g.URL); err == nil { + return nil + } + type plain GitUpstream + return json.Unmarshal(data, (*plain)(g)) +} + +// AllowsRepo reports whether repo, a path under URL, may be served. +func (g GitUpstream) AllowsRepo(repo string) bool { + if len(g.Repos) == 0 { + return true + } + for _, pattern := range g.Repos { + if matchRepoPattern(pattern, repo) { + return true + } + } + return false +} + +func matchRepoPattern(pattern, repo string) bool { + prefix, nested := strings.CutSuffix(pattern, "/**") + if !nested { + ok, _ := path.Match(pattern, repo) + return ok + } + segments := strings.Count(prefix, "/") + 1 + parts := strings.SplitN(repo, "/", segments+1) + if len(parts) <= segments { + return false + } + ok, _ := path.Match(prefix, strings.Join(parts[:segments], "/")) + return ok +} + +// GitConfig configures the Git repository cache behind /git/{name}/. +type GitConfig struct { + // CacheDir holds the cached bare repositories. It must be local disk + // used by this process only, which locks it. Default: ./cache/git. + CacheDir string `json:"cache_dir" yaml:"cache_dir"` + + // MaxSize bounds the cache, counting fetches in progress; the least + // recently used repositories not in use are evicted to stay within it. + // Empty or "0" disables the limit. + MaxSize string `json:"max_size" yaml:"max_size"` + + // MaxRepoSize is the most one repository may take; a clone or fetch that + // would pass it fails. "0" disables the limit. Default: 5GB. MaxSize + // caps it when set. + MaxRepoSize string `json:"max_repo_size" yaml:"max_repo_size"` + + // RefreshWait is how long a request waits for the upstream's branches + // and tags to be checked before it is served the cached copy. A fetch of + // new commits the check finds is waited for in full. Default: 5s. + RefreshWait string `json:"refresh_wait" yaml:"refresh_wait"` + + // StaleMaxAge is how long after the last successful upstream check a + // cached copy may still be served when the upstream fails or is slow. + // Default: 24h. + StaleMaxAge string `json:"stale_max_age" yaml:"stale_max_age"` + + maxSize int64 + maxRepoSize int64 + refreshWait time.Duration + staleMaxAge time.Duration +} + +// MaxSizeBytes returns the parsed MaxSize, 0 for no limit. Validate must run first. +func (g *GitConfig) MaxSizeBytes() int64 { return g.maxSize } + +// MaxRepoSizeBytes returns the parsed MaxRepoSize, 0 for no limit. Validate must run first. +func (g *GitConfig) MaxRepoSizeBytes() int64 { return g.maxRepoSize } + +// RefreshWaitDuration returns the parsed RefreshWait. Validate must run first. +func (g *GitConfig) RefreshWaitDuration() time.Duration { return g.refreshWait } + +// StaleMaxAgeDuration returns the parsed StaleMaxAge. Validate must run first. +func (g *GitConfig) StaleMaxAgeDuration() time.Duration { return g.staleMaxAge } + +// Validate checks the Git cache settings, applying defaults where unset. +func (g *GitConfig) Validate() error { + if g.CacheDir == "" { + g.CacheDir = defaultGitCacheDir + } + if g.MaxRepoSize == "" { + g.MaxRepoSize = defaultGitMaxRepoSize + } + if g.RefreshWait == "" { + g.RefreshWait = defaultGitRefreshWait + } + if g.StaleMaxAge == "" { + g.StaleMaxAge = defaultGitStaleMaxAge + } + + var err error + if g.maxSize, err = parseNonNegativeSize("git.max_size", g.MaxSize); err != nil { + return err + } + if g.maxRepoSize, err = parseNonNegativeSize("git.max_repo_size", g.MaxRepoSize); err != nil { + return err + } + if g.refreshWait, err = parseNonNegativeDuration("git.refresh_wait", g.RefreshWait); err != nil { + return err + } + if g.staleMaxAge, err = parseNonNegativeDuration("git.stale_max_age", g.StaleMaxAge); err != nil { + return err + } + return nil +} + +func parseNonNegativeSize(field, value string) (int64, error) { + size, err := ParseSize(value) + if err != nil { + return 0, fmt.Errorf("invalid %s: %w", field, err) + } + if size < 0 { + return 0, fmt.Errorf("invalid %s %q: must not be negative", field, value) + } + return size, nil +} + +func parseNonNegativeDuration(field, value string) (time.Duration, error) { + d, err := time.ParseDuration(value) + if err != nil { + return 0, fmt.Errorf("invalid %s %q: %w", field, value, err) + } + if d < 0 { + return 0, fmt.Errorf("invalid %s %q: must not be negative", field, value) + } + return d, nil +} + +// validateGitUpstreams checks upstream.git names, URLs and repository +// patterns. An upstream that upstream.auth has credentials for must list its +// repositories, so configuring a token does not open every repository it can +// read to everyone who can reach the proxy. +func (u *UpstreamConfig) validateGitUpstreams() error { + for name, upstream := range u.Git { + field := "upstream.git." + name + if name == "" || name == "." || name == ".." || name[0] == '.' || strings.ContainsAny(name, `/\\`) { + return fmt.Errorf("invalid upstream.git name %q", name) + } + if err := validateAbsoluteURL(field, upstream.URL); err != nil { + return err + } + if scheme, _, _ := strings.Cut(upstream.URL, "://"); !strings.EqualFold(scheme, "https") && !strings.EqualFold(scheme, "http") { + return fmt.Errorf("invalid %s %q: only http and https are supported", field, upstream.URL) + } + for _, pattern := range upstream.Repos { + if _, err := path.Match(strings.TrimSuffix(pattern, "/**"), ""); pattern == "" || err != nil { + return fmt.Errorf("invalid %s.repos pattern %q", field, pattern) + } + } + if len(upstream.Repos) == 0 && u.HasAuthWithin(upstream.URL) { + return fmt.Errorf("%s needs repos: upstream.auth has credentials for %s, "+ + "and everyone who can reach the proxy could read every repository they allow", field, upstream.URL) + } + } + return nil +} + +// HasAuthWithin reports whether an upstream.auth scope covers baseURL or a +// path under it. +func (u *UpstreamConfig) HasAuthWithin(baseURL string) bool { + base, err := parseAuthURL(baseURL) + if err != nil { + return false + } + for scope := range u.Auth { + configured, err := parseAuthURL(scope) + if err != nil { + continue + } + if authURLMatches(configured, base) || authURLMatches(base, configured) { + return true + } + } + return false +} diff --git a/internal/config/git_test.go b/internal/config/git_test.go new file mode 100644 index 0000000..6c9b47b --- /dev/null +++ b/internal/config/git_test.go @@ -0,0 +1,172 @@ +package config + +import ( + "encoding/json" + "os" + "path/filepath" + "slices" + "testing" + "time" + + "gopkg.in/yaml.v3" +) + +func TestGitUpstreamAcceptsURLOrObject(t *testing.T) { + want := map[string]GitUpstream{ + "github": {URL: "https://github.com"}, + "wikimedia": {URL: "https://gitlab.wikimedia.org", Repos: []string{"mediawiki/*"}}, + } + + var fromYAML struct { + Git map[string]GitUpstream `yaml:"git"` + } + err := yaml.Unmarshal([]byte(` +git: + github: "https://github.com" + wikimedia: + url: "https://gitlab.wikimedia.org" + repos: ["mediawiki/*"] +`), &fromYAML) + if err != nil { + t.Fatalf("YAML: %v", err) + } + + var fromJSON struct { + Git map[string]GitUpstream `json:"git"` + } + err = json.Unmarshal([]byte(`{"git": { + "github": "https://github.com", + "wikimedia": {"url": "https://gitlab.wikimedia.org", "repos": ["mediawiki/*"]} + }}`), &fromJSON) + if err != nil { + t.Fatalf("JSON: %v", err) + } + + for format, got := range map[string]map[string]GitUpstream{"YAML": fromYAML.Git, "JSON": fromJSON.Git} { + for name, w := range want { + if g := got[name]; g.URL != w.URL || !slices.Equal(g.Repos, w.Repos) { + t.Errorf("%s %s = %+v, want %+v", format, name, g, w) + } + } + } +} + +func TestGitUpstreamAllowsRepo(t *testing.T) { + open := GitUpstream{URL: "https://github.com"} + if !open.AllowsRepo("any/repo") { + t.Error("an upstream without repos refused a repository") + } + listed := GitUpstream{URL: "https://gitlab.example", Repos: []string{"mediawiki/*", "tools/one", "group/**"}} + for repo, want := range map[string]bool{ + "mediawiki/core": true, + "tools/one": true, + "tools/two": false, + "mediawiki/extensions/Echo": false, + "group/sub/project": true, + "group/project": true, + "group": false, + "groups/project": false, + } { + if got := listed.AllowsRepo(repo); got != want { + t.Errorf("AllowsRepo(%q) = %v, want %v", repo, got, want) + } + } +} + +func TestValidateGitUpstreams(t *testing.T) { + tests := []struct { + name string + upstream UpstreamConfig + wantErr bool + }{ + {name: "public", upstream: UpstreamConfig{Git: map[string]GitUpstream{"github": {URL: "https://github.com"}}}}, + {name: "relative URL", upstream: UpstreamConfig{Git: map[string]GitUpstream{"github": {URL: "github.com"}}}, wantErr: true}, + {name: "name with slash", upstream: UpstreamConfig{Git: map[string]GitUpstream{"a/b": {URL: "https://github.com"}}}, wantErr: true}, + {name: "name with leading dot", upstream: UpstreamConfig{Git: map[string]GitUpstream{".tmp": {URL: "https://github.com"}}}, wantErr: true}, + {name: "bad pattern", upstream: UpstreamConfig{Git: map[string]GitUpstream{"github": {URL: "https://github.com", Repos: []string{"["}}}}, wantErr: true}, + {name: "nested pattern", upstream: UpstreamConfig{Git: map[string]GitUpstream{"gerrit": {URL: "https://gerrit.example/r", Repos: []string{"mediawiki/**"}}}}}, + {name: "ssh URL", upstream: UpstreamConfig{Git: map[string]GitUpstream{"github": {URL: "ssh://git@github.com"}}}, wantErr: true}, + { + name: "credentials without repos", + upstream: UpstreamConfig{ + Git: map[string]GitUpstream{"private": {URL: "https://git.example"}}, + Auth: map[string]AuthConfig{"https://git.example": {Type: "bearer", Token: "t"}}, + }, + wantErr: true, + }, + { + name: "credentials for a path under the upstream", + upstream: UpstreamConfig{ + Git: map[string]GitUpstream{"private": {URL: "https://git.example"}}, + Auth: map[string]AuthConfig{"https://git.example/team": {Type: "bearer", Token: "t"}}, + }, + wantErr: true, + }, + { + name: "credentials with repos", + upstream: UpstreamConfig{ + Git: map[string]GitUpstream{"private": {URL: "https://git.example", Repos: []string{"team/*"}}}, + Auth: map[string]AuthConfig{"https://git.example": {Type: "bearer", Token: "t"}}, + }, + }, + { + name: "credentials for another host", + upstream: UpstreamConfig{ + Git: map[string]GitUpstream{"github": {URL: "https://github.com"}}, + Auth: map[string]AuthConfig{"https://npm.example": {Type: "bearer", Token: "t"}}, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if err := tt.upstream.Validate(); (err != nil) != tt.wantErr { + t.Errorf("Validate() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +func TestGitConfigDefaults(t *testing.T) { + var g GitConfig + if err := g.Validate(); err != nil { + t.Fatalf("Validate: %v", err) + } + if g.CacheDir != defaultGitCacheDir || g.MaxSizeBytes() != 0 || g.MaxRepoSizeBytes() != 5<<30 || + g.RefreshWaitDuration() != 5*time.Second || g.StaleMaxAgeDuration() != 24*time.Hour { + t.Errorf("defaults = %+v", g) + } + + for _, bad := range []GitConfig{{MaxSize: "lots"}, {RefreshWait: "soon"}, {StaleMaxAge: "-1h"}} { + if err := bad.Validate(); err == nil { + t.Errorf("Validate(%+v) accepted it", bad) + } + } +} + +func TestLoadGitConfig(t *testing.T) { + path := filepath.Join(t.TempDir(), "config.yaml") + data := []byte(` +upstream: + git: + github: https://github.com +git: + cache_dir: /data/git + max_size: 50GB +`) + if err := os.WriteFile(path, data, 0o600); err != nil { + t.Fatal(err) + } + cfg, err := Load(path) + if err != nil { + t.Fatalf("Load: %v", err) + } + t.Setenv("PROXY_GIT_REFRESH_WAIT", "2s") + cfg.LoadFromEnv() + if err := cfg.Validate(); err != nil { + t.Fatalf("Validate: %v", err) + } + if cfg.Upstream.Git["github"].URL != "https://github.com" || cfg.Git.CacheDir != "/data/git" || + cfg.Git.MaxSizeBytes() != 50<<30 || cfg.Git.RefreshWaitDuration() != 2*time.Second { + t.Errorf("loaded %+v and %+v", cfg.Upstream.Git, cfg.Git) + } +} diff --git a/internal/gitcache/budget.go b/internal/gitcache/budget.go new file mode 100644 index 0000000..9d4135e --- /dev/null +++ b/internal/gitcache/budget.go @@ -0,0 +1,126 @@ +package gitcache + +import ( + "context" + "sync/atomic" + "time" +) + +const ( + // maxReserveChunk caps the space a fetch reserves ahead of what it has + // received, so the cache is not locked for every read. Each fetch takes + // at most a sixteenth of MaxSize ahead, so fetches share a small cache. + maxReserveChunk = 4 << 20 + reserveChunks = 16 + // cutWait bounds how long a fetch waits for one that was cut off to + // give back its space. + cutWait = time.Minute +) + +// reservation holds space within MaxSize for the bytes a fetch receives, +// until they are counted in the repository's size. Fetches still running +// therefore count toward MaxSize, so the cache stays within it however many +// run at once. +type reservation struct { + c *Cache + // reserved is written under c.mu; reads that find it large enough skip + // the lock. + reserved atomic.Int64 + // cut, guarded by c.mu, marks a fetch stopped because the cache could + // not make room for it. + cut bool +} + +func (c *Cache) newReservation() *reservation { + return &reservation{c: c} +} + +// cover reserves space for a fetch that has received relayed bytes, +// evicting idle repositories as needed. It returns ErrCacheFull when evicting +// every idle repository would not make room, unless another fetch has just +// been cut off, whose space it waits for: one cut off waits for nothing, so +// two fetches that do not fit together do not both fail. +func (r *reservation) cover(ctx context.Context, relayed int64) error { + c := r.c + if c.opts.MaxSize <= 0 || relayed <= r.reserved.Load() { + return nil + } + var timeout <-chan time.Time + for { + c.mu.Lock() + if r.cut { + c.mu.Unlock() + return ErrCacheFull + } + need := relayed - r.reserved.Load() + removed, total := c.evictLocked(need) + fits := total+need <= c.opts.MaxSize + var freed chan struct{} + switch { + case fits: + chunk := min(maxReserveChunk, c.opts.MaxSize/reserveChunks) + n := min(need+chunk, c.opts.MaxSize-total) + r.reserved.Add(n) + c.reserved += n + case c.cutPending > 0: + freed = c.freed + default: + r.cut = true + c.cutPending++ + } + c.mu.Unlock() + c.removeEvicted(removed) + if fits { + return nil + } + if freed == nil { + return ErrCacheFull + } + if timeout == nil { + timeout = time.After(cutWait) + } + select { + case <-freed: + case <-timeout: + c.mu.Lock() + r.cut = true + c.cutPending++ + c.mu.Unlock() + return ErrCacheFull + case <-ctx.Done(): + return ctx.Err() + } + } +} + +// settle counts what the fetch left in the repository's size and gives back +// its reservation together, so the bytes are never counted twice or not at +// all. +func (r *reservation) settle(rp *repo, size int64) { + r.c.mu.Lock() + rp.size.Store(size) + r.releaseLocked() + r.c.mu.Unlock() +} + +// release gives back the reservation of a fetch that left nothing behind. +// It may follow settle. +func (r *reservation) release() { + r.c.mu.Lock() + r.releaseLocked() + r.c.mu.Unlock() +} + +func (r *reservation) releaseLocked() { + c := r.c + n := r.reserved.Swap(0) + if r.cut { + r.cut = false + c.cutPending-- + } else if n == 0 { + return + } + c.reserved -= n + close(c.freed) + c.freed = make(chan struct{}) +} diff --git a/internal/gitcache/budget_test.go b/internal/gitcache/budget_test.go new file mode 100644 index 0000000..506ef2f --- /dev/null +++ b/internal/gitcache/budget_test.go @@ -0,0 +1,218 @@ +package gitcache + +import ( + "context" + "crypto/rand" + "errors" + "net/http" + "os" + "path/filepath" + "sync" + "testing" + "time" +) + +// cachedRepo adds a repository of the given size to the cache, still in use +// when busy. +func cachedRepo(t *testing.T, tc *testCache, name string, size int64, busy bool) *repo { + t.Helper() + rp := tc.acquire("up/" + name + ".git") + if err := os.MkdirAll(rp.dir, dirPermissions); err != nil { + t.Fatal(err) + } + rp.size.Store(size) + if !busy { + tc.release(rp) + } + return rp +} + +func (tc *testCache) budget() (reserved int64, cutPending int) { + tc.mu.Lock() + defer tc.mu.Unlock() + return tc.reserved, tc.cutPending +} + +func TestCoverEvictsIdleRepositories(t *testing.T) { + tc := newTestCache(t, "", func(o *Options) { o.MaxSize = 100 }) + old := cachedRepo(t, tc, "old", 60, false) + busy := cachedRepo(t, tc, "busy", 30, true) + + res := tc.newReservation() + if err := res.cover(t.Context(), 20); err != nil { + t.Fatal(err) + } + if fileExists(old.dir) { + t.Error("idle repository was not evicted") + } + if !fileExists(busy.dir) { + t.Error("evicted a repository in use") + } + if reserved, _ := tc.budget(); reserved < 20 { + t.Errorf("reserved %d, want at least 20", reserved) + } + res.release() + if reserved, _ := tc.budget(); reserved != 0 { + t.Errorf("reserved %d after release", reserved) + } +} + +func TestCoverCutsOffOnceOnlyRepositoriesInUseAreLeft(t *testing.T) { + tc := newTestCache(t, "", func(o *Options) { o.MaxSize = 100 }) + busy := cachedRepo(t, tc, "busy", 90, true) + + res := tc.newReservation() + if err := res.cover(t.Context(), 20); !errors.Is(err, ErrCacheFull) { + t.Fatalf("cover: %v, want ErrCacheFull", err) + } + if !fileExists(busy.dir) { + t.Error("evicted a repository in use") + } + res.release() + if reserved, cutPending := tc.budget(); reserved != 0 || cutPending != 0 { + t.Errorf("reserved %d, cut off %d after release", reserved, cutPending) + } +} + +// TestCoverKeepsIdleRepositoriesItCannotUse cuts off a fetch that would not +// fit even with every idle repository evicted, so none is. +func TestCoverKeepsIdleRepositoriesItCannotUse(t *testing.T) { + tc := newTestCache(t, "", func(o *Options) { o.MaxSize = 100 }) + idle := cachedRepo(t, tc, "idle", 5, false) + cachedRepo(t, tc, "busy", 90, true) + + res := tc.newReservation() + if err := res.cover(t.Context(), 20); !errors.Is(err, ErrCacheFull) { + t.Fatalf("cover: %v, want ErrCacheFull", err) + } + if !fileExists(idle.dir) { + t.Error("evicted an idle repository without making room") + } + res.release() +} + +// TestCoverWaitsForCutOffFetch runs two fetches that do not fit together. +// Once one is cut off, the other waits for its space instead of failing too. +func TestCoverWaitsForCutOffFetch(t *testing.T) { + tc := newTestCache(t, "", func(o *Options) { o.MaxSize = 100 }) + cachedRepo(t, tc, "busy", 40, true) + + first, second := tc.newReservation(), tc.newReservation() + if err := first.cover(t.Context(), 30); err != nil { + t.Fatal(err) + } + if err := second.cover(t.Context(), 20); err != nil { + t.Fatal(err) + } + if err := first.cover(t.Context(), 40); !errors.Is(err, ErrCacheFull) { + t.Fatalf("first: %v, want ErrCacheFull", err) + } + + done := make(chan error, 1) + go func() { done <- second.cover(context.Background(), 30) }() + select { + case err := <-done: + t.Fatalf("second did not wait for the first to be released: %v", err) + case <-time.After(100 * time.Millisecond): + } + first.release() + if err := <-done; err != nil { + t.Fatalf("second: %v", err) + } + second.release() + if reserved, cutPending := tc.budget(); reserved != 0 || cutPending != 0 { + t.Errorf("reserved %d, cut off %d after release", reserved, cutPending) + } +} + +// TestConcurrentClonesStayWithinMaxSize clones two repositories at once that +// do not fit in MaxSize together. Both are kept in use, so neither can be +// evicted: one clone succeeds, and the other is cut off without being +// marked too large. +func TestConcurrentClonesStayWithinMaxSize(t *testing.T) { + upstream := newFakeUpstream(t) + for _, name := range []string{"a", "b"} { + work := upstream.newRepo(name) + big := make([]byte, 256<<10) + _, _ = rand.Read(big) + if err := os.WriteFile(filepath.Join(work, "big.bin"), big, 0o600); err != nil { + t.Fatal(err) + } + runGit(t, work, "add", "big.bin") + runGit(t, work, "commit", "--quiet", "-m", "big") + runGit(t, work, "push", "--quiet", "origin", "main") + } + size := repoSize(t, upstream, "a") + maxSize := size * 5 / 3 + tc := newTestCache(t, upstream.server.URL, func(o *Options) { o.MaxSize = maxSize }) + for _, name := range []string{"a", "b"} { + defer tc.release(tc.acquire("up/" + name + ".git")) + } + + // Sample what the cache counts while the clones run. + var peak int64 + stop := make(chan struct{}) + sampled := make(chan struct{}) + go func() { + defer close(sampled) + for { + tc.mu.Lock() + total := tc.reserved + for _, rp := range tc.repos { + total += rp.size.Load() + } + tc.mu.Unlock() + peak = max(peak, total) + select { + case <-stop: + return + case <-time.After(100 * time.Microsecond): + } + } + }() + + // Held back, both fetches are under way before either receives a pack. + upstream.delayFetches.Store(int64(200 * time.Millisecond)) + statuses := make(map[string]int) + var mu sync.Mutex + var wg sync.WaitGroup + for _, name := range []string{"a", "b"} { + wg.Go(func() { + resp, err := http.Get(tc.url(name) + "/info/refs?service=git-upload-pack") + if err != nil { + t.Error(err) + return + } + _ = resp.Body.Close() + mu.Lock() + statuses[name] = resp.StatusCode + mu.Unlock() + }) + } + wg.Wait() + close(stop) + <-sampled + + if (statuses["a"] == http.StatusOK) == (statuses["b"] == http.StatusOK) { + t.Fatalf("statuses %v, want one 200 and one 507", statuses) + } + for name, status := range statuses { + if status != http.StatusOK && status != http.StatusInsufficientStorage { + t.Errorf("%s: status %d, want 507", name, status) + } + if tc.isTooLarge("up/" + name + ".git") { + t.Errorf("%s was marked too large", name) + } + } + // What a clone writes beyond the bytes it receives, such as its index, + // is counted only once it is measured. + if slack := size / 4; peak > maxSize+slack { + t.Errorf("cache counted %d, past max_size %d", peak, maxSize) + } + if reserved, cutPending := tc.budget(); reserved != 0 || cutPending != 0 { + t.Errorf("reserved %d, cut off %d once the clones ended", reserved, cutPending) + } + if entries, _ := os.ReadDir(tc.tmpDir()); len(entries) != 0 { + t.Errorf("left %d entries in the temporary directory", len(entries)) + } +} diff --git a/internal/gitcache/cache.go b/internal/gitcache/cache.go new file mode 100644 index 0000000..4014fd5 --- /dev/null +++ b/internal/gitcache/cache.go @@ -0,0 +1,499 @@ +// Package gitcache serves Git repositories read-only over smart HTTP from a +// local cache of bare repositories, refreshed from configured upstreams. +package gitcache + +import ( + "cmp" + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "io/fs" + "log/slog" + "net/http" + "os" + "os/exec" + "path/filepath" + "regexp" + "runtime" + "slices" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/git-pkgs/proxy/internal/metrics" +) + +const ( + // tmpDirName holds clones in progress and evicted repositories being + // deleted. Upstream names cannot start with a dot, so it cannot clash + // with an upstream's directory, and the name is specific in case + // cache_dir is shared with other data. + tmpDirName = ".gitcache-tmp" + // lockFileName is locked while the cache is open, so a second process + // given the same cache_dir fails to start instead of deleting the first + // one's clones in progress. + lockFileName = ".gitcache-lock" + // usedStampInterval limits how often a repository's last use is written + // to disk. + usedStampInterval = time.Minute + // defaultLowSpeedTime is how many seconds a git transfer may receive + // nothing before it is abandoned. Fetches run with --quiet, so the + // upstream sends nothing while it counts objects; this is far longer. + defaultLowSpeedTime = 600 + ecosystem = "git" + upstreamHashLen = 8 + checkedStamp = "proxy-checked" + usedStamp = "proxy-used" + dirPermissions = 0o750 + lockFilePermissions = 0o600 +) + +// Options configures a Cache. +type Options struct { + CacheDir string + MaxSize int64 + MaxRepoSize int64 + RefreshWait time.Duration + StaleMaxAge time.Duration + // Client makes every upstream request, including those of the git + // processes the cache runs. Redirects are never followed. + Client *http.Client + Logger *slog.Logger + // Upstreams maps each configured upstream's name to its directory in + // CacheDir, as UpstreamDir names it. Only those directories are loaded + // at startup. A directory UpstreamDir would name for one of these names + // at another URL, left by repointing the upstream, is removed. Nothing + // else in CacheDir is touched, so it may hold other data. + Upstreams map[string]string + + // refCheckTimeout, lowSpeedTime, maxRequestBody and maxUploadPacks + // override their defaults in tests. + refCheckTimeout time.Duration + lowSpeedTime int + maxRequestBody int64 + maxUploadPacks int +} + +// Cache is the Git repository cache. internal/handler serves it at /git. +type Cache struct { + opts Options + client *http.Client + logger *slog.Logger + gitPath string + dir string + lock *os.File + relay *relay + fetches chan struct{} + uploadPacks chan struct{} + refCheckTimeout time.Duration + lowSpeedTime int + maxRequestBody int64 + + ctx context.Context + cancel context.CancelFunc + // refreshes counts the refreshes running, which Close waits for. + refreshes sync.WaitGroup + + mu sync.Mutex + // closed stops refreshes from starting once Close has begun. + closed bool + repos map[string]*repo + tooLarge map[string]time.Time + // overBudget records that eviction left the cache over MaxSize because + // the rest of it was in use. + overBudget bool + // reserved is the space reservations hold for fetches in progress, and + // cutPending counts those cut off and not yet released. freed is closed + // and replaced whenever a reservation gives back space or one cut off is + // released. + reserved int64 + cutPending int + freed chan struct{} + // failedWants records, by repository key and object ID, wants the + // upstream could not provide. + failedWants map[string]time.Time +} + +// repo is a cached repository, or one being cloned. users, refreshing and +// the map entry itself are guarded by Cache.mu. +type repo struct { + key string + dir string + + users int + refreshing *refreshCall + // writes serializes git fetches into the repository. + writes chan struct{} + + size atomic.Int64 + lastUsed atomic.Int64 + lastChecked atomic.Int64 + usedWritten atomic.Int64 +} + +var errLocked = errors.New("locked by another process") + +// maxConcurrentFetches caps clones and fetches running at once. Refreshes +// outlive the requests that start them, so without a cap a client could +// start any number and hang up. +const maxConcurrentFetches = 16 + +// uploadPacksPerCPU caps the upload-pack processes building packs at once, +// so a burst of clones cannot run the proxy out of memory. Each uses one +// delta-search thread (servingConfig); twice the CPUs keeps them busy while +// some packs wait on slow clients. +const uploadPacksPerCPU = 2 + +// New creates the cache, loading repositories already in CacheDir. +func New(opts Options) (*Cache, error) { + gitPath, err := exec.LookPath("git") + if err != nil { + return nil, fmt.Errorf("upstream.git needs git installed: %w", err) + } + dir, err := filepath.Abs(opts.CacheDir) + if err != nil { + return nil, err + } + if err := os.MkdirAll(dir, dirPermissions); err != nil { + return nil, err + } + lock, err := lockFile(filepath.Join(dir, lockFileName)) + if errors.Is(err, errLocked) { + return nil, fmt.Errorf("git.cache_dir %s is in use by another process; each needs its own", dir) + } + if err != nil { + return nil, fmt.Errorf("locking git.cache_dir %s: %w", dir, err) + } + logger := opts.Logger + if logger == nil { + logger = slog.New(slog.DiscardHandler) + } + // A repository larger than the whole cache would be evicted once + // released and cloned again by the next request. + if opts.MaxSize > 0 && (opts.MaxRepoSize <= 0 || opts.MaxRepoSize > opts.MaxSize) { + opts.MaxRepoSize = opts.MaxSize + } + client := *opts.Client + client.Timeout = 0 + client.CheckRedirect = func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse } + + uploadPacks := uploadPacksPerCPU * runtime.GOMAXPROCS(0) + if opts.maxUploadPacks > 0 { + uploadPacks = opts.maxUploadPacks + } + c := &Cache{ + opts: opts, + client: &client, + logger: logger, + gitPath: gitPath, + dir: dir, + lock: lock, + fetches: make(chan struct{}, maxConcurrentFetches), + uploadPacks: make(chan struct{}, uploadPacks), + refCheckTimeout: defaultRefCheckTimeout, + lowSpeedTime: defaultLowSpeedTime, + maxRequestBody: defaultMaxRequestBody, + repos: make(map[string]*repo), + tooLarge: make(map[string]time.Time), + failedWants: make(map[string]time.Time), + freed: make(chan struct{}), + } + if opts.refCheckTimeout > 0 { + c.refCheckTimeout = opts.refCheckTimeout + } + if opts.lowSpeedTime > 0 { + c.lowSpeedTime = opts.lowSpeedTime + } + if opts.maxRequestBody > 0 { + c.maxRequestBody = opts.maxRequestBody + } + if err := c.load(); err != nil { + _ = lock.Close() + return nil, err + } + if c.relay, err = newRelay(c.client, logger); err != nil { + _ = lock.Close() + return nil, fmt.Errorf("starting git relay: %w", err) + } + c.ctx, c.cancel = context.WithCancel(context.Background()) + // Applies a lowered git.max_size straight away. + c.evict() + return c, nil +} + +// Close stops the refreshes in progress and waits for them to finish, so +// none is still writing to the cache when it returns, then stops the relay. +// Fetches by ID run on request goroutines; stop the HTTP server first. +// Closing again does nothing. +func (c *Cache) Close() error { + c.mu.Lock() + if c.closed { + c.mu.Unlock() + return nil + } + c.closed = true + c.mu.Unlock() + c.cancel() + c.refreshes.Wait() + return errors.Join(c.relay.close(), c.lock.Close()) +} + +// UpstreamDir names the top-level cache directory of the upstream called +// name at baseURL. It includes a hash of the URL, so repointing an upstream +// starts a fresh cache rather than serving the old host's repositories. +func UpstreamDir(name, baseURL string) string { + digest := sha256.Sum256([]byte(strings.TrimSuffix(baseURL, "/"))) + return name + "-" + hex.EncodeToString(digest[:])[:upstreamHashLen] +} + +func (c *Cache) tmpDir() string { + return filepath.Join(c.dir, tmpDirName) +} + +// load clears what an earlier run left in the temporary directory and +// registers the repositories already cached. +func (c *Cache) load() error { + if err := os.RemoveAll(c.tmpDir()); err != nil { + return fmt.Errorf("clearing %s: %w", c.tmpDir(), err) + } + if err := os.MkdirAll(c.tmpDir(), dirPermissions); err != nil { + return err + } + c.removeRepointed() + for _, upstreamDir := range c.opts.Upstreams { + root := filepath.Join(c.dir, upstreamDir) + err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { + if err != nil || !d.IsDir() || !strings.HasSuffix(d.Name(), ".git") || path == root { + return err + } + rel, _ := filepath.Rel(c.dir, path) + rp := newRepo(filepath.ToSlash(rel), path) + rp.size.Store(dirSize(path)) + rp.lastChecked.Store(stampTime(path, checkedStamp)) + rp.lastUsed.Store(stampTime(path, usedStamp)) + rp.usedWritten.Store(rp.lastUsed.Load()) + c.repos[rp.key] = rp + return filepath.SkipDir + }) + if err != nil && !errors.Is(err, fs.ErrNotExist) { + return err + } + } + return nil +} + +// upstreamDirPattern matches a directory UpstreamDir names, capturing the +// upstream's name. +var upstreamDirPattern = regexp.MustCompile(`^(.+)-[0-9a-f]{8}$`) + +// removeRepointed removes the directories of configured upstreams at URLs +// they no longer have. Each is moved aside first, so nothing can be read +// from it half deleted. Directories of upstreams no longer configured are +// only logged: nothing else says they are the cache's. +func (c *Cache) removeRepointed() { + entries, err := os.ReadDir(c.dir) + if err != nil { + c.logger.Warn("git cache: cannot list the cache directory", "error", err) + return + } + current := make(map[string]bool, len(c.opts.Upstreams)) + for _, upstreamDir := range c.opts.Upstreams { + current[upstreamDir] = true + } + var unconfigured []string + for _, entry := range entries { + name := entry.Name() + match := upstreamDirPattern.FindStringSubmatch(name) + if !entry.IsDir() || match == nil || current[name] { + continue + } + if _, configured := c.opts.Upstreams[match[1]]; !configured { + unconfigured = append(unconfigured, name) + continue + } + trash, err := os.MkdirTemp(c.tmpDir(), "repointed-") + if err == nil { + err = os.Rename(filepath.Join(c.dir, name), filepath.Join(trash, name)) + } + if err != nil { + c.logger.Warn("git cache: failed to remove the cache of a repointed upstream", "directory", name, "error", err) + continue + } + c.logger.Info("git cache: removed the cache of a repointed upstream", "directory", name) + if err := os.RemoveAll(trash); err != nil { + c.logger.Warn("git cache: failed to delete a removed cache", "path", trash, "error", err) + } + } + if len(unconfigured) > 0 { + c.logger.Info("git cache: leaving directories that look like caches of upstreams no longer configured; delete them if they are", + "directories", unconfigured) + } +} + +// acquire returns the repository for key, counting the caller as a user +// until release, so it is not evicted while in use. +func (c *Cache) acquire(key string) *repo { + c.mu.Lock() + defer c.mu.Unlock() + rp := c.repos[key] + if rp == nil { + rp = newRepo(key, filepath.Join(c.dir, filepath.FromSlash(key))) + c.repos[key] = rp + } + rp.users++ + return rp +} + +func (c *Cache) release(rp *repo) { + now := time.Now() + rp.lastUsed.Store(now.UnixNano()) + exists := rp.exists() + if exists && now.Sub(time.Unix(0, rp.usedWritten.Load())) >= usedStampInterval { + rp.usedWritten.Store(now.UnixNano()) + touch(rp.dir, usedStamp, now) + } + + c.mu.Lock() + rp.users-- + idle := rp.users == 0 && c.repos[rp.key] == rp + // Forget repositories that never got cloned, such as ones missing upstream. + if idle && !exists { + delete(c.repos, rp.key) + } + _, tooLarge := c.tooLarge[rp.key] + retry := tooLarge || c.overBudget + c.mu.Unlock() + if idle && exists && retry { + c.evict() + } +} + +func newRepo(key, dir string) *repo { + return &repo{key: key, dir: dir, writes: make(chan struct{}, 1)} +} + +// lockWrites waits for other fetches into the repository to finish. +func (rp *repo) lockWrites(ctx context.Context) (func(), error) { + select { + case rp.writes <- struct{}{}: + return func() { <-rp.writes }, nil + case <-ctx.Done(): + return nil, ctx.Err() + } +} + +func (rp *repo) exists() bool { + _, err := os.Stat(rp.dir) + return err == nil +} + +// evict removes repositories nobody is using: the least recently used until +// the cache fits in MaxSize, and any that could not be refreshed within +// MaxRepoSize, so the next request clones them again with only their +// branches and tags. It runs after every fetch, and when a repository it had +// to skip is released: one too large to refresh, or any while the cache is +// over MaxSize. Fetches also evict as they receive bytes (reservation.cover). +func (c *Cache) evict() { + c.mu.Lock() + removed, _ := c.evictLocked(0) + c.mu.Unlock() + c.removeEvicted(removed) +} + +// evictLocked evicts until the cache, counting reservations, has room for +// extra more bytes, and returns the evicted repositories' directories, to be +// deleted once c.mu is released, and the size left. When evicting every idle +// repository would not make that room, it evicts only as evict does. A +// repository is moved aside under the lock, so a request that comes after it +// clones it again instead of reading it while it is deleted. +func (c *Cache) evictLocked(extra int64) ([]string, int64) { + total := c.reserved + var idleTotal int64 + idle := make([]*repo, 0, len(c.repos)) + for _, rp := range c.repos { + total += rp.size.Load() + if rp.users == 0 { + idle = append(idle, rp) + idleTotal += rp.size.Load() + } + } + if total-idleTotal+extra > c.opts.MaxSize { + extra = 0 + } + slices.SortFunc(idle, func(a, b *repo) int { + return cmp.Compare(a.lastUsed.Load(), b.lastUsed.Load()) + }) + var removed []string + for _, rp := range idle { + _, marked := c.tooLarge[rp.key] + tooLarge := marked && rp.exists() + if !tooLarge && (c.opts.MaxSize <= 0 || total+extra <= c.opts.MaxSize) { + continue + } + trash, err := os.MkdirTemp(c.tmpDir(), "evicted-") + if err == nil { + err = os.Rename(rp.dir, filepath.Join(trash, "repo")) + } + if err != nil && !errors.Is(err, fs.ErrNotExist) { + c.logger.Warn("git cache: failed to evict repository", "repository", rp.key, "error", err) + continue + } + delete(c.repos, rp.key) + total -= rp.size.Load() + removed = append(removed, trash) + reason := "lru" + if tooLarge { + // The clone that replaces it enforces the limit itself. + delete(c.tooLarge, rp.key) + reason = "too_large" + } + metrics.RecordArtifactEvicted(reason, ecosystem) + c.logger.Info("git cache: evicted repository", "repository", rp.key, "size", rp.size.Load(), "reason", reason) + } + c.overBudget = c.opts.MaxSize > 0 && total > c.opts.MaxSize + return removed, total +} + +func (c *Cache) removeEvicted(removed []string) { + for _, trash := range removed { + if err := os.RemoveAll(trash); err != nil { + c.logger.Warn("git cache: failed to delete evicted repository", "path", trash, "error", err) + } + } +} + +func dirSize(dir string) int64 { + var size int64 + _ = filepath.WalkDir(dir, func(_ string, d fs.DirEntry, err error) error { + if err == nil && d.Type().IsRegular() { + if info, err := d.Info(); err == nil { + size += info.Size() + } + } + return nil + }) + return size +} + +// Stamps are empty files whose modification time records an event, so it +// survives restarts. +func stampTime(dir, name string) int64 { + info, err := os.Stat(filepath.Join(dir, name)) + if err != nil { + return 0 + } + return info.ModTime().UnixNano() +} + +func touch(dir, name string, t time.Time) { + path := filepath.Join(dir, name) + if err := os.Chtimes(path, t, t); errors.Is(err, fs.ErrNotExist) { + if f, err := os.Create(path); err == nil { + _ = f.Close() + _ = os.Chtimes(path, t, t) + } + } +} diff --git a/internal/gitcache/cache_test.go b/internal/gitcache/cache_test.go new file mode 100644 index 0000000..1770361 --- /dev/null +++ b/internal/gitcache/cache_test.go @@ -0,0 +1,765 @@ +package gitcache + +import ( + "context" + "crypto/rand" + "errors" + "fmt" + "maps" + "net/http" + "net/http/httptest" + "net/http/httputil" + "net/url" + "os" + "path/filepath" + "slices" + "strings" + "sync" + "testing" + "time" + + "github.com/git-pkgs/registries/safehttp" +) + +func TestCloneMatchesUpstream(t *testing.T) { + for _, version := range []string{"0", "1", "2"} { + t.Run("protocol v"+version, func(t *testing.T) { + upstream := newFakeUpstream(t) + upstream.newRepo("repo") + tc := newTestCache(t, upstream.server.URL, nil) + + clone := tc.clone(t, "repo.git", "-c", "protocol.version="+version) + + want := runGit(t, "", "ls-remote", "--heads", "--tags", upstream.bare()) + if got := runGit(t, "", "-c", "protocol.version="+version, "ls-remote", "--heads", "--tags", tc.url("repo.git")); got != want { + t.Errorf("ls-remote through the cache:\n%s\nwant:\n%s", got, want) + } + if got, want := runGit(t, clone, "rev-parse", "HEAD"), runGit(t, upstream.bare(), "rev-parse", "main"); got != want { + t.Errorf("HEAD = %s, want %s", got, want) + } + runGit(t, clone, "fsck", "--no-dangling", "--no-progress") + }) + } +} + +func TestUpstreamWithoutProtocolV2(t *testing.T) { + upstream := newFakeUpstream(t) + upstream.v0Only = true + upstream.newRepo("repo") + tc := newTestCache(t, upstream.server.URL, nil) + + clone := tc.clone(t, "repo") + if got, want := runGit(t, clone, "rev-parse", "HEAD"), runGit(t, upstream.bare(), "rev-parse", "main"); got != want { + t.Errorf("HEAD = %s, want %s", got, want) + } +} + +func TestUnchangedRepositoryIsNotFetchedAgain(t *testing.T) { + upstream := newFakeUpstream(t) + upstream.newRepo("repo") + tc := newTestCache(t, upstream.server.URL, nil) + + tc.clone(t, "repo") + fetches := upstream.gitRequests.Load() + tc.clone(t, "repo") + if got := upstream.gitRequests.Load(); got != fetches { + t.Errorf("second clone made %d upstream fetch requests, want none", got-fetches) + } +} + +func TestPushedCommitIsServedOnNextClone(t *testing.T) { + upstream := newFakeUpstream(t) + work := upstream.newRepo("repo") + tc := newTestCache(t, upstream.server.URL, nil) + existing := tc.clone(t, "repo") + + head := upstream.commit(work, "second") + runGit(t, work, "push", "--quiet", "origin", "main") + + if got := runGit(t, tc.clone(t, "repo"), "rev-parse", "HEAD"); got != head { + t.Errorf("clone after push: HEAD = %s, want %s", got, head) + } + runGit(t, existing, "pull", "--quiet", "--ff-only") + if got := runGit(t, existing, "rev-parse", "HEAD"); got != head { + t.Errorf("pull after push: HEAD = %s, want %s", got, head) + } + if got := runGit(t, tc.clone(t, "repo", "--depth=1"), "rev-list", "--count", "HEAD"); got != "1" { + t.Errorf("shallow clone has %s commits, want 1", got) + } +} + +func TestOnlyBranchesAndTagsAreCachedAndPruned(t *testing.T) { + upstream := newFakeUpstream(t) + work := upstream.newRepo("repo") + runGit(t, work, "push", "--quiet", "origin", "main:refs/changes/01/1/1", "main:refs/heads/gone", "main:refs/tags/gone") + tc := newTestCache(t, upstream.server.URL, nil) + tc.clone(t, "repo") + + refs := tc.cachedRefs(t) + if strings.Contains(refs, "refs/changes/") { + t.Errorf("cached refs/changes:\n%s", refs) + } + if !strings.Contains(refs, "refs/heads/gone") || !strings.Contains(refs, "refs/tags/gone") { + t.Fatalf("branch and tag not cached:\n%s", refs) + } + + runGit(t, work, "push", "--quiet", "origin", ":refs/heads/gone", ":refs/tags/gone") + tc.clone(t, "repo") + if refs := tc.cachedRefs(t); strings.Contains(refs, "gone") { + t.Errorf("deleted branch or tag still cached:\n%s", refs) + } + + // Gerrit advertises a ref per patch set, so asking for every ref would + // list hundreds of thousands for a large repository. + bodies := upstream.lsRefsBodies() + if len(bodies) == 0 { + t.Fatal("no ls-refs request reached the upstream") + } + for _, body := range bodies { + if !strings.Contains(body, "ref-prefix refs/heads/") || !strings.Contains(body, "ref-prefix refs/tags/") { + t.Errorf("ls-refs asked for every ref, want branches and tags only:\n%q", body) + } + } +} + +func TestDefaultBranchChangeIsFollowed(t *testing.T) { + upstream := newFakeUpstream(t) + work := upstream.newRepo("repo") + tc := newTestCache(t, upstream.server.URL, nil) + tc.clone(t, "repo") + + runGit(t, work, "checkout", "--quiet", "-b", "trunk") + head := upstream.commit(work, "on trunk") + runGit(t, work, "push", "--quiet", "origin", "trunk") + runGit(t, upstream.bare(), "symbolic-ref", "HEAD", "refs/heads/trunk") + + clone := tc.clone(t, "repo") + if got := runGit(t, clone, "rev-parse", "HEAD"); got != head { + t.Errorf("HEAD = %s, want trunk's %s", got, head) + } +} + +func TestConcurrentColdClonesShareOneFetch(t *testing.T) { + upstream := newFakeUpstream(t) + upstream.newRepo("repo") + tc := newTestCache(t, upstream.server.URL, nil) + + release := upstream.holdRequests() + var wg sync.WaitGroup + for range 8 { + wg.Go(func() { + dir := filepath.Join(t.TempDir(), "clone") + if out, err := tryGit(t, "", "clone", "--quiet", tc.url("repo"), dir); err != nil { + t.Errorf("clone: %v\n%s", err, out) + } + }) + } + time.Sleep(200 * time.Millisecond) + release() + wg.Wait() + + if got := upstream.gitSessions.Load(); got != 1 { + t.Errorf("upstream saw %d fetches, want 1", got) + } +} + +func TestMissingRepositoryLeavesNothingBehind(t *testing.T) { + upstream := newFakeUpstream(t) + tc := newTestCache(t, upstream.server.URL, nil) + + resp, err := http.Get(tc.url("missing") + "/info/refs?service=git-upload-pack") + if err != nil { + t.Fatal(err) + } + _ = resp.Body.Close() + if resp.StatusCode != http.StatusNotFound { + t.Errorf("status %d, want 404", resp.StatusCode) + } + waitForRepos(t, tc.Cache, 0) + if _, err := os.Stat(filepath.Join(tc.dir, "up", "missing.git")); !os.IsNotExist(err) { + t.Errorf("repository directory left behind: %v", err) + } +} + +func TestStaleCopyServedOnlyWithinStaleMaxAge(t *testing.T) { + upstream := newFakeUpstream(t) + upstream.newRepo("repo") + tc := newTestCache(t, upstream.server.URL, nil) + tc.clone(t, "repo") + + upstream.status.Store(http.StatusServiceUnavailable) + tc.clone(t, "repo") + + waitIdle(t, tc.Cache) + rp := tc.acquire("up/repo.git") + rp.lastChecked.Store(time.Now().Add(-2 * time.Hour).UnixNano()) + tc.release(rp) + if out, err := tryGit(t, "", "ls-remote", tc.url("repo")); err == nil { + t.Errorf("served a copy older than stale_max_age:\n%s", out) + } +} + +func TestRefusedRefreshIsNotServedStale(t *testing.T) { + for _, status := range []int{http.StatusUnauthorized, http.StatusForbidden, http.StatusNotFound, http.StatusMovedPermanently} { + t.Run(fmt.Sprint(status), func(t *testing.T) { + upstream := newFakeUpstream(t) + upstream.newRepo("repo") + tc := newTestCache(t, upstream.server.URL, nil) + tc.clone(t, "repo") + + upstream.status.Store(int32(status)) + if out, err := tryGit(t, "", "ls-remote", tc.url("repo")); err == nil { + t.Errorf("served the cached copy after status %d:\n%s", status, out) + } + }) + } +} + +func TestSlowUpstreamServesCachedCopyAfterRefreshWait(t *testing.T) { + upstream := newFakeUpstream(t) + work := upstream.newRepo("repo") + tc := newTestCache(t, upstream.server.URL, func(o *Options) { o.RefreshWait = 100 * time.Millisecond }) + old := runGit(t, tc.clone(t, "repo"), "rev-parse", "HEAD") + head := upstream.commit(work, "second") + runGit(t, work, "push", "--quiet", "origin", "main") + + release := upstream.holdRequests() + defer release() + if got := runGit(t, tc.clone(t, "repo"), "rev-parse", "HEAD"); got != old { + t.Errorf("HEAD = %s, want the cached %s", got, old) + } + release() + waitFor(t, func() bool { return strings.Contains(tc.cachedRefs(t), head) }) +} + +// TestSlowFetchAfterPushIsWaitedFor has the upstream answer the ref check at +// once and the fetch after a push take longer than RefreshWait: a clone +// still gets the pushed commit. RefreshWait leaves the check time to finish +// on a slow machine. +func TestSlowFetchAfterPushIsWaitedFor(t *testing.T) { + upstream := newFakeUpstream(t) + work := upstream.newRepo("repo") + tc := newTestCache(t, upstream.server.URL, func(o *Options) { o.RefreshWait = 2 * time.Second }) + tc.clone(t, "repo") + head := upstream.commit(work, "second") + runGit(t, work, "push", "--quiet", "origin", "main") + + upstream.delayFetches.Store(int64(4 * time.Second)) + if got := runGit(t, tc.clone(t, "repo"), "rev-parse", "HEAD"); got != head { + t.Errorf("HEAD = %s, want the pushed %s", got, head) + } +} + +// TestStalledFetchAfterPushServesCachedCopy has the fetch after a push +// stall: once it is abandoned, the clone gets the cached copy. +func TestStalledFetchAfterPushServesCachedCopy(t *testing.T) { + upstream := newFakeUpstream(t) + work := upstream.newRepo("repo") + tc := newTestCache(t, upstream.server.URL, func(o *Options) { + o.RefreshWait = 100 * time.Millisecond + o.lowSpeedTime = 1 + }) + old := runGit(t, tc.clone(t, "repo"), "rev-parse", "HEAD") + upstream.commit(work, "second") + runGit(t, work, "push", "--quiet", "origin", "main") + + upstream.stallFetches.Store(true) + if got := runGit(t, tc.clone(t, "repo"), "rev-parse", "HEAD"); got != old { + t.Errorf("HEAD = %s, want the cached %s", got, old) + } +} + +func TestUpstreamCredentialsReachEveryUpstreamRequest(t *testing.T) { + upstream := newFakeUpstream(t) + upstream.requireAuth = "Bearer secret" + upstream.newRepo("repo") + client := &http.Client{Transport: headerTransport{"Authorization", "Bearer secret"}} + tc := newTestCache(t, upstream.server.URL, func(o *Options) { o.Client = client }) + + tc.clone(t, "repo") + if got := upstream.unauthed.Load(); got != 0 { + t.Errorf("%d upstream requests lacked credentials", got) + } + if upstream.gitRequests.Load() == 0 { + t.Error("no fetch went through the relay") + } +} + +type headerTransport struct{ name, value string } + +func (h headerTransport) RoundTrip(r *http.Request) (*http.Response, error) { + r = r.Clone(r.Context()) + r.Header.Set(h.name, h.value) + return http.DefaultTransport.RoundTrip(r) +} + +// TestUpstreamAddressChecksApplyToGit uses a client that refuses loopback +// addresses, as the proxy's does by default. git must not reach the loopback +// upstream around it, whether the ref check or only git's fetch is refused. +func TestUpstreamAddressChecksApplyToGit(t *testing.T) { + blocked := safehttp.New(nil, safehttp.Options{}) + for name, client := range map[string]*http.Client{ + "every request": blocked, + "git's fetch": {Transport: gitOnlyTransport{blocked.Transport}}, + } { + t.Run(name, func(t *testing.T) { + upstream := newFakeUpstream(t) + upstream.newRepo("repo") + tc := newTestCache(t, upstream.server.URL, func(o *Options) { o.Client = client }) + + if out, err := tryGit(t, "", "ls-remote", tc.url("repo")); err == nil { + t.Fatalf("clone through a blocked upstream succeeded:\n%s", out) + } + if got := upstream.gitRequests.Load(); got != 0 { + t.Errorf("git reached the upstream with %d requests", got) + } + }) + } +} + +// gitOnlyTransport sends the cache's own ref checks with the default +// transport, and git's relayed requests with next. +type gitOnlyTransport struct{ next http.RoundTripper } + +func (g gitOnlyTransport) RoundTrip(r *http.Request) (*http.Response, error) { + if r.UserAgent() == gitUserAgent { + return http.DefaultTransport.RoundTrip(r) + } + return g.next.RoundTrip(r) +} + +// TestRedirectIsNotFollowed redirects the ref check, then only git's fetch. +func TestRedirectIsNotFollowed(t *testing.T) { + for _, redirectRefCheck := range []bool{true, false} { + t.Run(fmt.Sprint("ref check redirected: ", redirectRefCheck), func(t *testing.T) { + target := newFakeUpstream(t) + target.newRepo("repo") + targetURL, _ := url.Parse(target.server.URL) + passThrough := httputil.NewSingleHostReverseProxy(targetURL) + redirect := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.UserAgent() == gitUserAgent && !redirectRefCheck { + passThrough.ServeHTTP(w, r) + return + } + http.Redirect(w, r, target.server.URL+r.URL.RequestURI(), http.StatusFound) + })) + defer redirect.Close() + tc := newTestCache(t, redirect.URL, nil) + + if out, err := tryGit(t, "", "ls-remote", tc.url("repo")); err == nil { + t.Fatalf("followed a redirect:\n%s", out) + } + if got := target.gitRequests.Load(); got != 0 { + t.Errorf("redirect target received %d requests from git", got) + } + }) + } +} + +func TestRepositoryOverMaxRepoSizeIsRefused(t *testing.T) { + upstream := newFakeUpstream(t) + work := upstream.newRepo("repo") + big := make([]byte, 256<<10) + _, _ = rand.Read(big) + if err := os.WriteFile(filepath.Join(work, "big.bin"), big, 0o600); err != nil { + t.Fatal(err) + } + runGit(t, work, "add", "big.bin") + runGit(t, work, "commit", "--quiet", "-m", "big") + runGit(t, work, "push", "--quiet", "origin", "main") + + // max_size caps max_repo_size. + for name, configure := range map[string]func(*Options){ + "max_repo_size": func(o *Options) { o.MaxRepoSize = 64 << 10 }, + "max_size": func(o *Options) { o.MaxSize = 64 << 10 }, + } { + t.Run(name, func(t *testing.T) { + tc := newTestCache(t, upstream.server.URL, configure) + + resp, err := http.Get(tc.url("repo") + "/info/refs?service=git-upload-pack") + if err != nil { + t.Fatal(err) + } + _ = resp.Body.Close() + if resp.StatusCode != http.StatusInsufficientStorage { + t.Errorf("status %d, want 507", resp.StatusCode) + } + waitForRepos(t, tc.Cache, 0) + if entries, _ := os.ReadDir(tc.tmpDir()); len(entries) != 0 { + t.Errorf("left %d entries in the temporary directory", len(entries)) + } + + requests := upstream.requests.Load() + if out, err := tryGit(t, "", "ls-remote", tc.url("repo")); err == nil { + t.Errorf("second attempt succeeded:\n%s", out) + } + if got := upstream.requests.Load(); got != requests { + t.Errorf("second attempt made %d upstream requests, want none", got-requests) + } + }) + } +} + +// TestEvictionRemovesLeastRecentlyUsedIdleRepositories makes the least +// recently used repository one a request is reading, so the next one is +// evicted instead. +func TestEvictionRemovesLeastRecentlyUsedIdleRepositories(t *testing.T) { + upstream := newFakeUpstream(t) + for _, name := range []string{"old", "busy", "new"} { + upstream.newRepo(name) + } + size := repoSize(t, upstream, "new") + tc := newTestCache(t, upstream.server.URL, func(o *Options) { o.MaxSize = 2*size + size/2 }) + + tc.clone(t, "busy") + busy := tc.acquire("up/busy.git") + defer tc.release(busy) + busy.lastUsed.Store(0) + tc.clone(t, "old") + waitFor(t, func() bool { return tc.usersOf("up/old.git") == 0 }) + tc.repo("up/old.git").lastUsed.Store(1) + + // The third clone takes the cache past MaxSize. + tc.clone(t, "new") + waitFor(t, func() bool { return !fileExists(filepath.Join(tc.dir, "up", "old.git")) }) + for _, name := range []string{"busy", "new"} { + if !fileExists(filepath.Join(tc.dir, "up", name+".git")) { + t.Errorf("%s was evicted", name) + } + } + tc.clone(t, "old") +} + +// repoSize is the size a cache gives repository name once cloned. +func repoSize(t *testing.T, upstream *fakeUpstream, name string) int64 { + t.Helper() + probe := newTestCache(t, upstream.server.URL, nil) + probe.clone(t, name) + return probe.repo("up/" + name + ".git").size.Load() +} + +func TestNewLoadsCachedRepositories(t *testing.T) { + upstream := newFakeUpstream(t) + upstream.newRepo("repo") + tc := newTestCache(t, upstream.server.URL, nil) + tc.clone(t, "repo") + if err := os.MkdirAll(filepath.Join(tc.tmpDir(), "clone-leftover"), 0o750); err != nil { + t.Fatal(err) + } + if err := tc.Close(); err != nil { + t.Fatal(err) + } + + reloaded, err := New(Options{ + Upstreams: map[string]string{"up": "up"}, + CacheDir: tc.dir, + StaleMaxAge: time.Hour, + Client: &http.Client{}, + }) + if err != nil { + t.Fatal(err) + } + defer func() { _ = reloaded.Close() }() + + rp := reloaded.repos["up/repo.git"] // nothing else uses reloaded yet + if rp == nil || rp.size.Load() == 0 || rp.lastChecked.Load() == 0 { + t.Fatalf("repository not loaded: %+v", rp) + } + if entries, _ := os.ReadDir(reloaded.tmpDir()); len(entries) != 0 { + t.Errorf("temporary directory not cleared: %v", entries) + } +} + +func waitForRepos(t *testing.T, c *Cache, n int) { + t.Helper() + waitFor(t, func() bool { + c.mu.Lock() + defer c.mu.Unlock() + return len(c.repos) == n + }) +} + +func waitFor(t *testing.T, cond func() bool) { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + for !cond() { + select { + case <-ctx.Done(): + t.Fatal("condition not met in time") + case <-time.After(20 * time.Millisecond): + } + } +} + +// TestRewoundCacheIsBroughtBackInLine changes the cache behind the upstream's +// back, so it only matches again if the next request refreshes branches and +// tags and prunes what the upstream does not have. +func TestRewoundCacheIsBroughtBackInLine(t *testing.T) { + upstream := newFakeUpstream(t) + work := upstream.newRepo("repo") + upstream.commit(work, "second") + runGit(t, work, "push", "--quiet", "origin", "main") + tc := newTestCache(t, upstream.server.URL, nil) + tc.clone(t, "repo") + + cached := filepath.Join(tc.dir, "up", "repo.git") + runGit(t, cached, "update-ref", "refs/heads/main", "main~1") + runGit(t, cached, "update-ref", "refs/heads/extra", "main") + runGit(t, cached, "tag", "extra", "main") + + want := runGit(t, "", "ls-remote", "--heads", "--tags", upstream.bare()) + if got := runGit(t, "", "ls-remote", "--heads", "--tags", tc.url("repo")); got != want { + t.Errorf("ls-remote through the cache:\n%s\nwant:\n%s", got, want) + } +} + +// TestConcurrentWarmClonesDuringRefresh serves clones from the cache while a +// fetch writes to it. +func TestConcurrentWarmClonesDuringRefresh(t *testing.T) { + upstream := newFakeUpstream(t) + work := upstream.newRepo("repo") + tc := newTestCache(t, upstream.server.URL, func(o *Options) { o.RefreshWait = time.Millisecond }) + old := runGit(t, tc.clone(t, "repo"), "rev-parse", "HEAD") + head := upstream.commit(work, "second") + runGit(t, work, "push", "--quiet", "origin", "main") + + var wg sync.WaitGroup + for range 8 { + wg.Go(func() { + dir := filepath.Join(t.TempDir(), "clone") + if out, err := tryGit(t, "", "clone", "--quiet", tc.url("repo"), dir); err != nil { + t.Errorf("clone: %v\n%s", err, out) + return + } + if got := runGit(t, dir, "rev-parse", "HEAD"); got != old && got != head { + t.Errorf("HEAD = %s, want %s or %s", got, old, head) + } + runGit(t, dir, "fsck", "--no-dangling", "--no-progress") + }) + } + wg.Wait() + waitFor(t, func() bool { return strings.Contains(tc.cachedRefs(t), head) }) +} + +// TestStalledRefCheckFailsInsteadOfHanging has the upstream send headers and +// then nothing, as a broken load balancer might. +func TestStalledRefCheckFailsInsteadOfHanging(t *testing.T) { + stall := make(chan struct{}) + defer close(stall) + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", advertisementType) + w.WriteHeader(http.StatusOK) + _ = http.NewResponseController(w).Flush() + select { + case <-stall: + case <-r.Context().Done(): + } + })) + defer upstream.Close() + tc := newTestCache(t, upstream.URL, func(o *Options) { o.refCheckTimeout = 200 * time.Millisecond }) + + start := time.Now() + if out, err := tryGit(t, "", "ls-remote", tc.url("repo")); err == nil { + t.Fatalf("ls-remote succeeded:\n%s", out) + } + if elapsed := time.Since(start); elapsed > 10*time.Second { + t.Errorf("took %s to fail", elapsed) + } +} + +// TestStalledFetchFailsInsteadOfHanging has the upstream answer the ref check +// but never git's fetch, as a connection that stalls mid-pack would. +func TestStalledFetchFailsInsteadOfHanging(t *testing.T) { + upstream := newFakeUpstream(t) + upstream.newRepo("repo") + upstream.stallFetches.Store(true) + tc := newTestCache(t, upstream.server.URL, func(o *Options) { o.lowSpeedTime = 1 }) + + start := time.Now() + if out, err := tryGit(t, "", "ls-remote", tc.url("repo")); err == nil { + t.Fatalf("ls-remote succeeded:\n%s", out) + } + if elapsed := time.Since(start); elapsed > 20*time.Second { + t.Errorf("took %s to fail", elapsed) + } +} + +// TestFetchEvictsToMaxSize checks eviction runs on its own once a fetch has +// grown the cache past MaxSize. +func TestFetchEvictsToMaxSize(t *testing.T) { + upstream := newFakeUpstream(t) + upstream.newRepo("first") + upstream.newRepo("second") + size := repoSize(t, upstream, "first") + + tc := newTestCache(t, upstream.server.URL, func(o *Options) { o.MaxSize = size + size/2 }) + tc.clone(t, "first") + waitIdle(t, tc.Cache) + tc.clone(t, "second") + waitFor(t, func() bool { return !fileExists(filepath.Join(tc.dir, "up", "first.git")) }) + if !fileExists(filepath.Join(tc.dir, "up", "second.git")) { + t.Error("evicted the repository just fetched") + } +} + +// TestReleaseEvictsWhatEvictionHadToSkip fills the cache past MaxSize while +// every repository is in use, so eviction after the fetch can remove none. +// The first release brings the cache back within MaxSize. +func TestReleaseEvictsWhatEvictionHadToSkip(t *testing.T) { + upstream := newFakeUpstream(t) + upstream.newRepo("first") + upstream.newRepo("second") + size := repoSize(t, upstream, "first") + + tc := newTestCache(t, upstream.server.URL, func(o *Options) { o.MaxSize = size + size/2 }) + first := tc.acquire("up/first.git") + tc.clone(t, "first") + second := tc.acquire("up/second.git") + defer tc.release(second) + tc.clone(t, "second") + waitFor(t, func() bool { + tc.mu.Lock() + defer tc.mu.Unlock() + return tc.overBudget + }) + + tc.release(first) + if fileExists(first.dir) { + t.Error("first is still cached once released") + } + if !fileExists(second.dir) { + t.Error("evicted second while in use") + } + tc.clone(t, "second") +} + +// TestNewRemovesRepointedUpstreams starts a cache whose directory holds an +// upstream's cache at its old URL, and one of an upstream no longer +// configured. Only the first is known to be the cache's to remove. +func TestNewRemovesRepointedUpstreams(t *testing.T) { + dir := t.TempDir() + current := UpstreamDir("up", "https://new.example") + repointed := UpstreamDir("up", "https://old.example") + unconfigured := UpstreamDir("gone", "https://gone.example") + // Named after a configured upstream's name, but another upstream's. + longer := UpstreamDir("up-api", "https://api.example") + for _, name := range []string{current, repointed, unconfigured, longer} { + runGit(t, "", "init", "--quiet", "--bare", filepath.Join(dir, name, "repo.git")) + } + c, err := New(Options{ + CacheDir: dir, + StaleMaxAge: time.Hour, + Client: &http.Client{}, + Upstreams: map[string]string{"up": current}, + }) + if err != nil { + t.Fatal(err) + } + defer func() { _ = c.Close() }() + + for name, want := range map[string]bool{current: true, repointed: false, unconfigured: true, longer: true} { + if got := fileExists(filepath.Join(dir, name)); got != want { + t.Errorf("%s present = %v, want %v", name, got, want) + } + } + if keys := slices.Collect(maps.Keys(c.repos)); len(keys) != 1 || keys[0] != current+"/repo.git" { + t.Errorf("loaded %v, want only %s/repo.git", keys, current) + } +} + +// TestCacheDirSharedWithOtherData points the cache at a directory that also +// holds an artifact store and a database, as git.cache_dir: ./cache would. +func TestCacheDirSharedWithOtherData(t *testing.T) { + dir := t.TempDir() + _ = os.MkdirAll(filepath.Join(dir, "artifacts", "npm"), 0o750) + _ = os.WriteFile(filepath.Join(dir, "artifacts", "npm", "pkg.tgz"), []byte("x"), 0o600) + _ = os.WriteFile(filepath.Join(dir, "proxy.db"), []byte("db"), 0o600) + runGit(t, "", "init", "--quiet", "--bare", filepath.Join(dir, "artifacts", "foo", "bar.git")) + c, err := New(Options{ + CacheDir: dir, + Client: &http.Client{}, + Upstreams: map[string]string{"github": UpstreamDir("github", "https://github.com")}, + }) + if err != nil { + t.Fatal(err) + } + defer func() { _ = c.Close() }() + for _, p := range []string{"artifacts/npm/pkg.tgz", "proxy.db", "artifacts/foo/bar.git"} { + if _, err := os.Stat(filepath.Join(dir, p)); err != nil { + t.Errorf("%s was deleted at startup: %v", p, err) + } + } + if len(c.repos) != 0 { + t.Errorf("loaded %v from outside the upstream directories", slices.Collect(maps.Keys(c.repos))) + } +} + +// TestCloseWaitsForRefreshes closes the cache while a clone is held up +// upstream. When Close returns, the clone must have finished, temporary +// directory included, so nothing is left writing to the cache. +func TestCloseWaitsForRefreshes(t *testing.T) { + upstream := newFakeUpstream(t) + upstream.newRepo("repo") + upstream.stallFetches.Store(true) + tc := newTestCache(t, upstream.server.URL, nil) + clones := func() []string { + names, _ := filepath.Glob(filepath.Join(tc.tmpDir(), "clone-*")) + return names + } + + go func() { _, _ = tryGit(t, "", "ls-remote", tc.url("repo")) }() + waitFor(t, func() bool { return len(clones()) > 0 }) + if err := tc.Close(); err != nil { + t.Fatal(err) + } + if left := clones(); len(left) != 0 { + t.Errorf("Close returned with the clone still running: %v", left) + } + + before := upstream.requests.Load() + if _, err := tc.Open(context.Background(), "up/other.git", upstream.server.URL+"/other", OpenOptions{}); !errors.Is(err, errClosed) { + t.Errorf("Open after Close: %v, want errClosed", err) + } + if got := upstream.requests.Load() - before; got != 0 { + t.Errorf("Open after Close made %d upstream requests", got) + } +} + +// TestCacheDirInUse opens a second cache on a directory one already uses, +// as a second pod on a shared volume would, then again once the first has +// closed. +func TestCacheDirInUse(t *testing.T) { + dir := t.TempDir() + open := func() (*Cache, error) { return New(Options{CacheDir: dir, Client: &http.Client{}}) } + first, err := open() + if err != nil { + t.Fatal(err) + } + if second, err := open(); err == nil { + _ = second.Close() + t.Fatal("a second cache opened on the same directory") + } else if !strings.Contains(err.Error(), "in use by another process") { + t.Errorf("error = %v, want one naming the directory as in use", err) + } + if err := first.Close(); err != nil { + t.Fatal(err) + } + again, err := open() + if err != nil { + t.Fatalf("reopening after Close: %v", err) + } + _ = again.Close() +} + +// TestCloseTwice checks closing again does nothing, as a cleanup that also +// closes the cache may. +func TestCloseTwice(t *testing.T) { + c, err := New(Options{CacheDir: t.TempDir(), Client: &http.Client{}}) + if err != nil { + t.Fatal(err) + } + if err := errors.Join(c.Close(), c.Close()); err != nil { + t.Errorf("closing twice: %v", err) + } +} diff --git a/internal/gitcache/git.go b/internal/gitcache/git.go new file mode 100644 index 0000000..0cc2c11 --- /dev/null +++ b/internal/gitcache/git.go @@ -0,0 +1,104 @@ +package gitcache + +import ( + "bytes" + "context" + "fmt" + "os" + "os/exec" + "slices" + "strconv" + "strings" + "time" +) + +const ( + maxStderr = 4 << 10 + gitWaitDelay = 5 * time.Second +) + +// gitConfig applies to every git process the cache runs. +var gitConfig = [][2]string{ + // git would follow a redirect from the relay straight to the network. + {"http.followRedirects", "false"}, + {"protocol.version", "2"}, + // Auto gc stays on, as on any Git server, but runs inside the fetch + // rather than detaching, so no git process outlives the fetch that + // started it, and sizes are measured after it. + {"gc.autoDetach", "false"}, + {"maintenance.autoDetach", "false"}, + {"init.defaultBranch", "main"}, + // Keep every fetched pack as a pack. index-pack writes it to a + // temporary file and moves it into place only once complete, whereas a + // small pack is otherwise unpacked object by object, and a transfer cut + // off at max_repo_size would leave the objects before the cut, such as a + // commit without its tree's blobs. + {"transfer.unpackLimit", "1"}, + // Serve any commit the cache holds when a client asks for it by object + // ID, as the upstream would, including ones a force-push left behind, + // and do not advertise the refs that keep fetched ones. + {"uploadpack.allowAnySHA1InWant", "true"}, + {"uploadpack.hideRefs", wantedPrefix}, + // With http.lowSpeedTime, added per cache, gives up on a transfer that + // receives nothing for that long, so a connection that stalls mid-pack + // cannot hold a refresh for fetchTimeout. + {"http.lowSpeedLimit", "1"}, + // index-pack streams blobs larger than this instead of holding them in + // memory, and pack-objects stores them without delta compression, so a + // large file costs a fetch or a clone little memory. + {"core.bigFileThreshold", "8m"}, +} + +// command builds a git process with an environment of its own: none of the +// proxy's environment is passed on, so no system or user config, credential +// helper, HTTP proxy setting or GIT_ variable applies, and the only transport +// git may use is plain HTTP, which is what the relay speaks. +func (c *Cache) command(ctx context.Context, extraEnv []string, args ...string) *exec.Cmd { + cmd := exec.CommandContext(ctx, c.gitPath, args...) + env := []string{ + "PATH=" + os.Getenv("PATH"), + "HOME=" + c.tmpDir(), + "LC_ALL=C", + "GIT_CONFIG_NOSYSTEM=1", + "GIT_CONFIG_GLOBAL=" + os.DevNull, + "GIT_TERMINAL_PROMPT=0", + "GIT_ALLOW_PROTOCOL=http", + } + config := slices.Concat(gitConfig, [][2]string{{"http.lowSpeedTime", strconv.Itoa(c.lowSpeedTime)}}) + env = append(env, "GIT_CONFIG_COUNT="+strconv.Itoa(len(config))) + for i, kv := range config { + env = append(env, + "GIT_CONFIG_KEY_"+strconv.Itoa(i)+"="+kv[0], + "GIT_CONFIG_VALUE_"+strconv.Itoa(i)+"="+kv[1]) + } + env = append(env, extraEnv...) + cmd.Env = env + cmd.WaitDelay = gitWaitDelay + return cmd +} + +// run runs git in the repository at dir and returns its standard output. +func (c *Cache) run(ctx context.Context, dir string, args ...string) (string, error) { + cmd := c.command(ctx, nil, append([]string{"-C", dir}, args...)...) + var stdout bytes.Buffer + stderr := &limitedBuffer{max: maxStderr} + cmd.Stdout = &stdout + cmd.Stderr = stderr + if err := cmd.Run(); err != nil { + return "", fmt.Errorf("git %s: %w: %s", args[0], err, strings.TrimSpace(stderr.String())) + } + return stdout.String(), nil +} + +// limitedBuffer keeps the first max bytes written to it. +type limitedBuffer struct { + bytes.Buffer + max int +} + +func (b *limitedBuffer) Write(p []byte) (int, error) { + if room := b.max - b.Len(); room > 0 { + b.Buffer.Write(p[:min(len(p), room)]) + } + return len(p), nil +} diff --git a/internal/gitcache/helpers_test.go b/internal/gitcache/helpers_test.go new file mode 100644 index 0000000..e2c4a27 --- /dev/null +++ b/internal/gitcache/helpers_test.go @@ -0,0 +1,354 @@ +package gitcache + +import ( + "bytes" + "compress/gzip" + "errors" + "io" + "log" + "net/http" + "net/http/cgi" + "net/http/httptest" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + "sync/atomic" + "testing" + "time" +) + +// gitEnv isolates the tests' own git commands from the machine's config. +func gitEnv(home string) []string { + return []string{ + "PATH=" + os.Getenv("PATH"), + "HOME=" + home, + "GIT_CONFIG_NOSYSTEM=1", + "GIT_TERMINAL_PROMPT=0", + "GIT_AUTHOR_NAME=test", "GIT_AUTHOR_EMAIL=test@example.com", + "GIT_COMMITTER_NAME=test", "GIT_COMMITTER_EMAIL=test@example.com", + } +} + +func runGit(t *testing.T, dir string, args ...string) string { + t.Helper() + out, err := tryGit(t, dir, args...) + if err != nil { + t.Fatalf("git %s: %v\n%s", strings.Join(args, " "), err, out) + } + return strings.TrimSpace(out) +} + +func tryGit(t *testing.T, dir string, args ...string) (string, error) { + t.Helper() + cmd := exec.Command("git", args...) + cmd.Dir = dir + cmd.Env = gitEnv(t.TempDir()) + out, err := cmd.CombinedOutput() + return string(out), err +} + +// httpBackendPath finds git http-backend, which serves the fake upstreams, +// and skips the test where git is installed without it. +func httpBackendPath(t *testing.T) string { + t.Helper() + dir := runGit(t, "", "--exec-path") + for _, name := range []string{"git-http-backend", "git-http-backend.exe"} { + if path := filepath.Join(dir, name); fileExists(path) { + return path + } + } + t.Skip("git http-backend is not installed") + return "" +} + +func fileExists(path string) bool { + _, err := os.Stat(path) + return err == nil +} + +// fakeUpstream serves bare repositories under root with git http-backend. +type fakeUpstream struct { + t *testing.T + root string + server *httptest.Server + + // status, when set, is returned instead of serving the request. + status atomic.Int32 + // requireAuth, when set, is the Authorization every request must carry. + requireAuth string + // v0Only drops Git-Protocol, so only protocol v0 is spoken. + v0Only bool + // hold, when set, blocks requests until it is closed. + holdMu sync.Mutex + hold chan struct{} + + requests atomic.Int32 + // gitRequests counts requests from git processes, which reach the + // upstream through the relay, and gitSessions the fetches they make. + gitRequests atomic.Int32 + gitSessions atomic.Int32 + unauthed atomic.Int32 + + // stallFetches makes git's own upload-pack requests hang, and + // delayFetches holds them for that long. + stallFetches atomic.Bool + delayFetches atomic.Int64 + + bodiesMu sync.Mutex + bodies []string +} + +// recordBody keeps the decoded body of a POST. It also gives the request a +// Content-Length, since net/http/cgi refuses chunked bodies, which git sends +// once a request outgrows http.postBuffer. +func (u *fakeUpstream) recordBody(r *http.Request) { + raw, err := io.ReadAll(r.Body) + if err != nil { + u.t.Errorf("reading request body: %v", err) + } + r.Body = io.NopCloser(bytes.NewReader(raw)) + r.ContentLength = int64(len(raw)) + r.TransferEncoding = nil + decoded := raw + if r.Header.Get("Content-Encoding") == "gzip" { + if zr, err := gzip.NewReader(bytes.NewReader(raw)); err == nil { + decoded, _ = io.ReadAll(zr) + } + } + u.bodiesMu.Lock() + u.bodies = append(u.bodies, string(decoded)) + u.bodiesMu.Unlock() +} + +// lsRefsBodies returns the ls-refs requests the upstream received. +func (u *fakeUpstream) lsRefsBodies() []string { + u.bodiesMu.Lock() + defer u.bodiesMu.Unlock() + var out []string + for _, body := range u.bodies { + if strings.Contains(body, "command=ls-refs") { + out = append(out, body) + } + } + return out +} + +func newFakeUpstream(t *testing.T) *fakeUpstream { + t.Helper() + u := &fakeUpstream{t: t, root: t.TempDir()} + backend := &cgi.Handler{ + Path: httpBackendPath(t), + Stderr: io.Discard, + Logger: log.New(io.Discard, "", 0), + Env: append(gitEnv(t.TempDir()), + "GIT_PROJECT_ROOT="+u.root, + "GIT_HTTP_EXPORT_ALL=1"), + } + u.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + u.requests.Add(1) + if r.UserAgent() != gitUserAgent { + u.gitRequests.Add(1) + if r.Method == http.MethodGet { + u.gitSessions.Add(1) + } + } + u.holdMu.Lock() + hold := u.hold + u.holdMu.Unlock() + if hold != nil { + <-hold + } + if status := u.status.Load(); status != 0 { + http.Error(w, "unavailable", int(status)) + return + } + if u.requireAuth != "" && r.Header.Get("Authorization") != u.requireAuth { + u.unauthed.Add(1) + http.Error(w, "unauthorized", http.StatusUnauthorized) + return + } + if u.v0Only { + r.Header.Del("Git-Protocol") + } + if delay := u.delayFetches.Load(); delay > 0 && r.UserAgent() != gitUserAgent && r.Method == http.MethodPost { + time.Sleep(time.Duration(delay)) + } + if u.stallFetches.Load() && r.UserAgent() != gitUserAgent && r.Method == http.MethodPost { + // The server notices the client leaving only once the body is read. + _, _ = io.Copy(io.Discard, r.Body) + <-r.Context().Done() + return + } + if r.Method == http.MethodPost { + u.recordBody(r) + } + backend.ServeHTTP(w, r) + })) + t.Cleanup(u.server.Close) + return u +} + +func (u *fakeUpstream) holdRequests() func() { + u.holdMu.Lock() + defer u.holdMu.Unlock() + hold := make(chan struct{}) + u.hold = hold + var once sync.Once + return func() { + once.Do(func() { + u.holdMu.Lock() + u.hold = nil + u.holdMu.Unlock() + close(hold) + }) + } +} + +// newRepo creates {name}.git upstream with one commit on main and a tag, and +// returns a working clone to commit and push from. +func (u *fakeUpstream) newRepo(name string) string { + u.t.Helper() + bare := filepath.Join(u.root, name+".git") + runGit(u.t, "", "init", "--quiet", "--bare", "--initial-branch=main", bare) + work := u.t.TempDir() + runGit(u.t, "", "clone", "--quiet", bare, work) + runGit(u.t, work, "checkout", "--quiet", "-b", "main") + u.commit(work, "first") + runGit(u.t, work, "tag", "-a", "-m", "v1", "v1") + runGit(u.t, work, "push", "--quiet", "origin", "main", "v1") + return work +} + +func (u *fakeUpstream) commit(work, msg string) string { + u.t.Helper() + if err := os.WriteFile(filepath.Join(work, "file.txt"), []byte(msg+"\n"), 0o600); err != nil { + u.t.Fatal(err) + } + runGit(u.t, work, "add", "file.txt") + runGit(u.t, work, "commit", "--quiet", "-m", msg) + return runGit(u.t, work, "rev-parse", "HEAD") +} + +// waitIdle waits until no request or refresh is using a repository, so a +// test can inspect or change the cache without racing a request finishing. +func waitIdle(t *testing.T, c *Cache) { + t.Helper() + waitFor(t, func() bool { + c.mu.Lock() + defer c.mu.Unlock() + for _, rp := range c.repos { + if rp.users > 0 { + return false + } + } + return true + }) +} + +// bare is the upstream repository named "repo". +// repo returns the cache's entry for key, reading the map under the lock. +func (tc *testCache) repo(key string) *repo { + tc.mu.Lock() + defer tc.mu.Unlock() + return tc.repos[key] +} + +// usersOf is the number of requests and refreshes using the repository. +func (tc *testCache) usersOf(key string) int { + tc.mu.Lock() + defer tc.mu.Unlock() + return tc.repos[key].users +} + +func (u *fakeUpstream) bare() string { + return filepath.Join(u.root, "repo.git") +} + +type testCache struct { + *Cache + server *httptest.Server +} + +func newTestCache(t *testing.T, upstreamURL string, configure func(*Options)) *testCache { + t.Helper() + opts := Options{ + Upstreams: map[string]string{"up": "up"}, + CacheDir: t.TempDir(), + RefreshWait: 5 * time.Second, + StaleMaxAge: time.Hour, + Client: &http.Client{}, + } + if configure != nil { + configure(&opts) + } + c, err := New(opts) + if err != nil { + t.Fatalf("New: %v", err) + } + server := httptest.NewServer(testFront(c, upstreamURL, true)) + t.Cleanup(func() { + server.Close() + _ = c.Close() + }) + return &testCache{Cache: c, server: server} +} + +// testFront serves /git/up/{repo}/info/refs and git-upload-pack from c, as +// internal/handler does once it has validated the request, with repo +// fetched from upstreamURL/{repo}. fetchWants is OpenOptions.FetchWants. +func testFront(c *Cache, upstreamURL string, fetchWants bool) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + path := strings.TrimPrefix(r.URL.Path, "/git/up/") + repo, infoRefs := strings.CutSuffix(path, "/info/refs") + if !infoRefs { + var ok bool + if repo, ok = strings.CutSuffix(path, "/git-upload-pack"); !ok || r.Method != http.MethodPost { + http.Error(w, "not served", http.StatusForbidden) + return + } + } else if r.URL.RawQuery != "service=git-upload-pack" { + http.Error(w, "not served", http.StatusForbidden) + return + } + lease, err := c.Open(r.Context(), "up/"+strings.TrimSuffix(repo, ".git")+".git", upstreamURL+"/"+repo, + OpenOptions{Refresh: infoRefs, FetchWants: fetchWants}) + switch { + case errors.Is(err, ErrNotFound): + http.Error(w, err.Error(), http.StatusNotFound) + return + case errors.Is(err, ErrTooLarge), errors.Is(err, ErrCacheFull): + http.Error(w, err.Error(), http.StatusInsufficientStorage) + return + case err != nil: + http.Error(w, err.Error(), http.StatusBadGateway) + return + } + defer lease.Release() + if infoRefs { + lease.ServeInfoRefs(w, r) + } else { + lease.ServeUploadPack(w, r) + } + }) +} + +func (tc *testCache) url(repo string) string { + return tc.server.URL + "/git/up/" + repo +} + +// clone clones repo through the cache with the given protocol version and +// returns the working copy. +func (tc *testCache) clone(t *testing.T, repo string, args ...string) string { + t.Helper() + dir := filepath.Join(t.TempDir(), "clone") + runGit(t, "", append(append([]string{"clone", "--quiet"}, args...), tc.url(repo), dir)...) + return dir +} + +// cachedRefs lists the refs of the cached copy of "repo". +func (tc *testCache) cachedRefs(t *testing.T) string { + t.Helper() + return runGit(t, filepath.Join(tc.dir, "up", "repo.git"), "for-each-ref", "--format=%(objectname) %(refname)") +} diff --git a/internal/gitcache/lock_unix.go b/internal/gitcache/lock_unix.go new file mode 100644 index 0000000..215027a --- /dev/null +++ b/internal/gitcache/lock_unix.go @@ -0,0 +1,24 @@ +//go:build !windows + +package gitcache + +import ( + "errors" + "os" + "syscall" +) + +func lockFile(path string) (*os.File, error) { + f, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE, lockFilePermissions) + if err != nil { + return nil, err + } + if err := syscall.Flock(int(f.Fd()), syscall.LOCK_EX|syscall.LOCK_NB); err != nil { + _ = f.Close() + if errors.Is(err, syscall.EWOULDBLOCK) { + return nil, errLocked + } + return nil, err + } + return f, nil +} diff --git a/internal/gitcache/lock_windows.go b/internal/gitcache/lock_windows.go new file mode 100644 index 0000000..99f46d5 --- /dev/null +++ b/internal/gitcache/lock_windows.go @@ -0,0 +1,27 @@ +package gitcache + +import ( + "errors" + "os" + "syscall" +) + +const errorSharingViolation syscall.Errno = 32 + +// lockFile opens path with no sharing, so no other handle can open it while +// this one is open. +func lockFile(path string) (*os.File, error) { + name, err := syscall.UTF16PtrFromString(path) + if err != nil { + return nil, err + } + h, err := syscall.CreateFile(name, syscall.GENERIC_READ|syscall.GENERIC_WRITE, 0, nil, + syscall.OPEN_ALWAYS, syscall.FILE_ATTRIBUTE_NORMAL, 0) + if errors.Is(err, errorSharingViolation) { + return nil, errLocked + } + if err != nil { + return nil, err + } + return os.NewFile(uintptr(h), path), nil +} diff --git a/internal/gitcache/pktline.go b/internal/gitcache/pktline.go new file mode 100644 index 0000000..5f6095b --- /dev/null +++ b/internal/gitcache/pktline.go @@ -0,0 +1,65 @@ +package gitcache + +import ( + "bufio" + "fmt" + "io" + "strconv" + "strings" +) + +// pkt-line framing (gitprotocol-common): four hex digits giving the packet +// length including themselves, then the payload. The lengths 0000, 0001 and +// 0002 are the flush, delimiter and response-end packets. +const ( + pktFlush = "0000" + pktDelim = "0001" + pktHeaderLen = 4 + pktMaxLen = 65520 + pktSpecialLast = 2 +) + +type pktKind int + +const ( + pktData pktKind = iota + pktFlushKind + pktDelimKind + pktEndKind +) + +func pktLine(payload string) string { + return fmt.Sprintf("%04x%s", len(payload)+pktHeaderLen, payload) +} + +type pktReader struct { + r *bufio.Reader +} + +func newPktReader(r io.Reader) *pktReader { + return &pktReader{r: bufio.NewReader(r)} +} + +// next returns the next packet. A data packet's payload has its trailing +// newline removed. +func (p *pktReader) next() (pktKind, string, error) { + var header [pktHeaderLen]byte + if _, err := io.ReadFull(p.r, header[:]); err != nil { + return 0, "", err + } + n, err := strconv.ParseUint(string(header[:]), 16, 16) + if err != nil { + return 0, "", fmt.Errorf("invalid pkt-line length %q", header) + } + if n <= pktSpecialLast { + return pktKind(n + 1), "", nil + } + if n < pktHeaderLen || n > pktMaxLen { + return 0, "", fmt.Errorf("invalid pkt-line length %d", n) + } + payload := make([]byte, n-pktHeaderLen) + if _, err := io.ReadFull(p.r, payload); err != nil { + return 0, "", err + } + return pktData, strings.TrimSuffix(string(payload), "\n"), nil +} diff --git a/internal/gitcache/refresh.go b/internal/gitcache/refresh.go new file mode 100644 index 0000000..694ea12 --- /dev/null +++ b/internal/gitcache/refresh.go @@ -0,0 +1,376 @@ +package gitcache + +import ( + "context" + "errors" + "fmt" + "maps" + "os" + "path/filepath" + "strings" + "sync" + "time" + + "github.com/git-pkgs/proxy/internal/metrics" +) + +const ( + // fetchTimeout bounds one refresh, including a cold clone of a large + // repository. + fetchTimeout = time.Hour + // defaultRefCheckTimeout bounds the ref check, so an upstream that + // stalls mid-response does not hold a repository's refresh for + // fetchTimeout. + defaultRefCheckTimeout = 2 * time.Minute + // tooLargeRetryAfter is how long a repository that passed + // git.max_repo_size is left alone before it is tried again. + tooLargeRetryAfter = time.Hour +) + +// errUpstreamUnanswered marks a fetch the upstream never answered, such as +// one during an outage. +var errUpstreamUnanswered = errors.New("upstream did not answer the fetch") + +// errClosed is returned for a refresh asked for once the cache is closing. +var errClosed = errors.New("git cache is closed") + +// refreshCall is a refresh in progress, shared by every request for the +// repository that arrives while it runs. +type refreshCall struct { + done chan struct{} + err error + // checked is closed once the upstream's branches and tags have been + // compared with the cached ones, and fetching then says whether they + // differ, so a fetch follows. + checked chan struct{} + fetching bool + checkedOnce sync.Once +} + +func newRefreshCall() *refreshCall { + return &refreshCall{done: make(chan struct{}), checked: make(chan struct{})} +} + +func (call *refreshCall) markChecked(fetching bool) { + call.checkedOnce.Do(func() { + call.fetching = fetching + close(call.checked) + }) +} + +// ensureFresh brings the repository up to date with its upstream before it is +// served. A repository checked successfully within StaleMaxAge is served +// from the cache if the upstream fails or its check takes longer than +// RefreshWait; the refresh carries on in the background either way. Once the +// check finds new commits, their fetch is waited for, so a clone right after +// a push gets them. One never cloned, or not checked within StaleMaxAge, is +// served only once the refresh succeeds. +func (c *Cache) ensureFresh(ctx context.Context, rp *repo, repoURL string) error { + call := c.startRefresh(rp, repoURL) + checked := time.Unix(0, rp.lastChecked.Load()) + if !rp.exists() || time.Since(checked) > c.opts.StaleMaxAge { + select { + case <-call.done: + return call.err + case <-ctx.Done(): + return ctx.Err() + } + } + + timer := time.NewTimer(c.opts.RefreshWait) + defer timer.Stop() + refChecked, timeout := call.checked, timer.C + for { + select { + case <-refChecked: + refChecked = nil + if call.fetching { + timeout = nil + } + case <-call.done: + if call.err != nil && servesStale(call.err) { + c.logger.Warn("git cache: refresh failed, serving cached copy", + "repository", rp.key, "checked", checked, "error", call.err) + return nil + } + return call.err + case <-timeout: + c.logger.Info("git cache: upstream check still running, serving cached copy", + "repository", rp.key, "checked", checked) + return nil + case <-ctx.Done(): + return ctx.Err() + } + } +} + +// servesStale reports whether the cached copy may be served after err: not +// when the upstream says the repository is gone or refuses access to it. +func servesStale(err error) bool { + return !errors.Is(err, ErrNotFound) && !errors.Is(err, ErrRefused) +} + +// startRefresh starts a refresh of the repository, or joins the one running. +// The refresh holds a use of the repository until it ends. +func (c *Cache) startRefresh(rp *repo, repoURL string) *refreshCall { + c.mu.Lock() + defer c.mu.Unlock() + if rp.refreshing != nil { + return rp.refreshing + } + call := newRefreshCall() + if c.closed { + call.err = errClosed + close(call.done) + return call + } + rp.refreshing = call + rp.users++ + c.refreshes.Add(1) + go func() { + defer c.refreshes.Done() + ctx, cancel := context.WithTimeout(c.ctx, fetchTimeout) + start := time.Now() + call.err = c.refresh(ctx, rp, repoURL, call.markChecked) + call.markChecked(false) + cancel() + metrics.RecordUpstreamFetch(ecosystem, time.Since(start)) + // A refresh cut short by Close is not an upstream failure. + if call.err != nil && c.ctx.Err() == nil { + metrics.RecordUpstreamError(ecosystem, errorType(call.err), metrics.FailureDetail{URL: repoURL, Err: call.err}) + c.logger.Warn("git cache: refresh failed", "repository", rp.key, "error", call.err) + } + c.mu.Lock() + rp.refreshing = nil + c.mu.Unlock() + c.release(rp) + close(call.done) + if call.err == nil { + c.evict() + } + }() + return call +} + +// refresh checks the upstream's branches and tags and fetches if they differ +// from the cached ones, cloning the repository if it is not cached yet. +// checked is called once a cached repository's refs have been compared. +func (c *Cache) refresh(ctx context.Context, rp *repo, repoURL string, checked func(fetching bool)) error { + if c.isTooLarge(rp.key) { + return ErrTooLarge + } + checkCtx, cancel := context.WithTimeout(ctx, c.refCheckTimeout) + remote, err := c.listUpstreamRefs(checkCtx, repoURL) + cancel() + if err != nil { + return err + } + if rp.exists() { + err = c.update(ctx, rp, repoURL, remote, checked) + } else { + err = c.clone(ctx, rp, repoURL, remote) + } + if errors.Is(err, ErrTooLarge) { + c.markTooLarge(rp.key) + } + if err != nil { + return err + } + now := time.Now() + rp.lastChecked.Store(now.UnixNano()) + touch(rp.dir, checkedStamp, now) + return nil +} + +// clone builds the repository in a temporary directory and moves it into +// place, so no request ever reads a partial clone. +func (c *Cache) clone(ctx context.Context, rp *repo, repoURL string, remote remoteRefs) error { + // Released after the temporary clone is removed. + res := c.newReservation() + defer res.release() + tmp, err := os.MkdirTemp(c.tmpDir(), "clone-") + if err != nil { + return err + } + defer func() { _ = os.RemoveAll(tmp) }() + + if _, err := c.run(ctx, tmp, "init", "--quiet", "--bare"); err != nil { + return err + } + if err := c.fetch(ctx, tmp, repoURL, c.opts.MaxRepoSize, res, refreshFlags, branchesAndTags...); err != nil { + return err + } + if err := c.setHead(ctx, tmp, remote, ""); err != nil { + return err + } + size := dirSize(tmp) + if c.opts.MaxRepoSize > 0 && size > c.opts.MaxRepoSize { + return ErrTooLarge + } + if err := os.MkdirAll(filepath.Dir(rp.dir), dirPermissions); err != nil { + return err + } + if err := os.Rename(tmp, rp.dir); err != nil { + return fmt.Errorf("moving clone into place: %w", err) + } + res.settle(rp, size) + c.logger.Info("git cache: cloned repository", "repository", rp.key, "size", size) + return nil +} + +// update fetches when the upstream's branches or tags differ from the cached +// ones, and follows a change of the upstream's default branch. +func (c *Cache) update(ctx context.Context, rp *repo, repoURL string, remote remoteRefs, checked func(fetching bool)) error { + unlock, err := rp.lockWrites(ctx) + if err != nil { + return err + } + defer unlock() + local, wanted, err := c.localRefs(ctx, rp.dir) + if err != nil { + return err + } + fetched := !maps.Equal(local, remote.refs) + checked(fetched) + if fetched { + limit, err := c.fetchLimit(rp) + if err != nil { + return err + } + res := c.newReservation() + err = c.fetch(ctx, rp.dir, repoURL, limit, res, refreshFlags, branchesAndTags...) + res.settle(rp, dirSize(rp.dir)) + if err != nil { + return err + } + } + if err := c.pruneWanted(ctx, rp.dir, wanted, fetched); err != nil { + c.logger.Warn("git cache: pruning wanted refs failed", "repository", rp.key, "error", err) + } + head, _ := c.run(ctx, rp.dir, "symbolic-ref", "--quiet", "HEAD") + return c.setHead(ctx, rp.dir, remote, strings.TrimSpace(head)) +} + +// branchesAndTags are the refspecs of a refresh. A bare clone has none of its +// own, and other refs, such as Gerrit's refs/changes, are left out. +var branchesAndTags = []string{"+refs/heads/*:refs/heads/*", "+refs/tags/*:refs/tags/*"} + +// refreshFlags delete the branches and tags the upstream no longer has, and +// update all refs together, so a client served while a refresh runs never +// sees only some of them moved. +var refreshFlags = []string{"--prune", "--atomic"} + +// fetchLimit returns how many more bytes the repository may take, 0 for no +// limit. +func (c *Cache) fetchLimit(rp *repo) (int64, error) { + if c.opts.MaxRepoSize <= 0 { + return 0, nil + } + limit := c.opts.MaxRepoSize - rp.size.Load() + if limit <= 0 { + return 0, ErrTooLarge + } + return limit, nil +} + +// fetch fetches refspecs from the upstream into the repository at dir, +// through the relay, adding flags to git fetch. limit caps the bytes +// fetched, 0 for no cap, and res holds space within MaxSize for them. +func (c *Cache) fetch(ctx context.Context, dir, repoURL string, limit int64, res *reservation, flags []string, refspecs ...string) error { + select { + case c.fetches <- struct{}{}: + defer func() { <-c.fetches }() + case <-ctx.Done(): + return ctx.Err() + } + + relayURL, route, unregister := c.relay.register(repoURL, limit, res) + defer unregister() + args := append([]string{"fetch", "--quiet", "--no-write-fetch-head"}, flags...) + _, err := c.run(ctx, dir, append(append(args, relayURL), refspecs...)...) + if err != nil { + // index-pack leaves its temporary pack behind when its input ends + // early. Fetches into a repository run one at a time, so any left + // now are this fetch's. + leftovers, _ := filepath.Glob(filepath.Join(dir, "objects", "pack", "tmp_*")) + for _, path := range leftovers { + _ = os.Remove(path) + } + } + if route.exceeded.Load() { + return ErrTooLarge + } + if route.full.Load() { + return ErrCacheFull + } + if err != nil && !route.answered.Load() { + return fmt.Errorf("%w: %w", errUpstreamUnanswered, err) + } + return err +} + +// setHead points HEAD at the upstream's default branch, when the upstream +// names one and it is among the cached branches. +func (c *Cache) setHead(ctx context.Context, dir string, remote remoteRefs, current string) error { + if remote.head == "" || remote.head == current { + return nil + } + if _, ok := remote.refs[remote.head]; !ok { + return nil + } + _, err := c.run(ctx, dir, "symbolic-ref", "HEAD", remote.head) + return err +} + +// localRefs lists the cached branches and tags, and the names of the wanted +// refs. +func (c *Cache) localRefs(ctx context.Context, dir string) (map[string]string, []string, error) { + out, err := c.run(ctx, dir, "for-each-ref", "--format=%(objectname) %(refname)", headsPrefix, tagsPrefix, wantedPrefix) + if err != nil { + return nil, nil, err + } + refs := make(map[string]string) + var wanted []string + for line := range strings.Lines(out) { + oid, name, ok := strings.Cut(strings.TrimSuffix(line, "\n"), " ") + switch { + case !ok: + case strings.HasPrefix(name, wantedPrefix): + wanted = append(wanted, name) + default: + refs[name] = oid + } + } + return refs, wanted, nil +} + +func errorType(err error) string { + switch { + case errors.Is(err, ErrNotFound): + return "not_found" + case errors.Is(err, ErrRefused): + return "refused" + case errors.Is(err, ErrTooLarge): + return "too_large" + case errors.Is(err, ErrCacheFull): + return "cache_full" + } + return "fetch_failed" +} + +func (c *Cache) isTooLarge(key string) bool { + c.mu.Lock() + defer c.mu.Unlock() + since, ok := c.tooLarge[key] + if ok && time.Since(since) >= tooLargeRetryAfter { + delete(c.tooLarge, key) + return false + } + return ok +} + +func (c *Cache) markTooLarge(key string) { + c.mu.Lock() + defer c.mu.Unlock() + c.tooLarge[key] = time.Now() +} diff --git a/internal/gitcache/relay.go b/internal/gitcache/relay.go new file mode 100644 index 0000000..38baa29 --- /dev/null +++ b/internal/gitcache/relay.go @@ -0,0 +1,189 @@ +package gitcache + +import ( + "context" + "crypto/rand" + "encoding/hex" + "errors" + "io" + "log/slog" + "net" + "net/http" + "strings" + "sync" + "sync/atomic" + "time" +) + +const ( + relayIDBytes = 16 + relayReadHeaderTimeout = 30 * time.Second +) + +// relay is a loopback HTTP server that every git fetch the cache runs goes +// through. git never reaches the network itself: each fetch gets a route to +// one upstream repository, and the relay forwards its requests with the +// proxy's upstream client, so address checks, upstream.auth, the access log +// and the no-redirect rule apply as they do to the proxy's own requests, and +// git never sees a credential. +type relay struct { + client *http.Client + logger *slog.Logger + listener net.Listener + server *http.Server + base string + + mu sync.Mutex + routes map[string]*relayRoute +} + +type relayRoute struct { + repoURL string + // limit caps the response bytes the route relays, 0 for no cap, and + // budget, when set, holds space within MaxSize for them. exceeded and + // full record which of them stopped the route. + limit int64 + budget *reservation + relayed atomic.Int64 + exceeded atomic.Bool + full atomic.Bool + // answered records that the upstream answered a git-upload-pack + // request, so a failed fetch was not an outage. + answered atomic.Bool +} + +func newRelay(client *http.Client, logger *slog.Logger) (*relay, error) { + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + if listener, err = net.Listen("tcp", "[::1]:0"); err != nil { + return nil, err + } + } + rl := &relay{ + client: client, + logger: logger, + listener: listener, + base: "http://" + listener.Addr().String(), + routes: make(map[string]*relayRoute), + } + rl.server = &http.Server{Handler: rl, ReadHeaderTimeout: relayReadHeaderTimeout} + go func() { + if err := rl.server.Serve(listener); err != nil && !errors.Is(err, http.ErrServerClosed) { + logger.Error("git relay stopped", "error", err) + } + }() + return rl, nil +} + +func (rl *relay) close() error { + return rl.server.Close() +} + +// register adds a route to repoURL and returns the URL git fetches from and +// a function that removes the route. +func (rl *relay) register(repoURL string, limit int64, budget *reservation) (string, *relayRoute, func()) { + var b [relayIDBytes]byte + _, _ = rand.Read(b[:]) + id := hex.EncodeToString(b[:]) + route := &relayRoute{repoURL: repoURL, limit: limit, budget: budget} + + rl.mu.Lock() + rl.routes[id] = route + rl.mu.Unlock() + + return rl.base + "/" + id, route, func() { + rl.mu.Lock() + delete(rl.routes, id) + rl.mu.Unlock() + } +} + +func (rl *relay) ServeHTTP(w http.ResponseWriter, r *http.Request) { + id, suffix, _ := strings.Cut(strings.TrimPrefix(r.URL.Path, "/"), "/") + rl.mu.Lock() + route := rl.routes[id] + rl.mu.Unlock() + if route == nil { + http.NotFound(w, r) + return + } + + var target string + switch { + case suffix == "info/refs" && r.Method == http.MethodGet && r.URL.RawQuery == "service=git-upload-pack": + target = route.repoURL + "/info/refs?service=git-upload-pack" + case suffix == "git-upload-pack" && r.Method == http.MethodPost: + target = route.repoURL + "/git-upload-pack" + default: + http.NotFound(w, r) + return + } + + out, err := http.NewRequestWithContext(r.Context(), r.Method, target, r.Body) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + out.ContentLength = r.ContentLength + for _, name := range []string{"Git-Protocol", "Content-Type", "Content-Encoding", "Accept", "User-Agent"} { + if value := r.Header.Get(name); value != "" { + out.Header.Set(name, value) + } + } + + resp, err := rl.client.Do(out) + if err != nil { + rl.logger.Warn("git relay: upstream request failed", "upstream", route.repoURL, "error", err) + http.Error(w, "upstream request failed", http.StatusBadGateway) + return + } + defer func() { _ = resp.Body.Close() }() + // A redirect passed on would send git to the Location itself. + if resp.StatusCode >= http.StatusMultipleChoices && resp.StatusCode < http.StatusBadRequest { + http.Error(w, "upstream redirect not followed", http.StatusBadGateway) + return + } + + if r.Method == http.MethodPost && resp.StatusCode == http.StatusOK { + route.answered.Store(true) + } + if ct := resp.Header.Get("Content-Type"); ct != "" { + w.Header().Set("Content-Type", ct) + } + w.WriteHeader(resp.StatusCode) + if _, err := io.Copy(w, &routeReader{ctx: r.Context(), r: resp.Body, route: route}); err != nil { + if route.exceeded.Load() || route.full.Load() { + // Cut the connection so git fails rather than keeping a pack + // that merely looks short. + panic(http.ErrAbortHandler) + } + if !errors.Is(err, context.Canceled) { + rl.logger.Warn("git relay: copying upstream response", "upstream", route.repoURL, "error", err) + } + } +} + +// routeReader counts relayed bytes against the route's limit and budget. +type routeReader struct { + ctx context.Context + r io.Reader + route *relayRoute +} + +func (rr *routeReader) Read(p []byte) (int, error) { + n, err := rr.r.Read(p) + total := rr.route.relayed.Add(int64(n)) + if rr.route.limit > 0 && total > rr.route.limit { + rr.route.exceeded.Store(true) + return 0, ErrTooLarge + } + if rr.route.budget != nil { + if err := rr.route.budget.cover(rr.ctx, total); err != nil { + if errors.Is(err, ErrCacheFull) { + rr.route.full.Store(true) + } + return 0, err + } + } + return n, err +} diff --git a/internal/gitcache/relay_test.go b/internal/gitcache/relay_test.go new file mode 100644 index 0000000..4ee9142 --- /dev/null +++ b/internal/gitcache/relay_test.go @@ -0,0 +1,96 @@ +package gitcache + +import ( + "io" + "log/slog" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" +) + +func newTestRelay(t *testing.T) *relay { + t.Helper() + rl, err := newRelay(&http.Client{}, slog.New(slog.DiscardHandler)) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = rl.close() }) + return rl +} + +func TestRelayOnlyForwardsFetchRequests(t *testing.T) { + var forwarded atomic.Int32 + upstream := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { + forwarded.Add(1) + })) + defer upstream.Close() + rl := newTestRelay(t) + base, _, unregister := rl.register(upstream.URL+"/repo", 0, nil) + + refused := []struct{ method, url string }{ + {http.MethodGet, rl.base + "/unknown/info/refs?service=git-upload-pack"}, + {http.MethodGet, base + "/info/refs?service=git-receive-pack"}, + {http.MethodPost, base + "/git-receive-pack"}, + {http.MethodGet, base + "/HEAD"}, + {http.MethodGet, base + "/git-upload-pack"}, + } + for _, r := range refused { + req, _ := http.NewRequest(r.method, r.url, nil) + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + _ = resp.Body.Close() + if resp.StatusCode != http.StatusNotFound { + t.Errorf("%s %s: status %d, want 404", r.method, r.url, resp.StatusCode) + } + } + if got := forwarded.Load(); got != 0 { + t.Errorf("forwarded %d refused requests", got) + } + + unregister() + resp, err := http.Get(base + "/info/refs?service=git-upload-pack") + if err != nil { + t.Fatal(err) + } + _ = resp.Body.Close() + if resp.StatusCode != http.StatusNotFound || forwarded.Load() != 0 { + t.Errorf("unregistered route: status %d, forwarded %d", resp.StatusCode, forwarded.Load()) + } +} + +// TestRelayStopsAtLimit checks a fetch over git.max_repo_size is cut off +// rather than downloaded in full and thrown away. +func TestRelayStopsAtLimit(t *testing.T) { + const size, limit = 4 << 20, 64 << 10 + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = io.Copy(w, io.LimitReader(zeros{}, size)) + })) + defer upstream.Close() + rl := newTestRelay(t) + base, route, unregister := rl.register(upstream.URL+"/repo", limit, nil) + defer unregister() + + resp, err := http.Post(base+"/git-upload-pack", requestType, strings.NewReader(pktFlush)) + if err != nil { + t.Fatal(err) + } + n, err := io.Copy(io.Discard, resp.Body) + _ = resp.Body.Close() + if err == nil { + t.Errorf("read %d bytes without an error", n) + } + if !route.exceeded.Load() || route.relayed.Load() > size/2 { + t.Errorf("exceeded %v after relaying %d of %d bytes", route.exceeded.Load(), route.relayed.Load(), size) + } +} + +type zeros struct{} + +func (zeros) Read(p []byte) (int, error) { + clear(p) + return len(p), nil +} diff --git a/internal/gitcache/serve.go b/internal/gitcache/serve.go new file mode 100644 index 0000000..6d88df4 --- /dev/null +++ b/internal/gitcache/serve.go @@ -0,0 +1,227 @@ +package gitcache + +import ( + "bytes" + "compress/gzip" + "context" + "errors" + "io" + "net/http" + "path/filepath" + "regexp" + "slices" + "strings" + "sync/atomic" +) + +// gitProtocolPattern is what a Git-Protocol header may hold, such as +// "version=2". It is passed to upload-pack in its environment. +var gitProtocolPattern = regexp.MustCompile(`^[A-Za-z0-9=:._-]*$`) + +var errInvalidKey = errors.New("invalid repository key") + +// Lease is a use of a cached repository. The repository is not evicted +// until the lease is released. +type Lease struct { + cache *Cache + repo *repo + repoURL string + hit bool + fetchWants bool +} + +// OpenOptions says what Open does besides returning the repository. +type OpenOptions struct { + // Refresh brings a cached repository up to date with the upstream first, + // or leaves it as it is when the upstream fails or is slow and the copy + // is recent enough (see Options). + Refresh bool + // FetchWants lets ServeUploadPack fetch commits a client asks for by ID + // that the cache lacks. Leave it off where a fetch by ID could reach + // commits the client may not read through this repository, such as + // GitHub forks with credentials configured. + FetchWants bool +} + +// Open returns a lease on the repository cached under key, a relative path +// such as github-1a2b3c4d/acme/widgets.git, whose upstream is repoURL. A +// repository not cached yet is cloned first. +func (c *Cache) Open(ctx context.Context, key, repoURL string, opts OpenOptions) (*Lease, error) { + if !strings.HasSuffix(key, ".git") || !filepath.IsLocal(filepath.FromSlash(key)) { + return nil, errInvalidKey + } + rp := c.acquire(key) + hit := rp.exists() + if opts.Refresh || !hit { + if err := c.ensureFresh(ctx, rp, repoURL); err != nil { + c.release(rp) + return nil, err + } + } + return &Lease{cache: c, repo: rp, repoURL: repoURL, hit: hit, fetchWants: opts.FetchWants}, nil +} + +// Hit reports whether the repository was already cached when it was opened. +func (l *Lease) Hit() bool { + return l.hit +} + +// Release ends the lease. +func (l *Lease) Release() { + l.cache.release(l.repo) +} + +// ServeInfoRefs answers GET info/refs?service=git-upload-pack. +func (l *Lease) ServeInfoRefs(w http.ResponseWriter, r *http.Request) { + protocol := gitProtocol(r) + setNoCache(w) + w.Header().Set("Content-Type", advertisementType) + // Protocol v2 starts with its version line; earlier versions with the + // service name. + if !isProtocolV2(protocol) { + _, _ = io.WriteString(w, pktLine("# service=git-upload-pack\n")+pktFlush) + } + l.cache.uploadPack(w, r, l.repo, protocol, nil, "--advertise-refs") +} + +// ServeUploadPack answers POST git-upload-pack. With OpenOptions.FetchWants, +// commits the request wants that the cache lacks are fetched from the +// upstream first. +func (l *Lease) ServeUploadPack(w http.ResponseWriter, r *http.Request) { + c := l.cache + // The raw body is limited too: a gzip stream of any length can decode + // to almost nothing. + raw := &bodyLimit{r: r.Body, remaining: c.maxRequestBody} + decoded := raw + if r.Header.Get("Content-Encoding") == "gzip" { + gz, err := gzip.NewReader(raw) + if err != nil { + http.Error(w, "invalid gzip body", http.StatusBadRequest) + return + } + defer func() { _ = gz.Close() }() + decoded = &bodyLimit{r: gz, remaining: c.maxRequestBody} + } + defer func() { + if raw.exceeded.Load() || decoded.exceeded.Load() { + c.logger.Warn("git cache: request body passed the limit", + "repository", l.repo.key, "limit", c.maxRequestBody) + } + }() + body := io.Reader(decoded) + if l.fetchWants { + var wants []string + wants, body = readWants(body) + if err := c.fetchMissingWants(r.Context(), l.repo, l.repoURL, wants); err != nil && r.Context().Err() == nil { + // upload-pack reports the missing objects to the client. + c.logger.Warn("git cache: fetching wanted objects failed", "repository", l.repo.key, "error", err) + } + } + var command string + command, body = requestCommand(body) + // ls-refs builds no pack, so it does not wait behind clones. + if command != "ls-refs" { + select { + case c.uploadPacks <- struct{}{}: + defer func() { <-c.uploadPacks }() + case <-r.Context().Done(): + return + } + } + setNoCache(w) + w.Header().Set("Content-Type", resultType) + c.uploadPack(w, r, l.repo, gitProtocol(r), body) +} + +// requestCommand returns the protocol v2 command an upload-pack request +// starts with, such as "fetch", or "" for an earlier protocol, with a reader +// that replays the whole body. +func requestCommand(body io.Reader) (string, io.Reader) { + var consumed bytes.Buffer + _, line, _ := newPktReader(io.TeeReader(body, &consumed)).next() + command, ok := strings.CutPrefix(line, "command=") + if !ok { + command = "" + } + return command, io.MultiReader(&consumed, body) +} + +// servingConfig gives each pack-objects that upload-pack runs one +// delta-search thread with a bounded window, where git defaults to a thread +// per CPU and no bound. +var servingConfig = []string{"-c", "pack.threads=1", "-c", "pack.windowMemory=64m"} + +// uploadPack runs git upload-pack on the cached repository in stateless mode, +// the way git http-backend does. +func (c *Cache) uploadPack(w http.ResponseWriter, r *http.Request, rp *repo, protocol string, stdin io.Reader, args ...string) { + var env []string + if protocol != "" { + env = append(env, "GIT_PROTOCOL="+protocol) + } + args = slices.Concat(servingConfig, []string{"upload-pack", "--stateless-rpc"}, args, []string{rp.dir}) + cmd := c.command(r.Context(), env, args...) + cmd.Stdin = stdin + cmd.Stdout = w + stderr := &limitedBuffer{max: maxStderr} + cmd.Stderr = stderr + if err := cmd.Run(); err != nil && r.Context().Err() == nil { + c.logger.Warn("git cache: upload-pack failed", + "repository", rp.key, "error", err, "stderr", strings.TrimSpace(stderr.String())) + } +} + +// defaultMaxRequestBody bounds an upload-pack request far above any real +// negotiation, so a small gzip body cannot feed upload-pack for as long as +// the request may run. +const defaultMaxRequestBody = 100 << 20 + +var errRequestTooLarge = errors.New("request body too large") + +// bodyLimit fails reads past remaining bytes, where io.LimitReader would end +// the body quietly and leave upload-pack a request cut short. +type bodyLimit struct { + r io.Reader + remaining int64 + exceeded atomic.Bool +} + +func (b *bodyLimit) Read(p []byte) (int, error) { + if b.exceeded.Load() { + return 0, errRequestTooLarge + } + // One byte past the limit tells a longer body from one that ends at it. + if int64(len(p)) > b.remaining+1 { + p = p[:b.remaining+1] + } + n, err := b.r.Read(p) + if int64(n) <= b.remaining { + b.remaining -= int64(n) + return n, err + } + n = int(b.remaining) + b.remaining = 0 + b.exceeded.Store(true) + return n, errRequestTooLarge +} + +func gitProtocol(r *http.Request) string { + if protocol := r.Header.Get("Git-Protocol"); gitProtocolPattern.MatchString(protocol) { + return protocol + } + return "" +} + +func isProtocolV2(protocol string) bool { + for field := range strings.SplitSeq(protocol, ":") { + if field == "version=2" { + return true + } + } + return false +} + +func setNoCache(w http.ResponseWriter) { + w.Header().Set("Expires", "Fri, 01 Jan 1980 00:00:00 GMT") + w.Header().Set("Pragma", "no-cache") + w.Header().Set("Cache-Control", "no-cache, max-age=0, must-revalidate") +} diff --git a/internal/gitcache/serve_test.go b/internal/gitcache/serve_test.go new file mode 100644 index 0000000..b5779ae --- /dev/null +++ b/internal/gitcache/serve_test.go @@ -0,0 +1,327 @@ +package gitcache + +import ( + "bytes" + "compress/gzip" + "fmt" + "io" + "net/http" + "os/exec" + "path/filepath" + "slices" + "strings" + "sync" + "testing" + "time" +) + +// TestRequestBodies sends the bodies git sends for large requests: gzipped, +// or chunked once one outgrows http.postBuffer. +func TestRequestBodies(t *testing.T) { + upstream := newFakeUpstream(t) + upstream.newRepo("repo") + tc := newTestCache(t, upstream.server.URL, nil) + tc.clone(t, "repo") + + lsRefs := pktLine("command=ls-refs\n") + pktDelim + pktLine("ref-prefix refs/heads/\n") + pktFlush + var gzipped bytes.Buffer + gz := gzip.NewWriter(&gzipped) + _, _ = io.WriteString(gz, lsRefs) + _ = gz.Close() + for _, tt := range []struct { + name string + body io.Reader + encoding string + }{ + // A reader of unknown length is sent chunked. + {"chunked", io.MultiReader(strings.NewReader(lsRefs)), ""}, + {"gzip", &gzipped, "gzip"}, + } { + req, _ := http.NewRequest(http.MethodPost, tc.url("repo")+"/git-upload-pack", tt.body) + req.Header.Set("Content-Type", requestType) + req.Header.Set("Git-Protocol", "version=2") + if tt.encoding != "" { + req.Header.Set("Content-Encoding", tt.encoding) + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + out, _ := io.ReadAll(resp.Body) + _ = resp.Body.Close() + if resp.StatusCode != http.StatusOK || !strings.Contains(string(out), "refs/heads/main") { + t.Errorf("%s: status %d, body %q", tt.name, resp.StatusCode, out) + } + } +} + +// TestRequestBodyLimit refuses a request past the limit, before or after +// gzip, including one that is small compressed. +func TestRequestBodyLimit(t *testing.T) { + const limit = 4096 + upstream := newFakeUpstream(t) + upstream.newRepo("repo") + tc := newTestCache(t, upstream.server.URL, func(o *Options) { o.maxRequestBody = limit }) + tc.clone(t, "repo") + + lsRefs := func(prefixes int) string { + return pktLine("command=ls-refs\n") + pktDelim + + strings.Repeat(pktLine("ref-prefix refs/heads/\n"), prefixes) + pktFlush + } + gzipped := func(body string) *bytes.Buffer { + var buf bytes.Buffer + gz := gzip.NewWriter(&buf) + _, _ = io.WriteString(gz, body) + _ = gz.Close() + return &buf + } + small, large := lsRefs(10), lsRefs(200) + if len(small) > limit || len(large) <= limit { + t.Fatalf("bodies of %d and %d bytes do not straddle the limit", len(small), len(large)) + } + compressed := gzipped(large) + if compressed.Len() > limit { + t.Fatalf("compressed body is %d bytes, over the limit", compressed.Len()) + } + // Empty gzip members before a small body: long on the wire, short decoded. + var padded bytes.Buffer + for empty := gzipped(""); padded.Len() <= limit; { + padded.Write(empty.Bytes()) + } + padded.Write(gzipped(small).Bytes()) + for _, tt := range []struct { + name string + body io.Reader + encoding string + served bool + }{ + {"small", strings.NewReader(small), "", true}, + {"small gzip", gzipped(small), "gzip", true}, + {"large", strings.NewReader(large), "", false}, + {"large gzip", compressed, "gzip", false}, + {"padded gzip", &padded, "gzip", false}, + } { + req, _ := http.NewRequest(http.MethodPost, tc.url("repo")+"/git-upload-pack", tt.body) + req.Header.Set("Content-Type", requestType) + req.Header.Set("Git-Protocol", "version=2") + if tt.encoding != "" { + req.Header.Set("Content-Encoding", tt.encoding) + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + out, _ := io.ReadAll(resp.Body) + _ = resp.Body.Close() + if served := strings.Contains(string(out), "refs/heads/main"); served != tt.served { + t.Errorf("%s: served %t, want %t (status %d, body %q)", tt.name, served, tt.served, resp.StatusCode, out) + } + } +} + +// TestUploadPackSlots holds the only upload-pack slot: a clone waits for it, +// while listing refs, which builds no pack, does not. +func TestUploadPackSlots(t *testing.T) { + upstream := newFakeUpstream(t) + upstream.newRepo("repo") + tc := newTestCache(t, upstream.server.URL, func(o *Options) { o.maxUploadPacks = 1 }) + tc.clone(t, "repo") + + start := func(args ...string) <-chan error { + done := make(chan error, 1) + go func() { + out, err := tryGit(t, "", args...) + if err != nil { + err = fmt.Errorf("%w\n%s", err, out) + } + done <- err + }() + return done + } + + tc.uploadPacks <- struct{}{} + held := true + release := func() { + if held { + held = false + <-tc.uploadPacks + } + } + // A failure while the slot is held must free it for the cache to close. + defer release() + select { + case err := <-start("-c", "protocol.version=2", "ls-remote", tc.url("repo")): + if err != nil { + t.Fatalf("ls-remote: %v", err) + } + case <-time.After(10 * time.Second): + t.Fatal("ls-remote waited for the slot") + } + clone := start("clone", "--quiet", tc.url("repo"), filepath.Join(t.TempDir(), "clone")) + select { + case err := <-clone: + t.Fatalf("clone finished while the slot was held: %v", err) + case <-time.After(time.Second): + } + release() + if err := <-clone; err != nil { + t.Errorf("clone: %v", err) + } +} + +func TestRequestCommand(t *testing.T) { + for _, tt := range []struct{ body, want string }{ + {pktLine("command=ls-refs\n") + pktDelim + pktFlush, "ls-refs"}, + {pktLine("command=fetch\n") + pktDelim + pktLine("done\n") + pktFlush, "fetch"}, + {pktLine("want "+strings.Repeat("a", 40)+"\n") + pktFlush, ""}, + {"", ""}, + } { + command, body := requestCommand(strings.NewReader(tt.body)) + replayed, _ := io.ReadAll(body) + if command != tt.want || string(replayed) != tt.body { + t.Errorf("requestCommand(%q) = %q, %q; want %q and the whole body", tt.body, command, replayed, tt.want) + } + } +} + +// TestCloneWithChunkedRequests clones a repository with enough refs that +// git's requests, to the cache and from the cache to the upstream, outgrow +// http.postBuffer and are sent chunked. +func TestCloneWithChunkedRequests(t *testing.T) { + const tags = 2000 + upstream := newFakeUpstream(t) + work := upstream.newRepo("repo") + var stream strings.Builder + for i := 1; i <= tags; i++ { + fmt.Fprintf(&stream, "commit refs/heads/main\nmark :%d\ncommitter test %d +0000\ndata 1\n\n", i, i) + if i == 1 { + stream.WriteString("from refs/heads/main^0\n") + } + fmt.Fprintf(&stream, "reset refs/tags/t%d\nfrom :%d\n\n", i, i) + } + importer := exec.Command("git", "fast-import", "--quiet") + importer.Dir, importer.Env, importer.Stdin = work, gitEnv(t.TempDir()), strings.NewReader(stream.String()) + if out, err := importer.CombinedOutput(); err != nil { + t.Fatalf("git fast-import: %v\n%s", err, out) + } + runGit(t, work, "push", "--quiet", "origin", "main", "--tags") + tc := newTestCache(t, upstream.server.URL, nil) + + // One want per tag is about 100KB, past a postBuffer just above git's + // minimum. + clone := tc.clone(t, "repo", "-c", "http.postBuffer=66000") + if got := strings.Count(runGit(t, clone, "tag", "--list"), "\n") + 1; got != tags+1 { + t.Errorf("cloned %d tags, want %d", got, tags+1) + } +} + +func TestParseV0Advertisement(t *testing.T) { + oid := strings.Repeat("a", 40) + peeled := strings.Repeat("b", 40) + adv := pktLine(oid+" HEAD\x00multi_ack symref=HEAD:refs/heads/trunk agent=git/2\n") + + pktLine(oid+" refs/heads/trunk\n") + + pktLine(oid+" refs/tags/v1\n") + + pktLine(peeled+" refs/tags/v1^{}\n") + + pktLine(oid+" refs/changes/01/1/1\n") + + pktLine(oid+" refs/pull/1/head\n") + + pktFlush + pkts := newPktReader(strings.NewReader(adv)) + kind, line, err := pkts.next() + if err != nil { + t.Fatal(err) + } + remote, err := parseV0Advertisement(pkts, kind, line) + if err != nil { + t.Fatal(err) + } + if remote.head != "refs/heads/trunk" { + t.Errorf("head = %q", remote.head) + } + want := map[string]string{"refs/heads/trunk": oid, "refs/tags/v1": oid} + if len(remote.refs) != len(want) || remote.refs["refs/heads/trunk"] != oid || remote.refs["refs/tags/v1"] != oid { + t.Errorf("refs = %v, want %v", remote.refs, want) + } +} + +// TestCommitOnNoBranchIsFetchedByID does what actions/checkout does: fetch +// one commit by object ID, here a pull request head that is on no branch. +func TestCommitOnNoBranchIsFetchedByID(t *testing.T) { + for _, version := range []string{"0", "2"} { + t.Run("protocol v"+version, func(t *testing.T) { + upstream := newFakeUpstream(t) + work := upstream.newRepo("repo") + runGit(t, upstream.bare(), "config", "uploadpack.allowAnySHA1InWant", "true") + runGit(t, work, "checkout", "--quiet", "-b", "pr") + sha := upstream.commit(work, "pull request") + runGit(t, work, "push", "--quiet", "origin", "pr:refs/pull/1/head") + tc := newTestCache(t, upstream.server.URL, nil) + tc.clone(t, "repo") + sessions := upstream.gitSessions.Load() + + var wg sync.WaitGroup + for range 4 { + wg.Go(func() { + dir := t.TempDir() + runGit(t, dir, "init", "--quiet") + if out, err := tryGit(t, dir, "-c", "protocol.version="+version, "fetch", "--quiet", "--depth=1", tc.url("repo"), sha); err != nil { + t.Errorf("fetch by ID: %v\n%s", err, out) + return + } + if got := runGit(t, dir, "rev-parse", "FETCH_HEAD"); got != sha { + t.Errorf("FETCH_HEAD = %s, want %s", got, sha) + } + }) + } + wg.Wait() + if got := upstream.gitSessions.Load() - sessions; got != 1 { + t.Errorf("upstream saw %d fetches for the commit, want 1", got) + } + if refs := runGit(t, "", "ls-remote", tc.url("repo")); strings.Contains(refs, wantedPrefix) { + t.Errorf("advertised %s refs:\n%s", wantedPrefix, refs) + } + + // A refresh that prunes branches keeps the commit. + runGit(t, work, "push", "--quiet", "origin", "main:refs/heads/new") + tc.clone(t, "repo") + if refs := tc.cachedRefs(t); !strings.Contains(refs, wantedPrefix) || !strings.Contains(refs, "/"+sha) { + t.Errorf("refresh dropped the fetched commit:\n%s", refs) + } + }) + } +} + +func TestReadWants(t *testing.T) { + a, b := strings.Repeat("a", 40), strings.Repeat("b", 40) + tests := map[string]struct { + body string + want []string + }{ + "v0": { + pktLine("want "+a+" multi_ack side-band-64k\n") + pktLine("want "+b+"\n") + pktLine("deepen 1\n") + pktFlush + pktLine("have "+a+"\n") + pktFlush, + []string{a, b}, + }, + "v2 fetch": { + pktLine("command=fetch\n") + pktLine("agent=git/2\n") + pktDelim + pktLine("thin-pack\n") + pktLine("want "+a+"\n") + pktLine("have "+b+"\n") + pktLine("want "+b+"\n") + pktLine("done\n") + pktFlush, + []string{a, b}, + }, + "v2 ls-refs": { + pktLine("command=ls-refs\n") + pktDelim + pktLine("want "+a+"\n") + pktFlush, + nil, + }, + "not an object ID": { + pktLine("want HEAD\n") + pktFlush, + nil, + }, + } + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + wants, replay := readWants(strings.NewReader(tt.body)) + if !slices.Equal(wants, tt.want) { + t.Errorf("wants = %v, want %v", wants, tt.want) + } + if got, _ := io.ReadAll(replay); string(got) != tt.body { + t.Errorf("replayed %q, want the whole body", got) + } + }) + } +} diff --git a/internal/gitcache/upstream.go b/internal/gitcache/upstream.go new file mode 100644 index 0000000..46a2667 --- /dev/null +++ b/internal/gitcache/upstream.go @@ -0,0 +1,214 @@ +package gitcache + +import ( + "context" + "errors" + "fmt" + "io" + "net/http" + "strings" +) + +const ( + advertisementType = "application/x-git-upload-pack-advertisement" + requestType = "application/x-git-upload-pack-request" + resultType = "application/x-git-upload-pack-result" + // Some servers only speak the smart protocol to clients that look like git. + gitUserAgent = "git/2 (git-pkgs-proxy)" + // maxRefsResponse bounds a ref listing; branches and tags of even large + // repositories take a few megabytes. + maxRefsResponse = 64 << 20 + + headsPrefix = "refs/heads/" + tagsPrefix = "refs/tags/" +) + +var ( + // ErrNotFound is returned when the upstream has no such repository. + ErrNotFound = errors.New("repository not found upstream") + // ErrRefused covers authentication failures and redirects. The cached + // copy is not served after one, since the upstream may have withdrawn + // access to it. + ErrRefused = errors.New("upstream refused the request") + // ErrTooLarge is returned for a repository larger than git.max_repo_size, + // or than git.max_size where that is smaller. + ErrTooLarge = errors.New("repository is larger than git.max_repo_size or git.max_size") + // ErrCacheFull is returned for a fetch cut off because repositories in + // use take what git.max_size leaves. + ErrCacheFull = errors.New("git cache is full") +) + +// remoteRefs is what a repository advertises: its branch and tag tips, and +// the branch HEAD points at. +type remoteRefs struct { + refs map[string]string + head string +} + +// listUpstreamRefs asks the upstream for its branches, tags and HEAD. With +// protocol v2 the server filters by prefix, which matters for Gerrit, where +// every patch set is a ref. Servers without v2 send every ref, and the rest +// are dropped here. +func (c *Cache) listUpstreamRefs(ctx context.Context, repoURL string) (remoteRefs, error) { + resp, err := c.upstreamRequest(ctx, http.MethodGet, repoURL+"/info/refs?service=git-upload-pack", "") + if err != nil { + return remoteRefs{}, err + } + defer func() { _ = resp.Body.Close() }() + if ct := resp.Header.Get("Content-Type"); !strings.HasPrefix(ct, advertisementType) { + return remoteRefs{}, fmt.Errorf("upstream did not answer as a smart HTTP Git server (content type %q)", ct) + } + + pkts := newPktReader(io.LimitReader(resp.Body, maxRefsResponse)) + kind, line, err := pkts.next() + if err == nil && kind == pktData && strings.HasPrefix(line, "# service=") { + if kind, _, err = pkts.next(); err == nil && kind != pktFlushKind { + err = errors.New("no flush after the service line") + } + if err == nil { + kind, line, err = pkts.next() + } + } + if err != nil { + return remoteRefs{}, fmt.Errorf("reading ref advertisement: %w", err) + } + + if kind == pktData && line == "version 2" { + return c.lsRefs(ctx, repoURL) + } + if kind == pktData && line == "version 1" { + if kind, line, err = pkts.next(); err != nil { + return remoteRefs{}, fmt.Errorf("reading ref advertisement: %w", err) + } + } + return parseV0Advertisement(pkts, kind, line) +} + +// lsRefs runs the protocol v2 ls-refs command. +func (c *Cache) lsRefs(ctx context.Context, repoURL string) (remoteRefs, error) { + body := pktLine("command=ls-refs\n") + pktDelim + + pktLine("symrefs\n") + + pktLine("ref-prefix HEAD\n") + + pktLine("ref-prefix "+headsPrefix+"\n") + + pktLine("ref-prefix "+tagsPrefix+"\n") + + pktFlush + resp, err := c.upstreamRequest(ctx, http.MethodPost, repoURL+"/git-upload-pack", body) + if err != nil { + return remoteRefs{}, err + } + defer func() { _ = resp.Body.Close() }() + + remote := remoteRefs{refs: make(map[string]string)} + pkts := newPktReader(io.LimitReader(resp.Body, maxRefsResponse)) + for { + kind, line, err := pkts.next() + if err != nil { + return remoteRefs{}, fmt.Errorf("reading ls-refs response: %w", err) + } + if kind == pktFlushKind { + return remote, nil + } + if kind != pktData { + continue + } + fields := strings.Split(line, " ") + if len(fields) < 2 { //nolint:mnd // object id and ref name + return remoteRefs{}, fmt.Errorf("invalid ls-refs line %q", line) + } + if fields[1] == "HEAD" { + for _, attr := range fields[2:] { + if target, ok := strings.CutPrefix(attr, "symref-target:"); ok { + remote.head = target + } + } + continue + } + remote.add(fields[1], fields[0]) + } +} + +// parseV0Advertisement reads a protocol v0 ref advertisement whose first +// packet has already been read. The first ref carries the capabilities, +// including where HEAD points. +func parseV0Advertisement(pkts *pktReader, kind pktKind, line string) (remoteRefs, error) { + remote := remoteRefs{refs: make(map[string]string)} + first := true + for kind != pktFlushKind { + if kind == pktData { + ref, caps, _ := strings.Cut(line, "\x00") + if first { + remote.head = symrefTarget(caps, "HEAD") + first = false + } + oid, name, ok := strings.Cut(ref, " ") + if !ok { + return remoteRefs{}, fmt.Errorf("invalid ref advertisement line %q", ref) + } + remote.add(name, oid) + } + var err error + if kind, line, err = pkts.next(); err != nil { + return remoteRefs{}, fmt.Errorf("reading ref advertisement: %w", err) + } + } + return remote, nil +} + +// add keeps branches and tags, leaving out peeled tag entries. +func (r remoteRefs) add(name, oid string) { + if (strings.HasPrefix(name, headsPrefix) || strings.HasPrefix(name, tagsPrefix)) && !strings.HasSuffix(name, "^{}") { + r.refs[name] = oid + } +} + +func symrefTarget(caps, ref string) string { + for _, capability := range strings.Fields(caps) { + if value, ok := strings.CutPrefix(capability, "symref="+ref+":"); ok { + return value + } + } + return "" +} + +// upstreamRequest sends a smart HTTP request through the upstream client and +// classifies refusals. A non-empty body is sent as an upload-pack request. +func (c *Cache) upstreamRequest(ctx context.Context, method, target, body string) (*http.Response, error) { + var reader io.Reader + if body != "" { + reader = strings.NewReader(body) + } + req, err := http.NewRequestWithContext(ctx, method, target, reader) + if err != nil { + return nil, err + } + req.Header.Set("Git-Protocol", "version=2") + req.Header.Set("User-Agent", gitUserAgent) + if body != "" { + req.Header.Set("Content-Type", requestType) + req.Header.Set("Accept", resultType) + } + resp, err := c.client.Do(req) + if err != nil { + return nil, fmt.Errorf("upstream request: %w", err) + } + if err := upstreamStatusError(resp.StatusCode); err != nil { + _ = resp.Body.Close() + return nil, err + } + return resp, nil +} + +func upstreamStatusError(status int) error { + switch { + case status == http.StatusOK: + return nil + case status == http.StatusNotFound || status == http.StatusGone: + return fmt.Errorf("%w: status %d", ErrNotFound, status) + case status == http.StatusUnauthorized || status == http.StatusForbidden: + return fmt.Errorf("%w: status %d", ErrRefused, status) + case status >= http.StatusMultipleChoices && status < http.StatusBadRequest: + return fmt.Errorf("%w: redirect (status %d) not followed", ErrRefused, status) + default: + return fmt.Errorf("upstream returned status %d", status) + } +} diff --git a/internal/gitcache/wants.go b/internal/gitcache/wants.go new file mode 100644 index 0000000..c715f2f --- /dev/null +++ b/internal/gitcache/wants.go @@ -0,0 +1,265 @@ +package gitcache + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "maps" + "slices" + "strconv" + "strings" + "time" +) + +const ( + // wantedPrefix holds commits fetched because a client asked for them by + // object ID, such as a pull request head that is on no branch, as + // {wantedPrefix}{unix time}/{object ID}. The refs keep them from being + // garbage collected, and are hidden from clients. + wantedPrefix = "refs/proxy/wanted/" + // wantedMaxAge is how long a commit fetched by ID is kept while no + // cached branch or tag includes it. + wantedMaxAge = 7 * 24 * time.Hour + // wantRetryAfter is how long a want the upstream could not provide is + // not asked for again. Short, since a commit can reach a CI job before + // it reaches every upstream replica. A want too large for what + // max_repo_size leaves waits tooLargeRetryAfter instead. + wantRetryAfter = time.Minute + // maxFailedWants bounds the record of failed wants. + maxFailedWants = 4096 + // maxWantScan bounds how much of a request body is read for its wants. + maxWantScan = 1 << 20 + // maxWantFetch bounds the missing objects one request may have fetched. + maxWantFetch = 32 + // maxWantSplit bounds the wants of a failed fetch that are fetched again + // one at a time; a larger batch is refused as a whole. + maxWantSplit = 4 +) + +// readWants reads the object IDs a fetch request wants, from its first +// section, and returns them with a reader that replays the whole body. In +// protocol v0 that section holds the wants; in v2 it is the whole request. +func readWants(body io.Reader) ([]string, io.Reader) { + var consumed bytes.Buffer + pkts := newPktReader(io.TeeReader(io.LimitReader(body, maxWantScan), &consumed)) + var wants []string + for first := true; ; first = false { + kind, line, err := pkts.next() + if err != nil || kind == pktFlushKind { + break + } + if kind != pktData { + continue + } + if command, ok := strings.CutPrefix(line, "command="); ok && first && command != "fetch" { + break + } + if rest, ok := strings.CutPrefix(line, "want "); ok { + if oid, _, _ := strings.Cut(rest, " "); isObjectID(oid) { + wants = append(wants, oid) + } + } + } + return wants, io.MultiReader(&consumed, body) +} + +// fetchMissingWants fetches the wanted objects the repository lacks. Wants +// for branch and tag tips are already there after the refresh; this covers +// commits a client names directly, as actions/checkout does. +func (c *Cache) fetchMissingWants(ctx context.Context, rp *repo, repoURL string, wants []string) error { + missing, err := c.missingObjects(ctx, rp.dir, wants) + if err != nil { + return err + } + missing = c.withoutFailedWants(rp.key, missing) + if len(missing) == 0 || len(missing) > maxWantFetch { + return nil + } + unlock, err := rp.lockWrites(ctx) + if err != nil { + return err + } + defer unlock() + // Another request may have fetched them while this one waited. + if missing, err = c.missingObjects(ctx, rp.dir, missing); err != nil || len(missing) == 0 { + return err + } + defer c.evict() + + err = c.fetchWants(ctx, rp, repoURL, missing) + if err == nil || ctx.Err() != nil { + return err + } + if len(missing) == 1 || len(missing) > maxWantSplit || + errors.Is(err, ErrTooLarge) || errors.Is(err, ErrCacheFull) || errors.Is(err, errUpstreamUnanswered) { + // Fetched one at a time, wants too large for what max_repo_size + // or max_size leaves would exhaust the same budget, and an upstream + // that did not answer would be waited on once per want. + c.recordFailedWants(rp.key, missing, err) + return err + } + // Find which wants failed, so a good one asked for with a bad one is + // not refused too. These fetches hold the repository's write lock, so + // only a small batch is split. + var errs []error + for _, oid := range missing { + if err := c.fetchWants(ctx, rp, repoURL, []string{oid}); err != nil { + if ctx.Err() != nil { + return err + } + c.recordFailedWants(rp.key, []string{oid}, err) + errs = append(errs, err) + } + } + return errors.Join(errs...) +} + +// fetchWants fetches oids from the upstream under wanted refs, within what +// max_repo_size leaves. A want that does not fit is refused, not the +// repository: transfer.unpackLimit makes a cut-off transfer leave nothing +// behind. +func (c *Cache) fetchWants(ctx context.Context, rp *repo, repoURL string, oids []string) error { + limit, err := c.fetchLimit(rp) + if err != nil { + return err + } + now := time.Now() + refspecs := make([]string, len(oids)) + for i, oid := range oids { + refspecs[i] = oid + ":" + wantedRef(oid, now) + } + res := c.newReservation() + err = c.fetch(ctx, rp.dir, repoURL, limit, res, []string{"--atomic"}, refspecs...) + res.settle(rp, dirSize(rp.dir)) + return err +} + +func wantedRef(oid string, at time.Time) string { + return wantedPrefix + strconv.FormatInt(at.Unix(), 10) + "/" + oid +} + +// pruneWanted deletes wanted refs that are no longer needed: those older +// than wantedMaxAge, and, after a fetch has moved branches or tags, those +// whose commit a cached branch or tag now includes. +func (c *Cache) pruneWanted(ctx context.Context, dir string, wanted []string, fetched bool) error { + if len(wanted) == 0 { + return nil + } + cutoff := time.Now().Add(-wantedMaxAge).Unix() + var drop []string + keep := make(map[string]string, len(wanted)) + for _, ref := range wanted { + created, oid, ok := strings.Cut(strings.TrimPrefix(ref, wantedPrefix), "/") + unix, err := strconv.ParseInt(created, 10, 64) + if !ok || err != nil || unix < cutoff || !isObjectID(oid) { + drop = append(drop, ref) + continue + } + keep[oid] = ref + } + var mergedErr error + if fetched && len(keep) > 0 { + // rev-list prints what the wanted commits, read from stdin, reach + // and no branch or tag does, so a wanted commit it leaves out is + // merged. + cmd := c.command(ctx, nil, "-C", dir, "rev-list", "--stdin", "--not", "--branches", "--tags") + cmd.Stdin = strings.NewReader(strings.Join(slices.Sorted(maps.Keys(keep)), "\n") + "\n") + out, err := cmd.Output() + if err != nil { + mergedErr = fmt.Errorf("listing unmerged wanted commits: %w", err) + } else { + unmerged := make(map[string]bool) + for line := range strings.Lines(string(out)) { + unmerged[strings.TrimSpace(line)] = true + } + for oid, ref := range keep { + if !unmerged[oid] { + drop = append(drop, ref) + } + } + } + } + if len(drop) == 0 { + return mergedErr + } + var stdin strings.Builder + for _, ref := range drop { + stdin.WriteString("delete " + ref + "\n") + } + cmd := c.command(ctx, nil, "-C", dir, "update-ref", "--stdin") + cmd.Stdin = strings.NewReader(stdin.String()) + if out, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("deleting wanted refs: %w: %s", err, strings.TrimSpace(string(out))) + } + return mergedErr +} + +// withoutFailedWants drops the object IDs a recent fetch for the +// repository could not get. +func (c *Cache) withoutFailedWants(key string, oids []string) []string { + c.mu.Lock() + defer c.mu.Unlock() + now := time.Now() + return slices.DeleteFunc(oids, func(oid string) bool { + until, ok := c.failedWants[key+" "+oid] + return ok && now.Before(until) + }) +} + +// recordFailedWants keeps oids from being fetched again for the repository +// for wantRetryAfter, or for tooLargeRetryAfter when they did not fit in +// max_repo_size. +func (c *Cache) recordFailedWants(key string, oids []string, err error) { + now := time.Now() + until := now.Add(wantRetryAfter) + if errors.Is(err, ErrTooLarge) { + until = now.Add(tooLargeRetryAfter) + } + c.mu.Lock() + defer c.mu.Unlock() + for k, expiry := range c.failedWants { + if !now.Before(expiry) { + delete(c.failedWants, k) + } + } + for _, oid := range oids { + if len(c.failedWants) >= maxFailedWants { + return + } + c.failedWants[key+" "+oid] = until + } +} + +// missingObjects returns the object IDs in oids the repository does not have. +func (c *Cache) missingObjects(ctx context.Context, dir string, oids []string) ([]string, error) { + if len(oids) == 0 { + return nil, nil + } + cmd := c.command(ctx, nil, "-C", dir, "cat-file", "--batch-check") + cmd.Stdin = strings.NewReader(strings.Join(oids, "\n") + "\n") + out, err := cmd.Output() + if err != nil { + return nil, err + } + var missing []string + for line := range strings.Lines(string(out)) { + if oid, ok := strings.CutSuffix(strings.TrimSuffix(line, "\n"), " missing"); ok { + missing = append(missing, oid) + } + } + return missing, nil +} + +func isObjectID(s string) bool { + if len(s) != 40 && len(s) != 64 { //nolint:mnd // SHA-1 and SHA-256 hex lengths + return false + } + for _, r := range s { + if (r < '0' || r > '9') && (r < 'a' || r > 'f') { + return false + } + } + return true +} diff --git a/internal/gitcache/wants_test.go b/internal/gitcache/wants_test.go new file mode 100644 index 0000000..f01df06 --- /dev/null +++ b/internal/gitcache/wants_test.go @@ -0,0 +1,319 @@ +package gitcache + +import ( + "crypto/rand" + "encoding/hex" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strconv" + "strings" + "testing" + "time" +) + +// pushPullRequest commits on a new branch from main, with data in a file if +// given, and pushes it only as refs/pull/{n}/head, so the commit is on no +// branch or tag upstream. +func (u *fakeUpstream) pushPullRequest(work string, n int, data []byte) string { + u.t.Helper() + branch := "pr" + strconv.Itoa(n) + runGit(u.t, work, "checkout", "--quiet", "-b", branch, "main") + if data != nil { + if err := os.WriteFile(filepath.Join(work, branch+".bin"), data, 0o600); err != nil { + u.t.Fatal(err) + } + runGit(u.t, work, "add", branch+".bin") + } + runGit(u.t, work, "commit", "--quiet", "--allow-empty", "-m", branch) + runGit(u.t, work, "push", "--quiet", "origin", "HEAD:refs/pull/"+strconv.Itoa(n)+"/head") + runGit(u.t, work, "checkout", "--quiet", "main") + return runGit(u.t, work, "rev-parse", branch) +} + +// fetchByID fetches one commit by ID from url, as actions/checkout does. +func fetchByID(t *testing.T, url, sha string) error { + t.Helper() + dir := t.TempDir() + runGit(t, dir, "init", "--quiet") + _, err := tryGit(t, dir, "fetch", "--quiet", "--depth=1", url, sha) + return err +} + +func randomBytes(n int) []byte { + b := make([]byte, n) + _, _ = rand.Read(b) + return b +} + +func (tc *testCache) wantedRefs(t *testing.T) []string { + t.Helper() + var refs []string + for line := range strings.Lines(tc.cachedRefs(t)) { + if _, name, _ := strings.Cut(strings.TrimSpace(line), " "); strings.HasPrefix(name, wantedPrefix) { + refs = append(refs, name) + } + } + return refs +} + +func TestMergedWantedCommitIsUnpinned(t *testing.T) { + upstream := newFakeUpstream(t) + work := upstream.newRepo("repo") + runGit(t, upstream.bare(), "config", "uploadpack.allowAnySHA1InWant", "true") + sha := upstream.pushPullRequest(work, 1, nil) + tc := newTestCache(t, upstream.server.URL, nil) + tc.clone(t, "repo") + if err := fetchByID(t, tc.url("repo"), sha); err != nil { + t.Fatalf("fetch by ID: %v", err) + } + if refs := tc.wantedRefs(t); len(refs) != 1 { + t.Fatalf("wanted refs %v, want one", refs) + } + + runGit(t, work, "push", "--quiet", "origin", "pr1:main") + tc.clone(t, "repo") + if refs := tc.wantedRefs(t); len(refs) != 0 { + t.Errorf("wanted refs %v after the commit was merged, want none", refs) + } +} + +func TestOldWantedCommitIsUnpinned(t *testing.T) { + upstream := newFakeUpstream(t) + work := upstream.newRepo("repo") + runGit(t, upstream.bare(), "config", "uploadpack.allowAnySHA1InWant", "true") + old := upstream.pushPullRequest(work, 1, nil) + recent := upstream.pushPullRequest(work, 2, nil) + tc := newTestCache(t, upstream.server.URL, nil) + tc.clone(t, "repo") + for _, sha := range []string{old, recent} { + if err := fetchByID(t, tc.url("repo"), sha); err != nil { + t.Fatalf("fetch by ID: %v", err) + } + } + + cached := filepath.Join(tc.dir, "up", "repo.git") + for _, ref := range tc.wantedRefs(t) { + if strings.HasSuffix(ref, "/"+old) { + runGit(t, cached, "update-ref", "-d", ref) + } + } + runGit(t, cached, "update-ref", wantedRef(old, time.Now().Add(-wantedMaxAge-time.Hour)), old) + + tc.clone(t, "repo") + refs := tc.wantedRefs(t) + if len(refs) != 1 || !strings.HasSuffix(refs[0], "/"+recent) { + t.Errorf("wanted refs %v, want only the recent commit's", refs) + } +} + +// TestRepositoryTooLargeToRefreshIsClonedAgain fills a repository with a +// commit fetched by ID, then grows its branch upstream past what is left of +// max_repo_size. Rather than stop following the upstream, the repository is +// dropped once idle and cloned again with only its branches and tags, which +// fit. +func TestRepositoryTooLargeToRefreshIsClonedAgain(t *testing.T) { + upstream := newFakeUpstream(t) + work := upstream.newRepo("repo") + runGit(t, upstream.bare(), "config", "uploadpack.allowAnySHA1InWant", "true") + pinned := upstream.pushPullRequest(work, 1, randomBytes(100<<10)) + base := repoSize(t, upstream, "repo") + tc := newTestCache(t, upstream.server.URL, func(o *Options) { o.MaxRepoSize = base + 150<<10 }) + tc.clone(t, "repo") + if err := fetchByID(t, tc.url("repo"), pinned); err != nil { + t.Fatalf("fetch by ID: %v", err) + } + + if err := os.WriteFile(filepath.Join(work, "grown.bin"), randomBytes(80<<10), 0o600); err != nil { + t.Fatal(err) + } + runGit(t, work, "add", "grown.bin") + head := upstream.commit(work, "grown") + runGit(t, work, "push", "--quiet", "origin", "main") + + // This clone's refresh does not fit. It is served the cached copy, or, + // if the repository is dropped between its requests, the new clone. + tc.clone(t, "repo") + if got := runGit(t, tc.clone(t, "repo"), "rev-parse", "HEAD"); got != head { + t.Errorf("HEAD = %s, want %s once cloned again", got, head) + } + waitIdle(t, tc.Cache) + if _, err := tryGit(t, filepath.Join(tc.dir, "up", "repo.git"), "cat-file", "-e", pinned); err == nil { + t.Error("the repository still holds the commit fetched by ID, so it was not cloned again") + } +} + +// TestTooLargeWantKeepsRepository asks for a commit by ID that does not fit +// in what max_repo_size leaves. The commit is refused, for a while, and the +// repository is kept, with nothing of the cut-off transfer left in it. +func TestTooLargeWantKeepsRepository(t *testing.T) { + upstream := newFakeUpstream(t) + work := upstream.newRepo("repo") + runGit(t, upstream.bare(), "config", "uploadpack.allowAnySHA1InWant", "true") + large := upstream.pushPullRequest(work, 1, randomBytes(200<<10)) + small := upstream.pushPullRequest(work, 2, nil) + base := repoSize(t, upstream, "repo") + tc := newTestCache(t, upstream.server.URL, func(o *Options) { o.MaxRepoSize = base + 100<<10 }) + tc.clone(t, "repo") + + if err := fetchByID(t, tc.url("repo"), large); err == nil { + t.Fatal("fetched a commit larger than max_repo_size leaves") + } + waitIdle(t, tc.Cache) + cached := filepath.Join(tc.dir, "up", "repo.git") + if !fileExists(cached) { + t.Fatal("the repository was dropped") + } + if _, err := tryGit(t, cached, "cat-file", "-e", large); err == nil { + t.Error("the cut-off transfer left the commit behind") + } + if leftovers, _ := filepath.Glob(filepath.Join(cached, "objects", "pack", "tmp_*")); len(leftovers) != 0 { + t.Errorf("the cut-off transfer left %v", leftovers) + } + tc.mu.Lock() + until := tc.failedWants["up/repo.git "+large] + tc.mu.Unlock() + if time.Until(until) <= wantRetryAfter { + t.Errorf("the large commit is refused until %s, want for tooLargeRetryAfter", until) + } + + before := upstream.gitRequests.Load() + if err := fetchByID(t, tc.url("repo"), large); err == nil { + t.Fatal("fetched the large commit on a second try") + } + if got := upstream.gitRequests.Load() - before; got != 0 { + t.Errorf("the second try made %d upstream fetch requests, want none", got) + } + if err := fetchByID(t, tc.url("repo"), small); err != nil { + t.Errorf("fetch of another commit by ID: %v", err) + } +} + +// TestWantsFetchedTogetherFailSeparately asks for a good and a bad commit +// in one request. The good one is still fetched, and only the bad one is +// refused for a while. +func TestWantsFetchedTogetherFailSeparately(t *testing.T) { + upstream := newFakeUpstream(t) + work := upstream.newRepo("repo") + runGit(t, upstream.bare(), "config", "uploadpack.allowAnySHA1InWant", "true") + good := upstream.pushPullRequest(work, 1, nil) + bad := hex.EncodeToString(randomBytes(20)) + tc := newTestCache(t, upstream.server.URL, nil) + tc.clone(t, "repo") + + dir := t.TempDir() + runGit(t, dir, "init", "--quiet") + if _, err := tryGit(t, dir, "fetch", "--quiet", tc.url("repo"), good, bad); err == nil { + t.Fatal("fetched a commit the upstream does not have") + } + waitIdle(t, tc.Cache) + if _, err := tryGit(t, filepath.Join(tc.dir, "up", "repo.git"), "cat-file", "-e", good); err != nil { + t.Error("the good commit was not fetched") + } + tc.mu.Lock() + _, goodRefused := tc.failedWants["up/repo.git "+good] + _, badRefused := tc.failedWants["up/repo.git "+bad] + tc.mu.Unlock() + if goodRefused || !badRefused { + t.Errorf("refused good=%v bad=%v, want only the bad commit", goodRefused, badRefused) + } +} + +func TestFailedWantIsNotRetriedAtOnce(t *testing.T) { + upstream := newFakeUpstream(t) + upstream.newRepo("repo") + tc := newTestCache(t, upstream.server.URL, nil) + tc.clone(t, "repo") + missing := hex.EncodeToString(randomBytes(20)) + + before := upstream.gitRequests.Load() + if err := fetchByID(t, tc.url("repo"), missing); err == nil { + t.Fatal("fetched a commit the upstream does not have") + } + if upstream.gitRequests.Load() == before { + t.Fatal("the first request did not ask the upstream") + } + before = upstream.gitRequests.Load() + if err := fetchByID(t, tc.url("repo"), missing); err == nil { + t.Fatal("fetched a commit the upstream does not have") + } + if got := upstream.gitRequests.Load() - before; got != 0 { + t.Errorf("the second request made %d upstream fetch requests, want none", got) + } +} + +func TestWantsAreNotFetchedWhenTurnedOff(t *testing.T) { + upstream := newFakeUpstream(t) + work := upstream.newRepo("repo") + runGit(t, upstream.bare(), "config", "uploadpack.allowAnySHA1InWant", "true") + sha := upstream.pushPullRequest(work, 1, nil) + tc := newTestCache(t, upstream.server.URL, nil) + server := httptest.NewServer(testFront(tc.Cache, upstream.server.URL, false)) + defer server.Close() + url := server.URL + "/git/up/repo" + runGit(t, "", "clone", "--quiet", url, filepath.Join(t.TempDir(), "clone")) + + sessions := upstream.gitSessions.Load() + if err := fetchByID(t, url, sha); err == nil { + t.Error("fetched a commit by ID with FetchWants off") + } + if got := upstream.gitSessions.Load() - sessions; got != 0 { + t.Errorf("upstream saw %d fetches, want none", got) + } +} + +// TestWantsDuringAnOutageAreNotFetchedOneByOne asks for two commits while +// the upstream is down. One failed fetch records both, rather than a fetch +// per commit, each waiting on the upstream. +func TestWantsDuringAnOutageAreNotFetchedOneByOne(t *testing.T) { + upstream := newFakeUpstream(t) + work := upstream.newRepo("repo") + runGit(t, upstream.bare(), "config", "uploadpack.allowAnySHA1InWant", "true") + first := upstream.pushPullRequest(work, 1, nil) + second := upstream.pushPullRequest(work, 2, nil) + tc := newTestCache(t, upstream.server.URL, nil) + tc.clone(t, "repo") + + upstream.status.Store(http.StatusServiceUnavailable) + sessions := upstream.gitSessions.Load() + dir := t.TempDir() + runGit(t, dir, "init", "--quiet") + if _, err := tryGit(t, dir, "fetch", "--quiet", tc.url("repo"), first, second); err == nil { + t.Fatal("fetched commits while the upstream was down") + } + if got := upstream.gitSessions.Load() - sessions; got != 1 { + t.Errorf("upstream saw %d fetches, want 1", got) + } + tc.mu.Lock() + _, firstRefused := tc.failedWants["up/repo.git "+first] + _, secondRefused := tc.failedWants["up/repo.git "+second] + tc.mu.Unlock() + if !firstRefused || !secondRefused { + t.Errorf("refused first=%v second=%v, want both", firstRefused, secondRefused) + } +} + +// TestLargeBatchOfFailedWantsIsNotSplit asks for more unknown commits at +// once than are fetched again one by one, which would hold the repository's +// write lock for a fetch per commit. +func TestLargeBatchOfFailedWantsIsNotSplit(t *testing.T) { + upstream := newFakeUpstream(t) + upstream.newRepo("repo") + tc := newTestCache(t, upstream.server.URL, nil) + tc.clone(t, "repo") + + args := []string{"fetch", "--quiet", tc.url("repo")} + for range maxWantSplit + 1 { + args = append(args, hex.EncodeToString(randomBytes(20))) + } + dir := t.TempDir() + runGit(t, dir, "init", "--quiet") + sessions := upstream.gitSessions.Load() + if _, err := tryGit(t, dir, args...); err == nil { + t.Fatal("fetched commits the upstream does not have") + } + if got := upstream.gitSessions.Load() - sessions; got != 1 { + t.Errorf("upstream saw %d fetches, want 1", got) + } +} diff --git a/internal/handler/git.go b/internal/handler/git.go new file mode 100644 index 0000000..3fbacfd --- /dev/null +++ b/internal/handler/git.go @@ -0,0 +1,215 @@ +package handler + +import ( + "errors" + "net/http" + "net/url" + "strings" + "time" + + "github.com/git-pkgs/proxy/internal/config" + "github.com/git-pkgs/proxy/internal/gitcache" + "github.com/git-pkgs/proxy/internal/metrics" +) + +const ( + gitEcosystem = "git" + + gitInfoRefs = "/info/refs" + gitUploadPack = "/git-upload-pack" + gitReceivePack = "/git-receive-pack" + gitRequestType = "application/x-git-upload-pack-request" + + // gitRequestTimeout replaces the server's read and write timeouts for Git + // requests: a cold clone of a large repository keeps the client waiting + // before the first byte, a large pack takes a while to send, and the rest + // of a request body is read only after any fetch it needs. + gitRequestTimeout = 2 * time.Hour +) + +// GitHandler serves Git repositories read-only over smart HTTP at +// /git/{name}/{repo}, from a local cache of the upstream.git hosts. Clients +// clone and fetch with ordinary git commands, usually through an insteadOf +// rewrite. Pushing and the dumb protocol are refused. +type GitHandler struct { + proxy *Proxy + cache *gitcache.Cache + upstreams map[string]gitUpstream +} + +type gitUpstream struct { + config.GitUpstream + // dir names the upstream's directory in the cache (gitcache.UpstreamDir). + dir string + // fetchWants is off when upstream.auth has credentials for the host. A + // fetch by ID there could reach commits outside the repos allowlist, + // as GitHub serves any commit in a repository's fork network. + fetchWants bool +} + +// NewGitHandler creates a Git handler serving upstream.git from cache. +func NewGitHandler(proxy *Proxy, cache *gitcache.Cache, upstream config.UpstreamConfig) *GitHandler { + h := &GitHandler{proxy: proxy, cache: cache, upstreams: make(map[string]gitUpstream, len(upstream.Git))} + for name, git := range upstream.Git { + git.URL = strings.TrimSuffix(git.URL, "/") + h.upstreams[name] = gitUpstream{ + GitUpstream: git, + dir: gitcache.UpstreamDir(name, git.URL), + fetchWants: !upstream.HasAuthWithin(git.URL), + } + } + return h +} + +// Routes returns the HTTP handler for Git requests. Mount this at /git. +func (h *GitHandler) Routes() http.Handler { + return http.HandlerFunc(h.serve) +} + +// gitRequest is a parsed /git/{name}/{repo}/{endpoint} request. +type gitRequest struct { + infoRefs bool + fetchWants bool + // key names the repository in the cache: {name}-{URL hash}/{repo}.git, + // with any .git the client gave removed from repo first, so both + // spellings share one copy. + key string + // repoURL is the upstream repository, spelled as the client spelled it. + repoURL string +} + +func (h *GitHandler) serve(w http.ResponseWriter, r *http.Request) { + req, status, msg := h.parse(r) + if status != 0 { + http.Error(w, msg, status) + return + } + deadline := time.Now().Add(gitRequestTimeout) + rc := http.NewResponseController(w) + if err := errors.Join(rc.SetReadDeadline(deadline), rc.SetWriteDeadline(deadline)); err != nil { + h.proxy.Logger.Warn("git: cannot extend the request deadlines", "error", err) + } + // upload-pack writes its response while its stdin is still attached to + // the request body. Without full duplex, an HTTP/1 server reads off and + // closes whatever body is left once the response is flushed. + if !req.infoRefs { + if err := rc.EnableFullDuplex(); err != nil { + h.proxy.Logger.Warn("git: cannot enable full duplex", "error", err) + } + } + + lease, err := h.cache.Open(r.Context(), req.key, req.repoURL, + gitcache.OpenOptions{Refresh: req.infoRefs, FetchWants: req.fetchWants}) + if err != nil { + h.serveError(w, r, err) + return + } + defer lease.Release() + if !req.infoRefs { + lease.ServeUploadPack(w, r) + return + } + if lease.Hit() { + metrics.RecordArtifactCacheHit(gitEcosystem) + } else { + metrics.RecordArtifactCacheMiss(gitEcosystem) + } + lease.ServeInfoRefs(w, r) +} + +func (h *GitHandler) parse(r *http.Request) (gitRequest, int, string) { + path := strings.TrimPrefix(r.URL.Path, "/") + var req gitRequest + switch { + case strings.HasSuffix(path, gitInfoRefs): + path = strings.TrimSuffix(path, gitInfoRefs) + req.infoRefs = true + switch r.URL.Query().Get("service") { + case "git-upload-pack": + case "git-receive-pack": + return req, http.StatusForbidden, "pushing is not supported" + default: + return req, http.StatusForbidden, "only the smart HTTP protocol is supported" + } + if r.Method != http.MethodGet { + return req, http.StatusMethodNotAllowed, "method not allowed" + } + case strings.HasSuffix(path, gitUploadPack): + path = strings.TrimSuffix(path, gitUploadPack) + if r.Method != http.MethodPost { + return req, http.StatusMethodNotAllowed, "method not allowed" + } + if r.Header.Get("Content-Type") != gitRequestType { + return req, http.StatusUnsupportedMediaType, "unsupported content type" + } + case strings.HasSuffix(path, gitReceivePack): + return req, http.StatusForbidden, "pushing is not supported" + default: + return req, http.StatusNotFound, "not found" + } + + name, repoPath, _ := strings.Cut(path, "/") + upstream, ok := h.upstreams[name] + if !ok { + return req, http.StatusNotFound, "unknown upstream" + } + segments := strings.Split(repoPath, "/") + last := len(segments) - 1 + suffix := "" + if trimmed, ok := strings.CutSuffix(segments[last], ".git"); ok { + segments[last], suffix = trimmed, ".git" + } + escaped := make([]string, len(segments)) + for i, segment := range segments { + if !validGitSegment(segment, i == last) { + return req, http.StatusNotFound, "invalid repository path" + } + escaped[i] = url.PathEscape(segment) + } + repo := strings.Join(segments, "/") + if !upstream.AllowsRepo(repo) { + // As for a missing repository, so a prober learns nothing about + // the allowlist. + return req, http.StatusNotFound, "repository not found" + } + req.key = upstream.dir + "/" + repo + ".git" + req.repoURL = upstream.URL + "/" + strings.Join(escaped, "/") + suffix + req.fetchWants = upstream.fetchWants + return req, 0, "" +} + +// validGitSegment checks one segment of a repository path, which becomes both +// part of a directory under the cache and part of the upstream URL. Only the +// last segment may end in .git, so one repository's directory never sits +// inside another's. +func validGitSegment(segment string, last bool) bool { + if segment == "" || segment == "." || segment == ".." || (!last && strings.HasSuffix(segment, ".git")) { + return false + } + for _, r := range segment { + if r < ' ' || r == 0x7f || r == '\\' { + return false + } + } + return true +} + +// serveError answers a request whose repository could not be opened. The +// cache has already logged why. +func (h *GitHandler) serveError(w http.ResponseWriter, r *http.Request, err error) { + if r.Context().Err() != nil { + return + } + switch { + case errors.Is(err, gitcache.ErrNotFound): + http.Error(w, "repository not found", http.StatusNotFound) + case errors.Is(err, gitcache.ErrRefused): + http.Error(w, "upstream refused the request", http.StatusBadGateway) + case errors.Is(err, gitcache.ErrTooLarge): + http.Error(w, "repository is larger than git.max_repo_size or git.max_size", http.StatusInsufficientStorage) + case errors.Is(err, gitcache.ErrCacheFull): + http.Error(w, "git cache is full", http.StatusInsufficientStorage) + default: + http.Error(w, "upstream unavailable", http.StatusBadGateway) + } +} diff --git a/internal/handler/git_test.go b/internal/handler/git_test.go new file mode 100644 index 0000000..ec0b630 --- /dev/null +++ b/internal/handler/git_test.go @@ -0,0 +1,247 @@ +package handler + +import ( + "fmt" + "io" + "log/slog" + "net/http" + "net/http/cgi" + "net/http/httptest" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/git-pkgs/proxy/internal/config" + "github.com/git-pkgs/proxy/internal/gitcache" +) + +func TestParseGitRequest(t *testing.T) { + h := NewGitHandler(nil, nil, config.UpstreamConfig{Git: map[string]config.GitUpstream{ + "up": {URL: "https://git.example/r/"}, + "limited": {URL: "https://git.example", Repos: []string{"allowed/*"}}, + }}) + up, limited := h.upstreams["up"].dir, h.upstreams["limited"].dir + tests := []struct { + method, target string + status int + key, repoURL string + }{ + {"GET", "/up/repo.git/info/refs?service=git-upload-pack", 0, up + "/repo.git", "https://git.example/r/repo.git"}, + {"GET", "/up/a/b/info/refs?service=git-upload-pack", 0, up + "/a/b.git", "https://git.example/r/a/b"}, + {"POST", "/up/a/b%20c/git-upload-pack", 0, up + "/a/b c.git", "https://git.example/r/a/b%20c"}, + {"GET", "/up/a.git/b/info/refs?service=git-upload-pack", http.StatusNotFound, "", ""}, + {"GET", "/up/a/../b/info/refs?service=git-upload-pack", http.StatusNotFound, "", ""}, + {"GET", "/up/a//b/info/refs?service=git-upload-pack", http.StatusNotFound, "", ""}, + {"GET", "/up/.git/info/refs?service=git-upload-pack", http.StatusNotFound, "", ""}, + {"GET", "/other/repo/info/refs?service=git-upload-pack", http.StatusNotFound, "", ""}, + {"GET", "/up/info/refs?service=git-upload-pack", http.StatusNotFound, "", ""}, + {"GET", "/up/repo/info/refs?service=git-receive-pack", http.StatusForbidden, "", ""}, + {"POST", "/up/repo/git-receive-pack", http.StatusForbidden, "", ""}, + {"GET", "/up/repo/info/refs", http.StatusForbidden, "", ""}, + {"GET", "/up/repo/objects/info/packs", http.StatusNotFound, "", ""}, + {"POST", "/up/repo/info/refs?service=git-upload-pack", http.StatusMethodNotAllowed, "", ""}, + {"GET", "/up/repo/git-upload-pack", http.StatusMethodNotAllowed, "", ""}, + {"GET", "/limited/allowed/repo/info/refs?service=git-upload-pack", 0, limited + "/allowed/repo.git", "https://git.example/allowed/repo"}, + {"GET", "/limited/other/repo/info/refs?service=git-upload-pack", http.StatusNotFound, "", ""}, + } + for _, tt := range tests { + t.Run(tt.method+" "+tt.target, func(t *testing.T) { + r := httptest.NewRequest(tt.method, tt.target, nil) + r.Header.Set("Content-Type", gitRequestType) + req, status, _ := h.parse(r) + if status != tt.status { + t.Fatalf("status %d, want %d", status, tt.status) + } + if status == 0 && (req.key != tt.key || req.repoURL != tt.repoURL) { + t.Errorf("key %q, URL %q; want %q, %q", req.key, req.repoURL, tt.key, tt.repoURL) + } + }) + } + + r := httptest.NewRequest(http.MethodPost, "/up/repo/git-upload-pack", nil) + if _, status, _ := h.parse(r); status != http.StatusUnsupportedMediaType { + t.Errorf("POST without the request content type: status %d, want 415", status) + } +} + +// TestGitUpstreamCacheDirectory checks the cache directory follows the +// upstream's URL, so repointing an upstream does not serve the old host's +// repositories. +func TestGitUpstreamCacheDirectory(t *testing.T) { + dir := func(url string) string { + h := NewGitHandler(nil, nil, config.UpstreamConfig{Git: map[string]config.GitUpstream{"up": {URL: url}}}) + return h.upstreams["up"].dir + } + if a, b := dir("https://git.example"), dir("https://other.example"); a == b || !strings.HasPrefix(a, "up-") { + t.Errorf("directories %q and %q, want distinct ones named after the upstream", a, b) + } + if a, b := dir("https://git.example"), dir("https://git.example/"); a != b { + t.Errorf("a trailing slash changed the directory: %q, %q", a, b) + } +} + +// TestGitFetchByIDOffWithCredentials checks commits are not fetched by ID +// for an upstream that upstream.auth has credentials for: there a fetch by +// ID could reach repositories outside its repos allowlist. +func TestGitFetchByIDOffWithCredentials(t *testing.T) { + h := NewGitHandler(nil, nil, config.UpstreamConfig{ + Git: map[string]config.GitUpstream{ + "public": {URL: "https://github.com"}, + "private": {URL: "https://git.example", Repos: []string{"team/*"}}, + "scoped": {URL: "https://other.example", Repos: []string{"team/*"}}, + }, + Auth: map[string]config.AuthConfig{ + "https://git.example": {Type: "bearer", Token: "t"}, + "https://other.example/team": {Type: "bearer", Token: "t"}, + }, + }) + for name, want := range map[string]bool{"public": true, "private": false, "scoped": false} { + if got := h.upstreams[name].fetchWants; got != want { + t.Errorf("%s: fetchWants = %v, want %v", name, got, want) + } + } +} + +func runGitCommand(t *testing.T, dir string, args ...string) string { + t.Helper() + cmd := exec.Command("git", args...) + cmd.Dir = dir + cmd.Env = []string{ + "PATH=" + os.Getenv("PATH"), "HOME=" + t.TempDir(), "GIT_CONFIG_NOSYSTEM=1", + "GIT_AUTHOR_NAME=test", "GIT_AUTHOR_EMAIL=test@example.com", + "GIT_COMMITTER_NAME=test", "GIT_COMMITTER_EMAIL=test@example.com", + } + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("git %s: %v\n%s", strings.Join(args, " "), err, out) + } + return strings.TrimSpace(string(out)) +} + +// gitTestSetup is a GitHandler in front of a cache of one upstream +// repository, "repo", served by git http-backend. +type gitTestSetup struct { + work string + server *httptest.Server + logs *syncLog + // slow delays every upstream request by half a second. + slow atomic.Bool +} + +func newGitTestSetup(t *testing.T, readTimeout time.Duration) *gitTestSetup { + t.Helper() + execPath := runGitCommand(t, "", "--exec-path") + backendPath := "" + for _, name := range []string{"git-http-backend", "git-http-backend.exe"} { + if path := filepath.Join(execPath, name); fileExists(path) { + backendPath = path + } + } + if backendPath == "" { + t.Skip("git http-backend is not installed") + } + root := t.TempDir() + bare := filepath.Join(root, "repo.git") + runGitCommand(t, "", "init", "--quiet", "--bare", "--initial-branch=main", bare) + runGitCommand(t, bare, "config", "uploadpack.allowAnySHA1InWant", "true") + g := &gitTestSetup{work: t.TempDir(), logs: &syncLog{}} + runGitCommand(t, "", "clone", "--quiet", bare, g.work) + runGitCommand(t, g.work, "commit", "--quiet", "--allow-empty", "-m", "first") + runGitCommand(t, g.work, "push", "--quiet", "origin", "HEAD:main") + + backend := &cgi.Handler{ + Path: backendPath, + Env: []string{"PATH=" + os.Getenv("PATH"), "GIT_PROJECT_ROOT=" + root, "GIT_HTTP_EXPORT_ALL=1"}, + } + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if g.slow.Load() { + time.Sleep(500 * time.Millisecond) + } + backend.ServeHTTP(w, r) + })) + t.Cleanup(upstream.Close) + + logger := slog.New(slog.NewTextHandler(g.logs, nil)) + cache, err := gitcache.New(gitcache.Options{ + CacheDir: t.TempDir(), + RefreshWait: 5 * time.Second, + StaleMaxAge: time.Hour, + Client: &http.Client{}, + Logger: logger, + Upstreams: map[string]string{"up": gitcache.UpstreamDir("up", upstream.URL)}, + }) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = cache.Close() }) + h := NewGitHandler(&Proxy{Logger: logger}, cache, config.UpstreamConfig{Git: map[string]config.GitUpstream{"up": {URL: upstream.URL}}}) + g.server = httptest.NewUnstartedServer(http.StripPrefix("/git", h.Routes())) + g.server.Config.ReadTimeout = readTimeout + g.server.Start() + t.Cleanup(g.server.Close) + return g +} + +func (g *gitTestSetup) url() string { + return g.server.URL + "/git/up/repo" +} + +type syncLog struct { + mu sync.Mutex + buf strings.Builder +} + +func (l *syncLog) Write(p []byte) (int, error) { + l.mu.Lock() + defer l.mu.Unlock() + return l.buf.Write(p) +} + +func (l *syncLog) String() string { + l.mu.Lock() + defer l.mu.Unlock() + return l.buf.String() +} + +// TestGitRequestBodyOutlastsServerReadTimeout sends a protocol v0 request +// whose haves come after a want the cache must fetch first, so upload-pack +// reads them after the server's read timeout has passed. +func TestGitRequestBodyOutlastsServerReadTimeout(t *testing.T) { + g := newGitTestSetup(t, 200*time.Millisecond) + runGitCommand(t, g.work, "commit", "--quiet", "--allow-empty", "-m", "pull request") + runGitCommand(t, g.work, "push", "--quiet", "origin", "HEAD:refs/pull/1/head") + sha := runGitCommand(t, g.work, "rev-parse", "HEAD") + runGitCommand(t, "", "clone", "--quiet", g.url(), filepath.Join(t.TempDir(), "clone")) + + var body strings.Builder + body.WriteString(pktLine("want "+sha+" side-band-64k\n") + "0000") + for i := range 2000 { + body.WriteString(pktLine(fmt.Sprintf("have %040x\n", i+1))) + } + body.WriteString(pktLine("done\n")) + + g.slow.Store(true) + resp, err := http.Post(g.url()+"/git-upload-pack", gitRequestType, strings.NewReader(body.String())) + if err != nil { + t.Fatal(err) + } + defer func() { _ = resp.Body.Close() }() + out, _ := io.ReadAll(resp.Body) + if resp.StatusCode != http.StatusOK || !strings.Contains(string(out), "PACK") { + t.Errorf("status %d, no pack in %d bytes: %.200q", resp.StatusCode, len(out), out) + } +} + +func pktLine(payload string) string { + return fmt.Sprintf("%04x%s", len(payload)+4, payload) +} + +func fileExists(path string) bool { + _, err := os.Stat(path) + return err == nil +} diff --git a/internal/server/git_test.go b/internal/server/git_test.go new file mode 100644 index 0000000..d52f1a9 --- /dev/null +++ b/internal/server/git_test.go @@ -0,0 +1,160 @@ +package server + +import ( + "bytes" + "context" + "errors" + "log/slog" + "net" + "net/http" + "net/http/cgi" + "net/http/httptest" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/git-pkgs/proxy/internal/config" +) + +func gitTestEnv(home string) []string { + return []string{ + "PATH=" + os.Getenv("PATH"), + "HOME=" + home, + "GIT_CONFIG_NOSYSTEM=1", + "GIT_TERMINAL_PROMPT=0", + "GIT_AUTHOR_NAME=test", "GIT_AUTHOR_EMAIL=test@example.com", + "GIT_COMMITTER_NAME=test", "GIT_COMMITTER_EMAIL=test@example.com", + } +} + +func gitCommand(t *testing.T, dir string, args ...string) string { + t.Helper() + cmd := exec.Command("git", args...) + cmd.Dir = dir + cmd.Env = gitTestEnv(t.TempDir()) + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("git %s: %v\n%s", strings.Join(args, " "), err, out) + } + return strings.TrimSpace(string(out)) +} + +// gitHTTPBackend finds git http-backend, which serves the test upstream, and +// skips the test where git is installed without it. +func gitHTTPBackend(t *testing.T) string { + t.Helper() + dir := gitCommand(t, "", "--exec-path") + for _, name := range []string{"git-http-backend", "git-http-backend.exe"} { + if _, err := os.Stat(filepath.Join(dir, name)); err == nil { + return filepath.Join(dir, name) + } + } + t.Skip("git http-backend is not installed") + return "" +} + +// syncBuffer is a log destination the server and the test share. +type syncBuffer struct { + mu sync.Mutex + buf bytes.Buffer +} + +func (b *syncBuffer) Write(p []byte) (int, error) { + b.mu.Lock() + defer b.mu.Unlock() + return b.buf.Write(p) +} + +func (b *syncBuffer) String() string { + b.mu.Lock() + defer b.mu.Unlock() + return b.buf.String() +} + +// TestGitRouteServesClonesWithUpstreamCredentials clones through the whole +// server, with credentials from upstream.auth that only the proxy holds. +func TestGitRouteServesClonesWithUpstreamCredentials(t *testing.T) { + root := t.TempDir() + bare := filepath.Join(root, "team", "repo.git") + gitCommand(t, "", "init", "--quiet", "--bare", "--initial-branch=main", bare) + work := t.TempDir() + gitCommand(t, "", "clone", "--quiet", bare, work) + if err := os.WriteFile(filepath.Join(work, "file.txt"), []byte("hello\n"), 0o600); err != nil { + t.Fatal(err) + } + gitCommand(t, work, "add", "file.txt") + gitCommand(t, work, "commit", "--quiet", "-m", "first") + gitCommand(t, work, "push", "--quiet", "origin", "HEAD:main") + head := gitCommand(t, work, "rev-parse", "HEAD") + + backend := &cgi.Handler{ + Path: gitHTTPBackend(t), + Env: append(gitTestEnv(t.TempDir()), "GIT_PROJECT_ROOT="+root, "GIT_HTTP_EXPORT_ALL=1"), + } + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("Authorization") != "Bearer git-secret" { + http.Error(w, "unauthorized", http.StatusUnauthorized) + return + } + backend.ServeHTTP(w, r) + })) + defer upstream.Close() + + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + tempDir := t.TempDir() + cfg := config.Default() + cfg.Listen = listener.Addr().String() + cfg.BaseURL = "http://" + cfg.Listen + cfg.Database.Path = filepath.Join(tempDir, "proxy.db") + cfg.Storage.URL = "file://" + filepath.Join(tempDir, "artifacts") + cfg.Upstream.AllowLoopback = true + cfg.Upstream.Git = map[string]config.GitUpstream{"private": {URL: upstream.URL, Repos: []string{"team/*"}}} + cfg.Upstream.Auth = map[string]config.AuthConfig{upstream.URL: {Type: "bearer", Token: "git-secret"}} + cfg.Git.CacheDir = filepath.Join(tempDir, "git") + if err := cfg.Validate(); err != nil { + t.Fatalf("validating config: %v", err) + } + + var logs syncBuffer + proxyServer, err := New(cfg, slog.New(slog.NewTextHandler(&logs, nil)), BuildInfo{Version: "test", Commit: "test"}) + if err != nil { + t.Fatalf("creating server: %v", err) + } + startErr := make(chan error, 1) + go func() { startErr <- proxyServer.Start(listener) }() + defer func() { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if err := proxyServer.Shutdown(ctx); err != nil { + t.Errorf("shutting down server: %v", err) + } + if err := <-startErr; !errors.Is(err, http.ErrServerClosed) { + t.Errorf("Start() error = %v, want %v", err, http.ErrServerClosed) + } + }() + + clone := filepath.Join(t.TempDir(), "clone") + gitCommand(t, "", "clone", "--quiet", cfg.BaseURL+"/git/private/team/repo.git", clone) + if got := gitCommand(t, clone, "rev-parse", "HEAD"); got != head { + t.Errorf("HEAD = %s, want %s", got, head) + } + if strings.Contains(logs.String(), "request deadlines") { + t.Errorf("request deadlines not extended:\n%s", logs.String()) + } + + resp, err := http.Get(cfg.BaseURL + "/git/private/other/repo.git/info/refs?service=git-upload-pack") + if err != nil { + t.Fatal(err) + } + _ = resp.Body.Close() + if resp.StatusCode != http.StatusNotFound { + t.Errorf("repository outside repos: status %d, want 404", resp.StatusCode) + } +} diff --git a/internal/server/middleware.go b/internal/server/middleware.go index 0de0d1b..0de0dba 100644 --- a/internal/server/middleware.go +++ b/internal/server/middleware.go @@ -107,7 +107,7 @@ func requestEcosystem(path string) string { switch segment { case "npm", "cargo", "hex", "pub", "pypi", "maven", "gradle", "nuget", "conan", "chef", "conda", "cran", "julia", "debian", "rpm", "arch", - "helm", "homebrew", "generic", "swift": + "helm", "homebrew", "generic", "swift", "git": return segment case "apk": return "alpine" diff --git a/internal/server/middleware_test.go b/internal/server/middleware_test.go index 7c9eab9..b9c67c7 100644 --- a/internal/server/middleware_test.go +++ b/internal/server/middleware_test.go @@ -314,7 +314,7 @@ func TestRequestEcosystemCoversEveryMountedRoute(t *testing.T) { "/conan/x": "conan", "/chef/x": "chef", "/conda/x": "conda", "/cran/x": "cran", "/julia/x": "julia", "/swift/x": "swift", "/v2/x": "oci", "/apk/x": "alpine", "/arch/x": "arch", "/debian/x": "debian", "/rpm/x": "rpm", - "/helm/x": "helm", "/homebrew/x": "homebrew", "/generic/x": "generic", + "/helm/x": "helm", "/homebrew/x": "homebrew", "/generic/x": "generic", "/git/x": "git", } for path, want := range mounts { if got := requestEcosystem(path); got != want { diff --git a/internal/server/server.go b/internal/server/server.go index b8a7a8b..8a0a683 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -74,6 +74,7 @@ import ( "github.com/git-pkgs/proxy/internal/database" "github.com/git-pkgs/proxy/internal/denylist" "github.com/git-pkgs/proxy/internal/enrichment" + "github.com/git-pkgs/proxy/internal/gitcache" "github.com/git-pkgs/proxy/internal/handler" upstreamhttp "github.com/git-pkgs/proxy/internal/httpclient" "github.com/git-pkgs/proxy/internal/metrics" @@ -122,6 +123,7 @@ type Server struct { metaStats metadataStatsCache sources sourceTracker scheduler *mirror.Scheduler + git *gitcache.Cache // browseLimits overrides defaultBrowseLimits when set; tests use it. browseLimits archives.StreamOptions @@ -270,6 +272,27 @@ func (s *Server) serve(listener net.Listener) error { proxy.DirectServeBaseURL = s.cfg.Storage.DirectServeBaseURL proxy.StreamArtifacts = !s.cfg.Storage.CacheArtifacts + if len(s.cfg.Upstream.Git) > 0 { + upstreamDirs := make(map[string]string, len(s.cfg.Upstream.Git)) + for name, upstream := range s.cfg.Upstream.Git { + upstreamDirs[name] = gitcache.UpstreamDir(name, upstream.URL) + } + gitCache, err := gitcache.New(gitcache.Options{ + CacheDir: s.cfg.Git.CacheDir, + MaxSize: s.cfg.Git.MaxSizeBytes(), + MaxRepoSize: s.cfg.Git.MaxRepoSizeBytes(), + RefreshWait: s.cfg.Git.RefreshWaitDuration(), + StaleMaxAge: s.cfg.Git.StaleMaxAgeDuration(), + Client: artifactClient, + Logger: s.logger, + Upstreams: upstreamDirs, + }) + if err != nil { + return fmt.Errorf("configuring git cache: %w", err) + } + s.git = gitCache + } + // Create router with Chi r := chi.NewRouter() @@ -496,6 +519,10 @@ func (s *Server) mountProtocolHandlers(r chi.Router, proxy *handler.Proxy) { r.Mount("/debian", http.StripPrefix("/debian", debianHandler.Routes())) r.Mount("/rpm", http.StripPrefix("/rpm", rpmHandler.Routes())) r.Mount("/generic", http.StripPrefix("/generic", genericHandler.Routes())) + if s.git != nil { + gitHandler := handler.NewGitHandler(proxy, s.git, s.cfg.Upstream) + r.Mount("/git", http.StripPrefix("/git", gitHandler.Routes())) + } } // configureScanning builds the scanner group from cfg and wires it into @@ -682,6 +709,12 @@ func (s *Server) Shutdown(ctx context.Context) error { s.waitForMirrorSchedules(ctx) + if s.git != nil { + if err := s.git.Close(); err != nil { + errs = append(errs, fmt.Errorf("git cache close: %w", err)) + } + } + if s.storage != nil { if err := s.storage.Close(); err != nil { errs = append(errs, fmt.Errorf("storage close: %w", err)) From d4addfe26eeb0244f43e265f265eea477df517c4 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Wed, 21 Oct 2026 15:43:32 +0200 Subject: [PATCH 341/633] Cut a patch release on the 1st of each month A scheduled workflow tags the next patch version when main is green, something has merged since the last tag and no pull request labelled breaking was merged. It dispatches release.yml and publish.yml on the new tag, since a tag created with GITHUB_TOKEN doesn't start them. Scheduled runs are dry runs until the MONTHLY_RELEASE variable is true. Closes #532 --- .github/workflows/monthly-release.yml | 109 ++++++++++++++++++++++++++ .github/workflows/release.yml | 3 + CONTRIBUTING.md | 8 ++ 3 files changed, 120 insertions(+) create mode 100644 .github/workflows/monthly-release.yml diff --git a/.github/workflows/monthly-release.yml b/.github/workflows/monthly-release.yml new file mode 100644 index 0000000..6afee9d --- /dev/null +++ b/.github/workflows/monthly-release.yml @@ -0,0 +1,109 @@ +name: Monthly patch release + +# Cuts a patch release on the 1st of each month when main is green and +# something has merged since the last release. Scheduled runs only report +# what they would do until the repository variable MONTHLY_RELEASE is "true". + +on: + schedule: + - cron: "0 6 1 * *" + workflow_dispatch: + inputs: + dry_run: + description: Only report what would be released + type: boolean + default: true + +permissions: {} + +concurrency: + group: monthly-release + +jobs: + release: + runs-on: ubuntu-latest + permissions: + contents: write + actions: write + checks: read + pull-requests: read + env: + GH_TOKEN: ${{ github.token }} + REPO: ${{ github.repository }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: main + fetch-depth: 0 + persist-credentials: false + + - name: Decide whether to release + id: plan + env: + EVENT: ${{ github.event_name }} + INPUT_DRY_RUN: ${{ inputs.dry_run }} + ENABLED: ${{ vars.MONTHLY_RELEASE }} + run: | + set -euo pipefail + if [ "$EVENT" = workflow_dispatch ]; then + dry_run=$INPUT_DRY_RUN + elif [ "$ENABLED" = true ]; then + dry_run=false + else + dry_run=true + fi + echo "dry_run=$dry_run" >> "$GITHUB_OUTPUT" + + sha=$(git rev-parse HEAD) + last=$(git describe --tags --abbrev=0 --match 'v[0-9]*.[0-9]*.[0-9]*' --exclude '*-*') + merged=$(git rev-list --count "$last..$sha") + if [ "$merged" -eq 0 ]; then + echo "Nothing merged since $last" + exit 0 + fi + + # Every check on main's HEAD must have finished green. This run + # shows up as a check on the same commit, so it is left out. + bad=$(gh api --paginate "repos/$REPO/commits/$sha/check-runs?per_page=100" \ + --jq ".check_runs[] + | select(.html_url | contains(\"/actions/runs/$GITHUB_RUN_ID/\") | not) + | select(.status != \"completed\" + or (.conclusion | IN(\"success\", \"skipped\", \"neutral\") | not)) + | .name") + if [ -n "$bad" ]; then + echo "::warning::main ($sha) is not green, skipping: $(echo "$bad" | sort -u | paste -sd, -)" + exit 0 + fi + + since=$(git log -1 --format=%cI "$last") + breaking=$(gh pr list -R "$REPO" --state merged --base main \ + --search "merged:>$since label:breaking" --json number --jq '.[].number') + if [ -n "$breaking" ]; then + echo "::warning::breaking changes merged since $last (#$(echo "$breaking" | paste -sd, - | sed 's/,/, #/g')), release a new minor version by hand" + exit 0 + fi + + IFS=. read -r major minor patch <<<"${last#v}" + tag="v$major.$minor.$((patch + 1))" + echo "$merged commits since $last, next release $tag at $sha" + echo "tag=$tag" >> "$GITHUB_OUTPUT" + echo "sha=$sha" >> "$GITHUB_OUTPUT" + + # A tag pushed with GITHUB_TOKEN doesn't start other workflows, so the + # release workflows are dispatched on the new tag instead. + - name: Tag and start the release workflows + if: steps.plan.outputs.tag != '' + env: + TAG: ${{ steps.plan.outputs.tag }} + SHA: ${{ steps.plan.outputs.sha }} + DRY_RUN: ${{ steps.plan.outputs.dry_run }} + run: | + set -euo pipefail + if [ "$DRY_RUN" = true ]; then + echo "Dry run: would tag $TAG at $SHA and start release.yml and publish.yml" + exit 0 + fi + gh api "repos/$REPO/git/refs" -f ref="refs/tags/$TAG" -f sha="$SHA" >/dev/null + gh workflow run release.yml -R "$REPO" --ref "$TAG" + gh workflow run publish.yml -R "$REPO" --ref "$TAG" + echo "Tagged $TAG at $SHA" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 5d32181..dddfef0 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -4,6 +4,8 @@ on: push: tags: - "v*" + # Started by monthly-release.yml on the tag it creates. + workflow_dispatch: permissions: contents: write @@ -11,6 +13,7 @@ permissions: jobs: release: + if: github.ref_type == 'tag' runs-on: ubuntu-latest steps: diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 2c69728..f680251 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -212,6 +212,14 @@ Add PyPI registry support - Add tests for wheel and sdist downloads ``` +## Releases + +A patch release is cut on the 1st of each month by `.github/workflows/monthly-release.yml` when `main` is green and something has merged since the last tag. It is skipped if a pull request labelled `breaking` was merged since then: label pull requests that change behaviour users rely on, and release those as a new minor version by hand. + +Scheduled runs only report what they would release until the repository variable `MONTHLY_RELEASE` is set to `true`. The workflow can also be started by hand from the Actions tab, as a dry run by default. + +To release by hand, push a `v*` tag. + ## Questions? Open an issue on GitHub if you have questions or need help getting started. From 0e25e46732ffecffee8cec7fd5938cad3d979ea3 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Thu, 22 Oct 2026 09:14:33 +0200 Subject: [PATCH 342/633] Keep the stale fallback for 404 and 401/403, leave npm to #503 Serving cached metadata when the upstream refuses is documented for generic upstreams, and GitHub rate limits are 403s, so the fallback stays as it is. #503 handles npm's definite answers. What is left here is the stale warning for Composer, Chef and hosted npm metadata, which went through the rewrite cache without it. --- docs/configuration.md | 2 +- internal/handler/handler.go | 14 -- internal/handler/npm.go | 9 +- internal/handler/stale_metadata_test.go | 230 ++++++------------------ 4 files changed, 62 insertions(+), 193 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 3bffadb..dccdf26 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -987,7 +987,7 @@ Or via environment variable: `PROXY_METADATA_TTL=10m`. Set to `"0"` to always revalidate with upstream (ETag-based conditional requests still avoid re-downloading unchanged content). -When upstream is unreachable and the cached entry is past its TTL, the proxy serves the stale cached copy with a `Warning: 110 - "Response is Stale"` header so clients can tell the data may be outdated. This covers timeouts, connection errors, 5xx and 429 responses. A 404 or a 401/403 from the upstream is an answer rather than an outage, so it is not covered: a removed package answers 404, and credentials the upstream rejects answer 502, instead of the client getting a cached version list that no longer exists or can no longer be read. +When upstream is unreachable and the cached entry is past its TTL, the proxy serves the stale cached copy with a `Warning: 110 - "Response is Stale"` header so clients can tell the data may be outdated. ### Metadata size limit diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 31451d3..6419d17 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -1197,10 +1197,6 @@ func JSONError(w http.ResponseWriter, status int, message string) { _, _ = fmt.Fprintf(w, `{"error":%q}`, message) } -// ErrUpstreamRefused indicates the upstream answered 401 or 403: the -// credentials the proxy sends are missing, wrong or no longer allowed. -var ErrUpstreamRefused = errors.New("upstream refused the request") - // staleWarning marks a response served from the metadata cache because the // upstream could not be reached. const staleWarning = `110 - "Response is Stale"` @@ -1406,13 +1402,6 @@ func (p *Proxy) fetchMetadataFromUpstream(ctx context.Context, ecosystem, cacheK return metadataResult{body: meta.body, contentType: meta.contentType, contentEncoding: meta.contentEncoding, digest: digest} } - // A 404 or 401/403 is the upstream's answer, not an outage: the package - // is gone, or the proxy may no longer read it. Serving the cached copy - // would hide that from the client, so only other failures fall back. - if errors.Is(err, ErrUpstreamNotFound) || errors.Is(err, ErrUpstreamRefused) { - return metadataResult{err: err} - } - // Upstream failed -- fall back to cache if available if !p.CacheMetadata || entry == nil { return metadataResult{err: fmt.Errorf("upstream failed and no cached metadata: %w", err)} @@ -1628,9 +1617,6 @@ func (p *Proxy) fetchUpstreamMetadata(ctx context.Context, upstreamURL string, e body, _ := io.ReadAll(io.LimitReader(resp.Body, upstreamNotFoundBodyLimit)) return nil, &upstreamNotFoundError{body: body} } - if resp.StatusCode == http.StatusUnauthorized || resp.StatusCode == http.StatusForbidden { - return nil, fmt.Errorf("%w: upstream returned %d", ErrUpstreamRefused, resp.StatusCode) - } if resp.StatusCode != http.StatusOK { return nil, fmt.Errorf("upstream returned %d", resp.StatusCode) } diff --git a/internal/handler/npm.go b/internal/handler/npm.go index af7bc2e..6530f5a 100644 --- a/internal/handler/npm.go +++ b/internal/handler/npm.go @@ -263,14 +263,12 @@ func (h *NPMHandler) handlePackageMetadata(w http.ResponseWriter, r *http.Reques } // If rewriting fails, just proxy the original h.proxy.Logger.Warn("failed to rewrite metadata, proxying original", "error", err) - markStale(w, upstream.stale) w.Header().Set(headerContentType, contentTypeJSON) w.WriteHeader(http.StatusOK) _, _ = w.Write(upstream.body) return } - markStale(w, upstream.stale) serveRewrittenMetadata(w, r, rewritten) } @@ -298,8 +296,8 @@ func (h *NPMHandler) packumentAccept() string { // filtered packument so cooldown and dist-tags match the root metadata response. func (h *NPMHandler) handleFilteredNPMVersionMetadata(w http.ResponseWriter, r *http.Request, packageName, specifier, accept, registryURL, cacheKey, metadataCacheKey string) { _, packumentPath := npmMetadataPath(packageName) - upstream := h.proxy.fetchMetadataDocument(r.Context(), "npm", cacheKey, registryURL+"/"+packumentPath, accept) - if err := upstream.err; err != nil { + packument, _, err := h.proxy.FetchOrCacheMetadata(r.Context(), "npm", cacheKey, registryURL+"/"+packumentPath, accept) + if err != nil { if errors.Is(err, ErrUpstreamNotFound) { JSONError(w, http.StatusNotFound, ErrPackageNotFound.Error()) return @@ -312,7 +310,7 @@ func (h *NPMHandler) handleFilteredNPMVersionMetadata(w http.ResponseWriter, r * return } - rewritten, err := h.proxy.cachedRewrite(r.Context(), "npm", h.proxyURL, metadataCacheKey, upstream.body, "", func(b []byte) ([]byte, []string, error) { + rewritten, err := h.proxy.cachedRewrite(r.Context(), "npm", h.proxyURL, metadataCacheKey, packument, "", func(b []byte) ([]byte, []string, error) { out, err := h.rewriteNPMVersionFromPackument(packageName, specifier, b) return out, nil, err }) @@ -337,7 +335,6 @@ func (h *NPMHandler) handleFilteredNPMVersionMetadata(w http.ResponseWriter, r * return } - markStale(w, upstream.stale) w.Header().Set(headerContentType, contentTypeJSON) w.WriteHeader(http.StatusOK) _, _ = w.Write(rewritten.body) diff --git a/internal/handler/stale_metadata_test.go b/internal/handler/stale_metadata_test.go index 1fd8238..9cd8e75 100644 --- a/internal/handler/stale_metadata_test.go +++ b/internal/handler/stale_metadata_test.go @@ -3,205 +3,91 @@ package handler import ( "net/http" "net/http/httptest" - "strings" "sync/atomic" "testing" "time" ) -const stalePackument = `{"name":"left-pad","versions":{"1.0.0":{"name":"left-pad","version":"1.0.0",` + - `"dist":{"tarball":"https://registry.npmjs.org/left-pad/-/left-pad-1.0.0.tgz"}}}}` - -// npmUpstreamThen serves the packument once, with an ETag, and answers every -// later request with then. -func npmUpstreamThen(t *testing.T, then http.HandlerFunc) *httptest.Server { +// upstreamThen serves body once and answers every later request with status. +func upstreamThen(t *testing.T, body string, status int) *httptest.Server { t.Helper() var requests atomic.Int32 - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { if requests.Add(1) == 1 { w.Header().Set("Content-Type", "application/json") - w.Header().Set("ETag", `"v1"`) - _, _ = w.Write([]byte(stalePackument)) + _, _ = w.Write([]byte(body)) return } - then(w, r) + w.WriteHeader(status) })) t.Cleanup(srv.Close) return srv } -func TestNPMMetadataAfterUpstreamAnswer(t *testing.T) { - tests := []struct { - name string - then http.HandlerFunc - wantStatus int - wantStale bool - }{ - {name: "upstream error", then: func(w http.ResponseWriter, _ *http.Request) { - w.WriteHeader(http.StatusBadGateway) - }, wantStatus: http.StatusOK, wantStale: true}, - {name: "timeout", then: func(w http.ResponseWriter, r *http.Request) { - select { - case <-r.Context().Done(): - case <-time.After(2 * time.Second): - } - }, wantStatus: http.StatusOK, wantStale: true}, - {name: "not modified", then: func(w http.ResponseWriter, _ *http.Request) { - w.WriteHeader(http.StatusNotModified) - }, wantStatus: http.StatusOK}, - {name: "unpublished", then: func(w http.ResponseWriter, _ *http.Request) { - http.NotFound(w, nil) - }, wantStatus: http.StatusNotFound}, - {name: "credentials rejected", then: func(w http.ResponseWriter, _ *http.Request) { - w.WriteHeader(http.StatusUnauthorized) - }, wantStatus: http.StatusBadGateway}, - {name: "access forbidden", then: func(w http.ResponseWriter, _ *http.Request) { - w.WriteHeader(http.StatusForbidden) - }, wantStatus: http.StatusBadGateway}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - upstream := npmUpstreamThen(t, tt.then) - proxy, _, _, _ := setupTestProxy(t) - proxy.CacheMetadata = true - proxy.MetadataTTL = time.Millisecond - proxy.HTTPClient = &http.Client{Timeout: 200 * time.Millisecond} - h := NewNPMHandler(proxy, "http://proxy.test", upstream.URL+"/").Routes() - - first := httptest.NewRecorder() - h.ServeHTTP(first, httptest.NewRequest(http.MethodGet, "/left-pad", nil)) - if first.Code != http.StatusOK { - t.Fatalf("first request: status = %d", first.Code) - } - time.Sleep(5 * time.Millisecond) - - w := httptest.NewRecorder() - h.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/left-pad", nil)) - if w.Code != tt.wantStatus { - t.Fatalf("status = %d, want %d, body %s", w.Code, tt.wantStatus, w.Body) - } - if stale := w.Header().Get("Warning") == staleWarning; stale != tt.wantStale { - t.Errorf("Warning = %q, want stale %v", w.Header().Get("Warning"), tt.wantStale) - } - servedCopy := strings.Contains(w.Body.String(), "left-pad-1.0.0.tgz") - if wantCopy := tt.wantStatus == http.StatusOK; servedCopy != wantCopy { - t.Errorf("served the cached packument = %v, want %v", servedCopy, wantCopy) - } - }) - } +// staleProxy returns a proxy whose cached metadata expires right away. +func staleProxy(t *testing.T, upstream *httptest.Server) *Proxy { + t.Helper() + proxy, _, _, _ := setupTestProxy(t) + proxy.CacheMetadata = true + proxy.MetadataTTL = time.Millisecond + proxy.HTTPClient = upstream.Client() + return proxy } -// With a denylist entry for the package, version requests read the filtered -// packument through handleFilteredNPMVersionMetadata. -func TestNPMFilteredVersionMetadataAfterUpstreamAnswer(t *testing.T) { - tests := []struct { - name string - status int - wantStatus int - wantStale bool - }{ - {name: "upstream error", status: http.StatusServiceUnavailable, wantStatus: http.StatusOK, wantStale: true}, - {name: "unpublished", status: http.StatusNotFound, wantStatus: http.StatusNotFound}, - {name: "credentials rejected", status: http.StatusUnauthorized, wantStatus: http.StatusBadGateway}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - upstream := npmUpstreamThen(t, func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(tt.status) }) - proxy, _, _, _ := setupTestProxy(t) - proxy.CacheMetadata = true - proxy.MetadataTTL = time.Millisecond - proxy.HTTPClient = upstream.Client() - setTestDenylist(t, proxy, "pkg:npm/left-pad@2.0.0") - h := NewNPMHandler(proxy, "http://proxy.test", upstream.URL+"/").Routes() - - first := httptest.NewRecorder() - h.ServeHTTP(first, httptest.NewRequest(http.MethodGet, "/left-pad", nil)) - if first.Code != http.StatusOK { - t.Fatalf("first request: status = %d", first.Code) - } - time.Sleep(5 * time.Millisecond) - - w := httptest.NewRecorder() - h.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/left-pad/1.0.0", nil)) - if w.Code != tt.wantStatus { - t.Fatalf("status = %d, want %d, body %s", w.Code, tt.wantStatus, w.Body) - } - if stale := w.Header().Get("Warning") == staleWarning; stale != tt.wantStale { - t.Errorf("Warning = %q, want stale %v", w.Header().Get("Warning"), tt.wantStale) - } - }) +// requestTwice requests path, lets the cached copy expire, and requests it +// again. +func requestTwice(t *testing.T, h http.Handler, path string) *httptest.ResponseRecorder { + t.Helper() + first := httptest.NewRecorder() + h.ServeHTTP(first, httptest.NewRequest(http.MethodGet, path, nil)) + if first.Code != http.StatusOK { + t.Fatalf("first request: status = %d, body %s", first.Code, first.Body) } + time.Sleep(5 * time.Millisecond) + w := httptest.NewRecorder() + h.ServeHTTP(w, httptest.NewRequest(http.MethodGet, path, nil)) + return w } -func TestProxyCachedPassesDefiniteAnswersThrough(t *testing.T) { - for _, status := range []int{http.StatusNotFound, http.StatusUnauthorized, http.StatusForbidden} { - t.Run(http.StatusText(status), func(t *testing.T) { - upstream := npmUpstreamThen(t, func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(status) }) - proxy, _, _, _ := setupTestProxy(t) - proxy.CacheMetadata = true - proxy.MetadataTTL = time.Millisecond - proxy.HTTPClient = upstream.Client() - - w := httptest.NewRecorder() - proxy.ProxyCached(w, httptest.NewRequest(http.MethodGet, "/doc", nil), upstream.URL+"/doc", "test-eco", "doc") - time.Sleep(5 * time.Millisecond) - w = httptest.NewRecorder() - proxy.ProxyCached(w, httptest.NewRequest(http.MethodGet, "/doc", nil), upstream.URL+"/doc", "test-eco", "doc") - - if w.Code == http.StatusOK || strings.Contains(w.Body.String(), "left-pad") { - t.Errorf("status = %d, body %q, want the upstream's answer instead of the cached copy", w.Code, w.Body) - } - }) - } -} +// staleStatuses are upstream answers the metadata cache falls back on. +var staleStatuses = []int{http.StatusBadGateway, http.StatusNotFound, http.StatusUnauthorized, http.StatusForbidden} const staleComposerDocument = `{"packages":{"acme/widget":[{"name":"acme/widget","version":"1.0.0",` + `"dist":{"type":"zip","url":"https://repo.packagist.org/files/acme/widget/1.0.0/widget.zip"}}]},"minified":"composer/2.0"}` -func TestComposerMetadataAfterUpstreamAnswer(t *testing.T) { - tests := []struct { - name string - status int - wantStatus int - wantStale bool - }{ - {name: "upstream error", status: http.StatusBadGateway, wantStatus: http.StatusOK, wantStale: true}, - {name: "removed", status: http.StatusNotFound, wantStatus: http.StatusNotFound}, - {name: "credentials rejected", status: http.StatusUnauthorized, wantStatus: http.StatusBadGateway}, - {name: "access forbidden", status: http.StatusForbidden, wantStatus: http.StatusBadGateway}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - var requests atomic.Int32 - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - if requests.Add(1) == 1 { - w.Header().Set("Content-Type", "application/json") - _, _ = w.Write([]byte(staleComposerDocument)) - return - } - w.WriteHeader(tt.status) - })) - t.Cleanup(upstream.Close) - proxy, _, _, _ := setupTestProxy(t) - proxy.CacheMetadata = true - proxy.MetadataTTL = time.Millisecond - proxy.HTTPClient = upstream.Client() - h := NewComposerHandlerWithUpstreams(proxy, "http://proxy.test", upstream.URL, upstream.URL).Routes() +func TestComposerStaleMetadataIsMarked(t *testing.T) { + for _, status := range staleStatuses { + t.Run(http.StatusText(status), func(t *testing.T) { + upstream := upstreamThen(t, staleComposerDocument, status) + h := NewComposerHandlerWithUpstreams(staleProxy(t, upstream), "http://proxy.test", upstream.URL, upstream.URL).Routes() - first := httptest.NewRecorder() - h.ServeHTTP(first, httptest.NewRequest(http.MethodGet, "/p2/acme/widget.json", nil)) - if first.Code != http.StatusOK { - t.Fatalf("first request: status = %d, body %s", first.Code, first.Body) + w := requestTwice(t, h, "/p2/acme/widget.json") + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want 200 from the cache, body %s", w.Code, w.Body) } - time.Sleep(5 * time.Millisecond) - - w := httptest.NewRecorder() - h.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/p2/acme/widget.json", nil)) - if w.Code != tt.wantStatus { - t.Fatalf("status = %d, want %d, body %s", w.Code, tt.wantStatus, w.Body) - } - if stale := w.Header().Get("Warning") == staleWarning; stale != tt.wantStale { - t.Errorf("Warning = %q, want stale %v", w.Header().Get("Warning"), tt.wantStale) + if got := w.Header().Get("Warning"); got != staleWarning { + t.Errorf("Warning = %q, want %q", got, staleWarning) + } + }) + } +} + +const staleChefUniverse = `{"apt":{"7.5.0":{"location_type":"supermarket",` + + `"location_path":"https://supermarket.chef.io/api/v1","download_url":` + + `"https://supermarket.chef.io/api/v1/cookbooks/apt/versions/7.5.0/download","dependencies":{}}}}` + +func TestChefStaleMetadataIsMarked(t *testing.T) { + for _, status := range staleStatuses { + t.Run(http.StatusText(status), func(t *testing.T) { + upstream := upstreamThen(t, staleChefUniverse, status) + h := NewChefHandler(staleProxy(t, upstream), chefTestProxyURL, upstream.URL).Routes() + + w := requestTwice(t, h, "/universe") + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want 200 from the cache, body %s", w.Code, w.Body) + } + if got := w.Header().Get("Warning"); got != staleWarning { + t.Errorf("Warning = %q, want %q", got, staleWarning) } }) } From 91b37ec1f8f0a29debbc47d370c241fc26832468 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Thu, 22 Oct 2026 12:13:20 +0200 Subject: [PATCH 343/633] Proxy FreeBSD pkg repositories at /freebsd/{name}/ Catalogues (meta.conf, packagesite and data) carry their signature inside the archive and go through the metadata cache byte for byte. Packages under Hashed/ have the content hash in their name, so they go to the artifact cache as pkg:bsd/freebsd/{name}@{version}. Other files, including the bootstrap files under Latest/, are passed through. pkg.freebsd.org is the built-in freebsd repository; more mirrors go under upstream.freebsd. Closes #536 --- README.md | 55 ++++ config.example.yaml | 8 + docs/configuration.md | 21 ++ go.mod | 2 +- go.sum | 4 +- internal/config/config.go | 9 + internal/config/config_test.go | 17 +- internal/handler/freebsd.go | 198 +++++++++++++++ internal/handler/freebsd_test.go | 251 +++++++++++++++++++ internal/handler/notfound_ecosystems_test.go | 2 + internal/retention/retention.go | 2 +- internal/server/dashboard.go | 18 ++ internal/server/middleware.go | 2 +- internal/server/middleware_test.go | 2 +- internal/server/server.go | 3 + internal/server/sources.go | 1 + internal/server/sources_test.go | 1 + 17 files changed, 589 insertions(+), 7 deletions(-) create mode 100644 internal/handler/freebsd.go create mode 100644 internal/handler/freebsd_test.go diff --git a/README.md b/README.md index 2c538ad..e322f3c 100644 --- a/README.md +++ b/README.md @@ -66,6 +66,7 @@ The proxy never uploads artifact bytes to a scanner. Each scanner is notified wi | RPM | RHEL/Fedora | | ✓ | | Alpine | Alpine Linux | | ✓ | | Arch | Arch Linux | | ✓ | +| FreeBSD | FreeBSD (ports and pkgbase) | | ✓ | | Chef Supermarket | Chef | | ✓ | | Generic | Any | | ✓ | | Git | Any | | ✓ | @@ -777,6 +778,59 @@ Avoid `mirror.archlinuxarm.org` as the upstream. It redirects each request to a changing regional mirror over plain HTTP, so the database and the packages can come from mirrors with a different sync state. +### FreeBSD / pkg + +Override the repositories in `/usr/local/etc/pkg/repos/FreeBSD.conf`. Settings +there are merged with `/etc/pkg/FreeBSD.conf`, so only the URL and mirror type +change and the signature settings stay. The default repository name `freebsd` +proxies the official mirror (`https://pkg.freebsd.org`). On FreeBSD 15: + +``` +FreeBSD-ports: { + url: "http://localhost:8080/freebsd/freebsd/${ABI}/quarterly", + mirror_type: "none" +} +FreeBSD-ports-kmods: { + url: "http://localhost:8080/freebsd/freebsd/${ABI}/kmods_quarterly_${VERSION_MINOR}", + mirror_type: "none" +} +FreeBSD-base: { + url: "http://localhost:8080/freebsd/freebsd/${ABI}/base_release_${VERSION_MINOR}", + mirror_type: "none" +} +``` + +FreeBSD 14 has a single repository called `FreeBSD`: + +``` +FreeBSD: { + url: "http://localhost:8080/freebsd/freebsd/${ABI}/quarterly", + mirror_type: "none" +} +``` + +Then: + +```bash +pkg update +``` + +Keep the path after `${ABI}` that `pkg -vv` shows for each repository +(`quarterly`, `latest`, `base_latest`, ...) and replace only the host part. + +The repository catalogues (`meta.conf`, `packagesite.pkg`, `data.pkg`) carry +their signature inside the archive and are served byte-for-byte unchanged +through the metadata cache (`metadata_ttl`, stale fallback). Enable +`cache_metadata` so `pkg update` keeps working while the upstream is down. +Packages under `Hashed/` have their content hash in the file name, so they are +stored in the shared artifact cache as `pkg:bsd/freebsd/{name}@{version}` and +remain available when the upstream is unreachable. + +Other files are passed through uncached. That includes the bootstrap package +under `Latest/`, whose signature file is rewritten with every repository build, +and package files without a hash in their name, which can be rebuilt under the +same name. + ### GitHub Releases / mise (aqua backend) Configure named generic upstreams: @@ -1171,6 +1225,7 @@ Recently cached: | `GET /v2/homebrew/core/*` | Homebrew core bottle manifests and blobs from GHCR | | `GET /apk/{repository}/*` | Alpine APK repository protocol | | `GET /arch/{repository}/*` | Arch Linux pacman repository protocol | +| `GET /freebsd/{repository}/*` | FreeBSD pkg repository protocol | | `GET /generic/{name}/*` | Generic HTTP download proxy (GitHub release assets, mise/aqua) | | `GET /git/{name}/{repo}/info/refs`, `POST /git/{name}/{repo}/git-upload-pack` | Git smart HTTP protocol, read-only | | `GET /debian/*` | Debian/APT repository protocol (main archive) | diff --git a/config.example.yaml b/config.example.yaml index 7534858..f785232 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -315,6 +315,14 @@ upstream: # archlinux: "https://geo.mirror.pkgbuild.com" # archlinuxarm: "https://.mirror.archlinuxarm.org" + # Named FreeBSD pkg mirrors (used by /freebsd/{name}/), for ports and + # pkgbase repositories alike. + # Defaults to {"freebsd": "https://pkg.freebsd.org"} when empty; + # configuring any entry replaces that default. + # freebsd: + # freebsd: "https://pkg.freebsd.org" + # poudriere: "https://packages.example.com" + # Named generic HTTP upstreams (used by /generic/{name}/). The remaining # request path and query are appended to the upstream URL. GitHub release # assets ({owner}/{repo}/releases/download/{tag}/{asset}) are cached diff --git a/docs/configuration.md b/docs/configuration.md index 6f7be45..2e763ee 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -124,6 +124,7 @@ The package rule covers every release and asset in that repository, but each cac | `conda` | yes | `pkg:conda/conda-forge/numpy`, with the channel as the namespace; the package name's case doesn't matter. A key without a channel is refused. | | `cran` | yes | `pkg:cran/ggplot2`, the package name as published, case included (`pkg:cran/Rcpp`). A rule covers source and binary builds alike. | | `deb` | yes | `pkg:deb/curl`; case doesn't matter. A vendor namespace such as `pkg:deb/debian/curl` is accepted and dropped, so a rule covers the package from every configured archive. Debian and Ubuntu archives only keep the current version per suite, so an evicted older one can't be fetched again. | +| `freebsd` | yes | `pkg:bsd/freebsd/zsh`, the package name as published, case included (`pkg:bsd/freebsd/FreeBSD-acct`). A rule covers the package in every configured repository and ABI. Only packages under `Hashed/` are cached, and mirrors keep only the current build, so an evicted older one can't be fetched again. | | `gem` | yes | `pkg:gem/rails`, the gem name as published, case included. | | `generic` | yes, explicit rules only | `pkg:generic/owner%2Frepo` for a GitHub Release repository such as `jqlang/jq`. The ecosystem rule covers all cached version-pinned generic downloads, including GitHub Release assets and configured `generic_artifacts`. The global `default` never applies; metadata/API responses are not artifact retention candidates. | | `golang` | yes | `pkg:golang/github.com/stretchr/testify`, the module path as in `go.mod`; case doesn't matter. A major version like `/v2` (or `.v3` on gopkg.in) is a module of its own and needs its own rule. The `!`-escaped form from proxy URLs and the module cache (`github.com/!azure/...`) is refused. | @@ -537,6 +538,26 @@ When `upstream.arch` is empty, a single repository named `archlinux` pointing at that default. Arch Linux ARM has no default: pick a regional HTTPS mirror, not the `mirror.archlinuxarm.org` redirector. +```yaml +upstream: + + # Named FreeBSD pkg mirrors, served at /freebsd/{name}/. + freebsd: + freebsd: "https://pkg.freebsd.org" +``` + +FreeBSD pkg repositories are read-only. Requests to `/freebsd/{name}/…` are +passed to the mirror unchanged, e.g. +`/freebsd/freebsd/FreeBSD:14:amd64/quarterly/packagesite.pkg`, so ports and +pkgbase repositories and repositories built with poudriere work. Catalogues +(`meta`, `meta.conf`, `packagesite` and `data` as `.pkg`, `.txz` or `.tzst`) +are cached with the metadata TTL and served byte-for-byte unchanged, which +keeps their embedded signatures valid. Packages under `Hashed/` use the shared +artifact cache; everything else, including `Latest/` and unhashed package +files, is passed through uncached. When `upstream.freebsd` is empty, a single +repository named `freebsd` pointing at `https://pkg.freebsd.org` is available; +configuring any entry replaces that default. + ### Go checksum database ```yaml diff --git a/go.mod b/go.mod index 3c1d4e2..0bcea25 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/git-pkgs/gcs v0.1.0 github.com/git-pkgs/integrity v0.2.0 github.com/git-pkgs/magic v0.5.0 - github.com/git-pkgs/purl v0.1.22 + github.com/git-pkgs/purl v0.1.23 github.com/git-pkgs/registries v0.11.0 github.com/git-pkgs/sbom v0.1.7 github.com/git-pkgs/spdx v0.3.3 diff --git a/go.sum b/go.sum index f486b50..f298c28 100644 --- a/go.sum +++ b/go.sum @@ -133,8 +133,8 @@ github.com/git-pkgs/magic v0.5.0 h1:Pm+4fkHR+K6kg9pwJM4hnwWUuBlkYdLmF1yIAI8lU9Q= github.com/git-pkgs/magic v0.5.0/go.mod h1:SXOqcsNmbmpZjJZHEEWnwxprbsFvpwWgTAZrgXp4Jm4= github.com/git-pkgs/pom v0.1.8 h1:mXCuWw6XkrKKZb6IurRZyIznYLPhV5AYg5ZO38Uj/l8= github.com/git-pkgs/pom v0.1.8/go.mod h1:sQBeYRzjCUn77fcABPn6I/a1aBHFxm55jTmkNxYtGtw= -github.com/git-pkgs/purl v0.1.22 h1:qT9FNXT/3/RbWnQCxMHn1CFeuruMsq0RBH154oLt1Xs= -github.com/git-pkgs/purl v0.1.22/go.mod h1:l8rCfvt5iXVrFIcoQ8+vCLOaHhrtEW2id0LENP+IzMw= +github.com/git-pkgs/purl v0.1.23 h1:RG7F73S4ZnbAnB/posWf2WS4OhdG0xUbyX8UIkEUhPw= +github.com/git-pkgs/purl v0.1.23/go.mod h1:l8rCfvt5iXVrFIcoQ8+vCLOaHhrtEW2id0LENP+IzMw= github.com/git-pkgs/registries v0.11.0 h1:kkB80NjO9UncyyxGfKmdPXWqUMNE/CkNw5Z/6gqW7Rs= github.com/git-pkgs/registries v0.11.0/go.mod h1:bKxgdy/PwnzQXL2FnOktmSQHTKk6B85MA4VtvvYItPc= github.com/git-pkgs/sbom v0.1.7 h1:4eboy/Yed/SC9ZcC/xT50iywHPzWbmCU7azDb5T1RbA= diff --git a/internal/config/config.go b/internal/config/config.go index cbb4e6e..ce8e451 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -750,6 +750,12 @@ type UpstreamConfig struct { // Default when empty: {"archlinux": "https://geo.mirror.pkgbuild.com"}. Arch map[string]string `json:"arch" yaml:"arch"` + // FreeBSD maps repository names to FreeBSD pkg mirror base URLs, served + // at /freebsd/{name}/. The remaining request path mirrors the upstream + // layout, e.g. /freebsd/freebsd/FreeBSD:14:amd64/latest/packagesite.pkg. + // Default when empty: {"freebsd": "https://pkg.freebsd.org"}. + FreeBSD map[string]string `json:"freebsd" yaml:"freebsd"` + // OCI maps names to OCI registry URLs. Requests to a named registry use // the repository prefix upstream/{name}/, for example // oci://proxy.example.com/upstream/ghcr/owner/chart. @@ -839,6 +845,9 @@ func (u *UpstreamConfig) Validate() error { if err := validateNamedUpstreams("upstream.arch", u.Arch); err != nil { return err } + if err := validateNamedUpstreams("upstream.freebsd", u.FreeBSD); err != nil { + return err + } if err := validateNamedUpstreams("upstream.go_sumdbs", u.GoSumDBs); err != nil { return err } diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 224c651..7e22dfc 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -1433,7 +1433,7 @@ func TestValidateNamedUpstreams(t *testing.T) { wantErr bool }{ { - name: "valid Helm, OCI, APK, Arch, and generic upstreams", + name: "valid Helm, OCI, APK, Arch, FreeBSD, and generic upstreams", modify: func(cfg *Config) { cfg.Upstream.Helm = map[string]string{"bitnami": "https://charts.bitnami.com/bitnami"} cfg.Upstream.OCI = map[string]string{"ghcr": "https://ghcr.io"} @@ -1442,6 +1442,7 @@ func TestValidateNamedUpstreams(t *testing.T) { "archlinux": "https://geo.mirror.pkgbuild.com", "archlinuxarm": "https://de3.mirror.archlinuxarm.org", } + cfg.Upstream.FreeBSD = map[string]string{"freebsd": "https://pkg.freebsd.org"} cfg.Upstream.Generic = map[string]string{ "github": "https://github.com", "github-api": "https://api.github.com", @@ -1616,6 +1617,20 @@ func TestValidateNamedUpstreams(t *testing.T) { }, wantErr: true, }, + { + name: "FreeBSD upstream name contains path separator", + modify: func(cfg *Config) { + cfg.Upstream.FreeBSD = map[string]string{"freebsd/ports": "https://pkg.freebsd.org"} + }, + wantErr: true, + }, + { + name: "FreeBSD upstream URL is not absolute", + modify: func(cfg *Config) { + cfg.Upstream.FreeBSD = map[string]string{"freebsd": "pkg.freebsd.org"} + }, + wantErr: true, + }, { name: "valid Debian repository", modify: func(cfg *Config) { diff --git a/internal/handler/freebsd.go b/internal/handler/freebsd.go new file mode 100644 index 0000000..419147b --- /dev/null +++ b/internal/handler/freebsd.go @@ -0,0 +1,198 @@ +package handler + +import ( + "crypto/sha256" + "encoding/hex" + "net/http" + "regexp" + "strings" + + "github.com/git-pkgs/proxy/internal/retention" + "github.com/git-pkgs/purl" +) + +const ( + freebsdEcosystem = "freebsd" + // defaultFreeBSDRepositoryName is the repository name used when no + // upstream.freebsd repositories are configured. + defaultFreeBSDRepositoryName = "freebsd" + // defaultFreeBSDUpstream is the official FreeBSD package mirror. + defaultFreeBSDUpstream = "https://pkg.freebsd.org" + freebsdNamespace = "freebsd" + freebsdMatchCount = 3 // full match + name + version + freebsdHashedDir = "Hashed" + freebsdPackageMedia = "application/octet-stream" +) + +// Packages are stored as pkg:bsd/freebsd/{name}, whichever configured +// repository they came from. +func init() { + retention.Default.Register(retention.Spec{Key: freebsdEcosystem, CanonicalPackage: freebsdRetentionKey}) +} + +// freebsdRetentionKey maps pkg:bsd/freebsd/{name} to the stored package PURL. +// Another namespace, or a name with a slash, can never match a stored package +// and is refused. +func freebsdRetentionKey(p *purl.PURL) (string, bool) { + if p == nil || p.Type != "bsd" || p.Namespace != freebsdNamespace || strings.Contains(p.Name, "/") { + return "", false + } + canonical := purl.MakePURLString(freebsdEcosystem, p.Name, "") + return canonical, canonical != "" +} + +// FreeBSDHandler serves FreeBSD pkg repositories, ports and pkgbase alike. +// Each configured upstream is mounted at /freebsd/{repository}/ and the rest +// of the path mirrors the upstream layout ({ABI}/{repo}/{file}). +// +// Repository catalogues (meta.conf, packagesite.pkg, data.pkg) carry their +// signature inside the archive, so they are served byte for byte through the +// metadata cache. Packages under Hashed/ have the content hash in their name +// and never change, so they go to the artifact cache. Everything else, +// including the unversioned bootstrap files under Latest/ whose detached +// signature is replaced on every repository build, is passed through. +type FreeBSDHandler struct { + proxy *Proxy + proxyURL string + repositories map[string]string +} + +// NewFreeBSDHandler creates a FreeBSD pkg repository handler. Without +// repositories it serves the official mirror as "freebsd". +func NewFreeBSDHandler(proxy *Proxy, proxyURL string, repositories map[string]string) *FreeBSDHandler { + h := &FreeBSDHandler{ + proxy: proxy, + proxyURL: strings.TrimSuffix(proxyURL, "/"), + repositories: make(map[string]string, len(repositories)), + } + for name, repositoryURL := range repositories { + h.repositories[name] = strings.TrimSuffix(repositoryURL, "/") + } + if len(h.repositories) == 0 { + h.repositories[defaultFreeBSDRepositoryName] = defaultFreeBSDUpstream + } + return h +} + +// Routes returns the HTTP handler for pkg requests. +// Mount this at /freebsd on your router. +func (h *FreeBSDHandler) Routes() http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet && r.Method != http.MethodHead { + http.Error(w, "method not allowed", http.StatusMethodNotAllowed) + return + } + + path := strings.TrimPrefix(r.URL.Path, "/") + + // A decoded '?' or '#' would cut the upstream path short and cache one + // file under another file's name. + if containsPathTraversal(path) || strings.ContainsAny(path, "?#") { + http.Error(w, "invalid path", http.StatusBadRequest) + return + } + + repository, rest, ok := strings.Cut(path, "/") + upstreamURL, found := h.repositories[repository] + if !ok || rest == "" || !found { + http.NotFound(w, r) + return + } + + switch { + case isFreeBSDCatalogue(rest): + h.handleMetadata(w, r, repository, upstreamURL, rest) + case isFreeBSDHashedPackage(rest): + h.handlePackageDownload(w, r, repository, upstreamURL, rest) + default: + h.proxyFile(w, r, upstreamURL, rest) + } + }) +} + +// freebsdCataloguePattern matches the files pkg reads to update a repository: +// meta and meta.conf, and the packagesite and data catalogues in the current +// (.pkg) and older (.txz, .tzst) packing formats. +var freebsdCataloguePattern = regexp.MustCompile(`^(?:meta(?:\.conf|\.txz|\.tzst|\.pkg)?|(?:packagesite|data)\.(?:pkg|txz|tzst))$`) + +func freebsdBase(path string) string { + return path[strings.LastIndex(path, "/")+1:] +} + +// isFreeBSDCatalogue reports whether path names a repository catalogue. The +// depth isn't checked, so repositories built with poudriere and served under +// another layout work too; package files are always {name}-{version}.pkg and +// can't take these names. +func isFreeBSDCatalogue(path string) bool { + return freebsdCataloguePattern.MatchString(freebsdBase(path)) +} + +// freebsdHashedPattern matches {name}-{version}~{hash}.pkg. Port versions +// never contain '-', so the version is what follows the last one, e.g. +// zsh-navigation-tools-2.2.7~2$imxi7k7w.pkg or +// FreeBSD-syslogd-man-14.snap20261019150821~2$k6xtent4.pkg. +var freebsdHashedPattern = regexp.MustCompile(`^(.+)-([^-~/]+)~[^~/]+\.pkg$`) + +// isFreeBSDHashedPackage reports whether path names a package in a Hashed/ +// directory: All/Hashed/ for ports, Hashed/ for pkgbase. +func isFreeBSDHashedPackage(path string) bool { + dir := strings.TrimSuffix(path, "/"+freebsdBase(path)) + return freebsdBase(dir) == freebsdHashedDir && freebsdHashedPattern.MatchString(freebsdBase(path)) +} + +// parseFreeBSDPackage returns the name and version of a hashed package file. +func parseFreeBSDPackage(filename string) (name, version string) { + matches := freebsdHashedPattern.FindStringSubmatch(filename) + if len(matches) != freebsdMatchCount { + return "", "" + } + return matches[1], matches[2] +} + +// handlePackageDownload caches hashed packages. The cache key holds the +// repository and full path, since every ABI and repository has its own +// builds. +func (h *FreeBSDHandler) handlePackageDownload(w http.ResponseWriter, r *http.Request, repository, upstreamURL, path string) { + name, version := parseFreeBSDPackage(freebsdBase(path)) + if name == "" { + h.proxyFile(w, r, upstreamURL, path) + return + } + + downloadURL := upstreamURL + "/" + path + cacheFilename := repository + "/" + path + + h.proxy.Logger.Info("freebsd package download", + "repository", repository, "name", name, "version", version) + + result, err := h.proxy.GetOrFetchArtifactFromURL( + r.Context(), freebsdEcosystem, name, version, cacheFilename, downloadURL) + if err != nil { + h.proxy.serveArtifactError(w, err, "failed to fetch package") + return + } + + if result.Artifact.MediaType == "" { + result.Artifact.MediaType = freebsdPackageMedia + } + ServeArtifactRequest(w, r, result) +} + +// handleMetadata serves catalogues unchanged through the metadata cache, +// which keeps their embedded signatures valid. +func (h *FreeBSDHandler) handleMetadata(w http.ResponseWriter, r *http.Request, repository, upstreamURL, path string) { + h.proxy.ProxyCached(w, r, upstreamURL+"/"+path, freebsdEcosystem, + h.metadataCacheKey(repository, upstreamURL, path), "*/*") +} + +// metadataCacheKey hashes repository, upstream URL and path, as APKHandler does. +func (h *FreeBSDHandler) metadataCacheKey(repository, upstreamURL, path string) string { + identity := repository + "\x00" + upstreamURL + "\x00" + path + digest := sha256.Sum256([]byte(identity)) + return hex.EncodeToString(digest[:]) +} + +// proxyFile passes other files through uncached. +func (h *FreeBSDHandler) proxyFile(w http.ResponseWriter, r *http.Request, upstreamURL, path string) { + h.proxy.ProxyFile(w, r, upstreamURL+"/"+path) +} diff --git a/internal/handler/freebsd_test.go b/internal/handler/freebsd_test.go new file mode 100644 index 0000000..b203871 --- /dev/null +++ b/internal/handler/freebsd_test.go @@ -0,0 +1,251 @@ +package handler + +import ( + "net/http" + "net/http/httptest" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/git-pkgs/purl" + "github.com/git-pkgs/registries/fetch" +) + +const freebsdABI = "/FreeBSD:14:amd64" + +func TestFreeBSDPathClassification(t *testing.T) { + tests := []struct { + path string + catalogue bool + hashed bool + }{ + {"FreeBSD:14:amd64/latest/meta.conf", true, false}, + {"FreeBSD:14:amd64/latest/meta", true, false}, + {"FreeBSD:14:amd64/latest/packagesite.pkg", true, false}, + {"FreeBSD:14:amd64/latest/data.pkg", true, false}, + {"FreeBSD:13:amd64/quarterly/packagesite.txz", true, false}, + {"142amd64-default/packagesite.tzst", true, false}, + {"FreeBSD:14:amd64/latest/All/Hashed/zsh-5.9_5~2$3rwh1rgp.pkg", false, true}, + {"FreeBSD:15:amd64/base_latest/Hashed/FreeBSD-acct-15.snap20261021214449~2$k6xtent4.pkg", false, true}, + {"FreeBSD:14:amd64/latest/All/zsh-5.9_5.pkg", false, false}, + {"FreeBSD:15:amd64/base_latest/FreeBSD-acct-15.snap20261021214449.pkg", false, false}, + {"FreeBSD:14:amd64/latest/Latest/pkg.pkg", false, false}, + {"FreeBSD:14:amd64/latest/Latest/pkg.pkg.sig", false, false}, + {"FreeBSD:14:amd64/latest/All/Hashed/no-hash.pkg", false, false}, + } + for _, tt := range tests { + if got := isFreeBSDCatalogue(tt.path); got != tt.catalogue { + t.Errorf("isFreeBSDCatalogue(%q) = %v, want %v", tt.path, got, tt.catalogue) + } + if got := isFreeBSDHashedPackage(tt.path); got != tt.hashed { + t.Errorf("isFreeBSDHashedPackage(%q) = %v, want %v", tt.path, got, tt.hashed) + } + } +} + +func TestParseFreeBSDPackage(t *testing.T) { + tests := []struct { + filename, name, version string + }{ + {"zsh-5.9_5~2$3rwh1rgp.pkg", "zsh", "5.9_5"}, + {"zsh-navigation-tools-2.2.7~2$imxi7k7w.pkg", "zsh-navigation-tools", "2.2.7"}, + {"py311-requests-2.32.3_1,1~2$abcdefgh.pkg", "py311-requests", "2.32.3_1,1"}, + {"FreeBSD-syslogd-man-14.snap20261019150821~2$k6xtent4.pkg", "FreeBSD-syslogd-man", "14.snap20261019150821"}, + {"nohash-1.0.pkg", "", ""}, + } + for _, tt := range tests { + name, version := parseFreeBSDPackage(tt.filename) + if name != tt.name || version != tt.version { + t.Errorf("parseFreeBSDPackage(%q) = %q, %q, want %q, %q", tt.filename, name, version, tt.name, tt.version) + } + } +} + +func TestFreeBSDHandler_Routes(t *testing.T) { + h := NewFreeBSDHandler(nil, "http://localhost:8080", nil) + assertRoutesBasics(t, h.Routes(), "/freebsd"+freebsdABI+"/latest/meta.conf", "/freebsd"+freebsdABI+"/../../../etc/passwd") +} + +func TestFreeBSDHandler_DefaultsToOfficialMirror(t *testing.T) { + h := NewFreeBSDHandler(nil, "http://localhost:8080", nil) + if got := h.repositories[defaultFreeBSDRepositoryName]; got != defaultFreeBSDUpstream { + t.Errorf("default repository = %q, want %q", got, defaultFreeBSDUpstream) + } +} + +func TestFreeBSDHandler_UnknownRepositoryReturns404(t *testing.T) { + proxy, _, _, _ := setupTestProxy(t) + h := NewFreeBSDHandler(proxy, "http://localhost:8080", map[string]string{"freebsd": "https://example.test"}) + for _, target := range []string{"/unknown" + freebsdABI + "/latest/meta.conf", "/freebsd", "/"} { + if w := serveFreeBSDRequest(h, target); w.Code != http.StatusNotFound { + t.Errorf("%s: status = %d, want 404", target, w.Code) + } + } +} + +// fakeFreeBSDMirror serves files and counts requests per path. It answers +// 503 for everything while down is set. +type fakeFreeBSDMirror struct { + *httptest.Server + down atomic.Bool + mu sync.Mutex + requests map[string]int +} + +func newFakeFreeBSDMirror(t *testing.T, files map[string]string) *fakeFreeBSDMirror { + t.Helper() + m := &fakeFreeBSDMirror{requests: map[string]int{}} + m.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if m.down.Load() { + http.Error(w, "unavailable", http.StatusServiceUnavailable) + return + } + body, ok := files[r.URL.Path] + if !ok { + http.NotFound(w, r) + return + } + m.mu.Lock() + m.requests[r.URL.Path]++ + m.mu.Unlock() + w.Header().Set("Content-Type", "application/octet-stream") + _, _ = w.Write([]byte(body)) + })) + t.Cleanup(m.Close) + return m +} + +func (m *fakeFreeBSDMirror) count(path string) int { + m.mu.Lock() + defer m.mu.Unlock() + return m.requests[path] +} + +func TestFreeBSDHandler_CataloguesServedUnchanged(t *testing.T) { + files := map[string]string{ + freebsdABI + "/latest/meta.conf": "version = 2;\npacking_format = \"tzst\";\n", + freebsdABI + "/latest/packagesite.pkg": "\x28\xb5\x2f\xfdpackagesite.yaml and its signature", + freebsdABI + "/latest/data.pkg": "\x28\xb5\x2f\xfddata and its signature", + } + mirror := newFakeFreeBSDMirror(t, files) + proxy, _, _, _ := setupTestProxy(t) + proxy.CacheMetadata = true + proxy.MetadataTTL = time.Hour + proxy.HTTPClient = mirror.Client() + h := NewFreeBSDHandler(proxy, "http://proxy.example", map[string]string{"freebsd": mirror.URL}) + + for round, down := range []bool{false, true} { + mirror.down.Store(down) + for path, want := range files { + w := serveFreeBSDRequest(h, "/freebsd"+path) + if w.Code != http.StatusOK || w.Body.String() != want { + t.Fatalf("round %d %s: status = %d, body %q, want 200 and the upstream bytes", round, path, w.Code, w.Body) + } + } + } + for path := range files { + if got := mirror.count(path); got != 1 { + t.Errorf("%s fetched %d times, want 1", path, got) + } + } +} + +func TestFreeBSDHandler_HashedPackagesCachedPerABI(t *testing.T) { + const file = "/latest/All/Hashed/zsh-5.9_5~2$3rwh1rgp.pkg" + files := map[string]string{ + "/FreeBSD:14:amd64" + file: "amd64 package", + "/FreeBSD:14:aarch64" + file: "aarch64 package", + } + mirror := newFakeFreeBSDMirror(t, files) + proxy, db, _, _ := setupTestProxy(t) + fetcher := fetch.NewFetcher(fetch.WithHTTPClient(mirror.Client()), fetch.WithMaxRetries(0)) + proxy.Fetcher = fetcher + t.Cleanup(func() { _ = fetcher.Close() }) + h := NewFreeBSDHandler(proxy, "http://proxy.example", map[string]string{"freebsd": mirror.URL}) + + for round, down := range []bool{false, true} { + mirror.down.Store(down) + for path, want := range files { + w := serveFreeBSDRequest(h, "/freebsd"+path) + if w.Code != http.StatusOK || w.Body.String() != want { + t.Fatalf("round %d %s: status = %d, body %q, want 200 and %q", round, path, w.Code, w.Body, want) + } + } + } + for path := range files { + if got := mirror.count(path); got != 1 { + t.Errorf("%s fetched %d times, want 1", path, got) + } + } + + record, err := db.GetArtifact("pkg:bsd/freebsd/zsh@5.9_5", "freebsd/FreeBSD:14:amd64"+file) + if err != nil || record == nil { + t.Errorf("artifact record for pkg:bsd/freebsd/zsh@5.9_5 = %v, %v", record, err) + } +} + +// Unhashed packages can be rebuilt under the same name, and the bootstrap +// files under Latest/ change with every build, so none of them are cached. +func TestFreeBSDHandler_UnhashedFilesPassedThrough(t *testing.T) { + files := map[string]string{ + freebsdABI + "/latest/Latest/pkg.pkg": "bootstrap package", + freebsdABI + "/latest/Latest/pkg.pkg.sig": "bootstrap signature", + freebsdABI + "/latest/All/zsh-5.9_5.pkg": "unhashed package", + } + mirror := newFakeFreeBSDMirror(t, files) + proxy, _, _, _ := setupTestProxy(t) + proxy.CacheMetadata = true + proxy.MetadataTTL = time.Hour + proxy.HTTPClient = mirror.Client() + h := NewFreeBSDHandler(proxy, "http://proxy.example", map[string]string{"freebsd": mirror.URL}) + + for range 2 { + for path, want := range files { + w := serveFreeBSDRequest(h, "/freebsd"+path) + if w.Code != http.StatusOK || w.Body.String() != want { + t.Fatalf("%s: status = %d, body %q, want 200 and %q", path, w.Code, w.Body, want) + } + } + } + for path := range files { + if got := mirror.count(path); got != 2 { + t.Errorf("%s fetched %d times, want 2 (not cached)", path, got) + } + } +} + +func serveFreeBSDRequest(h *FreeBSDHandler, target string) *httptest.ResponseRecorder { + w := httptest.NewRecorder() + h.Routes().ServeHTTP(w, httptest.NewRequest(http.MethodGet, target, nil)) + return w +} + +func TestFreeBSDRetentionKey(t *testing.T) { + tests := []struct { + key string + want string + ok bool + }{ + {"pkg:bsd/freebsd/zsh", "pkg:bsd/freebsd/zsh", true}, + {"pkg:bsd/FreeBSD/FreeBSD-acct", "pkg:bsd/freebsd/FreeBSD-acct", true}, + {"pkg:bsd/freebsd/libstdc++", "pkg:bsd/freebsd/libstdc%2B%2B", true}, + {"pkg:bsd/netbsd/zsh", "", false}, + {"pkg:bsd/zsh", "", false}, + {"pkg:bsd/freebsd/latest%2Fzsh", "", false}, + {"pkg:apk/freebsd/zsh", "", false}, + } + for _, tt := range tests { + p, err := purl.Parse(tt.key) + if err != nil { + t.Fatalf("Parse(%s): %v", tt.key, err) + } + got, ok := freebsdRetentionKey(p) + if ok != tt.ok || (ok && got != tt.want) { + t.Errorf("freebsdRetentionKey(%s) = %q, %v, want %q, %v", tt.key, got, ok, tt.want, tt.ok) + } + } + if _, ok := freebsdRetentionKey(nil); ok { + t.Error("freebsdRetentionKey(nil) accepted") + } +} diff --git a/internal/handler/notfound_ecosystems_test.go b/internal/handler/notfound_ecosystems_test.go index e8b3b9b..cc61928 100644 --- a/internal/handler/notfound_ecosystems_test.go +++ b/internal/handler/notfound_ecosystems_test.go @@ -23,6 +23,8 @@ func TestArtifactDownloadUpstreamNotFoundReturns404(t *testing.T) { func(p *Proxy) http.Handler { return NewAPKHandler(p, "http://localhost", nil).Routes() }}, {"arch", "/archlinux/core/os/x86_64/bash-5.2.037-1-x86_64.pkg.tar.zst", func(p *Proxy) http.Handler { return NewArchHandler(p, "http://localhost", nil).Routes() }}, + {"freebsd", "/freebsd/FreeBSD:14:amd64/latest/All/Hashed/zsh-5.9_5~2$3rwh1rgp.pkg", + func(p *Proxy) http.Handler { return NewFreeBSDHandler(p, "http://localhost", nil).Routes() }}, {"nuget", "/v3-flatcontainer/newtonsoft.json/13.0.3/newtonsoft.json.13.0.3.nupkg", func(p *Proxy) http.Handler { return NewNuGetHandler(p, "http://localhost").Routes() }}, {"pypi", "/packages/packages/ab/cd/ef0123456789/requests-2.31.0-py3-none-any.whl", diff --git a/internal/retention/retention.go b/internal/retention/retention.go index 465b220..a8ff835 100644 --- a/internal/retention/retention.go +++ b/internal/retention/retention.go @@ -22,7 +22,7 @@ import ( // outside this list is a typo; a key in it whose handler has not registered // yet is not supported by this version. var KnownKeys = []string{ - "alpine", "cargo", "composer", "conan", "conda", "cran", "deb", "gem", + "alpine", "cargo", "composer", "conan", "conda", "cran", "deb", "freebsd", "gem", "generic", "golang", "helm", "hex", "julia", "maven", "npm", "nuget", "oci", "pub", "pypi", "rpm", "swift", } diff --git a/internal/server/dashboard.go b/internal/server/dashboard.go index fece033..00a0f11 100644 --- a/internal/server/dashboard.go +++ b/internal/server/dashboard.go @@ -145,6 +145,7 @@ func supportedEcosystems() []string { "conda", "cran", "deb", + "freebsd", "gem", "golang", "hex", @@ -213,6 +214,8 @@ func ecosystemColorClasses(ecosystem string) string { return "bg-lime-100 text-lime-800 dark:bg-lime-900/50 dark:text-lime-300" case "arch": return "bg-blue-100 text-blue-800 dark:bg-blue-900/50 dark:text-blue-300" + case "freebsd": + return "bg-red-200 text-red-900 dark:bg-red-950/60 dark:text-red-200" default: return "bg-gray-100 text-gray-700 dark:bg-gray-800 dark:text-gray-300" } @@ -553,5 +556,20 @@ Server = ` + baseURL + `/arch/archlinux/$repo/os/$arch # Then run: pacman -Syu`), }, + { + ID: "freebsd", + Name: "FreeBSD pkg", + Language: "FreeBSD", + Endpoint: "/freebsd/", + Instructions: template.HTML(`

Configure pkg to use the proxy. On FreeBSD 15, do the same for FreeBSD-ports-kmods and FreeBSD-base, keeping their paths; FreeBSD 14 calls the ports repository FreeBSD.

+
# /usr/local/etc/pkg/repos/FreeBSD.conf
+FreeBSD-ports: {
+  url: "` + baseURL + `/freebsd/freebsd/${ABI}/quarterly",
+  mirror_type: "none"
+}
+
+# Then run:
+pkg update
`), + }, } } diff --git a/internal/server/middleware.go b/internal/server/middleware.go index 0de0dba..abcf004 100644 --- a/internal/server/middleware.go +++ b/internal/server/middleware.go @@ -107,7 +107,7 @@ func requestEcosystem(path string) string { switch segment { case "npm", "cargo", "hex", "pub", "pypi", "maven", "gradle", "nuget", "conan", "chef", "conda", "cran", "julia", "debian", "rpm", "arch", - "helm", "homebrew", "generic", "swift", "git": + "freebsd", "helm", "homebrew", "generic", "swift", "git": return segment case "apk": return "alpine" diff --git a/internal/server/middleware_test.go b/internal/server/middleware_test.go index b9c67c7..7bdd0f9 100644 --- a/internal/server/middleware_test.go +++ b/internal/server/middleware_test.go @@ -313,7 +313,7 @@ func TestRequestEcosystemCoversEveryMountedRoute(t *testing.T) { "/gradle/x": "gradle", "/nuget/x": "nuget", "/composer/x": "packagist", "/conan/x": "conan", "/chef/x": "chef", "/conda/x": "conda", "/cran/x": "cran", "/julia/x": "julia", "/swift/x": "swift", "/v2/x": "oci", - "/apk/x": "alpine", "/arch/x": "arch", "/debian/x": "debian", "/rpm/x": "rpm", + "/apk/x": "alpine", "/arch/x": "arch", "/freebsd/x": "freebsd", "/debian/x": "debian", "/rpm/x": "rpm", "/helm/x": "helm", "/homebrew/x": "homebrew", "/generic/x": "generic", "/git/x": "git", } for path, want := range mounts { diff --git a/internal/server/server.go b/internal/server/server.go index 8a0a683..adbf9af 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -21,6 +21,7 @@ // - /v2/* - OCI/Docker container registry protocol // - /apk/* - Alpine APK repository protocol // - /arch/* - Arch Linux pacman repository protocol +// - /freebsd/* - FreeBSD pkg repository protocol // - /debian/* - Debian/APT repository protocol // - /rpm/* - RPM/Yum repository protocol // @@ -484,6 +485,7 @@ func (s *Server) mountProtocolHandlers(r chi.Router, proxy *handler.Proxy) { proxy, s.cfg.BaseURL, s.cfg.Upstream.Helm, s.cfg.Upstream.OCIDefault, s.cfg.Upstream.OCI) apkHandler := handler.NewAPKHandler(proxy, s.cfg.BaseURL, s.cfg.Upstream.APK) archHandler := handler.NewArchHandler(proxy, s.cfg.BaseURL, s.cfg.Upstream.Arch) + freebsdHandler := handler.NewFreeBSDHandler(proxy, s.cfg.BaseURL, s.cfg.Upstream.FreeBSD) debianHandler := handler.NewDebianHandler( proxy, s.cfg.BaseURL, @@ -516,6 +518,7 @@ func (s *Server) mountProtocolHandlers(r chi.Router, proxy *handler.Proxy) { r.Mount("/helm", http.StripPrefix("/helm", helmHandler.Routes())) r.Mount("/apk", http.StripPrefix("/apk", apkHandler.Routes())) r.Mount("/arch", http.StripPrefix("/arch", archHandler.Routes())) + r.Mount("/freebsd", http.StripPrefix("/freebsd", freebsdHandler.Routes())) r.Mount("/debian", http.StripPrefix("/debian", debianHandler.Routes())) r.Mount("/rpm", http.StripPrefix("/rpm", rpmHandler.Routes())) r.Mount("/generic", http.StripPrefix("/generic", genericHandler.Routes())) diff --git a/internal/server/sources.go b/internal/server/sources.go index 2a0e47c..90f3981 100644 --- a/internal/server/sources.go +++ b/internal/server/sources.go @@ -65,6 +65,7 @@ var knownClients = map[string]string{ "urlgrabber": "dnf", "apk": "apk", "pacman": "pacman", + "pkg": "pkg", "conda": "conda", "mamba": "conda", "conan": "conan", diff --git a/internal/server/sources_test.go b/internal/server/sources_test.go index 560ca93..c37313c 100644 --- a/internal/server/sources_test.go +++ b/internal/server/sources_test.go @@ -22,6 +22,7 @@ func TestClientName(t *testing.T) { {"Debian APT-HTTP/1.3 (2.6.1)", "apt"}, {"bundler/2.4.22 rubygems/3.4.22", "bundler"}, {"pacman/7.0.0 (Linux x86_64) libalpm/15.0.0", "pacman"}, + {"pkg/2.8.4", "pkg"}, // Unknown and hostile input collapses, so a User-Agent cannot mint a // new Prometheus time series. {"", unknownClient}, From b8a4cf71dee8d2764d68f6b3541429d2cf29d4d8 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Thu, 22 Oct 2026 13:03:16 +0200 Subject: [PATCH 344/633] Keep the long config test and dashboard list under the maintidx limit The FreeBSD config cases get their own test, and the dashboard entry moves into a helper. --- internal/config/config_test.go | 38 ++++++++++++++++++++-------------- internal/server/dashboard.go | 19 ++++++++++------- 2 files changed, 34 insertions(+), 23 deletions(-) diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 7e22dfc..71fe754 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -1433,7 +1433,7 @@ func TestValidateNamedUpstreams(t *testing.T) { wantErr bool }{ { - name: "valid Helm, OCI, APK, Arch, FreeBSD, and generic upstreams", + name: "valid Helm, OCI, APK, Arch, and generic upstreams", modify: func(cfg *Config) { cfg.Upstream.Helm = map[string]string{"bitnami": "https://charts.bitnami.com/bitnami"} cfg.Upstream.OCI = map[string]string{"ghcr": "https://ghcr.io"} @@ -1442,7 +1442,6 @@ func TestValidateNamedUpstreams(t *testing.T) { "archlinux": "https://geo.mirror.pkgbuild.com", "archlinuxarm": "https://de3.mirror.archlinuxarm.org", } - cfg.Upstream.FreeBSD = map[string]string{"freebsd": "https://pkg.freebsd.org"} cfg.Upstream.Generic = map[string]string{ "github": "https://github.com", "github-api": "https://api.github.com", @@ -1617,20 +1616,6 @@ func TestValidateNamedUpstreams(t *testing.T) { }, wantErr: true, }, - { - name: "FreeBSD upstream name contains path separator", - modify: func(cfg *Config) { - cfg.Upstream.FreeBSD = map[string]string{"freebsd/ports": "https://pkg.freebsd.org"} - }, - wantErr: true, - }, - { - name: "FreeBSD upstream URL is not absolute", - modify: func(cfg *Config) { - cfg.Upstream.FreeBSD = map[string]string{"freebsd": "pkg.freebsd.org"} - }, - wantErr: true, - }, { name: "valid Debian repository", modify: func(cfg *Config) { @@ -1916,3 +1901,24 @@ func TestGenericUpstreamNamedHosted(t *testing.T) { t.Errorf("upstream.generic named hosted with hosting on: %v, want it refused", err) } } + +func TestValidateFreeBSDUpstreams(t *testing.T) { + tests := []struct { + name string + freebsd map[string]string + wantErr bool + }{ + {"valid", map[string]string{"freebsd": "https://pkg.freebsd.org", "poudriere": "https://packages.example.com"}, false}, + {"name contains path separator", map[string]string{"freebsd/ports": "https://pkg.freebsd.org"}, true}, + {"URL is not absolute", map[string]string{"freebsd": "pkg.freebsd.org"}, true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cfg := Default() + cfg.Upstream.FreeBSD = tt.freebsd + if err := cfg.Validate(); (err != nil) != tt.wantErr { + t.Errorf("Validate() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} diff --git a/internal/server/dashboard.go b/internal/server/dashboard.go index 00a0f11..da866a6 100644 --- a/internal/server/dashboard.go +++ b/internal/server/dashboard.go @@ -556,12 +556,18 @@ Server = ` + baseURL + `/arch/archlinux/$repo/os/$arch # Then run: pacman -Syu`), }, - { - ID: "freebsd", - Name: "FreeBSD pkg", - Language: "FreeBSD", - Endpoint: "/freebsd/", - Instructions: template.HTML(`

Configure pkg to use the proxy. On FreeBSD 15, do the same for FreeBSD-ports-kmods and FreeBSD-base, keeping their paths; FreeBSD 14 calls the ports repository FreeBSD.

+ freebsdRegistryConfig(baseURL), + } +} + +// freebsdRegistryConfig describes the FreeBSD pkg endpoint for the dashboard. +func freebsdRegistryConfig(baseURL string) RegistryConfig { + return RegistryConfig{ + ID: "freebsd", + Name: "FreeBSD pkg", + Language: "FreeBSD", + Endpoint: "/freebsd/", + Instructions: template.HTML(`

Configure pkg to use the proxy. On FreeBSD 15, do the same for FreeBSD-ports-kmods and FreeBSD-base, keeping their paths; FreeBSD 14 calls the ports repository FreeBSD.

# /usr/local/etc/pkg/repos/FreeBSD.conf
 FreeBSD-ports: {
   url: "` + baseURL + `/freebsd/freebsd/${ABI}/quarterly",
@@ -570,6 +576,5 @@ FreeBSD-ports: {
 
 # Then run:
 pkg update
`), - }, } } From 25a54144271640cd6009d0eaabfadd7dc101653c Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Thu, 22 Oct 2026 13:22:13 +0200 Subject: [PATCH 345/633] Release FreeBSD binaries The proxy builds without cgo for FreeBSD, and the test suite passes on FreeBSD 15.1. --- .goreleaser.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.goreleaser.yaml b/.goreleaser.yaml index e5881db..9aec8f9 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -15,6 +15,7 @@ builds: - linux - darwin - windows + - freebsd goarch: - amd64 - arm64 From f44ce1e60fd94362928d5aee9bec83904d19adc9 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Thu, 22 Oct 2026 15:38:35 +0200 Subject: [PATCH 346/633] List mirror versions from the configured upstream proxy mirror and mirror API jobs listed the versions of unversioned PURLs from the public registries, ignoring upstream.*. They now list them from the configured upstream through the proxy's upstream client, and npm packages from their npm_routes entry. Where the upstream has no listing API, or the ecosystem has no listing at all, the error asks for a version. Refs #524 --- cmd/proxy/main.go | 7 + docs/configuration.md | 4 + internal/handler/package_routes.go | 7 + internal/mirror/job.go | 4 +- internal/mirror/mirror.go | 4 + internal/mirror/schedule.go | 2 +- internal/mirror/source.go | 47 +++---- internal/mirror/versions.go | 52 +++++++ internal/mirror/versions_test.go | 98 +++++++++++++ internal/server/mirror_schedule.go | 3 +- internal/server/mirror_versions.go | 94 +++++++++++++ internal/server/mirror_versions_test.go | 174 ++++++++++++++++++++++++ internal/server/server.go | 3 +- 13 files changed, 462 insertions(+), 37 deletions(-) create mode 100644 internal/mirror/versions.go create mode 100644 internal/mirror/versions_test.go create mode 100644 internal/server/mirror_versions.go create mode 100644 internal/server/mirror_versions_test.go diff --git a/cmd/proxy/main.go b/cmd/proxy/main.go index e3505f2..55f4836 100644 --- a/cmd/proxy/main.go +++ b/cmd/proxy/main.go @@ -558,6 +558,13 @@ func runMirror() { m := mirror.New(proxy, db, store, logger, *concurrency) m.Images = images + m.Versions = server.NewMirrorVersions(cfg.Upstream, proxy.HTTPClient) + switch s := source.(type) { + case *mirror.PURLSource: + s.Versions = m.Versions + case *mirror.SBOMSource: + s.Versions = m.Versions + } if cfg.Scanning.Enabled { refuseUnscannedMirroring(m) } diff --git a/docs/configuration.md b/docs/configuration.md index 6f7be45..e2e8f4f 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1403,6 +1403,10 @@ Positional arguments are treated as PURLs: proxy mirror pkg:npm/lodash@4.17.21 pkg:cargo/serde@1.0.0 ``` +### Unversioned PURLs + +A PURL without a version, here or in a mirror API job, mirrors every version the configured upstream lists, through the same client, so `upstream.auth` and `upstream.allow_private_hosts` apply. npm packages in `upstream.npm_routes` are listed from their route. Listing works for npm, gem, golang, hex (from `upstream.hex_api`), pub, pypi, maven, nuget, composer, cran and brew. For cargo, conda and julia the upstream doesn't serve the API used for listing, so versions come from the public registry while the upstream is the default one; with another upstream, and for Composer packages in `upstream.composer_routes` or any other ecosystem, give a version. + ### Container images `pkg:docker` and `pkg:oci` PURLs mirror a container image the way a pull does: the manifest or index, the platform manifests, then config and layer blobs. Everything is stored under the keys a pull through `/v2` reads, so a later `docker pull` or containerd pull is served from cache, also when the upstream is unreachable. diff --git a/internal/handler/package_routes.go b/internal/handler/package_routes.go index 8af7099..cb7763b 100644 --- a/internal/handler/package_routes.go +++ b/internal/handler/package_routes.go @@ -72,3 +72,10 @@ func (r packageRoutes) cacheFilename(packageName, filename string) string { } return filename } + +// RoutedPackageURL returns the upstream URL a route in routes sends +// packageName to, as the npm and Composer handlers pick it. +func RoutedPackageURL(routes map[string]string, packageName string) (string, bool) { + route, ok := newPackageRoutes(routes).match(packageName) + return route.url, ok +} diff --git a/internal/mirror/job.go b/internal/mirror/job.go index 3d82966..70a2a2a 100644 --- a/internal/mirror/job.go +++ b/internal/mirror/job.go @@ -201,9 +201,9 @@ func (js *JobStore) sourceFromRequest(req JobRequest) (Source, error) { //nolint case len(req.PURLs) > 0 && len(req.SBOM) > 0: return nil, fmt.Errorf("request must include only one of purls or sbom") case len(req.PURLs) > 0: - return &PURLSource{PURLs: req.PURLs, Platforms: req.Platforms}, nil + return &PURLSource{PURLs: req.PURLs, Versions: js.mirror.Versions, Platforms: req.Platforms}, nil case len(req.SBOM) > 0: - return &SBOMSource{Data: req.SBOM, Platforms: req.Platforms}, nil + return &SBOMSource{Data: req.SBOM, Versions: js.mirror.Versions, Platforms: req.Platforms}, nil case req.Registry != "": return nil, fmt.Errorf("registry mirroring is not yet implemented; use purls instead") default: diff --git a/internal/mirror/mirror.go b/internal/mirror/mirror.go index e946825..4b79ce9 100644 --- a/internal/mirror/mirror.go +++ b/internal/mirror/mirror.go @@ -27,6 +27,10 @@ type Mirror struct { // other unsupported ecosystem. Images ImageMirrorer + // Versions lists the versions of unversioned PURLs in mirror jobs and + // schedules. Nil lists them from the public registries. + Versions VersionLister + // RefusePackages, when set, fails every item Images does not take with // this reason instead of fetching it, so nothing is cached. RefusePackages string diff --git a/internal/mirror/schedule.go b/internal/mirror/schedule.go index 4ab0d2b..b224ea2 100644 --- a/internal/mirror/schedule.go +++ b/internal/mirror/schedule.go @@ -76,7 +76,7 @@ func (s *Scheduler) loop(ctx context.Context, index int, schedule Schedule) { func (s *Scheduler) run(ctx context.Context, index int, schedule Schedule) { started := time.Now() - progress, err := s.mirror.Run(ctx, &PURLSource{PURLs: schedule.PURLs, Platforms: schedule.Platforms}) + progress, err := s.mirror.Run(ctx, &PURLSource{PURLs: schedule.PURLs, Versions: s.mirror.Versions, Platforms: schedule.Platforms}) if ctx.Err() != nil { return } diff --git a/internal/mirror/source.go b/internal/mirror/source.go index 8fcf11c..3e40873 100644 --- a/internal/mirror/source.go +++ b/internal/mirror/source.go @@ -5,8 +5,6 @@ import ( "fmt" "github.com/git-pkgs/purl" - "github.com/git-pkgs/registries" - _ "github.com/git-pkgs/registries/all" "github.com/git-pkgs/sbom" ) @@ -32,18 +30,21 @@ type Source interface { } // PURLSource yields packages from PURL strings. -// Versioned PURLs produce a single item. Unversioned PURLs look up all versions from the registry. +// Versioned PURLs produce a single item. Unversioned PURLs produce one item +// per version Versions lists. type PURLSource struct { - PURLs []string - RegClient *registries.Client + PURLs []string + // Versions lists the versions of unversioned PURLs. Nil lists them from + // the public registries. + Versions VersionLister // Platforms limit which platforms of container images are mirrored. Platforms []string } func (s *PURLSource) Enumerate(ctx context.Context, fn func(PackageVersion) error) error { - client := s.RegClient - if client == nil { - client = registries.DefaultClient() + versions := s.Versions + if versions == nil { + versions = RegistryVersions{} } for _, purlStr := range s.PURLs { @@ -73,11 +74,11 @@ func (s *PURLSource) Enumerate(ctx context.Context, fn func(PackageVersion) erro } // Unversioned: enumerate all versions - versions, err := s.fetchVersions(ctx, client, ecosystem, name) + listed, err := versions.ListVersions(ctx, ecosystem, name) if err != nil { return fmt.Errorf("fetching versions for %s/%s: %w", ecosystem, name, err) } - for _, v := range versions { + for _, v := range listed { if err := fn(PackageVersion{Ecosystem: ecosystem, Name: name, Version: v}); err != nil { return err } @@ -86,27 +87,13 @@ func (s *PURLSource) Enumerate(ctx context.Context, fn func(PackageVersion) erro return nil } -func (s *PURLSource) fetchVersions(ctx context.Context, client *registries.Client, ecosystem, name string) ([]string, error) { - reg, err := registries.New(purl.EcosystemToPURLType(ecosystem), "", client) - if err != nil { - return nil, err - } - versions, err := reg.FetchVersions(ctx, name) - if err != nil { - return nil, err - } - result := make([]string, len(versions)) - for i, v := range versions { - result[i] = v.Number - } - return result, nil -} - // SBOMSource extracts package versions from CycloneDX or SPDX SBOM data. type SBOMSource struct { - Data []byte - Name string - RegClient *registries.Client + Data []byte + Name string + // Versions lists the versions of unversioned PURLs. Nil lists them from + // the public registries. + Versions VersionLister // Platforms limit which platforms of container images are mirrored. Platforms []string } @@ -120,7 +107,7 @@ func (s *SBOMSource) Enumerate(ctx context.Context, fn func(PackageVersion) erro return fmt.Errorf("parsing SBOM: %w", err) } - inner := &PURLSource{PURLs: purls, RegClient: s.RegClient, Platforms: s.Platforms} + inner := &PURLSource{PURLs: purls, Versions: s.Versions, Platforms: s.Platforms} return inner.Enumerate(ctx, fn) } diff --git a/internal/mirror/versions.go b/internal/mirror/versions.go new file mode 100644 index 0000000..f62102b --- /dev/null +++ b/internal/mirror/versions.go @@ -0,0 +1,52 @@ +package mirror + +import ( + "context" + + "github.com/git-pkgs/purl" + "github.com/git-pkgs/registries" + _ "github.com/git-pkgs/registries/all" +) + +// VersionLister lists the versions of a package, for PURLs without one. +type VersionLister interface { + ListVersions(ctx context.Context, ecosystem, name string) ([]string, error) +} + +// RegistryVersions lists versions through the registries clients. +type RegistryVersions struct { + // Client makes the requests. Nil uses registries.DefaultClient. + Client *registries.Client + // URL returns the registry base URL to list a package from, by PURL + // type. Nil, or an empty URL, uses the public registry. + URL func(purlType, name string) (string, error) +} + +// ListVersions implements VersionLister. +func (r RegistryVersions) ListVersions(ctx context.Context, ecosystem, name string) ([]string, error) { + purlType := purl.EcosystemToPURLType(ecosystem) + var baseURL string + if r.URL != nil { + var err error + if baseURL, err = r.URL(purlType, name); err != nil { + return nil, err + } + } + client := r.Client + if client == nil { + client = registries.DefaultClient() + } + reg, err := registries.New(purlType, baseURL, client) + if err != nil { + return nil, err + } + versions, err := reg.FetchVersions(ctx, name) + if err != nil { + return nil, err + } + result := make([]string, len(versions)) + for i, v := range versions { + result[i] = v.Number + } + return result, nil +} diff --git a/internal/mirror/versions_test.go b/internal/mirror/versions_test.go new file mode 100644 index 0000000..9c04a21 --- /dev/null +++ b/internal/mirror/versions_test.go @@ -0,0 +1,98 @@ +package mirror + +import ( + "context" + "encoding/json" + "errors" + "reflect" + "testing" +) + +// fakeVersions lists fixed versions and records what it was asked for. +type fakeVersions struct { + versions []string + err error + asked []string +} + +func (f *fakeVersions) ListVersions(_ context.Context, ecosystem, name string) ([]string, error) { + f.asked = append(f.asked, ecosystem+"/"+name) + return f.versions, f.err +} + +func enumerate(t *testing.T, source Source) []PackageVersion { + t.Helper() + var items []PackageVersion + if err := source.Enumerate(context.Background(), func(pv PackageVersion) error { + items = append(items, pv) + return nil + }); err != nil { + t.Fatalf("Enumerate() error = %v", err) + } + return items +} + +func TestPURLSourceListsVersionsThroughLister(t *testing.T) { + versions := &fakeVersions{versions: []string{"1.0.0", "1.1.0"}} + source := &PURLSource{PURLs: []string{"pkg:npm/%40acme/widget"}, Versions: versions} + + items := enumerate(t, source) + + want := []PackageVersion{ + {Ecosystem: "npm", Name: "@acme/widget", Version: "1.0.0"}, + {Ecosystem: "npm", Name: "@acme/widget", Version: "1.1.0"}, + } + if !reflect.DeepEqual(items, want) { + t.Errorf("items = %v, want %v", items, want) + } + if !reflect.DeepEqual(versions.asked, []string{"npm/@acme/widget"}) { + t.Errorf("lister asked for %v, want npm/@acme/widget", versions.asked) + } +} + +func TestPURLSourceReportsListerError(t *testing.T) { + source := &PURLSource{PURLs: []string{"pkg:npm/left-pad"}, Versions: &fakeVersions{err: errors.New("upstream down")}} + err := source.Enumerate(context.Background(), func(PackageVersion) error { return nil }) + if err == nil { + t.Fatal("Enumerate() succeeded although listing versions failed") + } +} + +func TestSBOMSourceListsVersionsThroughLister(t *testing.T) { + bom := map[string]any{ + "bomFormat": "CycloneDX", + "specVersion": "1.4", + "components": []map[string]any{{"type": "library", "name": "left-pad", "purl": "pkg:npm/left-pad"}}, + } + versions := &fakeVersions{versions: []string{"1.3.0"}} + source := &SBOMSource{Data: marshalJSON(t, bom), Versions: versions} + + items := enumerate(t, source) + + if len(items) != 1 || items[0].Version != "1.3.0" { + t.Errorf("items = %v, want left-pad@1.3.0 from the lister", items) + } +} + +func TestSourceFromRequestUsesMirrorVersions(t *testing.T) { + m := setupTestMirror(t, 1) + versions := &fakeVersions{} + m.Versions = versions + js := NewJobStore(context.Background(), m) + + source, err := js.sourceFromRequest(JobRequest{PURLs: []string{"pkg:npm/left-pad"}}) + if err != nil { + t.Fatalf("sourceFromRequest() error = %v", err) + } + if got := source.(*PURLSource).Versions; got != versions { + t.Errorf("PURL source lists versions through %v, want the mirror's lister", got) + } + + source, err = js.sourceFromRequest(JobRequest{SBOM: json.RawMessage(`{"bomFormat":"CycloneDX","components":[]}`)}) + if err != nil { + t.Fatalf("sourceFromRequest() error = %v", err) + } + if got := source.(*SBOMSource).Versions; got != versions { + t.Errorf("SBOM source lists versions through %v, want the mirror's lister", got) + } +} diff --git a/internal/server/mirror_schedule.go b/internal/server/mirror_schedule.go index 3f69e7f..3f0b77e 100644 --- a/internal/server/mirror_schedule.go +++ b/internal/server/mirror_schedule.go @@ -17,8 +17,7 @@ func (s *Server) startMirrorSchedules(ctx context.Context, proxy *handler.Proxy) return } // One worker, so a schedule never fans out against the registry. - m := mirror.New(proxy, s.db, s.storage, s.logger, 1) - m.Images = s.newContainerHandler(proxy) + m := s.newMirror(proxy, 1) schedules := make([]mirror.Schedule, len(s.cfg.Mirror.Schedules)) for i, schedule := range s.cfg.Mirror.Schedules { schedules[i] = mirror.Schedule{PURLs: schedule.Images, Platforms: schedule.Platforms, Interval: schedule.IntervalDuration()} diff --git a/internal/server/mirror_versions.go b/internal/server/mirror_versions.go new file mode 100644 index 0000000..9b981fd --- /dev/null +++ b/internal/server/mirror_versions.go @@ -0,0 +1,94 @@ +package server + +import ( + "cmp" + "fmt" + "net/http" + "strings" + + "github.com/git-pkgs/proxy/internal/config" + "github.com/git-pkgs/proxy/internal/handler" + "github.com/git-pkgs/proxy/internal/mirror" + "github.com/git-pkgs/registries" +) + +// NewMirrorVersions lists versions for proxy mirror from the configured +// upstreams, through client, so upstream.auth and the private address rules +// apply as they do when serving. +func NewMirrorVersions(upstream config.UpstreamConfig, client *http.Client) mirror.VersionLister { //nolint:ireturn // the mirror takes any lister + return mirror.RegistryVersions{ + Client: registries.NewClient(registries.WithHTTPClient(client)), + URL: func(purlType, name string) (string, error) { + return mirrorVersionsURL(upstream, purlType, name) + }, + } +} + +// newMirror builds the mirror for mirror API jobs and schedules, which fetch +// and list versions from the configured upstreams. +func (s *Server) newMirror(proxy *handler.Proxy, workers int) *mirror.Mirror { + m := mirror.New(proxy, s.db, s.storage, s.logger, workers) + m.Images = s.newContainerHandler(proxy) + m.Versions = NewMirrorVersions(s.cfg.Upstream, proxy.HTTPClient) + return m +} + +// mirrorVersionsURL returns the registry base URL to list a package's +// versions from: the upstream the proxy serves it from, where that upstream +// has the API the registries client reads. An empty URL means the public +// registry, used only while the upstream is the public one too. +func mirrorVersionsURL(u config.UpstreamConfig, purlType, name string) (string, error) { + d := config.Default().Upstream + switch purlType { + case "npm": + if routed, ok := handler.RoutedPackageURL(u.NPMRoutes, name); ok { + return routed, nil + } + return upstreamOr(u.NPM, d.NPM), nil + case "gem": + return upstreamOr(u.Gem, d.Gem), nil + case "golang": + return upstreamOr(u.Go, d.Go), nil + case "hex": + return upstreamOr(u.HexAPI, d.HexAPI), nil + case "pub": + return upstreamOr(u.Pub, d.Pub), nil + case "pypi": + return upstreamOr(u.PyPI, d.PyPI), nil + case "maven": + return upstreamOr(u.Maven, d.Maven), nil + case "nuget": + return upstreamOr(u.NuGet, d.NuGet) + "/v3", nil + case "composer": + if _, ok := handler.RoutedPackageURL(u.ComposerRoutes, name); ok { + return "", fmt.Errorf("listing versions of packages in upstream.composer_routes isn't supported; give a version") + } + return upstreamOr(u.Composer, d.Composer), nil + case "cran": + return upstreamOr(u.CRAN, d.CRAN), nil + case "brew": + return strings.TrimSuffix(upstreamOr(u.HomebrewAPI, d.HomebrewAPI), "/api"), nil + case "cargo": + return publicOnly("upstream.cargo", u.Cargo, d.Cargo) + case "conda": + return publicOnly("upstream.conda", u.Conda, d.Conda) + case "julia": + return publicOnly("upstream.julia", u.Julia, d.Julia) + default: + return "", fmt.Errorf("listing %s versions isn't supported; give a version", purlType) + } +} + +func upstreamOr(configured, fallback string) string { + return strings.TrimSuffix(cmp.Or(configured, fallback), "/") +} + +// publicOnly is for upstreams that don't serve the API the registries +// client reads: versions come from the public registry while the upstream +// is the public one, and can't be listed otherwise. +func publicOnly(setting, configured, fallback string) (string, error) { + if upstreamOr(configured, fallback) != strings.TrimSuffix(fallback, "/") { + return "", fmt.Errorf("listing versions from a custom %s isn't supported; give a version", setting) + } + return "", nil +} diff --git a/internal/server/mirror_versions_test.go b/internal/server/mirror_versions_test.go new file mode 100644 index 0000000..898c212 --- /dev/null +++ b/internal/server/mirror_versions_test.go @@ -0,0 +1,174 @@ +package server + +import ( + "context" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "slices" + "strings" + "sync/atomic" + "testing" + + "github.com/git-pkgs/proxy/internal/config" +) + +// newPackumentUpstream serves an npm packument with the given versions for +// path and counts the requests it gets. +func newPackumentUpstream(t *testing.T, path string, versions ...string) (*httptest.Server, *atomic.Int32) { + t.Helper() + var hits atomic.Int32 + entries := make([]string, len(versions)) + for i, v := range versions { + entries[i] = `"` + v + `":{"version":"` + v + `"}` + } + body := `{"name":"x","versions":{` + strings.Join(entries, ",") + `}}` + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if !strings.EqualFold(r.URL.EscapedPath(), path) { + http.NotFound(w, r) + return + } + hits.Add(1) + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, body) + })) + t.Cleanup(upstream.Close) + return upstream, &hits +} + +func listMirrorVersions(t *testing.T, cfg *config.Config, ecosystem, name string) ([]string, error) { + t.Helper() + cfg.Upstream.AllowLoopback = true + proxy, _ := NewMirrorProxy(cfg, nil, nil, slog.New(slog.NewTextHandler(io.Discard, nil))) + versions, err := NewMirrorVersions(cfg.Upstream, proxy.HTTPClient).ListVersions(context.Background(), ecosystem, name) + slices.Sort(versions) + return versions, err +} + +func TestMirrorVersionsComeFromConfiguredUpstream(t *testing.T) { + upstream, hits := newPackumentUpstream(t, "/left-pad", "1.0.0", "1.1.0") + cfg := config.Default() + cfg.Upstream.NPM = upstream.URL + + versions, err := listMirrorVersions(t, cfg, "npm", "left-pad") + if err != nil { + t.Fatalf("ListVersions() error = %v", err) + } + if !slices.Equal(versions, []string{"1.0.0", "1.1.0"}) { + t.Errorf("versions = %v, want the configured upstream's 1.0.0 and 1.1.0", versions) + } + if hits.Load() == 0 { + t.Error("the configured npm upstream was never asked") + } +} + +func TestMirrorVersionsFollowNPMRoutes(t *testing.T) { + public, publicHits := newPackumentUpstream(t, "/@acme%2fwidget", "9.9.9") + private, _ := newPackumentUpstream(t, "/@acme%2fwidget", "1.0.0") + cfg := config.Default() + cfg.Upstream.NPM = public.URL + cfg.Upstream.NPMRoutes = map[string]string{"@acme/*": private.URL} + + versions, err := listMirrorVersions(t, cfg, "npm", "@acme/widget") + if err != nil { + t.Fatalf("ListVersions() error = %v", err) + } + if !slices.Equal(versions, []string{"1.0.0"}) { + t.Errorf("versions = %v, want only the routed registry's 1.0.0", versions) + } + if publicHits.Load() != 0 { + t.Error("a routed package was listed from upstream.npm") + } +} + +func TestMirrorVersionsURL(t *testing.T) { + custom := config.Default().Upstream + custom.NPM = "https://npm.example.com/" + custom.Gem = "https://gems.example.com" + custom.Go = "https://goproxy.example.com" + custom.HexAPI = "https://hex.example.com" + custom.Pub = "https://pub.example.com" + custom.PyPI = "https://pypi.example.com" + custom.Maven = "https://maven.example.com/maven2" + custom.NuGet = "https://nuget.example.com" + custom.Composer = "https://packagist.example.com" + custom.ComposerRoutes = map[string]string{"acme/*": "https://composer.example.com"} + custom.CRAN = "https://cran.example.com" + custom.HomebrewAPI = "https://brew.example.com/api" + custom.Cargo = "https://index.example.com" + custom.Conda = "https://conda.example.com" + custom.Julia = "https://julia.example.com" + + tests := []struct { + name string + upstream config.UpstreamConfig + ecosystem string + pkg string + want string + wantErr string + }{ + {"npm default", config.Default().Upstream, "npm", "left-pad", "https://registry.npmjs.org", ""}, + {"npm unset", config.UpstreamConfig{}, "npm", "left-pad", "https://registry.npmjs.org", ""}, + {"npm", custom, "npm", "left-pad", "https://npm.example.com", ""}, + {"gem", custom, "gem", "rails", "https://gems.example.com", ""}, + {"golang", custom, "golang", "golang.org/x/mod", "https://goproxy.example.com", ""}, + {"hex uses the API URL", custom, "hex", "jason", "https://hex.example.com", ""}, + {"pub", custom, "pub", "http", "https://pub.example.com", ""}, + {"pypi", custom, "pypi", "requests", "https://pypi.example.com", ""}, + {"maven", custom, "maven", "org.example:lib", "https://maven.example.com/maven2", ""}, + {"nuget lists through v3", custom, "nuget", "Newtonsoft.Json", "https://nuget.example.com/v3", ""}, + {"composer", custom, "composer", "monolog/monolog", "https://packagist.example.com", ""}, + {"composer route", custom, "composer", "acme/widget", "", "upstream.composer_routes"}, + {"cran", custom, "cran", "ggplot2", "https://cran.example.com", ""}, + {"brew", custom, "brew", "wget", "https://brew.example.com", ""}, + {"cargo default lists from crates.io", config.Default().Upstream, "cargo", "serde", "", ""}, + {"cargo index", custom, "cargo", "serde", "", "upstream.cargo"}, + {"conda default", config.Default().Upstream, "conda", "numpy", "", ""}, + {"conda channel", custom, "conda", "numpy", "", "upstream.conda"}, + {"julia default", config.Default().Upstream, "julia", "JSON", "", ""}, + {"julia server", custom, "julia", "JSON", "", "upstream.julia"}, + {"helm", custom, "helm", "nginx", "", "give a version"}, + {"swift", custom, "swift", "apple/swift-nio", "", "give a version"}, + {"apk", custom, "apk", "curl", "", "give a version"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := mirrorVersionsURL(tt.upstream, tt.ecosystem, tt.pkg) + if tt.wantErr != "" { + if err == nil || !strings.Contains(err.Error(), tt.wantErr) { + t.Fatalf("error = %v, want one mentioning %q", err, tt.wantErr) + } + return + } + if err != nil { + t.Fatalf("error = %v", err) + } + if got != tt.want { + t.Errorf("URL = %q, want %q", got, tt.want) + } + }) + } +} + +func TestServerMirrorListsFromConfiguredUpstream(t *testing.T) { + upstream, hits := newPackumentUpstream(t, "/left-pad", "1.0.0") + cfg := config.Default() + cfg.Upstream.NPM = upstream.URL + cfg.Upstream.AllowLoopback = true + logger := slog.New(slog.NewTextHandler(io.Discard, nil)) + proxy, _ := NewMirrorProxy(cfg, nil, nil, logger) + s := &Server{cfg: cfg, logger: logger, ecr: newECRTokens(logger)} + + m := s.newMirror(proxy, 1) + if m.Versions == nil { + t.Fatal("the server's mirror lists versions from the public registries") + } + versions, err := m.Versions.ListVersions(context.Background(), "npm", "left-pad") + if err != nil { + t.Fatalf("ListVersions() error = %v", err) + } + if !slices.Equal(versions, []string{"1.0.0"}) || hits.Load() == 0 { + t.Errorf("versions = %v, want 1.0.0 from the configured upstream", versions) + } +} diff --git a/internal/server/server.go b/internal/server/server.go index 8a0a683..96d3b24 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -380,8 +380,7 @@ func (s *Server) serve(listener net.Listener) error { // Mirror API endpoints (opt-in via mirror_api config or PROXY_MIRROR_API env) if s.cfg.MirrorAPI { - mirrorSvc := mirror.New(proxy, s.db, s.storage, s.logger, 4) //nolint:mnd // default concurrency - mirrorSvc.Images = s.newContainerHandler(proxy) + mirrorSvc := s.newMirror(proxy, 4) //nolint:mnd // default concurrency jobStore := mirror.NewJobStore(bgCtx, mirrorSvc) mirrorAPI := NewMirrorAPIHandler(jobStore) r.Post("/api/mirror", mirrorAPI.HandleCreate) From baeccd4408a9a4cca66da4bd36fea9acf9afa08d Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Thu, 22 Oct 2026 15:57:14 +0200 Subject: [PATCH 347/633] Mirror gems and composer packages under the handler names proxy mirror filed pkg:gem and pkg:composer packages under rubygems and packagist, the names purl gives their registries, while the handlers store them under gem and composer. Since the resolver accepts both spellings, mirrored gems were downloaded and filed under rubygems, so the packages list filtered by gem didn't show them. The mirror now uses the handler names. The package list, count and search match every spelling of the requested ecosystem, so rows already written as rubygems or packagist, by the mirror or by git-pkgs, show up too without a migration. Fixes #415 --- internal/database/ecosystem_filter.go | 34 +++++++++++++++++++ internal/database/queries.go | 20 ++++++----- .../database/queries_packages_list_test.go | 32 +++++++++++++++++ internal/mirror/source.go | 14 +++++++- internal/mirror/source_test.go | 24 +++++++++++++ 5 files changed, 115 insertions(+), 9 deletions(-) create mode 100644 internal/database/ecosystem_filter.go diff --git a/internal/database/ecosystem_filter.go b/internal/database/ecosystem_filter.go new file mode 100644 index 0000000..17d66ba --- /dev/null +++ b/internal/database/ecosystem_filter.go @@ -0,0 +1,34 @@ +package database + +import ( + "slices" + "strings" + + "github.com/git-pkgs/purl" +) + +// ecosystemSpellings returns the names an ecosystem's packages can be stored +// under. The proxy's handlers write "gem" and "composer" where git-pkgs, and +// proxy mirror before it used the handler names, write "rubygems" and +// "packagist". +func ecosystemSpellings(ecosystem string) []string { + purlType := purl.EcosystemToPURLType(ecosystem) + spellings := []string{ecosystem} + for _, s := range []string{purlType, purl.PURLTypeToEcosystem(purlType)} { + if s != "" && !slices.Contains(spellings, s) { + spellings = append(spellings, s) + } + } + return spellings +} + +// ecosystemFilter returns a condition matching p.ecosystem against every +// spelling of ecosystem, and its arguments. +func ecosystemFilter(ecosystem string) (string, []any) { + spellings := ecosystemSpellings(ecosystem) + args := make([]any, len(spellings)) + for i, s := range spellings { + args[i] = s + } + return "p.ecosystem IN (" + strings.TrimSuffix(strings.Repeat("?, ", len(spellings)), ", ") + ")", args +} diff --git a/internal/database/queries.go b/internal/database/queries.go index 9560d88..3afb134 100644 --- a/internal/database/queries.go +++ b/internal/database/queries.go @@ -795,6 +795,7 @@ func (db *DB) SearchPackages(query string, ecosystem string, limit int, offset i var args []any if ecosystem != "" { + condition, ecosystemArgs := ecosystemFilter(ecosystem) sqlQuery = db.Rebind(` SELECT p.ecosystem, p.name, p.latest_version, p.license, COALESCE(SUM(a.hit_count), 0) as hits, @@ -803,12 +804,12 @@ func (db *DB) SearchPackages(query string, ecosystem string, limit int, offset i FROM packages p LEFT JOIN versions v ON v.package_purl = p.purl LEFT JOIN artifacts a ON a.version_purl = v.purl - WHERE p.name LIKE ? AND p.ecosystem = ? AND a.storage_path IS NOT NULL + WHERE p.name LIKE ? AND ` + condition + ` AND a.storage_path IS NOT NULL GROUP BY p.purl, p.ecosystem, p.name, p.latest_version, p.license ORDER BY hits DESC LIMIT ? OFFSET ? `) - args = []any{searchPattern, ecosystem, limit, offset} + args = append(append([]any{searchPattern}, ecosystemArgs...), limit, offset) } else { sqlQuery = db.Rebind(` SELECT p.ecosystem, p.name, p.latest_version, p.license, @@ -847,14 +848,15 @@ func (db *DB) CountSearchResults(query string, ecosystem string) (int64, error) var args []any if ecosystem != "" { + condition, ecosystemArgs := ecosystemFilter(ecosystem) sqlQuery = db.Rebind(` SELECT COUNT(DISTINCT p.purl) FROM packages p LEFT JOIN versions v ON v.package_purl = p.purl LEFT JOIN artifacts a ON a.version_purl = v.purl - WHERE p.name LIKE ? AND p.ecosystem = ? AND a.storage_path IS NOT NULL + WHERE p.name LIKE ? AND ` + condition + ` AND a.storage_path IS NOT NULL `) - args = []any{searchPattern, ecosystem} + args = append([]any{searchPattern}, ecosystemArgs...) } else { sqlQuery = db.Rebind(` SELECT COUNT(DISTINCT p.purl) @@ -1093,8 +1095,9 @@ func (db *DB) ListCachedPackages(ecosystem string, sortBy string, limit int, off whereClause := "WHERE a.storage_path IS NOT NULL" args := []any{} if ecosystem != "" { - whereClause += " AND p.ecosystem = ?" - args = append(args, ecosystem) + condition, ecosystemArgs := ecosystemFilter(ecosystem) + whereClause += " AND " + condition + args = append(args, ecosystemArgs...) } groupByClause := "GROUP BY p.purl, p.ecosystem, p.name, p.latest_version, p.license" @@ -1176,8 +1179,9 @@ func (db *DB) CountCachedPackages(ecosystem string) (int64, error) { whereClause := "WHERE a.storage_path IS NOT NULL" args := []any{} if ecosystem != "" { - whereClause += " AND p.ecosystem = ?" - args = append(args, ecosystem) + condition, ecosystemArgs := ecosystemFilter(ecosystem) + whereClause += " AND " + condition + args = append(args, ecosystemArgs...) } query := fmt.Sprintf(` diff --git a/internal/database/queries_packages_list_test.go b/internal/database/queries_packages_list_test.go index e30d364..325fee7 100644 --- a/internal/database/queries_packages_list_test.go +++ b/internal/database/queries_packages_list_test.go @@ -218,3 +218,35 @@ func TestListCachedPackages(t *testing.T) { } }) } + +// git-pkgs, and proxy mirror before it used the handler names, wrote +// "rubygems" where the gem handler writes "gem". Filtering by either name +// finds both. +func TestCachedPackageFiltersMatchEcosystemAliases(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + seedArtifact(t, db, "rubygems", "rake", "13.2.1", 100, 1) + seedArtifact(t, db, "gem", "rails", "8.0.0", 100, 1) + seedArtifact(t, db, "npm", "express", "1.0.0", 100, 1) + + for _, ecosystem := range []string{"gem", "rubygems"} { + listed, err := db.ListCachedPackages(ecosystem, "name", 50, 0) + if err != nil { + t.Fatal(err) + } + var names []string + for _, p := range listed { + names = append(names, p.Name) + } + if len(names) != 2 || names[0] != "rails" || names[1] != "rake" { + t.Errorf("ListCachedPackages(%q) = %v, want rails and rake", ecosystem, names) + } + if count, err := db.CountCachedPackages(ecosystem); err != nil || count != 2 { + t.Errorf("CountCachedPackages(%q) = %d, %v, want 2", ecosystem, count, err) + } + found, err := db.SearchPackages("ra", ecosystem, 50, 0) + if err != nil || len(found) != 2 { + t.Errorf("SearchPackages(ra, %q) = %d results, %v, want 2", ecosystem, len(found), err) + } + } + }) +} diff --git a/internal/mirror/source.go b/internal/mirror/source.go index 8fcf11c..8a4100f 100644 --- a/internal/mirror/source.go +++ b/internal/mirror/source.go @@ -26,6 +26,18 @@ func (pv PackageVersion) String() string { return fmt.Sprintf("pkg:%s/%s@%s", pv.Ecosystem, pv.Name, pv.Version) } +// handlerEcosystem returns the ecosystem name the proxy's handlers store +// packages of a PURL type under. purl names gem and composer after their +// registries, rubygems and packagist; the handlers use the type. +func handlerEcosystem(purlType string) string { + switch purlType { + case "gem", "composer": + return purlType + default: + return purl.PURLTypeToEcosystem(purlType) + } +} + // Source produces PackageVersion items for mirroring. type Source interface { Enumerate(ctx context.Context, fn func(PackageVersion) error) error @@ -52,7 +64,7 @@ func (s *PURLSource) Enumerate(ctx context.Context, fn func(PackageVersion) erro return fmt.Errorf("parsing PURL %q: %w", purlStr, err) } - ecosystem := purl.PURLTypeToEcosystem(p.Type) + ecosystem := handlerEcosystem(p.Type) name := p.Name if p.Namespace != "" { name = p.Namespace + "/" + p.Name diff --git a/internal/mirror/source_test.go b/internal/mirror/source_test.go index 24fbbc0..9c69afe 100644 --- a/internal/mirror/source_test.go +++ b/internal/mirror/source_test.go @@ -228,3 +228,27 @@ func marshalJSON(t *testing.T, v any) []byte { } return data } + +// The mirror files packages under the ecosystem names the handlers use, so +// a mirrored package and the same package cached by a client share one name. +func TestPURLSourceUsesHandlerEcosystemNames(t *testing.T) { + source := &PURLSource{PURLs: []string{ + "pkg:gem/rake@13.2.1", + "pkg:composer/psr/log@3.0.2", + "pkg:apk/curl@8.14.1-r2", + "pkg:npm/left-pad@1.3.0", + }} + + var got []string + if err := source.Enumerate(context.Background(), func(pv PackageVersion) error { + got = append(got, pv.Ecosystem) + return nil + }); err != nil { + t.Fatalf("Enumerate() error = %v", err) + } + + want := []string{"gem", "composer", "alpine", "npm"} + if strings.Join(got, ",") != strings.Join(want, ",") { + t.Errorf("ecosystems = %v, want %v", got, want) + } +} From e92afe967e1250eff3d01828798854e4b326e198 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Thu, 22 Oct 2026 16:00:58 +0200 Subject: [PATCH 348/633] Mirror packages from the configured upstreams proxy mirror and mirror API jobs resolved download URLs with a bare resolver, so npm, cargo, gem, golang, hex, pub, maven and nuget packages came from the public registries whatever upstream.* says. The resolver now takes the configured upstreams as base URLs. npm packages in upstream.npm_routes are fetched from their route and stored under the route's filename prefix, the way the npm handler does, so a public package can't stand in for a routed one and clients are served the mirrored tarball. Refs #414 --- docs/configuration.md | 2 + internal/handler/handler.go | 38 +++++++- internal/handler/package_routes.go | 22 ++++- internal/server/mirror_fetch_test.go | 135 +++++++++++++++++++++++++++ internal/server/mirror_resolver.go | 52 +++++++++++ internal/server/server.go | 7 +- 6 files changed, 246 insertions(+), 10 deletions(-) create mode 100644 internal/server/mirror_fetch_test.go create mode 100644 internal/server/mirror_resolver.go diff --git a/docs/configuration.md b/docs/configuration.md index 6f7be45..0c5f7b2 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1385,6 +1385,8 @@ Pushed archives are not touched: a whole-version delete skips them, and naming o The `proxy mirror` command pre-populates the cache from various sources. It accepts the same storage and database flags as `serve`. +Packages are downloaded from the configured upstreams: npm, cargo, gem, golang, hex, pub, maven and nuget from `upstream.npm`, `upstream.cargo_download`, `upstream.gem`, `upstream.go`, `upstream.hex`, `upstream.pub`, `upstream.maven` and `upstream.nuget`. npm packages in `upstream.npm_routes` come from their route and are cached under the route's keys, so clients get them from the cache as they would after an install. Mirror API jobs do the same. + | Flag | Default | Description | |------|---------|-------------| | `--sbom` | | Path to CycloneDX or SPDX SBOM file | diff --git a/internal/handler/handler.go b/internal/handler/handler.go index ff5466a..307b90d 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -152,10 +152,14 @@ func (p *Proxy) ReadMetadata(r io.Reader) ([]byte, error) { // Proxy provides shared functionality for protocol handlers. type Proxy struct { - DB *database.DB - Storage storage.Storage - Fetcher fetch.FetcherInterface - Resolver *fetch.Resolver + DB *database.DB + Storage storage.Storage + Fetcher fetch.FetcherInterface + Resolver *fetch.Resolver + // RouteFor returns how to fetch a package that doesn't come from the + // ecosystem's default upstream, such as an npm package in + // upstream.npm_routes, or nil to use Resolver. + RouteFor func(ecosystem, name string) *ArtifactRoute Logger *slog.Logger Cooldown CooldownPolicy Denylist *denylist.Policy @@ -291,6 +295,9 @@ func (p *Proxy) GetOrFetchArtifact(ctx context.Context, ecosystem, name, version if filename == "" { return nil, errors.New("resolved artifact has no filename") } + if route := p.routeFor(ecosystem, name); route != nil { + filename = route.FilenamePrefix + filename + } } if p.StreamArtifacts { if info == nil { @@ -521,8 +528,29 @@ func (p *Proxy) rejectUnusableCacheRecord(artifact *database.CachedArtifact, ver } } +// ArtifactRoute fetches a package from an upstream other than its +// ecosystem's default one. +type ArtifactRoute struct { + // Resolver finds the package's download URLs on that upstream. + Resolver *fetch.Resolver + // FilenamePrefix qualifies the artifact filename the way the handler + // does for the route, so clients are served what was fetched. + FilenamePrefix string +} + +func (p *Proxy) routeFor(ecosystem, name string) *ArtifactRoute { + if p.RouteFor == nil { + return nil + } + return p.RouteFor(ecosystem, name) +} + func (p *Proxy) resolveArtifact(ctx context.Context, ecosystem, name, version string) (*fetch.ArtifactInfo, error) { - info, err := p.Resolver.Resolve(ctx, ecosystem, name, version) + resolver := p.Resolver + if route := p.routeFor(ecosystem, name); route != nil { + resolver = route.Resolver + } + info, err := resolver.Resolve(ctx, ecosystem, name, version) if err != nil { if errors.Is(err, fetch.ErrNotFound) { return nil, ErrUpstreamNotFound diff --git a/internal/handler/package_routes.go b/internal/handler/package_routes.go index 8af7099..c56c3fb 100644 --- a/internal/handler/package_routes.go +++ b/internal/handler/package_routes.go @@ -30,11 +30,10 @@ func newPackageRoutes(patterns map[string]string) packageRoutes { routes := make([]packageRoute, 0, len(patterns)) for pattern, upstreamURL := range patterns { upstreamURL = strings.TrimSuffix(upstreamURL, "/") - digest := sha256.Sum256([]byte(upstreamURL)) routes = append(routes, packageRoute{ pattern: strings.ToLower(pattern), url: upstreamURL, - id: hex.EncodeToString(digest[:])[:routeIDLength], + id: routeID(upstreamURL), }) } sort.Slice(routes, func(i, j int) bool { @@ -46,6 +45,18 @@ func newPackageRoutes(patterns map[string]string) packageRoutes { return packageRoutes{routes: routes} } +// routeID identifies a route's upstream in cache keys and filenames. +func routeID(upstreamURL string) string { + digest := sha256.Sum256([]byte(strings.TrimSuffix(upstreamURL, "/"))) + return hex.EncodeToString(digest[:])[:routeIDLength] +} + +// RoutedFilenamePrefix returns what the npm and Composer handlers put in +// front of the artifact filenames of a package routed to routeURL. +func RoutedFilenamePrefix(routeURL string) string { + return routeID(routeURL) + "/" +} + // match returns the route for a package name, if any pattern matches it. func (r packageRoutes) match(packageName string) (packageRoute, bool) { name := strings.ToLower(packageName) @@ -72,3 +83,10 @@ func (r packageRoutes) cacheFilename(packageName, filename string) string { } return filename } + +// RoutedPackageURL returns the upstream URL a route in routes sends +// packageName to, as the npm and Composer handlers pick it. +func RoutedPackageURL(routes map[string]string, packageName string) (string, bool) { + route, ok := newPackageRoutes(routes).match(packageName) + return route.url, ok +} diff --git a/internal/server/mirror_fetch_test.go b/internal/server/mirror_fetch_test.go new file mode 100644 index 0000000..2b3970d --- /dev/null +++ b/internal/server/mirror_fetch_test.go @@ -0,0 +1,135 @@ +package server + +import ( + "context" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + + "github.com/git-pkgs/proxy/internal/config" + "github.com/git-pkgs/proxy/internal/database" + "github.com/git-pkgs/proxy/internal/handler" + "github.com/git-pkgs/proxy/internal/mirror" + "github.com/git-pkgs/proxy/internal/storage" +) + +// newTarballUpstream serves body for path, compared case-insensitively since +// scoped npm names arrive with an escaped slash, and counts the requests. +func newTarballUpstream(t *testing.T, path, body string) (*httptest.Server, *atomic.Int32) { + t.Helper() + var hits atomic.Int32 + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if !strings.EqualFold(r.URL.EscapedPath(), path) && !strings.EqualFold(r.URL.Path, path) { + http.NotFound(w, r) + return + } + hits.Add(1) + _, _ = io.WriteString(w, body) + })) + t.Cleanup(upstream.Close) + return upstream, &hits +} + +// mirrorWithConfig mirrors purls with the proxy proxy mirror builds for cfg. +func mirrorWithConfig(t *testing.T, cfg *config.Config, purls ...string) (*mirror.Progress, *handler.Proxy) { + t.Helper() + cfg.Upstream.AllowLoopback = true + db, err := database.Create(t.TempDir() + "/test.db") + if err != nil { + t.Fatal(err) + } + if err := db.MigrateSchema(); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = db.Close() }) + store, err := storage.OpenBucket(context.Background(), "file://"+t.TempDir()) + if err != nil { + t.Fatal(err) + } + logger := slog.New(slog.NewTextHandler(io.Discard, nil)) + proxy, _ := NewMirrorProxy(cfg, db, store, logger) + progress, err := mirror.New(proxy, db, store, logger, 1).Run(context.Background(), &mirror.PURLSource{PURLs: purls}) + if err != nil { + t.Fatalf("Run() error = %v", err) + } + return progress, proxy +} + +func TestMirrorFetchesFromConfiguredUpstream(t *testing.T) { + upstream, hits := newTarballUpstream(t, "/left-pad/-/left-pad-1.3.0.tgz", "tarball") + cfg := config.Default() + cfg.Upstream.NPM = upstream.URL + + progress, _ := mirrorWithConfig(t, cfg, "pkg:npm/left-pad@1.3.0") + + if progress.Completed != 1 || hits.Load() != 1 { + t.Errorf("completed = %d, upstream hits = %d, errors = %v; want the tarball from upstream.npm", progress.Completed, hits.Load(), progress.Errors) + } +} + +func TestMirrorFetchesRoutedNPMPackagesFromTheirRoute(t *testing.T) { + const tarballPath = "/@acme/widget/-/widget-1.0.0.tgz" + public, publicHits := newTarballUpstream(t, tarballPath, "public") + private, privateHits := newTarballUpstream(t, tarballPath, "private") + cfg := config.Default() + cfg.Upstream.NPM = public.URL + cfg.Upstream.NPMRoutes = map[string]string{"@acme/*": private.URL} + + progress, proxy := mirrorWithConfig(t, cfg, "pkg:npm/%40acme/widget@1.0.0") + + if progress.Completed != 1 || privateHits.Load() != 1 { + t.Errorf("completed = %d, route hits = %d, errors = %v; want the tarball from the route", progress.Completed, privateHits.Load(), progress.Errors) + } + if publicHits.Load() != 0 { + t.Error("a routed package was mirrored from upstream.npm") + } + + // A client gets the mirrored tarball from the cache once the route is down. + private.Close() + npm := handler.NewNPMHandler(proxy, "http://proxy.test", cfg.Upstream.NPM).WithPackageRoutes(cfg.Upstream.NPMRoutes) + rec := httptest.NewRecorder() + npm.Routes().ServeHTTP(rec, httptest.NewRequest(http.MethodGet, tarballPath, nil)) + if rec.Code != http.StatusOK || rec.Body.String() != "private" { + t.Errorf("tarball from cache: status = %d, body = %q, want 200 and the mirrored tarball", rec.Code, rec.Body) + } +} + +func TestMirrorResolverBaseURLs(t *testing.T) { + u := config.Default().Upstream + u.NPM = "https://npm.example.com/" + u.CargoDownload = "https://crates.example.com/crates" + u.Gem = "https://gems.example.com" + u.Go = "https://goproxy.example.com" + u.Hex = "https://hex.example.com" + u.Pub = "https://pub.example.com" + u.Maven = "https://maven.example.com/maven2" + u.NuGet = "https://nuget.example.com" + + resolver := newMirrorResolver(u) + tests := []struct { + ecosystem, name, version, want string + }{ + {"npm", "left-pad", "1.3.0", "https://npm.example.com/left-pad/-/left-pad-1.3.0.tgz"}, + {"cargo", "serde", "1.0.0", "https://crates.example.com/crates/serde/serde-1.0.0.crate"}, + {"gem", "rake", "13.2.1", "https://gems.example.com/downloads/rake-13.2.1.gem"}, + {"golang", "golang.org/x/mod", "v0.20.0", "https://goproxy.example.com/golang.org/x/mod/@v/v0.20.0.zip"}, + {"hex", "jason", "1.4.4", "https://hex.example.com/tarballs/jason-1.4.4.tar"}, + {"pub", "http", "1.2.2", "https://pub.example.com/packages/http/versions/1.2.2.tar.gz"}, + {"maven", "org.example:lib", "1.0", "https://maven.example.com/maven2/org/example/lib/1.0/lib-1.0.jar"}, + {"nuget", "Newtonsoft.Json", "13.0.3", "https://nuget.example.com/v3-flatcontainer/newtonsoft.json/13.0.3/newtonsoft.json.13.0.3.nupkg"}, + } + for _, tt := range tests { + info, err := resolver.Resolve(context.Background(), tt.ecosystem, tt.name, tt.version) + if err != nil { + t.Errorf("%s: Resolve() error = %v", tt.ecosystem, err) + continue + } + if info.URL != tt.want { + t.Errorf("%s: URL = %q, want %q", tt.ecosystem, info.URL, tt.want) + } + } +} diff --git a/internal/server/mirror_resolver.go b/internal/server/mirror_resolver.go new file mode 100644 index 0000000..6195976 --- /dev/null +++ b/internal/server/mirror_resolver.go @@ -0,0 +1,52 @@ +package server + +import ( + "github.com/git-pkgs/proxy/internal/config" + "github.com/git-pkgs/proxy/internal/handler" + "github.com/git-pkgs/registries/fetch" +) + +// newMirrorResolver builds the resolver proxy mirror and mirror jobs find +// download URLs with, using the configured upstreams. Handlers serving +// clients don't use it; they take the URLs from upstream metadata. +func newMirrorResolver(u config.UpstreamConfig) *fetch.Resolver { + return fetch.NewResolver(fetch.WithBaseURLs(map[string]string{ + "npm": u.NPM, + "cargo": u.CargoDownload, + "gem": u.Gem, + "golang": u.Go, + "hex": u.Hex, + "pub": u.Pub, + "maven": u.Maven, + "nuget": u.NuGet, + })) +} + +// configureMirrorResolvers points proxy's resolver at the configured +// upstreams, and npm packages in upstream.npm_routes at their route, so a +// public package can't stand in for a routed one. +func configureMirrorResolvers(proxy *handler.Proxy, u config.UpstreamConfig) { + proxy.Resolver = newMirrorResolver(u) + if len(u.NPMRoutes) == 0 { + return + } + routed := make(map[string]*handler.ArtifactRoute, len(u.NPMRoutes)) + for _, routeURL := range u.NPMRoutes { + routeUpstream := u + routeUpstream.NPM = routeURL + routed[routeURL] = &handler.ArtifactRoute{ + Resolver: newMirrorResolver(routeUpstream), + FilenamePrefix: handler.RoutedFilenamePrefix(routeURL), + } + } + proxy.RouteFor = func(ecosystem, name string) *handler.ArtifactRoute { + if ecosystem != "npm" { + return nil + } + routeURL, ok := handler.RoutedPackageURL(u.NPMRoutes, name) + if !ok { + return nil + } + return routed[routeURL] + } +} diff --git a/internal/server/server.go b/internal/server/server.go index 8a0a683..21f7ede 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -234,13 +234,13 @@ func (s *Server) serve(listener net.Listener) error { baseFetcher := fetch.NewFetcher(fetch.WithHTTPClient(artifactClient), fetch.WithMaxArtifactSize(s.cfg.ParseArtifactMaxSize())) fetcher := fetch.NewCircuitBreakerFetcher(baseFetcher) s.breakers = newBreakerMonitor(fetcher, s.logger) - resolver := fetch.NewResolver() cd := &cooldown.Config{ Default: s.cfg.Cooldown.Default, Ecosystems: s.cfg.Cooldown.Ecosystems, Packages: s.cfg.Cooldown.NormalizedPackages(), } - proxy := handler.NewProxy(s.db, s.storage, fetcher, resolver, s.logger) + proxy := handler.NewProxy(s.db, s.storage, fetcher, nil, s.logger) + configureMirrorResolvers(proxy, s.cfg.Upstream) proxy.HTTPClient = metadataClient proxy.RateLimits = s.rateLimits cooldownPolicy, err := cooldown.NewPolicy(cd, s.cfg.Cooldown.PackagePatterns) @@ -575,7 +575,8 @@ func NewMirrorProxy(cfg *config.Config, db *database.DB, store storage.Storage, s := &Server{cfg: cfg, db: db, storage: store, logger: logger, ecr: newECRTokens(logger)} metadataClient, artifactClient := s.newUpstreamClients(0) fetcher := fetch.NewFetcher(fetch.WithHTTPClient(artifactClient), fetch.WithMaxArtifactSize(cfg.ParseArtifactMaxSize())) - proxy := handler.NewProxy(db, store, fetcher, fetch.NewResolver(), logger) + proxy := handler.NewProxy(db, store, fetcher, nil, logger) + configureMirrorResolvers(proxy, cfg.Upstream) proxy.HTTPClient = metadataClient proxy.RateLimits = s.rateLimits return proxy, s.newContainerHandler(proxy) From c08c25fd040222b41841ab8bdf23074f89d156bb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 22 Oct 2026 15:13:38 +0000 Subject: [PATCH 349/633] Bump goreleaser/goreleaser-action from 7.2.3 to 7.3.0 Bumps [goreleaser/goreleaser-action](https://github.com/goreleaser/goreleaser-action) from 7.2.3 to 7.3.0. - [Release notes](https://github.com/goreleaser/goreleaser-action/releases) - [Commits](https://github.com/goreleaser/goreleaser-action/compare/f06c13b6b1a9625abc9e6e439d9c05a8f2190e94...a4f02ab5355a589de60d917a659ce617ed19b3de) --- updated-dependencies: - dependency-name: goreleaser/goreleaser-action dependency-version: 7.3.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index dddfef0..64c1e47 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -30,7 +30,7 @@ jobs: go-version-file: go.mod cache: false - - uses: goreleaser/goreleaser-action@f06c13b6b1a9625abc9e6e439d9c05a8f2190e94 # v7.2.3 + - uses: goreleaser/goreleaser-action@a4f02ab5355a589de60d917a659ce617ed19b3de # v7.3.0 with: version: "~> v2" args: release --clean From d154757ca3750f47ae78c839515c6697549222af Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 22 Oct 2026 15:13:43 +0000 Subject: [PATCH 350/633] Bump zizmorcore/zizmor-action from 0.6.4 to 0.7.1 Bumps [zizmorcore/zizmor-action](https://github.com/zizmorcore/zizmor-action) from 0.6.4 to 0.7.1. - [Release notes](https://github.com/zizmorcore/zizmor-action/releases) - [Commits](https://github.com/zizmorcore/zizmor-action/compare/cc914d7f3750a2d13d75c7f184a1060aa0e9d482...85b2b9fb8e3325e7a23d7d0a9d7ba5ae0f999e3e) --- updated-dependencies: - dependency-name: zizmorcore/zizmor-action dependency-version: 0.7.1 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/zizmor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/zizmor.yml b/.github/workflows/zizmor.yml index 1f6df79..02a0722 100644 --- a/.github/workflows/zizmor.yml +++ b/.github/workflows/zizmor.yml @@ -34,4 +34,4 @@ jobs: persist-credentials: false - name: Run zizmor - uses: zizmorcore/zizmor-action@cc914d7f3750a2d13d75c7f184a1060aa0e9d482 # v0.6.4 + uses: zizmorcore/zizmor-action@85b2b9fb8e3325e7a23d7d0a9d7ba5ae0f999e3e # v0.7.1 From f9d4da84008a9fa5ed21966e90afa5d1d40451f9 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Thu, 22 Oct 2026 20:41:51 +0200 Subject: [PATCH 351/633] Route every Maven artifact file through the download path Only files with a known artifact or checksum extension went through the download path; anything else, such as the .exe binaries protoc and gRPC plugins publish or .hpi and .so files, was relayed from upstream as is. Those files skipped the denylist, cooldown, scanning and the cache. Any file in a version directory named {artifact}-{version}..., the way Maven names what it publishes (timestamped for SNAPSHOTs), now takes the download path. Directory listings and catalogs are still proxied. --- internal/handler/download_test.go | 103 ++++++++++++++++++++++++++++++ internal/handler/maven.go | 16 ++++- internal/handler/maven_test.go | 25 ++++++++ 3 files changed, 142 insertions(+), 2 deletions(-) diff --git a/internal/handler/download_test.go b/internal/handler/download_test.go index ce44775..e0b45eb 100644 --- a/internal/handler/download_test.go +++ b/internal/handler/download_test.go @@ -871,6 +871,109 @@ func TestMavenHandler_ChecksumsCachedAsArtifacts(t *testing.T) { } } +// Any file named after its coordinates goes through the download path, so +// policy applies to it whatever its extension. +func TestMavenHandler_DeniedVersionBlocksEveryArtifactFile(t *testing.T) { + proxy, _, _, fetcher := setupTestProxy(t) + setTestDenylist(t, proxy, "pkg:maven/com.google.protobuf/protoc@3.21.0") + + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + t.Errorf("denied file %s proxied to upstream", r.URL.Path) + })) + defer upstream.Close() + + h := NewMavenHandler(proxy, "http://localhost", upstream.URL, upstream.URL) + proxy.HTTPClient = upstream.Client() + + for _, name := range []string{ + "protoc-3.21.0-linux-x86_64.exe", + "protoc-3.21.0-linux-x86_64.exe.sha1", + "protoc-3.21.0.hpi", + "protoc-3.21.0.pom", + } { + fetcher.fetchCalled = false + w := httptest.NewRecorder() + h.Routes().ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/com/google/protobuf/protoc/3.21.0/"+name, nil)) + if w.Code != http.StatusForbidden { + t.Errorf("GET %s: status = %d, want %d", name, w.Code, http.StatusForbidden) + } + if fetcher.fetchCalled { + t.Errorf("GET %s: fetched a denied file", name) + } + } +} + +func TestMavenHandler_UncommonArtifactFilesCached(t *testing.T) { + proxy, _, _, fetcher := setupTestProxy(t) + + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + t.Errorf("artifact file %s proxied without caching", r.URL.Path) + })) + defer upstream.Close() + + h := NewMavenHandler(proxy, "http://localhost", upstream.URL, upstream.URL) + proxy.HTTPClient = upstream.Client() + + for _, path := range []string{ + "/com/google/protobuf/protoc/3.21.0/protoc-3.21.0-linux-x86_64.exe", + "/com/google/protobuf/protoc/3.21.0/protoc-3.21.0-linux-x86_64.exe.sha1", + "/org/example/plugin/1.0/plugin-1.0.hpi", + "/org/example/native/1.0/native-1.0-linux.so", + "/org/example/lib/1.0-SNAPSHOT/lib-1.0-20240101.120000-1.exe", + } { + for i := range 2 { + fetcher.artifact = &fetch.Artifact{ + Body: io.NopCloser(strings.NewReader("binary")), + ContentType: "application/octet-stream", + } + fetcher.fetchCalled = false + + w := httptest.NewRecorder() + h.Routes().ServeHTTP(w, httptest.NewRequest(http.MethodGet, path, nil)) + + if w.Code != http.StatusOK || w.Body.String() != "binary" { + t.Errorf("GET %s: status %d body %q", path, w.Code, w.Body.String()) + } + if fetcher.fetchCalled != (i == 0) { + t.Errorf("GET %s request %d: fetched = %v, want %v", path, i+1, fetcher.fetchCalled, i == 0) + } + } + } +} + +func TestMavenHandler_NonArtifactPathsProxied(t *testing.T) { + proxy, _, _, fetcher := setupTestProxy(t) + + var proxied []string + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + proxied = append(proxied, r.URL.Path) + _, _ = fmt.Fprint(w, "listing") + })) + defer upstream.Close() + + h := NewMavenHandler(proxy, "http://localhost", upstream.URL, upstream.URL) + proxy.HTTPClient = upstream.Client() + + paths := []string{ + "/com/google/protobuf/protoc/3.21.0/", + "/com/google/protobuf/protoc/", + "/archetype-catalog.xml", + } + for _, path := range paths { + w := httptest.NewRecorder() + h.Routes().ServeHTTP(w, httptest.NewRequest(http.MethodGet, path, nil)) + if w.Code != http.StatusOK || w.Body.String() != "listing" { + t.Errorf("GET %s: status %d body %q", path, w.Code, w.Body.String()) + } + } + if len(proxied) != len(paths) { + t.Errorf("proxied %v, want %v", proxied, paths) + } + if fetcher.fetchCalled { + t.Error("non-artifact path went through the download path") + } +} + func TestMavenHandler_CacheMiss(t *testing.T) { proxy, _, _, fetcher := setupTestProxy(t) fetcher.artifact = &fetch.Artifact{ diff --git a/internal/handler/maven.go b/internal/handler/maven.go index 8fa170f..4b677ac 100644 --- a/internal/handler/maven.go +++ b/internal/handler/maven.go @@ -135,12 +135,12 @@ func (h *MavenHandler) handleRequest(w http.ResponseWriter, r *http.Request) { return } - if h.isArtifactFile(filename) || h.isChecksumFile(filename) { + if h.isArtifactFile(filename) || h.isChecksumFile(filename) || h.isCoordinateFile(urlPath) { h.handleDownload(w, r, source, urlPath) return } - // Proxy everything else (directory listings etc.) + // Proxy everything else (directory listings, catalogs etc.) h.proxyUpstream(w, r, source, urlPath) } @@ -268,6 +268,18 @@ func (h *MavenHandler) isArtifactFile(filename string) bool { return false } +// isCoordinateFile reports whether urlPath is a file named after the +// artifact and version of its directory, {artifact}-{version}..., as Maven +// names every file it publishes, whatever the extension. A SNAPSHOT's +// files carry a timestamp in place of "SNAPSHOT". +func (h *MavenHandler) isCoordinateFile(urlPath string) bool { + _, artifact, version, filename := h.parsePath(urlPath) + if artifact == "" || version == "" { + return false + } + return strings.HasPrefix(filename, artifact+"-"+strings.TrimSuffix(version, "SNAPSHOT")) +} + // isVersionListingFile matches maven-metadata.xml and its checksums. The // listing changes with every release, so it is cached for the metadata TTL // rather than kept like an artifact. diff --git a/internal/handler/maven_test.go b/internal/handler/maven_test.go index beaa8f7..9d34b09 100644 --- a/internal/handler/maven_test.go +++ b/internal/handler/maven_test.go @@ -117,3 +117,28 @@ func TestMavenIsChecksumFile(t *testing.T) { } } } + +func TestMavenIsCoordinateFile(t *testing.T) { + h := &MavenHandler{} + + tests := []struct { + path string + want bool + }{ + {"com/google/protobuf/protoc/3.21.0/protoc-3.21.0-linux-x86_64.exe", true}, + {"com/google/protobuf/protoc/3.21.0/protoc-3.21.0-linux-x86_64.exe.sha1", true}, + {"org/example/plugin/1.0/plugin-1.0.hpi", true}, + {"org/example/lib/1.0-SNAPSHOT/lib-1.0-20240101.120000-1.exe", true}, + {"org/example/lib/1.0-SNAPSHOT/lib-1.0-SNAPSHOT.exe", true}, + {"com/google/protobuf/protoc/3.21.0/", false}, + {"com/google/protobuf/protoc/3.21.0/other-3.21.0.exe", false}, + {"com/google/protobuf/protoc/3.21.0/protoc-3.20.0.exe", false}, + {"archetype-catalog.xml", false}, + } + + for _, tt := range tests { + if got := h.isCoordinateFile(tt.path); got != tt.want { + t.Errorf("isCoordinateFile(%q) = %v, want %v", tt.path, got, tt.want) + } + } +} From 01692142a70ed1dc5d3c2ea5a3f37802d1d51a39 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Thu, 22 Oct 2026 20:42:17 +0200 Subject: [PATCH 352/633] Cache conda packages per subdir Conda download caching keyed artifacts on channel, name, version and filename only, but the same filename can hold different bytes in each subdir (python_abi-3.11-4_cp311.conda in linux-64 and osx-arm64, for example). A client on one platform could be served another platform's package. The subdir is now part of the cached filename, so each one gets its own entry. Entries cached under the old key are no longer used and are re-fetched on first request. --- internal/handler/conda.go | 5 +- internal/handler/conda_subdir_test.go | 61 +++++++++++++++++++++++++ internal/handler/download_test.go | 4 +- internal/server/retention_conda_test.go | 6 +-- 4 files changed, 70 insertions(+), 6 deletions(-) create mode 100644 internal/handler/conda_subdir_test.go diff --git a/internal/handler/conda.go b/internal/handler/conda.go index aecd01a..7f235e1 100644 --- a/internal/handler/conda.go +++ b/internal/handler/conda.go @@ -113,8 +113,11 @@ func (h *CondaHandler) handleDownload(w http.ResponseWriter, r *http.Request) { "channel", channel, "arch", arch, "name", name, "version", version) upstreamURL := h.upstreamURL + r.URL.Path + // The same filename can hold different bytes in each subdir, so the + // subdir is part of the cache key. + cacheFilename := arch + "/" + filename - result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "conda", packageName, version, filename, upstreamURL) + result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "conda", packageName, version, cacheFilename, upstreamURL) if err != nil { h.proxy.serveArtifactError(w, err, "failed to fetch package") return diff --git a/internal/handler/conda_subdir_test.go b/internal/handler/conda_subdir_test.go new file mode 100644 index 0000000..1a81e48 --- /dev/null +++ b/internal/handler/conda_subdir_test.go @@ -0,0 +1,61 @@ +package handler + +import ( + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + + "github.com/git-pkgs/registries/fetch" +) + +// The same conda filename can hold different bytes in each subdir, so each +// subdir must get its own cache entry. +func TestCondaDownloadCachesPerSubdir(t *testing.T) { + const filename = "python_abi-3.11-4_cp311.conda" + packages := map[string]string{ + "/conda-forge/linux-64/" + filename: "linux-64 bytes", + "/conda-forge/osx-arm64/" + filename: "osx-arm64 bytes", + } + + var available atomic.Bool + available.Store(true) + var requests atomic.Int32 + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + data, ok := packages[r.URL.Path] + if !available.Load() || !ok { + http.NotFound(w, r) + return + } + requests.Add(1) + _, _ = w.Write([]byte(data)) + })) + defer upstream.Close() + + proxy, _, _, _ := setupTestProxy(t) + fetcher := fetch.NewFetcher(fetch.WithHTTPClient(upstream.Client()), fetch.WithMaxRetries(0)) + proxy.Fetcher = fetcher + t.Cleanup(func() { _ = fetcher.Close() }) + routes := NewCondaHandlerWithUpstream(proxy, "http://proxy.local", upstream.URL).Routes() + + get := func(path string) string { + w := httptest.NewRecorder() + routes.ServeHTTP(w, httptest.NewRequest(http.MethodGet, path, nil)) + if w.Code != http.StatusOK { + t.Fatalf("%s: status = %d: %s", path, w.Code, w.Body.String()) + } + return w.Body.String() + } + + for _, round := range []string{"fetch", "cached"} { + for path, want := range packages { + if got := get(path); got != want { + t.Errorf("%s %s: body = %q, want %q", round, path, got, want) + } + } + available.Store(false) + } + if got := requests.Load(); got != 2 { + t.Errorf("upstream requests = %d, want 2 (one per subdir)", got) + } +} diff --git a/internal/handler/download_test.go b/internal/handler/download_test.go index ce44775..5144212 100644 --- a/internal/handler/download_test.go +++ b/internal/handler/download_test.go @@ -488,7 +488,7 @@ func TestPubHandler_CacheMiss(t *testing.T) { func TestCondaHandler_DownloadCacheHit(t *testing.T) { proxy, db, store, _ := setupTestProxy(t) - seedPackageWithPURL(t, db, store, "conda", "main/numpy", "1.24.0", "numpy-1.24.0-py311h64a7726_0.conda", "conda pkg") + seedPackageWithPURL(t, db, store, "conda", "main/numpy", "1.24.0", "linux-64/numpy-1.24.0-py311h64a7726_0.conda", "conda pkg") h := NewCondaHandler(proxy, "http://localhost") srv := httptest.NewServer(h.Routes()) @@ -511,7 +511,7 @@ func TestCondaHandler_DownloadCacheHit(t *testing.T) { func TestCondaHandler_DownloadTarBz2CacheHit(t *testing.T) { proxy, db, store, _ := setupTestProxy(t) - seedPackageWithPURL(t, db, store, "conda", "main/scipy", "1.11.0", "scipy-1.11.0-py311hb2e3ea1_0.tar.bz2", "tar bz2 data") + seedPackageWithPURL(t, db, store, "conda", "main/scipy", "1.11.0", "linux-64/scipy-1.11.0-py311hb2e3ea1_0.tar.bz2", "tar bz2 data") h := NewCondaHandler(proxy, "http://localhost") srv := httptest.NewServer(h.Routes()) diff --git a/internal/server/retention_conda_test.go b/internal/server/retention_conda_test.go index 230ecbb..d687e22 100644 --- a/internal/server/retention_conda_test.go +++ b/internal/server/retention_conda_test.go @@ -69,7 +69,7 @@ func condaGet(t *testing.T, url string) { } } -// condaArtifact returns the record of the package file named filename. +// condaArtifact returns the record cached as filename, which is {subdir}/{file}. func condaArtifact(t *testing.T, db *database.DB, filename string) database.Artifact { t.Helper() var art database.Artifact @@ -103,7 +103,7 @@ func TestCondaRegistersForRetention(t *testing.T) { func TestCondaRetentionEvictsAndRefetches(t *testing.T) { upstream, downloads := condaRetentionUpstream(t) srv, db := newCondaRetentionProxy(t, upstream) - const file = "requests-2.31.0-pyhd8ed1ab_0.conda" + const file = "noarch/requests-2.31.0-pyhd8ed1ab_0.conda" condaGet(t, srv.URL+condaRequestsPath) condaGet(t, srv.URL+condaRequestsPath) @@ -162,7 +162,7 @@ func TestCondaRetentionRefusesKeysWithoutChannel(t *testing.T) { func TestCondaRetentionPackageRule(t *testing.T) { upstream, _ := condaRetentionUpstream(t) srv, db := newCondaRetentionProxy(t, upstream) - const forgeNumpy, anacondaNumpy, requests = "numpy-1.24.0-py311h64a7726_0.conda", "numpy-1.24.0-py311h08b1b3b_0.conda", "requests-2.31.0-pyhd8ed1ab_0.conda" + const forgeNumpy, anacondaNumpy, requests = "linux-64/numpy-1.24.0-py311h64a7726_0.conda", "linux-64/numpy-1.24.0-py311h08b1b3b_0.conda", "noarch/requests-2.31.0-pyhd8ed1ab_0.conda" condaGet(t, srv.URL+condaForgeNumpyPath) condaGet(t, srv.URL+condaAnacondaNumpy) condaGet(t, srv.URL+condaRequestsPath) From 0e6ec299b397fdc246a0ff38fb3453709136e50d Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Thu, 22 Oct 2026 20:42:19 +0200 Subject: [PATCH 353/633] Refuse dot segments in package paths and upstream auth matching Routers clean literal dot segments, but encoded ones such as %2e%2e reach the handlers. Maven, conda, cran, go and gem append the decoded path to the upstream URL, so the upstream resolved it outside the configured base path. Package routes now answer 400 for a '.' or '..' segment, also when encoded twice or written with a backslash. Upstream credentials matched such a URL by its path prefix, so /scope/../other still got the credentials configured for /scope. URLs with dot segments now get no credentials. --- internal/config/config.go | 24 ++++++++ internal/config/config_test.go | 4 ++ internal/server/dot_segments_test.go | 91 ++++++++++++++++++++++++++++ internal/server/middleware.go | 17 ++++++ internal/server/server.go | 2 +- 5 files changed, 137 insertions(+), 1 deletion(-) create mode 100644 internal/server/dot_segments_test.go diff --git a/internal/config/config.go b/internal/config/config.go index cbb4e6e..96487d4 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -1068,6 +1068,12 @@ func parseAuthURL(value string) (*url.URL, error) { } func authURLMatches(configured, target *url.URL) bool { + // An upstream resolves dot segments, so /scope/../other would leave the + // configured path while still matching it by prefix. Real package URLs + // never contain them, so they get no credentials at all. + if HasDotSegment(target.Path) { + return false + } if !strings.EqualFold(configured.Scheme, target.Scheme) || !strings.EqualFold(configured.Hostname(), target.Hostname()) || authURLPort(configured) != authURLPort(target) { @@ -1085,6 +1091,24 @@ func authURLMatches(configured, target *url.URL) bool { return targetPath == configuredPath || strings.HasPrefix(targetPath, configuredPath+"/") } +// HasDotSegment reports whether the URL path p, or p decoded once more, has a +// '.' or '..' segment. Backslashes count as separators, since some upstream +// servers treat them as such. +func HasDotSegment(p string) bool { + candidates := []string{p} + if decoded, err := url.PathUnescape(p); err == nil { + candidates = append(candidates, decoded) + } + for _, candidate := range candidates { + for segment := range strings.SplitSeq(strings.ReplaceAll(candidate, `\`, "/"), "/") { + if segment == "." || segment == ".." { + return true + } + } + } + return false +} + func authURLPort(value *url.URL) string { if port := value.Port(); port != "" { return port diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 224c651..bf27f1e 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -1322,6 +1322,10 @@ func TestUpstreamAuthForURLMatchesURLComponents(t *testing.T) { {name: "lookalike host rejected", url: "https://registry.example.com.evil.test/package"}, {name: "different scheme rejected", url: "http://registry.example.com/package"}, {name: "different port rejected", url: "https://registry.example.com:8443/package"}, + {name: "dot dot segment rejected", url: "https://registry.example.com/private/../package"}, + {name: "encoded dot dot segment rejected", url: "https://registry.example.com/private/%2e%2e/package"}, + {name: "double encoded dot dot segment rejected", url: "https://registry.example.com/private/%252e%252e/package"}, + {name: "dot segment rejected", url: "https://registry.example.com/private/./package"}, } for _, tt := range tests { diff --git a/internal/server/dot_segments_test.go b/internal/server/dot_segments_test.go new file mode 100644 index 0000000..c0979c3 --- /dev/null +++ b/internal/server/dot_segments_test.go @@ -0,0 +1,91 @@ +package server + +import ( + "context" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "testing" + + "github.com/git-pkgs/proxy/internal/config" + "github.com/git-pkgs/proxy/internal/database" + "github.com/git-pkgs/proxy/internal/handler" + "github.com/git-pkgs/proxy/internal/storage" + "github.com/git-pkgs/registries/fetch" + "github.com/go-chi/chi/v5" +) + +// TestProtocolHandlersRejectDotSegments sends paths with encoded '.' and '..' +// segments to every mounted ecosystem. Several handlers append the decoded +// path to the upstream URL, where a '..' leaves the configured upstream path. +func TestProtocolHandlersRejectDotSegments(t *testing.T) { + db, err := database.Create(t.TempDir() + "/test.db") + if err != nil { + t.Fatalf("creating database: %v", err) + } + t.Cleanup(func() { _ = db.Close() }) + store, err := storage.OpenBucket(context.Background(), "file://"+t.TempDir()) + if err != nil { + t.Fatalf("opening storage: %v", err) + } + + transport := &countingTransport{} + client := &http.Client{Transport: transport} + fetcher := fetch.NewFetcher(fetch.WithHTTPClient(client), fetch.WithMaxRetries(0)) + t.Cleanup(func() { _ = fetcher.Close() }) + proxy := handler.NewProxy(db, store, fetcher, fetch.NewResolver(), slog.New(slog.NewTextHandler(io.Discard, nil))) + proxy.HTTPClient = client + + cfg := config.Default() + cfg.BaseURL = "http://proxy.example" + s := &Server{cfg: cfg} + r := chi.NewRouter() + s.mountProtocolHandlers(r, proxy) + + for _, target := range []string{ + "/maven/%2e%2e/%2e%2e/secret", + "/maven/com/%2e%2e/%2e%2e/x/maven-metadata.xml", + "/maven/com/%2E%2E/x/1.0/x-1.0.jar", + "/maven/com/%2e/x/1.0/x-1.0.jar", + "/maven/com/%252e%252e/x/maven-metadata.xml", + "/conda/c/%2e%2e/x.json", + "/cran/src/contrib/%2e%2e%2fx.tar.gz", + "/cran/src/contrib/..%5cx.tar.gz", + "/go/a/%2e%2e/%2e%2e/x/@v/list", + "/gem/info/%2e%2e%2f%2e%2e%2fx", + "/gem/gems/%2e%2e%2fx-1.0.gem", + "/nuget/v3-flatcontainer/%2e%2e/x/index.json", + "/debian/pool/%2e%2e/x.deb", + "/rpm/%2e%2e/x.rpm", + } { + w := httptest.NewRecorder() + r.ServeHTTP(w, httptest.NewRequest(http.MethodGet, target, nil)) + if w.Code != http.StatusBadRequest { + t.Errorf("%s: status = %d, want 400", target, w.Code) + } + } + if n := transport.hits.Load(); n != 0 { + t.Errorf("upstream was contacted %d times", n) + } +} + +// TestDotSegmentGuardKeepsDottedNames keeps names that only contain dots, +// such as versions and file extensions, working. +func TestDotSegmentGuardKeepsDottedNames(t *testing.T) { + for _, target := range []string{ + "/maven/com/example/x/1.0/x-1.0.jar", + "/pypi/simple/zope.interface/", + "/npm/.bin", + "/generic/github/o/r/releases/download/v1/...tar.gz", + "/maven/com/example/x/1..0/x-1..0.jar", + } { + var reached bool + next := http.HandlerFunc(func(http.ResponseWriter, *http.Request) { reached = true }) + w := httptest.NewRecorder() + rejectDotSegments(next).ServeHTTP(w, httptest.NewRequest(http.MethodGet, target, nil)) + if !reached { + t.Errorf("%s: status = %d, want the request passed on", target, w.Code) + } + } +} diff --git a/internal/server/middleware.go b/internal/server/middleware.go index 0de0dba..b472973 100644 --- a/internal/server/middleware.go +++ b/internal/server/middleware.go @@ -8,6 +8,7 @@ import ( "time" "github.com/git-pkgs/proxy/internal/accesslog" + "github.com/git-pkgs/proxy/internal/config" "github.com/git-pkgs/proxy/internal/metrics" "github.com/go-chi/chi/v5/middleware" ) @@ -140,6 +141,22 @@ func rejectQueryInPath(next http.Handler) http.Handler { }) } +// rejectDotSegments refuses requests whose decoded path has a '.' or '..' +// segment. Routers clean literal dot segments, but encoded ones such as +// %2e%2e reach the handlers, and several of them append the path to the +// upstream URL, where the upstream resolves it outside the configured base +// path. Backslashes count as separators, and the path is decoded once more +// so double-encoded segments are refused too. +func rejectDotSegments(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if config.HasDotSegment(r.URL.Path) { + http.Error(w, "invalid path", http.StatusBadRequest) + return + } + next.ServeHTTP(w, r) + }) +} + // sandboxUpstreamContent keeps package-route responses from running as pages // on the proxy's origin, which also serves /ui. Their bodies, and usually // their Content-Type, come from upstreams or from whoever pushed a hosted diff --git a/internal/server/server.go b/internal/server/server.go index 8a0a683..9a4c506 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -493,7 +493,7 @@ func (s *Server) mountProtocolHandlers(r chi.Router, proxy *handler.Proxy) { rpmHandler := handler.NewRPMHandlerWithUpstream(proxy, s.cfg.BaseURL, s.cfg.Upstream.RPM) genericHandler := handler.NewGenericHandler(proxy, s.cfg.Upstream.Generic, s.cfg.Upstream.GenericArtifacts) - r = r.With(sandboxUpstreamContent, rejectQueryInPath) + r = r.With(sandboxUpstreamContent, rejectQueryInPath, rejectDotSegments) r.Mount("/npm", http.StripPrefix("/npm", npmHandler.Routes())) r.Mount("/cargo", http.StripPrefix("/cargo", cargoHandler.Routes())) r.Mount("/gem", http.StripPrefix("/gem", gemHandler.Routes())) From 20c8878dd924c5a108ff77476ca5c9f2e5920984 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Thu, 22 Oct 2026 20:44:02 +0200 Subject: [PATCH 354/633] Match denylist entries the way each ecosystem spells names Denylist entries were compared as plain PURL strings, but handlers build PURLs from download URLs that spell packages differently, so some denied versions were still served. Entries and checked PURLs now go through one per-ecosystem comparison form: PyPI names follow PEP 503 (case and runs of -, _ and . are equivalent), NuGet ids and versions ignore case, a gem version also matches its platform builds, and a conda entry matches the package in every channel unless it carries a channel qualifier. Cache keys are unchanged. --- docs/configuration.md | 15 ++++- internal/denylist/denylist.go | 98 +++++++++++++++++++++++++++--- internal/denylist/denylist_test.go | 40 +++++++++++- internal/handler/denylist_test.go | 56 +++++++++++++++++ 4 files changed, 196 insertions(+), 13 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 6f7be45..88d7470 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -969,11 +969,22 @@ denylist: - "pkg:npm/%40scope/example@4.5.6" ``` -Entries must be versioned PURLs without qualifiers or subpaths. Package names -are canonicalized (including PyPI case/separator normalization and npm scopes); +Entries must be versioned PURLs without subpaths or qualifiers (except a conda +`channel`, below). Package names are canonicalized (including npm scopes); versions match exactly, not by range or glob. Invalid entries prevent startup. An empty list disables the policy. Restart the proxy after changing the list. +Names and versions are compared the way each ecosystem treats them: + +- PyPI names ignore case, and runs of `-`, `_` and `.` are equivalent (PEP 503), + so `pkg:pypi/zope.interface@6.0` also covers `zope_interface-6.0-*.whl`. +- NuGet ids and versions ignore case. +- A RubyGems entry also covers that version's platform gems: + `pkg:gem/nokogiri@1.16.0` denies `nokogiri-1.16.0-x86_64-linux.gem`. +- A conda entry covers the package in every channel. Add a `channel` qualifier, + such as `pkg:conda/numpy@1.26.4?channel=conda-forge`, to deny it in one + channel only. + Denied versions are removed from npm packuments, Cargo sparse indexes, and PyPI simple HTML/JSON and release metadata, including responses served from cached metadata during an upstream outage. No publication timestamp or enabled cooldown diff --git a/internal/denylist/denylist.go b/internal/denylist/denylist.go index c4384ad..3237496 100644 --- a/internal/denylist/denylist.go +++ b/internal/denylist/denylist.go @@ -3,6 +3,7 @@ package denylist import ( "fmt" + "regexp" "strings" "github.com/git-pkgs/purl" @@ -14,7 +15,8 @@ type Policy struct { } // New validates and canonicalizes versioned PURLs. Qualifiers and subpaths are -// rejected because the proxy cannot reliably distinguish them at every endpoint. +// rejected because the proxy cannot reliably distinguish them at every endpoint, +// except a conda channel, which download URLs always name. func New(packages []string) (*Policy, error) { if len(packages) == 0 { return nil, nil @@ -25,19 +27,32 @@ func New(packages []string) (*Policy, error) { if err != nil { return nil, fmt.Errorf("invalid denylist package %q: %w", value, err) } - if pkg.Version == "" || len(pkg.Qualifiers) != 0 || pkg.Subpath != "" || + if pkg.Version == "" || !allowedQualifiers(pkg) || pkg.Subpath != "" || strings.ContainsAny(pkg.Namespace+pkg.Name+pkg.Version, "*?[]<>=| \t\r\n") { return nil, fmt.Errorf("invalid denylist package %q: expected an exact versioned PURL without qualifiers or subpath", value) } - key := pkg.WithoutVersion().String() + id := identify(pkg) + key := id.key + if id.channel != "" { + key = id.channelKey() + } if p.packages[key] == nil { p.packages[key] = make(map[string]bool) } - p.packages[key][pkg.Version] = true + p.packages[key][id.version] = true } return p, nil } +func allowedQualifiers(pkg *purl.PURL) bool { + for _, q := range pkg.Qualifiers { + if pkg.Type != "conda" || q.Key != "channel" { + return false + } + } + return true +} + // Denied reports whether a versioned PURL is denied. func (p *Policy) Denied(value string) bool { if p == nil || len(p.packages) == 0 { @@ -47,21 +62,84 @@ func (p *Policy) Denied(value string) bool { if err != nil { return false } - return p.packages[pkg.WithoutVersion().String()][pkg.Version] + id := identify(pkg) + versions := []string{id.version} + // A platform gem's version carries the platform ("1.16.0-x86_64-linux"), + // so also match the plain version. + if pkg.Type == "gem" { + if base, _, found := strings.Cut(id.version, "-"); found && base != "" { + versions = append(versions, base) + } + } + for _, version := range versions { + if p.packages[id.key][version] || (id.channel != "" && p.packages[id.channelKey()][version]) { + return true + } + } + return false } // Versions returns a caller-owned set of denied versions for a package PURL. func (p *Policy) Versions(packagePURL string) map[string]bool { - if p == nil { + if p == nil || len(p.packages) == 0 { return nil } - versions := p.packages[packagePURL] - if len(versions) == 0 { + pkg, err := purl.Parse(packagePURL) + if err != nil { return nil } - result := make(map[string]bool, len(versions)) - for version := range versions { + id := identify(pkg) + result := make(map[string]bool) + for version := range p.packages[id.key] { result[version] = true } + if id.channel != "" { + for version := range p.packages[id.channelKey()] { + result[version] = true + } + } + if len(result) == 0 { + return nil + } return result } + +// identity is the form a PURL is compared in: the same for denylist entries +// and for the PURLs handlers build, however each spells the package. +type identity struct { + key string // versionless PURL without channel + channel string // conda channel, if any + version string +} + +func (id identity) channelKey() string { + return id.key + "?channel=" + id.channel +} + +var pypiSeparators = regexp.MustCompile(`[-_.]+`) + +func identify(pkg *purl.PURL) identity { + namespace, name, version := pkg.Namespace, pkg.Name, pkg.Version + var channel string + switch pkg.Type { + case "pypi": + // PEP 503: case-insensitive, runs of -, _ and . are equivalent. + name = pypiSeparators.ReplaceAllString(strings.ToLower(name), "-") + case "nuget": + // NuGet ids and versions are case-insensitive. + name, version = strings.ToLower(name), strings.ToLower(version) + case "conda": + // The conda handler names packages "channel/name"; entries may give the + // channel as a qualifier or namespace. An entry without one matches + // every channel. + channel = pkg.Qualifier("channel") + if namespace != "" { + channel, namespace = namespace, "" + } + if i := strings.LastIndex(name, "/"); i >= 0 { + channel, name = name[:i], name[i+1:] + } + } + key := purl.New(pkg.Type, namespace, name, "", nil).String() + return identity{key: key, channel: channel, version: version} +} diff --git a/internal/denylist/denylist_test.go b/internal/denylist/denylist_test.go index 380f1ee..ba74ddb 100644 --- a/internal/denylist/denylist_test.go +++ b/internal/denylist/denylist_test.go @@ -29,7 +29,7 @@ func TestPolicy(t *testing.T) { } func TestInvalidPolicy(t *testing.T) { - for _, value := range []string{"", "not-a-purl", "pkg:npm/foo", "pkg:npm/foo@*", "pkg:npm/foo@>=1", "pkg:npm/foo@1?arch=x86", "pkg:npm/foo@1#src"} { + for _, value := range []string{"", "not-a-purl", "pkg:npm/foo", "pkg:npm/foo@*", "pkg:npm/foo@>=1", "pkg:npm/foo@1?arch=x86", "pkg:npm/foo@1#src", "pkg:conda/numpy@1?subdir=linux-64", "pkg:pypi/foo@1?channel=x"} { t.Run(value, func(t *testing.T) { if _, err := New([]string{value}); err == nil { t.Fatal("invalid denylist accepted") @@ -37,3 +37,41 @@ func TestInvalidPolicy(t *testing.T) { }) } } + +func TestEcosystemSpellings(t *testing.T) { + p, err := New([]string{ + "pkg:pypi/zope.interface@6.0", + "pkg:nuget/Newtonsoft.Json@13.0.1-Beta", + "pkg:gem/nokogiri@1.16.0", + "pkg:gem/sorbet-static@0.5.1-universal-darwin-22", + "pkg:conda/numpy@1.26.4", + "pkg:conda/scipy@1.0?channel=conda-forge", + "pkg:conda/bioconda/samtools@1.0", + }) + if err != nil { + t.Fatal(err) + } + for _, value := range []string{ + "pkg:pypi/zope-interface@6.0", "pkg:pypi/Zope__Interface@6.0", + "pkg:nuget/newtonsoft.json@13.0.1-beta", + "pkg:gem/nokogiri@1.16.0", "pkg:gem/nokogiri@1.16.0-x86_64-linux", + "pkg:gem/sorbet-static@0.5.1-universal-darwin-22", + "pkg:conda/numpy@1.26.4", "pkg:conda/conda-forge%2Fnumpy@1.26.4", + "pkg:conda/conda-forge%2Fscipy@1.0", "pkg:conda/bioconda%2Fsamtools@1.0", + } { + if !p.Denied(value) { + t.Errorf("not denied: %s", value) + } + } + for _, value := range []string{ + "pkg:pypi/zopeinterface@6.0", "pkg:gem/nokogiri@1.16.1-x86_64-linux", "pkg:gem/Nokogiri@1.16.0", + "pkg:conda/bioconda%2Fscipy@1.0", "pkg:conda/scipy@1.0", "pkg:conda/conda-forge%2Fsamtools@1.0", + } { + if p.Denied(value) { + t.Errorf("unexpected denial: %s", value) + } + } + if !p.Versions("pkg:pypi/zope_interface")["6.0"] { + t.Error("pypi versions not found under another spelling") + } +} diff --git a/internal/handler/denylist_test.go b/internal/handler/denylist_test.go index e08d662..e55e36f 100644 --- a/internal/handler/denylist_test.go +++ b/internal/handler/denylist_test.go @@ -252,3 +252,59 @@ func TestDenylistMalformedMetadataFailsClosed(t *testing.T) { }) } } + +// Denylist entries must match however the ecosystem spells the name or +// version in download URLs. +func TestDenylistMatchesEcosystemSpellings(t *testing.T) { + pypi := func(p *Proxy) http.Handler { + return NewPyPIHandlerWithUpstreams(p, "http://proxy.test", "", "").Routes() + } + for _, tc := range []struct { + name, entry, path string + ecosystem, pkg string + version, filename string + handler func(*Proxy) http.Handler + }{ + {"pypi wheel", "pkg:pypi/zope.interface@6.0", "/packages/a/b/zope_interface-6.0-py3-none-any.whl", + "pypi", "zope_interface", "6.0", "zope_interface-6.0-py3-none-any.whl", pypi}, + {"pypi dotted sdist", "pkg:pypi/zope-interface@6.0", "/packages/a/b/zope.interface-6.0.tar.gz", + "pypi", "zope.interface", "6.0", "zope.interface-6.0.tar.gz", pypi}, + {"nuget", "pkg:nuget/Newtonsoft.Json@13.0.1-Beta", "/v3-flatcontainer/newtonsoft.json/13.0.1-beta/newtonsoft.json.13.0.1-beta.nupkg", + "nuget", "newtonsoft.json", "13.0.1-beta", "newtonsoft.json.13.0.1-beta.nupkg", + func(p *Proxy) http.Handler { return NewNuGetHandler(p, "http://proxy.test").Routes() }}, + {"gem platform", "pkg:gem/nokogiri@1.16.0", "/gems/nokogiri-1.16.0-x86_64-linux.gem", + "gem", "nokogiri", "1.16.0-x86_64-linux", "nokogiri-1.16.0-x86_64-linux.gem", + func(p *Proxy) http.Handler { return NewGemHandler(p, "http://proxy.test").Routes() }}, + {"conda any channel", "pkg:conda/numpy@1.26.4", "/conda-forge/linux-64/numpy-1.26.4-py311_0.conda", + "conda", "conda-forge/numpy", "1.26.4", "numpy-1.26.4-py311_0.conda", + func(p *Proxy) http.Handler { return NewCondaHandler(p, "http://proxy.test").Routes() }}, + {"conda channel qualifier", "pkg:conda/numpy@1.26.4?channel=conda-forge", "/conda-forge/linux-64/numpy-1.26.4-py311_0.conda", + "conda", "conda-forge/numpy", "1.26.4", "numpy-1.26.4-py311_0.conda", + func(p *Proxy) http.Handler { return NewCondaHandler(p, "http://proxy.test").Routes() }}, + } { + t.Run(tc.name, func(t *testing.T) { + // The seeded artifact would be served if the entry didn't match. + p, db, store, _ := setupTestProxy(t) + seedPackage(t, db, store, tc.ecosystem, tc.pkg, tc.version, tc.filename, "cached content") + setTestDenylist(t, p, tc.entry) + w := httptest.NewRecorder() + tc.handler(p).ServeHTTP(w, httptest.NewRequest(http.MethodGet, tc.path, nil)) + if w.Code != http.StatusForbidden { + t.Fatalf("denied download status=%d body=%s", w.Code, w.Body.String()) + } + }) + } +} + +func TestPyPIDenylistSimpleMatchesSeparators(t *testing.T) { + body := `xy` + h, p := setupPyPIHandler(t, func(r *http.Request) (*http.Response, error) { + return pypiHTTPResponse(r, "text/html", body), nil + }) + setTestDenylist(t, p, "pkg:pypi/zope.interface@6.0") + w := httptest.NewRecorder() + h.Routes().ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/simple/zope-interface/", nil)) + if w.Code != http.StatusOK || strings.Contains(w.Body.String(), "6.0") || !strings.Contains(w.Body.String(), "7.0") { + t.Fatalf("status=%d body=%s", w.Code, w.Body.String()) + } +} From 0ed360b8f44465ecfe40b905945bc61f81f201b8 Mon Sep 17 00:00:00 2001 From: montehurd Date: Thu, 22 Oct 2026 18:55:05 -0700 Subject: [PATCH 355/633] Document how pull requests are reviewed and merged Record the rules agreed in #514: one approval from a maintainer other than the author plus green CI, a "request changes" review blocks the merge, and bigger changes start with an issue. --- CONTRIBUTING.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f680251..9bb2fa1 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -212,6 +212,14 @@ Add PyPI registry support - Add tests for wheel and sdist downloads ``` +## Reviewing and Merging + +Maintainers are the members of the proxy team in the git-pkgs org. Any maintainer can review and merge. A pull request needs green CI and an approval from a maintainer other than its author. A "request changes" review from any maintainer blocks it until that reviewer approves or withdraws the review. + +Using AI to write or review is fine, but whoever approves a pull request should understand it and stand behind it. If it touches an ecosystem you don't use, you can ask someone who does to look before you approve. + +Open an issue before a bigger change: anything that changes behaviour users rely on, new configuration that would be hard to remove, or a change that fans out into other git-pkgs modules. If no maintainer objects within a few days, open the pull request; it's reviewed like any other. Objections are worked out in the issue, with @andrew breaking ties for now. Where possible, deprecate before removing: an old configuration key keeps working, with a warning, in at least one release before the one that removes it. + ## Releases A patch release is cut on the 1st of each month by `.github/workflows/monthly-release.yml` when `main` is green and something has merged since the last tag. It is skipped if a pull request labelled `breaking` was merged since then: label pull requests that change behaviour users rely on, and release those as a new minor version by hand. From 1fbee102b141797ca16420d3645416cae08370e9 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Fri, 23 Oct 2026 09:47:07 +0200 Subject: [PATCH 356/633] Ignore path parameters when checking for dot segments Tomcat and Jetty drop ";params" from a path segment before resolving it, so "..;" passed the dot-segment check and still got the credentials configured for the upstream path. The check now ignores ";params", and package routes refuse ';' like '?' and '#', which also stops "1.0;x" from passing the denylist as a different version. Composer archives stay exempt, since git branch names may contain ';'. --- internal/config/config.go | 5 +++-- internal/config/config_test.go | 3 +++ internal/server/dot_segments_test.go | 10 ++++++++-- internal/server/middleware.go | 16 +++++++++------- internal/server/query_in_path_test.go | 22 ++++++++++++++-------- 5 files changed, 37 insertions(+), 19 deletions(-) diff --git a/internal/config/config.go b/internal/config/config.go index 96487d4..14c4c20 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -1092,8 +1092,8 @@ func authURLMatches(configured, target *url.URL) bool { } // HasDotSegment reports whether the URL path p, or p decoded once more, has a -// '.' or '..' segment. Backslashes count as separators, since some upstream -// servers treat them as such. +// '.' or '..' segment. Backslashes count as separators, and ";params" are +// ignored, since some upstream servers (Tomcat, Jetty) treat them that way. func HasDotSegment(p string) bool { candidates := []string{p} if decoded, err := url.PathUnescape(p); err == nil { @@ -1101,6 +1101,7 @@ func HasDotSegment(p string) bool { } for _, candidate := range candidates { for segment := range strings.SplitSeq(strings.ReplaceAll(candidate, `\`, "/"), "/") { + segment, _, _ = strings.Cut(segment, ";") if segment == "." || segment == ".." { return true } diff --git a/internal/config/config_test.go b/internal/config/config_test.go index bf27f1e..7a689f8 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -1326,6 +1326,9 @@ func TestUpstreamAuthForURLMatchesURLComponents(t *testing.T) { {name: "encoded dot dot segment rejected", url: "https://registry.example.com/private/%2e%2e/package"}, {name: "double encoded dot dot segment rejected", url: "https://registry.example.com/private/%252e%252e/package"}, {name: "dot segment rejected", url: "https://registry.example.com/private/./package"}, + {name: "dot dot segment with parameter rejected", url: "https://registry.example.com/private/..;/package"}, + {name: "encoded dot dot segment with parameter rejected", url: "https://registry.example.com/private/%2e%2e;x/package"}, + {name: "dot dot segment with encoded parameter rejected", url: "https://registry.example.com/private/..%3b/package"}, } for _, tt := range tests { diff --git a/internal/server/dot_segments_test.go b/internal/server/dot_segments_test.go index c0979c3..429ccc0 100644 --- a/internal/server/dot_segments_test.go +++ b/internal/server/dot_segments_test.go @@ -17,8 +17,9 @@ import ( ) // TestProtocolHandlersRejectDotSegments sends paths with encoded '.' and '..' -// segments to every mounted ecosystem. Several handlers append the decoded -// path to the upstream URL, where a '..' leaves the configured upstream path. +// segments, and with ';' path parameters, to every mounted ecosystem. Several +// handlers append the decoded path to the upstream URL, where a '..' leaves +// the configured upstream path, and Tomcat and Jetty drop ";params" first. func TestProtocolHandlersRejectDotSegments(t *testing.T) { db, err := database.Create(t.TempDir() + "/test.db") if err != nil { @@ -58,6 +59,11 @@ func TestProtocolHandlersRejectDotSegments(t *testing.T) { "/nuget/v3-flatcontainer/%2e%2e/x/index.json", "/debian/pool/%2e%2e/x.deb", "/rpm/%2e%2e/x.rpm", + "/maven/..;/..;/other-repo/com/x/x/1.0/x-1.0.jar", + "/maven/com/%2e%2e;/x/1.0/x-1.0.jar", + "/maven/com/..%3b/x/1.0/x-1.0.jar", + "/maven/com/evil/evil/1.0;x/evil-1.0.jar", + "/generic/github/o/r/releases/download/v1/tool.tar.gz;x", } { w := httptest.NewRecorder() r.ServeHTTP(w, httptest.NewRequest(http.MethodGet, target, nil)) diff --git a/internal/server/middleware.go b/internal/server/middleware.go index b472973..c179744 100644 --- a/internal/server/middleware.go +++ b/internal/server/middleware.go @@ -125,15 +125,17 @@ func requestEcosystem(path string) string { } } -// rejectQueryInPath refuses requests whose decoded path contains '?' or '#'. -// Handlers build upstream URLs from the decoded path, where such a character -// would end the path: busybox.apk%3F/foo-9.9-r0.apk fetches busybox but is -// parsed and cached as foo, under the wrong PURL and past the denylist. -// Composer archives are exempt: their upstream URL comes from the package -// metadata, and git branch versions such as dev-fix/#42 may contain '#'. +// rejectQueryInPath refuses requests whose decoded path contains '?', '#' +// or ';'. Handlers build upstream URLs from the decoded path, where '?' or +// '#' would end the path: busybox.apk%3F/foo-9.9-r0.apk fetches busybox but +// is parsed and cached as foo, under the wrong PURL and past the denylist. +// Tomcat and Jetty drop ";params" from a path segment, so 1.0;x/evil-1.0.jar +// would be served as version 1.0 while the proxy checks 1.0;x. Composer +// archives are exempt: their upstream URL comes from the package metadata, +// and git branch versions such as dev-fix/#42 may contain '#' or ';'. func rejectQueryInPath(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if strings.ContainsAny(r.URL.Path, "?#") && !strings.HasPrefix(r.URL.Path, "/composer/files/") { + if strings.ContainsAny(r.URL.Path, "?#;") && !strings.HasPrefix(r.URL.Path, "/composer/files/") { http.Error(w, "invalid path", http.StatusBadRequest) return } diff --git a/internal/server/query_in_path_test.go b/internal/server/query_in_path_test.go index 7c70a13..44ab58e 100644 --- a/internal/server/query_in_path_test.go +++ b/internal/server/query_in_path_test.go @@ -93,15 +93,21 @@ func TestProtocolHandlersRejectQueryInPath(t *testing.T) { } // TestComposerFilesAllowHashInVersion keeps Composer archive downloads of git -// branches with '#' in their name working: the version is escaped in the dist -// URL and the archive is resolved from metadata, not from the request path. +// branches with '#' or ';' in their name working: the version is escaped in +// the dist URL and the archive is resolved from metadata, not from the +// request path. func TestComposerFilesAllowHashInVersion(t *testing.T) { - var reached atomic.Bool - next := http.HandlerFunc(func(http.ResponseWriter, *http.Request) { reached.Store(true) }) + for _, target := range []string{ + "/composer/files/acme/lib/dev-fix%2F%2342/abc123.zip", + "/composer/files/acme/lib/dev-fix%3B42/abc123.zip", + } { + var reached atomic.Bool + next := http.HandlerFunc(func(http.ResponseWriter, *http.Request) { reached.Store(true) }) - w := httptest.NewRecorder() - rejectQueryInPath(next).ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/composer/files/acme/lib/dev-fix%2F%2342/abc123.zip", nil)) - if w.Code == http.StatusBadRequest || !reached.Load() { - t.Errorf("status = %d, reached = %v; want the request passed on", w.Code, reached.Load()) + w := httptest.NewRecorder() + rejectQueryInPath(next).ServeHTTP(w, httptest.NewRequest(http.MethodGet, target, nil)) + if w.Code == http.StatusBadRequest || !reached.Load() { + t.Errorf("%s: status = %d, reached = %v; want the request passed on", target, w.Code, reached.Load()) + } } } From d28c0befbfa0fc02149449627f88d9bee6046179 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Fri, 23 Oct 2026 09:57:37 +0200 Subject: [PATCH 357/633] Build with Go 1.27.3 everywhere CI, the release binaries and swagger take their Go version from go.mod, which pinned 1.26.7 and so missed the stdlib fixes in 1.26.8 to 1.26.10. A toolchain line now sets 1.27.3 for them, and the Docker builder uses the same version. The go line stays at 1.26.7, so source builds with an older local Go still work. golangci-lint 2.13.1 can't read Go 1.27.2+ export data, so CI moves to 2.15.0. Its errcheck flags two test cleanups that dropped the server's exit error; they now check it. --- .github/workflows/ci.yml | 2 +- Dockerfile | 2 +- go.mod | 2 ++ internal/server/gradle_read_only_test.go | 5 ++++- internal/server/hosted_api_test.go | 5 ++++- 5 files changed, 12 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c4b7776..64ecd80 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -44,7 +44,7 @@ jobs: go-version-file: go.mod - name: Install golangci-lint - run: go install github.com/golangci/golangci-lint/v2/cmd/golangci-lint@v2.13.1 + run: go install github.com/golangci/golangci-lint/v2/cmd/golangci-lint@v2.15.0 - name: golangci-lint run: golangci-lint run ./... diff --git a/Dockerfile b/Dockerfile index 4113999..1a934ed 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM --platform=$BUILDPLATFORM golang:1.26.7-alpine AS builder +FROM --platform=$BUILDPLATFORM golang:1.27.3-alpine AS builder WORKDIR /src diff --git a/go.mod b/go.mod index 3c1d4e2..2cca4af 100644 --- a/go.mod +++ b/go.mod @@ -2,6 +2,8 @@ module github.com/git-pkgs/proxy go 1.26.7 +toolchain go1.27.3 + require ( github.com/BurntSushi/toml v1.6.0 github.com/Masterminds/semver/v3 v3.5.0 diff --git a/internal/server/gradle_read_only_test.go b/internal/server/gradle_read_only_test.go index 0911bff..1c927f5 100644 --- a/internal/server/gradle_read_only_test.go +++ b/internal/server/gradle_read_only_test.go @@ -2,6 +2,7 @@ package server import ( "context" + "errors" "io" "log/slog" "net" @@ -43,7 +44,9 @@ func startGradleServer(t *testing.T, readOnly bool) string { ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() _ = proxyServer.Shutdown(ctx) - <-startErr + if err := <-startErr; !errors.Is(err, http.ErrServerClosed) { + t.Errorf("server stopped with %v", err) + } }) return cfg.BaseURL } diff --git a/internal/server/hosted_api_test.go b/internal/server/hosted_api_test.go index 38b34b3..17b38bb 100644 --- a/internal/server/hosted_api_test.go +++ b/internal/server/hosted_api_test.go @@ -5,6 +5,7 @@ import ( "crypto/sha256" "encoding/hex" "encoding/json" + "errors" "io" "log/slog" "net" @@ -277,7 +278,9 @@ func startHostedServer(t *testing.T, hosted config.HostedConfig) string { ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() _ = proxyServer.Shutdown(ctx) - <-startErr + if err := <-startErr; !errors.Is(err, http.ErrServerClosed) { + t.Errorf("server stopped with %v", err) + } }) return cfg.BaseURL } From d9d3bd2ff19da38d2966921298d83da256ac5be2 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Fri, 23 Oct 2026 09:50:22 +0200 Subject: [PATCH 358/633] Check unhashed FreeBSD packages against the denylist Package files outside Hashed/, such as All/{name}-{version}.pkg from repositories built without pkg repo --hash and the bootstrap Latest/pkg.pkg, were relayed without any checks. They are still passed through uncached, since their names can be reused for rebuilds, but a versioned name is now checked against the denylist first, and while a scanner covers freebsd they are refused, because only cached artifacts can be scanned. --- README.md | 5 +- docs/configuration.md | 10 +- internal/handler/freebsd.go | 64 ++++++++++++- internal/handler/freebsd_test.go | 156 +++++++++++++++++++++++++++++++ internal/scanner/group.go | 5 + internal/scanner/group_test.go | 27 ++++++ 6 files changed, 261 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index e322f3c..0553e4a 100644 --- a/README.md +++ b/README.md @@ -829,7 +829,10 @@ remain available when the upstream is unreachable. Other files are passed through uncached. That includes the bootstrap package under `Latest/`, whose signature file is rewritten with every repository build, and package files without a hash in their name, which can be rebuilt under the -same name. +same name. The denylist still applies to unhashed package files. Since only +cached artifacts can be scanned, both kinds of package file are refused while +a scanner covers `freebsd`; use repositories built with `pkg repo --hash` and +bootstrap pkg without the proxy. ### GitHub Releases / mise (aqua backend) diff --git a/docs/configuration.md b/docs/configuration.md index 2e763ee..017d991 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -553,8 +553,14 @@ pkgbase repositories and repositories built with poudriere work. Catalogues (`meta`, `meta.conf`, `packagesite` and `data` as `.pkg`, `.txz` or `.tzst`) are cached with the metadata TTL and served byte-for-byte unchanged, which keeps their embedded signatures valid. Packages under `Hashed/` use the shared -artifact cache; everything else, including `Latest/` and unhashed package -files, is passed through uncached. When `upstream.freebsd` is empty, a single +artifact cache. Other package files (`.pkg`, `.txz`, `.tzst`), unhashed ones +and the bootstrap package under `Latest/`, can be rebuilt under the same name, +so they are passed through uncached after the [denylist](#version-denylist) +check on their `{name}-{version}`. Only cached artifacts are scanned, so while +a scanner covers the `freebsd` ecosystem these files are refused with +`403 Forbidden`: serve repositories built with `pkg repo --hash` (as the +official mirror is) and bootstrap pkg without the proxy. Everything else, such +as signatures and keys, is passed through uncached. When `upstream.freebsd` is empty, a single repository named `freebsd` pointing at `https://pkg.freebsd.org` is available; configuring any entry replaces that default. diff --git a/internal/handler/freebsd.go b/internal/handler/freebsd.go index 419147b..aceffa4 100644 --- a/internal/handler/freebsd.go +++ b/internal/handler/freebsd.go @@ -3,6 +3,8 @@ package handler import ( "crypto/sha256" "encoding/hex" + "errors" + "fmt" "net/http" "regexp" "strings" @@ -48,9 +50,11 @@ func freebsdRetentionKey(p *purl.PURL) (string, bool) { // Repository catalogues (meta.conf, packagesite.pkg, data.pkg) carry their // signature inside the archive, so they are served byte for byte through the // metadata cache. Packages under Hashed/ have the content hash in their name -// and never change, so they go to the artifact cache. Everything else, -// including the unversioned bootstrap files under Latest/ whose detached -// signature is replaced on every repository build, is passed through. +// and never change, so they go to the artifact cache. Other package files, +// unhashed ones and the bootstrap package under Latest/, can be rebuilt under +// the same name and are passed through after the denylist; while a scanner +// covers freebsd they are refused, since only cached artifacts are scanned. +// Everything else, such as signatures and keys, is passed through. type FreeBSDHandler struct { proxy *Proxy proxyURL string @@ -104,6 +108,8 @@ func (h *FreeBSDHandler) Routes() http.Handler { h.handleMetadata(w, r, repository, upstreamURL, rest) case isFreeBSDHashedPackage(rest): h.handlePackageDownload(w, r, repository, upstreamURL, rest) + case isFreeBSDPackageFile(rest): + h.handleUncachedPackage(w, r, upstreamURL, rest) default: h.proxyFile(w, r, upstreamURL, rest) } @@ -149,6 +155,58 @@ func parseFreeBSDPackage(filename string) (name, version string) { return matches[1], matches[2] } +// freebsdUnhashedPattern matches {name}-{version} package files in the current +// (.pkg) and older (.txz, .tzst) packing formats, split at the last '-' like +// freebsdHashedPattern. +var freebsdUnhashedPattern = regexp.MustCompile(`^(.+)-([^-~/]+)\.(?:pkg|txz|tzst)$`) + +// freebsdPackageFilePattern matches every package file name, including +// Latest/pkg.pkg, which has no version. Catalogues match it too and must be +// routed first. +var freebsdPackageFilePattern = regexp.MustCompile(`\.(?:pkg|txz|tzst)$`) + +// isFreeBSDPackageFile reports whether path names a package file. +func isFreeBSDPackageFile(path string) bool { + return freebsdPackageFilePattern.MatchString(freebsdBase(path)) +} + +// parseFreeBSDPackageFile returns the name and version of any package file, +// hashed or not, or empty strings for a file without a version such as +// Latest/pkg.pkg. +func parseFreeBSDPackageFile(filename string) (name, version string) { + if name, version = parseFreeBSDPackage(filename); name != "" { + return name, version + } + matches := freebsdUnhashedPattern.FindStringSubmatch(filename) + if len(matches) != freebsdMatchCount { + return "", "" + } + return matches[1], matches[2] +} + +// errFreeBSDUnscannable refuses package files that can't be cached while +// scanning covers freebsd. +var errFreeBSDUnscannable = errors.New("package file can't be scanned: only packages under Hashed/ are cached and scanned; build the repository with pkg repo --hash") + +// handleUncachedPackage passes a package file outside Hashed/ through. Its +// name may be reused for a rebuild with other bytes, which pkg would then +// reject against the catalogue checksum, so it is never cached. The denylist +// applies when the name carries a version. Scanners only see cached +// artifacts, so the file is refused while one covers freebsd. +func (h *FreeBSDHandler) handleUncachedPackage(w http.ResponseWriter, r *http.Request, upstreamURL, path string) { + name, version := parseFreeBSDPackageFile(freebsdBase(path)) + if name != "" && h.proxy.versionDenied(freebsdEcosystem, name, version) { + h.proxy.serveArtifactError(w, fmt.Errorf("%w: %s", ErrVersionDenied, canonicalVersionPURL(freebsdEcosystem, name, version)), "") + return + } + if h.proxy.Scanners.Covers(freebsdEcosystem) { + h.proxy.Logger.Warn("refusing unscannable freebsd package", "path", path) + JSONError(w, http.StatusForbidden, errFreeBSDUnscannable.Error()) + return + } + h.proxyFile(w, r, upstreamURL, path) +} + // handlePackageDownload caches hashed packages. The cache key holds the // repository and full path, since every ABI and repository has its own // builds. diff --git a/internal/handler/freebsd_test.go b/internal/handler/freebsd_test.go index b203871..4702f34 100644 --- a/internal/handler/freebsd_test.go +++ b/internal/handler/freebsd_test.go @@ -1,13 +1,18 @@ package handler import ( + "io" + "log/slog" "net/http" "net/http/httptest" + "strings" "sync" "sync/atomic" "testing" "time" + "github.com/git-pkgs/proxy/internal/config" + "github.com/git-pkgs/proxy/internal/scanner" "github.com/git-pkgs/purl" "github.com/git-pkgs/registries/fetch" ) @@ -62,6 +67,34 @@ func TestParseFreeBSDPackage(t *testing.T) { } } +func TestParseFreeBSDPackageFile(t *testing.T) { + tests := []struct { + filename, name, version string + pkgfile bool + }{ + {"zsh-5.9_5~2$3rwh1rgp.pkg", "zsh", "5.9_5", true}, + {"zsh-5.9_5.pkg", "zsh", "5.9_5", true}, + {"zsh-navigation-tools-2.2.7.pkg", "zsh-navigation-tools", "2.2.7", true}, + {"py311-requests-2.32.3_1,1.txz", "py311-requests", "2.32.3_1,1", true}, + {"FreeBSD-acct-15.snap20261021214449.tzst", "FreeBSD-acct", "15.snap20261021214449", true}, + {"pkg.pkg", "", "", true}, + {"pkg.txz", "", "", true}, + {"pkg.pkg.sig", "", "", false}, + {"pkg.pkg.pub", "", "", false}, + {"zsh-5.9_5.pkg.sig", "", "", false}, + {"zsh-5.9_5.tar", "", "", false}, + } + for _, tt := range tests { + name, version := parseFreeBSDPackageFile(tt.filename) + if name != tt.name || version != tt.version { + t.Errorf("parseFreeBSDPackageFile(%q) = %q, %q, want %q, %q", tt.filename, name, version, tt.name, tt.version) + } + if got := isFreeBSDPackageFile("FreeBSD:14:amd64/latest/All/" + tt.filename); got != tt.pkgfile { + t.Errorf("isFreeBSDPackageFile(%q) = %v, want %v", tt.filename, got, tt.pkgfile) + } + } +} + func TestFreeBSDHandler_Routes(t *testing.T) { h := NewFreeBSDHandler(nil, "http://localhost:8080", nil) assertRoutesBasics(t, h.Routes(), "/freebsd"+freebsdABI+"/latest/meta.conf", "/freebsd"+freebsdABI+"/../../../etc/passwd") @@ -187,6 +220,7 @@ func TestFreeBSDHandler_HashedPackagesCachedPerABI(t *testing.T) { // Unhashed packages can be rebuilt under the same name, and the bootstrap // files under Latest/ change with every build, so none of them are cached. +// Without a denylist entry or a scanner they are passed through. func TestFreeBSDHandler_UnhashedFilesPassedThrough(t *testing.T) { files := map[string]string{ freebsdABI + "/latest/Latest/pkg.pkg": "bootstrap package", @@ -215,6 +249,128 @@ func TestFreeBSDHandler_UnhashedFilesPassedThrough(t *testing.T) { } } +// Package files without a hash in their name are checked against the +// denylist like hashed ones, before anything is fetched. +func TestFreeBSDHandler_DenylistCoversEveryPackageFile(t *testing.T) { + denied := []string{ + freebsdABI + "/latest/All/Hashed/zsh-5.9_5~2$3rwh1rgp.pkg", + freebsdABI + "/latest/All/zsh-5.9_5.pkg", + freebsdABI + "/latest/All/zsh-5.9_5~2$3rwh1rgp.pkg", + "/142amd64-default/All/zsh-5.9_5.txz", + "/142amd64-default/All/zsh-5.9_5.tzst", + } + allowed := map[string]string{ + freebsdABI + "/latest/All/zsh-5.9_4.pkg": "older build", + freebsdABI + "/latest/All/zsh-navigation-tools-5.9_5.pkg": "another package", + } + files := map[string]string{} + for _, path := range denied { + files[path] = "denied package" + } + for path, body := range allowed { + files[path] = body + } + mirror := newFakeFreeBSDMirror(t, files) + proxy, _, _, _ := setupTestProxy(t) + proxy.HTTPClient = mirror.Client() + fetcher := fetch.NewFetcher(fetch.WithHTTPClient(mirror.Client()), fetch.WithMaxRetries(0)) + proxy.Fetcher = fetcher + t.Cleanup(func() { _ = fetcher.Close() }) + setTestDenylist(t, proxy, "pkg:bsd/freebsd/zsh@5.9_5") + h := NewFreeBSDHandler(proxy, "http://proxy.example", map[string]string{"freebsd": mirror.URL}) + + for _, path := range denied { + if w := serveFreeBSDRequest(h, "/freebsd"+path); w.Code != http.StatusForbidden { + t.Errorf("%s: status = %d, want 403", path, w.Code) + } + if got := mirror.count(path); got != 0 { + t.Errorf("%s fetched %d times, want 0", path, got) + } + } + for path, want := range allowed { + if w := serveFreeBSDRequest(h, "/freebsd"+path); w.Code != http.StatusOK || w.Body.String() != want { + t.Errorf("%s: status = %d, body %q, want 200 and %q", path, w.Code, w.Body, want) + } + } +} + +// Package files the proxy can't cache can't be scanned either, so they are +// refused while a scanner covers freebsd. Catalogues, signatures and hashed +// packages are unaffected. +func TestFreeBSDHandler_ScanningRefusesUncachablePackages(t *testing.T) { + refused := []string{ + freebsdABI + "/latest/All/zsh-5.9_5.pkg", + freebsdABI + "/latest/Latest/pkg.pkg", + "/142amd64-default/Latest/pkg.txz", + "/142amd64-default/All/zsh-5.9_5.tzst", + } + served := map[string]string{ + freebsdABI + "/latest/meta.conf": "version = 2;\n", + freebsdABI + "/latest/Latest/pkg.pkg.sig": "bootstrap signature", + freebsdABI + "/latest/Latest/pkg.pkg.pub": "bootstrap key", + } + files := map[string]string{} + for _, path := range refused { + files[path] = "unscanned package" + } + for path, body := range served { + files[path] = body + } + + for _, ecosystems := range [][]string{nil, {"freebsd"}} { + mirror := newFakeFreeBSDMirror(t, files) + proxy, _, _, _ := setupTestProxy(t) + proxy.HTTPClient = mirror.Client() + proxy.Scanners = newFreeBSDScanGroup(t, ecosystems) + h := NewFreeBSDHandler(proxy, "http://proxy.example", map[string]string{"freebsd": mirror.URL}) + + for _, path := range refused { + w := serveFreeBSDRequest(h, "/freebsd"+path) + if w.Code != http.StatusForbidden || !strings.Contains(w.Body.String(), "scan") { + t.Errorf("ecosystems %v %s: status = %d, body %q, want 403 naming the scan", ecosystems, path, w.Code, w.Body) + } + if got := mirror.count(path); got != 0 { + t.Errorf("ecosystems %v %s fetched %d times, want 0", ecosystems, path, got) + } + } + for path, want := range served { + if w := serveFreeBSDRequest(h, "/freebsd"+path); w.Code != http.StatusOK || w.Body.String() != want { + t.Errorf("ecosystems %v %s: status = %d, body %q, want 200 and %q", ecosystems, path, w.Code, w.Body, want) + } + } + } +} + +// A scanner restricted to other ecosystems never sees FreeBSD packages, so +// unhashed files are passed through as without scanning. +func TestFreeBSDHandler_ScannerForOtherEcosystemsPassesThrough(t *testing.T) { + const path = freebsdABI + "/latest/All/zsh-5.9_5.pkg" + mirror := newFakeFreeBSDMirror(t, map[string]string{path: "unhashed package"}) + proxy, _, _, _ := setupTestProxy(t) + proxy.HTTPClient = mirror.Client() + proxy.Scanners = newFreeBSDScanGroup(t, []string{"npm"}) + h := NewFreeBSDHandler(proxy, "http://proxy.example", map[string]string{"freebsd": mirror.URL}) + + if w := serveFreeBSDRequest(h, "/freebsd"+path); w.Code != http.StatusOK || w.Body.String() != "unhashed package" { + t.Errorf("status = %d, body %q, want 200 and the upstream bytes", w.Code, w.Body) + } +} + +func newFreeBSDScanGroup(t *testing.T, ecosystems []string) *scanner.Group { + t.Helper() + g, err := scanner.NewGroup(config.ScanningConfig{ + Enabled: true, + SigningKey: "test-signing-key", + Scanners: []config.ScannerConfig{ + {Name: "test-scanner", URL: "http://scanner.invalid/scan", Ecosystems: ecosystems}, + }, + }, slog.New(slog.NewTextHandler(io.Discard, nil))) + if err != nil { + t.Fatalf("scanner.NewGroup() error: %v", err) + } + return g +} + func serveFreeBSDRequest(h *FreeBSDHandler, target string) *httptest.ResponseRecorder { w := httptest.NewRecorder() h.Routes().ServeHTTP(w, httptest.NewRequest(http.MethodGet, target, nil)) diff --git a/internal/scanner/group.go b/internal/scanner/group.go index a31d324..227a037 100644 --- a/internal/scanner/group.go +++ b/internal/scanner/group.go @@ -86,6 +86,11 @@ func (g *Group) Enabled() bool { return g != nil && len(g.entries) > 0 } +// Covers reports whether any scanner, in either mode, applies to ecosystem. +func (g *Group) Covers(ecosystem string) bool { + return g.Enabled() && len(g.applicable(ecosystem)) > 0 +} + // Timeout returns the per-scan-call timeout used to bound the signed fetch // URL's validity. func (g *Group) Timeout() time.Duration { diff --git a/internal/scanner/group_test.go b/internal/scanner/group_test.go index e5b40ee..48cbebc 100644 --- a/internal/scanner/group_test.go +++ b/internal/scanner/group_test.go @@ -77,6 +77,33 @@ func TestGroup_Enabled(t *testing.T) { } } +func TestGroup_Covers(t *testing.T) { + var nilGroup *Group + if nilGroup.Covers("npm") { + t.Error("nil group Covers(npm) = true, want false") + } + g, err := NewGroup(config.ScanningConfig{ + Enabled: true, + SigningKey: "test-signing-key", + Scanners: []config.ScannerConfig{ + {Name: "trivy", URL: "http://trivy.invalid", Mode: "monitor", Ecosystems: []string{"npm"}}, + {Name: "clamav", URL: "http://clamav.invalid", Ecosystems: []string{"pypi"}}, + }, + }, discardLogger()) + if err != nil { + t.Fatalf("NewGroup() error: %v", err) + } + for ecosystem, want := range map[string]bool{"npm": true, "pypi": true, "freebsd": false} { + if got := g.Covers(ecosystem); got != want { + t.Errorf("Covers(%s) = %v, want %v", ecosystem, got, want) + } + } + all := newTestGroup([]entry{{mode: modeBlock}}, false) + if !all.Covers("freebsd") { + t.Error("unrestricted scanner: Covers(freebsd) = false, want true") + } +} + func TestGroup_NewGroup_InvalidMode(t *testing.T) { _, err := NewGroup(config.ScanningConfig{ Enabled: true, From e1650e202d929b5768d586c4016e6dc1de33da04 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Fri, 23 Oct 2026 09:55:06 +0200 Subject: [PATCH 359/633] Match more denylist spellings Denylist entries now also match Composer versions with Packagist's leading v, OCI repositories written as a path, Swift registry packages in either form (also when streaming), Go versions with escaped uppercase letters, and gems whose names contain a dash followed by a digit. OCI manifests are checked by tag and digest, and denied blobs and Swift archives answer 403 instead of 500 or 502. --- docs/configuration.md | 15 ++++ internal/denylist/denylist.go | 35 ++++++++ internal/denylist/denylist_test.go | 15 ++++ internal/handler/container.go | 11 ++- internal/handler/container_manifest.go | 42 +++++++-- internal/handler/denylist_test.go | 113 ++++++++++++++++++++++++- internal/handler/gem.go | 39 +++++++-- internal/handler/gem_test.go | 6 ++ internal/handler/go.go | 9 ++ internal/handler/handler.go | 7 +- internal/handler/swift.go | 4 + 11 files changed, 277 insertions(+), 19 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 88d7470..2d122cc 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -981,6 +981,21 @@ Names and versions are compared the way each ecosystem treats them: - NuGet ids and versions ignore case. - A RubyGems entry also covers that version's platform gems: `pkg:gem/nokogiri@1.16.0` denies `nokogiri-1.16.0-x86_64-linux.gem`. +- Composer versions ignore a leading `v`, which Packagist keeps from the tag: + `pkg:composer/symfony/console@6.1.7` also covers `v6.1.7`. +- Go versions are matched as written, `pkg:golang/example.com/mod@v1.0.0-RC1`, + though download URLs escape uppercase letters (`v1.0.0-!r!c1`). +- A Swift entry names the registry scope and package, ignoring case: + `pkg:swift/apple/swift-nio@2.62.0`. The stored form, + `pkg:generic/swift-registry/apple.swift-nio@2.62.0`, works too. +- An OCI entry names the repository as it is pulled through the proxy and a + digest or tag: `pkg:oci/library/alpine@sha256:` refuses the blob or + manifest with that digest, including a manifest a tag resolves to, and + `pkg:oci/library/alpine@3.19` refuses pulling that tag. Repositories on a + named upstream keep their prefix, as in `pkg:oci/upstream/ghcr/org/image@...`. + An entry covers only the digest it names: denying an image index does not + deny the platform manifests and layers it lists to a client that requests + them by digest directly. - A conda entry covers the package in every channel. Add a `channel` qualifier, such as `pkg:conda/numpy@1.26.4?channel=conda-forge`, to deny it in one channel only. diff --git a/internal/denylist/denylist.go b/internal/denylist/denylist.go index 3237496..77b076a 100644 --- a/internal/denylist/denylist.go +++ b/internal/denylist/denylist.go @@ -118,6 +118,20 @@ func (id identity) channelKey() string { var pypiSeparators = regexp.MustCompile(`[-_.]+`) +// swiftRegistryIdentity returns the registry scope and name of a Swift +// package. A source coordinate such as pkg:swift/github.com/apple/swift-nio +// names no registry identity. +func swiftRegistryIdentity(pkg *purl.PURL) (scope, name string, ok bool) { + switch { + case pkg.Type == "swift" && pkg.Namespace != "" && !strings.ContainsAny(pkg.Namespace, "./"): + return pkg.Namespace, pkg.Name, true + case pkg.Type == "generic" && strings.EqualFold(pkg.Namespace, "swift-registry"): + scope, name, found := strings.Cut(pkg.Name, ".") + return scope, name, found && scope != "" && name != "" + } + return "", "", false +} + func identify(pkg *purl.PURL) identity { namespace, name, version := pkg.Namespace, pkg.Name, pkg.Version var channel string @@ -139,6 +153,27 @@ func identify(pkg *purl.PURL) identity { if i := strings.LastIndex(name, "/"); i >= 0 { channel, name = name[:i], name[i+1:] } + case "composer": + // Packagist keeps a tag's leading v ("v6.1.7"); Composer treats it + // as the same version. + if len(version) > 1 && (version[0] == 'v' || version[0] == 'V') && version[1] >= '0' && version[1] <= '9' { + version = version[1:] + } + case "oci": + // Handlers name the repository as one name, library%2Falpine; + // entries write it as a path, library/alpine. + if namespace != "" { + name, namespace = namespace+"/"+name, "" + } + case "swift", "generic": + // Swift registry archives are stored as + // pkg:generic/swift-registry/{scope}.{name}; entries may write + // pkg:swift/{scope}/{name}. Registry identities ignore case. + if scope, pkgName, ok := swiftRegistryIdentity(pkg); ok { + namespace, name = strings.ToLower(scope), strings.ToLower(pkgName) + key := purl.New("swift", namespace, name, "", nil).String() + return identity{key: key, version: version} + } } key := purl.New(pkg.Type, namespace, name, "", nil).String() return identity{key: key, channel: channel, version: version} diff --git a/internal/denylist/denylist_test.go b/internal/denylist/denylist_test.go index ba74ddb..a110608 100644 --- a/internal/denylist/denylist_test.go +++ b/internal/denylist/denylist_test.go @@ -47,6 +47,12 @@ func TestEcosystemSpellings(t *testing.T) { "pkg:conda/numpy@1.26.4", "pkg:conda/scipy@1.0?channel=conda-forge", "pkg:conda/bioconda/samtools@1.0", + "pkg:composer/symfony/console@6.1.7", + "pkg:composer/monolog/monolog@v3.5.0", + "pkg:oci/library/alpine@sha256:abc", + "pkg:oci/upstream%2Fghcr%2Forg%2Fimage@sha256:def", + "pkg:swift/Apple/Swift-NIO@2.0.0", + "pkg:generic/swift-registry/apple.example@1.0.0", }) if err != nil { t.Fatal(err) @@ -58,6 +64,12 @@ func TestEcosystemSpellings(t *testing.T) { "pkg:gem/sorbet-static@0.5.1-universal-darwin-22", "pkg:conda/numpy@1.26.4", "pkg:conda/conda-forge%2Fnumpy@1.26.4", "pkg:conda/conda-forge%2Fscipy@1.0", "pkg:conda/bioconda%2Fsamtools@1.0", + "pkg:composer/symfony/console@v6.1.7", "pkg:composer/symfony/console@V6.1.7", + "pkg:composer/monolog/monolog@3.5.0", + "pkg:oci/library%2Falpine@sha256:abc", "pkg:oci/library/alpine@sha256:abc", + "pkg:oci/upstream/ghcr/org/image@sha256:def", + "pkg:generic/swift-registry/apple.swift-nio@2.0.0", "pkg:swift/apple/swift-nio@2.0.0", + "pkg:swift/apple/example@1.0.0", } { if !p.Denied(value) { t.Errorf("not denied: %s", value) @@ -66,6 +78,9 @@ func TestEcosystemSpellings(t *testing.T) { for _, value := range []string{ "pkg:pypi/zopeinterface@6.0", "pkg:gem/nokogiri@1.16.1-x86_64-linux", "pkg:gem/Nokogiri@1.16.0", "pkg:conda/bioconda%2Fscipy@1.0", "pkg:conda/scipy@1.0", "pkg:conda/conda-forge%2Fsamtools@1.0", + "pkg:composer/symfony/console@6.1.70", "pkg:composer/symfony/console@vv6.1.7", "pkg:npm/demo@v6.1.7", + "pkg:oci/alpine@sha256:abc", "pkg:oci/library%2Falpine@sha256:abd", + "pkg:generic/other/apple.swift-nio@2.0.0", "pkg:swift/github.com/apple/swift-nio@2.0.0", } { if p.Denied(value) { t.Errorf("unexpected denial: %s", value) diff --git a/internal/handler/container.go b/internal/handler/container.go index 1d90605..57c30d8 100644 --- a/internal/handler/container.go +++ b/internal/handler/container.go @@ -397,6 +397,10 @@ func (h *ContainerHandler) handleBlobDownload(w http.ResponseWriter, r *http.Req filename := digest cached, err := h.proxy.GetCachedArtifact(r.Context(), "oci", cacheName, digest, filename) + if errors.Is(err, ErrVersionDenied) { + h.containerError(w, http.StatusForbidden, "DENIED", err.Error()) + return + } if err != nil { h.proxy.Logger.Error("failed to check blob cache", "error", err) h.containerError(w, http.StatusInternalServerError, "INTERNAL_ERROR", "failed to check blob cache") @@ -471,14 +475,17 @@ func (h *ContainerHandler) handleManifest(w http.ResponseWriter, r *http.Request return } - registryURL, upstreamName, _, ok := h.registryForRequest(r, name) + registryURL, upstreamName, cacheName, ok := h.registryForRequest(r, name) if !ok { h.containerError(w, http.StatusNotFound, "NAME_UNKNOWN", "unknown upstream registry") return } h.proxy.Logger.Info("container manifest request", "name", upstreamName, "reference", reference) - h.serveManifest(w, r, registryURL, upstreamName, reference) + if h.manifestDenied(w, cacheName, reference) { + return + } + h.serveManifest(w, r, registryURL, upstreamName, cacheName, reference) } // handleTagsList caches tag list responses for offline OCI pulls. diff --git a/internal/handler/container_manifest.go b/internal/handler/container_manifest.go index 62609da..82eed45 100644 --- a/internal/handler/container_manifest.go +++ b/internal/handler/container_manifest.go @@ -49,14 +49,16 @@ type cachedContainerManifest struct { fetchedAt time.Time } -func (h *ContainerHandler) serveManifest(w http.ResponseWriter, r *http.Request, registryURL, name, reference string) { +// serveManifest serves a manifest unless the denylist names the digest it +// resolves to under cacheName, the name blobs are stored under. +func (h *ContainerHandler) serveManifest(w http.ResponseWriter, r *http.Request, registryURL, name, cacheName, reference string) { accept := containerManifestAccept(r) cacheAccept := normalizeContainerManifestAccept(accept) cached, fromFallback := h.loadContainerManifestForAccept(r.Context(), registryURL, name, reference, accept, cacheAccept) immutable := manifestDigestReferencePattern.MatchString(reference) if cached != nil && (immutable || h.containerManifestFresh(cached)) { - writeContainerManifest(w, r, cached, false) + h.writeManifest(w, r, cacheName, cached, false) return } @@ -73,7 +75,7 @@ func (h *ContainerHandler) serveManifest(w http.ResponseWriter, r *http.Request, resp, err := h.proxy.HTTPClient.Do(req) if err != nil { - h.serveStaleManifestOrError(w, r, reference, cached, err) + h.serveStaleManifestOrError(w, r, cacheName, reference, cached, err) return } defer func() { _ = resp.Body.Close() }() @@ -87,12 +89,12 @@ func (h *ContainerHandler) serveManifest(w http.ResponseWriter, r *http.Request, } else { h.storeContainerManifestForAccept(r.Context(), registryURL, name, reference, accept, cacheAccept, cached) } - writeContainerManifest(w, r, cached, false) + h.writeManifest(w, r, cacheName, cached, false) return } if resp.StatusCode != http.StatusOK { if shouldServeStaleManifest(resp.StatusCode) && (cached != nil || referrersTagPattern.MatchString(reference)) { - h.serveStaleManifestOrError(w, r, reference, cached, fmt.Errorf("upstream status %d", resp.StatusCode)) + h.serveStaleManifestOrError(w, r, cacheName, reference, cached, fmt.Errorf("upstream status %d", resp.StatusCode)) return } h.proxy.relayResponse(w, r, resp, copyContainerManifestHeaders) @@ -100,13 +102,16 @@ func (h *ContainerHandler) serveManifest(w http.ResponseWriter, r *http.Request, } if r.Method == http.MethodHead { + if h.manifestDenied(w, cacheName, resp.Header.Get("Docker-Content-Digest")) { + return + } h.proxy.relayResponse(w, r, resp, copyContainerManifestHeaders) return } body, err := h.proxy.ReadMetadata(resp.Body) if err != nil { - h.serveStaleManifestOrError(w, r, reference, cached, fmt.Errorf("reading manifest: %w", err)) + h.serveStaleManifestOrError(w, r, cacheName, reference, cached, fmt.Errorf("reading manifest: %w", err)) return } computedDigest := sha256Digest(body) @@ -136,13 +141,13 @@ func (h *ContainerHandler) serveManifest(w http.ResponseWriter, r *http.Request, if manifest.contentDigest != reference && manifestDigestReferencePattern.MatchString(manifest.contentDigest) { h.storeContainerManifestForAccept(r.Context(), registryURL, name, manifest.contentDigest, accept, cacheAccept, manifest) } - writeContainerManifest(w, r, manifest, false) + h.writeManifest(w, r, cacheName, manifest, false) } -func (h *ContainerHandler) serveStaleManifestOrError(w http.ResponseWriter, r *http.Request, reference string, cached *cachedContainerManifest, err error) { +func (h *ContainerHandler) serveStaleManifestOrError(w http.ResponseWriter, r *http.Request, cacheName, reference string, cached *cachedContainerManifest, err error) { if cached != nil { h.proxy.Logger.Warn("upstream manifest fetch failed, serving stale cache", "error", err) - writeContainerManifest(w, r, cached, true) + h.writeManifest(w, r, cacheName, cached, true) return } if referrersTagPattern.MatchString(reference) { @@ -157,6 +162,25 @@ func (h *ContainerHandler) serveStaleManifestOrError(w http.ResponseWriter, r *h h.containerError(w, http.StatusBadGateway, "INTERNAL_ERROR", "failed to fetch from upstream") } +// manifestDenied writes a 403 and reports true when the denylist names +// reference, a tag or digest, under the repository's cache name. +func (h *ContainerHandler) manifestDenied(w http.ResponseWriter, cacheName, reference string) bool { + if reference == "" || !h.proxy.versionDenied("oci", cacheName, reference) { + return false + } + h.containerError(w, http.StatusForbidden, "DENIED", + ErrVersionDenied.Error()+": "+canonicalVersionPURL("oci", cacheName, reference)) + return true +} + +// writeManifest writes manifest unless the denylist names its digest. +func (h *ContainerHandler) writeManifest(w http.ResponseWriter, r *http.Request, cacheName string, manifest *cachedContainerManifest, stale bool) { + if h.manifestDenied(w, cacheName, manifest.contentDigest) { + return + } + writeContainerManifest(w, r, manifest, stale) +} + func (h *ContainerHandler) containerManifestFresh(manifest *cachedContainerManifest) bool { return h.proxy.MetadataTTL > 0 && !manifest.fetchedAt.IsZero() && time.Since(manifest.fetchedAt) < h.proxy.MetadataTTL } diff --git a/internal/handler/denylist_test.go b/internal/handler/denylist_test.go index e55e36f..8e92471 100644 --- a/internal/handler/denylist_test.go +++ b/internal/handler/denylist_test.go @@ -14,6 +14,7 @@ import ( "github.com/git-pkgs/cooldown" "github.com/git-pkgs/proxy/internal/denylist" + "github.com/git-pkgs/registries/fetch" ) func setTestDenylist(t testing.TB, p *Proxy, packages ...string) { @@ -281,10 +282,22 @@ func TestDenylistMatchesEcosystemSpellings(t *testing.T) { {"conda channel qualifier", "pkg:conda/numpy@1.26.4?channel=conda-forge", "/conda-forge/linux-64/numpy-1.26.4-py311_0.conda", "conda", "conda-forge/numpy", "1.26.4", "numpy-1.26.4-py311_0.conda", func(p *Proxy) http.Handler { return NewCondaHandler(p, "http://proxy.test").Routes() }}, + {"gem name with digit segment", "pkg:gem/devise-2fa@1.0.0", "/gems/devise-2fa-1.0.0.gem", + "gem", "devise-2fa", "1.0.0", "devise-2fa-1.0.0.gem", + func(p *Proxy) http.Handler { return NewGemHandler(p, "http://proxy.test").Routes() }}, + {"go case-escaped version", "pkg:golang/github.com/example/mod@v1.0.0-RC1", "/github.com/example/mod/@v/v1.0.0-!r!c1.zip", + "golang", "github.com/example/mod", "v1.0.0-!r!c1", "mod@v1.0.0-!r!c1.zip", + func(p *Proxy) http.Handler { return NewGoHandler(p, "http://proxy.test").Routes() }}, + {"oci repository path", "pkg:oci/library/alpine@" + ociTestDigest, "/library/alpine/blobs/" + ociTestDigest, + "oci", "library/alpine", ociTestDigest, ociTestDigest, + func(p *Proxy) http.Handler { + return NewContainerHandlerWithRegistry(p, "http://proxy.test", "https://registry.invalid").Routes() + }}, } { t.Run(tc.name, func(t *testing.T) { // The seeded artifact would be served if the entry didn't match. - p, db, store, _ := setupTestProxy(t) + p, db, store, fetcher := setupTestProxy(t) + fetcher.fetchErr = errors.New("offline") seedPackage(t, db, store, tc.ecosystem, tc.pkg, tc.version, tc.filename, "cached content") setTestDenylist(t, p, tc.entry) w := httptest.NewRecorder() @@ -296,6 +309,104 @@ func TestDenylistMatchesEcosystemSpellings(t *testing.T) { } } +const ociTestDigest = "sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" + +// Packagist keeps a tag's leading v in the version it lists, and so in the +// download URL the proxy hands out. +func TestComposerDenylistMatchesTagPrefix(t *testing.T) { + var srv *httptest.Server + srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/p2/symfony/console.json": + _, _ = fmt.Fprintf(w, `{"packages":{"symfony/console":[{"version":"v6.1.7","version_normalized":"6.1.7.0","dist":{"url":%q,"type":"zip"}}]}}`, + srv.URL+"/archives/console.zip") + case "/archives/console.zip": + _, _ = io.WriteString(w, "archive") + default: + http.NotFound(w, r) + } + })) + t.Cleanup(srv.Close) + p, _, _, _ := setupTestProxy(t) + p.Fetcher = fetch.NewFetcher(fetch.WithHTTPClient(srv.Client()), fetch.WithMaxRetries(0)) + setTestDenylist(t, p, "pkg:composer/symfony/console@6.1.7") + h := NewComposerHandlerWithUpstreams(p, "http://proxy.test", srv.URL, srv.URL).Routes() + w := httptest.NewRecorder() + h.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/files/symfony/console/v6.1.7/console.zip", nil)) + if w.Code != http.StatusForbidden { + t.Fatalf("status=%d body=%s", w.Code, w.Body.String()) + } +} + +// Swift archives are cached under pkg:generic/swift-registry/{scope}.{name}, +// which is also what streaming checks. +func TestSwiftDenylistBlocksArchive(t *testing.T) { + for _, stream := range []bool{false, true} { + for _, entry := range []string{"pkg:swift/Apple/Example@1.2.3", "pkg:generic/swift-registry/apple.example@1.2.3"} { + t.Run(fmt.Sprintf("stream=%t/%s", stream, entry), func(t *testing.T) { + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = io.WriteString(w, `{"id":"apple.example","version":"1.2.3","resources":[{"name":"source-archive","type":"application/zip","checksum":"`+strings.Repeat("0", 64)+`"}]}`) + })) + t.Cleanup(upstream.Close) + p, _, _, fetcher := setupTestProxy(t) + fetcher.artifact = &fetch.Artifact{Body: io.NopCloser(strings.NewReader("archive")), Size: 7} + p.StreamArtifacts = stream + setTestDenylist(t, p, entry) + req := httptest.NewRequest(http.MethodGet, "/apple/example/1.2.3.zip", nil) + req.Header.Set("Accept", swiftAcceptArchive) + w := httptest.NewRecorder() + NewSwiftHandler(p, "http://proxy.test", upstream.URL).Routes().ServeHTTP(w, req) + if w.Code != http.StatusForbidden || fetcher.fetchCalled { + t.Fatalf("status=%d fetched=%t body=%s", w.Code, fetcher.fetchCalled, w.Body.String()) + } + }) + } + } +} + +// A denied digest is refused as a manifest as well as a blob, whether the +// client names it or a tag resolves to it. +func TestContainerDenylistBlocksManifests(t *testing.T) { + manifest := `{"schemaVersion":2}` + digest := sha256Digest([]byte(manifest)) + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set(headerContentType, "application/vnd.oci.image.manifest.v1+json") + w.Header().Set("Docker-Content-Digest", digest) + _, _ = io.WriteString(w, manifest) + })) + t.Cleanup(upstream.Close) + for _, tc := range []struct{ entry, reference string }{ + {"pkg:oci/library/alpine@" + digest, digest}, + {"pkg:oci/library/alpine@" + digest, "3.19"}, + {"pkg:oci/library/alpine@3.19", "3.19"}, + } { + t.Run(tc.entry+"/"+tc.reference, func(t *testing.T) { + p := testProxy() + p.HTTPClient = upstream.Client() + h := NewContainerHandlerWithRegistry(p, "http://proxy.test", upstream.URL).Routes() + // Cache the manifest first, then deny it. + w := httptest.NewRecorder() + h.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/library/alpine/manifests/"+tc.reference, nil)) + if w.Code != http.StatusOK { + t.Fatalf("allowed status=%d", w.Code) + } + setTestDenylist(t, p, tc.entry) + for _, method := range []string{http.MethodGet, http.MethodHead} { + w = httptest.NewRecorder() + h.ServeHTTP(w, httptest.NewRequest(method, "/library/alpine/manifests/"+tc.reference, nil)) + if w.Code != http.StatusForbidden { + t.Fatalf("%s status=%d body=%s", method, w.Code, w.Body.String()) + } + } + w = httptest.NewRecorder() + h.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/library/alpine/manifests/other", nil)) + if strings.HasPrefix(tc.entry, "pkg:oci/library/alpine@3.19") && w.Code != http.StatusOK { + t.Fatalf("other tag status=%d", w.Code) + } + }) + } +} + func TestPyPIDenylistSimpleMatchesSeparators(t *testing.T) { body := `xy` h, p := setupPyPIHandler(t, func(r *http.Request) (*http.Response, error) { diff --git a/internal/handler/gem.go b/internal/handler/gem.go index 2d2136b..e1acb12 100644 --- a/internal/handler/gem.go +++ b/internal/handler/gem.go @@ -5,6 +5,7 @@ import ( "encoding/json" "fmt" "net/http" + "regexp" "strings" "time" @@ -88,19 +89,45 @@ func (h *GemHandler) handleDownload(w http.ResponseWriter, r *http.Request) { // e.g., "rails-7.1.0.gem" -> ("rails", "7.1.0") // e.g., "aws-sdk-s3-1.142.0.gem" -> ("aws-sdk-s3", "1.142.0") // e.g., "sorbet-static-0.5.11002-universal-darwin-22.gem" -> ("sorbet-static", "0.5.11002-universal-darwin-22") +// e.g., "devise-2fa-1.0.0.gem" -> ("devise-2fa", "1.0.0") // -// The version starts at the first hyphen followed by a digit. Platforms can -// end in a number (universal-java-17, arm64-darwin-23), so splitting at the -// last one would cut the platform; only gem names with a "-" segment -// are misread this way, and those are rare. +// A gem file is named {name}-{version}[-{platform}]. RubyGems writes a +// version as dot-separated alphanumeric segments starting with a digit, +// turning any "-" into ".pre.", and a platform starts with a letter (its +// CPU, or java), though it can end in a number (universal-java-17), so +// splitting at the last "-" would cut the platform. The version +// starts at the first "-" that leaves such a version and platform, +// so a name with a "-" segment of its own, such as devise-2fa, is +// kept whole. A filename no split fits is split at the first "-". func (h *GemHandler) parseGemFilename(filename string) (name, version string) { base := strings.TrimSuffix(filename, ".gem") + first := -1 for i := 0; i+1 < len(base); i++ { - if base[i] == '-' && base[i+1] >= '0' && base[i+1] <= '9' { + if base[i] != '-' || base[i+1] < '0' || base[i+1] > '9' { + continue + } + if first < 0 { + first = i + } + if gemVersionAndPlatform(base[i+1:]) { return base[:i], base[i+1:] } } - return "", "" + if first < 0 { + return "", "" + } + return base[:first], base[first+1:] +} + +var ( + gemVersionPattern = regexp.MustCompile(`^[0-9]+(\.[0-9A-Za-z]+)*$`) + gemPlatformPattern = regexp.MustCompile(`^[A-Za-z][0-9A-Za-z_]*(-[0-9A-Za-z_]+)*$`) +) + +// gemVersionAndPlatform reports whether s reads as {version}[-{platform}]. +func gemVersionAndPlatform(s string) bool { + version, platform, hasPlatform := strings.Cut(s, "-") + return gemVersionPattern.MatchString(version) && (!hasPlatform || gemPlatformPattern.MatchString(platform)) } // handleCompactIndex serves the compact index for a gem, filtering versions diff --git a/internal/handler/gem_test.go b/internal/handler/gem_test.go index ce63deb..9b71074 100644 --- a/internal/handler/gem_test.go +++ b/internal/handler/gem_test.go @@ -30,6 +30,12 @@ func TestGemParseFilename(t *testing.T) { {"google-protobuf-3.25.3-arm64-darwin-23.gem", "google-protobuf", "3.25.3-arm64-darwin-23"}, {"nokogiri-1.16.0-x64-mingw-ucrt.gem", "nokogiri", "1.16.0-x64-mingw-ucrt"}, {"rake-13.1.0.pre.1.gem", "rake", "13.1.0.pre.1"}, + {"devise-2fa-1.0.0.gem", "devise-2fa", "1.0.0"}, + {"devise-2fa-1.0.0-java.gem", "devise-2fa", "1.0.0-java"}, + {"foo-2-1.0.0.gem", "foo-2", "1.0.0"}, + {"foo-2-1.gem", "foo-2", "1"}, + {"foo-2-x86_64-linux.gem", "foo", "2-x86_64-linux"}, + {"odd-1_0.gem", "odd", "1_0"}, {"no-version.gem", "", ""}, {"invalid", "", ""}, } diff --git a/internal/handler/go.go b/internal/handler/go.go index 7a1eb86..c8ae81a 100644 --- a/internal/handler/go.go +++ b/internal/handler/go.go @@ -10,6 +10,7 @@ import ( "github.com/git-pkgs/proxy/internal/database" "github.com/git-pkgs/proxy/internal/retention" "github.com/git-pkgs/registries/fetch" + gomodule "golang.org/x/mod/module" ) const ( @@ -126,6 +127,14 @@ func (h *GoHandler) handleDownload(w http.ResponseWriter, r *http.Request, modul h.proxy.Logger.Info("go module download request", "module", decodedModule, "version", version) + // The URL escapes uppercase letters in the version (v1.0.0-!r!c1 for + // v1.0.0-RC1), and the zip is cached under that form, so the denylist + // is checked against the version itself. + if unescaped, err := gomodule.UnescapeVersion(version); err == nil && h.proxy.versionDenied("golang", decodedModule, unescaped) { + http.Error(w, ErrVersionDenied.Error()+": "+canonicalVersionPURL("golang", decodedModule, unescaped), http.StatusForbidden) + return + } + downloadURL := h.upstreamURL + r.URL.Path h.discardOtherModulesZip(decodedModule, module, version, filename) result, err := h.proxy.GetOrFetchArtifactFromURL( diff --git a/internal/handler/handler.go b/internal/handler/handler.go index ff5466a..164e161 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -1924,10 +1924,15 @@ func (p *Proxy) getOrFetchArtifactFromURL(ctx context.Context, ecosystem, name, } func (p *Proxy) getOrFetchArtifactFromURLWithCachePURLs(ctx context.Context, ecosystem, name, version, filename, pkgPURL, versionPURL, downloadURL string, headers http.Header, upstreamHash, declared string) (*CacheResult, error) { - // Some protocols use qualified cache keys. Check their package identity too. + // Some protocols use qualified cache keys. Check their package identity + // too, and the cache key itself for those, such as Swift, whose name has + // no PURL of its own. if p.versionDenied(ecosystem, name, version) { return nil, fmt.Errorf("%w: %s", ErrVersionDenied, canonicalVersionPURL(ecosystem, name, version)) } + if p.Denylist.Denied(versionPURL) { + return nil, fmt.Errorf("%w: %s", ErrVersionDenied, versionPURL) + } if p.StreamArtifacts { return p.streamFromUpstream(ctx, ecosystem, name, version, filename, versionPURL, downloadURL, upstreamHash, declared, func(fetchCtx context.Context) (*fetch.Artifact, error) { diff --git a/internal/handler/swift.go b/internal/handler/swift.go index b506f1c..d7d403f 100644 --- a/internal/handler/swift.go +++ b/internal/handler/swift.go @@ -628,6 +628,10 @@ func (h *SwiftHandler) writeArtifactError(w http.ResponseWriter, err error) { writeSwiftProblem(w, http.StatusNotFound, "release not found") return } + if errors.Is(err, ErrArtifactBlocked) || errors.Is(err, ErrVersionDenied) { + writeSwiftProblem(w, http.StatusForbidden, err.Error()) + return + } if writeUpstreamHeld(w, err) { return } From 96cc1ee87f6a76d2420c26d80a998e0790c2369f Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Fri, 23 Oct 2026 12:07:01 +0200 Subject: [PATCH 360/633] Serve conda sharded repodata through cooldown and the cache The shard index (repodata_shards.msgpack.zst) and the shards of CEP 16 fell through to the uncached passthrough, so clients that use them, such as pixi, got no cooldown filtering and no offline copy. Without cooldown, the index and shards are now served through the metadata cache. With cooldown, shards have to be filtered, but clients cache them by the hash in the index and never fetch a hash twice. The proxy rewrites the index instead: each hash is XORed with a mask derived from channel, subdir and the current hour, and shards_base_url points at cooldown-shards/{hour}/. That route undoes the mask, fetches the upstream shard and drops records still in cooldown. The unmasked shard paths answer 404 while cooldown is on. When the index puts packages or shards somewhere the proxy doesn't route, or has an unknown version, the proxy answers 404 and clients fall back to repodata.json, which is filtered as before. Fixes #529 --- README.md | 2 + docs/configuration.md | 2 + go.mod | 4 +- go.sum | 4 + internal/handler/conda.go | 31 +- internal/handler/conda_shards.go | 469 ++++++++++++++++++++++++++ internal/handler/conda_shards_test.go | 275 +++++++++++++++ internal/handler/handler.go | 23 +- 8 files changed, 794 insertions(+), 16 deletions(-) create mode 100644 internal/handler/conda_shards.go create mode 100644 internal/handler/conda_shards_test.go diff --git a/README.md b/README.md index 0553e4a..6e0e9d8 100644 --- a/README.md +++ b/README.md @@ -411,6 +411,8 @@ Or set via command: conda config --add channels http://localhost:8080/conda/main ``` +pixi and other rattler-based clients use sharded repodata ([CEP 16](https://github.com/conda/ceps/blob/main/cep-0016.md)) where a channel offers it, as conda-forge does. Point them at the same channel URLs, for example `pixi init -c http://localhost:8080/conda/conda-forge`. The shard index and shards are cached as metadata when `cache_metadata` is on. + ### CRAN (R) Set the repository in R: diff --git a/docs/configuration.md b/docs/configuration.md index 4da6f63..e151ea8 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -982,6 +982,8 @@ Resolution order: exact package override, then package pattern, then ecosystem o Currently supported for npm, PyPI, pub.dev, Composer, Cargo, NuGet, Conda, RubyGems, and Hex. These ecosystems include publish timestamps in their metadata. +Conda cooldown covers `repodata.json` and sharded repodata (CEP 16). Clients cache shards by the hash in the shard index, so with cooldown on the proxy serves a rewritten index whose shard hashes change every hour and filters each shard as it is fetched. Versions leaving cooldown show up within the hour, and clients fetch the shards they need again once an hour. If a channel's shard index points packages or shards somewhere the proxy doesn't route, the proxy answers 404 for it and clients fall back to `repodata.json`. + Note: Hex cooldown requires disabling registry signature verification since the proxy re-encodes the protobuf payload without the original signature. Set `HEX_NO_VERIFY_REPO_ORIGIN=1` or configure your repo with `no_verify: true`. ## Version denylist diff --git a/go.mod b/go.mod index e2f860e..7d17bfd 100644 --- a/go.mod +++ b/go.mod @@ -24,11 +24,13 @@ require ( github.com/git-pkgs/vulns v0.2.4 github.com/go-chi/chi/v5 v5.5.0 github.com/jmoiron/sqlx v1.4.0 + github.com/klauspost/compress v1.20.1 github.com/lib/pq v1.12.3 github.com/opencontainers/go-digest v1.0.0 github.com/prometheus/client_golang v1.25.0 github.com/prometheus/client_model v0.6.3 github.com/swaggo/swag v1.16.6 + github.com/vmihailenco/msgpack/v5 v5.4.1 gocloud.dev v0.47.0 golang.org/x/mod v0.42.0 golang.org/x/net v0.61.0 @@ -90,7 +92,6 @@ require ( github.com/googleapis/enterprise-certificate-proxy v0.3.21 // indirect github.com/googleapis/gax-go/v2 v2.24.1 // indirect github.com/josharian/intern v1.0.0 // indirect - github.com/klauspost/compress v1.20.1 // indirect github.com/kylelemons/godebug v1.1.0 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-isatty v0.0.24 // indirect @@ -108,6 +109,7 @@ require ( github.com/rs/dnscache v0.0.0-20230804202142-fc85eb664529 // indirect github.com/rubyist/circuitbreaker v2.2.1+incompatible // indirect github.com/ulikunitz/xz v0.5.17 // indirect + github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect go.opentelemetry.io/otel v1.46.0 // indirect go.opentelemetry.io/otel/metric v1.46.0 // indirect diff --git a/go.sum b/go.sum index f298c28..ee37652 100644 --- a/go.sum +++ b/go.sum @@ -275,6 +275,10 @@ github.com/swaggo/swag v1.16.6 h1:qBNcx53ZaX+M5dxVyTrgQ0PJ/ACK+NzhwcbieTt+9yI= github.com/swaggo/swag v1.16.6/go.mod h1:ngP2etMK5a0P3QBizic5MEwpRmluJZPHjXcMoj4Xesg= github.com/ulikunitz/xz v0.5.17 h1:flR0y/x1hgM8EGV1AW3Xll6T413G0glV8UfBwR617V4= github.com/ulikunitz/xz v0.5.17/go.mod h1:H9Rt/W6/Qj27PGauhQc6nfCDy7vHpzsOThBSaYDoEhw= +github.com/vmihailenco/msgpack/v5 v5.4.1 h1:cQriyiUvjTwOHg8QZaPihLWeRAAVoCpE00IUPn0Bjt8= +github.com/vmihailenco/msgpack/v5 v5.4.1/go.mod h1:GaZTsDaehaPpQVyxrf5mtQlH+pc21PIudVV/E3rRQok= +github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g= +github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= go.opentelemetry.io/contrib/detectors/gcp v1.46.0 h1:PI8dGkqDaQkwJ8kOopqMhDTbrnK3UIeG/RCHH4HErbo= diff --git a/internal/handler/conda.go b/internal/handler/conda.go index 7f235e1..0f229fc 100644 --- a/internal/handler/conda.go +++ b/internal/handler/conda.go @@ -82,6 +82,11 @@ func (h *CondaHandler) Routes() http.Handler { mux.HandleFunc("GET /{channel}/{arch}/repodata.json.bz2", h.proxyCached) mux.HandleFunc("GET /{channel}/{arch}/current_repodata.json", h.handleRepodata) + // Sharded repodata (CEP 16) + mux.HandleFunc("GET /{channel}/{arch}/"+condaShardIndexFile, h.handleShardIndex) + mux.HandleFunc("GET /{channel}/{arch}/shards/{filename}", h.handleShard) + mux.HandleFunc("GET /{channel}/{arch}/"+condaCooldownShardDir+"/{bucket}/{rest...}", h.handleCooldownShard) + // Package downloads (cache these) mux.HandleFunc("GET /{channel}/{arch}/{filename}", h.handleDownload) @@ -94,6 +99,11 @@ func (h *CondaHandler) handleDownload(w http.ResponseWriter, r *http.Request) { arch := r.PathValue("arch") filename := r.PathValue("filename") + if strings.HasSuffix(filename, condaShardSuffix) { + h.handleShard(w, r) + return + } + // Only cache actual package files if !h.isPackageFile(filename) { h.proxyUpstream(w, r) @@ -288,22 +298,31 @@ type repodataEntry struct { // Entries without a name or timestamp are kept. func (h *CondaHandler) inCooldown(filename string, raw []byte) bool { var entry repodataEntry - if err := json.Unmarshal(raw, &entry); err != nil || entry.Name == "" || entry.Timestamp == 0 { + if err := json.Unmarshal(raw, &entry); err != nil { return false } - publishedAt := time.Unix(int64(entry.Timestamp)/condaTimestampDivisor, 0) - if h.proxy.Cooldown.IsAllowed("conda", canonicalPackagePURL("conda", entry.Name), publishedAt) { + return h.recordInCooldown(filename, entry.Name, entry.Version, entry.Timestamp) +} + +// recordInCooldown reports whether a package record with a millisecond +// timestamp was published too recently. Records without a name or +// timestamp are kept. +func (h *CondaHandler) recordInCooldown(filename, name, version string, timestamp float64) bool { + if name == "" || timestamp == 0 { + return false + } + publishedAt := time.Unix(int64(timestamp)/condaTimestampDivisor, 0) + if h.proxy.Cooldown.IsAllowed("conda", canonicalPackagePURL("conda", name), publishedAt) { return false } h.proxy.Logger.Info("cooldown: filtering conda package", - "name", entry.Name, "version", entry.Version, "filename", filename) + "name", name, "version", version, "filename", filename) return true } // proxyCached forwards a metadata request with caching. func (h *CondaHandler) proxyCached(w http.ResponseWriter, r *http.Request) { - cacheKey := strings.TrimPrefix(r.URL.Path, "/") - cacheKey = strings.ReplaceAll(cacheKey, "/", "_") + cacheKey := condaCacheKey(r.URL.Path) // Large JSON indexes can exceed the metadata limit uncompressed. Conda // clients decode Content-Encoding, so cache and relay gzip verbatim. The // already-compressed repodata.json.bz2 route must stay on identity. diff --git a/internal/handler/conda_shards.go b/internal/handler/conda_shards.go new file mode 100644 index 0000000..f28f348 --- /dev/null +++ b/internal/handler/conda_shards.go @@ -0,0 +1,469 @@ +package handler + +import ( + "bytes" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "net/http" + "net/url" + "path" + "strconv" + "strings" + "sync" + "time" + + "github.com/klauspost/compress/zstd" + "github.com/vmihailenco/msgpack/v5" +) + +// Sharded repodata (CEP 16) splits repodata.json into an index, +// /repodata_shards.msgpack.zst, that maps each package name to the +// sha256 of its shard, and one zstd-compressed msgpack shard per package +// stored under that hash. +// +// Without cooldown the index and the shards are passed through and cached as +// metadata. With cooldown, shards have to be filtered, and that changes their +// bytes. Clients cache shards by the hash in the index and never fetch the same +// hash again, so a filtered shard can't be served under its upstream hash. The +// proxy rewrites the index instead: every hash is XORed with a mask derived +// from the channel, the subdir and the current hour, and shards_base_url points +// at cooldown-shards/{hour}/. A shard request there undoes the mask, fetches +// the upstream shard and filters it. Each hour the keys change, so clients +// fetch shards again and see versions that have left cooldown since. +// +// When the index can't be served this way, the proxy answers 404 and clients +// fall back to repodata.json, which is filtered too. +const ( + condaShardIndexFile = "repodata_shards.msgpack.zst" + condaShardSuffix = ".msgpack.zst" + condaCooldownShardDir = "cooldown-shards" + condaShardsSubdir = "shards/" + condaShardBucket = time.Hour + condaShardIndexFormat = 1 + condaShardHashLen = sha256.Size + // maxCondaShardSize bounds a decompressed index or shard. conda-forge's + // largest index is a few MB. + maxCondaShardSize = 256 << 20 +) + +var ( + errCondaShardLayout = errors.New("unsupported sharded repodata layout") + + condaZstd = sync.OnceValues(func() (*zstd.Decoder, error) { + return zstd.NewReader(nil, zstd.WithDecoderMaxMemory(maxCondaShardSize), zstd.WithDecoderConcurrency(0)) + }) + condaZstdEncoder = sync.OnceValues(func() (*zstd.Encoder, error) { + return zstd.NewWriter(nil) + }) +) + +func (h *CondaHandler) cooldownEnabled() bool { + return h.proxy.Cooldown != nil && h.proxy.Cooldown.Enabled() +} + +// handleShardIndex serves repodata_shards.msgpack.zst. +func (h *CondaHandler) handleShardIndex(w http.ResponseWriter, r *http.Request) { + channel, arch := r.PathValue("channel"), r.PathValue("arch") + upstreamURL := h.upstreamURL + r.URL.Path + cacheKey := condaCacheKey(r.URL.Path) + + body, contentType, contentEncoding, err := h.proxy.fetchOrCacheMetadata(r.Context(), "conda", cacheKey, upstreamURL, "identity", nil, "*/*") + if err != nil { + h.proxy.writeMetadataFetchError(w, err) + return + } + + if !h.cooldownEnabled() { + index, err := decodeCondaShardIndex(body) + if err == nil { + _, err = index.shardsLayout(upstreamURL) + } + if err != nil { + h.proxy.Logger.Info("serving no sharded repodata", "url", upstreamURL, "error", err) + http.NotFound(w, r) + return + } + h.proxy.writeMetadataCachedResponseWithEncoding(w, r, "conda", cacheKey, body, contentType, contentEncoding) + return + } + + now := time.Now() + bucket := strconv.FormatInt(now.Unix()/int64(condaShardBucket/time.Second), 10) + rewritten, err := rewriteCondaShardIndex(body, upstreamURL, condaShardMask(channel, arch, bucket), bucket) + if err != nil { + h.proxy.Logger.Info("serving no sharded repodata", "url", upstreamURL, "error", err) + http.NotFound(w, r) + return + } + + next := now.Truncate(condaShardBucket).Add(condaShardBucket) + maxAge := max(int(next.Sub(now)/time.Second), 1) + w.Header().Set(headerContentType, "application/octet-stream") + w.Header().Set("Cache-Control", fmt.Sprintf("public, max-age=%d", maxAge)) + _, _ = w.Write(rewritten) +} + +// handleShard serves a shard under its upstream hash. While cooldown is on, +// the index never points here, and the unfiltered shard is refused. +func (h *CondaHandler) handleShard(w http.ResponseWriter, r *http.Request) { + if h.cooldownEnabled() || !isCondaShardFile(path.Base(r.URL.Path)) { + http.NotFound(w, r) + return + } + h.proxyCached(w, r) +} + +// handleCooldownShard serves a shard named in a rewritten index, filtered by +// cooldown. +func (h *CondaHandler) handleCooldownShard(w http.ResponseWriter, r *http.Request) { + channel, arch, bucket := r.PathValue("channel"), r.PathValue("arch"), r.PathValue("bucket") + dir, file := path.Split(r.PathValue("rest")) + if _, err := strconv.ParseUint(bucket, 10, 64); err != nil || (dir != "" && dir != condaShardsSubdir) || !isCondaShardFile(file) { + http.NotFound(w, r) + return + } + + masked, _ := hex.DecodeString(strings.TrimSuffix(file, condaShardSuffix)) + mask := condaShardMask(channel, arch, bucket) + for i := range masked { + masked[i] ^= mask[i] + } + upstreamPath := "/" + channel + "/" + arch + "/" + dir + hex.EncodeToString(masked) + condaShardSuffix + + body, _, _, err := h.proxy.fetchOrCacheMetadata(r.Context(), "conda", condaCacheKey(upstreamPath), h.upstreamURL+upstreamPath, "identity", nil, "*/*") + if err != nil { + h.proxy.writeMetadataFetchError(w, err) + return + } + + if h.cooldownEnabled() { + body, err = h.filterCondaShard(body) + if err != nil { + h.proxy.Logger.Error("failed to filter conda shard", "path", upstreamPath, "error", err) + http.Error(w, "failed to filter shard", http.StatusBadGateway) + return + } + } + w.Header().Set(headerContentType, "application/octet-stream") + _, _ = w.Write(body) +} + +func isCondaShardFile(name string) bool { + hash, ok := strings.CutSuffix(name, condaShardSuffix) + if !ok || len(hash) != 2*condaShardHashLen { + return false + } + _, err := hex.DecodeString(hash) + return err == nil && strings.ToLower(hash) == hash +} + +// condaShardMask is XORed with shard hashes in a rewritten index. It keeps +// filtered shards apart from upstream ones in client caches, which are keyed +// by hash alone. +func condaShardMask(channel, arch, bucket string) [condaShardHashLen]byte { + return sha256.Sum256([]byte("git-pkgs proxy conda cooldown shard\x00" + channel + "\x00" + arch + "\x00" + bucket)) +} + +// msgpackEntry is one member of a msgpack map, kept in its upstream order +// with its value undecoded. +type msgpackEntry struct { + key string + value msgpack.RawMessage +} + +func decodeMsgpackMap(b []byte) ([]msgpackEntry, error) { + dec := msgpack.NewDecoder(bytes.NewReader(b)) + n, err := dec.DecodeMapLen() + if err != nil { + return nil, err + } + if n < 0 { + return nil, errors.New("expected a msgpack map") + } + entries := make([]msgpackEntry, 0, n) + for range n { + key, err := dec.DecodeString() + if err != nil { + return nil, err + } + value, err := dec.DecodeRaw() + if err != nil { + return nil, err + } + entries = append(entries, msgpackEntry{key: key, value: value}) + } + return entries, nil +} + +func encodeMsgpackMap(entries []msgpackEntry) ([]byte, error) { + var buf bytes.Buffer + enc := msgpack.NewEncoder(&buf) + if err := enc.EncodeMapLen(len(entries)); err != nil { + return nil, err + } + for _, e := range entries { + if err := enc.EncodeString(e.key); err != nil { + return nil, err + } + if err := enc.Encode(e.value); err != nil { + return nil, err + } + } + return buf.Bytes(), nil +} + +func lookupMsgpackEntry(entries []msgpackEntry, key string) (int, bool) { + for i, e := range entries { + if e.key == key { + return i, true + } + } + return -1, false +} + +func decompressCondaShard(body []byte) ([]byte, error) { + dec, err := condaZstd() + if err != nil { + return nil, err + } + return dec.DecodeAll(body, nil) +} + +func compressCondaShard(body []byte) ([]byte, error) { + enc, err := condaZstdEncoder() + if err != nil { + return nil, err + } + return enc.EncodeAll(body, nil), nil +} + +// condaShardIndex is a decoded repodata_shards.msgpack.zst. +type condaShardIndex struct { + top []msgpackEntry + info []msgpackEntry +} + +func decodeCondaShardIndex(body []byte) (*condaShardIndex, error) { + raw, err := decompressCondaShard(body) + if err != nil { + return nil, err + } + top, err := decodeMsgpackMap(raw) + if err != nil { + return nil, err + } + i, ok := lookupMsgpackEntry(top, "version") + if !ok { + return nil, errors.New("sharded index has no version") + } + var version int + if err := msgpack.Unmarshal(top[i].value, &version); err != nil || version != condaShardIndexFormat { + return nil, fmt.Errorf("unsupported sharded index version %v", top[i].value) + } + i, ok = lookupMsgpackEntry(top, "info") + if !ok { + return nil, errors.New("sharded index has no info") + } + info, err := decodeMsgpackMap(top[i].value) + if err != nil { + return nil, err + } + return &condaShardIndex{top: top, info: info}, nil +} + +func (idx *condaShardIndex) infoString(key string) (string, error) { + i, ok := lookupMsgpackEntry(idx.info, key) + if !ok { + return "", nil + } + var s string + err := msgpack.Unmarshal(idx.info[i].value, &s) + return s, err +} + +// shardsLayout checks that packages and shards sit where the proxy routes +// them, below the subdir of indexURL, and returns the shards directory +// relative to the subdir: "" or "shards/". +func (idx *condaShardIndex) shardsLayout(indexURL string) (string, error) { + base, err := url.Parse(indexURL) + if err != nil { + return "", err + } + subdir := base.ResolveReference(&url.URL{Path: "./"}).String() + resolve := func(ref string) (string, error) { + if ref == "" { + return subdir, nil + } + u, err := url.Parse(ref) + if err != nil { + return "", err + } + return base.ResolveReference(u).String(), nil + } + + baseURL, err := idx.infoString("base_url") + if err != nil { + return "", err + } + if resolved, err := resolve(baseURL); err != nil || resolved != subdir { + return "", fmt.Errorf("%w: base_url %q", errCondaShardLayout, baseURL) + } + + shardsBaseURL, err := idx.infoString("shards_base_url") + if err != nil { + return "", err + } + resolved, err := resolve(shardsBaseURL) + if err != nil { + return "", err + } + rel, ok := strings.CutPrefix(resolved, subdir) + if !ok || (rel != "" && rel != condaShardsSubdir) { + return "", fmt.Errorf("%w: shards_base_url %q", errCondaShardLayout, shardsBaseURL) + } + return rel, nil +} + +// rewriteCondaShardIndex masks every shard hash and points shards_base_url +// at the cooldown shard route for bucket. +func rewriteCondaShardIndex(body []byte, indexURL string, mask [condaShardHashLen]byte, bucket string) ([]byte, error) { + idx, err := decodeCondaShardIndex(body) + if err != nil { + return nil, err + } + layout, err := idx.shardsLayout(indexURL) + if err != nil { + return nil, err + } + + shardsBase, err := msgpack.Marshal("./" + condaCooldownShardDir + "/" + bucket + "/" + layout) + if err != nil { + return nil, err + } + if i, ok := lookupMsgpackEntry(idx.info, "shards_base_url"); ok { + idx.info[i].value = shardsBase + } else { + idx.info = append(idx.info, msgpackEntry{key: "shards_base_url", value: shardsBase}) + } + info, err := encodeMsgpackMap(idx.info) + if err != nil { + return nil, err + } + i, _ := lookupMsgpackEntry(idx.top, "info") + idx.top[i].value = info + + i, ok := lookupMsgpackEntry(idx.top, "shards") + if !ok { + return nil, errors.New("sharded index has no shards") + } + shards, err := decodeMsgpackMap(idx.top[i].value) + if err != nil { + return nil, err + } + for j := range shards { + var hash []byte + if err := msgpack.Unmarshal(shards[j].value, &hash); err != nil || len(hash) != condaShardHashLen { + return nil, fmt.Errorf("shard hash for %q is not a sha256", shards[j].key) + } + for k := range hash { + hash[k] ^= mask[k] + } + if shards[j].value, err = msgpack.Marshal(hash); err != nil { + return nil, err + } + } + if idx.top[i].value, err = encodeMsgpackMap(shards); err != nil { + return nil, err + } + + raw, err := encodeMsgpackMap(idx.top) + if err != nil { + return nil, err + } + return compressCondaShard(raw) +} + +// condaShardRecord holds the fields cooldown needs from a shard record. +// Timestamps are milliseconds, encoded as an integer or a float. +type condaShardRecord struct { + Name string `msgpack:"name"` + Version string `msgpack:"version"` + Timestamp any `msgpack:"timestamp"` +} + +// filterCondaShard leaves out records in packages and packages.conda that +// are still inside their cooldown. Kept records keep their upstream bytes. +func (h *CondaHandler) filterCondaShard(body []byte) ([]byte, error) { + raw, err := decompressCondaShard(body) + if err != nil { + return nil, err + } + top, err := decodeMsgpackMap(raw) + if err != nil { + return nil, err + } + for i := range top { + if top[i].key != "packages" && top[i].key != "packages.conda" { + continue + } + records, err := decodeMsgpackMap(top[i].value) + if err != nil { + return nil, err + } + kept := records[:0] + for _, rec := range records { + var fields condaShardRecord + if err := msgpack.Unmarshal(rec.value, &fields); err != nil { + return nil, fmt.Errorf("record %q: %w", rec.key, err) + } + if h.recordInCooldown(rec.key, fields.Name, fields.Version, msgpackNumber(fields.Timestamp)) { + continue + } + kept = append(kept, rec) + } + if top[i].value, err = encodeMsgpackMap(kept); err != nil { + return nil, err + } + } + out, err := encodeMsgpackMap(top) + if err != nil { + return nil, err + } + return compressCondaShard(out) +} + +// msgpackNumber returns a decoded msgpack integer or float as float64, and 0 +// for anything else. +func msgpackNumber(v any) float64 { + switch n := v.(type) { + case int8: + return float64(n) + case int16: + return float64(n) + case int32: + return float64(n) + case int64: + return float64(n) + case uint8: + return float64(n) + case uint16: + return float64(n) + case uint32: + return float64(n) + case uint64: + return float64(n) + case float32: + return float64(n) + case float64: + return n + default: + return 0 + } +} + +// condaCacheKey is the metadata cache key for a request path. +func condaCacheKey(urlPath string) string { + return strings.ReplaceAll(strings.TrimPrefix(urlPath, "/"), "/", "_") +} diff --git a/internal/handler/conda_shards_test.go b/internal/handler/conda_shards_test.go new file mode 100644 index 0000000..43b507f --- /dev/null +++ b/internal/handler/conda_shards_test.go @@ -0,0 +1,275 @@ +package handler + +import ( + "bytes" + "crypto/sha256" + "encoding/hex" + "net/http" + "net/http/httptest" + "sort" + "strings" + "testing" + "time" + + "github.com/git-pkgs/cooldown" + "github.com/vmihailenco/msgpack/v5" +) + +func zstdMsgpack(t *testing.T, v any) []byte { + t.Helper() + raw, err := msgpack.Marshal(v) + if err != nil { + t.Fatal(err) + } + out, err := compressCondaShard(raw) + if err != nil { + t.Fatal(err) + } + return out +} + +func decodeZstdMsgpack(t *testing.T, body []byte, v any) { + t.Helper() + raw, err := decompressCondaShard(body) + if err != nil { + t.Fatal(err) + } + if err := msgpack.Unmarshal(raw, v); err != nil { + t.Fatal(err) + } +} + +type shardFixture struct { + upstream *httptest.Server + shardHash string + requests []string +} + +// newShardUpstream serves a sharded index for conda-forge/noarch with one +// shard for testpkg, holding an old and a recent build. +func newShardUpstream(t *testing.T, info map[string]any) *shardFixture { + t.Helper() + now := time.Now() + shard := zstdMsgpack(t, map[string]any{ + "packages": map[string]any{ + "testpkg-1.0.0-0.tar.bz2": map[string]any{ + "name": "testpkg", "version": "1.0.0", "timestamp": now.Add(-7 * 24 * time.Hour).UnixMilli(), + "sha256": bytes.Repeat([]byte{1}, 32), + }, + }, + "packages.conda": map[string]any{ + "testpkg-1.0.0-0.conda": map[string]any{ + "name": "testpkg", "version": "1.0.0", "timestamp": now.Add(-7 * 24 * time.Hour).UnixMilli(), + }, + "testpkg-2.0.0-0.conda": map[string]any{ + "name": "testpkg", "version": "2.0.0", "timestamp": now.Add(-time.Hour).UnixMilli(), + }, + }, + "removed": []string{"testpkg-0.1.0-0.tar.bz2"}, + }) + sum := sha256.Sum256(shard) + f := &shardFixture{shardHash: hex.EncodeToString(sum[:])} + + layout := "" + if s, _ := info["shards_base_url"].(string); strings.Contains(s, "shards/") { + layout = "shards/" + } + index := zstdMsgpack(t, map[string]any{ + "version": 1, + "info": info, + "shards": map[string]any{"testpkg": sum[:]}, + }) + + f.upstream = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + f.requests = append(f.requests, r.URL.Path) + switch r.URL.Path { + case "/conda-forge/noarch/" + condaShardIndexFile: + _, _ = w.Write(index) + case "/conda-forge/noarch/" + layout + f.shardHash + condaShardSuffix: + _, _ = w.Write(shard) + default: + http.NotFound(w, r) + } + })) + t.Cleanup(f.upstream.Close) + return f +} + +func shardTestHandler(f *shardFixture, cooldownDefault string) http.Handler { + proxy := testProxy() + if cooldownDefault != "" { + proxy.Cooldown = &cooldown.Config{Default: cooldownDefault} + } + return NewCondaHandlerWithUpstream(proxy, "http://proxy.local", f.upstream.URL).Routes() +} + +func serveShardRequest(h http.Handler, path string) *httptest.ResponseRecorder { + w := httptest.NewRecorder() + h.ServeHTTP(w, httptest.NewRequest(http.MethodGet, path, nil)) + return w +} + +type shardRecords struct { + Packages map[string]map[string]any `msgpack:"packages"` + PackagesConda map[string]map[string]any `msgpack:"packages.conda"` + Removed []string `msgpack:"removed"` +} + +func (s shardRecords) filenames() []string { + var names []string + for k := range s.Packages { + names = append(names, k) + } + for k := range s.PackagesConda { + names = append(names, k) + } + sort.Strings(names) + return names +} + +type shardIndex struct { + Version int `msgpack:"version"` + Info map[string]any `msgpack:"info"` + Shards map[string][]byte `msgpack:"shards"` +} + +func TestCondaShardsWithoutCooldown(t *testing.T) { + for _, shardsBaseURL := range []string{"", "./shards/"} { + t.Run(shardsBaseURL, func(t *testing.T) { + f := newShardUpstream(t, map[string]any{"base_url": "", "shards_base_url": shardsBaseURL, "subdir": "noarch"}) + h := shardTestHandler(f, "") + + w := serveShardRequest(h, "/conda-forge/noarch/"+condaShardIndexFile) + if w.Code != http.StatusOK { + t.Fatalf("index status = %d", w.Code) + } + var index shardIndex + decodeZstdMsgpack(t, w.Body.Bytes(), &index) + if got := hex.EncodeToString(index.Shards["testpkg"]); got != f.shardHash { + t.Fatalf("index hash = %s, want upstream %s", got, f.shardHash) + } + if index.Info["shards_base_url"] != shardsBaseURL { + t.Fatalf("shards_base_url = %v, want %q", index.Info["shards_base_url"], shardsBaseURL) + } + + dir := strings.TrimPrefix(shardsBaseURL, "./") + w = serveShardRequest(h, "/conda-forge/noarch/"+dir+f.shardHash+condaShardSuffix) + if w.Code != http.StatusOK { + t.Fatalf("shard status = %d", w.Code) + } + var shard shardRecords + decodeZstdMsgpack(t, w.Body.Bytes(), &shard) + if len(shard.filenames()) != 3 { + t.Fatalf("records = %v, want all 3", shard.filenames()) + } + }) + } +} + +func TestCondaShardsWithCooldown(t *testing.T) { + f := newShardUpstream(t, map[string]any{"base_url": "", "shards_base_url": "./shards/", "subdir": "noarch", "created_at": "2026-01-01T00:00:00Z"}) + h := shardTestHandler(f, "3d") + + w := serveShardRequest(h, "/conda-forge/noarch/"+condaShardIndexFile) + if w.Code != http.StatusOK { + t.Fatalf("index status = %d", w.Code) + } + if !strings.HasPrefix(w.Header().Get("Cache-Control"), "public, max-age=") { + t.Errorf("Cache-Control = %q", w.Header().Get("Cache-Control")) + } + var index shardIndex + decodeZstdMsgpack(t, w.Body.Bytes(), &index) + masked := hex.EncodeToString(index.Shards["testpkg"]) + if masked == f.shardHash || len(masked) != 64 { + t.Fatalf("index hash = %s, want a masked hash", masked) + } + if index.Info["created_at"] != "2026-01-01T00:00:00Z" || index.Info["subdir"] != "noarch" { + t.Errorf("info lost fields: %v", index.Info) + } + shardsBaseURL, _ := index.Info["shards_base_url"].(string) + if !strings.HasPrefix(shardsBaseURL, "./"+condaCooldownShardDir+"/") || !strings.HasSuffix(shardsBaseURL, "/shards/") { + t.Fatalf("shards_base_url = %q", shardsBaseURL) + } + + w = serveShardRequest(h, "/conda-forge/noarch/"+strings.TrimPrefix(shardsBaseURL, "./")+masked+condaShardSuffix) + if w.Code != http.StatusOK { + t.Fatalf("shard status = %d: %s", w.Code, w.Body) + } + var shard shardRecords + decodeZstdMsgpack(t, w.Body.Bytes(), &shard) + want := []string{"testpkg-1.0.0-0.conda", "testpkg-1.0.0-0.tar.bz2"} + if got := shard.filenames(); strings.Join(got, ",") != strings.Join(want, ",") { + t.Fatalf("records = %v, want %v", got, want) + } + if len(shard.Removed) != 1 { + t.Errorf("removed = %v, want it kept", shard.Removed) + } + if _, ok := shard.Packages["testpkg-1.0.0-0.tar.bz2"]["sha256"].([]byte); !ok { + t.Errorf("sha256 is no longer binary: %T", shard.Packages["testpkg-1.0.0-0.tar.bz2"]["sha256"]) + } + + // The unfiltered shard stays out of reach while cooldown is on. + w = serveShardRequest(h, "/conda-forge/noarch/shards/"+f.shardHash+condaShardSuffix) + if w.Code != http.StatusNotFound { + t.Fatalf("plain shard status = %d, want 404", w.Code) + } +} + +func TestCondaCooldownShardRejectsBadPaths(t *testing.T) { + f := newShardUpstream(t, map[string]any{"base_url": "", "shards_base_url": ""}) + h := shardTestHandler(f, "3d") + hash := strings.Repeat("a", 64) + condaShardSuffix + for _, p := range []string{ + "/conda-forge/noarch/cooldown-shards/abc/" + hash, + "/conda-forge/noarch/cooldown-shards/1/other/" + hash, + "/conda-forge/noarch/cooldown-shards/1/" + strings.Repeat("A", 64) + condaShardSuffix, + "/conda-forge/noarch/cooldown-shards/1/x.msgpack.zst", + } { + if w := serveShardRequest(h, p); w.Code != http.StatusNotFound { + t.Errorf("%s: status = %d, want 404", p, w.Code) + } + } + if len(f.requests) != 0 { + t.Errorf("upstream was asked for %v", f.requests) + } +} + +func TestCondaShardIndexUnsupportedLayout(t *testing.T) { + for name, info := range map[string]map[string]any{ + "packages elsewhere": {"base_url": "https://cdn.example.com/conda-forge/noarch/", "shards_base_url": ""}, + "shards elsewhere": {"base_url": "", "shards_base_url": "https://cdn.example.com/shards/"}, + "nested shards": {"base_url": "", "shards_base_url": "./a/b/"}, + } { + t.Run(name, func(t *testing.T) { + f := newShardUpstream(t, info) + for _, cd := range []string{"", "3d"} { + w := serveShardRequest(shardTestHandler(f, cd), "/conda-forge/noarch/"+condaShardIndexFile) + if w.Code != http.StatusNotFound { + t.Errorf("cooldown %q: status = %d, want 404 so clients use repodata.json", cd, w.Code) + } + } + }) + } +} + +func TestCondaShardIndexUnknownVersion(t *testing.T) { + index := zstdMsgpack(t, map[string]any{"version": 2, "info": map[string]any{}, "shards": map[string]any{}}) + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write(index) + })) + defer upstream.Close() + + proxy := testProxy() + proxy.Cooldown = &cooldown.Config{Default: "3d"} + h := NewCondaHandlerWithUpstream(proxy, "http://proxy.local", upstream.URL).Routes() + if w := serveShardRequest(h, "/conda-forge/noarch/"+condaShardIndexFile); w.Code != http.StatusNotFound { + t.Fatalf("status = %d, want 404", w.Code) + } +} + +func TestCondaShardMaskDependsOnBucket(t *testing.T) { + a := condaShardMask("conda-forge", "noarch", "1") + if a == condaShardMask("conda-forge", "noarch", "2") || a == condaShardMask("conda-forge", "linux-64", "1") { + t.Fatal("mask must change with bucket and subdir") + } +} diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 022f3d8..2d83d19 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -1800,21 +1800,26 @@ func (p *Proxy) proxyCachedWithEncoding(w http.ResponseWriter, r *http.Request, body, contentType, contentEncoding, err := p.fetchOrCacheMetadata(r.Context(), ecosystem, cacheKey, upstreamURL, acceptEncoding, nil, acceptHeaders...) if err != nil { - if errors.Is(err, ErrUpstreamNotFound) { - http.Error(w, "not found", http.StatusNotFound) - return - } - if writeUpstreamHeld(w, err) { - return - } - p.Logger.Error("metadata fetch failed", "error", err) - http.Error(w, "failed to fetch from upstream", http.StatusBadGateway) + p.writeMetadataFetchError(w, err) return } p.writeMetadataCachedResponseWithEncoding(w, r, ecosystem, cacheKey, body, contentType, contentEncoding) } +// writeMetadataFetchError answers a failed metadata fetch. +func (p *Proxy) writeMetadataFetchError(w http.ResponseWriter, err error) { + if errors.Is(err, ErrUpstreamNotFound) { + http.Error(w, "not found", http.StatusNotFound) + return + } + if writeUpstreamHeld(w, err) { + return + } + p.Logger.Error("metadata fetch failed", "error", err) + http.Error(w, "failed to fetch from upstream", http.StatusBadGateway) +} + // writeMetadataCachedResponse writes a cached metadata response and handles // conditional request headers using metadata cache validators. func (p *Proxy) writeMetadataCachedResponse(w http.ResponseWriter, r *http.Request, ecosystem, cacheKey string, body []byte, contentType string) { From 4d74ad61136e4a3b5c526a49c1b3123251ab37af Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Fri, 23 Oct 2026 12:40:19 +0200 Subject: [PATCH 361/633] Apply cooldown to Go modules With cooldown on, @v/list leaves out versions whose .info time is inside the window, and @latest falls back to the newest release out of cooldown when upstream's answer is too new. Each listed version's .info is read once and its time is stored in the versions table, as npm does, except for modules whose path has uppercase letters, since golang PURLs are lowercased. A .info, .mod or .zip for a version in cooldown, and a query such as a branch name that resolves to one, gets 403. The go command moves on to the next GOPROXY entry after 404 or 410, so a 404 would let GOPROXY=proxy,direct fetch the version directly. Go has no publish time: .info carries the commit time of the revision, so a version tagged on an old commit counts as old. The docs say so. Part of #132 --- README.md | 4 +- docs/configuration.md | 4 +- internal/handler/go.go | 27 ++- internal/handler/go_cooldown.go | 252 +++++++++++++++++++++++++++ internal/handler/go_cooldown_test.go | 191 ++++++++++++++++++++ 5 files changed, 473 insertions(+), 5 deletions(-) create mode 100644 internal/handler/go_cooldown.go create mode 100644 internal/handler/go_cooldown_test.go diff --git a/README.md b/README.md index 0553e4a..9f54b74 100644 --- a/README.md +++ b/README.md @@ -47,7 +47,7 @@ The proxy never uploads artifact bytes to a scanner. Each scanner is notified wi | npm | JavaScript | Yes | ✓ | | Cargo | Rust | Yes | ✓ | | RubyGems | Ruby | Yes | ✓ | -| Go proxy | Go | | ✓ | +| Go proxy | Go | Yes† | ✓ | | Hex | Elixir | Yes* | ✓ | | pub.dev | Dart | Yes | ✓ | | PyPI | Python | Yes | ✓ | @@ -77,6 +77,8 @@ Cooldown requires publish timestamps in metadata. Registries without a "Yes" in \* Hex cooldown requires disabling registry signature verification (`HEX_NO_VERIFY_REPO_ORIGIN=1`) since the proxy re-encodes the protobuf payload. +† Go cooldown uses the time in each version's `.info`, which is the commit time of the tagged revision, not when the tag was pushed. + ## Install ```bash diff --git a/docs/configuration.md b/docs/configuration.md index 4da6f63..eb99cd9 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -980,7 +980,9 @@ Exact `packages` entries take precedence over patterns. When several patterns ma Resolution order: exact package override, then package pattern, then ecosystem override, then global default. This lets you set a conservative default while exempting trusted package families. -Currently supported for npm, PyPI, pub.dev, Composer, Cargo, NuGet, Conda, RubyGems, and Hex. These ecosystems include publish timestamps in their metadata. +Currently supported for npm, PyPI, pub.dev, Composer, Cargo, NuGet, Conda, RubyGems, Hex and Go. These ecosystems include publish timestamps in their metadata. + +Go has no publish time. The module proxy's `.info` gives the commit time of the revision a version points at, so a version tagged on an old commit counts as old. The proxy reads each listed version's `.info` once, stores the time, and leaves versions in cooldown out of `@v/list` and `@latest`. When upstream's `@latest` is in cooldown, it serves the newest release out of cooldown instead. Requests for a version in cooldown, including its `.info`, `.mod` and `.zip`, and a branch or commit query that resolves to one, get 403 rather than 404, because the go command moves on to the next `GOPROXY` entry, such as `direct`, after a 404. Note: Hex cooldown requires disabling registry signature verification since the proxy re-encodes the protobuf payload without the original signature. Set `HEX_NO_VERIFY_REPO_ORIGIN=1` or configure your repo with `no_verify: true`. diff --git a/internal/handler/go.go b/internal/handler/go.go index c8ae81a..85cf7e9 100644 --- a/internal/handler/go.go +++ b/internal/handler/go.go @@ -84,17 +84,28 @@ func (h *GoHandler) handleRequest(w http.ResponseWriter, r *http.Request) { rest := path[idx+4:] // after "/@v/" decodedMod := decodeGoModule(module) + cooldown := h.cooldownEnabled() switch { + case rest == "list" && cooldown: + h.serveCooldownList(w, r, module, decodedMod) + case rest == "list": // GET /{module}/@v/list - list versions h.proxyCached(w, r, decodedMod+"/@v/list") + case strings.HasSuffix(rest, ".info") && cooldown: + h.serveCooldownInfo(w, r, module, decodedMod, rest) + case strings.HasSuffix(rest, ".info"): // GET /{module}/@v/{version}.info - version metadata h.proxyCached(w, r, decodedMod+"/@v/"+rest) case strings.HasSuffix(rest, ".mod"): // GET /{module}/@v/{version}.mod - go.mod file + if version, err := gomodule.UnescapeVersion(strings.TrimSuffix(rest, ".mod")); err == nil && h.versionInCooldown(r.Context(), module, decodedMod, version) { + h.refuseCooldown(w, decodedMod, version) + return + } h.proxyCached(w, r, decodedMod+"/@v/"+rest) case strings.HasSuffix(rest, ".zip"): @@ -111,6 +122,10 @@ func (h *GoHandler) handleRequest(w http.ResponseWriter, r *http.Request) { // Check for @latest if strings.HasSuffix(path, "/@latest") { module := strings.TrimSuffix(path, "/@latest") + if h.cooldownEnabled() { + h.serveCooldownLatest(w, r, module, decodeGoModule(module)) + return + } h.proxyCached(w, r, decodeGoModule(module)+"/@latest") return } @@ -130,9 +145,15 @@ func (h *GoHandler) handleDownload(w http.ResponseWriter, r *http.Request, modul // The URL escapes uppercase letters in the version (v1.0.0-!r!c1 for // v1.0.0-RC1), and the zip is cached under that form, so the denylist // is checked against the version itself. - if unescaped, err := gomodule.UnescapeVersion(version); err == nil && h.proxy.versionDenied("golang", decodedModule, unescaped) { - http.Error(w, ErrVersionDenied.Error()+": "+canonicalVersionPURL("golang", decodedModule, unescaped), http.StatusForbidden) - return + if unescaped, err := gomodule.UnescapeVersion(version); err == nil { + if h.proxy.versionDenied("golang", decodedModule, unescaped) { + http.Error(w, ErrVersionDenied.Error()+": "+canonicalVersionPURL("golang", decodedModule, unescaped), http.StatusForbidden) + return + } + if h.versionInCooldown(r.Context(), module, decodedModule, unescaped) { + h.refuseCooldown(w, decodedModule, unescaped) + return + } } downloadURL := h.upstreamURL + r.URL.Path diff --git a/internal/handler/go_cooldown.go b/internal/handler/go_cooldown.go new file mode 100644 index 0000000..f73a897 --- /dev/null +++ b/internal/handler/go_cooldown.go @@ -0,0 +1,252 @@ +package handler + +import ( + "context" + "encoding/json" + "errors" + "net/http" + "strings" + "time" + + gomodule "golang.org/x/mod/module" + "golang.org/x/mod/semver" + "golang.org/x/sync/errgroup" +) + +// Go cooldown measures a version by the Time in its .info, which the module +// proxy takes from the commit the version points at. That is when the code +// was committed, not when the version was tagged or first fetched. +// +// Versions in cooldown are left out of @v/list and @latest, and their .info, +// .mod and .zip are answered with 403. The go command falls back to the next +// GOPROXY entry on 404 and 410, so with GOPROXY=proxy,direct a 404 would let +// it fetch the version directly. + +// goInfoFetchLimit bounds the .info requests made at once to filter a list. +const goInfoFetchLimit = 8 + +const errGoCooldown = "version is in cooldown" + +// goInfo is the JSON served for a version's .info and for @latest. +type goInfo struct { + Version string `json:"Version"` + Time time.Time `json:"Time"` +} + +func (h *GoHandler) cooldownEnabled() bool { + return h.proxy.Cooldown != nil && h.proxy.Cooldown.Enabled() +} + +func (h *GoHandler) allowedAt(module string, published time.Time) bool { + if published.IsZero() { + return true + } + return h.proxy.Cooldown.IsAllowed("golang", canonicalPackagePURL("golang", module), published) +} + +// writeGoMetadataError answers a failed upstream metadata fetch. +func (h *GoHandler) writeGoMetadataError(w http.ResponseWriter, err error) { + if errors.Is(err, ErrUpstreamNotFound) { + http.Error(w, "not found", http.StatusNotFound) + return + } + if writeUpstreamHeld(w, err) { + return + } + h.proxy.Logger.Error("go metadata fetch failed", "error", err) + http.Error(w, "failed to fetch from upstream", http.StatusBadGateway) +} + +func (h *GoHandler) fetchGoMetadata(ctx context.Context, escapedModule, module, rest string) ([]byte, error) { + body, _, err := h.proxy.FetchOrCacheMetadata(ctx, "golang", module+"/"+rest, h.upstreamURL+"/"+escapedModule+"/"+rest, "*/*") + return body, err +} + +// storesGoTimes reports whether publish times for module can be kept in the +// versions table. Its rows are keyed by PURL, which lowercases golang names, +// so modules that differ only in case would share them; such modules read +// their times from the metadata cache instead. +func (h *GoHandler) storesGoTimes(module string) bool { + return h.proxy.DB != nil && module == strings.ToLower(module) +} + +// publishTime returns a version's time from the versions table, or from its +// .info, which is then stored. A version whose time can't be read gets the +// zero time and is let through, as in the other ecosystems. +func (h *GoHandler) publishTime(ctx context.Context, escapedModule, module, version string) time.Time { + versionPURL := canonicalVersionPURL("golang", module, version) + if h.storesGoTimes(module) { + if ver, err := h.proxy.DB.GetVersionByPURL(versionPURL); err == nil && ver != nil && ver.PublishedAt.Valid { + return ver.PublishedAt.Time + } + } + + escapedVersion, err := gomodule.EscapeVersion(version) + if err != nil { + return time.Time{} + } + body, err := h.fetchGoMetadata(ctx, escapedModule, module, "@v/"+escapedVersion+".info") + if err != nil { + h.proxy.Logger.Warn("cooldown: could not fetch go version info", + "module", module, "version", version, "error", err) + return time.Time{} + } + var info goInfo + if err := json.Unmarshal(body, &info); err != nil || info.Time.IsZero() { + return time.Time{} + } + h.storePublishTime(module, version, info.Time) + return info.Time +} + +func (h *GoHandler) storePublishTime(module, version string, published time.Time) { + if !h.storesGoTimes(module) { + return + } + if err := h.proxy.DB.SetVersionPublishedAt(canonicalVersionPURL("golang", module, version), canonicalPackagePURL("golang", module), published); err != nil { + h.proxy.Logger.Warn("cooldown: could not store go publish time", + "module", module, "version", version, "error", err) + } +} + +// versionInCooldown reports whether an exact version is still in cooldown. +func (h *GoHandler) versionInCooldown(ctx context.Context, escapedModule, module, version string) bool { + if !h.cooldownEnabled() { + return false + } + return !h.allowedAt(module, h.publishTime(ctx, escapedModule, module, version)) +} + +func (h *GoHandler) refuseCooldown(w http.ResponseWriter, module, version string) { + h.proxy.Logger.Info("cooldown: withholding go module version", "module", module, "version", version) + http.Error(w, errGoCooldown+": "+canonicalVersionPURL("golang", module, version), http.StatusForbidden) +} + +// allowedVersions returns the listed versions of module that are out of +// cooldown, in their upstream order. +func (h *GoHandler) allowedVersions(ctx context.Context, escapedModule, module string) ([]string, error) { + body, err := h.fetchGoMetadata(ctx, escapedModule, module, "@v/list") + if err != nil { + return nil, err + } + var versions []string + for line := range strings.SplitSeq(string(body), "\n") { + if v := strings.TrimSpace(line); v != "" { + versions = append(versions, v) + } + } + + keep := make([]bool, len(versions)) + g, gctx := errgroup.WithContext(ctx) + g.SetLimit(goInfoFetchLimit) + for i, v := range versions { + g.Go(func() error { + keep[i] = h.allowedAt(module, h.publishTime(gctx, escapedModule, module, v)) + return nil + }) + } + _ = g.Wait() + + allowed := versions[:0] + for i, v := range versions { + if keep[i] { + allowed = append(allowed, v) + } else { + h.proxy.Logger.Info("cooldown: filtering go module version", "module", module, "version", v) + } + } + return allowed, nil +} + +// serveCooldownList serves @v/list without versions in cooldown. +func (h *GoHandler) serveCooldownList(w http.ResponseWriter, r *http.Request, escapedModule, module string) { + allowed, err := h.allowedVersions(r.Context(), escapedModule, module) + if err != nil { + h.writeGoMetadataError(w, err) + return + } + w.Header().Set(headerContentType, "text/plain; charset=utf-8") + if len(allowed) > 0 { + _, _ = w.Write([]byte(strings.Join(allowed, "\n") + "\n")) + } +} + +// serveCooldownInfo serves a .info, refusing it when the version it names is +// in cooldown. rest is the request after /@v/: an escaped version or a query +// such as a branch name, followed by .info. +func (h *GoHandler) serveCooldownInfo(w http.ResponseWriter, r *http.Request, escapedModule, module, rest string) { + body, err := h.fetchGoMetadata(r.Context(), escapedModule, module, "@v/"+rest) + if err != nil { + h.writeGoMetadataError(w, err) + return + } + var info goInfo + if err := json.Unmarshal(body, &info); err == nil && info.Version != "" { + if !h.allowedAt(module, info.Time) { + h.refuseCooldown(w, module, info.Version) + return + } + if !info.Time.IsZero() && semver.IsValid(info.Version) { + h.storePublishTime(module, info.Version, info.Time) + } + } + w.Header().Set(headerContentType, contentTypeJSON) + _, _ = w.Write(body) +} + +// serveCooldownLatest serves @latest. When upstream's answer is in cooldown, +// it serves the newest listed version out of cooldown instead: the highest +// release, or else the highest pre-release, which is how the go command +// picks from a list. +func (h *GoHandler) serveCooldownLatest(w http.ResponseWriter, r *http.Request, escapedModule, module string) { + body, err := h.fetchGoMetadata(r.Context(), escapedModule, module, "@latest") + if err != nil { + h.writeGoMetadataError(w, err) + return + } + var info goInfo + if err := json.Unmarshal(body, &info); err != nil || h.allowedAt(module, info.Time) { + w.Header().Set(headerContentType, contentTypeJSON) + _, _ = w.Write(body) + return + } + + allowed, err := h.allowedVersions(r.Context(), escapedModule, module) + if err != nil { + h.writeGoMetadataError(w, err) + return + } + best := newestGoVersion(allowed) + if best == "" { + h.refuseCooldown(w, module, info.Version) + return + } + escapedVersion, err := gomodule.EscapeVersion(best) + if err != nil { + h.refuseCooldown(w, module, info.Version) + return + } + h.serveCooldownInfo(w, r, escapedModule, module, escapedVersion+".info") +} + +// newestGoVersion returns the highest release in versions, or the highest +// pre-release when there is no release. +func newestGoVersion(versions []string) string { + var release, prerelease string + for _, v := range versions { + if !semver.IsValid(v) { + continue + } + if semver.Prerelease(v) == "" { + if release == "" || semver.Compare(v, release) > 0 { + release = v + } + } else if prerelease == "" || semver.Compare(v, prerelease) > 0 { + prerelease = v + } + } + if release != "" { + return release + } + return prerelease +} diff --git a/internal/handler/go_cooldown_test.go b/internal/handler/go_cooldown_test.go new file mode 100644 index 0000000..c528195 --- /dev/null +++ b/internal/handler/go_cooldown_test.go @@ -0,0 +1,191 @@ +package handler + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + "time" + + "github.com/git-pkgs/cooldown" +) + +type goCooldownUpstream struct { + server *httptest.Server + mu sync.Mutex + hits map[string]int +} + +func (u *goCooldownUpstream) count(path string) int { + u.mu.Lock() + defer u.mu.Unlock() + return u.hits[path] +} + +// newGoCooldownUpstream serves example.com/mod with an old v1.0.0, a +// recent v1.1.0 and a recent v1.2.0-rc.1, and @latest pointing at v1.1.0. +func newGoCooldownUpstream(t *testing.T, latest string) *goCooldownUpstream { + t.Helper() + now := time.Now() + times := map[string]time.Time{ + "v1.0.0": now.Add(-10 * 24 * time.Hour), + "v1.1.0": now.Add(-time.Hour), + "v1.2.0-rc.1": now.Add(-2 * time.Hour), + } + u := &goCooldownUpstream{hits: map[string]int{}} + u.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + u.mu.Lock() + u.hits[r.URL.Path]++ + u.mu.Unlock() + rest, ok := strings.CutPrefix(r.URL.Path, "/example.com/mod/") + if !ok { + http.NotFound(w, r) + return + } + writeInfo := func(v string) { + _ = json.NewEncoder(w).Encode(goInfo{Version: v, Time: times[v]}) + } + switch { + case rest == "@v/list": + _, _ = w.Write([]byte("v1.0.0\nv1.1.0\nv1.2.0-rc.1\n")) + case rest == "@latest": + writeInfo(latest) + case rest == "@v/main.info": + writeInfo("v1.1.0") + case strings.HasSuffix(rest, ".info"): + v := strings.TrimSuffix(strings.TrimPrefix(rest, "@v/"), ".info") + if _, ok := times[v]; !ok { + http.NotFound(w, r) + return + } + writeInfo(v) + case strings.HasSuffix(rest, ".mod"): + _, _ = w.Write([]byte("module example.com/mod\n")) + default: + http.NotFound(w, r) + } + })) + t.Cleanup(u.server.Close) + return u +} + +func goCooldownHandler(t *testing.T, u *goCooldownUpstream) (http.Handler, *mockFetcher, *Proxy) { + t.Helper() + proxy, _, _, fetcher := setupTestProxy(t) + proxy.Cooldown = &cooldown.Config{Default: "3d"} + return NewGoHandlerWithUpstream(proxy, "http://localhost:8080", u.server.URL).Routes(), fetcher, proxy +} + +func goGet(h http.Handler, path string) *httptest.ResponseRecorder { + w := httptest.NewRecorder() + h.ServeHTTP(w, httptest.NewRequest(http.MethodGet, path, nil)) + return w +} + +func TestGoCooldownFiltersList(t *testing.T) { + u := newGoCooldownUpstream(t, "v1.1.0") + h, _, _ := goCooldownHandler(t, u) + + w := goGet(h, "/example.com/mod/@v/list") + if w.Code != http.StatusOK { + t.Fatalf("status = %d", w.Code) + } + if got := w.Body.String(); got != "v1.0.0\n" { + t.Fatalf("list = %q, want only v1.0.0", got) + } + + // Publish times are stored, so a second list needs no .info requests. + _ = goGet(h, "/example.com/mod/@v/list") + if n := u.count("/example.com/mod/@v/v1.0.0.info"); n != 1 { + t.Errorf("v1.0.0.info fetched %d times, want 1", n) + } +} + +func TestGoCooldownRefusesVersions(t *testing.T) { + u := newGoCooldownUpstream(t, "v1.1.0") + h, fetcher, _ := goCooldownHandler(t, u) + + for _, p := range []string{ + "/example.com/mod/@v/v1.1.0.info", + "/example.com/mod/@v/v1.1.0.mod", + "/example.com/mod/@v/v1.1.0.zip", + "/example.com/mod/@v/main.info", + } { + w := goGet(h, p) + if w.Code != http.StatusForbidden || !strings.Contains(w.Body.String(), errGoCooldown) { + t.Errorf("%s: status = %d body %q, want 403 cooldown", p, w.Code, w.Body.String()) + } + } + if fetcher.fetchCalled { + t.Error("zip in cooldown was fetched") + } + + for _, p := range []string{"/example.com/mod/@v/v1.0.0.info", "/example.com/mod/@v/v1.0.0.mod"} { + if w := goGet(h, p); w.Code != http.StatusOK { + t.Errorf("%s: status = %d, want 200", p, w.Code) + } + } +} + +func TestGoCooldownLatest(t *testing.T) { + u := newGoCooldownUpstream(t, "v1.1.0") + h, _, _ := goCooldownHandler(t, u) + + w := goGet(h, "/example.com/mod/@latest") + if w.Code != http.StatusOK { + t.Fatalf("status = %d: %s", w.Code, w.Body) + } + var info goInfo + if err := json.Unmarshal(w.Body.Bytes(), &info); err != nil { + t.Fatal(err) + } + if info.Version != "v1.0.0" { + t.Fatalf("@latest = %s, want v1.0.0", info.Version) + } +} + +func TestGoCooldownLatestAllowed(t *testing.T) { + u := newGoCooldownUpstream(t, "v1.0.0") + h, _, _ := goCooldownHandler(t, u) + + w := goGet(h, "/example.com/mod/@latest") + var info goInfo + if err := json.Unmarshal(w.Body.Bytes(), &info); err != nil || info.Version != "v1.0.0" { + t.Fatalf("@latest = %d %s, want v1.0.0", w.Code, w.Body) + } + if n := u.count("/example.com/mod/@v/list"); n != 0 { + t.Errorf("list fetched %d times for an allowed @latest", n) + } +} + +func TestGoCooldownDisabledPassesThrough(t *testing.T) { + u := newGoCooldownUpstream(t, "v1.1.0") + h, _, proxy := goCooldownHandler(t, u) + proxy.Cooldown = nil + + if w := goGet(h, "/example.com/mod/@v/list"); w.Body.String() != "v1.0.0\nv1.1.0\nv1.2.0-rc.1\n" { + t.Fatalf("list = %q", w.Body.String()) + } + if w := goGet(h, "/example.com/mod/@v/v1.1.0.info"); w.Code != http.StatusOK { + t.Fatalf("info status = %d", w.Code) + } +} + +func TestNewestGoVersion(t *testing.T) { + tests := []struct { + versions []string + want string + }{ + {[]string{"v1.0.0", "v1.10.0", "v1.9.0", "v2.0.0-rc.1"}, "v1.10.0"}, + {[]string{"v1.0.0-alpha", "v1.0.0-beta"}, "v1.0.0-beta"}, + {[]string{"v2.0.0+incompatible", "v1.5.0"}, "v2.0.0+incompatible"}, + {nil, ""}, + } + for _, tt := range tests { + if got := newestGoVersion(tt.versions); got != tt.want { + t.Errorf("newestGoVersion(%v) = %q, want %q", tt.versions, got, tt.want) + } + } +} From a7879137277536718ca3d5258865a717e82ac357 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Fri, 23 Oct 2026 12:46:39 +0200 Subject: [PATCH 362/633] Fail closed in Go cooldown and key stored times by escaped path A failed .info fetch counted as an unknown time, so a rate-limited or flaky upstream let versions in cooldown through. It now fails the list or download with 502; only a version upstream doesn't know, or whose .info has no time, is let through. Times are stored under the escaped module path, which keeps modules that differ only in case apart. Modules with uppercase letters were not stored before and fetched every .info again on each list request. Only canonical versions are stored, since a query such as a branch name can resolve to a later commit. A .mod or .zip whose version doesn't unescape is answered with 404 instead of being passed upstream unchecked. --- docs/configuration.md | 2 +- internal/handler/go.go | 29 +++++---- internal/handler/go_cooldown.go | 89 +++++++++++++++++++--------- internal/handler/go_cooldown_test.go | 69 +++++++++++++++++++++ 4 files changed, 149 insertions(+), 40 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index eb99cd9..d13ee87 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -982,7 +982,7 @@ Resolution order: exact package override, then package pattern, then ecosystem o Currently supported for npm, PyPI, pub.dev, Composer, Cargo, NuGet, Conda, RubyGems, Hex and Go. These ecosystems include publish timestamps in their metadata. -Go has no publish time. The module proxy's `.info` gives the commit time of the revision a version points at, so a version tagged on an old commit counts as old. The proxy reads each listed version's `.info` once, stores the time, and leaves versions in cooldown out of `@v/list` and `@latest`. When upstream's `@latest` is in cooldown, it serves the newest release out of cooldown instead. Requests for a version in cooldown, including its `.info`, `.mod` and `.zip`, and a branch or commit query that resolves to one, get 403 rather than 404, because the go command moves on to the next `GOPROXY` entry, such as `direct`, after a 404. +Go has no publish time. The module proxy's `.info` gives the commit time of the revision a version points at, so a version tagged on an old commit counts as old. The proxy reads each listed version's `.info` once, stores the time, and leaves versions in cooldown out of `@v/list` and `@latest`. When upstream's `@latest` is in cooldown, it serves the newest release out of cooldown instead. Requests for a version in cooldown, including its `.info`, `.mod` and `.zip`, and a branch or commit query that resolves to one, get 403 rather than 404, because the go command moves on to the next `GOPROXY` entry, such as `direct`, after a 404. If a version's `.info` can't be fetched, the list or download fails with 502 instead of letting the version through. Note: Hex cooldown requires disabling registry signature verification since the proxy re-encodes the protobuf payload without the original signature. Set `HEX_NO_VERIFY_REPO_ORIGIN=1` or configure your repo with `no_verify: true`. diff --git a/internal/handler/go.go b/internal/handler/go.go index 85cf7e9..e544cd7 100644 --- a/internal/handler/go.go +++ b/internal/handler/go.go @@ -102,8 +102,12 @@ func (h *GoHandler) handleRequest(w http.ResponseWriter, r *http.Request) { case strings.HasSuffix(rest, ".mod"): // GET /{module}/@v/{version}.mod - go.mod file - if version, err := gomodule.UnescapeVersion(strings.TrimSuffix(rest, ".mod")); err == nil && h.versionInCooldown(r.Context(), module, decodedMod, version) { - h.refuseCooldown(w, decodedMod, version) + version, err := gomodule.UnescapeVersion(strings.TrimSuffix(rest, ".mod")) + if err != nil { + http.NotFound(w, r) + return + } + if h.checkCooldown(w, r, module, decodedMod, version) { return } h.proxyCached(w, r, decodedMod+"/@v/"+rest) @@ -145,15 +149,18 @@ func (h *GoHandler) handleDownload(w http.ResponseWriter, r *http.Request, modul // The URL escapes uppercase letters in the version (v1.0.0-!r!c1 for // v1.0.0-RC1), and the zip is cached under that form, so the denylist // is checked against the version itself. - if unescaped, err := gomodule.UnescapeVersion(version); err == nil { - if h.proxy.versionDenied("golang", decodedModule, unescaped) { - http.Error(w, ErrVersionDenied.Error()+": "+canonicalVersionPURL("golang", decodedModule, unescaped), http.StatusForbidden) - return - } - if h.versionInCooldown(r.Context(), module, decodedModule, unescaped) { - h.refuseCooldown(w, decodedModule, unescaped) - return - } + // A version that doesn't unescape can't be checked, so it isn't fetched. + unescaped, err := gomodule.UnescapeVersion(version) + if err != nil { + http.NotFound(w, r) + return + } + if h.proxy.versionDenied("golang", decodedModule, unescaped) { + http.Error(w, ErrVersionDenied.Error()+": "+canonicalVersionPURL("golang", decodedModule, unescaped), http.StatusForbidden) + return + } + if h.checkCooldown(w, r, module, decodedModule, unescaped) { + return } downloadURL := h.upstreamURL + r.URL.Path diff --git a/internal/handler/go_cooldown.go b/internal/handler/go_cooldown.go index f73a897..c3aa0cb 100644 --- a/internal/handler/go_cooldown.go +++ b/internal/handler/go_cooldown.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "errors" + "fmt" "net/http" "strings" "time" @@ -62,59 +63,85 @@ func (h *GoHandler) fetchGoMetadata(ctx context.Context, escapedModule, module, return body, err } -// storesGoTimes reports whether publish times for module can be kept in the -// versions table. Its rows are keyed by PURL, which lowercases golang names, -// so modules that differ only in case would share them; such modules read -// their times from the metadata cache instead. -func (h *GoHandler) storesGoTimes(module string) bool { - return h.proxy.DB != nil && module == strings.ToLower(module) +// goTimeKey returns the versions-table PURLs a module's times are stored +// under. golang PURLs lowercase the name, so the key uses the escaped module +// path (github.com/!azure for github.com/Azure), which keeps modules that +// differ only in case apart and is the plain path for lowercase modules. +// Only canonical versions are stored: a query such as a branch name can +// resolve to another commit later. +func (h *GoHandler) goTimeKey(module, version string) (versionPURL, packagePURL string, ok bool) { + if h.proxy.DB == nil || gomodule.CanonicalVersion(version) != version { + return "", "", false + } + escaped, err := gomodule.EscapePath(module) + if err != nil { + return "", "", false + } + return canonicalVersionPURL("golang", escaped, version), canonicalPackagePURL("golang", escaped), true } // publishTime returns a version's time from the versions table, or from its -// .info, which is then stored. A version whose time can't be read gets the -// zero time and is let through, as in the other ecosystems. -func (h *GoHandler) publishTime(ctx context.Context, escapedModule, module, version string) time.Time { - versionPURL := canonicalVersionPURL("golang", module, version) - if h.storesGoTimes(module) { +// .info, which is then stored. A version upstream doesn't know, or whose +// .info has no time, gets the zero time and is let through, as in the other +// ecosystems. A failed fetch is an error, so a flaky or rate-limited +// upstream can't let a version through. +func (h *GoHandler) publishTime(ctx context.Context, escapedModule, module, version string) (time.Time, error) { + if versionPURL, _, ok := h.goTimeKey(module, version); ok { if ver, err := h.proxy.DB.GetVersionByPURL(versionPURL); err == nil && ver != nil && ver.PublishedAt.Valid { - return ver.PublishedAt.Time + return ver.PublishedAt.Time, nil } } escapedVersion, err := gomodule.EscapeVersion(version) if err != nil { - return time.Time{} + return time.Time{}, nil } body, err := h.fetchGoMetadata(ctx, escapedModule, module, "@v/"+escapedVersion+".info") + if errors.Is(err, ErrUpstreamNotFound) { + return time.Time{}, nil + } if err != nil { - h.proxy.Logger.Warn("cooldown: could not fetch go version info", - "module", module, "version", version, "error", err) - return time.Time{} + return time.Time{}, err } var info goInfo - if err := json.Unmarshal(body, &info); err != nil || info.Time.IsZero() { - return time.Time{} + if err := json.Unmarshal(body, &info); err != nil { + return time.Time{}, err } - h.storePublishTime(module, version, info.Time) - return info.Time + if !info.Time.IsZero() { + h.storePublishTime(module, version, info.Time) + } + return info.Time, nil } func (h *GoHandler) storePublishTime(module, version string, published time.Time) { - if !h.storesGoTimes(module) { + versionPURL, packagePURL, ok := h.goTimeKey(module, version) + if !ok { return } - if err := h.proxy.DB.SetVersionPublishedAt(canonicalVersionPURL("golang", module, version), canonicalPackagePURL("golang", module), published); err != nil { + if err := h.proxy.DB.SetVersionPublishedAt(versionPURL, packagePURL, published); err != nil { h.proxy.Logger.Warn("cooldown: could not store go publish time", "module", module, "version", version, "error", err) } } -// versionInCooldown reports whether an exact version is still in cooldown. -func (h *GoHandler) versionInCooldown(ctx context.Context, escapedModule, module, version string) bool { +// checkCooldown answers a request for an exact version in cooldown, or whose +// time couldn't be read, and reports whether it did. +func (h *GoHandler) checkCooldown(w http.ResponseWriter, r *http.Request, escapedModule, module, version string) bool { if !h.cooldownEnabled() { return false } - return !h.allowedAt(module, h.publishTime(ctx, escapedModule, module, version)) + published, err := h.publishTime(r.Context(), escapedModule, module, version) + if err != nil { + h.proxy.Logger.Warn("cooldown: could not read go version time", + "module", module, "version", version, "error", err) + http.Error(w, "failed to check cooldown", http.StatusBadGateway) + return true + } + if !h.allowedAt(module, published) { + h.refuseCooldown(w, module, version) + return true + } + return false } func (h *GoHandler) refuseCooldown(w http.ResponseWriter, module, version string) { @@ -141,11 +168,17 @@ func (h *GoHandler) allowedVersions(ctx context.Context, escapedModule, module s g.SetLimit(goInfoFetchLimit) for i, v := range versions { g.Go(func() error { - keep[i] = h.allowedAt(module, h.publishTime(gctx, escapedModule, module, v)) + published, err := h.publishTime(gctx, escapedModule, module, v) + if err != nil { + return fmt.Errorf("reading time of %s: %w", v, err) + } + keep[i] = h.allowedAt(module, published) return nil }) } - _ = g.Wait() + if err := g.Wait(); err != nil { + return nil, err + } allowed := versions[:0] for i, v := range versions { @@ -186,7 +219,7 @@ func (h *GoHandler) serveCooldownInfo(w http.ResponseWriter, r *http.Request, es h.refuseCooldown(w, module, info.Version) return } - if !info.Time.IsZero() && semver.IsValid(info.Version) { + if !info.Time.IsZero() { h.storePublishTime(module, info.Version, info.Time) } } diff --git a/internal/handler/go_cooldown_test.go b/internal/handler/go_cooldown_test.go index c528195..bedc334 100644 --- a/internal/handler/go_cooldown_test.go +++ b/internal/handler/go_cooldown_test.go @@ -189,3 +189,72 @@ func TestNewestGoVersion(t *testing.T) { } } } + +func TestGoCooldownFailsClosedOnUpstreamErrors(t *testing.T) { + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case strings.HasSuffix(r.URL.Path, "/@v/list"): + _, _ = w.Write([]byte("v1.0.0\n")) + case strings.HasSuffix(r.URL.Path, ".info"): + http.Error(w, "slow down", http.StatusTooManyRequests) + default: + _, _ = w.Write([]byte("module example.com/mod\n")) + } + })) + defer upstream.Close() + + proxy, _, _, fetcher := setupTestProxy(t) + proxy.Cooldown = &cooldown.Config{Default: "3d"} + h := NewGoHandlerWithUpstream(proxy, "http://localhost:8080", upstream.URL).Routes() + + for _, p := range []string{ + "/example.com/mod/@v/list", + "/example.com/mod/@v/v1.0.0.mod", + "/example.com/mod/@v/v1.0.0.zip", + } { + if w := goGet(h, p); w.Code != http.StatusBadGateway { + t.Errorf("%s: status = %d, want 502", p, w.Code) + } + } + if fetcher.fetchCalled { + t.Error("zip was fetched without a known time") + } +} + +func TestGoCooldownStoresTimesPerExactModule(t *testing.T) { + u := newGoCooldownUpstream(t, "v1.1.0") + proxy, db, _, _ := setupTestProxy(t) + proxy.Cooldown = &cooldown.Config{Default: "3d"} + h := NewGoHandlerWithUpstream(proxy, "http://localhost:8080", u.server.URL).Routes() + + _ = goGet(h, "/example.com/mod/@v/list") + ver, err := db.GetVersionByPURL(canonicalVersionPURL("golang", "example.com/mod", "v1.0.0")) + if err != nil || ver == nil || !ver.PublishedAt.Valid { + t.Fatalf("time of v1.0.0 not stored: %v %v", ver, err) + } + + gh := &GoHandler{proxy: proxy} + upper, _, ok := gh.goTimeKey("github.com/Azure/sdk", "v1.0.0") + lower, _, _ := gh.goTimeKey("github.com/azure/sdk", "v1.0.0") + if !ok || upper == lower { + t.Errorf("modules differing in case share a key: %q %q", upper, lower) + } + for _, query := range []string{"master", "v1.2", "v1.0.0+meta"} { + if _, _, ok := gh.goTimeKey("example.com/mod", query); ok { + t.Errorf("time of non-canonical %q would be stored", query) + } + } +} + +func TestGoRejectsBadlyEscapedVersions(t *testing.T) { + u := newGoCooldownUpstream(t, "v1.1.0") + h, fetcher, _ := goCooldownHandler(t, u) + for _, p := range []string{"/example.com/mod/@v/V1.0.0.zip", "/example.com/mod/@v/V1.0.0.mod"} { + if w := goGet(h, p); w.Code != http.StatusNotFound { + t.Errorf("%s: status = %d, want 404", p, w.Code) + } + } + if fetcher.fetchCalled || u.count("/example.com/mod/@v/V1.0.0.mod") != 0 { + t.Error("badly escaped version reached upstream") + } +} From 5b1c53b24d98cbdec244c1d251399d63f948ebce Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Fri, 23 Oct 2026 13:09:31 +0200 Subject: [PATCH 363/633] Refresh the Julia name map only from the advertised registry Each request for a General registry tarball whose hash differed from the loaded name map started a goroutine that downloaded the whole registry from upstream again, with no limit on concurrent loads and no deadline. Alternating between two valid snapshots made every request cost a full download and parse, and left the map at whichever snapshot came last, older ones included. The refresh now only loads the hash /registries currently lists, read through the metadata cache, runs one load at a time and skips while one is running, and stops after five minutes. Registry.toml is read up to 64 MB. --- internal/handler/julia.go | 69 ++++++++----- internal/handler/julia_refresh_test.go | 135 +++++++++++++++++++++++++ 2 files changed, 179 insertions(+), 25 deletions(-) create mode 100644 internal/handler/julia_refresh_test.go diff --git a/internal/handler/julia.go b/internal/handler/julia.go index 749335b..aaac731 100644 --- a/internal/handler/julia.go +++ b/internal/handler/julia.go @@ -12,6 +12,7 @@ import ( "regexp" "strings" "sync" + "time" "github.com/BurntSushi/toml" "github.com/git-pkgs/proxy/internal/retention" @@ -22,6 +23,11 @@ const ( juliaGeneralRegistryUUID = "23338594-aafe-5451-b93e-139f81909106" juliaArtifactName = "_artifact" juliaRegistryName = "_registry" + + // juliaRefreshTimeout bounds a background reload of the name map. + juliaRefreshTimeout = 5 * time.Minute + // maxJuliaRegistryTomlSize bounds Registry.toml. General's is a few MB. + maxJuliaRegistryTomlSize = 64 << 20 ) var ( @@ -110,7 +116,7 @@ func (h *JuliaHandler) handleRegistry(w http.ResponseWriter, r *http.Request) { return } - go h.refreshNamesFromRegistry(uuid, hash) + go h.refreshNamesFromRegistry(context.WithoutCancel(r.Context()), uuid, hash) ServeArtifactRequest(w, r, result) } @@ -211,24 +217,16 @@ func (h *JuliaHandler) loadNamesFromUpstream(ctx context.Context) error { return h.loadRegistryTarball(ctx, juliaGeneralRegistryUUID, hash) } -// fetchGeneralRegistryHash reads /registries and returns the current tree hash -// for the General registry. +// fetchGeneralRegistryHash reads /registries, through the metadata cache that +// also serves it to clients, and returns the current tree hash for the +// General registry. func (h *JuliaHandler) fetchGeneralRegistryHash(ctx context.Context) (string, error) { - req, err := http.NewRequestWithContext(ctx, http.MethodGet, h.upstreamURL+"/registries", nil) + body, _, err := h.proxy.FetchOrCacheMetadata(ctx, "julia", "registries", h.upstreamURL+"/registries", "*/*") if err != nil { - return "", err - } - resp, err := h.proxy.HTTPClient.Do(req) - if err != nil { - return "", err - } - defer func() { _ = resp.Body.Close() }() - - if resp.StatusCode != http.StatusOK { - return "", fmt.Errorf("upstream /registries returned %d", resp.StatusCode) + return "", fmt.Errorf("reading /registries: %w", err) } - scanner := bufio.NewScanner(resp.Body) + scanner := bufio.NewScanner(bytes.NewReader(body)) for scanner.Scan() { uuid, hash, ok := parseRegistryLine(scanner.Text()) if ok && uuid == juliaGeneralRegistryUUID { @@ -241,23 +239,41 @@ func (h *JuliaHandler) fetchGeneralRegistryHash(ctx context.Context) (string, er return "", fmt.Errorf("general registry not listed in /registries") } -// refreshNamesFromRegistry reloads the UUID→name map from a registry tarball -// that has just been cached. Errors are logged but do not affect the response. -func (h *JuliaHandler) refreshNamesFromRegistry(uuid, hash string) { - if uuid != juliaGeneralRegistryUUID { +// refreshNamesFromRegistry reloads the UUID→name map after a General registry +// tarball has been served. Only the hash /registries currently lists replaces +// the map, so requests for older snapshots neither roll it back nor trigger a +// download each. One reload runs at a time; a request that finds one running +// skips its own. Errors are logged but do not affect the response. +func (h *JuliaHandler) refreshNamesFromRegistry(ctx context.Context, uuid, hash string) { + if uuid != juliaGeneralRegistryUUID || h.currentNamesHash() == hash { return } - h.mu.RLock() - current := h.namesHash - h.mu.RUnlock() - if current == hash { + if !h.loadMu.TryLock() { return } - if err := h.loadRegistryTarball(context.Background(), uuid, hash); err != nil { + defer h.loadMu.Unlock() + + ctx, cancel := context.WithTimeout(ctx, juliaRefreshTimeout) + defer cancel() + advertised, err := h.fetchGeneralRegistryHash(ctx) + if err != nil { + h.proxy.Logger.Warn("failed to refresh julia name map", "error", err) + return + } + if advertised != hash || h.currentNamesHash() == hash { + return + } + if err := h.loadRegistryTarball(ctx, uuid, hash); err != nil { h.proxy.Logger.Warn("failed to refresh julia name map", "error", err) } } +func (h *JuliaHandler) currentNamesHash() string { + h.mu.RLock() + defer h.mu.RUnlock() + return h.namesHash +} + // loadRegistryTarball downloads a registry tarball and replaces the name map // with the contents of its Registry.toml. func (h *JuliaHandler) loadRegistryTarball(ctx context.Context, uuid, hash string) error { @@ -312,10 +328,13 @@ func extractRegistryNames(r io.Reader) (map[string]string, error) { continue } - data, err := io.ReadAll(tr) + data, err := io.ReadAll(io.LimitReader(tr, maxJuliaRegistryTomlSize+1)) if err != nil { return nil, err } + if len(data) > maxJuliaRegistryTomlSize { + return nil, fmt.Errorf("registry file Registry.toml exceeds %d bytes", maxJuliaRegistryTomlSize) + } return parseRegistryToml(data) } } diff --git a/internal/handler/julia_refresh_test.go b/internal/handler/julia_refresh_test.go new file mode 100644 index 0000000..4ee080b --- /dev/null +++ b/internal/handler/julia_refresh_test.go @@ -0,0 +1,135 @@ +package handler + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "context" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + "time" +) + +func juliaRegistryTarball(t *testing.T, packageName string) []byte { + t.Helper() + body := "name = \"General\"\n[packages]\n295af30f-e4ad-537b-8983-00126c2a3abe = { name = \"" + packageName + "\", path = \"R/X\" }\n" + var buf bytes.Buffer + gw := gzip.NewWriter(&buf) + tw := tar.NewWriter(gw) + if err := tw.WriteHeader(&tar.Header{Name: "Registry.toml", Mode: 0o644, Size: int64(len(body))}); err != nil { + t.Fatal(err) + } + _, _ = tw.Write([]byte(body)) + _ = tw.Close() + _ = gw.Close() + return buf.Bytes() +} + +type fakeJuliaUpstream struct { + mu sync.Mutex + advertised string + tarballs map[string][]byte + downloads map[string]int +} + +func (u *fakeJuliaUpstream) ServeHTTP(w http.ResponseWriter, r *http.Request) { + u.mu.Lock() + defer u.mu.Unlock() + if r.URL.Path == "/registries" { + _, _ = w.Write([]byte("/registry/" + juliaGeneralRegistryUUID + "/" + u.advertised + "\n")) + return + } + hash := r.URL.Path[strings.LastIndex(r.URL.Path, "/")+1:] + body, ok := u.tarballs[hash] + if !ok { + http.NotFound(w, r) + return + } + u.downloads[hash]++ + _, _ = w.Write(body) +} + +func (u *fakeJuliaUpstream) count(hash string) int { + u.mu.Lock() + defer u.mu.Unlock() + return u.downloads[hash] +} + +func TestJuliaRefreshOnlyLoadsAdvertisedRegistry(t *testing.T) { + oldHash, newHash := strings.Repeat("a", 40), strings.Repeat("b", 40) + up := &fakeJuliaUpstream{ + advertised: newHash, + tarballs: map[string][]byte{ + oldHash: juliaRegistryTarball(t, "OldName"), + newHash: juliaRegistryTarball(t, "NewName"), + }, + downloads: map[string]int{}, + } + server := httptest.NewServer(up) + defer server.Close() + + proxy, _, _, _ := setupTestProxy(t) + h := NewJuliaHandlerWithUpstream(proxy, server.URL) + ctx := context.Background() + + // An older snapshot neither replaces the map nor is downloaded for it. + for range 5 { + h.refreshNamesFromRegistry(ctx, juliaGeneralRegistryUUID, oldHash) + } + if n := up.count(oldHash); n != 0 { + t.Fatalf("old registry downloaded %d times for the name map", n) + } + + h.refreshNamesFromRegistry(ctx, juliaGeneralRegistryUUID, newHash) + if got := h.resolveName("295af30f-e4ad-537b-8983-00126c2a3abe"); got != "NewName" { + t.Fatalf("name = %q, want NewName", got) + } + + // The loaded hash is not fetched again, and an old one doesn't roll back. + h.refreshNamesFromRegistry(ctx, juliaGeneralRegistryUUID, newHash) + h.refreshNamesFromRegistry(ctx, juliaGeneralRegistryUUID, oldHash) + if n := up.count(newHash); n != 1 { + t.Errorf("current registry downloaded %d times, want 1", n) + } + if got := h.resolveName("295af30f-e4ad-537b-8983-00126c2a3abe"); got != "NewName" { + t.Errorf("name rolled back to %q", got) + } +} + +func TestJuliaRefreshSkipsWhileLoading(t *testing.T) { + proxy, _, _, _ := setupTestProxy(t) + h := NewJuliaHandlerWithUpstream(proxy, "http://127.0.0.1:1") + h.loadMu.Lock() + done := make(chan struct{}) + go func() { + h.refreshNamesFromRegistry(context.Background(), juliaGeneralRegistryUUID, strings.Repeat("c", 40)) + close(done) + }() + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatal("refresh waited for a running load instead of skipping") + } + h.loadMu.Unlock() +} + +func TestJuliaExtractRegistryNamesLimitsRegistryToml(t *testing.T) { + var buf bytes.Buffer + gw := gzip.NewWriter(&buf) + tw := tar.NewWriter(gw) + size := int64(maxJuliaRegistryTomlSize + 1) + if err := tw.WriteHeader(&tar.Header{Name: "Registry.toml", Mode: 0o644, Size: size}); err != nil { + t.Fatal(err) + } + if _, err := tw.Write(bytes.Repeat([]byte("#"), int(size))); err != nil { + t.Fatal(err) + } + _ = tw.Close() + _ = gw.Close() + if _, err := extractRegistryNames(&buf); err == nil { + t.Fatal("oversized Registry.toml was accepted") + } +} From 8dcd194c39c38d78559ec80ffcbc526e05e597bf Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Fri, 23 Oct 2026 13:22:49 +0200 Subject: [PATCH 364/633] Fix npm search routing and abbreviated packuments under cooldown `npm search` requests /-/v1/search?text=... . The path has no "/-/" after the leading slash is trimmed, so it fell through to the package metadata handler: upstream got /-/v1%2Fsearch without the query string, and the answer was cached under "-/v1/search", so every later search was served the first one's results. Search and /-/ping (used by `npm ping`, which lost its query string the same way) are now relayed to upstream uncached with their query strings, like the signing-keys endpoint. Packuments share one metadata cache row whatever format they were fetched in. Without cooldown the proxy asks for the abbreviated format, which has no "time" map. After cooldown was turned on, a still-fresh abbreviated packument kept being served from cache, and used as the stale copy when upstream failed, so cooldown filtered nothing and the tarball check let recent versions through. When full metadata is required (cooldown or npm_full_metadata on), packument lookups now pass a validator that rejects an abbreviated document (top-level "modified", no "time"), so it counts as a cache miss, is refetched in full and is never used as a stale fallback. Behaviour without cooldown is unchanged. fetchMetadataDocument takes the validate callback that fetchOrCacheMetadata already has; other callers pass nil. --- README.md | 1 + docs/configuration.md | 4 +- internal/handler/chef.go | 2 +- internal/handler/composer.go | 4 +- internal/handler/composer_hosted.go | 2 +- internal/handler/handler.go | 7 +- .../handler/metadata_not_modified_test.go | 6 +- internal/handler/npm.go | 69 ++++++++- internal/handler/npm_hosted.go | 2 +- internal/handler/npm_packument_accept_test.go | 132 ++++++++++++++++++ internal/handler/npm_test.go | 69 +++++++++ 11 files changed, 282 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index 0553e4a..3433402 100644 --- a/README.md +++ b/README.md @@ -1207,6 +1207,7 @@ Recently cached: | `GET /npm/*` | npm registry protocol | | `POST /npm/-/npm/v1/security/*` | npm/pnpm/Yarn audit endpoints, passed through to upstream | | `GET /npm/-/npm/v1/keys` | npm registry signing keys, passed through to upstream | +| `GET /npm/-/v1/search`, `GET /npm/-/ping` | `npm search` and `npm ping`, passed through to upstream uncached | | `GET /cargo/*` | Cargo sparse index protocol | | `GET /gem/*` | RubyGems protocol | | `GET /go/*` | Go module proxy protocol | diff --git a/docs/configuration.md b/docs/configuration.md index 4da6f63..5a6f292 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -353,7 +353,7 @@ upstream: - Composer metadata of routed packages carries an empty `notification-url`, so Composer does not report their installs to the default upstream's `notify-batch` endpoint. - Use `upstream.auth` for the registry's credentials. The proxy has no authentication of its own, so every client that can reach it can download routed packages. - Search (`/composer/search.json`) and the package list (`/composer/packages/list.json`) still come from the default upstream only. -- npm audit and signing-key requests still go to `upstream.npm`. An audit request lists every package in the dependency tree, routed ones included. +- npm audit, signing-key, search and ping requests still go to `upstream.npm`. An audit request lists every package in the dependency tree, routed ones included. - Routes can only be set in the configuration file, not with environment variables. Helm HTTP repositories and additional OCI registries are configured as named maps: @@ -982,6 +982,8 @@ Resolution order: exact package override, then package pattern, then ecosystem o Currently supported for npm, PyPI, pub.dev, Composer, Cargo, NuGet, Conda, RubyGems, and Hex. These ecosystems include publish timestamps in their metadata. +With cooldown on, the proxy asks npm for full packuments, since the abbreviated ones it uses otherwise carry no publish times. Abbreviated packuments cached before cooldown was turned on are fetched again in full rather than served, including as a stale copy when the upstream is down. + Note: Hex cooldown requires disabling registry signature verification since the proxy re-encodes the protobuf payload without the original signature. Set `HEX_NO_VERIFY_REPO_ORIGIN=1` or configure your repo with `no_verify: true`. ## Version denylist diff --git a/internal/handler/chef.go b/internal/handler/chef.go index 764d8a3..cf2002d 100644 --- a/internal/handler/chef.go +++ b/internal/handler/chef.go @@ -133,7 +133,7 @@ func (h *ChefHandler) serveDocument(w http.ResponseWriter, r *http.Request, docu return } - upstream := h.proxy.fetchMetadataDocument(r.Context(), chefEcosystem, cacheKey, upstreamURL, contentTypeJSON) + upstream := h.proxy.fetchMetadataDocument(r.Context(), chefEcosystem, cacheKey, upstreamURL, contentTypeJSON, nil) if err := upstream.err; err != nil { if errors.Is(err, ErrUpstreamNotFound) { http.Error(w, "not found", http.StatusNotFound) diff --git a/internal/handler/composer.go b/internal/handler/composer.go index 643b287..223784c 100644 --- a/internal/handler/composer.go +++ b/internal/handler/composer.go @@ -169,7 +169,7 @@ func (h *ComposerHandler) handlePackageMetadata(w http.ResponseWriter, r *http.R return } - upstream := h.proxy.fetchMetadataDocument(r.Context(), "composer", cacheKey, upstreamURL, contentTypeJSON) + upstream := h.proxy.fetchMetadataDocument(r.Context(), "composer", cacheKey, upstreamURL, contentTypeJSON, nil) if err := upstream.err; err != nil { if errors.Is(err, ErrUpstreamNotFound) { http.Error(w, "not found", http.StatusNotFound) @@ -753,7 +753,7 @@ func (h *ComposerHandler) cachedDist(ctx context.Context, files []composerMetada return composerDist{}, "", false } for _, f := range files { - doc := h.proxy.fetchMetadataDocument(ctx, "composer", f.cacheKey, f.url, contentTypeJSON) + doc := h.proxy.fetchMetadataDocument(ctx, "composer", f.cacheKey, f.url, contentTypeJSON, nil) if doc.err != nil { continue } diff --git a/internal/handler/composer_hosted.go b/internal/handler/composer_hosted.go index 689cf8f..e94d11c 100644 --- a/internal/handler/composer_hosted.go +++ b/internal/handler/composer_hosted.go @@ -78,7 +78,7 @@ func pushedComposerVersion(local []composerLocalEntry, version string) bool { // upstream does not know the package, the document holds only the pushed // versions. func (h *ComposerHandler) serveMetadataWithLocal(w http.ResponseWriter, r *http.Request, packageName, upstreamURL, cacheKey string, local []composerLocalEntry) { - upstream := h.proxy.fetchMetadataDocument(r.Context(), "composer", cacheKey, upstreamURL, contentTypeJSON) + upstream := h.proxy.fetchMetadataDocument(r.Context(), "composer", cacheKey, upstreamURL, contentTypeJSON, nil) var out []byte var err error switch { diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 022f3d8..8bcac24 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -1335,9 +1335,10 @@ func (p *Proxy) fetchOrCacheMetadata(ctx context.Context, ecosystem, cacheKey, u // fetchMetadataDocument is FetchOrCacheMetadata for callers that rewrite the // document and need to know which one they got: the result carries its -// digest along with the body. -func (p *Proxy) fetchMetadataDocument(ctx context.Context, ecosystem, cacheKey, upstreamURL, accept string) metadataResult { - return p.lookupMetadata(ctx, ecosystem, cacheKey, upstreamURL, accept, "", nil) +// digest along with the body. validate, which may be nil, is as for +// fetchOrCacheMetadata. +func (p *Proxy) fetchMetadataDocument(ctx context.Context, ecosystem, cacheKey, upstreamURL, accept string, validate func([]byte) error) metadataResult { + return p.lookupMetadata(ctx, ecosystem, cacheKey, upstreamURL, accept, "", validate) } // lookupMetadata implements fetchOrCacheMetadata and fetchMetadataDocument. diff --git a/internal/handler/metadata_not_modified_test.go b/internal/handler/metadata_not_modified_test.go index a16c82c..fd690df 100644 --- a/internal/handler/metadata_not_modified_test.go +++ b/internal/handler/metadata_not_modified_test.go @@ -36,7 +36,7 @@ func TestMetadataNotModifiedKeepsStoredDocument(t *testing.T) { proxy.HTTPClient = upstream.Client() ctx := context.Background() - first := proxy.fetchMetadataDocument(ctx, "test", "pkg", upstream.URL+"/pkg", contentTypeJSON) + first := proxy.fetchMetadataDocument(ctx, "test", "pkg", upstream.URL+"/pkg", contentTypeJSON, nil) if first.err != nil { t.Fatal(first.err) } @@ -46,7 +46,7 @@ func TestMetadataNotModifiedKeepsStoredDocument(t *testing.T) { } for range 2 { - res := proxy.fetchMetadataDocument(ctx, "test", "pkg", upstream.URL+"/pkg", contentTypeJSON) + res := proxy.fetchMetadataDocument(ctx, "test", "pkg", upstream.URL+"/pkg", contentTypeJSON, nil) if res.err != nil || string(res.body) != body || res.digest != first.digest { t.Fatalf("revalidated fetch = %q, digest %q, err %v; want %q, digest %q", res.body, res.digest, res.err, body, first.digest) } @@ -99,7 +99,7 @@ func TestMetadataNotModifiedStoresRowWithoutDigest(t *testing.T) { t.Fatal(err) } - res := proxy.fetchMetadataDocument(context.Background(), "test", "pkg", upstream.URL+"/pkg", contentTypeJSON) + res := proxy.fetchMetadataDocument(context.Background(), "test", "pkg", upstream.URL+"/pkg", contentTypeJSON, nil) if res.err != nil || string(res.body) != `{"v":1}` { t.Fatalf("fetch = %q, %v", res.body, res.err) } diff --git a/internal/handler/npm.go b/internal/handler/npm.go index a874c61..0e2d8f2 100644 --- a/internal/handler/npm.go +++ b/internal/handler/npm.go @@ -32,6 +32,11 @@ const ( // npmKeysPath serves the registry signing keys `npm audit signatures` reads. npmKeysPath = "/-/npm/v1/keys" + // npmSearchPath is what `npm search` queries, with the terms in the query + // string. npmPingPath is what `npm ping` checks. + npmSearchPath = "/-/v1/search" + npmPingPath = "/-/ping" + // npmSecurityMaxBody caps the audit payload. Buffering it lets an oversized // body be refused before the upstream request starts. npmSecurityMaxBody = 16 << 20 @@ -88,10 +93,21 @@ func (h *NPMHandler) Routes() http.Handler { return } - if r.URL.Path == npmKeysPath { + switch r.URL.Path { + case npmKeysPath: h.proxy.ProxyUpstream(w, r, h.upstreamURL+npmKeysPath, []string{headerAccept, headerAcceptEncoding}) return + case npmSearchPath, npmPingPath: + // Every query has its own answer, so these are relayed uncached + // and keep their query string. + upstreamURL := h.upstreamURL + r.URL.Path + if r.URL.RawQuery != "" { + upstreamURL += "?" + r.URL.RawQuery + } + h.proxy.ProxyUpstream(w, r, upstreamURL, + []string{headerAccept, headerAcceptEncoding}) + return } path := strings.TrimPrefix(r.URL.Path, "/") @@ -224,7 +240,11 @@ func (h *NPMHandler) handlePackageMetadata(w http.ResponseWriter, r *http.Reques return } - upstream := h.proxy.fetchMetadataDocument(r.Context(), "npm", metadataCacheKey, upstreamURL, accept) + var validate func([]byte) error + if !versionLookup { + validate = h.packumentValidator() + } + upstream := h.proxy.fetchMetadataDocument(r.Context(), "npm", metadataCacheKey, upstreamURL, accept, validate) if err := upstream.err; err != nil { if errors.Is(err, ErrUpstreamNotFound) { if versionLookup { @@ -292,11 +312,52 @@ func (h *NPMHandler) packumentAccept() string { return npmAcceptDefault } +// errAbbreviatedPackument rejects an abbreviated packument where the full one +// is needed. +var errAbbreviatedPackument = errors.New("abbreviated packument where full metadata is required") + +// packumentValidator returns the check a packument must pass to be served or +// cached, or nil when any document will do. Packuments share one cache row +// whatever format they were fetched in, so one cached in abbreviated form +// before cooldown or full metadata was turned on would otherwise still be +// served, and used for the cooldown tarball check, while it stayed fresh or +// as a stale copy when upstream failed. It has no "time" map, so cooldown +// would let every version through. Rejecting it makes the lookup fetch the +// full document instead. +func (h *NPMHandler) packumentValidator() func([]byte) error { + if h.packumentAccept() != contentTypeJSON { + return nil + } + return requireFullPackument +} + +// requireFullPackument rejects an abbreviated packument: one with the +// top-level "modified" field the abbreviated format carries and no "time" +// map. A full document without a time map, as some registries serve, is +// let through as before, and so is anything that is not a JSON object, +// leaving it to the usual handling. +func requireFullPackument(body []byte) error { + var hasTime, hasModified bool + err := forEachJSONMember(body, func(m jsonMember) error { + switch key := m.key(body); { + case jsonKeyIs(key, "time"): + hasTime = true + case jsonKeyIs(key, "modified"): + hasModified = true + } + return nil + }) + if err == nil && hasModified && !hasTime { + return errAbbreviatedPackument + } + return nil +} + // handleFilteredNPMVersionMetadata serves /pkg/ and /pkg/ from the // filtered packument so cooldown and dist-tags match the root metadata response. func (h *NPMHandler) handleFilteredNPMVersionMetadata(w http.ResponseWriter, r *http.Request, packageName, specifier, accept, registryURL, cacheKey, metadataCacheKey string) { _, packumentPath := npmMetadataPath(packageName) - packument, _, err := h.proxy.FetchOrCacheMetadata(r.Context(), "npm", cacheKey, registryURL+"/"+packumentPath, accept) + packument, _, _, err := h.proxy.fetchOrCacheMetadata(r.Context(), "npm", cacheKey, registryURL+"/"+packumentPath, "", h.packumentValidator(), accept) if err != nil { if errors.Is(err, ErrUpstreamNotFound) { JSONError(w, http.StatusNotFound, ErrPackageNotFound.Error()) @@ -859,7 +920,7 @@ func (h *NPMHandler) handleDownload(w http.ResponseWriter, r *http.Request) { registryURL, cacheKey := h.sourceFor(packageName) metadata := sync.OnceValues(func() ([]byte, error) { upstreamURL := registryURL + "/" + url.PathEscape(packageName) - body, _, err := h.proxy.FetchOrCacheMetadata(r.Context(), "npm", cacheKey, upstreamURL, h.packumentAccept()) + body, _, _, err := h.proxy.fetchOrCacheMetadata(r.Context(), "npm", cacheKey, upstreamURL, "", h.packumentValidator(), h.packumentAccept()) return body, err }) if h.versionInCooldown(packageName, version, metadata) { diff --git a/internal/handler/npm_hosted.go b/internal/handler/npm_hosted.go index 7532cbd..4c786e8 100644 --- a/internal/handler/npm_hosted.go +++ b/internal/handler/npm_hosted.go @@ -98,7 +98,7 @@ func (h *NPMHandler) serveLocalMetadata(w http.ResponseWriter, r *http.Request, // When the upstream does not know the package, the packument holds only the // pushed versions. func (h *NPMHandler) serveMetadataWithLocal(w http.ResponseWriter, r *http.Request, packageName, upstreamURL, accept, metadataCacheKey string, local []npmLocalEntry) { - upstream := h.proxy.fetchMetadataDocument(r.Context(), "npm", metadataCacheKey, upstreamURL, accept) + upstream := h.proxy.fetchMetadataDocument(r.Context(), "npm", metadataCacheKey, upstreamURL, accept, h.packumentValidator()) var out []byte var err error switch { diff --git a/internal/handler/npm_packument_accept_test.go b/internal/handler/npm_packument_accept_test.go index 7158cef..1a87fdd 100644 --- a/internal/handler/npm_packument_accept_test.go +++ b/internal/handler/npm_packument_accept_test.go @@ -1,15 +1,18 @@ package handler import ( + "encoding/json" "fmt" "io" "net/http" "net/http/httptest" "strings" "sync" + "sync/atomic" "testing" "time" + "github.com/git-pkgs/cooldown" "github.com/git-pkgs/registries/fetch" ) @@ -76,3 +79,132 @@ func TestNPMTarballKeepsCachedPackumentFormat(t *testing.T) { t.Errorf("cached packument Content-Type = %q, want %q", entry.ContentType.String, abbreviated) } } + +// abbreviatedThenFullRegistry serves a packument with a 2.0.0 published an +// hour ago and a 1.0.0 published a month ago. The abbreviated form, like +// registry.npmjs.org's, has no time map. fail makes requests for the full +// document fail. +func abbreviatedThenFullRegistry(t *testing.T, fail *atomic.Bool, fullRequests *atomic.Int64) *httptest.Server { + t.Helper() + const abbreviated = "application/vnd.npm.install-v1+json" + var upstream *httptest.Server + upstream = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/demo" { + w.WriteHeader(http.StatusNotFound) + return + } + versions := fmt.Sprintf(`{"1.0.0":{"name":"demo","version":"1.0.0","dist":{"tarball":%q}},"2.0.0":{"name":"demo","version":"2.0.0","dist":{"tarball":%q}}}`, + upstream.URL+"/demo/-/demo-1.0.0.tgz", upstream.URL+"/demo/-/demo-2.0.0.tgz") + if strings.HasPrefix(r.Header.Get("Accept"), abbreviated) { + w.Header().Set(headerContentType, abbreviated) + _, _ = fmt.Fprintf(w, `{"name":"demo","modified":%q,"dist-tags":{"latest":"2.0.0"},"versions":%s}`, + time.Now().Format(time.RFC3339), versions) + return + } + fullRequests.Add(1) + if fail.Load() { + w.WriteHeader(http.StatusServiceUnavailable) + return + } + w.Header().Set(headerContentType, contentTypeJSON) + _, _ = fmt.Fprintf(w, `{"name":"demo","dist-tags":{"latest":"2.0.0"},"time":{"1.0.0":%q,"2.0.0":%q},"versions":%s}`, + time.Now().Add(-30*24*time.Hour).Format(time.RFC3339), time.Now().Add(-time.Hour).Format(time.RFC3339), versions) + })) + t.Cleanup(upstream.Close) + return upstream +} + +// seedAbbreviatedPackument caches the abbreviated packument for demo through +// the metadata route while cooldown is off, then turns cooldown on. +func seedAbbreviatedPackument(t *testing.T, upstream *httptest.Server) *NPMHandler { + t.Helper() + proxy, db, _, artifactFetcher := setupTestProxy(t) + proxy.CacheMetadata = true + proxy.MetadataTTL = time.Hour + proxy.HTTPClient = upstream.Client() + artifactFetcher.artifact = &fetch.Artifact{Body: io.NopCloser(strings.NewReader("package")), ContentType: "application/gzip"} + h := NewNPMHandler(proxy, "http://proxy.test", upstream.URL) + + if w := serveNPM(t, h, http.MethodGet, "/demo", nil, nil); w.Code != http.StatusOK { + t.Fatalf("seeding metadata: status %d: %s", w.Code, w.Body.String()) + } + entry, err := db.GetMetadataCache("npm", "demo") + if err != nil || entry == nil || entry.ContentType.String != "application/vnd.npm.install-v1+json" { + t.Fatalf("expected an abbreviated packument in the cache, got %+v (%v)", entry, err) + } + + proxy.Cooldown = &cooldown.Config{Default: "7d"} + return h +} + +// A packument cached in abbreviated form before cooldown was turned on has +// no publish times, so cooldown could not filter anything while it stayed +// fresh. With cooldown on it must be refetched in full. +func TestNPMCooldownRefetchesCachedAbbreviatedPackument(t *testing.T) { + var fail atomic.Bool + var fullRequests atomic.Int64 + upstream := abbreviatedThenFullRegistry(t, &fail, &fullRequests) + h := seedAbbreviatedPackument(t, upstream) + + w := serveNPM(t, h, http.MethodGet, "/demo", nil, nil) + if w.Code != http.StatusOK { + t.Fatalf("metadata: status %d: %s", w.Code, w.Body.String()) + } + var doc struct { + DistTags map[string]string `json:"dist-tags"` + Versions map[string]json.RawMessage `json:"versions"` + } + if err := json.Unmarshal(w.Body.Bytes(), &doc); err != nil { + t.Fatalf("decoding metadata: %v", err) + } + if _, ok := doc.Versions["2.0.0"]; ok { + t.Error("version 2.0.0, published an hour ago, was not filtered by cooldown") + } + if _, ok := doc.Versions["1.0.0"]; !ok { + t.Error("version 1.0.0 missing") + } + if doc.DistTags["latest"] != "1.0.0" { + t.Errorf("dist-tags.latest = %q, want 1.0.0", doc.DistTags["latest"]) + } + if fullRequests.Load() == 0 { + t.Error("full packument was never fetched from upstream") + } + + if w := serveNPM(t, h, http.MethodGet, "/demo/-/demo-2.0.0.tgz", nil, nil); w.Code != http.StatusNotFound { + t.Errorf("tarball 2.0.0: status %d, want %d (withheld by cooldown)", w.Code, http.StatusNotFound) + } +} + +// The tarball check alone must not trust the abbreviated packument either. +func TestNPMCooldownTarballRefetchesCachedAbbreviatedPackument(t *testing.T) { + var fail atomic.Bool + var fullRequests atomic.Int64 + upstream := abbreviatedThenFullRegistry(t, &fail, &fullRequests) + h := seedAbbreviatedPackument(t, upstream) + + if w := serveNPM(t, h, http.MethodGet, "/demo/-/demo-2.0.0.tgz", nil, nil); w.Code != http.StatusNotFound { + t.Errorf("tarball 2.0.0: status %d, want %d (withheld by cooldown)", w.Code, http.StatusNotFound) + } + if fullRequests.Load() == 0 { + t.Error("full packument was never fetched from upstream") + } +} + +// When upstream cannot supply the full packument, the abbreviated one must not +// stand in for it as a stale copy: it would let every version through. +func TestNPMCooldownSkipsAbbreviatedStaleFallback(t *testing.T) { + var fail atomic.Bool + var fullRequests atomic.Int64 + upstream := abbreviatedThenFullRegistry(t, &fail, &fullRequests) + h := seedAbbreviatedPackument(t, upstream) + h.proxy.MetadataTTL = time.Nanosecond // force a refetch + fail.Store(true) + + w := serveNPM(t, h, http.MethodGet, "/demo", nil, nil) + if w.Code != http.StatusBadGateway { + t.Errorf("metadata: status %d, want %d; body: %s", w.Code, http.StatusBadGateway, w.Body.String()) + } + if strings.Contains(w.Body.String(), "2.0.0") { + t.Errorf("served the abbreviated packument as a stale copy: %s", w.Body.String()) + } +} diff --git a/internal/handler/npm_test.go b/internal/handler/npm_test.go index c8a4eb3..23e223f 100644 --- a/internal/handler/npm_test.go +++ b/internal/handler/npm_test.go @@ -1328,3 +1328,72 @@ func TestNPMVersionMetadataDenied(t *testing.T) { t.Fatalf("status = %d, want 403; body: %s", w.Code, w.Body.String()) } } + +// `npm search` asks for /-/v1/search?text=.... The path used to fall through to +// the package dispatch, which dropped the query string, escaped the path into +// a package name and cached the answer, so every later search got the first +// one's results. +func TestNPMSearchRelaysQueryUncached(t *testing.T) { + var ( + requests int + gotPaths []string + ) + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests++ + gotPaths = append(gotPaths, r.URL.EscapedPath()) + if r.URL.EscapedPath() != "/-/v1/search" { + w.WriteHeader(http.StatusNotFound) + return + } + w.Header().Set("Content-Type", "application/json") + _, _ = fmt.Fprintf(w, `{"objects":[],"total":0,"query":%q}`, r.URL.RawQuery) + })) + defer upstream.Close() + + proxy, db, _, _ := setupTestProxy(t) + proxy.CacheMetadata = true + proxy.MetadataTTL = time.Hour + proxy.HTTPClient = upstream.Client() + h := NewNPMHandler(proxy, "http://proxy.test", upstream.URL) + + for _, query := range []string{"text=foo&size=20", "text=bar&size=20", "text=foo&size=20"} { + w := serveNPM(t, h, http.MethodGet, "/-/v1/search?"+query, nil, nil) + if w.Code != http.StatusOK { + t.Fatalf("search %q: status = %d; body: %s", query, w.Code, w.Body.String()) + } + if want := fmt.Sprintf(`{"objects":[],"total":0,"query":%q}`, query); w.Body.String() != want { + t.Errorf("search %q: body = %s, want %s", query, w.Body.String(), want) + } + } + if requests != 3 { + t.Errorf("upstream requests = %d (%q), want 3: searches must not be cached", requests, gotPaths) + } + for _, key := range []string{"-/v1/search", "-", "-/v1"} { + if entry, _ := db.GetMetadataCache("npm", key); entry != nil { + t.Errorf("search response cached under %q", key) + } + } +} + +// `npm ping` reads /-/ping, which used to be looked up as a package. +func TestNPMPingRelaysUpstream(t *testing.T) { + var gotPath, gotQuery string + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotPath, gotQuery = r.URL.EscapedPath(), r.URL.RawQuery + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, "{}") + })) + defer upstream.Close() + + proxy, _, _, _ := setupTestProxy(t) + proxy.HTTPClient = upstream.Client() + h := NewNPMHandler(proxy, "http://proxy.test", upstream.URL) + + w := serveNPM(t, h, http.MethodGet, "/-/ping?write=true", nil, nil) + if w.Code != http.StatusOK || w.Body.String() != "{}" { + t.Fatalf("ping = %d %q, want 200 {}", w.Code, w.Body.String()) + } + if gotPath != "/-/ping" || gotQuery != "write=true" { + t.Errorf("upstream request = %q?%q, want /-/ping?write=true", gotPath, gotQuery) + } +} From be50be27e20309d3840f6fb231ae4c31b732b8f1 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Fri, 23 Oct 2026 13:22:56 +0200 Subject: [PATCH 365/633] Serve cached Composer releases without a metadata lookup Every Composer archive download resolved its dist from the package metadata before checking the artifact cache. With cache_metadata off, the default, that meant an uncoalesced upstream GET of the p2 document for every download of an archive the proxy already had: three downloads of a cached archive made three metadata requests. With the cache on, each download after the TTL revalidated and rescanned the document. The lookup is only needed for the publish time cooldown checks downloads against, since lock files name archive URLs directly. Like the npm handler, record that time in the versions table when the dist is first resolved, and on later downloads of a tagged release read it from there and serve the cached archive straight away. Without a stored time, as for archives cached by an older proxy, the download takes the metadata path, which then records it. The time is stored even with cooldown off, so enabling cooldown later keeps the fast path. The cache still enforces the denylist and verifies the stored digest, and an archive is only ever cached under a name that was checked against the metadata, so those guarantees hold. Branches, whose archive and time change with every push, routed packages, whose versions row would not say which repository the time came from, and requests with a query string such as ?sha= keep resolving from the metadata. --- docs/configuration.md | 4 +- internal/handler/composer.go | 94 +++++++- .../handler/composer_download_lookup_test.go | 203 +++++++++++++++++- 3 files changed, 283 insertions(+), 18 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 4da6f63..660421f 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -349,7 +349,7 @@ upstream: - An npm route URL is the registry base that serves `/{package}`. Tarball URLs in its packuments must be on the same host and below the same path. - A routed package is not looked up anywhere else. If the private registry does not have it, the client gets a 404. If the registry fails or rejects the credentials, the client gets a 502. A package with the same name on the public registry is never served in its place (dependency confusion). - Metadata and artifacts of routed packages are cached under keys that include the route's URL. Entries cached from the public registry before a route was added are never served for a routed package. -- With cooldown enabled, the publish time of a routed npm version always comes from the route's metadata. Times recorded for the public package of the same name, or for a route's previous URL, are not used, and routed downloads do not record times of their own. +- With cooldown enabled, the publish time of a routed npm or Composer version always comes from the route's metadata. Times recorded for the public package of the same name, or for a route's previous URL, are not used, and routed downloads do not record times of their own. - Composer metadata of routed packages carries an empty `notification-url`, so Composer does not report their installs to the default upstream's `notify-batch` endpoint. - Use `upstream.auth` for the registry's credentials. The proxy has no authentication of its own, so every client that can reach it can download routed packages. - Search (`/composer/search.json`) and the package list (`/composer/packages/list.json`) still come from the default upstream only. @@ -1209,7 +1209,7 @@ cache_metadata: true Or via environment variable: `PROXY_CACHE_METADATA=true`. -Composer archive downloads look up their upstream URL in the same cached metadata, so a cached archive is served without asking Packagist again and stays installable while it is down. Only a version or file the cached metadata does not list yet is looked up upstream. +Composer archive downloads look up their upstream URL in the same cached metadata, so an archive is resolved without asking Packagist again and stays installable while it is down. Only a version or file the cached metadata does not list yet is looked up upstream. With or without `cache_metadata`, a cached archive of a tagged release from the default repository is served without any metadata lookup; with cooldown enabled it is checked against the publish time recorded when the archive was first resolved. Branch (dev) versions, routed packages and archives picked by a query string such as `?sha=` are always resolved from the metadata. The `proxy mirror` command always enables metadata caching regardless of this setting. diff --git a/internal/handler/composer.go b/internal/handler/composer.go index 643b287..768b89a 100644 --- a/internal/handler/composer.go +++ b/internal/handler/composer.go @@ -383,13 +383,8 @@ func (h *ComposerHandler) shouldFilterVersion(packagePURL, packageName, version return false } - var timeStr string - if err := json.Unmarshal(published, &timeStr); err != nil { - return false - } - - publishedAt, err := time.Parse(time.RFC3339, timeStr) - if err != nil { + publishedAt, ok := composerPublishTime(published) + if !ok { return false } @@ -402,6 +397,17 @@ func (h *ComposerHandler) shouldFilterVersion(packagePURL, packageName, version return false } +// composerPublishTime parses the raw "time" field of a version, reporting +// false when it is missing or not an RFC 3339 string. +func composerPublishTime(published []byte) (time.Time, bool) { + var timeStr string + if err := json.Unmarshal(published, &timeStr); err != nil { + return time.Time{}, false + } + publishedAt, err := time.Parse(time.RFC3339, timeStr) + return publishedAt, err == nil +} + // disableDefaultNotification stops Composer from reporting installs of a routed // package to the default repository's notify-batch URL, which would disclose // the package name to it. Composer only falls back to notify-batch when a @@ -633,6 +639,8 @@ func (h *ComposerHandler) handleDownload(w http.ResponseWriter, r *http.Request) requested := path.Base(r.URL.EscapedPath()) if r.URL.RawQuery != "" { requested += "?" + r.URL.RawQuery + } else if h.serveCachedRelease(w, r, packageName, version, filename) { + return } files := h.metadataFilesForVersion(vendor, pkg, version) dist, downloadURL, ok := h.cachedDist(r.Context(), files, packageName, version, requested) @@ -641,6 +649,7 @@ func (h *ComposerHandler) handleDownload(w http.ResponseWriter, r *http.Request) return } } + h.recordPublishTime(packageName, version, dist.published) // Lock files name archive URLs directly, so a version filtered out of the // metadata still has to be withheld here. @@ -682,6 +691,77 @@ func (h *ComposerHandler) handleDownload(w http.ResponseWriter, r *http.Request) ServeArtifactRequest(w, r, result) } +// releaseTimeStored reports whether the publish time of version is kept in +// the versions table, so a download can be checked against cooldown without +// the metadata. That holds for tagged releases of packages from the default +// repository. A branch is left out: its archive and time change with every +// push. So are routed packages, as for npm: the versions row is keyed by the +// plain PURL, which does not say which repository the version came from, so +// a time read from the default repository would be trusted for a private +// version of the same name, and the other way round. +func (h *ComposerHandler) releaseTimeStored(packageName, version string) bool { + if h.proxy.DB == nil || isDevVersion(version) { + return false + } + _, routed := h.routes.match(packageName) + return !routed +} + +// serveCachedRelease serves a tagged release's archive from the cache +// without looking up its dist in the metadata, and reports whether it wrote a +// response. The archive was only cached under its name after that name was +// checked against the metadata, and the cache verifies it against the digest +// recorded then. Cooldown is checked against the publish time recorded when +// the dist was first resolved; without one, as for an archive cached by an +// older proxy, it returns false and the caller takes the metadata path, which +// records it. Requests with a query string, which pick a commit, always take +// that path. +func (h *ComposerHandler) serveCachedRelease(w http.ResponseWriter, r *http.Request, packageName, version, filename string) bool { + if !h.releaseTimeStored(packageName, version) { + return false + } + if h.proxy.Cooldown != nil && h.proxy.Cooldown.Enabled() { + ver, err := h.proxy.DB.GetVersionByPURL(canonicalVersionPURL("composer", packageName, version)) + if err != nil || ver == nil || !ver.PublishedAt.Valid { + return false + } + if !h.proxy.Cooldown.IsAllowed("composer", canonicalPackagePURL("composer", packageName), ver.PublishedAt.Time) { + h.proxy.Logger.Info("cooldown: withholding composer archive", + "package", packageName, "version", version) + http.Error(w, "version not found", http.StatusNotFound) + return true + } + } + cached, err := h.proxy.GetCachedArtifact(r.Context(), "composer", packageName, version, filename) + if err != nil { + h.proxy.serveArtifactError(w, err, "failed to read cached package") + return true + } + if cached == nil { + return false + } + ServeArtifactRequest(w, r, cached) + return true +} + +// recordPublishTime stores the publish time a release's metadata lists, for +// serveCachedRelease. It is stored whether or not cooldown is enabled, so +// enabling it later does not send cached archives back to the metadata. +func (h *ComposerHandler) recordPublishTime(packageName, version string, published []byte) { + if !h.releaseTimeStored(packageName, version) { + return + } + publishedAt, ok := composerPublishTime(published) + if !ok { + return + } + if err := h.proxy.DB.SetVersionPublishedAt(canonicalVersionPURL("composer", packageName, version), + canonicalPackagePURL("composer", packageName), publishedAt); err != nil { + h.proxy.Logger.Warn("failed to store composer publish time", + "package", packageName, "version", version, "error", err) + } +} + // discardOtherCommit clears a cached branch archive named for commit that was // fetched from another commit's URL. Proxy versions before the pinned-commit // fix cached the branch head under whatever commit a lock file asked for. The diff --git a/internal/handler/composer_download_lookup_test.go b/internal/handler/composer_download_lookup_test.go index 5ac4c75..5635000 100644 --- a/internal/handler/composer_download_lookup_test.go +++ b/internal/handler/composer_download_lookup_test.go @@ -10,23 +10,29 @@ import ( "testing" "time" + "github.com/git-pkgs/cooldown" "github.com/git-pkgs/registries/fetch" ) // composerLookupUpstream serves v/p's metadata listing versions and the // archives they point at, counts metadata and archive requests, and fails -// every request while down is set. +// every request while down is set. A version listed in published carries that +// publish time. type composerLookupUpstream struct { - h http.Handler - versions atomic.Value // []string - down atomic.Bool - metadata atomic.Int32 - archives atomic.Int32 + h http.Handler + handler *ComposerHandler + proxy *Proxy + versions atomic.Value // []string + published map[string]time.Time + down atomic.Bool + metadata atomic.Int32 + archives atomic.Int32 + routeURL string } func setupComposerLookup(t *testing.T, cacheMetadata bool, versions ...string) *composerLookupUpstream { t.Helper() - u := &composerLookupUpstream{} + u := &composerLookupUpstream{published: map[string]time.Time{}} u.versions.Store(versions) var srv *httptest.Server srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -39,7 +45,11 @@ func setupComposerLookup(t *testing.T, cacheMetadata bool, versions ...string) * u.metadata.Add(1) var entries []string for _, v := range u.versions.Load().([]string) { - entries = append(entries, fmt.Sprintf(`{"version":%q,"dist":{"url":%q,"type":"zip"}}`, v, srv.URL+"/archives/"+v+".zip")) + var published string + if at, ok := u.published[v]; ok { + published = fmt.Sprintf(`"time":%q,`, at.Format(time.RFC3339)) + } + entries = append(entries, fmt.Sprintf(`{"version":%q,%s"dist":{"url":%q,"type":"zip"}}`, v, published, srv.URL+"/archives/"+v+".zip")) } _, _ = fmt.Fprintf(w, `{"packages":{"v/p":[%s]}}`, strings.Join(entries, ",")) case r.URL.Path == "/p2/v/p~dev.json": @@ -60,10 +70,19 @@ func setupComposerLookup(t *testing.T, cacheMetadata bool, versions ...string) * fetcher := fetch.NewFetcher(fetch.WithHTTPClient(srv.Client()), fetch.WithMaxRetries(0)) proxy.Fetcher = fetcher t.Cleanup(func() { _ = fetcher.Close() }) - u.h = NewComposerHandlerWithUpstreams(proxy, "http://proxy", srv.URL, srv.URL).Routes() + u.proxy = proxy + u.handler = NewComposerHandlerWithUpstreams(proxy, "http://proxy", srv.URL, srv.URL) + u.h = u.handler.Routes() + u.routeURL = srv.URL return u } +// route sends v/p to a custom repository, which is the same server under +// another name. +func (u *composerLookupUpstream) route() { + u.h = u.handler.WithPackageRoutes(map[string]string{"v/*": u.routeURL}).Routes() +} + func (u *composerLookupUpstream) get(t *testing.T, target string) *httptest.ResponseRecorder { t.Helper() w := httptest.NewRecorder() @@ -126,3 +145,169 @@ func TestComposerDownloadWithoutMetadataCache(t *testing.T) { t.Errorf("upstream metadata requests = %d, want 1", n) } } + +// storedPublishTime returns the publish time recorded for v/p at version. +func (u *composerLookupUpstream) storedPublishTime(t *testing.T, version string) (time.Time, bool) { + t.Helper() + ver, err := u.proxy.DB.GetVersionByPURL(canonicalVersionPURL("composer", "v/p", version)) + if err != nil { + t.Fatalf("reading versions row: %v", err) + } + if ver == nil || !ver.PublishedAt.Valid { + return time.Time{}, false + } + return ver.PublishedAt.Time, true +} + +func (u *composerLookupUpstream) wantMetadataRequests(t *testing.T, want int32) { + t.Helper() + if n := u.metadata.Load(); n != want { + t.Errorf("upstream metadata requests = %d, want %d", n, want) + } +} + +// TestComposerCachedReleaseSkipsMetadata checks that a tagged release whose +// archive is cached is served without asking upstream for the metadata +// again, with and without cache_metadata, and with cooldown, which takes the +// publish time recorded when the archive was first resolved. +func TestComposerCachedReleaseSkipsMetadata(t *testing.T) { + for _, tt := range []struct { + name string + cacheMetadata bool + cooldown bool + }{ + {"metadata not cached", false, false}, + {"metadata cached", true, false}, + {"cooldown", false, true}, + } { + t.Run(tt.name, func(t *testing.T) { + u := setupComposerLookup(t, tt.cacheMetadata, "1.0.0") + u.published["1.0.0"] = time.Now().Add(-30 * 24 * time.Hour) + if tt.cooldown { + u.proxy.Cooldown = &cooldown.Config{Default: "3d"} + } + u.download(t, "1.0.0") + // With cache_metadata the cached copy expires; the archive must + // not need it. + u.proxy.MetadataTTL = 0 + u.download(t, "1.0.0") + u.download(t, "1.0.0") + u.wantMetadataRequests(t, 1) + if n := u.archives.Load(); n != 1 { + t.Errorf("upstream archive requests = %d, want 1", n) + } + }) + } +} + +// TestComposerDownloadStoresPublishTime checks that resolving a tagged +// release records its publish time, so later downloads can check cooldown +// without the metadata. +func TestComposerDownloadStoresPublishTime(t *testing.T) { + u := setupComposerLookup(t, false, "1.0.0") + published := time.Date(2024, 5, 6, 7, 8, 9, 0, time.UTC) + u.published["1.0.0"] = published + u.download(t, "1.0.0") + got, ok := u.storedPublishTime(t, "1.0.0") + if !ok || !got.Equal(published) { + t.Errorf("stored publish time = %v, %v, want %v", got, ok, published) + } +} + +// TestComposerCachedReleaseCooldownFromStoredTime checks that a cached +// archive whose stored publish time is still inside the cooldown window is +// withheld without the metadata, including when cooldown was enabled or +// raised after the archive was cached. +func TestComposerCachedReleaseCooldownFromStoredTime(t *testing.T) { + u := setupComposerLookup(t, false, "1.0.0", "2.0.0") + u.published["1.0.0"] = time.Now().Add(-5 * 24 * time.Hour) + u.published["2.0.0"] = time.Now().Add(-1 * time.Hour) + u.download(t, "2.0.0") + u.proxy.Cooldown = &cooldown.Config{Default: "3d"} + u.download(t, "1.0.0") + + if w := u.get(t, "/files/v/p/2.0.0/2.0.0.zip"); w.Code != http.StatusNotFound { + t.Errorf("2.0.0 after enabling cooldown: status %d, want 404", w.Code) + } + u.proxy.Cooldown = &cooldown.Config{Default: "7d"} + if w := u.get(t, "/files/v/p/1.0.0/1.0.0.zip"); w.Code != http.StatusNotFound { + t.Errorf("1.0.0 after raising cooldown: status %d, want 404", w.Code) + } + u.wantMetadataRequests(t, 2) +} + +// TestComposerCachedReleaseWithoutStoredTime checks that with cooldown a +// cached archive the proxy recorded no publish time for, such as one cached +// by an older version, is checked against the metadata, which then records +// the time. +func TestComposerCachedReleaseWithoutStoredTime(t *testing.T) { + u := setupComposerLookup(t, false, "1.0.0") + u.published["1.0.0"] = time.Now().Add(-1 * time.Hour) + u.download(t, "1.0.0") + if _, err := u.proxy.DB.Exec(`UPDATE versions SET published_at = NULL`); err != nil { + t.Fatal(err) + } + + u.proxy.Cooldown = &cooldown.Config{Default: "3d"} + if w := u.get(t, "/files/v/p/1.0.0/1.0.0.zip"); w.Code != http.StatusNotFound { + t.Errorf("status %d, want 404", w.Code) + } + u.wantMetadataRequests(t, 2) + if _, ok := u.storedPublishTime(t, "1.0.0"); !ok { + t.Error("publish time not recorded from the metadata") + } +} + +// TestComposerCachedDevVersionUsesMetadata checks that a branch, whose +// archive and publish time change with every push, is still resolved from +// the metadata on each download and records no publish time. +func TestComposerCachedDevVersionUsesMetadata(t *testing.T) { + u := setupComposerLookup(t, false, "dev-main") + u.published["dev-main"] = time.Now().Add(-30 * 24 * time.Hour) + u.download(t, "dev-main") + before := u.metadata.Load() + u.download(t, "dev-main") + if u.metadata.Load() == before { + t.Error("cached branch archive served without consulting the metadata") + } + if _, ok := u.storedPublishTime(t, "dev-main"); ok { + t.Error("publish time recorded for a branch") + } +} + +// TestComposerCachedRoutedReleaseUsesMetadata checks that a routed package +// keeps resolving every download from its route's metadata: the versions +// table is keyed by a PURL that does not say which repository the version +// came from, so it neither stores nor trusts a publish time for it. +func TestComposerCachedRoutedReleaseUsesMetadata(t *testing.T) { + u := setupComposerLookup(t, false, "1.0.0") + u.published["1.0.0"] = time.Now().Add(-1 * time.Hour) + u.route() + u.download(t, "1.0.0") + u.download(t, "1.0.0") + u.wantMetadataRequests(t, 2) + if _, ok := u.storedPublishTime(t, "1.0.0"); ok { + t.Error("publish time recorded for a routed package") + } + + // A time stored for the public package of the same name is not used. + if err := u.proxy.DB.SetVersionPublishedAt(canonicalVersionPURL("composer", "v/p", "1.0.0"), + canonicalPackagePURL("composer", "v/p"), time.Now().Add(-30*24*time.Hour)); err != nil { + t.Fatal(err) + } + u.proxy.Cooldown = &cooldown.Config{Default: "3d"} + if w := u.get(t, "/files/v/p/1.0.0/1.0.0.zip"); w.Code != http.StatusNotFound { + t.Errorf("routed release in cooldown: status %d, want 404", w.Code) + } +} + +// TestComposerCachedReleaseDenied checks that the denylist still blocks a +// cached release served without the metadata. +func TestComposerCachedReleaseDenied(t *testing.T) { + u := setupComposerLookup(t, false, "1.0.0") + u.download(t, "1.0.0") + setTestDenylist(t, u.proxy, "pkg:composer/v/p@1.0.0") + if w := u.get(t, "/files/v/p/1.0.0/1.0.0.zip"); w.Code != http.StatusForbidden { + t.Errorf("status %d, want 403", w.Code) + } +} From f4cab2d3028c24211f3380ebec7b3bc8d718ee68 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Fri, 23 Oct 2026 13:24:58 +0200 Subject: [PATCH 366/633] Cache cooldown metadata rewrites for a minute With cooldown on, every npm, Composer and Chef metadata request rewrote the whole document, because a cached rewrite could keep hiding a version after its cooldown ended. For a 24.5 MB packument that is about 45 ms and 31 MB of allocations per request, and the result was served uncompressed and without Last-Modified, so Composer, which revalidates p2 files with If-Modified-Since only, never got a 304. The rewrite cache now keeps cooldown rewrites with an expiry of one minute, so a version leaving cooldown shows up at most that late. Concurrent requests share one rewrite, clients get the gzip copy, and the rewrite is dated through metadataDate like any other. Its ETag is the hash of its output, so replicas agree on it. The cache is keyed by the digest the metadata cache already recorded, instead of hashing the document again, which also lets storedRewrite serve cooldown rewrites without reading the stored document. Without the rewrite cache, cooldown rewrites behave as before. --- docs/configuration.md | 2 +- internal/handler/metadata_access_test.go | 2 +- internal/handler/metadata_gzip_test.go | 7 +- internal/handler/metadata_validators_test.go | 52 +++++-- internal/handler/npm_rewrite_test.go | 8 +- internal/handler/rewrite_cache.go | 150 ++++++++++++++----- internal/handler/rewrite_cache_test.go | 68 ++++++--- 7 files changed, 208 insertions(+), 81 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 4da6f63..d039c30 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1249,7 +1249,7 @@ Or via environment variable: `PROXY_ARTIFACT_MAX_SIZE=2GB`. ### Rewritten metadata cache -The npm and Composer handlers rewrite every metadata document they serve, so that download URLs point at the proxy. That means decoding the whole document and encoding it again, and Composer's minified documents are also expanded, which together cost milliseconds per typical package and far more for very large ones. The proxy keeps rewritten documents in memory, so each distinct upstream document is rewritten once and requests arriving while it is being rewritten wait for that rewrite. New bytes from upstream are rewritten again. With version cooldown enabled the cache is bypassed, because cooldown filtering depends on the current time. +The npm and Composer handlers rewrite every metadata document they serve, so that download URLs point at the proxy. That means decoding the whole document and encoding it again, and Composer's minified documents are also expanded, which together cost milliseconds per typical package and far more for very large ones. The proxy keeps rewritten documents in memory, so each distinct upstream document is rewritten once and requests arriving while it is being rewritten wait for that rewrite. New bytes from upstream are rewritten again. With version cooldown enabled, a cached rewrite is kept for one minute only, because cooldown filtering depends on the current time: a version leaving cooldown shows up at most a minute late. Such documents are dated like any other, so clients that revalidate with `If-Modified-Since`, such as Composer, get `304 Not Modified`. Each cached document is also kept gzip-compressed, when that makes it smaller, and served with `Content-Encoding: gzip` to clients that accept it. A large packument is compressed once rather than on every request, by the proxy or by a reverse proxy in front of it. The compressed copy counts against the cache size and typically adds about an eighth to each entry. Documents rewritten outside the cache are served uncompressed. diff --git a/internal/handler/metadata_access_test.go b/internal/handler/metadata_access_test.go index eda97e4..0a64b80 100644 --- a/internal/handler/metadata_access_test.go +++ b/internal/handler/metadata_access_test.go @@ -143,7 +143,7 @@ func TestStoredRewrite_MarksAccess(t *testing.T) { body := []byte(`{}`) proxy.cacheMetadataBlob(t.Context(), "npm", "demo", metadataStoragePath("npm", "demo"), &upstreamMetadata{body: body}) key := rewriteCacheKey("npm", "http://proxy", "demo", body) - if _, err := proxy.rewrites.rewrite(t.Context(), key, body, func(b []byte) ([]byte, error) { return b, nil }); err != nil { + if _, err := proxy.rewrites.rewrite(t.Context(), key, body, 0, func(b []byte) ([]byte, error) { return b, nil }); err != nil { t.Fatal(err) } old := ageMetadataAccess(t, db, "npm", "demo") diff --git a/internal/handler/metadata_gzip_test.go b/internal/handler/metadata_gzip_test.go index 5826441..98a3a4c 100644 --- a/internal/handler/metadata_gzip_test.go +++ b/internal/handler/metadata_gzip_test.go @@ -149,7 +149,10 @@ func TestRewrittenMetadataWithoutGzip(t *testing.T) { func TestRewrittenMetadataGzipOnlyFromRewriteCache(t *testing.T) { setups := map[string]func(*Proxy){ "cache disabled": func(p *Proxy) { p.SetMetadataRewriteCacheSize(0) }, - "cooldown": func(p *Proxy) { p.Cooldown = &cooldown.Config{Default: "3d"} }, + "cooldown without cache": func(p *Proxy) { + p.SetMetadataRewriteCacheSize(0) + p.Cooldown = &cooldown.Config{Default: "3d"} + }, } for _, c := range validatorCases { for name, setup := range setups { @@ -180,7 +183,7 @@ func TestRewriteCache_KeepsGzipCopyOnlyWhenSmaller(t *testing.T) { for _, doc := range [][]byte{large, []byte("abc")} { key := rewriteCacheKey("npm", "http://proxy", string(doc[:3]), doc) - if _, err := c.rewrite(t.Context(), key, doc, identity); err != nil { + if _, err := c.rewrite(t.Context(), key, doc, 0, identity); err != nil { t.Fatal(err) } } diff --git a/internal/handler/metadata_validators_test.go b/internal/handler/metadata_validators_test.go index e34395d..f778c5f 100644 --- a/internal/handler/metadata_validators_test.go +++ b/internal/handler/metadata_validators_test.go @@ -429,8 +429,8 @@ func TestRewrittenMetadataLastModifiedAcrossRestart(t *testing.T) { } // TestRewrittenMetadataValidatorsWithCooldown checks that with cooldown on, -// rewritten metadata keeps an exact ETag but has no Last-Modified: cooldown -// changes the rewrite as versions age in, while upstream stays the same. +// rewritten metadata from the rewrite cache has an ETag and a Last-Modified, +// so clients that only send If-Modified-Since, such as Composer, get 304. func TestRewrittenMetadataValidatorsWithCooldown(t *testing.T) { for _, c := range validatorCases { t.Run(c.ecosystem, func(t *testing.T) { @@ -444,15 +444,34 @@ func TestRewrittenMetadataValidatorsWithCooldown(t *testing.T) { if etag == "" { t.Fatal("no ETag with cooldown on") } - if lm := first.Header().Get("Last-Modified"); lm != "" { - t.Errorf("Last-Modified = %q with cooldown on, want none", lm) + lm := first.Header().Get("Last-Modified") + if lm == "" { + t.Fatal("no Last-Modified with cooldown on") } if rec := serve(handler, c.path, http.Header{"If-None-Match": {etag}}); rec.Code != http.StatusNotModified { t.Errorf("If-None-Match: status %d, want 304", rec.Code) } - future := time.Now().Add(time.Hour).Format(http.TimeFormat) - if rec := serve(handler, c.path, http.Header{"If-Modified-Since": {future}}); rec.Code != http.StatusOK { - t.Errorf("If-Modified-Since: status %d, want 200", rec.Code) + if rec := serve(handler, c.path, http.Header{"If-Modified-Since": {lm}}); rec.Code != http.StatusNotModified { + t.Errorf("If-Modified-Since: status %d, want 304", rec.Code) + } + }) + } +} + +// TestRewrittenMetadataWithCooldownIsGzipped checks that with cooldown on, +// clients accepting gzip get the cached compressed copy. +func TestRewrittenMetadataWithCooldownIsGzipped(t *testing.T) { + for _, c := range validatorCases { + t.Run(c.ecosystem, func(t *testing.T) { + u := newValidatorUpstream(t) + compressibleDocuments(u) + proxy := validatorProxy(t, u, time.Hour) + proxy.Cooldown = &cooldown.Config{Default: "3d"} + handler := c.handler(proxy, u) + for range 2 { + if rec := serve(handler, c.path, http.Header{"Accept-Encoding": {"gzip"}}); rec.Code != http.StatusOK || rec.Header().Get("Content-Encoding") != "gzip" { + t.Errorf("status %d, Content-Encoding %q; want 200 gzip", rec.Code, rec.Header().Get("Content-Encoding")) + } } }) } @@ -481,7 +500,7 @@ func TestRewriteCacheETag(t *testing.T) { exclaim := func(b []byte) ([]byte, error) { return []byte(string(b) + "!"), nil } key := rewriteCacheKey("npm", "http://proxy", "demo", []byte("doc")) - m, err := c.rewrite(t.Context(), key, []byte("doc"), exclaim) + m, err := c.rewrite(t.Context(), key, []byte("doc"), 0, exclaim) if err != nil || string(m.body) != "doc!" { t.Fatalf("rewrite = %q, %v", m.body, err) } @@ -685,33 +704,39 @@ func cooldownUpstream(t *testing.T) *validatorUpstream { // TestRewrittenMetadataETagWithCooldownFollowsKeptVersions checks that with // cooldown on, the ETag changes when a version leaves cooldown although the -// upstream document stays the same, and comes back when the same versions do. +// upstream document stays the same, once the cached rewrite has expired, and +// comes back when the same versions do. func TestRewrittenMetadataETagWithCooldownFollowsKeptVersions(t *testing.T) { for _, c := range validatorCases { t.Run(c.ecosystem, func(t *testing.T) { u := cooldownUpstream(t) - proxy := validatorProxy(t, u, time.Hour) + proxy, _, clock := validatorProxyWithDB(t, u, time.Hour) handler := c.handler(proxy, u) proxy.Cooldown = &cooldown.Config{Default: "1d"} etag := serve(handler, c.path, nil).Header().Get("ETag") proxy.Cooldown = &cooldown.Config{Default: "1h"} + if rec := serve(handler, c.path, http.Header{"If-None-Match": {etag}}); rec.Code != http.StatusNotModified { + t.Errorf("within the rewrite's lifetime: status %d, want 304 from the cached rewrite", rec.Code) + } + clock.advance(cooldownRewriteTTL) rec := serve(handler, c.path, http.Header{"If-None-Match": {etag}}) if rec.Code != http.StatusOK || rec.Header().Get("ETag") == etag { t.Errorf("after a version left cooldown: status %d, ETag %q; want 200 with a new ETag", rec.Code, rec.Header().Get("ETag")) } proxy.Cooldown = &cooldown.Config{Default: "1d"} + clock.advance(cooldownRewriteTTL) expectNotModified(t, serve(handler, c.path, http.Header{"If-None-Match": {etag}}), etag) }) } } // TestRewrittenMetadataETagWithCooldownComesFromInputs checks that with -// cooldown on, the ETag is derived from what fixes the rewrite rather than -// from the rewritten body, when the metadata cache knows the upstream -// digest, and falls back to the body when it does not. +// cooldown on and no rewrite cache, the ETag is derived from what fixes the +// rewrite rather than from the rewritten body, when the metadata cache knows +// the upstream digest, and falls back to the body when it does not. func TestRewrittenMetadataETagWithCooldownComesFromInputs(t *testing.T) { for _, c := range validatorCases { for _, cacheMetadata := range []bool{true, false} { @@ -719,6 +744,7 @@ func TestRewrittenMetadataETagWithCooldownComesFromInputs(t *testing.T) { u := cooldownUpstream(t) proxy := validatorProxy(t, u, time.Hour) proxy.CacheMetadata = cacheMetadata + proxy.SetMetadataRewriteCacheSize(0) proxy.Cooldown = &cooldown.Config{Default: "1d"} handler := c.handler(proxy, u) diff --git a/internal/handler/npm_rewrite_test.go b/internal/handler/npm_rewrite_test.go index 60616d8..8521082 100644 --- a/internal/handler/npm_rewrite_test.go +++ b/internal/handler/npm_rewrite_test.go @@ -249,7 +249,7 @@ func TestNPMStoredRewriteSkippedWhenStale(t *testing.T) { body := []byte(`{}`) proxy.cacheMetadataBlob(t.Context(), "npm", "demo", metadataStoragePath("npm", "demo"), &upstreamMetadata{body: body}) key := rewriteCacheKey("npm", "http://proxy", "demo", body) - if _, err := proxy.rewrites.rewrite(t.Context(), key, body, func(b []byte) ([]byte, error) { return b, nil }); err != nil { + if _, err := proxy.rewrites.rewrite(t.Context(), key, body, 0, func(b []byte) ([]byte, error) { return b, nil }); err != nil { t.Fatal(err) } if _, ok := proxy.storedRewrite("npm", "demo", "http://proxy", "demo"); !ok { @@ -268,8 +268,8 @@ func TestNPMStoredRewriteSkippedWhenStale(t *testing.T) { proxy.Cooldown = &cooldown.Config{Default: "3d"} entry.FetchedAt.Time = time.Now() _ = db.UpsertMetadataCache(entry) - if _, ok := proxy.storedRewrite("npm", "demo", "http://proxy", "demo"); ok { - t.Error("cooldown on: expected no stored rewrite") + if _, ok := proxy.storedRewrite("npm", "demo", "http://proxy", "demo"); !ok { + t.Error("cooldown on, fresh row: expected the stored rewrite") } } @@ -284,7 +284,7 @@ func TestNPMStoredRewriteRecordsMetadataHit(t *testing.T) { body := []byte(`{"name":"demo"}`) proxy.cacheMetadataBlob(t.Context(), "npm", "hit-demo", metadataStoragePath("npm", "hit-demo"), &upstreamMetadata{body: body}) key := rewriteCacheKey("npm", "http://proxy", "hit-demo", body) - if _, err := proxy.rewrites.rewrite(t.Context(), key, body, func(b []byte) ([]byte, error) { return b, nil }); err != nil { + if _, err := proxy.rewrites.rewrite(t.Context(), key, body, 0, func(b []byte) ([]byte, error) { return b, nil }); err != nil { t.Fatal(err) } diff --git a/internal/handler/rewrite_cache.go b/internal/handler/rewrite_cache.go index 83ce2a1..0569a9c 100644 --- a/internal/handler/rewrite_cache.go +++ b/internal/handler/rewrite_cache.go @@ -33,7 +33,10 @@ import ( // Entries are keyed by a hash of the raw document, so new bytes from upstream // are rewritten again rather than served stale. Everything else a rewrite // depends on is fixed for the life of the process: the proxy URL is part of -// the key, and the denylist is loaded at startup. +// the key, and the denylist is loaded at startup. The exception is cooldown, +// which depends on the time of the request: with cooldown on, entries expire +// after cooldownRewriteTTL, so a version is held back at most that long past +// its cooldown. // // Each entry keeps the ETag of its output, so clients can revalidate their // copy without the output being hashed again on every request. Dates are not @@ -45,6 +48,8 @@ import ( // on every request, by the proxy or by a reverse proxy in front of it. type rewriteCache struct { maxBytes int64 + // now reads the time entries expire by; nil means time.Now. + now func() time.Time mu sync.Mutex size int64 @@ -56,8 +61,15 @@ type rewriteCache struct { type rewriteEntry struct { key string rewrittenMetadata + // expires is when the entry stops being served, or zero for never. + expires time.Time } +// cooldownRewriteTTL is how long a rewrite made with cooldown on is served. +// Versions leave cooldown as time passes, so a cached rewrite would otherwise +// keep hiding them. +const cooldownRewriteTTL = time.Minute + // size is how much of the cache's room the entry takes. func (e *rewriteEntry) size() int64 { return int64(len(e.body) + len(e.gzipped)) @@ -159,29 +171,59 @@ func (c *rewriteCache) get(key string) (rewrittenMetadata, bool) { } c.mu.Lock() defer c.mu.Unlock() - el, ok := c.entries[key] + e, ok := c.lookup(key, c.clock()) if !ok { return rewrittenMetadata{}, false } + return e.rewrittenMetadata, true +} + +// lookup returns the live entry for key and marks it recently used. An +// expired entry is dropped. Called with mu held. +func (c *rewriteCache) lookup(key string, now time.Time) (*rewriteEntry, bool) { + el, ok := c.entries[key] + if !ok { + return nil, false + } + e := el.Value.(*rewriteEntry) + if !e.expires.IsZero() && !now.Before(e.expires) { + c.remove(el) + return nil, false + } c.order.MoveToFront(el) - return el.Value.(*rewriteEntry).rewrittenMetadata, true + return e, true +} + +func (c *rewriteCache) clock() time.Time { + if c.now == nil { + return time.Now() + } + return c.now() +} + +// remove drops an entry. Called with mu held. +func (c *rewriteCache) remove(el *list.Element) { + e := el.Value.(*rewriteEntry) + c.order.Remove(el) + delete(c.entries, e.key) + c.size -= e.size() } // rewrite returns rewrite(in) with its ETag and gzip copy, from the cache -// when it can. Callers must treat the returned bytes as read-only: a cached -// result is shared. A caller waiting on another's rewrite leaves when ctx -// ends; the rewrite itself always runs to completion, so the result is cached -// for the next request. -func (c *rewriteCache) rewrite(ctx context.Context, key string, in []byte, rewrite func([]byte) ([]byte, error)) (rewrittenMetadata, error) { +// when it can. A ttl above zero makes the result expire after that long. +// Callers must treat the returned bytes as read-only: a cached result is +// shared. A caller waiting on another's rewrite leaves when ctx ends; the +// rewrite itself always runs to completion, so the result is cached for the +// next request. +func (c *rewriteCache) rewrite(ctx context.Context, key string, in []byte, ttl time.Duration, rewrite func([]byte) ([]byte, error)) (rewrittenMetadata, error) { if c == nil { return rewriteWithETag(in, rewrite) } c.mu.Lock() - if el, ok := c.entries[key]; ok { - c.order.MoveToFront(el) + if e, ok := c.lookup(key, c.clock()); ok { c.mu.Unlock() - return el.Value.(*rewriteEntry).rewrittenMetadata, nil + return e.rewrittenMetadata, nil } if f, ok := c.inFlight[key]; ok { c.mu.Unlock() @@ -196,11 +238,17 @@ func (c *rewriteCache) rewrite(ctx context.Context, key string, in []byte, rewri c.inFlight[key] = f c.mu.Unlock() + // The entry expires counting from before the rewrite, so it never + // outlives the time its cooldown decisions were made at by more than ttl. + var expires time.Time + if ttl > 0 { + expires = c.clock().Add(ttl) + } defer func() { c.mu.Lock() delete(c.inFlight, key) if f.err == nil { - c.add(&rewriteEntry{key: key, rewrittenMetadata: f.m}) + c.add(&rewriteEntry{key: key, rewrittenMetadata: f.m, expires: expires}) } c.mu.Unlock() close(f.done) @@ -243,45 +291,61 @@ func (c *rewriteCache) add(e *rewriteEntry) { if n > c.maxBytes { return } + if old, ok := c.entries[e.key]; ok { + c.remove(old) + } c.entries[e.key] = c.order.PushFront(e) c.size += n for c.size > c.maxBytes { - oldest := c.order.Back() - e := oldest.Value.(*rewriteEntry) - c.order.Remove(oldest) - delete(c.entries, e.key) - c.size -= e.size() + c.remove(c.order.Back()) } } // cachedRewrite rewrites a metadata document through the proxy's rewrite // cache and returns it with its validators. digest identifies in when the -// metadata cache stored it, and is empty otherwise. Cooldown filtering depends -// on the current time, so with cooldown on a cached rewrite could keep hiding -// a version past its cooldown; those rewrites always run, with no -// Last-Modified, and when digest is known their ETag comes from their inputs -// rather than from hashing each output again. Their ETag also changes with -// every restart, so replicas would never agree on a date for them. +// metadata cache stored it, and is empty otherwise; when known it keys the +// cache, so the document isn't hashed again. +// +// With cooldown on, the cached rewrite expires after cooldownRewriteTTL. Its +// ETag is the hash of its output, and it is dated like any other, so clients +// revalidate with either validator and get the gzip copy. +// +// Without the rewrite cache, cooldown rewrites run on every request and go +// undated: when digest is known their ETag comes from their inputs rather +// than from hashing each output again, and that ETag changes with every +// restart, so replicas would never agree on a date for it. func (p *Proxy) cachedRewrite(ctx context.Context, ecosystem, proxyURL, name string, in []byte, digest string, rewrite keepingRewrite) (rewrittenMetadata, error) { - if p.cooldownEnabled() && digest != "" { + rewriteOnly := func(b []byte) ([]byte, error) { + out, _, err := rewrite(b) + return out, err + } + cooldown := p.cooldownEnabled() + + if p.rewrites != nil { + key := rewriteCacheKeyForDigest(ecosystem, proxyURL, name, digest) + if digest == "" { + key = rewriteCacheKey(ecosystem, proxyURL, name, in) + } + var ttl time.Duration + if cooldown { + ttl = cooldownRewriteTTL + } + m, err := p.rewrites.rewrite(ctx, key, in, ttl, rewriteOnly) + if err == nil { + m.lastModified = p.metadataDate(ecosystem, proxyURL, name, m.etag) + } + return m, err + } + + if cooldown && digest != "" { out, kept, err := rewrite(in) if err != nil { return rewrittenMetadata{}, err } return rewrittenMetadata{body: out, metadataValidators: metadataValidators{etag: keptVersionsETag(digest, proxyURL, name, kept)}}, nil } - rewriteOnly := func(b []byte) ([]byte, error) { - out, _, err := rewrite(b) - return out, err - } - var m rewrittenMetadata - var err error - if p.rewrites == nil || p.cooldownEnabled() { - m, err = rewriteWithETag(in, rewriteOnly) - } else { - m, err = p.rewrites.rewrite(ctx, rewriteCacheKey(ecosystem, proxyURL, name, in), in, rewriteOnly) - } - if err == nil && !p.cooldownEnabled() { + m, err := rewriteWithETag(in, rewriteOnly) + if err == nil && !cooldown { m.lastModified = p.metadataDate(ecosystem, proxyURL, name, m.etag) } return m, err @@ -304,12 +368,13 @@ func (p *Proxy) cooldownEnabled() bool { // storedRewrite returns the cached rewrite of the metadata stored for // ecosystem and cacheKey, when that metadata is still within its TTL and a -// rewrite of it is cached. It reads only the cache row, never the stored -// document, so a repeated request for a large packument costs a database -// lookup instead of reading and hashing the whole document again. When it -// reports false the caller takes the usual fetch and rewrite path. +// rewrite of it is cached and has not expired. It reads only the cache row, +// never the stored document, so a repeated request for a large packument +// costs a database lookup instead of reading and hashing the whole document +// again. When it reports false the caller takes the usual fetch and rewrite +// path. func (p *Proxy) storedRewrite(ecosystem, cacheKey, proxyURL, name string) (rewrittenMetadata, bool) { - if p.rewrites == nil || p.cooldownEnabled() { + if p.rewrites == nil { return rewrittenMetadata{}, false } if !p.CacheMetadata || p.DB == nil || p.MetadataTTL <= 0 { @@ -414,4 +479,7 @@ func notModified(r *http.Request, etag string, lastModified time.Time) bool { // room for maxBytes of output. Zero or less disables it. func (p *Proxy) SetMetadataRewriteCacheSize(maxBytes int64) { p.rewrites = newRewriteCache(maxBytes) + if p.rewrites != nil { + p.rewrites.now = p.clock + } } diff --git a/internal/handler/rewrite_cache_test.go b/internal/handler/rewrite_cache_test.go index 5c2b842..373e845 100644 --- a/internal/handler/rewrite_cache_test.go +++ b/internal/handler/rewrite_cache_test.go @@ -1,6 +1,7 @@ package handler import ( + "bytes" "context" "errors" "net/http" @@ -9,6 +10,7 @@ import ( "sync" "sync/atomic" "testing" + "time" "github.com/git-pkgs/cooldown" ) @@ -42,7 +44,7 @@ func TestRewriteCache_RepeatServesCachedRewrite(t *testing.T) { key := rewriteCacheKey("npm", "http://proxy", "left-pad", []byte("doc")) for range 3 { - m, err := c.rewrite(context.Background(), key, []byte("doc"), r.fn) + m, err := c.rewrite(context.Background(), key, []byte("doc"), 0, r.fn) if err != nil || string(m.body) != "DOC" { t.Fatalf("rewrite = %q, %v", m.body, err) } @@ -58,7 +60,7 @@ func TestRewriteCache_NewUpstreamBytesRewriteAgain(t *testing.T) { for _, doc := range []string{"v1", "v2"} { key := rewriteCacheKey("npm", "http://proxy", "left-pad", []byte(doc)) - m, err := c.rewrite(context.Background(), key, []byte(doc), r.fn) + m, err := c.rewrite(context.Background(), key, []byte(doc), 0, r.fn) if err != nil || string(m.body) != strings.ToUpper(doc) { t.Fatalf("rewrite(%s) = %q, %v", doc, m.body, err) } @@ -80,7 +82,7 @@ func TestRewriteCache_ConcurrentRequestsShareRewrite(t *testing.T) { outs := make(chan string, n) run := func() { - m, err := c.rewrite(context.Background(), key, []byte("doc"), r.fn) + m, err := c.rewrite(context.Background(), key, []byte("doc"), 0, r.fn) if err != nil { t.Error(err) } @@ -113,20 +115,20 @@ func TestRewriteCache_WaiterLeavingKeepsRewrite(t *testing.T) { first := make(chan error, 1) go func() { - _, err := c.rewrite(context.Background(), key, []byte("doc"), r.fn) + _, err := c.rewrite(context.Background(), key, []byte("doc"), 0, r.fn) first <- err }() <-r.entered ctx, cancel := context.WithCancel(context.Background()) cancel() - if _, err := c.rewrite(ctx, key, []byte("doc"), r.fn); !errors.Is(err, context.Canceled) { + if _, err := c.rewrite(ctx, key, []byte("doc"), 0, r.fn); !errors.Is(err, context.Canceled) { t.Errorf("waiter err = %v, want context.Canceled", err) } close(r.release) if err := <-first; err != nil { t.Fatal(err) } - if m, err := c.rewrite(context.Background(), key, []byte("doc"), r.fn); err != nil || string(m.body) != "DOC" { + if m, err := c.rewrite(context.Background(), key, []byte("doc"), 0, r.fn); err != nil || string(m.body) != "DOC" { t.Errorf("cached rewrite = %q, %v", m.body, err) } if got := r.calls.Load(); got != 1 { @@ -140,18 +142,18 @@ func TestRewriteCache_EvictsLeastRecentlyUsed(t *testing.T) { keyFor := func(doc string) string { return rewriteCacheKey("npm", "http://proxy", doc, []byte(doc)) } for _, doc := range []string{"aaa", "bbb"} { - _, _ = c.rewrite(context.Background(), keyFor(doc), []byte(doc), r.fn) + _, _ = c.rewrite(context.Background(), keyFor(doc), []byte(doc), 0, r.fn) } - _, _ = c.rewrite(context.Background(), keyFor("aaa"), []byte("aaa"), r.fn) // aaa is now the most recent - _, _ = c.rewrite(context.Background(), keyFor("ccc"), []byte("ccc"), r.fn) // evicts bbb + _, _ = c.rewrite(context.Background(), keyFor("aaa"), []byte("aaa"), 0, r.fn) // aaa is now the most recent + _, _ = c.rewrite(context.Background(), keyFor("ccc"), []byte("ccc"), 0, r.fn) // evicts bbb before := r.calls.Load() - _, _ = c.rewrite(context.Background(), keyFor("aaa"), []byte("aaa"), r.fn) - _, _ = c.rewrite(context.Background(), keyFor("ccc"), []byte("ccc"), r.fn) + _, _ = c.rewrite(context.Background(), keyFor("aaa"), []byte("aaa"), 0, r.fn) + _, _ = c.rewrite(context.Background(), keyFor("ccc"), []byte("ccc"), 0, r.fn) if got := r.calls.Load() - before; got != 0 { t.Errorf("aaa and ccc rewritten %d times, want both still cached", got) } - _, _ = c.rewrite(context.Background(), keyFor("bbb"), []byte("bbb"), r.fn) + _, _ = c.rewrite(context.Background(), keyFor("bbb"), []byte("bbb"), 0, r.fn) if got := r.calls.Load() - before; got != 1 { t.Errorf("bbb should have been evicted and rewritten") } @@ -162,7 +164,7 @@ func TestRewriteCache_OutputLargerThanCacheIsNotStored(t *testing.T) { var r countingRewrite key := rewriteCacheKey("npm", "http://proxy", "big", []byte("big")) - _, _ = c.rewrite(context.Background(), key, []byte("big"), r.fn) + _, _ = c.rewrite(context.Background(), key, []byte("big"), 0, r.fn) if c.len() != 0 { t.Errorf("cached %d entries, want 0", c.len()) } @@ -175,7 +177,7 @@ func TestRewriteCache_ErrorsAreNotCached(t *testing.T) { key := rewriteCacheKey("npm", "http://proxy", "broken", []byte("doc")) for range 2 { - if _, err := c.rewrite(context.Background(), key, []byte("doc"), failing); err == nil { + if _, err := c.rewrite(context.Background(), key, []byte("doc"), 0, failing); err == nil { t.Fatal("expected the rewrite error") } } @@ -197,11 +199,14 @@ func TestCachedRewrite_DisabledRewritesEveryTime(t *testing.T) { } } -// TestCachedRewrite_CooldownBypassesCache asserts that with cooldown on every -// request is rewritten: cooldown filtering depends on the current time, so a -// cached rewrite could keep hiding a version after its cooldown ends. -func TestCachedRewrite_CooldownBypassesCache(t *testing.T) { +// TestCachedRewrite_CooldownExpires asserts that with cooldown on a rewrite +// is cached for cooldownRewriteTTL only: cooldown filtering depends on the +// current time, so a cached rewrite would otherwise keep hiding a version +// after its cooldown ends. +func TestCachedRewrite_CooldownExpires(t *testing.T) { proxy, _, _, _ := setupTestProxy(t) + clock := &testClock{t: time.Date(2026, 10, 10, 12, 0, 0, 0, time.UTC)} + proxy.now = clock.now proxy.SetMetadataRewriteCacheSize(1 << 20) proxy.Cooldown = &cooldown.Config{Default: "3d"} var r countingRewrite @@ -209,8 +214,33 @@ func TestCachedRewrite_CooldownBypassesCache(t *testing.T) { for range 2 { _, _ = proxy.cachedRewrite(context.Background(), "npm", "http://proxy", "left-pad", []byte("doc"), "", r.keeping) } + if got := r.calls.Load(); got != 1 { + t.Errorf("rewrites within the TTL = %d, want 1", got) + } + + clock.advance(cooldownRewriteTTL) + _, _ = proxy.cachedRewrite(context.Background(), "npm", "http://proxy", "left-pad", []byte("doc"), "", r.keeping) if got := r.calls.Load(); got != 2 { - t.Errorf("rewrites = %d, want 2", got) + t.Errorf("rewrites after the TTL = %d, want 2", got) + } +} + +// TestRewriteCacheExpiredEntryFreesRoom asserts that an expired entry is +// dropped when looked up, so its size no longer counts. +func TestRewriteCacheExpiredEntryFreesRoom(t *testing.T) { + c := newRewriteCache(1 << 20) + clock := &testClock{t: time.Date(2026, 10, 10, 12, 0, 0, 0, time.UTC)} + c.now = clock.now + upper := func(b []byte) ([]byte, error) { return bytes.ToUpper(b), nil } + if _, err := c.rewrite(context.Background(), "k", []byte("doc"), time.Minute, upper); err != nil { + t.Fatal(err) + } + clock.advance(time.Minute) + if _, ok := c.get("k"); ok { + t.Fatal("expired entry was served") + } + if c.size != 0 || c.order.Len() != 0 { + t.Errorf("size %d, entries %d after expiry; want 0", c.size, c.order.Len()) } } From d02ad9b2bac1b270c22986f4d15a3dbf7d771e75 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Fri, 23 Oct 2026 16:15:24 +0200 Subject: [PATCH 367/633] Coordinate instances sharing a Postgres database Instances started together on an empty database created the schema at the same time and all but one failed on duplicate types. Schema creation and migrations now run under a Postgres advisory lock. Eviction, reclaim, retention and the Gradle build cache sweep take a per-job advisory lock for each pass, and an instance that finds the job running elsewhere skips the pass. Before, every instance evicted the whole overshoot over max_size at once and emptied the cache. Mirror schedules record their last start in a new job_runs table, so each schedule runs once per interval instead of once per instance. Refs #543 --- deploy/charts/proxy/values.yaml | 9 +- docs/configuration.md | 14 +- internal/database/database.go | 27 ++- internal/database/database_test.go | 2 +- internal/database/lock.go | 98 ++++++++++ internal/database/lock_test.go | 239 +++++++++++++++++++++++ internal/database/schema.go | 39 ++++ internal/mirror/schedule.go | 37 ++++ internal/mirror/schedule_test.go | 56 ++++++ internal/server/eviction.go | 4 +- internal/server/exclusive.go | 21 ++ internal/server/exclusive_test.go | 60 ++++++ internal/server/gradle_cache_eviction.go | 3 +- internal/server/mirror_schedule.go | 1 + internal/server/reclaim.go | 4 +- internal/server/retention.go | 10 +- 16 files changed, 603 insertions(+), 21 deletions(-) create mode 100644 internal/database/lock.go create mode 100644 internal/database/lock_test.go create mode 100644 internal/server/exclusive.go create mode 100644 internal/server/exclusive_test.go diff --git a/deploy/charts/proxy/values.yaml b/deploy/charts/proxy/values.yaml index 9a74e47..cb7a0bf 100644 --- a/deploy/charts/proxy/values.yaml +++ b/deploy/charts/proxy/values.yaml @@ -1,7 +1,8 @@ -# More than one replica requires config.data.database on Postgres, -# config.data.storage on object storage, deploymentStrategy: RollingUpdate, -# and a PVC access mode other than ReadWriteOnce (or persistence disabled). -# With upstream.git, each replica also needs its own git.cache_dir. +# More than one replica requires config.data.database on Postgres and +# deploymentStrategy: RollingUpdate. Storage can be object storage (with +# persistence disabled) or a file:// URL on a ReadWriteMany volume +# (persistence.accessModes: [ReadWriteMany]). With upstream.git, each +# replica also needs its own git.cache_dir. replicaCount: 1 nameOverride: "" diff --git a/docs/configuration.md b/docs/configuration.md index 4da6f63..875f8fc 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -197,6 +197,18 @@ database: |--------|-------------|------|-------------| | `database.url` | `PROXY_DATABASE_URL` | `-database-url` | PostgreSQL connection URL | +### Several instances + +Several instances can serve one cache when they share a PostgreSQL database and storage. The storage can be S3 or a `file://` directory on a volume every instance mounts, such as NFS, CephFS or a Kubernetes `ReadWriteMany` volume. Each fetch writes its own file and renames it into place, and records are updated with compare-and-swap, so concurrent fetches and a crashed instance leave no partial files or records that don't match a file. SQLite only supports a single instance. + +Instances coordinate through PostgreSQL advisory locks: + +- Instances started together create the schema and run migrations one after the other. +- Eviction, reclaim, retention and the Gradle build cache sweep run on one instance at a time. An instance that finds a job running elsewhere skips that pass. +- A [scheduled mirror](#scheduled-refresh) runs on whichever instance reaches it first in each interval. + +Cache misses are only coalesced within an instance. Two instances missing the same file both fetch it from upstream; the extra copy is queued for deletion like any replaced file. Each instance also needs its own [`git.cache_dir`](#git-repositories). + ### Hit counts Every cache hit updates the artifact's hit count and last-access time, which the stats pages and LRU eviction use. Rather than writing each hit as its own transaction, the proxy counts hits in memory and writes them together every `hit_flush_interval`. SQLite allows one writer at a time, and the proxy uses a single SQLite connection, so with a write per hit, concurrent downloads queue behind each other. @@ -1496,7 +1508,7 @@ mirror: Each run checks the tag with a HEAD request. If the tag still points to the cached digest, nothing else is fetched. If it moved, the new manifests and blobs are fetched first and the tag is switched only when all of them are cached, so a failed run leaves the previous image pullable. Images of one schedule are mirrored one after the other. -- The first run starts within ten minutes of startup, later runs every `interval` ±10 %, so restarts and several instances don't hit the registry at once. Each instance runs the schedules on its own. +- The first run starts within ten minutes of startup, later runs every `interval` ±10 %, so restarts and several instances don't hit the registry at once. Instances sharing a PostgreSQL database run each schedule once per interval: the first to reach it records the run, and the others skip it. - Digest-pinned PURLs never move and are rejected. A `pkg:oci` digest with `tag=` is accepted; the tag is refreshed and a warning is logged when it no longer matches the digest. - The previous digest stays cached, so pulls by digest keep working, until eviction or retention removes it. - Docker Hub does not count HEAD requests against its pull limit, but every changed manifest is a pull. Set `platforms` and `upstream.auth` for `registry-1.docker.io` to stay under the anonymous limit. diff --git a/internal/database/database.go b/internal/database/database.go index f438f67..5a7e127 100644 --- a/internal/database/database.go +++ b/internal/database/database.go @@ -1,6 +1,7 @@ package database import ( + "context" "fmt" "os" "path/filepath" @@ -125,18 +126,24 @@ func OpenPostgresOrCreate(url string) (*DB, error) { return nil, err } - var exists bool - err = db.Get(&exists, "SELECT EXISTS (SELECT FROM information_schema.tables WHERE table_name = 'schema_info')") + // Instances started together on an empty database would otherwise + // create the schema at the same time and fail on each other's tables. + err = db.WithLock(context.Background(), schemaLock, func() error { + var exists bool + if err := db.Get(&exists, "SELECT EXISTS (SELECT FROM information_schema.tables WHERE table_name = 'schema_info')"); err != nil { + return fmt.Errorf("checking schema: %w", err) + } + if exists { + return nil + } + if err := db.CreateSchema(); err != nil { + return fmt.Errorf("creating schema: %w", err) + } + return nil + }) if err != nil { _ = db.Close() - return nil, fmt.Errorf("checking schema: %w", err) - } - - if !exists { - if err := db.CreateSchema(); err != nil { - _ = db.Close() - return nil, fmt.Errorf("creating schema: %w", err) - } + return nil, err } return db, nil diff --git a/internal/database/database_test.go b/internal/database/database_test.go index f09a86a..e5b9899 100644 --- a/internal/database/database_test.go +++ b/internal/database/database_test.go @@ -613,7 +613,7 @@ func createTestPostgresDB(t *testing.T) *DB { // Drop and recreate every table CreateSchema creates for clean test state; // leftover migration records make the next CreateSchema fail on the // migrations primary key. - tables := []string{"artifacts", "pending_deletes", "versions", "packages", "vulnerabilities", "metadata_cache", "metadata_dates", "migrations", "schema_info"} + tables := []string{"artifacts", "pending_deletes", "versions", "packages", "vulnerabilities", "metadata_cache", "metadata_dates", "job_runs", "migrations", "schema_info"} for _, table := range tables { _, _ = db.Exec("DROP TABLE IF EXISTS " + table + " CASCADE") } diff --git a/internal/database/lock.go b/internal/database/lock.go new file mode 100644 index 0000000..b245c62 --- /dev/null +++ b/internal/database/lock.go @@ -0,0 +1,98 @@ +package database + +import ( + "context" + "database/sql" + "database/sql/driver" + "fmt" + "hash/fnv" + "time" +) + +// Several instances can share one Postgres database. Advisory locks keep +// them from creating the schema, migrating it or running a background job at +// the same time. SQLite is single-instance, so there every lock is granted +// at once. + +// lockKey maps a lock name to the 64-bit key Postgres advisory locks take. +func lockKey(name string) int64 { + h := fnv.New64a() + _, _ = h.Write([]byte("git-pkgs/proxy:" + name)) + return int64(h.Sum64()) //nolint:gosec // a key, wrapping is fine +} + +// WithLock runs fn while holding the named lock, waiting for another +// instance that holds it. +func (db *DB) WithLock(ctx context.Context, name string, fn func() error) error { + if db.dialect != DialectPostgres { + return fn() + } + conn, err := db.Conn(ctx) + if err != nil { + return fmt.Errorf("taking connection for lock %s: %w", name, err) + } + defer func() { _ = conn.Close() }() + if _, err := conn.ExecContext(ctx, "SELECT pg_advisory_lock($1)", lockKey(name)); err != nil { + return fmt.Errorf("taking lock %s: %w", name, err) + } + defer unlock(conn, name) + return fn() +} + +// TryLock takes the named lock if no other instance holds it. When ok is +// true the caller must call release once done. The lock lives in a +// connection held for that time, and Postgres frees it if the process dies. +func (db *DB) TryLock(ctx context.Context, name string) (release func(), ok bool, err error) { + if db.dialect != DialectPostgres { + return func() {}, true, nil + } + conn, err := db.Conn(ctx) + if err != nil { + return nil, false, fmt.Errorf("taking connection for lock %s: %w", name, err) + } + if err := conn.QueryRowContext(ctx, "SELECT pg_try_advisory_lock($1)", lockKey(name)).Scan(&ok); err != nil { + _ = conn.Close() + return nil, false, fmt.Errorf("taking lock %s: %w", name, err) + } + if !ok { + _ = conn.Close() + return nil, false, nil + } + return func() { + unlock(conn, name) + _ = conn.Close() + }, true, nil +} + +// unlockTimeout bounds the unlock. If it fails, closing the connection still +// ends the session that holds the lock. +const unlockTimeout = 10 * time.Second + +func unlock(conn *sql.Conn, name string) { + ctx, cancel := context.WithTimeout(context.Background(), unlockTimeout) + defer cancel() + if _, err := conn.ExecContext(ctx, "SELECT pg_advisory_unlock($1)", lockKey(name)); err != nil { + // Drop the connection so the session, and with it the lock, ends. + _ = conn.Raw(func(any) error { return driver.ErrBadConn }) + } +} + +// ClaimJobRun records that the named job starts now unless it started less +// than minGap ago, and reports whether the caller should run it. Instances +// sharing the database each keep their own timer for a scheduled job, and +// this lets only the first of them run it per interval. +func (db *DB) ClaimJobRun(name string, minGap time.Duration) (bool, error) { + now := time.Now().UTC() + res, err := db.Exec(db.Rebind(` + INSERT INTO job_runs (name, started_at) VALUES (?, ?) + ON CONFLICT(name) DO UPDATE SET started_at = excluded.started_at + WHERE job_runs.started_at <= ?`), name, now, now.Add(-minGap)) + if err != nil { + return false, fmt.Errorf("claiming job run %s: %w", name, err) + } + n, err := res.RowsAffected() + if err != nil { + return false, fmt.Errorf("claiming job run %s: %w", name, err) + } + return n > 0, nil +} diff --git a/internal/database/lock_test.go b/internal/database/lock_test.go new file mode 100644 index 0000000..c49906d --- /dev/null +++ b/internal/database/lock_test.go @@ -0,0 +1,239 @@ +package database + +import ( + "context" + "net/url" + "os" + "strings" + "sync" + "sync/atomic" + "testing" + "time" +) + +func TestTryLockSQLiteAlwaysGranted(t *testing.T) { + db := createTestDB(t) + release, ok, err := db.TryLock(context.Background(), "job:test") + if err != nil || !ok { + t.Fatalf("TryLock = %v, %v; want granted", ok, err) + } + // SQLite is single-instance, so a second holder is granted too. + release2, ok, err := db.TryLock(context.Background(), "job:test") + if err != nil || !ok { + t.Fatalf("second TryLock = %v, %v; want granted", ok, err) + } + release2() + release() +} + +func TestClaimJobRun(t *testing.T) { + forEachDialect(t, func(t *testing.T, db *DB) { + claimed, err := db.ClaimJobRun("mirror-schedule:a", time.Hour) + if err != nil || !claimed { + t.Fatalf("first claim = %v, %v; want claimed", claimed, err) + } + claimed, err = db.ClaimJobRun("mirror-schedule:a", time.Hour) + if err != nil || claimed { + t.Fatalf("claim within the gap = %v, %v; want not claimed", claimed, err) + } + claimed, err = db.ClaimJobRun("mirror-schedule:b", time.Hour) + if err != nil || !claimed { + t.Fatalf("claim of another job = %v, %v; want claimed", claimed, err) + } + // A run started longer ago than the gap can be claimed again. + claimed, err = db.ClaimJobRun("mirror-schedule:a", 0) + if err != nil || !claimed { + t.Fatalf("claim after the gap = %v, %v; want claimed", claimed, err) + } + }) +} + +// forEachDialect runs fn against SQLite and, when PROXY_DATABASE_URL is set, +// against Postgres. +func forEachDialect(t *testing.T, fn func(t *testing.T, db *DB)) { + t.Run("sqlite", func(t *testing.T) { fn(t, createTestDB(t)) }) + t.Run("postgres", func(t *testing.T) { + db := createTestPostgresDB(t) + if db == nil { + t.Skip("PROXY_DATABASE_URL not set") + } + defer func() { _ = db.Close() }() + fn(t, db) + }) +} + +func openTestPostgres(t *testing.T) *DB { + t.Helper() + dbURL := os.Getenv("PROXY_DATABASE_URL") + if dbURL == "" { + t.Skip("PROXY_DATABASE_URL not set") + } + db, err := OpenPostgres(dbURL) + if err != nil { + t.Fatalf("OpenPostgres: %v", err) + } + t.Cleanup(func() { _ = db.Close() }) + return db +} + +func TestTryLockPostgresExcludesOtherInstance(t *testing.T) { + a := openTestPostgres(t) + b := openTestPostgres(t) + ctx := context.Background() + + release, ok, err := a.TryLock(ctx, "job:test-exclusive") + if err != nil || !ok { + t.Fatalf("TryLock on a = %v, %v; want granted", ok, err) + } + if _, ok, err := b.TryLock(ctx, "job:test-exclusive"); err != nil || ok { + t.Fatalf("TryLock on b while a holds it = %v, %v; want refused", ok, err) + } + // Another lock name is independent. + releaseOther, ok, err := b.TryLock(ctx, "job:test-other") + if err != nil || !ok { + t.Fatalf("TryLock on another name = %v, %v; want granted", ok, err) + } + releaseOther() + + release() + release, ok, err = b.TryLock(ctx, "job:test-exclusive") + if err != nil || !ok { + t.Fatalf("TryLock on b after a released = %v, %v; want granted", ok, err) + } + release() +} + +func TestTryLockPostgresFreedWhenSessionEnds(t *testing.T) { + a := openTestPostgres(t) + b := openTestPostgres(t) + ctx := context.Background() + + if _, ok, err := a.TryLock(ctx, "job:test-crash"); err != nil || !ok { + t.Fatalf("TryLock on a = %v, %v; want granted", ok, err) + } + // End a's session holding the lock, as a crash would. + if _, err := b.Exec(`SELECT pg_terminate_backend(pid) FROM pg_locks + WHERE locktype = 'advisory' AND pid <> pg_backend_pid()`); err != nil { + t.Fatalf("terminating a's session: %v", err) + } + waitUntil(t, func() bool { + release, ok, err := b.TryLock(ctx, "job:test-crash") + if ok { + release() + } + return err == nil && ok + }) +} + +func TestWithLockPostgresSerialises(t *testing.T) { + const instances = 4 + dbs := make([]*DB, instances) + for i := range dbs { + dbs[i] = openTestPostgres(t) + } + var inside, overlapped atomic.Int32 + var wg sync.WaitGroup + for _, db := range dbs { + wg.Go(func() { + err := db.WithLock(context.Background(), "test-serialise", func() error { + if inside.Add(1) > 1 { + overlapped.Store(1) + } + time.Sleep(20 * time.Millisecond) + inside.Add(-1) + return nil + }) + if err != nil { + t.Errorf("WithLock: %v", err) + } + }) + } + wg.Wait() + if overlapped.Load() != 0 { + t.Error("two instances held the lock at once") + } +} + +// TestOpenPostgresOrCreateConcurrent starts several instances at once on an +// empty database. Without the schema lock all but one failed on each +// other's tables. +func TestOpenPostgresOrCreateConcurrent(t *testing.T) { + admin := openTestPostgres(t) + name := "proxy_lock_test_" + strings.ReplaceAll(time.Now().UTC().Format("150405.000000"), ".", "_") + if _, err := admin.Exec("CREATE DATABASE " + name); err != nil { + t.Skipf("cannot create a scratch database: %v", err) + } + t.Cleanup(func() { _, _ = admin.Exec("DROP DATABASE IF EXISTS " + name + " WITH (FORCE)") }) + + u, err := url.Parse(os.Getenv("PROXY_DATABASE_URL")) + if err != nil { + t.Fatalf("parsing PROXY_DATABASE_URL: %v", err) + } + u.Path = "/" + name + scratch := u.String() + + const instances = 5 + errs := make([]error, instances) + var wg sync.WaitGroup + start := make(chan struct{}) + for i := range instances { + wg.Go(func() { + <-start + db, err := OpenPostgresOrCreate(scratch) + if err == nil { + err = db.MigrateSchema() + _ = db.Close() + } + errs[i] = err + }) + } + close(start) + wg.Wait() + for i, err := range errs { + if err != nil { + t.Errorf("instance %d: %v", i, err) + } + } + + db, err := OpenPostgres(scratch) + if err != nil { + t.Fatalf("OpenPostgres: %v", err) + } + defer func() { _ = db.Close() }() + var n int + if err := db.Get(&n, "SELECT COUNT(*) FROM schema_info"); err != nil || n != 1 { + t.Errorf("schema_info rows = %d, %v; want 1", n, err) + } + if err := db.Get(&n, "SELECT COUNT(*) FROM migrations"); err != nil || n != len(migrations) { + t.Errorf("migrations rows = %d, %v; want %d", n, err, len(migrations)) + } +} + +func waitUntil(t *testing.T, cond func() bool) { + t.Helper() + deadline := time.Now().Add(5 * time.Second) + for !cond() { + if time.Now().After(deadline) { + t.Fatal("timed out") + } + time.Sleep(10 * time.Millisecond) + } +} + +func TestMigrateAddsJobRuns(t *testing.T) { + forEachDialect(t, func(t *testing.T, db *DB) { + // A database from before the job_runs migration. + if _, err := db.Exec("DROP TABLE job_runs"); err != nil { + t.Fatalf("dropping job_runs: %v", err) + } + if _, err := db.Exec(db.Rebind("DELETE FROM migrations WHERE name = ?"), "015_add_job_runs"); err != nil { + t.Fatalf("forgetting the migration: %v", err) + } + if err := db.MigrateSchema(); err != nil { + t.Fatalf("MigrateSchema: %v", err) + } + if claimed, err := db.ClaimJobRun("mirror-schedule:a", time.Hour); err != nil || !claimed { + t.Fatalf("ClaimJobRun after migration = %v, %v; want claimed", claimed, err) + } + }) +} diff --git a/internal/database/schema.go b/internal/database/schema.go index dc9fc80..7c1922b 100644 --- a/internal/database/schema.go +++ b/internal/database/schema.go @@ -1,6 +1,7 @@ package database import ( + "context" "fmt" "strings" "time" @@ -86,6 +87,11 @@ CREATE TABLE IF NOT EXISTS pending_deletes ( ); CREATE INDEX IF NOT EXISTS idx_pending_deletes_queued_at ON pending_deletes(queued_at); +CREATE TABLE IF NOT EXISTS job_runs ( + name TEXT NOT NULL PRIMARY KEY, + started_at DATETIME NOT NULL +); + CREATE TABLE IF NOT EXISTS vulnerabilities ( id INTEGER PRIMARY KEY, vuln_id TEXT NOT NULL, @@ -210,6 +216,11 @@ CREATE TABLE IF NOT EXISTS pending_deletes ( ); CREATE INDEX IF NOT EXISTS idx_pending_deletes_queued_at ON pending_deletes(queued_at); +CREATE TABLE IF NOT EXISTS job_runs ( + name TEXT NOT NULL PRIMARY KEY, + started_at TIMESTAMP NOT NULL +); + CREATE TABLE IF NOT EXISTS vulnerabilities ( id SERIAL PRIMARY KEY, vuln_id TEXT NOT NULL, @@ -420,6 +431,7 @@ var migrations = []migration{ {"012_add_artifact_source", migrateAddArtifactSource}, {"013_add_artifact_manifest", migrateAddArtifactManifest}, {"014_add_metadata_last_accessed", migrateAddMetadataLastAccessed}, + {"015_add_job_runs", migrateAddJobRuns}, } // isTableNotFound returns true if the error indicates a missing table. @@ -489,9 +501,19 @@ func (db *DB) recordAllMigrations() error { return nil } +// schemaLock serialises schema creation and migrations across instances +// sharing a Postgres database. +const schemaLock = "schema" + // MigrateSchema applies any unapplied migrations in order. // For a fully migrated database this executes a single SELECT query. +// Instances sharing a Postgres database run it one after the other, so each +// sees the migrations the one before it recorded. func (db *DB) MigrateSchema() error { + return db.WithLock(context.Background(), schemaLock, db.migrateSchema) +} + +func (db *DB) migrateSchema() error { applied, err := db.appliedMigrations() if err != nil { return err @@ -709,6 +731,23 @@ func migrateAddPendingDeletes(db *DB) error { return nil } +// migrateAddJobRuns creates the table that records when a scheduled job +// last started, so instances sharing the database run it once per interval. +func migrateAddJobRuns(db *DB) error { + ts := sqliteDatetime + if db.dialect == DialectPostgres { + ts = postgresTimestamp + } + query := fmt.Sprintf(`CREATE TABLE IF NOT EXISTS job_runs ( + name TEXT NOT NULL PRIMARY KEY, + started_at %s NOT NULL + )`, ts) + if _, err := db.Exec(query); err != nil { + return fmt.Errorf("creating job_runs table: %w", err) + } + return nil +} + // migrateAddPendingDeletesQueuedAtIndex indexes the column reclaim orders // the queue by, which grows large after a retention sweep. func migrateAddPendingDeletesQueuedAtIndex(db *DB) error { diff --git a/internal/mirror/schedule.go b/internal/mirror/schedule.go index 4ab0d2b..55d6ab4 100644 --- a/internal/mirror/schedule.go +++ b/internal/mirror/schedule.go @@ -2,6 +2,9 @@ package mirror import ( "context" + "crypto/sha256" + "encoding/hex" + "fmt" "log/slog" "math/rand/v2" "sync" @@ -27,6 +30,11 @@ type Scheduler struct { schedules []Schedule logger *slog.Logger wg sync.WaitGroup + // Claim, when set, reports whether this instance should run a schedule + // that no instance sharing the database started within minGap. Each + // instance keeps its own timer, so without it every instance would run + // every schedule. + Claim func(key string, minGap time.Duration) (bool, error) // randN returns a value in [0, n); tests replace it. randN func(n int64) int64 } @@ -75,6 +83,20 @@ func (s *Scheduler) loop(ctx context.Context, index int, schedule Schedule) { } func (s *Scheduler) run(ctx context.Context, index int, schedule Schedule) { + if s.Claim != nil { + // Later runs come at least interval - 2*spread apart, so a run + // started more recently than that was another instance's. + minGap := schedule.Interval - 2*(schedule.Interval/scheduleJitter) //nolint:mnd // two spreads of margin + claimed, err := s.Claim(scheduleKey(schedule), minGap) + if err != nil { + s.logger.Warn("mirror schedule: failed to claim run, skipping it", "schedule", index, "error", err) + return + } + if !claimed { + s.logger.Debug("mirror schedule: another instance ran it, skipping", "schedule", index) + return + } + } started := time.Now() progress, err := s.mirror.Run(ctx, &PURLSource{PURLs: schedule.PURLs, Platforms: schedule.Platforms}) if ctx.Err() != nil { @@ -103,3 +125,18 @@ func scheduleDelay(interval time.Duration, first bool, randN func(int64) int64) spread := interval / scheduleJitter return interval - spread + time.Duration(randN(int64(2*spread)+1)) //nolint:mnd // spread on both sides } + +// scheduleKey names a schedule by what it mirrors and how often, so +// instances with the same configuration agree on it and a changed schedule +// does not inherit the old one's last run. +func scheduleKey(schedule Schedule) string { + h := sha256.New() + for _, purl := range schedule.PURLs { + _, _ = fmt.Fprintf(h, "purl %s\n", purl) + } + for _, platform := range schedule.Platforms { + _, _ = fmt.Fprintf(h, "platform %s\n", platform) + } + _, _ = fmt.Fprintf(h, "interval %s\n", schedule.Interval) + return "mirror-schedule:" + hex.EncodeToString(h.Sum(nil))[:32] +} diff --git a/internal/mirror/schedule_test.go b/internal/mirror/schedule_test.go index c869c8a..53bdea2 100644 --- a/internal/mirror/schedule_test.go +++ b/internal/mirror/schedule_test.go @@ -175,3 +175,59 @@ func TestSchedulerSkipsScheduleWithoutInterval(t *testing.T) { t.Errorf("calls = %d, want 0", got) } } + +func TestSchedulerSkipsRunClaimedElsewhere(t *testing.T) { + images := &scheduleImages{} + schedule := Schedule{PURLs: []string{"pkg:docker/alpine@3"}, Interval: time.Hour} + s := newTestScheduler(images, schedule) + var mu sync.Mutex + var keys []string + var gaps []time.Duration + claim := true + s.Claim = func(key string, minGap time.Duration) (bool, error) { + mu.Lock() + defer mu.Unlock() + keys = append(keys, key) + gaps = append(gaps, minGap) + return claim, nil + } + + s.run(context.Background(), 0, schedule) + if got := images.calls.Load(); got != 1 { + t.Fatalf("calls after a claimed run = %d, want 1", got) + } + claim = false + s.run(context.Background(), 0, schedule) + if got := images.calls.Load(); got != 1 { + t.Fatalf("calls after a run claimed elsewhere = %d, want 1", got) + } + s.Claim = func(string, time.Duration) (bool, error) { return false, errors.New("database down") } + s.run(context.Background(), 0, schedule) + if got := images.calls.Load(); got != 1 { + t.Fatalf("calls after a failed claim = %d, want 1", got) + } + + if keys[0] != keys[1] { + t.Errorf("claim keys differ between runs: %q, %q", keys[0], keys[1]) + } + // Runs come at least 0.9 intervals apart; the gap stays below that. + if want := 48 * time.Minute; gaps[0] != want { + t.Errorf("minGap = %s, want %s", gaps[0], want) + } +} + +func TestScheduleKey(t *testing.T) { + base := Schedule{PURLs: []string{"pkg:docker/alpine@3"}, Platforms: []string{"linux/amd64"}, Interval: time.Hour} + if scheduleKey(base) != scheduleKey(Schedule{PURLs: []string{"pkg:docker/alpine@3"}, Platforms: []string{"linux/amd64"}, Interval: time.Hour}) { + t.Error("same schedule gave different keys") + } + for name, other := range map[string]Schedule{ + "purls": {PURLs: []string{"pkg:docker/alpine@4"}, Platforms: base.Platforms, Interval: base.Interval}, + "platforms": {PURLs: base.PURLs, Platforms: []string{"linux/arm64"}, Interval: base.Interval}, + "interval": {PURLs: base.PURLs, Platforms: base.Platforms, Interval: 2 * time.Hour}, + } { + if scheduleKey(other) == scheduleKey(base) { + t.Errorf("changing %s kept the key", name) + } + } +} diff --git a/internal/server/eviction.go b/internal/server/eviction.go index 5bbae41..66887d1 100644 --- a/internal/server/eviction.go +++ b/internal/server/eviction.go @@ -39,7 +39,9 @@ func (s *Server) startEvictionLoop(ctx context.Context) { } func (s *Server) runEviction(ctx context.Context, maxSize int64) { - evictLRU(ctx, s.db, s.storage, s.logger, maxSize) + s.runExclusive(ctx, "eviction", func() { + evictLRU(ctx, s.db, s.storage, s.logger, maxSize) + }) } // evictLRU removes the least recently used cached artifacts and metadata diff --git a/internal/server/exclusive.go b/internal/server/exclusive.go new file mode 100644 index 0000000..5067afd --- /dev/null +++ b/internal/server/exclusive.go @@ -0,0 +1,21 @@ +package server + +import "context" + +// runExclusive runs one pass of a background job unless another instance +// sharing the database is running it. That instance does the work, so this +// pass is skipped rather than waited for. Jobs that act on shared state, such +// as eviction against max_size, would otherwise each do all of it. +func (s *Server) runExclusive(ctx context.Context, job string, run func()) { + release, ok, err := s.db.TryLock(ctx, "job:"+job) + if err != nil { + s.logger.Warn("background job: failed to take lock, skipping this pass", "job", job, "error", err) + return + } + if !ok { + s.logger.Debug("background job: running on another instance, skipping this pass", "job", job) + return + } + defer release() + run() +} diff --git a/internal/server/exclusive_test.go b/internal/server/exclusive_test.go new file mode 100644 index 0000000..abd506b --- /dev/null +++ b/internal/server/exclusive_test.go @@ -0,0 +1,60 @@ +package server + +import ( + "context" + "io" + "log/slog" + "os" + "path/filepath" + "testing" + + "github.com/git-pkgs/proxy/internal/database" +) + +func TestRunExclusiveSQLiteRuns(t *testing.T) { + db, err := database.Create(filepath.Join(t.TempDir(), "test.db")) + if err != nil { + t.Fatalf("creating database: %v", err) + } + defer func() { _ = db.Close() }() + s := &Server{db: db, logger: slog.New(slog.NewTextHandler(io.Discard, nil))} + + ran := false + s.runExclusive(context.Background(), "eviction", func() { ran = true }) + if !ran { + t.Error("job did not run on SQLite") + } +} + +// TestRunExclusiveSkipsJobRunningElsewhere checks that a pass is skipped +// while another instance sharing the Postgres database runs the job, and +// runs once that instance is done. +func TestRunExclusiveSkipsJobRunningElsewhere(t *testing.T) { + url := os.Getenv("PROXY_DATABASE_URL") + if url == "" { + t.Skip("PROXY_DATABASE_URL not set") + } + open := func() *Server { + db, err := database.OpenPostgres(url) + if err != nil { + t.Fatalf("OpenPostgres: %v", err) + } + t.Cleanup(func() { _ = db.Close() }) + return &Server{db: db, logger: slog.New(slog.NewTextHandler(io.Discard, nil))} + } + a, b := open(), open() + + ranOnB := 0 + a.runExclusive(context.Background(), "eviction", func() { + b.runExclusive(context.Background(), "eviction", func() { ranOnB++ }) + // A different job is not held up. + b.runExclusive(context.Background(), "retention", func() { ranOnB += 10 }) + }) + if ranOnB != 10 { + t.Fatalf("while a ran eviction, b ran %d (want only retention, 10)", ranOnB) + } + b.runExclusive(context.Background(), "eviction", func() { ranOnB++ }) + if ranOnB != 11 { + t.Errorf("after a finished, b's eviction did not run") + } +} diff --git a/internal/server/gradle_cache_eviction.go b/internal/server/gradle_cache_eviction.go index 7f546d1..10980bb 100644 --- a/internal/server/gradle_cache_eviction.go +++ b/internal/server/gradle_cache_eviction.go @@ -34,7 +34,7 @@ func (s *Server) startGradleBuildCacheEviction(ctx context.Context) { "max_size_bytes", maxSize, "interval", interval) - sweep := func() { + sweepOnce := func() { deletedCount, freedBytes, err := sweepGradleBuildCache(ctx, s.storage, lister, maxAge, maxSize, time.Now()) if err != nil { s.logger.Warn("gradle cache eviction sweep failed", "error", err) @@ -47,6 +47,7 @@ func (s *Server) startGradleBuildCacheEviction(ctx context.Context) { } } + sweep := func() { s.runExclusive(ctx, "gradle-cache-eviction", sweepOnce) } sweep() go func() { diff --git a/internal/server/mirror_schedule.go b/internal/server/mirror_schedule.go index 3f69e7f..fb32f24 100644 --- a/internal/server/mirror_schedule.go +++ b/internal/server/mirror_schedule.go @@ -24,6 +24,7 @@ func (s *Server) startMirrorSchedules(ctx context.Context, proxy *handler.Proxy) schedules[i] = mirror.Schedule{PURLs: schedule.Images, Platforms: schedule.Platforms, Interval: schedule.IntervalDuration()} } s.scheduler = mirror.NewScheduler(m, schedules, s.logger) + s.scheduler.Claim = s.db.ClaimJobRun s.scheduler.Start(ctx) s.logger.Info("mirror schedules enabled", "schedules", len(schedules)) } diff --git a/internal/server/reclaim.go b/internal/server/reclaim.go index aa986a1..9437023 100644 --- a/internal/server/reclaim.go +++ b/internal/server/reclaim.go @@ -34,7 +34,9 @@ func (s *Server) startReclaimLoop(ctx context.Context) { case <-ctx.Done(): return case <-ticker.C: - reclaimDue(ctx, s.db, s.storage, s.logger, time.Now().Add(-grace), reclaimTickBudget) + s.runExclusive(ctx, "reclaim", func() { + reclaimDue(ctx, s.db, s.storage, s.logger, time.Now().Add(-grace), reclaimTickBudget) + }) } } } diff --git a/internal/server/retention.go b/internal/server/retention.go index 9a2e9fb..520b4cc 100644 --- a/internal/server/retention.go +++ b/internal/server/retention.go @@ -47,18 +47,24 @@ func (s *Server) startRetentionLoop(ctx context.Context) { ticker := time.NewTicker(s.cfg.ParseRetentionSweepInterval()) defer ticker.Stop() - sweepRetention(ctx, s.db, s.logger, reg, rules, time.Now()) + s.sweepRetention(ctx, reg, rules) for { select { case <-ctx.Done(): return case <-ticker.C: - sweepRetention(ctx, s.db, s.logger, reg, rules, time.Now()) + s.sweepRetention(ctx, reg, rules) } } } +func (s *Server) sweepRetention(ctx context.Context, reg *retention.Registry, rules retention.Rules) { + s.runExclusive(ctx, "retention", func() { + sweepRetention(ctx, s.db, s.logger, reg, rules, time.Now()) + }) +} + // notCovered returns the known ecosystem keys the global default cannot reach. func notCovered(covered []string) []string { var missing []string From b907ef0abf5ad2575327f771aecdbb71c690f2f1 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Fri, 23 Oct 2026 16:23:54 +0200 Subject: [PATCH 368/633] Run the Postgres tests in CI The tests gated on PROXY_DATABASE_URL were always skipped in CI. A new job runs them against a Postgres 17 service container. --- .github/workflows/ci.yml | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 64ecd80..c8d0bc7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -31,6 +31,40 @@ jobs: - name: Test run: go test -v -race ./... + postgres: + name: test (postgres) + runs-on: ubuntu-latest + services: + postgres: + image: postgres:17@sha256:b8faeff7712663c2773a190181e1246e37e364b887611c04658c28b44e2408fe + env: + POSTGRES_PASSWORD: test + POSTGRES_DB: proxy + ports: + - 5432:5432 + options: >- + --health-cmd "pg_isready -U postgres -d proxy" + --health-interval 2s + --health-timeout 5s + --health-retries 30 + + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Set up Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: go.mod + + # The packages with tests that need PROXY_DATABASE_URL. -p 1 because + # their tests drop and recreate tables in the one shared database. + - name: Test + env: + PROXY_DATABASE_URL: postgres://postgres:test@localhost:5432/proxy?sslmode=disable + run: go test -v -race -p 1 ./internal/database/... ./internal/handler/... ./internal/server/... + lint: runs-on: ubuntu-latest steps: From 3fcdb0b652d7f20c818ca632dcfecd077ecbbbdf Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Fri, 23 Oct 2026 16:30:43 +0200 Subject: [PATCH 369/633] Close the SQLite test databases in the lock tests Windows cannot remove the temporary directory while the file is open. --- internal/database/lock_test.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/internal/database/lock_test.go b/internal/database/lock_test.go index c49906d..00750a6 100644 --- a/internal/database/lock_test.go +++ b/internal/database/lock_test.go @@ -13,6 +13,7 @@ import ( func TestTryLockSQLiteAlwaysGranted(t *testing.T) { db := createTestDB(t) + defer func() { _ = db.Close() }() release, ok, err := db.TryLock(context.Background(), "job:test") if err != nil || !ok { t.Fatalf("TryLock = %v, %v; want granted", ok, err) @@ -51,7 +52,11 @@ func TestClaimJobRun(t *testing.T) { // forEachDialect runs fn against SQLite and, when PROXY_DATABASE_URL is set, // against Postgres. func forEachDialect(t *testing.T, fn func(t *testing.T, db *DB)) { - t.Run("sqlite", func(t *testing.T) { fn(t, createTestDB(t)) }) + t.Run("sqlite", func(t *testing.T) { + db := createTestDB(t) + defer func() { _ = db.Close() }() + fn(t, db) + }) t.Run("postgres", func(t *testing.T) { db := createTestPostgresDB(t) if db == nil { From 84111c387adde558ae0a2ab725404828c9400cb7 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Fri, 23 Oct 2026 16:54:54 +0200 Subject: [PATCH 370/633] Fetch each cache miss once across instances Instances sharing a Postgres database each fetched a missed artifact from upstream; only callers within one instance shared a fetch. The instance that fetches now holds a lease on the miss in a new fetch_leases table and renews it while it downloads. Another instance missing the same key polls the cache and serves the stored copy, or takes the lease over when the holder releases it without storing anything or stops renewing it. Metadata misses use the same lease when metadata caching is on with a TTL above zero; without a stored copy there is nothing to wait for. proxy_fetch_lease_waits_total counts the misses that waited. Reclaim deletes leases left by stopped instances. Refs #543 --- README.md | 1 + docs/configuration.md | 4 +- internal/database/database_test.go | 2 +- internal/database/fetch_leases.go | 68 +++++++ internal/database/lock_test.go | 50 +++++ internal/database/schema.go | 31 ++++ internal/handler/fetch_lease.go | 136 ++++++++++++++ internal/handler/fetch_lease_test.go | 265 +++++++++++++++++++++++++++ internal/handler/handler.go | 50 ++++- internal/metrics/metrics.go | 15 ++ internal/server/reclaim.go | 4 + 11 files changed, 620 insertions(+), 6 deletions(-) create mode 100644 internal/database/fetch_leases.go create mode 100644 internal/handler/fetch_lease.go create mode 100644 internal/handler/fetch_lease_test.go diff --git a/README.md b/README.md index 0553e4a..e5a5105 100644 --- a/README.md +++ b/README.md @@ -1474,6 +1474,7 @@ The proxy exposes Prometheus metrics at `GET /metrics`. All metric names are pre | `proxy_metadata_cache_entries` | gauge | `ecosystem` | Number of cached metadata documents per ecosystem. | | `proxy_artifacts_evicted_total` | counter | `reason`, `ecosystem` | Artifacts evicted by the size limit (`lru`) or by age (`retention`) | | `proxy_metadata_evicted_total` | counter | `ecosystem` | Cached metadata documents evicted by the size limit | +| `proxy_fetch_lease_waits_total` | counter | `kind` | Cache misses (`artifact` or `metadata`) that waited for another instance sharing the database to fetch them | | `proxy_upstream_fetch_duration_seconds` | histogram | `ecosystem` | Time spent fetching from upstream | | `proxy_upstream_errors_total` | counter | `ecosystem`, `error_type` | Upstream fetch failures | | `proxy_storage_operation_duration_seconds` | histogram | `operation` | Storage read/write latency | diff --git a/docs/configuration.md b/docs/configuration.md index 875f8fc..c2417c1 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -207,7 +207,9 @@ Instances coordinate through PostgreSQL advisory locks: - Eviction, reclaim, retention and the Gradle build cache sweep run on one instance at a time. An instance that finds a job running elsewhere skips that pass. - A [scheduled mirror](#scheduled-refresh) runs on whichever instance reaches it first in each interval. -Cache misses are only coalesced within an instance. Two instances missing the same file both fetch it from upstream; the extra copy is queued for deletion like any replaced file. Each instance also needs its own [`git.cache_dir`](#git-repositories). +Cache misses are fetched from upstream once across instances. The instance that fetches holds a lease on the miss in the `fetch_leases` table and renews it while it downloads. Another instance missing the same file waits until it is stored and serves the stored copy; `proxy_fetch_lease_waits_total` counts those waits. If the fetch fails, the waiting instance fetches itself, and a lease of an instance that stopped expires after 30 seconds. Metadata is shared this way only with [`cache_metadata`](#metadata-caching) and a `metadata_ttl` above zero, since without a stored copy there is nothing to wait for. A cache hit costs no extra query; a miss costs one. + +Each instance needs its own [`git.cache_dir`](#git-repositories). ### Hit counts diff --git a/internal/database/database_test.go b/internal/database/database_test.go index e5b9899..8b29ced 100644 --- a/internal/database/database_test.go +++ b/internal/database/database_test.go @@ -613,7 +613,7 @@ func createTestPostgresDB(t *testing.T) *DB { // Drop and recreate every table CreateSchema creates for clean test state; // leftover migration records make the next CreateSchema fail on the // migrations primary key. - tables := []string{"artifacts", "pending_deletes", "versions", "packages", "vulnerabilities", "metadata_cache", "metadata_dates", "job_runs", "migrations", "schema_info"} + tables := []string{"artifacts", "pending_deletes", "versions", "packages", "vulnerabilities", "metadata_cache", "metadata_dates", "job_runs", "fetch_leases", "migrations", "schema_info"} for _, table := range tables { _, _ = db.Exec("DROP TABLE IF EXISTS " + table + " CASCADE") } diff --git a/internal/database/fetch_leases.go b/internal/database/fetch_leases.go new file mode 100644 index 0000000..545b84e --- /dev/null +++ b/internal/database/fetch_leases.go @@ -0,0 +1,68 @@ +package database + +import ( + "fmt" + "time" +) + +// Fetch leases let instances sharing a Postgres database fetch each cache +// miss from upstream once. The instance that fetches holds the lease for the +// miss's key and renews it until the result is stored; the others wait for +// the stored result. Expiry uses the database clock, so instances whose +// clocks disagree still agree on when a lease has run out. SQLite is +// single-instance, so these are only used on Postgres. + +// AcquireFetchLease takes the lease on key for owner unless another owner +// holds one that has not expired, and reports whether it did. +func (db *DB) AcquireFetchLease(key, owner string, ttl time.Duration) (bool, error) { + res, err := db.Exec(` + INSERT INTO fetch_leases (key, owner, expires_at) + VALUES ($1, $2, now() + $3 * interval '1 millisecond') + ON CONFLICT (key) DO UPDATE SET owner = excluded.owner, expires_at = excluded.expires_at + WHERE fetch_leases.expires_at < now()`, key, owner, ttl.Milliseconds()) + if err != nil { + return false, fmt.Errorf("acquiring fetch lease: %w", err) + } + n, err := res.RowsAffected() + if err != nil { + return false, fmt.Errorf("acquiring fetch lease: %w", err) + } + return n > 0, nil +} + +// RenewFetchLease extends owner's lease on key by ttl from now, and reports +// whether owner still held it. +func (db *DB) RenewFetchLease(key, owner string, ttl time.Duration) (bool, error) { + res, err := db.Exec(` + UPDATE fetch_leases SET expires_at = now() + $3 * interval '1 millisecond' + WHERE key = $1 AND owner = $2`, key, owner, ttl.Milliseconds()) + if err != nil { + return false, fmt.Errorf("renewing fetch lease: %w", err) + } + n, err := res.RowsAffected() + if err != nil { + return false, fmt.Errorf("renewing fetch lease: %w", err) + } + return n > 0, nil +} + +// ReleaseFetchLease drops owner's lease on key. +func (db *DB) ReleaseFetchLease(key, owner string) error { + if _, err := db.Exec(`DELETE FROM fetch_leases WHERE key = $1 AND owner = $2`, key, owner); err != nil { + return fmt.Errorf("releasing fetch lease: %w", err) + } + return nil +} + +// DeleteExpiredFetchLeases removes leases that expired more than grace ago, +// left by instances that stopped while fetching. +func (db *DB) DeleteExpiredFetchLeases(grace time.Duration) (int64, error) { + if db.dialect != DialectPostgres { + return 0, nil + } + res, err := db.Exec(`DELETE FROM fetch_leases WHERE expires_at < now() - $1 * interval '1 millisecond'`, grace.Milliseconds()) + if err != nil { + return 0, fmt.Errorf("deleting expired fetch leases: %w", err) + } + return res.RowsAffected() +} diff --git a/internal/database/lock_test.go b/internal/database/lock_test.go index 00750a6..9fe0186 100644 --- a/internal/database/lock_test.go +++ b/internal/database/lock_test.go @@ -242,3 +242,53 @@ func TestMigrateAddsJobRuns(t *testing.T) { } }) } + +func TestFetchLeases(t *testing.T) { + db := createTestPostgresDB(t) + if db == nil { + t.Skip("PROXY_DATABASE_URL not set") + } + defer func() { _ = db.Close() }() + + acquire := func(owner string, ttl time.Duration) bool { + t.Helper() + ok, err := db.AcquireFetchLease("k", owner, ttl) + if err != nil { + t.Fatalf("AcquireFetchLease(%s): %v", owner, err) + } + return ok + } + if !acquire("a", time.Minute) { + t.Fatal("a could not take a free lease") + } + if acquire("b", time.Minute) { + t.Fatal("b took a lease a holds") + } + if held, err := db.RenewFetchLease("k", "b", time.Minute); err != nil || held { + t.Fatalf("b renewed a's lease: %v, %v", held, err) + } + if held, err := db.RenewFetchLease("k", "a", time.Minute); err != nil || !held { + t.Fatalf("a could not renew its lease: %v, %v", held, err) + } + // Only the owner can release it. + if err := db.ReleaseFetchLease("k", "b"); err != nil { + t.Fatal(err) + } + if acquire("b", time.Minute) { + t.Fatal("b's release dropped a's lease") + } + if err := db.ReleaseFetchLease("k", "a"); err != nil { + t.Fatal(err) + } + if !acquire("b", time.Millisecond) { + t.Fatal("b could not take the lease a released") + } + time.Sleep(20 * time.Millisecond) + if !acquire("c", time.Millisecond) { + t.Fatal("c could not take b's expired lease") + } + time.Sleep(20 * time.Millisecond) + if n, err := db.DeleteExpiredFetchLeases(0); err != nil || n != 1 { + t.Fatalf("DeleteExpiredFetchLeases = %d, %v; want 1", n, err) + } +} diff --git a/internal/database/schema.go b/internal/database/schema.go index 7c1922b..9b10039 100644 --- a/internal/database/schema.go +++ b/internal/database/schema.go @@ -92,6 +92,12 @@ CREATE TABLE IF NOT EXISTS job_runs ( started_at DATETIME NOT NULL ); +CREATE TABLE IF NOT EXISTS fetch_leases ( + key TEXT NOT NULL PRIMARY KEY, + owner TEXT NOT NULL, + expires_at DATETIME NOT NULL +); + CREATE TABLE IF NOT EXISTS vulnerabilities ( id INTEGER PRIMARY KEY, vuln_id TEXT NOT NULL, @@ -221,6 +227,12 @@ CREATE TABLE IF NOT EXISTS job_runs ( started_at TIMESTAMP NOT NULL ); +CREATE TABLE IF NOT EXISTS fetch_leases ( + key TEXT NOT NULL PRIMARY KEY, + owner TEXT NOT NULL, + expires_at TIMESTAMP NOT NULL +); + CREATE TABLE IF NOT EXISTS vulnerabilities ( id SERIAL PRIMARY KEY, vuln_id TEXT NOT NULL, @@ -432,6 +444,7 @@ var migrations = []migration{ {"013_add_artifact_manifest", migrateAddArtifactManifest}, {"014_add_metadata_last_accessed", migrateAddMetadataLastAccessed}, {"015_add_job_runs", migrateAddJobRuns}, + {"016_add_fetch_leases", migrateAddFetchLeases}, } // isTableNotFound returns true if the error indicates a missing table. @@ -748,6 +761,24 @@ func migrateAddJobRuns(db *DB) error { return nil } +// migrateAddFetchLeases creates the table instances sharing a Postgres +// database use to fetch each cache miss from upstream only once. +func migrateAddFetchLeases(db *DB) error { + ts := sqliteDatetime + if db.dialect == DialectPostgres { + ts = postgresTimestamp + } + query := fmt.Sprintf(`CREATE TABLE IF NOT EXISTS fetch_leases ( + key TEXT NOT NULL PRIMARY KEY, + owner TEXT NOT NULL, + expires_at %s NOT NULL + )`, ts) + if _, err := db.Exec(query); err != nil { + return fmt.Errorf("creating fetch_leases table: %w", err) + } + return nil +} + // migrateAddPendingDeletesQueuedAtIndex indexes the column reclaim orders // the queue by, which grows large after a retention sweep. func migrateAddPendingDeletesQueuedAtIndex(db *DB) error { diff --git a/internal/handler/fetch_lease.go b/internal/handler/fetch_lease.go new file mode 100644 index 0000000..47dcbf5 --- /dev/null +++ b/internal/handler/fetch_lease.go @@ -0,0 +1,136 @@ +package handler + +import ( + "context" + "crypto/rand" + "crypto/sha256" + "encoding/hex" + "time" + + "github.com/git-pkgs/proxy/internal/database" + "github.com/git-pkgs/proxy/internal/metrics" +) + +// Instances sharing a Postgres database coalesce cache misses through fetch +// leases, as coalesceFetch and coalesceMetadata do within one instance. The +// instance that runs the fetch holds the lease for the miss and renews it +// while it downloads. Another instance missing the same key waits for the +// stored result and serves that, or takes the lease over if the holder +// releases it without storing anything or stops renewing it. +var ( + fetchLeaseTTL = 30 * time.Second + fetchLeaseRenew = 10 * time.Second + // The wait for another instance's fetch polls the cache, starting + // quickly for small files and backing off for large ones. + fetchLeasePollMin = 50 * time.Millisecond + fetchLeasePollMax = 2 * time.Second +) + +// fetchLeaseKey turns a coalescing key, which can hold long URLs and NUL +// separators, into a fixed-size table key. +func fetchLeaseKey(kind, key string) string { + sum := sha256.Sum256([]byte(kind + "\x00" + key)) + return hex.EncodeToString(sum[:]) +} + +func newLeaseOwner() string { + var b [16]byte + _, _ = rand.Read(b[:]) + return hex.EncodeToString(b[:]) +} + +// fetchLeasesEnabled reports whether misses are coalesced across instances. +func (p *Proxy) fetchLeasesEnabled() bool { + return p.DB != nil && p.DB.Dialect() == database.DialectPostgres +} + +// awaitFetchLease takes the lease on key before this instance fetches it from +// upstream. While another instance holds the lease, it polls ready, which +// reports whether that instance's result is now stored and, if so, keeps it +// for the caller. It returns: +// - stored true when ready found the result: serve it, nothing to release. +// - release when this instance should fetch: call it once the result is +// stored or the fetch failed. +// - err when ctx ended while waiting. +// +// A failing lease query is logged and the fetch goes ahead without a lease, +// so a database hiccup costs a duplicate fetch rather than the request. +func (p *Proxy) awaitFetchLease(ctx context.Context, kind, key string, ready func() bool) (release func(), stored bool, err error) { + noop := func() {} + if !p.fetchLeasesEnabled() { + return noop, false, nil + } + leaseKey := fetchLeaseKey(kind, key) + owner := newLeaseOwner() + delay := fetchLeasePollMin + waited := false + for { + acquired, err := p.DB.AcquireFetchLease(leaseKey, owner, fetchLeaseTTL) + if err != nil { + p.Logger.Warn("fetch lease unavailable, fetching without it", "kind", kind, "error", err) + return noop, false, nil + } + if acquired { + // The holder may have stored the result and released the lease + // between this caller's last poll and taking it. + if waited && ready() { + p.releaseFetchLease(leaseKey, owner) + return noop, true, nil + } + return p.holdFetchLease(leaseKey, owner), false, nil + } + if !waited { + metrics.RecordFetchLeaseWait(kind) + waited = true + } + timer := time.NewTimer(delay) + select { + case <-ctx.Done(): + timer.Stop() + return nil, false, ctx.Err() + case <-timer.C: + } + if ready() { + return noop, true, nil + } + delay = min(delay+delay, fetchLeasePollMax) + } +} + +// holdFetchLease renews owner's lease on leaseKey until the returned release +// is called, which also drops it. +func (p *Proxy) holdFetchLease(leaseKey, owner string) func() { + stop := make(chan struct{}) + done := make(chan struct{}) + go func() { + defer close(done) + ticker := time.NewTicker(fetchLeaseRenew) + defer ticker.Stop() + for { + select { + case <-stop: + return + case <-ticker.C: + // A lease that lapsed, say after a long pause, lets another + // instance fetch too. That costs a duplicate fetch, nothing + // else, so the fetch here carries on either way. + if held, err := p.DB.RenewFetchLease(leaseKey, owner, fetchLeaseTTL); err != nil || !held { + p.Logger.Debug("fetch lease lapsed", "error", err) + } + } + } + }() + return func() { + close(stop) + <-done + p.releaseFetchLease(leaseKey, owner) + } +} + +func (p *Proxy) releaseFetchLease(leaseKey, owner string) { + if err := p.DB.ReleaseFetchLease(leaseKey, owner); err != nil { + // The lease expires on its own; until then other instances wait + // on it and then fetch. + p.Logger.Warn("failed to release fetch lease", "error", err) + } +} diff --git a/internal/handler/fetch_lease_test.go b/internal/handler/fetch_lease_test.go new file mode 100644 index 0000000..a71f138 --- /dev/null +++ b/internal/handler/fetch_lease_test.go @@ -0,0 +1,265 @@ +package handler + +import ( + "context" + "errors" + "io" + "log/slog" + "net/http" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/git-pkgs/proxy/internal/database" + "github.com/git-pkgs/proxy/internal/metrics" + "github.com/git-pkgs/registries/fetch" + "github.com/prometheus/client_golang/prometheus/testutil" +) + +// gateFetcher counts upstream fetches, signals entered when the first one +// starts and holds every fetch until release is closed. +type gateFetcher struct { + calls atomic.Int64 + content string + err error + entered chan struct{} + release chan struct{} + once sync.Once + opened sync.Once +} + +// newGateFetcher returns a gateFetcher that is released when the test ends +// at the latest, so a failing test does not leave fetches hanging. +func newGateFetcher(t *testing.T, content string, err error) *gateFetcher { + f := &gateFetcher{content: content, err: err, entered: make(chan struct{}), release: make(chan struct{})} + t.Cleanup(f.open) + return f +} + +func (f *gateFetcher) open() { f.opened.Do(func() { close(f.release) }) } + +func (f *gateFetcher) Fetch(ctx context.Context, url string) (*fetch.Artifact, error) { + return f.FetchWithHeaders(ctx, url, nil) +} + +func (f *gateFetcher) FetchWithHeaders(ctx context.Context, _ string, _ http.Header) (*fetch.Artifact, error) { + f.calls.Add(1) + f.once.Do(func() { close(f.entered) }) + select { + case <-f.release: + case <-ctx.Done(): + return nil, ctx.Err() + } + if f.err != nil { + return nil, f.err + } + return &fetch.Artifact{Body: io.NopCloser(strings.NewReader(f.content)), ContentType: "application/gzip"}, nil +} + +func (f *gateFetcher) Head(context.Context, string) (int64, string, error) { return 0, "", nil } + +// sharedInstances returns two proxies sharing one Postgres database and one +// storage, as two replicas would. It skips without PROXY_DATABASE_URL. +func sharedInstances(t *testing.T) (*Proxy, *Proxy, *database.DB) { + t.Helper() + db := openTestPostgres(t) + if db == nil { + t.Skip("PROXY_DATABASE_URL not set") + } + store := newMockStorage() + logger := slog.New(slog.NewTextHandler(io.Discard, nil)) + a := NewProxy(db, store, &mockFetcher{}, fetch.NewResolver(), logger) + b := NewProxy(db, store, &mockFetcher{}, fetch.NewResolver(), logger) + return a, b, db +} + +const leaseTestURL = "https://registry.npmjs.org/left-pad/-/left-pad-1.3.0.tgz" + +func fetchLeftPad(p *Proxy) (string, error) { + res, err := p.GetOrFetchArtifactFromURL(context.Background(), "npm", "left-pad", "1.3.0", "left-pad-1.3.0.tgz", leaseTestURL) + if err != nil { + return "", err + } + defer func() { _ = res.Reader.Close() }() + b, err := io.ReadAll(res.Reader) + return string(b), err +} + +type fetchResult struct { + body string + err error +} + +func goFetchLeftPad(p *Proxy) chan fetchResult { + ch := make(chan fetchResult, 1) + go func() { + body, err := fetchLeftPad(p) + ch <- fetchResult{body, err} + }() + return ch +} + +// waitForLeaseWaits returns once the lease wait counter for kind has passed +// from. +func waitForLeaseWaits(t *testing.T, kind string, from float64) { + t.Helper() + deadline := time.Now().Add(5 * time.Second) + for testutil.ToFloat64(metrics.FetchLeaseWaits.WithLabelValues(kind)) <= from { + if time.Now().After(deadline) { + t.Fatal("timed out waiting for the second instance to wait on the lease") + } + time.Sleep(time.Millisecond) + } +} + +func TestFetchLeaseCoalescesArtifactAcrossInstances(t *testing.T) { + a, b, _ := sharedInstances(t) + const content = "left-pad tarball" + fa := newGateFetcher(t, content, nil) + fb := newGateFetcher(t, content, nil) + fb.open() + a.Fetcher, b.Fetcher = fa, fb + + waits := testutil.ToFloat64(metrics.FetchLeaseWaits.WithLabelValues(metrics.CacheKindArtifact)) + resA := goFetchLeftPad(a) + <-fa.entered + resB := goFetchLeftPad(b) + waitForLeaseWaits(t, metrics.CacheKindArtifact, waits) + fa.open() + + for name, ch := range map[string]chan fetchResult{"a": resA, "b": resB} { + r := <-ch + if r.err != nil || r.body != content { + t.Errorf("instance %s: body %q, err %v", name, r.body, r.err) + } + } + if got := fa.calls.Load() + fb.calls.Load(); got != 1 { + t.Errorf("upstream fetches = %d (a %d, b %d), want 1", got, fa.calls.Load(), fb.calls.Load()) + } +} + +// TestFetchLeaseFailedFetchLetsOtherInstanceFetch checks that an instance +// waiting on a fetch that fails takes the lease over and fetches itself, +// rather than waiting until the lease expires. +func TestFetchLeaseFailedFetchLetsOtherInstanceFetch(t *testing.T) { + a, b, _ := sharedInstances(t) + const content = "left-pad tarball" + fa := newGateFetcher(t, "", errors.New("upstream down")) + fb := newGateFetcher(t, content, nil) + fb.open() + a.Fetcher, b.Fetcher = fa, fb + + waits := testutil.ToFloat64(metrics.FetchLeaseWaits.WithLabelValues(metrics.CacheKindArtifact)) + resA := goFetchLeftPad(a) + <-fa.entered + resB := goFetchLeftPad(b) + waitForLeaseWaits(t, metrics.CacheKindArtifact, waits) + started := time.Now() + fa.open() + + if r := <-resA; r.err == nil { + t.Error("instance a: want the upstream error") + } + r := <-resB + if r.err != nil || r.body != content { + t.Fatalf("instance b: body %q, err %v", r.body, r.err) + } + if fb.calls.Load() != 1 { + t.Errorf("b's upstream fetches = %d, want 1", fb.calls.Load()) + } + if waited := time.Since(started); waited >= fetchLeaseTTL { + t.Errorf("b waited %s, as long as the lease lasts", waited) + } +} + +// TestFetchLeaseExpiredIsTakenOver checks that a lease left by an instance +// that stopped while fetching holds the others up only until it expires. +func TestFetchLeaseExpiredIsTakenOver(t *testing.T) { + _, b, db := sharedInstances(t) + const content = "left-pad tarball" + fb := newGateFetcher(t, content, nil) + fb.open() + b.Fetcher = fb + + key := artifactCoalesceKey("pkg:npm/left-pad@1.3.0", "left-pad-1.3.0.tgz", leaseTestURL, "", "") + if ok, err := db.AcquireFetchLease(fetchLeaseKey(metrics.CacheKindArtifact, key), "stopped-instance", 300*time.Millisecond); err != nil || !ok { + t.Fatalf("taking the lease for the stopped instance: %v, %v", ok, err) + } + started := time.Now() + body, err := fetchLeftPad(b) + if err != nil || body != content { + t.Fatalf("body %q, err %v", body, err) + } + if waited := time.Since(started); waited < 250*time.Millisecond { + t.Errorf("fetched after %s, before the lease expired; is the test key right?", waited) + } +} + +func TestFetchLeaseCoalescesMetadataAcrossInstances(t *testing.T) { + a, b, _ := sharedInstances(t) + u := newMetadataUpstream(t, http.StatusOK) + var released sync.Once + release := func() { released.Do(func() { close(u.release) }) } + t.Cleanup(release) + for _, p := range []*Proxy{a, b} { + p.HTTPClient = u.Client() + p.CacheMetadata = true + p.MetadataTTL = time.Minute + } + + waits := testutil.ToFloat64(metrics.FetchLeaseWaits.WithLabelValues(metrics.CacheKindMetadata)) + resA := startMetadataCalls(a, context.Background(), 1, u.URL+"/left-pad", "application/json")[0] + <-u.entered + resB := startMetadataCalls(b, context.Background(), 1, u.URL+"/left-pad", "application/json")[0] + waitForLeaseWaits(t, metrics.CacheKindMetadata, waits) + release() + + for name, ch := range map[string]chan metadataCall{"a": resA, "b": resB} { + r := <-ch + if r.err != nil || !strings.Contains(string(r.body), "application/json") { + t.Errorf("instance %s: body %q, err %v", name, r.body, r.err) + } + } + if got := u.calls.Load(); got != 1 { + t.Errorf("upstream metadata requests = %d, want 1", got) + } +} + +// TestFetchLeaseNotUsedWithoutMetadataCache checks that metadata misses are +// not held up when there is no stored copy to wait for. +func TestFetchLeaseNotUsedWithoutMetadataCache(t *testing.T) { + a, b, _ := sharedInstances(t) + u := newMetadataUpstream(t, http.StatusOK) + a.HTTPClient, b.HTTPClient = u.Client(), u.Client() + + resA := startMetadataCalls(a, context.Background(), 1, u.URL+"/left-pad", "application/json")[0] + <-u.entered + resB := startMetadataCalls(b, context.Background(), 1, u.URL+"/left-pad", "application/json")[0] + deadline := time.Now().Add(5 * time.Second) + for u.calls.Load() < 2 { + if time.Now().After(deadline) { + t.Fatal("b did not fetch while a was fetching") + } + time.Sleep(time.Millisecond) + } + close(u.release) + for _, ch := range []chan metadataCall{resA, resB} { + if r := <-ch; r.err != nil { + t.Errorf("unexpected error: %v", r.err) + } + } +} + +func TestFetchLeaseNotUsedOnSQLite(t *testing.T) { + proxy, _, _, _ := setupTestProxy(t) + release, stored, err := proxy.awaitFetchLease(context.Background(), metrics.CacheKindArtifact, "k", func() bool { + t.Error("ready polled on SQLite") + return false + }) + if err != nil || stored { + t.Fatalf("awaitFetchLease = %v, %v; want to fetch at once", stored, err) + } + release() +} diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 022f3d8..2aa1515 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -815,12 +815,38 @@ func (p *Proxy) runSharedFetch(ctx context.Context, key string, f *inflightFetch // in between would otherwise be repeated. Serve that record only if its // bytes are still present: a record can outlive them, and refetching is // the same recovery the cache lookup makes. - if stored, path, ok := recheck(); ok { - if res, err := p.openStoredArtifact(ctx, stored, path); err == nil { - f.val, f.err = fetchedArtifact{artifact: stored, storagePath: path}, nil - return res, nil + served := func() (*CacheResult, bool) { + stored, path, ok := recheck() + if !ok { + return nil, false } + res, err := p.openStoredArtifact(ctx, stored, path) + if err != nil { + return nil, false + } + f.val, f.err = fetchedArtifact{artifact: stored, storagePath: path}, nil + return res, true } + if res, ok := served(); ok { + return res, nil + } + + // Another instance sharing the database may be fetching the same + // artifact. Wait for it to store the artifact rather than fetch it too. + var waitedFor *CacheResult + release, storedElsewhere, err := p.awaitFetchLease(ctx, metrics.CacheKindArtifact, key, func() bool { + res, ok := served() + waitedFor = res + return ok + }) + if err != nil { + f.err = err + return nil, err + } + if storedElsewhere { + return waitedFor, nil + } + defer release() commitCtx := ctx if upstreamFetchDetached(ctx) && p.acquireDetachSlot() { @@ -1369,6 +1395,22 @@ func (p *Proxy) coalescedMetadataMiss(ctx context.Context, ecosystem, cacheKey, if hit != nil { return *hit } + // Another instance sharing the database may be fetching the same + // document. Only a stored copy within TTL can be served after + // waiting, so without that there is nothing to wait for. + if p.CacheMetadata && p.MetadataTTL > 0 { + release, stored, err := p.awaitFetchLease(fetchCtx, metrics.CacheKindMetadata, key, func() bool { + entry, hit = p.cachedMetadataState(fetchCtx, ecosystem, cacheKey, validate) + return hit != nil + }) + if err != nil { + return metadataResult{err: err} + } + if stored { + return *hit + } + defer release() + } return p.fetchMetadataFromUpstream(fetchCtx, ecosystem, cacheKey, upstreamURL, accept, acceptEncoding, validate, entry) }) // The fetch ran the first caller's validate. A caller that joined it runs diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go index 8fb857c..708b5b3 100644 --- a/internal/metrics/metrics.go +++ b/internal/metrics/metrics.go @@ -86,6 +86,14 @@ var ( []string{"ecosystem"}, ) + FetchLeaseWaits = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "proxy_fetch_lease_waits_total", + Help: "Cache misses that waited for another instance's upstream fetch, by kind (artifact or metadata)", + }, + []string{"kind"}, + ) + // Upstream metrics UpstreamFetchDuration = prometheus.NewHistogramVec( prometheus.HistogramOpts{ @@ -303,6 +311,7 @@ func init() { CachedArtifacts, ArtifactsEvicted, MetadataEvicted, + FetchLeaseWaits, UpstreamFetchDuration, UpstreamErrors, CircuitBreakerState, @@ -400,6 +409,12 @@ func RecordMetadataEvicted(ecosystem string) { MetadataEvicted.WithLabelValues(purl.NormalizeEcosystem(ecosystem)).Inc() } +// RecordFetchLeaseWait counts a cache miss that found another instance +// sharing the database already fetching it. +func RecordFetchLeaseWait(kind string) { + FetchLeaseWaits.WithLabelValues(kind).Inc() +} + // RecordArtifactCacheMiss counts an artifact that had to be fetched upstream. func RecordArtifactCacheMiss(ecosystem string) { CacheMisses.WithLabelValues(purl.NormalizeEcosystem(ecosystem), CacheKindArtifact).Inc() diff --git a/internal/server/reclaim.go b/internal/server/reclaim.go index 9437023..58aefed 100644 --- a/internal/server/reclaim.go +++ b/internal/server/reclaim.go @@ -36,6 +36,10 @@ func (s *Server) startReclaimLoop(ctx context.Context) { case <-ticker.C: s.runExclusive(ctx, "reclaim", func() { reclaimDue(ctx, s.db, s.storage, s.logger, time.Now().Add(-grace), reclaimTickBudget) + // Leases left by instances that stopped while fetching. + if _, err := s.db.DeleteExpiredFetchLeases(reclaimMinGrace); err != nil { + s.logger.Warn("reclaim: failed to delete expired fetch leases", "error", err) + } }) } } From f0548d107f50b0691bec62b2d584b13d7d324bac Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Fri, 23 Oct 2026 17:34:15 +0100 Subject: [PATCH 371/633] Document that the OCI handler is pull-only --- docs/configuration.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/configuration.md b/docs/configuration.md index fa77ec6..2274184 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -449,6 +449,9 @@ while `upstream.oci` selects named registries through the `upstream/{name}/` repository prefix. For example, `oci://proxy.example.com/upstream/ghcr/owner/chart` uses the `ghcr` registry with `owner/chart` as its repository. +The OCI handler only serves pulls (`GET` and `HEAD`). Pushes, such as a +BuildKit `--cache-to type=registry` export, need a separate registry. + containerd mirror requests carry the original registry host in an `ns` query parameter (see the containerd section in the README). The proxy only looks the host up and never connects to it: `docker.io`, `index.docker.io`, From eb5ff1a9da7cde2ee7e2606afe94d1696c05b1c5 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Fri, 23 Oct 2026 19:06:07 +0200 Subject: [PATCH 372/633] Accept only relative paths in a conda shard index's base URLs Clients resolve base_url and shards_base_url against the proxy's URL. A scheme or host sends them around the proxy, and a leading slash leaves its mount, so neither layout is served. --- internal/handler/conda_shards.go | 36 +++++++++++---------------- internal/handler/conda_shards_test.go | 32 +++++++++++++++++------- 2 files changed, 37 insertions(+), 31 deletions(-) diff --git a/internal/handler/conda_shards.go b/internal/handler/conda_shards.go index f28f348..f2de475 100644 --- a/internal/handler/conda_shards.go +++ b/internal/handler/conda_shards.go @@ -18,23 +18,12 @@ import ( "github.com/vmihailenco/msgpack/v5" ) -// Sharded repodata (CEP 16) splits repodata.json into an index, -// /repodata_shards.msgpack.zst, that maps each package name to the -// sha256 of its shard, and one zstd-compressed msgpack shard per package -// stored under that hash. -// -// Without cooldown the index and the shards are passed through and cached as -// metadata. With cooldown, shards have to be filtered, and that changes their -// bytes. Clients cache shards by the hash in the index and never fetch the same -// hash again, so a filtered shard can't be served under its upstream hash. The -// proxy rewrites the index instead: every hash is XORed with a mask derived -// from the channel, the subdir and the current hour, and shards_base_url points -// at cooldown-shards/{hour}/. A shard request there undoes the mask, fetches -// the upstream shard and filters it. Each hour the keys change, so clients -// fetch shards again and see versions that have left cooldown since. -// -// When the index can't be served this way, the proxy answers 404 and clients -// fall back to repodata.json, which is filtered too. +// Sharded repodata (CEP 16) maps each package name to the sha256 of its +// shard. With cooldown the proxy filters shards, which changes their bytes, +// but clients cache shards by hash and never fetch the same hash again. So +// the rewritten index XORs every hash with a mask that changes each hour: +// filtered shards get hashes of their own, and clients fetch them again once +// versions have left cooldown. const ( condaShardIndexFile = "repodata_shards.msgpack.zst" condaShardSuffix = ".msgpack.zst" @@ -285,7 +274,10 @@ func (idx *condaShardIndex) infoString(key string) (string, error) { // shardsLayout checks that packages and shards sit where the proxy routes // them, below the subdir of indexURL, and returns the shards directory -// relative to the subdir: "" or "shards/". +// relative to the subdir: "" or "shards/". Clients resolve base_url and +// shards_base_url against the proxy's URL, so only relative paths stay on +// the proxy: a scheme or host leads clients around it, a leading / outside +// its mount. func (idx *condaShardIndex) shardsLayout(indexURL string) (string, error) { base, err := url.Parse(indexURL) if err != nil { @@ -300,6 +292,9 @@ func (idx *condaShardIndex) shardsLayout(indexURL string) (string, error) { if err != nil { return "", err } + if u.Scheme != "" || u.Host != "" || strings.HasPrefix(u.Path, "/") { + return "", errCondaShardLayout + } return base.ResolveReference(u).String(), nil } @@ -316,11 +311,8 @@ func (idx *condaShardIndex) shardsLayout(indexURL string) (string, error) { return "", err } resolved, err := resolve(shardsBaseURL) - if err != nil { - return "", err - } rel, ok := strings.CutPrefix(resolved, subdir) - if !ok || (rel != "" && rel != condaShardsSubdir) { + if err != nil || !ok || (rel != "" && rel != condaShardsSubdir) { return "", fmt.Errorf("%w: shards_base_url %q", errCondaShardLayout, shardsBaseURL) } return rel, nil diff --git a/internal/handler/conda_shards_test.go b/internal/handler/conda_shards_test.go index 43b507f..27b0ad7 100644 --- a/internal/handler/conda_shards_test.go +++ b/internal/handler/conda_shards_test.go @@ -74,12 +74,7 @@ func newShardUpstream(t *testing.T, info map[string]any) *shardFixture { if s, _ := info["shards_base_url"].(string); strings.Contains(s, "shards/") { layout = "shards/" } - index := zstdMsgpack(t, map[string]any{ - "version": 1, - "info": info, - "shards": map[string]any{"testpkg": sum[:]}, - }) - + var index []byte f.upstream = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { f.requests = append(f.requests, r.URL.Path) switch r.URL.Path { @@ -92,6 +87,20 @@ func newShardUpstream(t *testing.T, info map[string]any) *shardFixture { } })) t.Cleanup(f.upstream.Close) + + // {upstream} in an info value stands for the upstream's URL, {host} for + // its host. + placeholders := strings.NewReplacer("{upstream}", f.upstream.URL, "{host}", strings.TrimPrefix(f.upstream.URL, "http://")) + for k, v := range info { + if s, ok := v.(string); ok { + info[k] = placeholders.Replace(s) + } + } + index = zstdMsgpack(t, map[string]any{ + "version": 1, + "info": info, + "shards": map[string]any{"testpkg": sum[:]}, + }) return f } @@ -236,9 +245,14 @@ func TestCondaCooldownShardRejectsBadPaths(t *testing.T) { func TestCondaShardIndexUnsupportedLayout(t *testing.T) { for name, info := range map[string]map[string]any{ - "packages elsewhere": {"base_url": "https://cdn.example.com/conda-forge/noarch/", "shards_base_url": ""}, - "shards elsewhere": {"base_url": "", "shards_base_url": "https://cdn.example.com/shards/"}, - "nested shards": {"base_url": "", "shards_base_url": "./a/b/"}, + "packages elsewhere": {"base_url": "https://cdn.example.com/conda-forge/noarch/", "shards_base_url": ""}, + "shards elsewhere": {"base_url": "", "shards_base_url": "https://cdn.example.com/shards/"}, + "nested shards": {"base_url": "", "shards_base_url": "./a/b/"}, + "packages upstream": {"base_url": "{upstream}/conda-forge/noarch/", "shards_base_url": ""}, + "shards upstream": {"base_url": "", "shards_base_url": "{upstream}/conda-forge/noarch/"}, + "no scheme upstream": {"base_url": "//{host}/conda-forge/noarch/", "shards_base_url": ""}, + "root-relative": {"base_url": "/conda-forge/noarch/", "shards_base_url": ""}, + "root-relative shards": {"base_url": "", "shards_base_url": "/conda-forge/noarch/shards/"}, } { t.Run(name, func(t *testing.T) { f := newShardUpstream(t, info) From 743289c8ca5a575bd753904b3931fee9a37e8a60 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Fri, 23 Oct 2026 19:07:32 +0200 Subject: [PATCH 373/633] Let an interrupted mirror schedule run again A schedule's claim was kept when its run was cut off, so after a restart every instance skipped it for most of an interval. The scheduler now drops the claim when the run ends with its context cancelled. --- docs/configuration.md | 2 +- internal/database/lock.go | 9 ++++++ internal/database/lock_test.go | 12 ++++++++ internal/mirror/schedule.go | 11 ++++++- internal/mirror/schedule_test.go | 48 ++++++++++++++++++++++++++++++ internal/server/mirror_schedule.go | 1 + 6 files changed, 81 insertions(+), 2 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 875f8fc..6826bf0 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1508,7 +1508,7 @@ mirror: Each run checks the tag with a HEAD request. If the tag still points to the cached digest, nothing else is fetched. If it moved, the new manifests and blobs are fetched first and the tag is switched only when all of them are cached, so a failed run leaves the previous image pullable. Images of one schedule are mirrored one after the other. -- The first run starts within ten minutes of startup, later runs every `interval` ±10 %, so restarts and several instances don't hit the registry at once. Instances sharing a PostgreSQL database run each schedule once per interval: the first to reach it records the run, and the others skip it. +- The first run starts within ten minutes of startup, later runs every `interval` ±10 %, so restarts and several instances don't hit the registry at once. Instances sharing a PostgreSQL database run each schedule once per interval: the first to reach it records the run, and the others skip it. A run cut off by a shutdown is not recorded, so the next instance to reach the schedule runs it. - Digest-pinned PURLs never move and are rejected. A `pkg:oci` digest with `tag=` is accepted; the tag is refreshed and a warning is logged when it no longer matches the digest. - The previous digest stays cached, so pulls by digest keep working, until eviction or retention removes it. - Docker Hub does not count HEAD requests against its pull limit, but every changed manifest is a pull. Set `platforms` and `upstream.auth` for `registry-1.docker.io` to stay under the anonymous limit. diff --git a/internal/database/lock.go b/internal/database/lock.go index b245c62..f4b1f70 100644 --- a/internal/database/lock.go +++ b/internal/database/lock.go @@ -96,3 +96,12 @@ func (db *DB) ClaimJobRun(name string, minGap time.Duration) (bool, error) { } return n > 0, nil } + +// ReleaseJobRun drops the named job's last claim, so the next attempt on any +// instance runs it. It is for a run that was interrupted before it finished. +func (db *DB) ReleaseJobRun(name string) error { + if _, err := db.Exec(db.Rebind(`DELETE FROM job_runs WHERE name = ?`), name); err != nil { + return fmt.Errorf("releasing job run %s: %w", name, err) + } + return nil +} diff --git a/internal/database/lock_test.go b/internal/database/lock_test.go index 00750a6..81292e2 100644 --- a/internal/database/lock_test.go +++ b/internal/database/lock_test.go @@ -46,6 +46,18 @@ func TestClaimJobRun(t *testing.T) { if err != nil || !claimed { t.Fatalf("claim after the gap = %v, %v; want claimed", claimed, err) } + // A released run can be claimed again within the gap. + if err := db.ReleaseJobRun("mirror-schedule:a"); err != nil { + t.Fatal(err) + } + claimed, err = db.ClaimJobRun("mirror-schedule:a", time.Hour) + if err != nil || !claimed { + t.Fatalf("claim after release = %v, %v; want claimed", claimed, err) + } + claimed, err = db.ClaimJobRun("mirror-schedule:b", time.Hour) + if err != nil || claimed { + t.Fatalf("other job after release = %v, %v; want still claimed", claimed, err) + } }) } diff --git a/internal/mirror/schedule.go b/internal/mirror/schedule.go index 55d6ab4..89bb512 100644 --- a/internal/mirror/schedule.go +++ b/internal/mirror/schedule.go @@ -35,6 +35,9 @@ type Scheduler struct { // instance keeps its own timer, so without it every instance would run // every schedule. Claim func(key string, minGap time.Duration) (bool, error) + // Release, when set, drops the claim of a run that was interrupted, so + // the next attempt runs it rather than waiting an interval. + Release func(key string) error // randN returns a value in [0, n); tests replace it. randN func(n int64) int64 } @@ -83,11 +86,12 @@ func (s *Scheduler) loop(ctx context.Context, index int, schedule Schedule) { } func (s *Scheduler) run(ctx context.Context, index int, schedule Schedule) { + key := scheduleKey(schedule) if s.Claim != nil { // Later runs come at least interval - 2*spread apart, so a run // started more recently than that was another instance's. minGap := schedule.Interval - 2*(schedule.Interval/scheduleJitter) //nolint:mnd // two spreads of margin - claimed, err := s.Claim(scheduleKey(schedule), minGap) + claimed, err := s.Claim(key, minGap) if err != nil { s.logger.Warn("mirror schedule: failed to claim run, skipping it", "schedule", index, "error", err) return @@ -100,6 +104,11 @@ func (s *Scheduler) run(ctx context.Context, index int, schedule Schedule) { started := time.Now() progress, err := s.mirror.Run(ctx, &PURLSource{PURLs: schedule.PURLs, Platforms: schedule.Platforms}) if ctx.Err() != nil { + if s.Claim != nil && s.Release != nil { + if err := s.Release(key); err != nil { + s.logger.Warn("mirror schedule: failed to release interrupted run", "schedule", index, "error", err) + } + } return } if err != nil { diff --git a/internal/mirror/schedule_test.go b/internal/mirror/schedule_test.go index 53bdea2..b5b0143 100644 --- a/internal/mirror/schedule_test.go +++ b/internal/mirror/schedule_test.go @@ -216,6 +216,54 @@ func TestSchedulerSkipsRunClaimedElsewhere(t *testing.T) { } } +// TestSchedulerReleasesInterruptedRun checks that a run cut off by shutdown +// does not count as done: the next attempt, on a restarted instance or +// another one, runs the schedule again. +func TestSchedulerReleasesInterruptedRun(t *testing.T) { + images := &scheduleImages{block: make(chan struct{})} + schedule := Schedule{PURLs: []string{"pkg:docker/alpine@3"}, Interval: time.Hour} + s := newTestScheduler(images, schedule) + // The claims of a shared database: a claimed key is not claimed again. + var mu sync.Mutex + claims := map[string]bool{} + s.Claim = func(key string, _ time.Duration) (bool, error) { + mu.Lock() + defer mu.Unlock() + if claims[key] { + return false, nil + } + claims[key] = true + return true, nil + } + s.Release = func(key string) error { + mu.Lock() + defer mu.Unlock() + delete(claims, key) + return nil + } + + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + go func() { + s.run(ctx, 0, schedule) + close(done) + }() + waitFor(t, "the run to start", func() bool { return images.calls.Load() == 1 }) + cancel() + <-done + + close(images.block) + s.run(context.Background(), 0, schedule) + if got := images.calls.Load(); got != 2 { + t.Fatalf("calls = %d, want 2: the interrupted run blocked the next one", got) + } + // A finished run keeps its claim. + s.run(context.Background(), 0, schedule) + if got := images.calls.Load(); got != 2 { + t.Fatalf("calls = %d, want 2 after a finished run", got) + } +} + func TestScheduleKey(t *testing.T) { base := Schedule{PURLs: []string{"pkg:docker/alpine@3"}, Platforms: []string{"linux/amd64"}, Interval: time.Hour} if scheduleKey(base) != scheduleKey(Schedule{PURLs: []string{"pkg:docker/alpine@3"}, Platforms: []string{"linux/amd64"}, Interval: time.Hour}) { diff --git a/internal/server/mirror_schedule.go b/internal/server/mirror_schedule.go index fb32f24..3b02337 100644 --- a/internal/server/mirror_schedule.go +++ b/internal/server/mirror_schedule.go @@ -25,6 +25,7 @@ func (s *Server) startMirrorSchedules(ctx context.Context, proxy *handler.Proxy) } s.scheduler = mirror.NewScheduler(m, schedules, s.logger) s.scheduler.Claim = s.db.ClaimJobRun + s.scheduler.Release = s.db.ReleaseJobRun s.scheduler.Start(ctx) s.logger.Info("mirror schedules enabled", "schedules", len(schedules)) } From 64162c77d838de2c67ce856fb4a26c71cd1c9a0d Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Fri, 23 Oct 2026 19:19:22 +0200 Subject: [PATCH 374/633] Keep detached lease waits alive and stop waiting on a failed fetch A detached fetch waiting on another instance's lease ran the wait under the request context, so a client leaving cancelled every caller joined to it. The wait now runs past the client, bounded by the holder's fetch and the lease expiry. When the holder's fetch failed, each waiting instance took the lease in turn and repeated it, so against a hanging upstream the k-th instance waited about k timeouts. The holder now marks the lease failed. The instances waiting on it then serve their stale metadata copy if they have one, and otherwise fetch at the same time. --- docs/configuration.md | 2 +- internal/database/fetch_leases.go | 32 ++++- internal/database/lock_test.go | 41 ++++++ internal/database/schema.go | 9 +- internal/handler/fetch_lease.go | 85 +++++++++--- internal/handler/fetch_lease_test.go | 185 +++++++++++++++++++++++++-- internal/handler/handler.go | 66 ++++++++-- 7 files changed, 371 insertions(+), 49 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index c2417c1..50327ce 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -207,7 +207,7 @@ Instances coordinate through PostgreSQL advisory locks: - Eviction, reclaim, retention and the Gradle build cache sweep run on one instance at a time. An instance that finds a job running elsewhere skips that pass. - A [scheduled mirror](#scheduled-refresh) runs on whichever instance reaches it first in each interval. -Cache misses are fetched from upstream once across instances. The instance that fetches holds a lease on the miss in the `fetch_leases` table and renews it while it downloads. Another instance missing the same file waits until it is stored and serves the stored copy; `proxy_fetch_lease_waits_total` counts those waits. If the fetch fails, the waiting instance fetches itself, and a lease of an instance that stopped expires after 30 seconds. Metadata is shared this way only with [`cache_metadata`](#metadata-caching) and a `metadata_ttl` above zero, since without a stored copy there is nothing to wait for. A cache hit costs no extra query; a miss costs one. +Cache misses are fetched from upstream once across instances. The instance that fetches holds a lease on the miss in the `fetch_leases` table and renews it while it downloads. Another instance missing the same file waits until it is stored and serves the stored copy; `proxy_fetch_lease_waits_total` counts those waits. If the fetch fails, the instances waiting on it stop waiting together: each serves its stale metadata copy if it has one, and otherwise fetches itself, so a failing upstream costs every instance one attempt at the same time rather than one after another. A lease of an instance that stopped expires after 30 seconds. Metadata is shared this way only with [`cache_metadata`](#metadata-caching) and a `metadata_ttl` above zero, since without a stored copy there is nothing to wait for. A cache hit costs no extra query; a miss costs one. Each instance needs its own [`git.cache_dir`](#git-repositories). diff --git a/internal/database/fetch_leases.go b/internal/database/fetch_leases.go index 545b84e..56390e3 100644 --- a/internal/database/fetch_leases.go +++ b/internal/database/fetch_leases.go @@ -1,6 +1,8 @@ package database import ( + "database/sql" + "errors" "fmt" "time" ) @@ -13,13 +15,13 @@ import ( // single-instance, so these are only used on Postgres. // AcquireFetchLease takes the lease on key for owner unless another owner -// holds one that has not expired, and reports whether it did. +// holds one that has neither expired nor failed, and reports whether it did. func (db *DB) AcquireFetchLease(key, owner string, ttl time.Duration) (bool, error) { res, err := db.Exec(` INSERT INTO fetch_leases (key, owner, expires_at) VALUES ($1, $2, now() + $3 * interval '1 millisecond') - ON CONFLICT (key) DO UPDATE SET owner = excluded.owner, expires_at = excluded.expires_at - WHERE fetch_leases.expires_at < now()`, key, owner, ttl.Milliseconds()) + ON CONFLICT (key) DO UPDATE SET owner = excluded.owner, expires_at = excluded.expires_at, failed = FALSE + WHERE fetch_leases.expires_at < now() OR fetch_leases.failed`, key, owner, ttl.Milliseconds()) if err != nil { return false, fmt.Errorf("acquiring fetch lease: %w", err) } @@ -46,6 +48,30 @@ func (db *DB) RenewFetchLease(key, owner string, ttl time.Duration) (bool, error return n > 0, nil } +// FetchLeaseHolder returns the owner of the lease on key, and whether its +// fetch failed. ok is false when no lease is held or it has expired. +func (db *DB) FetchLeaseHolder(key string) (owner string, failed, ok bool, err error) { + err = db.QueryRow(` + SELECT owner, failed FROM fetch_leases + WHERE key = $1 AND expires_at >= now()`, key).Scan(&owner, &failed) + if errors.Is(err, sql.ErrNoRows) { + return "", false, false, nil + } + if err != nil { + return "", false, false, fmt.Errorf("reading fetch lease: %w", err) + } + return owner, failed, true, nil +} + +// FailFetchLease marks owner's lease on key as failed. Instances waiting on +// it stop waiting, and the next AcquireFetchLease takes it at once. +func (db *DB) FailFetchLease(key, owner string) error { + if _, err := db.Exec(`UPDATE fetch_leases SET failed = TRUE WHERE key = $1 AND owner = $2`, key, owner); err != nil { + return fmt.Errorf("failing fetch lease: %w", err) + } + return nil +} + // ReleaseFetchLease drops owner's lease on key. func (db *DB) ReleaseFetchLease(key, owner string) error { if _, err := db.Exec(`DELETE FROM fetch_leases WHERE key = $1 AND owner = $2`, key, owner); err != nil { diff --git a/internal/database/lock_test.go b/internal/database/lock_test.go index 9fe0186..11dd619 100644 --- a/internal/database/lock_test.go +++ b/internal/database/lock_test.go @@ -292,3 +292,44 @@ func TestFetchLeases(t *testing.T) { t.Fatalf("DeleteExpiredFetchLeases = %d, %v; want 1", n, err) } } + +func TestFetchLeaseFailure(t *testing.T) { + db := createTestPostgresDB(t) + if db == nil { + t.Skip("PROXY_DATABASE_URL not set") + } + defer func() { _ = db.Close() }() + + if ok, err := db.AcquireFetchLease("k", "a", time.Minute); err != nil || !ok { + t.Fatalf("a could not take a free lease: %v, %v", ok, err) + } + if owner, failed, ok, err := db.FetchLeaseHolder("k"); err != nil || !ok || owner != "a" || failed { + t.Fatalf("FetchLeaseHolder = %q, failed %v, ok %v, %v; want a's live lease", owner, failed, ok, err) + } + // A failed lease stays readable for those waiting on it, and is free to + // take. + if err := db.FailFetchLease("k", "b"); err != nil { + t.Fatal(err) + } + if _, failed, _, _ := db.FetchLeaseHolder("k"); failed { + t.Fatal("b failed a's lease") + } + if err := db.FailFetchLease("k", "a"); err != nil { + t.Fatal(err) + } + if owner, failed, ok, err := db.FetchLeaseHolder("k"); err != nil || !ok || owner != "a" || !failed { + t.Fatalf("FetchLeaseHolder = %q, failed %v, ok %v, %v; want a's failed lease", owner, failed, ok, err) + } + if ok, err := db.AcquireFetchLease("k", "b", time.Minute); err != nil || !ok { + t.Fatalf("b could not take a's failed lease: %v, %v", ok, err) + } + if owner, failed, _, _ := db.FetchLeaseHolder("k"); owner != "b" || failed { + t.Fatalf("after takeover: owner %q, failed %v", owner, failed) + } + if err := db.ReleaseFetchLease("k", "b"); err != nil { + t.Fatal(err) + } + if _, _, ok, err := db.FetchLeaseHolder("k"); err != nil || ok { + t.Fatalf("FetchLeaseHolder after release: ok %v, %v", ok, err) + } +} diff --git a/internal/database/schema.go b/internal/database/schema.go index 9b10039..74cc00f 100644 --- a/internal/database/schema.go +++ b/internal/database/schema.go @@ -95,7 +95,8 @@ CREATE TABLE IF NOT EXISTS job_runs ( CREATE TABLE IF NOT EXISTS fetch_leases ( key TEXT NOT NULL PRIMARY KEY, owner TEXT NOT NULL, - expires_at DATETIME NOT NULL + expires_at DATETIME NOT NULL, + failed BOOLEAN NOT NULL DEFAULT FALSE ); CREATE TABLE IF NOT EXISTS vulnerabilities ( @@ -230,7 +231,8 @@ CREATE TABLE IF NOT EXISTS job_runs ( CREATE TABLE IF NOT EXISTS fetch_leases ( key TEXT NOT NULL PRIMARY KEY, owner TEXT NOT NULL, - expires_at TIMESTAMP NOT NULL + expires_at TIMESTAMP NOT NULL, + failed BOOLEAN NOT NULL DEFAULT FALSE ); CREATE TABLE IF NOT EXISTS vulnerabilities ( @@ -771,7 +773,8 @@ func migrateAddFetchLeases(db *DB) error { query := fmt.Sprintf(`CREATE TABLE IF NOT EXISTS fetch_leases ( key TEXT NOT NULL PRIMARY KEY, owner TEXT NOT NULL, - expires_at %s NOT NULL + expires_at %s NOT NULL, + failed BOOLEAN NOT NULL DEFAULT FALSE )`, ts) if _, err := db.Exec(query); err != nil { return fmt.Errorf("creating fetch_leases table: %w", err) diff --git a/internal/handler/fetch_lease.go b/internal/handler/fetch_lease.go index 47dcbf5..e49b7ef 100644 --- a/internal/handler/fetch_lease.go +++ b/internal/handler/fetch_lease.go @@ -44,21 +44,32 @@ func (p *Proxy) fetchLeasesEnabled() bool { return p.DB != nil && p.DB.Dialect() == database.DialectPostgres } +// leaseOutcome says how a caller goes on after awaitFetchLease. +type leaseOutcome int + +const ( + // leaseFetch: this instance fetches, and calls release afterwards. + leaseFetch leaseOutcome = iota + // leaseStored: ready found the result another instance stored. + leaseStored + // leaseFailed: the fetch this caller waited on failed. It fetches + // without the lease, as every instance that waited does at the same + // time, or serves a stale copy, rather than queueing behind the others. + leaseFailed +) + // awaitFetchLease takes the lease on key before this instance fetches it from // upstream. While another instance holds the lease, it polls ready, which // reports whether that instance's result is now stored and, if so, keeps it -// for the caller. It returns: -// - stored true when ready found the result: serve it, nothing to release. -// - release when this instance should fetch: call it once the result is -// stored or the fetch failed. -// - err when ctx ended while waiting. +// for the caller. release is to be called with whether the fetch failed once +// it is done; err is set when ctx ended while waiting. // // A failing lease query is logged and the fetch goes ahead without a lease, // so a database hiccup costs a duplicate fetch rather than the request. -func (p *Proxy) awaitFetchLease(ctx context.Context, kind, key string, ready func() bool) (release func(), stored bool, err error) { - noop := func() {} +func (p *Proxy) awaitFetchLease(ctx context.Context, kind, key string, ready func() bool) (release func(failed bool), outcome leaseOutcome, err error) { + noop := func(bool) {} if !p.fetchLeasesEnabled() { - return noop, false, nil + return noop, leaseFetch, nil } leaseKey := fetchLeaseKey(kind, key) owner := newLeaseOwner() @@ -68,38 +79,67 @@ func (p *Proxy) awaitFetchLease(ctx context.Context, kind, key string, ready fun acquired, err := p.DB.AcquireFetchLease(leaseKey, owner, fetchLeaseTTL) if err != nil { p.Logger.Warn("fetch lease unavailable, fetching without it", "kind", kind, "error", err) - return noop, false, nil + return noop, leaseFetch, nil } if acquired { // The holder may have stored the result and released the lease // between this caller's last poll and taking it. if waited && ready() { p.releaseFetchLease(leaseKey, owner) - return noop, true, nil + return noop, leaseStored, nil } - return p.holdFetchLease(leaseKey, owner), false, nil + return p.holdFetchLease(leaseKey, owner), leaseFetch, nil } if !waited { metrics.RecordFetchLeaseWait(kind) waited = true } - timer := time.NewTimer(delay) + outcome, done, err := p.waitForLeaseHolder(ctx, kind, leaseKey, ready, &delay) + if done || err != nil { + return noop, outcome, err + } + } +} + +// waitForLeaseHolder polls ready while another instance holds the lease on +// leaseKey. It is done when ready found the result, when the fetch it waited +// on failed, or when the lease can't be read; otherwise it returns once the +// lease is free to take. +func (p *Proxy) waitForLeaseHolder(ctx context.Context, kind, leaseKey string, ready func() bool, delay *time.Duration) (outcome leaseOutcome, done bool, err error) { + holder := "" + for { + current, failed, held, err := p.DB.FetchLeaseHolder(leaseKey) + if err != nil { + p.Logger.Warn("fetch lease unavailable, fetching without it", "kind", kind, "error", err) + return leaseFetch, true, nil + } + if failed && current == holder { + return leaseFailed, true, nil + } + if !held || failed { + // Released, expired, or failed before this caller waited on + // it: free to take. + return leaseFetch, false, nil + } + holder = current + timer := time.NewTimer(*delay) select { case <-ctx.Done(): timer.Stop() - return nil, false, ctx.Err() + return leaseFetch, true, ctx.Err() case <-timer.C: } if ready() { - return noop, true, nil + return leaseStored, true, nil } - delay = min(delay+delay, fetchLeasePollMax) + *delay = min(*delay+*delay, fetchLeasePollMax) } } // holdFetchLease renews owner's lease on leaseKey until the returned release -// is called, which also drops it. -func (p *Proxy) holdFetchLease(leaseKey, owner string) func() { +// is called. release drops the lease, or marks it failed so instances +// waiting on it stop waiting. +func (p *Proxy) holdFetchLease(leaseKey, owner string) func(failed bool) { stop := make(chan struct{}) done := make(chan struct{}) go func() { @@ -120,10 +160,17 @@ func (p *Proxy) holdFetchLease(leaseKey, owner string) func() { } } }() - return func() { + return func(failed bool) { close(stop) <-done - p.releaseFetchLease(leaseKey, owner) + if !failed { + p.releaseFetchLease(leaseKey, owner) + return + } + if err := p.DB.FailFetchLease(leaseKey, owner); err != nil { + // Waiting instances take the lease once it expires. + p.Logger.Warn("failed to mark fetch lease failed", "error", err) + } } } diff --git a/internal/handler/fetch_lease_test.go b/internal/handler/fetch_lease_test.go index a71f138..0be8d42 100644 --- a/internal/handler/fetch_lease_test.go +++ b/internal/handler/fetch_lease_test.go @@ -6,6 +6,7 @@ import ( "io" "log/slog" "net/http" + "net/http/httptest" "strings" "sync" "sync/atomic" @@ -78,7 +79,11 @@ func sharedInstances(t *testing.T) (*Proxy, *Proxy, *database.DB) { const leaseTestURL = "https://registry.npmjs.org/left-pad/-/left-pad-1.3.0.tgz" func fetchLeftPad(p *Proxy) (string, error) { - res, err := p.GetOrFetchArtifactFromURL(context.Background(), "npm", "left-pad", "1.3.0", "left-pad-1.3.0.tgz", leaseTestURL) + return fetchLeftPadCtx(context.Background(), p) +} + +func fetchLeftPadCtx(ctx context.Context, p *Proxy) (string, error) { + res, err := p.GetOrFetchArtifactFromURL(ctx, "npm", "left-pad", "1.3.0", "left-pad-1.3.0.tgz", leaseTestURL) if err != nil { return "", err } @@ -92,10 +97,10 @@ type fetchResult struct { err error } -func goFetchLeftPad(p *Proxy) chan fetchResult { +func goFetchLeftPad(ctx context.Context, p *Proxy) chan fetchResult { ch := make(chan fetchResult, 1) go func() { - body, err := fetchLeftPad(p) + body, err := fetchLeftPadCtx(ctx, p) ch <- fetchResult{body, err} }() return ch @@ -114,6 +119,28 @@ func waitForLeaseWaits(t *testing.T, kind string, from float64) { } } +// waitForArtifactWaiters returns once n callers have joined p's fetch of +// left-pad. +func waitForArtifactWaiters(t *testing.T, p *Proxy, n int) { + t.Helper() + deadline := time.Now().Add(5 * time.Second) + for { + p.fetchMu.Lock() + joined := 0 + for _, f := range p.inFlight { + joined += f.waiters + } + p.fetchMu.Unlock() + if joined >= n { + return + } + if time.Now().After(deadline) { + t.Fatal("timed out waiting for a caller to join the fetch") + } + time.Sleep(time.Millisecond) + } +} + func TestFetchLeaseCoalescesArtifactAcrossInstances(t *testing.T) { a, b, _ := sharedInstances(t) const content = "left-pad tarball" @@ -123,9 +150,9 @@ func TestFetchLeaseCoalescesArtifactAcrossInstances(t *testing.T) { a.Fetcher, b.Fetcher = fa, fb waits := testutil.ToFloat64(metrics.FetchLeaseWaits.WithLabelValues(metrics.CacheKindArtifact)) - resA := goFetchLeftPad(a) + resA := goFetchLeftPad(context.Background(), a) <-fa.entered - resB := goFetchLeftPad(b) + resB := goFetchLeftPad(context.Background(), b) waitForLeaseWaits(t, metrics.CacheKindArtifact, waits) fa.open() @@ -152,9 +179,9 @@ func TestFetchLeaseFailedFetchLetsOtherInstanceFetch(t *testing.T) { a.Fetcher, b.Fetcher = fa, fb waits := testutil.ToFloat64(metrics.FetchLeaseWaits.WithLabelValues(metrics.CacheKindArtifact)) - resA := goFetchLeftPad(a) + resA := goFetchLeftPad(context.Background(), a) <-fa.entered - resB := goFetchLeftPad(b) + resB := goFetchLeftPad(context.Background(), b) waitForLeaseWaits(t, metrics.CacheKindArtifact, waits) started := time.Now() fa.open() @@ -252,14 +279,150 @@ func TestFetchLeaseNotUsedWithoutMetadataCache(t *testing.T) { } } +// TestFetchLeaseWaitOutlivesDetachedCaller checks that a caller waiting on +// another instance's fetch keeps waiting after its client goes away when its +// fetch is detached, so callers that joined it still get the artifact. +func TestFetchLeaseWaitOutlivesDetachedCaller(t *testing.T) { + a, b, _ := sharedInstances(t) + const content = "left-pad tarball" + fa := newGateFetcher(t, content, nil) + fb := newGateFetcher(t, content, nil) + fb.open() + a.Fetcher, b.Fetcher = fa, fb + + waits := testutil.ToFloat64(metrics.FetchLeaseWaits.WithLabelValues(metrics.CacheKindArtifact)) + resA := goFetchLeftPad(context.Background(), a) + <-fa.entered + leadCtx, cancelLead := context.WithCancel(DetachUpstreamFetch(context.Background())) + defer cancelLead() + resLead := goFetchLeftPad(leadCtx, b) + waitForLeaseWaits(t, metrics.CacheKindArtifact, waits) + resJoined := goFetchLeftPad(context.Background(), b) + waitForArtifactWaiters(t, b, 1) + cancelLead() + fa.open() + <-resLead + + for name, ch := range map[string]chan fetchResult{"a": resA, "b's joined caller": resJoined} { + r := <-ch + if r.err != nil || r.body != content { + t.Errorf("%s: body %q, err %v", name, r.body, r.err) + } + } + if got := fb.calls.Load(); got != 0 { + t.Errorf("b fetched %d times, want 0", got) + } +} + +// TestFetchLeaseFailedFetchWaitersFetchInParallel checks that once a fetch +// fails, the instances waiting on it fetch at the same time rather than one +// after another, so a hanging upstream costs each one timeout. +func TestFetchLeaseFailedFetchWaitersFetchInParallel(t *testing.T) { + a, b, db := sharedInstances(t) + c := NewProxy(db, b.Storage, &mockFetcher{}, fetch.NewResolver(), b.Logger) + const content = "left-pad tarball" + fa := newGateFetcher(t, "", errors.New("upstream down")) + fb := newGateFetcher(t, content, nil) + fc := newGateFetcher(t, content, nil) + a.Fetcher, b.Fetcher, c.Fetcher = fa, fb, fc + + waits := testutil.ToFloat64(metrics.FetchLeaseWaits.WithLabelValues(metrics.CacheKindArtifact)) + resA := goFetchLeftPad(context.Background(), a) + <-fa.entered + resB := goFetchLeftPad(context.Background(), b) + waitForLeaseWaits(t, metrics.CacheKindArtifact, waits) + resC := goFetchLeftPad(context.Background(), c) + waitForLeaseWaits(t, metrics.CacheKindArtifact, waits+1) + fa.open() + if r := <-resA; r.err == nil { + t.Fatal("instance a: want the upstream error") + } + + for name, entered := range map[string]chan struct{}{"b": fb.entered, "c": fc.entered} { + select { + case <-entered: + case <-time.After(5 * time.Second): + t.Fatalf("%s did not fetch while the other waiting instance was fetching", name) + } + } + fb.open() + fc.open() + for name, ch := range map[string]chan fetchResult{"b": resB, "c": resC} { + if r := <-ch; r.err != nil || r.body != content { + t.Errorf("instance %s: body %q, err %v", name, r.body, r.err) + } + } +} + +// TestFetchLeaseFailedMetadataServesStale checks that an instance waiting on +// a metadata fetch that fails serves its stale copy at once, as the holder +// does, instead of asking the failing upstream again. +func TestFetchLeaseFailedMetadataServesStale(t *testing.T) { + a, b, _ := sharedInstances(t) + var status atomic.Int32 + status.Store(http.StatusOK) + var calls atomic.Int32 + entered := make(chan struct{}, 1) + release := make(chan struct{}) + var released sync.Once + open := func() { released.Do(func() { close(release) }) } + t.Cleanup(open) + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls.Add(1) + if code := int(status.Load()); code != http.StatusOK { + entered <- struct{}{} + <-release + w.WriteHeader(code) + return + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"name":"left-pad"}`)) + })) + t.Cleanup(upstream.Close) + for _, p := range []*Proxy{a, b} { + p.HTTPClient = upstream.Client() + p.CacheMetadata = true + p.MetadataTTL = 50 * time.Millisecond + } + fetchMeta := func(p *Proxy) chan metadataCall { + ch := make(chan metadataCall, 1) + go func() { + body, _, err := p.FetchOrCacheMetadata(context.Background(), "npm", "left-pad", upstream.URL+"/left-pad", "application/json") + ch <- metadataCall{body, err} + }() + return ch + } + if r := <-fetchMeta(a); r.err != nil { + t.Fatalf("seeding the cache: %v", r.err) + } + time.Sleep(100 * time.Millisecond) + + status.Store(http.StatusServiceUnavailable) + waits := testutil.ToFloat64(metrics.FetchLeaseWaits.WithLabelValues(metrics.CacheKindMetadata)) + resA := fetchMeta(a) + <-entered + resB := fetchMeta(b) + waitForLeaseWaits(t, metrics.CacheKindMetadata, waits) + open() + + for name, ch := range map[string]chan metadataCall{"a": resA, "b": resB} { + if r := <-ch; r.err != nil || string(r.body) != `{"name":"left-pad"}` { + t.Errorf("instance %s: body %q, err %v; want the stale copy", name, r.body, r.err) + } + } + if got := calls.Load(); got != 2 { + t.Errorf("upstream requests = %d, want 2: the seed and a's failed refresh", got) + } +} + func TestFetchLeaseNotUsedOnSQLite(t *testing.T) { proxy, _, _, _ := setupTestProxy(t) - release, stored, err := proxy.awaitFetchLease(context.Background(), metrics.CacheKindArtifact, "k", func() bool { + release, outcome, err := proxy.awaitFetchLease(context.Background(), metrics.CacheKindArtifact, "k", func() bool { t.Error("ready polled on SQLite") return false }) - if err != nil || stored { - t.Fatalf("awaitFetchLease = %v, %v; want to fetch at once", stored, err) + if err != nil || outcome != leaseFetch { + t.Fatalf("awaitFetchLease = %v, %v; want to fetch at once", outcome, err) } - release() + release(false) } diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 2aa1515..542dd47 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -744,6 +744,10 @@ type inflightFetch struct { done chan struct{} val fetchedArtifact err error + + // waiters counts callers that joined the fetch, guarded by fetchMu. + // Tests read it to know a caller has joined. + waiters int } // coalesceFetch runs commit at most once for concurrent callers sharing key, @@ -776,7 +780,9 @@ func (p *Proxy) coalesceFetch(ctx context.Context, key string, recheck func() (a p.inFlight = make(map[string]*inflightFetch) } f, joined := p.inFlight[key] - if !joined { + if joined { + f.waiters++ + } else { f = &inflightFetch{done: make(chan struct{})} p.inFlight[key] = f } @@ -815,7 +821,7 @@ func (p *Proxy) runSharedFetch(ctx context.Context, key string, f *inflightFetch // in between would otherwise be repeated. Serve that record only if its // bytes are still present: a record can outlive them, and refetching is // the same recovery the cache lookup makes. - served := func() (*CacheResult, bool) { + served := func(ctx context.Context) (*CacheResult, bool) { stored, path, ok := recheck() if !ok { return nil, false @@ -827,15 +833,22 @@ func (p *Proxy) runSharedFetch(ctx context.Context, key string, f *inflightFetch f.val, f.err = fetchedArtifact{artifact: stored, storagePath: path}, nil return res, true } - if res, ok := served(); ok { + if res, ok := served(ctx); ok { return res, nil } // Another instance sharing the database may be fetching the same // artifact. Wait for it to store the artifact rather than fetch it too. + // The wait makes no upstream request, so a detached fetch waits past + // its client too, bounded by the holder's fetch and the lease expiry; + // otherwise callers that joined this one would get its cancellation. + waitCtx := ctx + if upstreamFetchDetached(ctx) { + waitCtx = context.WithoutCancel(ctx) + } var waitedFor *CacheResult - release, storedElsewhere, err := p.awaitFetchLease(ctx, metrics.CacheKindArtifact, key, func() bool { - res, ok := served() + release, outcome, err := p.awaitFetchLease(waitCtx, metrics.CacheKindArtifact, key, func() bool { + res, ok := served(waitCtx) waitedFor = res return ok }) @@ -843,10 +856,14 @@ func (p *Proxy) runSharedFetch(ctx context.Context, key string, f *inflightFetch f.err = err return nil, err } - if storedElsewhere { + if outcome == leaseStored { return waitedFor, nil } - defer release() + // With leaseFailed, release is a no-op and this instance fetches on its + // own, so a failing upstream costs each instance one attempt in + // parallel. + failed := true + defer func() { release(failed) }() commitCtx := ctx if upstreamFetchDetached(ctx) && p.acquireDetachSlot() { @@ -854,6 +871,7 @@ func (p *Proxy) runSharedFetch(ctx context.Context, key string, f *inflightFetch commitCtx = context.WithoutCancel(ctx) } stored, path, err := commit(commitCtx) + failed = err != nil f.val, f.err = fetchedArtifact{artifact: stored, storagePath: path}, err if err != nil { return nil, err @@ -1398,20 +1416,35 @@ func (p *Proxy) coalescedMetadataMiss(ctx context.Context, ecosystem, cacheKey, // Another instance sharing the database may be fetching the same // document. Only a stored copy within TTL can be served after // waiting, so without that there is nothing to wait for. + release := func(bool) {} if p.CacheMetadata && p.MetadataTTL > 0 { - release, stored, err := p.awaitFetchLease(fetchCtx, metrics.CacheKindMetadata, key, func() bool { + var outcome leaseOutcome + var err error + release, outcome, err = p.awaitFetchLease(fetchCtx, metrics.CacheKindMetadata, key, func() bool { entry, hit = p.cachedMetadataState(fetchCtx, ecosystem, cacheKey, validate) return hit != nil }) if err != nil { return metadataResult{err: err} } - if stored { + switch outcome { + case leaseStored: return *hit + case leaseFailed: + // Serve the stale copy at once, as the holder did, rather + // than trying the failing upstream again. Without one, fetch. + if entry != nil { + if res := p.staleMetadata(fetchCtx, ecosystem, cacheKey, validate, entry, errFetchFailedElsewhere); res.err == nil { + return res + } + } + case leaseFetch: } - defer release() } - return p.fetchMetadataFromUpstream(fetchCtx, ecosystem, cacheKey, upstreamURL, accept, acceptEncoding, validate, entry) + res := metadataResult{err: errSharedFetchAbandoned} + defer func() { release(res.err != nil || res.stale) }() + res = p.fetchMetadataFromUpstream(fetchCtx, ecosystem, cacheKey, upstreamURL, accept, acceptEncoding, validate, entry) + return res }) // The fetch ran the first caller's validate. A caller that joined it runs // its own on the shared bytes, since validate may also decode them for it. @@ -1470,7 +1503,16 @@ func (p *Proxy) fetchMetadataFromUpstream(ctx context.Context, ecosystem, cacheK return metadataResult{body: meta.body, contentType: meta.contentType, contentEncoding: meta.contentEncoding, digest: digest} } - // Upstream failed -- fall back to cache if available + return p.staleMetadata(ctx, ecosystem, cacheKey, validate, entry, err) +} + +// errFetchFailedElsewhere is the upstream error behind a stale copy served +// because another instance's fetch for it failed. +var errFetchFailedElsewhere = errors.New("upstream fetch failed on another instance") + +// staleMetadata falls back to the cached copy in entry after the upstream +// fetch failed with err. +func (p *Proxy) staleMetadata(ctx context.Context, ecosystem, cacheKey string, validate func([]byte) error, entry *database.MetadataCacheEntry, err error) metadataResult { if !p.CacheMetadata || entry == nil { return metadataResult{err: fmt.Errorf("upstream failed and no cached metadata: %w", err)} } From 0acc41490e1af28de33f259e455aec83983ba706 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Fri, 23 Oct 2026 19:34:09 +0200 Subject: [PATCH 375/633] Stop mirror jobs before a job store test cleans up The job store tests left their jobs running after the test, and on Windows the test's directories could not be removed while a job was still writing to them. JobStore.Wait now waits for jobs to return, and the tests cancel their jobs and wait for them. --- internal/mirror/job.go | 10 ++++++- internal/mirror/job_test.go | 47 ++++++++++++++++-------------- internal/server/mirror_api_test.go | 9 +++++- 3 files changed, 42 insertions(+), 24 deletions(-) diff --git a/internal/mirror/job.go b/internal/mirror/job.go index 3d82966..202cf1e 100644 --- a/internal/mirror/job.go +++ b/internal/mirror/job.go @@ -50,6 +50,8 @@ type JobStore struct { jobs map[string]*Job mirror *Mirror parentCtx context.Context + // running counts the jobs whose goroutine has not returned yet. + running sync.WaitGroup } // NewJobStore creates a new job store. The parent context is used as the base @@ -83,7 +85,7 @@ func (js *JobStore) Create(req JobRequest) (string, error) { js.jobs[id] = job js.mu.Unlock() - go js.runJob(ctx, cancel, job, source) + js.running.Go(func() { js.runJob(ctx, cancel, job, source) }) return id, nil } @@ -125,6 +127,12 @@ func (js *JobStore) Cancel(id string) bool { return true } +// Wait blocks until every job has returned. Cancel the parent context first +// to stop running jobs. +func (js *JobStore) Wait() { + js.running.Wait() +} + // Cleanup removes completed/failed/canceled jobs older than jobTTL. func (js *JobStore) Cleanup() { js.mu.Lock() diff --git a/internal/mirror/job_test.go b/internal/mirror/job_test.go index c0ac3f1..7b4c88e 100644 --- a/internal/mirror/job_test.go +++ b/internal/mirror/job_test.go @@ -7,9 +7,22 @@ import ( "time" ) -func TestJobStoreCreateAndGet(t *testing.T) { +// newTestJobStore returns a job store whose jobs are stopped and waited for +// before the test's database and directories are removed. +func newTestJobStore(t *testing.T) *JobStore { + t.Helper() m := setupTestMirror(t, 1) - js := NewJobStore(context.Background(), m) + ctx, cancel := context.WithCancel(context.Background()) + js := NewJobStore(ctx, m) + t.Cleanup(func() { + cancel() + js.Wait() + }) + return js +} + +func TestJobStoreCreateAndGet(t *testing.T) { + js := newTestJobStore(t) id, err := js.Create(JobRequest{ PURLs: []string{"pkg:npm/lodash@4.17.21"}, @@ -35,8 +48,7 @@ func TestJobStoreCreateAndGet(t *testing.T) { } func TestJobStoreGetNotFound(t *testing.T) { - m := setupTestMirror(t, 1) - js := NewJobStore(context.Background(), m) + js := newTestJobStore(t) job := js.Get("nonexistent") if job != nil { @@ -45,8 +57,7 @@ func TestJobStoreGetNotFound(t *testing.T) { } func TestJobStoreCancelNotFound(t *testing.T) { - m := setupTestMirror(t, 1) - js := NewJobStore(context.Background(), m) + js := newTestJobStore(t) if js.Cancel("nonexistent") { t.Error("expected Cancel to return false for nonexistent job") @@ -54,8 +65,7 @@ func TestJobStoreCancelNotFound(t *testing.T) { } func TestJobStoreCreateInvalidRequest(t *testing.T) { - m := setupTestMirror(t, 1) - js := NewJobStore(context.Background(), m) + js := newTestJobStore(t) _, err := js.Create(JobRequest{}) if err == nil { @@ -64,8 +74,7 @@ func TestJobStoreCreateInvalidRequest(t *testing.T) { } func TestJobStoreMultipleJobs(t *testing.T) { - m := setupTestMirror(t, 1) - js := NewJobStore(context.Background(), m) + js := newTestJobStore(t) id1, err := js.Create(JobRequest{PURLs: []string{"pkg:npm/lodash@4.17.21"}}) if err != nil { @@ -89,8 +98,7 @@ func TestJobStoreMultipleJobs(t *testing.T) { } func TestSourceFromRequestPURLs(t *testing.T) { - m := setupTestMirror(t, 1) - js := NewJobStore(context.Background(), m) + js := newTestJobStore(t) source, err := js.sourceFromRequest(JobRequest{PURLs: []string{"pkg:npm/lodash@1.0.0"}}) if err != nil { @@ -102,8 +110,7 @@ func TestSourceFromRequestPURLs(t *testing.T) { } func TestSourceFromRequestSBOM(t *testing.T) { - m := setupTestMirror(t, 1) - js := NewJobStore(context.Background(), m) + js := newTestJobStore(t) sbom := json.RawMessage(`{"bomFormat":"CycloneDX","components":[]}`) source, err := js.sourceFromRequest(JobRequest{SBOM: sbom}) @@ -120,8 +127,7 @@ func TestSourceFromRequestSBOM(t *testing.T) { } func TestSourceFromRequestRejectsPURLsAndSBOM(t *testing.T) { - m := setupTestMirror(t, 1) - js := NewJobStore(context.Background(), m) + js := newTestJobStore(t) _, err := js.sourceFromRequest(JobRequest{ PURLs: []string{"pkg:npm/lodash@4.17.21"}, @@ -133,8 +139,7 @@ func TestSourceFromRequestRejectsPURLsAndSBOM(t *testing.T) { } func TestSourceFromRequestRegistryRejected(t *testing.T) { - m := setupTestMirror(t, 1) - js := NewJobStore(context.Background(), m) + js := newTestJobStore(t) _, err := js.sourceFromRequest(JobRequest{Registry: "npm"}) if err == nil { @@ -143,8 +148,7 @@ func TestSourceFromRequestRegistryRejected(t *testing.T) { } func TestJobStoreCleanup(t *testing.T) { - m := setupTestMirror(t, 1) - js := NewJobStore(context.Background(), m) + js := newTestJobStore(t) // Add a completed job with old CreatedAt js.mu.Lock() @@ -179,8 +183,7 @@ func TestJobStoreCleanup(t *testing.T) { } func TestJobStoreCancelPreservesStateAfterRunJob(t *testing.T) { - m := setupTestMirror(t, 1) - js := NewJobStore(context.Background(), m) + js := newTestJobStore(t) // Create a job with a PURL that will fail (no real upstream in test) id, err := js.Create(JobRequest{PURLs: []string{"pkg:npm/nonexistent-pkg@0.0.0"}}) diff --git a/internal/server/mirror_api_test.go b/internal/server/mirror_api_test.go index c960348..05dc2af 100644 --- a/internal/server/mirror_api_test.go +++ b/internal/server/mirror_api_test.go @@ -43,7 +43,14 @@ func setupMirrorAPI(t *testing.T) *MirrorAPIHandler { proxy := handler.NewProxy(db, store, fetcher, resolver, logger) m := mirror.New(proxy, db, store, logger, 1) - js := mirror.NewJobStore(context.Background(), m) + // Jobs are stopped and waited for before the database and directories + // are removed. + ctx, cancel := context.WithCancel(context.Background()) + js := mirror.NewJobStore(ctx, m) + t.Cleanup(func() { + cancel() + js.Wait() + }) return NewMirrorAPIHandler(js) } From 35df59f8f272607464f7c7c38e607456bfc8553f Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Fri, 23 Oct 2026 19:34:09 +0200 Subject: [PATCH 376/633] Wait for background jobs before closing the database at shutdown Shutdown cancelled the eviction, retention, reclaim, backfill and stats loops but closed storage and the database without waiting for them, and the stats loop never stopped. A pass still running kept the database open, which on Windows left proxy.db locked after a server test. Shutdown now waits up to 30 seconds for them, and for mirror jobs. --- internal/server/gradle_cache_eviction.go | 4 +- internal/server/server.go | 57 +++++++++++++++++++----- 2 files changed, 47 insertions(+), 14 deletions(-) diff --git a/internal/server/gradle_cache_eviction.go b/internal/server/gradle_cache_eviction.go index 7f546d1..20cdff2 100644 --- a/internal/server/gradle_cache_eviction.go +++ b/internal/server/gradle_cache_eviction.go @@ -49,7 +49,7 @@ func (s *Server) startGradleBuildCacheEviction(ctx context.Context) { sweep() - go func() { + s.background.Go(func() { ticker := time.NewTicker(interval) defer ticker.Stop() @@ -61,7 +61,7 @@ func (s *Server) startGradleBuildCacheEviction(ctx context.Context) { sweep() } } - }() + }) } func sweepGradleBuildCache( diff --git a/internal/server/server.go b/internal/server/server.go index 5b0ebf7..14e09b8 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -65,6 +65,7 @@ import ( "net/url" "strconv" "strings" + "sync" "time" "github.com/git-pkgs/archives" @@ -130,6 +131,10 @@ type Server struct { browseLimits archives.StreamOptions compares compareLimiter + + // background counts the goroutines Start runs until cancel, which + // Shutdown waits for before it closes storage and the database. + background sync.WaitGroup } // New creates a new Server with the given configuration. @@ -388,7 +393,12 @@ func (s *Server) serve(listener net.Listener) error { r.Post("/api/mirror", mirrorAPI.HandleCreate) r.Get("/api/mirror/{id}", mirrorAPI.HandleGet) r.Delete("/api/mirror/{id}", mirrorAPI.HandleCancel) - go jobStore.StartCleanup(bgCtx) + s.background.Go(func() { jobStore.StartCleanup(bgCtx) }) + // Mirror jobs stop with bgCtx; their fetches still use the database. + s.background.Go(func() { + <-bgCtx.Done() + jobStore.Wait() + }) } s.startMirrorSchedules(bgCtx, proxy) @@ -415,11 +425,11 @@ func (s *Server) serve(listener net.Listener) error { for _, warning := range s.cfg.EnvWarnings() { s.logger.Warn(warning) } - go s.updateCacheStatsMetrics() - go backfillMetadataAccess(bgCtx, s.db, s.logger, metadataBackfillBatch) - go s.startEvictionLoop(bgCtx) - go s.startRetentionLoop(bgCtx) - go s.startReclaimLoop(bgCtx) + s.background.Go(func() { s.updateCacheStatsMetrics(bgCtx) }) + s.background.Go(func() { backfillMetadataAccess(bgCtx, s.db, s.logger, metadataBackfillBatch) }) + s.background.Go(func() { s.startEvictionLoop(bgCtx) }) + s.background.Go(func() { s.startRetentionLoop(bgCtx) }) + s.background.Go(func() { s.startReclaimLoop(bgCtx) }) if listener != nil { return s.http.Serve(listener) @@ -631,17 +641,18 @@ func upstreamSafeHTTPOptions(upstream config.UpstreamConfig) safehttp.Options { } // updateCacheStatsMetrics periodically updates cache statistics in Prometheus metrics. -func (s *Server) updateCacheStatsMetrics() { +func (s *Server) updateCacheStatsMetrics(ctx context.Context) { ticker := time.NewTicker(1 * time.Minute) defer ticker.Stop() - // Update once immediately - s.updateCacheStats() - s.updateMetadataCacheStats() - - for range ticker.C { + for { s.updateCacheStats() s.updateMetadataCacheStats() + select { + case <-ctx.Done(): + return + case <-ticker.C: + } } } @@ -712,6 +723,7 @@ func (s *Server) Shutdown(ctx context.Context) error { } s.waitForMirrorSchedules(ctx) + s.waitForBackground(ctx) if s.git != nil { if err := s.git.Close(); err != nil { @@ -743,6 +755,27 @@ func (s *Server) Shutdown(ctx context.Context) error { return nil } +// backgroundShutdownWait bounds how long shutdown waits for background jobs +// to notice they were cancelled. +const backgroundShutdownWait = 30 * time.Second + +// waitForBackground waits for the goroutines Start ran to return, so none of +// them still uses storage or the database when they are closed. +func (s *Server) waitForBackground(ctx context.Context) { + ctx, cancel := context.WithTimeout(ctx, backgroundShutdownWait) + defer cancel() + done := make(chan struct{}) + go func() { + s.background.Wait() + close(done) + }() + select { + case <-done: + case <-ctx.Done(): + s.logger.Warn("background jobs still running at shutdown") + } +} + // authForURL returns the authentication header for a given URL based on config. func (s *Server) authForURL(url string) (headerName, headerValue string) { auth := s.cfg.Upstream.AuthForURL(url) From cdd3bea338885184cc2bf5da88a48ef287dead02 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Fri, 23 Oct 2026 19:34:09 +0200 Subject: [PATCH 377/633] Cut the relay connection on any upstream copy error A response that ended early for another reason than the size limit was passed on as complete. TestRelayStopsAtLimit failed that way in CI. The test upstream now reads the request body as a git server does, and the test prints the relay's log when it fails. --- internal/gitcache/relay.go | 12 +++++++----- internal/gitcache/relay_test.go | 27 ++++++++++++++++++++++++--- 2 files changed, 31 insertions(+), 8 deletions(-) diff --git a/internal/gitcache/relay.go b/internal/gitcache/relay.go index 38baa29..28f87f9 100644 --- a/internal/gitcache/relay.go +++ b/internal/gitcache/relay.go @@ -152,14 +152,16 @@ func (rl *relay) ServeHTTP(w http.ResponseWriter, r *http.Request) { } w.WriteHeader(resp.StatusCode) if _, err := io.Copy(w, &routeReader{ctx: r.Context(), r: resp.Body, route: route}); err != nil { - if route.exceeded.Load() || route.full.Load() { - // Cut the connection so git fails rather than keeping a pack - // that merely looks short. - panic(http.ErrAbortHandler) + if r.Context().Err() != nil { + // git went away; there is no one to tell. + return } - if !errors.Is(err, context.Canceled) { + if !route.exceeded.Load() && !route.full.Load() { rl.logger.Warn("git relay: copying upstream response", "upstream", route.repoURL, "error", err) } + // Cut the connection so git fails rather than taking a response + // that ended early for one that is complete. + panic(http.ErrAbortHandler) } } diff --git a/internal/gitcache/relay_test.go b/internal/gitcache/relay_test.go index 4ee9142..ea1d063 100644 --- a/internal/gitcache/relay_test.go +++ b/internal/gitcache/relay_test.go @@ -6,6 +6,7 @@ import ( "net/http" "net/http/httptest" "strings" + "sync" "sync/atomic" "testing" ) @@ -66,11 +67,18 @@ func TestRelayOnlyForwardsFetchRequests(t *testing.T) { // rather than downloaded in full and thrown away. func TestRelayStopsAtLimit(t *testing.T) { const size, limit = 4 << 20, 64 << 10 - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = io.Copy(io.Discard, r.Body) _, _ = io.Copy(w, io.LimitReader(zeros{}, size)) })) defer upstream.Close() - rl := newTestRelay(t) + var logs strings.Builder + var logMu sync.Mutex + rl, err := newRelay(&http.Client{}, slog.New(slog.NewTextHandler(lockedWriter{&logMu, &logs}, nil))) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = rl.close() }) base, route, unregister := rl.register(upstream.URL+"/repo", limit, nil) defer unregister() @@ -84,10 +92,23 @@ func TestRelayStopsAtLimit(t *testing.T) { t.Errorf("read %d bytes without an error", n) } if !route.exceeded.Load() || route.relayed.Load() > size/2 { - t.Errorf("exceeded %v after relaying %d of %d bytes", route.exceeded.Load(), route.relayed.Load(), size) + logMu.Lock() + defer logMu.Unlock() + t.Errorf("exceeded %v after relaying %d of %d bytes; relay log:\n%s", route.exceeded.Load(), route.relayed.Load(), size, logs.String()) } } +type lockedWriter struct { + mu *sync.Mutex + w io.Writer +} + +func (l lockedWriter) Write(p []byte) (int, error) { + l.mu.Lock() + defer l.mu.Unlock() + return l.w.Write(p) +} + type zeros struct{} func (zeros) Read(p []byte) (int, error) { From e0cb94641f00c8bdfa2fafea9adfe631265eb934 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Fri, 23 Oct 2026 19:34:09 +0200 Subject: [PATCH 378/633] Make the git cache's RefreshWait test independent of timing TestSlowFetchAfterPushIsWaitedFor relied on the ref check finishing within two seconds, which a slow Windows runner missed. The test now holds the fetch and ends RefreshWait itself. When the check and the wait end together, ensureFresh now waits for the fetch rather than picking either at random. Failing gitcache tests print the cache's log. --- internal/gitcache/cache.go | 4 ++- internal/gitcache/cache_test.go | 35 ++++++++++++++---- internal/gitcache/helpers_test.go | 59 +++++++++++++++++++++++++++++++ internal/gitcache/refresh.go | 21 +++++++++-- 4 files changed, 109 insertions(+), 10 deletions(-) diff --git a/internal/gitcache/cache.go b/internal/gitcache/cache.go index 4014fd5..73d0529 100644 --- a/internal/gitcache/cache.go +++ b/internal/gitcache/cache.go @@ -70,8 +70,10 @@ type Options struct { Upstreams map[string]string // refCheckTimeout, lowSpeedTime, maxRequestBody and maxUploadPacks - // override their defaults in tests. + // override their defaults in tests, and refreshWaitOver, when set, + // replaces the RefreshWait timer. refCheckTimeout time.Duration + refreshWaitOver <-chan time.Time lowSpeedTime int maxRequestBody int64 maxUploadPacks int diff --git a/internal/gitcache/cache_test.go b/internal/gitcache/cache_test.go index 1770361..4d43c5d 100644 --- a/internal/gitcache/cache_test.go +++ b/internal/gitcache/cache_test.go @@ -233,19 +233,42 @@ func TestSlowUpstreamServesCachedCopyAfterRefreshWait(t *testing.T) { } // TestSlowFetchAfterPushIsWaitedFor has the upstream answer the ref check at -// once and the fetch after a push take longer than RefreshWait: a clone -// still gets the pushed commit. RefreshWait leaves the check time to finish -// on a slow machine. +// once and the fetch after a push still run when RefreshWait is over: a +// clone still gets the pushed commit. func TestSlowFetchAfterPushIsWaitedFor(t *testing.T) { upstream := newFakeUpstream(t) work := upstream.newRepo("repo") - tc := newTestCache(t, upstream.server.URL, func(o *Options) { o.RefreshWait = 2 * time.Second }) + waitOver := make(chan time.Time) + tc := newTestCache(t, upstream.server.URL, func(o *Options) { o.refreshWaitOver = waitOver }) tc.clone(t, "repo") head := upstream.commit(work, "second") runGit(t, work, "push", "--quiet", "origin", "main") - upstream.delayFetches.Store(int64(4 * time.Second)) - if got := runGit(t, tc.clone(t, "repo"), "rev-parse", "HEAD"); got != head { + arrived, release := upstream.holdFetches() + defer release() + dir := filepath.Join(t.TempDir(), "clone") + cloned := make(chan error, 1) + go func() { + out, err := tryGit(t, "", "clone", "--quiet", tc.url("repo"), dir) + if err != nil { + err = fmt.Errorf("%w\n%s", err, out) + } + cloned <- err + }() + // The fetch starts only after the check found the push. + select { + case <-arrived: + case err := <-cloned: + t.Fatalf("clone finished before the fetch started: %v", err) + case <-time.After(30 * time.Second): + t.Fatal("the fetch after the push never reached the upstream") + } + close(waitOver) + release() + if err := <-cloned; err != nil { + t.Fatal(err) + } + if got := runGit(t, dir, "rev-parse", "HEAD"); got != head { t.Errorf("HEAD = %s, want the pushed %s", got, head) } } diff --git a/internal/gitcache/helpers_test.go b/internal/gitcache/helpers_test.go index e2c4a27..d5d82b7 100644 --- a/internal/gitcache/helpers_test.go +++ b/internal/gitcache/helpers_test.go @@ -6,6 +6,7 @@ import ( "errors" "io" "log" + "log/slog" "net/http" "net/http/cgi" "net/http/httptest" @@ -95,6 +96,11 @@ type fakeUpstream struct { // delayFetches holds them for that long. stallFetches atomic.Bool delayFetches atomic.Int64 + // fetchHold, while set, holds git's own upload-pack requests until it + // is closed; fetchArrived is closed when the first one arrives. + fetchHold chan struct{} + fetchArrived chan struct{} + fetchOnce *sync.Once bodiesMu sync.Mutex bodies []string @@ -172,6 +178,15 @@ func newFakeUpstream(t *testing.T) *fakeUpstream { if u.v0Only { r.Header.Del("Git-Protocol") } + if r.UserAgent() != gitUserAgent && r.Method == http.MethodPost { + u.holdMu.Lock() + fetchHold, fetchArrived, fetchOnce := u.fetchHold, u.fetchArrived, u.fetchOnce + u.holdMu.Unlock() + if fetchHold != nil { + fetchOnce.Do(func() { close(fetchArrived) }) + <-fetchHold + } + } if delay := u.delayFetches.Load(); delay > 0 && r.UserAgent() != gitUserAgent && r.Method == http.MethodPost { time.Sleep(time.Duration(delay)) } @@ -190,6 +205,24 @@ func newFakeUpstream(t *testing.T) *fakeUpstream { return u } +// holdFetches holds git's own upload-pack requests until release is called. +// arrived is closed when the first one reaches the upstream. +func (u *fakeUpstream) holdFetches() (arrived <-chan struct{}, release func()) { + u.holdMu.Lock() + defer u.holdMu.Unlock() + hold, first := make(chan struct{}), make(chan struct{}) + u.fetchHold, u.fetchArrived, u.fetchOnce = hold, first, &sync.Once{} + var once sync.Once + return first, func() { + once.Do(func() { + u.holdMu.Lock() + u.fetchHold = nil + u.holdMu.Unlock() + close(hold) + }) + } +} + func (u *fakeUpstream) holdRequests() func() { u.holdMu.Lock() defer u.holdMu.Unlock() @@ -273,12 +306,20 @@ type testCache struct { func newTestCache(t *testing.T, upstreamURL string, configure func(*Options)) *testCache { t.Helper() + // The cache's log is shown when the test fails. + logs := &lockedBuffer{} + t.Cleanup(func() { + if t.Failed() { + t.Logf("git cache log:\n%s", logs.String()) + } + }) opts := Options{ Upstreams: map[string]string{"up": "up"}, CacheDir: t.TempDir(), RefreshWait: 5 * time.Second, StaleMaxAge: time.Hour, Client: &http.Client{}, + Logger: slog.New(slog.NewTextHandler(logs, &slog.HandlerOptions{Level: slog.LevelDebug})), } if configure != nil { configure(&opts) @@ -352,3 +393,21 @@ func (tc *testCache) cachedRefs(t *testing.T) string { t.Helper() return runGit(t, filepath.Join(tc.dir, "up", "repo.git"), "for-each-ref", "--format=%(objectname) %(refname)") } + +// lockedBuffer is a bytes.Buffer safe for concurrent writers. +type lockedBuffer struct { + mu sync.Mutex + buf bytes.Buffer +} + +func (b *lockedBuffer) Write(p []byte) (int, error) { + b.mu.Lock() + defer b.mu.Unlock() + return b.buf.Write(p) +} + +func (b *lockedBuffer) String() string { + b.mu.Lock() + defer b.mu.Unlock() + return b.buf.String() +} diff --git a/internal/gitcache/refresh.go b/internal/gitcache/refresh.go index 694ea12..448b3d9 100644 --- a/internal/gitcache/refresh.go +++ b/internal/gitcache/refresh.go @@ -77,9 +77,13 @@ func (c *Cache) ensureFresh(ctx context.Context, rp *repo, repoURL string) error } } - timer := time.NewTimer(c.opts.RefreshWait) - defer timer.Stop() - refChecked, timeout := call.checked, timer.C + timeout := c.opts.refreshWaitOver + if timeout == nil { + timer := time.NewTimer(c.opts.RefreshWait) + defer timer.Stop() + timeout = timer.C + } + refChecked := call.checked for { select { case <-refChecked: @@ -95,6 +99,17 @@ func (c *Cache) ensureFresh(ctx context.Context, rp *repo, repoURL string) error } return call.err case <-timeout: + // select picks at random when the check finished just as the + // wait ran out; a check found new commits in time still counts. + select { + case <-refChecked: + refChecked = nil + if call.fetching { + timeout = nil + continue + } + default: + } c.logger.Info("git cache: upstream check still running, serving cached copy", "repository", rp.key, "checked", checked) return nil From 562faac436f4979c205ea6a75b57efd230697a06 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Fri, 23 Oct 2026 19:38:28 +0200 Subject: [PATCH 379/633] Split Maven metadata fetch from its response Cooldown needs to read maven-metadata.xml before serving it, with the same Gradle Plugin Portal fallback and error handling. --- internal/handler/maven.go | 43 +++++++++++++++++++++++++-------------- 1 file changed, 28 insertions(+), 15 deletions(-) diff --git a/internal/handler/maven.go b/internal/handler/maven.go index 4b677ac..44bd6e7 100644 --- a/internal/handler/maven.go +++ b/internal/handler/maven.go @@ -1,6 +1,7 @@ package handler import ( + "context" "crypto/sha256" "encoding/hex" "errors" @@ -170,32 +171,44 @@ func (s mavenSource) cacheFilename(filename string) string { const mavenUpstreamHashBytes = 8 func (h *MavenHandler) handleMetadata(w http.ResponseWriter, r *http.Request, source mavenSource, urlPath string) { - cacheKey := source.metadataCacheKey(urlPath) + body, contentType, cacheKey, err := h.fetchMetadata(r.Context(), source, urlPath) + if err != nil { + h.writeMetadataError(w, err) + return + } + + h.proxy.writeMetadataCachedResponse(w, r, "maven", cacheKey, body, contentType) +} + +// fetchMetadata fetches a metadata file from the source's upstream, or from +// its fallback when the upstream doesn't have it, and returns it with the +// key it is cached under. +func (h *MavenHandler) fetchMetadata(ctx context.Context, source mavenSource, urlPath string) (body []byte, contentType, cacheKey string, err error) { + cacheKey = source.metadataCacheKey(urlPath) upstreamURL := fmt.Sprintf("%s/%s", source.upstreamURL, urlPath) - body, contentType, err := h.proxy.FetchOrCacheMetadata(r.Context(), "maven", cacheKey, upstreamURL, "*/*") + body, contentType, err = h.proxy.FetchOrCacheMetadata(ctx, "maven", cacheKey, upstreamURL, "*/*") if err != nil && source.fallbackURL != "" { if errors.Is(err, ErrUpstreamNotFound) { pluginPortalURL := fmt.Sprintf("%s/%s", source.fallbackURL, urlPath) h.proxy.Logger.Info("maven metadata unavailable in primary upstream, trying Gradle Plugin Portal", "path", urlPath) - body, contentType, err = h.proxy.FetchOrCacheMetadata(r.Context(), "maven", cacheKey, pluginPortalURL, "*/*") + body, contentType, err = h.proxy.FetchOrCacheMetadata(ctx, "maven", cacheKey, pluginPortalURL, "*/*") } } - if err != nil { - if errors.Is(err, ErrUpstreamNotFound) { - http.Error(w, "not found", http.StatusNotFound) - return - } - if writeUpstreamHeld(w, err) { - return - } - h.proxy.Logger.Error("metadata fetch failed", "error", err) - http.Error(w, "failed to fetch from upstream", http.StatusBadGateway) + return body, contentType, cacheKey, err +} + +func (h *MavenHandler) writeMetadataError(w http.ResponseWriter, err error) { + if errors.Is(err, ErrUpstreamNotFound) { + http.Error(w, "not found", http.StatusNotFound) return } - - h.proxy.writeMetadataCachedResponse(w, r, "maven", cacheKey, body, contentType) + if writeUpstreamHeld(w, err) { + return + } + h.proxy.Logger.Error("metadata fetch failed", "error", err) + http.Error(w, "failed to fetch from upstream", http.StatusBadGateway) } // handleDownload serves an artifact file, fetching and caching from upstream if needed. From 74f8eb378b1597254183da9467379d9a815395f3 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Fri, 23 Oct 2026 19:38:28 +0200 Subject: [PATCH 380/633] Apply cooldown to Maven A version's publish time is the Last-Modified of its .pom, read with a HEAD request once per version and stored in the versions table. Named repositories store theirs under a repository_url qualifier. Versions in cooldown are left out of maven-metadata.xml, and are moved to the last versions left, and the metadata's checksum files are computed from what is served. Files of a version in cooldown get 403. Only versions that can still be in cooldown are checked: none when is older than the window, otherwise from the end of the list back to the first old enough version. Part of #132. --- internal/handler/maven.go | 8 + internal/handler/maven_cooldown.go | 443 ++++++++++++++++++++++++ internal/handler/maven_cooldown_test.go | 383 ++++++++++++++++++++ 3 files changed, 834 insertions(+) create mode 100644 internal/handler/maven_cooldown.go create mode 100644 internal/handler/maven_cooldown_test.go diff --git a/internal/handler/maven.go b/internal/handler/maven.go index 44bd6e7..4885906 100644 --- a/internal/handler/maven.go +++ b/internal/handler/maven.go @@ -171,6 +171,10 @@ func (s mavenSource) cacheFilename(filename string) string { const mavenUpstreamHashBytes = 8 func (h *MavenHandler) handleMetadata(w http.ResponseWriter, r *http.Request, source mavenSource, urlPath string) { + if h.cooldownEnabled() && h.serveCooldownMetadata(w, r, source, urlPath) { + return + } + body, contentType, cacheKey, err := h.fetchMetadata(r.Context(), source, urlPath) if err != nil { h.writeMetadataError(w, err) @@ -221,6 +225,10 @@ func (h *MavenHandler) handleDownload(w http.ResponseWriter, r *http.Request, so return } + if h.checkCooldown(w, r, source, urlPath, version) { + return + } + // Maven uses group:artifact as the package name name := fmt.Sprintf("%s:%s", group, artifact) diff --git a/internal/handler/maven_cooldown.go b/internal/handler/maven_cooldown.go new file mode 100644 index 0000000..ca5a03a --- /dev/null +++ b/internal/handler/maven_cooldown.go @@ -0,0 +1,443 @@ +package handler + +import ( + "bytes" + "context" + "crypto/md5" + "crypto/sha1" + "crypto/sha256" + "crypto/sha512" + "encoding/hex" + "encoding/xml" + "errors" + "fmt" + "hash" + "io" + "net/http" + "net/url" + "path" + "slices" + "strings" + "time" +) + +// Maven cooldown measures a version by the Last-Modified header of its .pom. +// On Maven Central that is the time the version was deployed. The header is +// read once per version and stored in the versions table, since release +// files don't change. +// +// maven-metadata.xml has a single for the whole artifact, so +// the proxy only looks at versions that can still be in cooldown: none when +// is older than the window, and otherwise the versions from +// the end of the list back to the first one that is old enough. Maven +// Central blocks clients that send bursts of requests, so the proxy doesn't +// read every version's time. +// +// SNAPSHOT versions are left alone: their files change after publishing. + +const errMavenCooldown = "version is in cooldown" + +const ( + mavenMetadataFile = "maven-metadata.xml" + mavenSnapshot = "-SNAPSHOT" + // mavenLastUpdatedLayout is the UTC timestamp format of . + mavenLastUpdatedLayout = "20060102150405" +) + +// mavenChecksums are the checksum files served beside maven-metadata.xml. +// With cooldown on they are computed from the metadata as served. +var mavenChecksums = map[string]func() hash.Hash{ + ".md5": md5.New, + ".sha1": sha1.New, + ".sha256": sha256.New, + ".sha512": sha512.New, +} + +func (h *MavenHandler) cooldownEnabled() bool { + return h.proxy.Cooldown != nil && h.proxy.Cooldown.Enabled() +} + +func mavenPackagePURL(group, artifact string) string { + return canonicalPackagePURL("maven", group+":"+artifact) +} + +// mavenCoordinates returns the group and artifact of an artifact directory +// such as com/google/guava/guava. +func mavenCoordinates(artifactDir string) (group, artifact string, ok bool) { + i := strings.LastIndex(artifactDir, "/") + if i <= 0 || i == len(artifactDir)-1 { + return "", "", false + } + return strings.ReplaceAll(artifactDir[:i], "/", "."), artifactDir[i+1:], true +} + +// usableMavenVersion reports whether version can be put in a URL path as +// one segment. Versions come from upstream metadata. +func usableMavenVersion(version string) bool { + return version != "" && version != "." && version != ".." && !strings.ContainsAny(version, "/\\?#%") +} + +// mavenTimeKey returns the versions-table PURLs a version's time is stored +// under. A named repository's keys carry its upstream as repository_url: +// the same coordinates there can have been published at another time than +// in the default repository. +func (h *MavenHandler) mavenTimeKey(source mavenSource, group, artifact, version string) (versionPURL, packagePURL string, ok bool) { + if h.proxy.DB == nil { + return "", "", false + } + name := group + ":" + artifact + versionPURL, packagePURL = canonicalVersionPURL("maven", name, version), canonicalPackagePURL("maven", name) + if source.name != "" { + qualifier := "?repository_url=" + url.QueryEscape(source.upstreamURL) + versionPURL += qualifier + packagePURL += qualifier + } + return versionPURL, packagePURL, true +} + +// publishTime returns a version's publish time from the versions table, or +// from the Last-Modified of its .pom, which is then stored. A version +// without a .pom, or whose .pom has no Last-Modified, gets the zero time and +// is let through, as in the other ecosystems. A failed request is an error, +// so a rate-limited upstream can't let a version through. +func (h *MavenHandler) publishTime(ctx context.Context, source mavenSource, artifactDir, version string) (time.Time, error) { + group, artifact, ok := mavenCoordinates(artifactDir) + if !ok || !usableMavenVersion(version) { + return time.Time{}, nil + } + versionPURL, packagePURL, store := h.mavenTimeKey(source, group, artifact, version) + if store { + if ver, err := h.proxy.DB.GetVersionByPURL(versionPURL); err == nil && ver != nil && ver.PublishedAt.Valid { + return ver.PublishedAt.Time, nil + } + } + + pomPath := artifactDir + "/" + version + "/" + artifact + "-" + version + ".pom" + published, found, err := h.pomLastModified(ctx, source.upstreamURL+"/"+pomPath) + if err == nil && !found && source.fallbackURL != "" { + published, _, err = h.pomLastModified(ctx, source.fallbackURL+"/"+pomPath) + } + if err != nil { + return time.Time{}, err + } + if store && !published.IsZero() { + if err := h.proxy.DB.SetVersionPublishedAt(versionPURL, packagePURL, published); err != nil { + h.proxy.Logger.Warn("cooldown: could not store maven publish time", + "group", group, "artifact", artifact, "version", version, "error", err) + } + } + return published, nil +} + +// pomLastModified sends a HEAD request for a .pom and returns its +// Last-Modified. found is false when upstream doesn't have the file. Only a +// 2xx response's header is used: an error page's date says nothing about the +// version. +func (h *MavenHandler) pomLastModified(ctx context.Context, pomURL string) (published time.Time, found bool, err error) { + req, err := http.NewRequestWithContext(ctx, http.MethodHead, pomURL, nil) + if err != nil { + return time.Time{}, false, err + } + resp, err := h.proxy.HTTPClient.Do(req) + if err != nil { + return time.Time{}, false, fmt.Errorf("reading pom time: %w", err) + } + _ = resp.Body.Close() + switch { + case resp.StatusCode == http.StatusNotFound || resp.StatusCode == http.StatusGone: + return time.Time{}, false, nil + case resp.StatusCode < 200 || resp.StatusCode > 299: + return time.Time{}, false, fmt.Errorf("reading pom time: upstream returned %d", resp.StatusCode) + } + published, err = http.ParseTime(resp.Header.Get(headerLastModified)) + if err != nil { + return time.Time{}, true, nil + } + return published, true, nil +} + +// checkCooldown answers a request for a file of a version in cooldown, or +// whose time couldn't be read, and reports whether it did. The answer is +// 403, as for a version on the denylist. +func (h *MavenHandler) checkCooldown(w http.ResponseWriter, r *http.Request, source mavenSource, urlPath, version string) bool { + if !h.cooldownEnabled() || strings.HasSuffix(version, mavenSnapshot) { + return false + } + artifactDir := path.Dir(path.Dir(urlPath)) + group, artifact, ok := mavenCoordinates(artifactDir) + if !ok || h.proxy.Cooldown.For("maven", mavenPackagePURL(group, artifact)) == 0 { + return false + } + published, err := h.publishTime(r.Context(), source, artifactDir, version) + if err != nil { + if writeUpstreamHeld(w, err) { + return true + } + h.proxy.Logger.Warn("cooldown: could not read maven version time", + "group", group, "artifact", artifact, "version", version, "error", err) + http.Error(w, "failed to check cooldown", http.StatusBadGateway) + return true + } + if !h.proxy.Cooldown.IsAllowed("maven", mavenPackagePURL(group, artifact), published) { + h.proxy.Logger.Info("cooldown: withholding maven version", + "group", group, "artifact", artifact, "version", version) + http.Error(w, errMavenCooldown+": "+canonicalVersionPURL("maven", group+":"+artifact, version), http.StatusForbidden) + return true + } + return false +} + +// serveCooldownMetadata serves maven-metadata.xml without versions in +// cooldown, and its checksum files computed from what is served. It reports +// false for other files beside the metadata, such as a signature, when the +// metadata is served unchanged, so they are proxied as usual. +func (h *MavenHandler) serveCooldownMetadata(w http.ResponseWriter, r *http.Request, source mavenSource, urlPath string) bool { + filename := path.Base(urlPath) + suffix, ok := strings.CutPrefix(filename, mavenMetadataFile) + if !ok { + return false + } + metadataPath := path.Join(path.Dir(urlPath), mavenMetadataFile) + body, contentType, cacheKey, err := h.fetchMetadata(r.Context(), source, metadataPath) + if err != nil { + h.writeMetadataError(w, err) + return true + } + + m, err := h.proxy.cachedRewrite(r.Context(), "maven", h.proxyURL, cacheKey, body, "", func(in []byte) ([]byte, []string, error) { + // The rewrite is shared by requests waiting on it and cached, so it + // finishes even if the request that started it goes away. + out, err := h.filterMetadata(context.WithoutCancel(r.Context()), source, path.Dir(urlPath), in) + return out, nil, err + }) + if err != nil { + if r.Context().Err() != nil { + return true + } + if writeUpstreamHeld(w, err) { + return true + } + h.proxy.Logger.Warn("cooldown: could not filter maven metadata", "path", metadataPath, "error", err) + http.Error(w, "failed to check cooldown", http.StatusBadGateway) + return true + } + + switch { + case suffix == "": + if contentType == "" || contentType == contentTypeJSON { + contentType = "text/xml" + } + serveMavenMetadata(w, r, m, contentType) + case mavenChecksums[suffix] != nil: + sum := mavenChecksums[suffix]() + _, _ = sum.Write(m.body) + w.Header().Set(headerContentType, "text/plain") + _, _ = io.WriteString(w, hex.EncodeToString(sum.Sum(nil))) + case bytes.Equal(m.body, body): + return false + default: + // A signature of the upstream metadata doesn't match what is served. + http.Error(w, "not found", http.StatusNotFound) + } + return true +} + +// serveMavenMetadata writes filtered metadata with its validators, or 304 +// when they show the client's copy is current. +func serveMavenMetadata(w http.ResponseWriter, r *http.Request, m rewrittenMetadata, contentType string) { + w.Header().Set(headerETag, m.etag) + if !m.lastModified.IsZero() { + w.Header().Set(headerLastModified, m.lastModified.UTC().Format(http.TimeFormat)) + } + if notModified(r, m.etag, m.lastModified) { + w.WriteHeader(http.StatusNotModified) + return + } + w.Header().Set(headerContentType, contentType) + _, _ = w.Write(m.body) +} + +// filterMetadata removes versions in cooldown from an artifact's +// maven-metadata.xml and points and at the newest +// versions left. Metadata without a version list, such as a group's plugin +// list or a SNAPSHOT's build list, is returned unchanged. The document is +// edited in place rather than encoded again, so everything else in it stays +// as upstream sent it. +func (h *MavenHandler) filterMetadata(ctx context.Context, source mavenSource, artifactDir string, body []byte) ([]byte, error) { + doc, err := parseMavenMetadata(body) + if err != nil { + return nil, err + } + group, artifact, ok := mavenCoordinates(artifactDir) + if !ok || len(doc.versions) == 0 { + return body, nil + } + packagePURL := mavenPackagePURL(group, artifact) + // No version can be newer than the metadata listing it. + if lastUpdated, err := time.Parse(mavenLastUpdatedLayout, doc.lastUpdated); err == nil && + h.proxy.Cooldown.IsAllowed("maven", packagePURL, lastUpdated) { + return body, nil + } + + removed := map[string]bool{} + for i := len(doc.versions) - 1; i >= 0; i-- { + v := doc.versions[i].value + if strings.HasSuffix(v, mavenSnapshot) { + continue + } + published, err := h.publishTime(ctx, source, artifactDir, v) + if err != nil { + return nil, fmt.Errorf("reading time of %s: %w", v, err) + } + if published.IsZero() { + continue + } + if h.proxy.Cooldown.IsAllowed("maven", packagePURL, published) { + break + } + h.proxy.Logger.Info("cooldown: filtering maven version", "group", group, "artifact", artifact, "version", v) + removed[v] = true + } + if len(removed) == 0 { + return body, nil + } + return doc.without(body, removed), nil +} + +// mavenElement is an element of maven-metadata.xml: its text and where it +// starts and ends in the document. +type mavenElement struct { + value string + start, end int +} + +type mavenMetadata struct { + versions []mavenElement + latest, release *mavenElement + lastUpdated string +} + +// parseMavenMetadata reads the version list, , and +// of a maven-metadata.xml, with their offsets. +func parseMavenMetadata(body []byte) (*mavenMetadata, error) { + dec := xml.NewDecoder(bytes.NewReader(body)) + // Versions are ASCII, so any declared charset can be read as bytes. + dec.CharsetReader = func(_ string, in io.Reader) (io.Reader, error) { return in, nil } + + doc := &mavenMetadata{} + var stack []string + var current *mavenElement + for { + offset := int(dec.InputOffset()) + tok, err := dec.Token() + if errors.Is(err, io.EOF) { + break + } + if err != nil { + return nil, fmt.Errorf("parsing maven metadata: %w", err) + } + switch t := tok.(type) { + case xml.StartElement: + stack = append(stack, t.Name.Local) + switch strings.Join(stack, "/") { + case "metadata/versioning/versions/version", + "metadata/versioning/latest", + "metadata/versioning/release", + "metadata/versioning/lastUpdated": + current = &mavenElement{start: offset} + } + case xml.CharData: + if current != nil { + current.value += string(t) + } + case xml.EndElement: + if current != nil { + current.value = strings.TrimSpace(current.value) + current.end = int(dec.InputOffset()) + switch t.Name.Local { + case "version": + doc.versions = append(doc.versions, *current) + case "latest": + doc.latest = current + case "release": + doc.release = current + case "lastUpdated": + doc.lastUpdated = current.value + } + current = nil + } + if len(stack) > 0 { + stack = stack[:len(stack)-1] + } + } + } + return doc, nil +} + +// without returns body with the removed versions taken out of the version +// list, and and set to the last versions left in list +// order, which is how the repository chose them. Either is dropped when no +// version is left for it. +func (doc *mavenMetadata) without(body []byte, removed map[string]bool) []byte { + type edit struct { + start, end int + text string + } + var edits []edit + drop := func(e mavenElement) { + edits = append(edits, edit{start: lineStart(body, e.start), end: e.end}) + } + var latest, release string + for _, v := range doc.versions { + if removed[v.value] { + drop(v) + continue + } + latest = v.value + if !strings.HasSuffix(v.value, mavenSnapshot) { + release = v.value + } + } + for _, field := range []struct { + el *mavenElement + name string + next string + }{{doc.latest, "latest", latest}, {doc.release, "release", release}} { + if field.el == nil || !removed[field.el.value] { + continue + } + if field.next == "" { + drop(*field.el) + continue + } + edits = append(edits, edit{field.el.start, field.el.end, "<" + field.name + ">" + field.next + ""}) + } + + // Edits don't overlap; apply them in document order. + slices.SortFunc(edits, func(a, b edit) int { return a.start - b.start }) + out := make([]byte, 0, len(body)) + pos := 0 + for _, e := range edits { + out = append(out, body[pos:e.start]...) + out = append(out, e.text...) + pos = e.end + } + return append(out, body[pos:]...) +} + +// lineStart moves an element's start back over the indentation and line +// break before it, so removing the element removes its line. +func lineStart(body []byte, start int) int { + i := start + for i > 0 && (body[i-1] == ' ' || body[i-1] == '\t') { + i-- + } + if i > 0 && body[i-1] == '\n' { + i-- + if i > 0 && body[i-1] == '\r' { + i-- + } + return i + } + return start +} diff --git a/internal/handler/maven_cooldown_test.go b/internal/handler/maven_cooldown_test.go new file mode 100644 index 0000000..74c4268 --- /dev/null +++ b/internal/handler/maven_cooldown_test.go @@ -0,0 +1,383 @@ +package handler + +import ( + "crypto/md5" + "crypto/sha1" + "crypto/sha256" + "crypto/sha512" + "encoding/hex" + "hash" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + "time" + + "github.com/git-pkgs/cooldown" + "github.com/git-pkgs/registries/fetch" +) + +const mavenTestDir = "/com/example/lib/" + +// mavenCooldownUpstream serves com.example:lib. Each version's .pom has the +// Last-Modified in times; pomStatus overrides the status of a version's +// .pom. Requests for .pom files are counted. +type mavenCooldownUpstream struct { + server *httptest.Server + versions []string + latest string + release string + lastUpdated time.Time + times map[string]time.Time + pomStatus map[string]int + + mu sync.Mutex + poms map[string]int +} + +func newMavenCooldownUpstream(t *testing.T) *mavenCooldownUpstream { + t.Helper() + now := time.Now() + u := &mavenCooldownUpstream{ + versions: []string{"1.0", "1.1", "2.0", "2.1", "3.0-SNAPSHOT"}, + latest: "3.0-SNAPSHOT", + release: "2.1", + lastUpdated: now.Add(-time.Hour), + times: map[string]time.Time{ + "1.0": now.Add(-30 * 24 * time.Hour), + "1.1": now.Add(-10 * 24 * time.Hour), + "2.0": now.Add(-2 * time.Hour), + "2.1": now.Add(-time.Hour), + "3.0-SNAPSHOT": now.Add(-time.Minute), + }, + pomStatus: map[string]int{}, + poms: map[string]int{}, + } + u.server = httptest.NewServer(http.HandlerFunc(u.serve)) + t.Cleanup(u.server.Close) + return u +} + +// serve answers for the default upstream and, under /portal, for the +// Gradle Plugin Portal fallback alike. +func (u *mavenCooldownUpstream) serve(w http.ResponseWriter, r *http.Request) { + rest, ok := strings.CutPrefix(strings.TrimPrefix(r.URL.Path, "/portal"), mavenTestDir) + if !ok { + http.NotFound(w, r) + return + } + switch rest { + case "maven-metadata.xml": + _, _ = w.Write([]byte(u.metadata())) + return + case "maven-metadata.xml.asc": + _, _ = w.Write([]byte("signature")) + return + } + version, file, _ := strings.Cut(rest, "/") + published, known := u.times[version] + if !known { + http.NotFound(w, r) + return + } + if strings.HasSuffix(file, ".pom") { + u.mu.Lock() + u.poms[version]++ + u.mu.Unlock() + w.Header().Set(headerLastModified, published.UTC().Format(http.TimeFormat)) + if status := u.pomStatus[version]; status != 0 { + // An error page dated long ago must not count as an old version. + w.Header().Set(headerLastModified, time.Unix(0, 0).UTC().Format(http.TimeFormat)) + w.WriteHeader(status) + return + } + } + _, _ = w.Write([]byte(file + " of " + version)) +} + +func (u *mavenCooldownUpstream) metadata() string { + var b strings.Builder + b.WriteString("\n\n com.example\n lib\n \n") + b.WriteString(" " + u.latest + "\n " + u.release + "\n \n") + for _, v := range u.versions { + b.WriteString(" " + v + "\n") + } + b.WriteString(" \n " + u.lastUpdated.UTC().Format(mavenLastUpdatedLayout) + "\n \n\n") + return b.String() +} + +// pomRequests returns how many requests each version's .pom got. +func (u *mavenCooldownUpstream) pomRequests() map[string]int { + u.mu.Lock() + defer u.mu.Unlock() + out := make(map[string]int, len(u.poms)) + for v, n := range u.poms { + out[v] = n + } + return out +} + +func (u *mavenCooldownUpstream) totalPOMRequests() int { + total := 0 + for _, n := range u.pomRequests() { + total += n + } + return total +} + +func newMavenCooldownHandler(t *testing.T, u *mavenCooldownUpstream) (http.Handler, *Proxy) { + t.Helper() + proxy, _, store, _ := setupTestProxy(t) + store.seekable = true + proxy.HTTPClient = u.server.Client() + fetcher := fetch.NewFetcher(fetch.WithHTTPClient(u.server.Client()), fetch.WithMaxRetries(0)) + proxy.Fetcher = fetcher + t.Cleanup(func() { _ = fetcher.Close() }) + proxy.Cooldown = &cooldown.Config{Default: "3d"} + return NewMavenHandler(proxy, "http://localhost", u.server.URL, u.server.URL+"/portal").Routes(), proxy +} + +func mavenMetadataVersions(t *testing.T, body string) []string { + t.Helper() + doc, err := parseMavenMetadata([]byte(body)) + if err != nil { + t.Fatalf("served metadata does not parse: %v\n%s", err, body) + } + var versions []string + for _, v := range doc.versions { + versions = append(versions, v.value) + } + return versions +} + +func TestMavenCooldownFiltersMetadata(t *testing.T) { + u := newMavenCooldownUpstream(t) + h, proxy := newMavenCooldownHandler(t, u) + + w := getMaven(h, mavenTestDir+"maven-metadata.xml") + if w.Code != http.StatusOK { + t.Fatalf("status = %d: %s", w.Code, w.Body) + } + body := w.Body.String() + if got := strings.Join(mavenMetadataVersions(t, body), " "); got != "1.0 1.1 3.0-SNAPSHOT" { + t.Errorf("versions = %s, want 1.0 1.1 3.0-SNAPSHOT", got) + } + if !strings.Contains(body, "1.1") || !strings.Contains(body, "3.0-SNAPSHOT") { + t.Errorf("release or latest not recomputed:\n%s", body) + } + if strings.Contains(body, "\n\n") { + t.Errorf("removed versions left blank lines:\n%s", body) + } + + // The walk stops at 1.1, the first version old enough, and skips the + // SNAPSHOT. + want := map[string]int{"2.1": 1, "2.0": 1, "1.1": 1} + if got := u.pomRequests(); len(got) != len(want) || got["2.1"] != 1 || got["2.0"] != 1 || got["1.1"] != 1 { + t.Errorf("pom requests = %v, want %v", got, want) + } + + // Times are stored, so the next request makes no .pom requests. + ver, err := proxy.DB.GetVersionByPURL(canonicalVersionPURL("maven", "com.example:lib", "2.1")) + if err != nil || ver == nil || !ver.PublishedAt.Valid { + t.Fatalf("time of 2.1 not stored: %v %v", ver, err) + } + if w := getMaven(h, mavenTestDir+"maven-metadata.xml"); w.Body.String() != body { + t.Errorf("second response differs:\n%s", w.Body) + } + if n := u.totalPOMRequests(); n != 3 { + t.Errorf("pom requests after second metadata request = %d, want 3", n) + } +} + +func TestMavenCooldownRecomputesLatest(t *testing.T) { + u := newMavenCooldownUpstream(t) + u.versions = []string{"1.0", "1.1", "2.0", "2.1"} + u.latest = "2.1" + h, _ := newMavenCooldownHandler(t, u) + + body := getMaven(h, mavenTestDir+"maven-metadata.xml").Body.String() + if !strings.Contains(body, "1.1") || !strings.Contains(body, "1.1") { + t.Errorf("latest and release should be 1.1:\n%s", body) + } +} + +func TestMavenCooldownDropsLatestWhenNothingIsLeft(t *testing.T) { + u := newMavenCooldownUpstream(t) + u.versions = []string{"2.0", "2.1"} + u.latest = "2.1" + h, _ := newMavenCooldownHandler(t, u) + + body := getMaven(h, mavenTestDir+"maven-metadata.xml").Body.String() + if strings.Contains(body, "") || strings.Contains(body, "") || strings.Contains(body, "") { + t.Errorf("no version is out of cooldown:\n%s", body) + } + if !strings.Contains(body, "\n ") { + t.Errorf("version list not emptied in place:\n%s", body) + } +} + +func TestMavenCooldownChecksumsMatchMetadata(t *testing.T) { + u := newMavenCooldownUpstream(t) + h, _ := newMavenCooldownHandler(t, u) + + body := getMaven(h, mavenTestDir+"maven-metadata.xml").Body.Bytes() + for suffix, newHash := range map[string]func() hash.Hash{ + ".md5": md5.New, ".sha1": sha1.New, ".sha256": sha256.New, ".sha512": sha512.New, + } { + sum := newHash() + _, _ = sum.Write(body) + w := getMaven(h, mavenTestDir+"maven-metadata.xml"+suffix) + if w.Code != http.StatusOK || w.Body.String() != hex.EncodeToString(sum.Sum(nil)) { + t.Errorf("%s: status = %d body %q, want checksum of served metadata", suffix, w.Code, w.Body) + } + } + // A signature of the upstream metadata would not match. + if w := getMaven(h, mavenTestDir+"maven-metadata.xml.asc"); w.Code != http.StatusNotFound { + t.Errorf(".asc status = %d, want 404", w.Code) + } + + // Unchanged metadata keeps its signature. + u.lastUpdated = time.Now().Add(-4 * 24 * time.Hour) + if w := getMaven(h, mavenTestDir+"maven-metadata.xml.asc"); w.Code != http.StatusOK || w.Body.String() != "signature" { + t.Errorf("unchanged .asc: status = %d body %q, want upstream's", w.Code, w.Body) + } +} + +func TestMavenCooldownRefusesFiles(t *testing.T) { + u := newMavenCooldownUpstream(t) + h, _ := newMavenCooldownHandler(t, u) + + for _, file := range []string{"2.1/lib-2.1.pom", "2.1/lib-2.1.jar", "2.1/lib-2.1.jar.sha1", "2.0/lib-2.0-sources.jar"} { + w := getMaven(h, mavenTestDir+file) + if w.Code != http.StatusForbidden || !strings.Contains(w.Body.String(), errMavenCooldown) { + t.Errorf("%s: status = %d body %q, want 403 cooldown", file, w.Code, w.Body) + } + } + for _, file := range []string{"1.1/lib-1.1.jar", "3.0-SNAPSHOT/lib-3.0-SNAPSHOT.jar"} { + if w := getMaven(h, mavenTestDir+file); w.Code != http.StatusOK { + t.Errorf("%s: status = %d, want 200", file, w.Code) + } + } + // One .pom request per version, none for the SNAPSHOT. + if got := u.pomRequests(); got["2.1"] != 1 || got["2.0"] != 1 || got["1.1"] != 1 || got["3.0-SNAPSHOT"] != 0 { + t.Errorf("pom requests = %v", got) + } +} + +func TestMavenCooldownOldLastUpdatedSkipsPOMs(t *testing.T) { + u := newMavenCooldownUpstream(t) + u.lastUpdated = time.Now().Add(-4 * 24 * time.Hour) + // The times would filter 2.0 and 2.1 if they were read. + h, _ := newMavenCooldownHandler(t, u) + + w := getMaven(h, mavenTestDir+"maven-metadata.xml") + if w.Body.String() != u.metadata() { + t.Errorf("metadata changed:\n%s", w.Body) + } + if n := u.totalPOMRequests(); n != 0 { + t.Errorf("pom requests = %d, want 0", n) + } +} + +func TestMavenCooldownIgnoresNon2xxPOMResponses(t *testing.T) { + u := newMavenCooldownUpstream(t) + u.pomStatus["2.1"] = http.StatusNotFound + h, proxy := newMavenCooldownHandler(t, u) + + // A missing .pom gives no time: 2.1 stays listed, and the walk goes on + // to 2.0 rather than stopping at the error page's old date. + versions := mavenMetadataVersions(t, getMaven(h, mavenTestDir+"maven-metadata.xml").Body.String()) + if got := strings.Join(versions, " "); got != "1.0 1.1 2.1 3.0-SNAPSHOT" { + t.Errorf("versions = %s, want 1.0 1.1 2.1 3.0-SNAPSHOT", got) + } + if ver, _ := proxy.DB.GetVersionByPURL(canonicalVersionPURL("maven", "com.example:lib", "2.1")); ver != nil && ver.PublishedAt.Valid { + t.Errorf("time of a 404 stored: %v", ver.PublishedAt.Time) + } + + // Any other error fails closed. + u.pomStatus["2.1"] = http.StatusTooManyRequests + for _, p := range []string{"maven-metadata.xml", "maven-metadata.xml.sha1", "2.1/lib-2.1.jar"} { + if w := getMaven(h, mavenTestDir+p); w.Code != http.StatusBadGateway { + t.Errorf("%s: status = %d, want 502", p, w.Code) + } + } + if ver, _ := proxy.DB.GetVersionByPURL(canonicalVersionPURL("maven", "com.example:lib", "2.1")); ver != nil && ver.PublishedAt.Valid { + t.Errorf("time of a 429 stored: %v", ver.PublishedAt.Time) + } +} + +func TestMavenCooldownDisabledPassesThrough(t *testing.T) { + u := newMavenCooldownUpstream(t) + h, proxy := newMavenCooldownHandler(t, u) + proxy.Cooldown = nil + + if w := getMaven(h, mavenTestDir+"maven-metadata.xml"); w.Body.String() != u.metadata() { + t.Errorf("metadata changed:\n%s", w.Body) + } + if w := getMaven(h, mavenTestDir+"2.1/lib-2.1.jar"); w.Code != http.StatusOK { + t.Errorf("jar status = %d, want 200", w.Code) + } + if n := u.totalPOMRequests(); n != 0 { + t.Errorf("pom requests = %d, want 0", n) + } +} + +func TestMavenCooldownPackageOverride(t *testing.T) { + u := newMavenCooldownUpstream(t) + h, proxy := newMavenCooldownHandler(t, u) + proxy.Cooldown = &cooldown.Config{Default: "3d", Packages: map[string]string{"pkg:maven/com.example/lib": "0"}} + + if w := getMaven(h, mavenTestDir+"maven-metadata.xml"); w.Body.String() != u.metadata() { + t.Errorf("metadata changed:\n%s", w.Body) + } + if w := getMaven(h, mavenTestDir+"2.1/lib-2.1.jar"); w.Code != http.StatusOK { + t.Errorf("jar status = %d, want 200", w.Code) + } + if n := u.totalPOMRequests(); n != 0 { + t.Errorf("pom requests = %d, want 0", n) + } +} + +func TestMavenCooldownNamedRepositoryStoresItsOwnTimes(t *testing.T) { + u := newMavenCooldownUpstream(t) + proxy, _, _, _ := setupTestProxy(t) + proxy.HTTPClient = u.server.Client() + proxy.Cooldown = &cooldown.Config{Default: "3d"} + // The named repository is the same server under /portal, which serves + // the same files. + h := NewMavenHandler(proxy, "http://localhost", u.server.URL, u.server.URL+"/portal"). + WithRepositories(map[string]string{"mirror": u.server.URL + "/portal"}).Routes() + + for range 2 { + versions := mavenMetadataVersions(t, getMaven(h, "/mirror"+mavenTestDir+"maven-metadata.xml").Body.String()) + if got := strings.Join(versions, " "); got != "1.0 1.1 3.0-SNAPSHOT" { + t.Fatalf("versions = %s, want 1.0 1.1 3.0-SNAPSHOT", got) + } + } + if w := getMaven(h, "/mirror"+mavenTestDir+"2.1/lib-2.1.jar"); w.Code != http.StatusForbidden { + t.Errorf("jar status = %d, want 403", w.Code) + } + if n := u.totalPOMRequests(); n != 3 { + t.Errorf("pom requests = %d, want 3", n) + } + + // The default repository doesn't use the named one's times. + _ = getMaven(h, mavenTestDir+"maven-metadata.xml") + if n := u.totalPOMRequests(); n != 6 { + t.Errorf("pom requests after default metadata = %d, want 6", n) + } +} + +func TestMavenMetadataWithoutCompact(t *testing.T) { + in := `2212` + doc, err := parseMavenMetadata([]byte(in)) + if err != nil { + t.Fatal(err) + } + got := string(doc.without([]byte(in), map[string]bool{"2": true})) + want := `111` + if got != want { + t.Errorf("got %s\nwant %s", got, want) + } +} From e6c4a3e57ae3ffc1eb2d13f0edfd2e12f5626b6a Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Fri, 23 Oct 2026 19:38:28 +0200 Subject: [PATCH 381/633] Document Maven cooldown Includes the caveats from #132: Nexus and Artifactory dates, Google's Maven Central mirror, version-ordered lists and SNAPSHOTs. --- README.md | 4 +++- docs/configuration.md | 11 ++++++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 3421dff..aac9ff5 100644 --- a/README.md +++ b/README.md @@ -51,7 +51,7 @@ The proxy never uploads artifact bytes to a scanner. Each scanner is notified wi | Hex | Elixir | Yes* | ✓ | | pub.dev | Dart | Yes | ✓ | | PyPI | Python | Yes | ✓ | -| Maven | Java | | ✓ | +| Maven | Java | Yes‡ | ✓ | | Gradle Build Cache | Java/Kotlin | | ✓ | | NuGet | .NET | Yes | ✓ | | Composer | PHP | Yes | ✓ | @@ -79,6 +79,8 @@ Cooldown requires publish timestamps in metadata. Registries without a "Yes" in † Go cooldown uses the time in each version's `.info`, which is the commit time of the tagged revision, not when the tag was pushed. +‡ Maven cooldown uses the `Last-Modified` header of each version's `.pom`. On Maven Central that is the deploy time; a Nexus or Artifactory upstream may report when it cached the file instead. SNAPSHOT versions are not held back. + ## Install ```bash diff --git a/docs/configuration.md b/docs/configuration.md index fa77ec6..d932292 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -980,10 +980,19 @@ Exact `packages` entries take precedence over patterns. When several patterns ma Resolution order: exact package override, then package pattern, then ecosystem override, then global default. This lets you set a conservative default while exempting trusted package families. -Currently supported for npm, PyPI, pub.dev, Composer, Cargo, NuGet, Conda, RubyGems, Hex and Go. These ecosystems include publish timestamps in their metadata. +Currently supported for npm, PyPI, pub.dev, Composer, Cargo, NuGet, Conda, RubyGems, Hex, Go and Maven. Most of these ecosystems include publish timestamps in their metadata; Go and Maven are described below. Go has no publish time. The module proxy's `.info` gives the commit time of the revision a version points at, so a version tagged on an old commit counts as old. The proxy reads each listed version's `.info` once, stores the time, and leaves versions in cooldown out of `@v/list` and `@latest`. When upstream's `@latest` is in cooldown, it serves the newest release out of cooldown instead. Requests for a version in cooldown, including its `.info`, `.mod` and `.zip`, and a branch or commit query that resolves to one, get 403 rather than 404, because the go command moves on to the next `GOPROXY` entry, such as `direct`, after a 404. If a version's `.info` can't be fetched, the list or download fails with 502 instead of letting the version through. +Maven metadata has no publish time per version. The proxy uses the `Last-Modified` header of each version's `.pom`, which on Maven Central is the time the version was deployed. It reads the header once per version with a `HEAD` request and stores the time, for each repository under `upstream.maven_repositories` separately. To keep requests to the upstream down, it only checks versions that can still be in cooldown: none when the metadata's `` is older than the cooldown, and otherwise the versions from the end of the list back to the first one that is old enough. Versions in cooldown are left out of `maven-metadata.xml`, `` and `` are set to the last versions left, and the metadata's `.md5`, `.sha1`, `.sha256` and `.sha512` files are computed from what is served. Requests for any file of a version in cooldown get 403, as for a version on the denylist. If a `.pom` can't be checked, the metadata or download fails with 502 instead of letting the version through; a version without a `.pom` is let through. + +Maven caveats: + +- A Nexus or Artifactory upstream may report when it cached the `.pom` rather than when it was published, so a version can stay in cooldown longer than it should. +- On Google's Maven Central mirror, `Last-Modified` is when the file reached the mirror's bucket, a few hours after it was published, so versions stay in cooldown a little longer. The upload time is in `x-goog-meta-last-modified`, which the proxy does not read. It is missing for files from the mirror's 2019 bulk import. +- Maven Central lists versions in version order, not in the order they were published. A patch to an older release line published after a newer release that is out of cooldown stays listed while it is in cooldown, but its files are still refused. +- SNAPSHOT versions are not held back. + With cooldown on, the proxy asks npm for full packuments, since the abbreviated ones it uses otherwise carry no publish times. Abbreviated packuments cached before cooldown was turned on are fetched again in full rather than served, including as a stale copy when the upstream is down. Note: Hex cooldown requires disabling registry signature verification since the proxy re-encodes the protobuf payload without the original signature. Set `HEX_NO_VERIFY_REPO_ORIGIN=1` or configure your repo with `no_verify: true`. From 78f614e82cf7b03222f248e5ec05f9e13611044f Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Fri, 23 Oct 2026 19:50:34 +0200 Subject: [PATCH 382/633] Read a Maven version's time from its .pom download A build fetching pinned versions never reads maven-metadata.xml, so each version's time was read with a HEAD before its download, roughly doubling the requests to Maven Central, which answered a fresh build with 429 and a 30-minute Retry-After. The .pom's own response now gives the time; Maven fetches the .pom before a version's other files, so their check reads the stored time. A HEAD is only sent for a .pom served from the cache or a file requested before its .pom. A 429 on that HEAD now answers 503 with the upstream's Retry-After, as other held upstream requests do, instead of 502. --- internal/handler/handler.go | 41 ++++++++++++++- internal/handler/maven.go | 14 ++++-- internal/handler/maven_cooldown.go | 34 +++++++++++++ internal/handler/maven_cooldown_test.go | 66 ++++++++++++++++++++++++- 4 files changed, 148 insertions(+), 7 deletions(-) diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 8bcac24..b5c34ef 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -1965,7 +1965,7 @@ func (p *Proxy) getOrFetchArtifactFromURLWithCachePURLs(ctx context.Context, eco if p.StreamArtifacts { return p.streamFromUpstream(ctx, ecosystem, name, version, filename, versionPURL, downloadURL, upstreamHash, declared, func(fetchCtx context.Context) (*fetch.Artifact, error) { - return p.Fetcher.FetchWithHeaders(fetchCtx, downloadURL, headers) + return p.fetchUpstream(fetchCtx, downloadURL, headers) }) } if cached, err := p.getCachedArtifactWithUpstreamHash(ctx, pkgPURL, versionPURL, filename, upstreamHash); err != nil { @@ -2040,7 +2040,7 @@ func (p *Proxy) fetchAndCacheFromURL(ctx context.Context, ecosystem, name, versi "ecosystem", ecosystem, "name", name, "version", version, "url", downloadURL) fetchStart := time.Now() - artifact, err := p.Fetcher.FetchWithHeaders(ctx, downloadURL, headers) + artifact, err := p.fetchUpstream(ctx, downloadURL, headers) metrics.RecordUpstreamFetch(ecosystem, time.Since(fetchStart)) if err != nil { metrics.RecordUpstreamError(ecosystem, "fetch_failed", metrics.FailureDetail{ @@ -2054,6 +2054,43 @@ func (p *Proxy) fetchAndCacheFromURL(ctx context.Context, ecosystem, name, versi return p.storeArtifact(ctx, ecosystem, name, version, filename, pkgPURL, versionPURL, downloadURL, upstreamHash, declared, artifact) } +type upstreamLastModifiedKey struct{} + +// WithUpstreamLastModified returns a context under which an artifact fetched +// from upstream reports its Last-Modified to fn, so a handler can learn a +// version's publish time from the download itself rather than with a +// request of its own. fn is not called for a cache hit, or when the +// response has no usable Last-Modified. +func WithUpstreamLastModified(ctx context.Context, fn func(time.Time)) context.Context { + return context.WithValue(ctx, upstreamLastModifiedKey{}, fn) +} + +// observedFetcher is the part of fetch.Fetcher and its circuit breaker that +// reports response headers. +type observedFetcher interface { + FetchObservedWithHeaders(ctx context.Context, url string, headers http.Header) (*fetch.ObservedArtifact, error) +} + +// fetchUpstream fetches downloadURL, reporting its Last-Modified when ctx +// asks for it. +func (p *Proxy) fetchUpstream(ctx context.Context, downloadURL string, headers http.Header) (*fetch.Artifact, error) { + report, _ := ctx.Value(upstreamLastModifiedKey{}).(func(time.Time)) + observer, ok := p.Fetcher.(observedFetcher) + if report == nil || !ok { + return p.Fetcher.FetchWithHeaders(ctx, downloadURL, headers) + } + observed, err := observer.FetchObservedWithHeaders(ctx, downloadURL, headers) + if err != nil { + return nil, err + } + if observed.Observation != nil { + if t, err := http.ParseTime(observed.Observation.Headers.Get(headerLastModified)); err == nil { + report(t) + } + } + return observed.Artifact, nil +} + // streamFromUpstream fetches an artifact when StreamArtifacts is set and hands its // body to the caller without storing it. // diff --git a/internal/handler/maven.go b/internal/handler/maven.go index 4885906..c201e74 100644 --- a/internal/handler/maven.go +++ b/internal/handler/maven.go @@ -225,7 +225,11 @@ func (h *MavenHandler) handleDownload(w http.ResponseWriter, r *http.Request, so return } - if h.checkCooldown(w, r, source, urlPath, version) { + // The .pom of a version whose time isn't known yet carries it in its + // own Last-Modified, so it is checked once fetched, without a request + // of its own. Maven fetches the .pom before a version's other files. + ctx, checkAfter := h.cooldownFromDownload(r.Context(), source, urlPath, version) + if !checkAfter && h.checkCooldown(w, r, source, urlPath, version) { return } @@ -238,19 +242,23 @@ func (h *MavenHandler) handleDownload(w http.ResponseWriter, r *http.Request, so upstreamURL := fmt.Sprintf("%s/%s", source.upstreamURL, urlPath) cacheFilename := source.cacheFilename(filename) - result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "maven", name, version, cacheFilename, upstreamURL) + result, err := h.proxy.GetOrFetchArtifactFromURL(ctx, "maven", name, version, cacheFilename, upstreamURL) if err != nil && source.fallbackURL != "" { if errors.Is(err, ErrUpstreamNotFound) { pluginPortalURL := fmt.Sprintf("%s/%s", source.fallbackURL, urlPath) h.proxy.Logger.Info("maven artifact not found in primary upstream, trying Gradle Plugin Portal", "group", group, "artifact", artifact, "version", version, "filename", filename) - result, err = h.proxy.GetOrFetchArtifactFromURL(r.Context(), "maven", name, version, cacheFilename, pluginPortalURL) + result, err = h.proxy.GetOrFetchArtifactFromURL(ctx, "maven", name, version, cacheFilename, pluginPortalURL) } } if err != nil { h.proxy.serveArtifactError(w, err, "failed to fetch artifact") return } + if checkAfter && h.checkCooldown(w, r, source, urlPath, version) { + _ = result.Reader.Close() + return + } ServeArtifactRequest(w, r, result) } diff --git a/internal/handler/maven_cooldown.go b/internal/handler/maven_cooldown.go index ca5a03a..d3388c7 100644 --- a/internal/handler/maven_cooldown.go +++ b/internal/handler/maven_cooldown.go @@ -129,6 +129,34 @@ func (h *MavenHandler) publishTime(ctx context.Context, source mavenSource, arti return published, nil } +// cooldownFromDownload returns a context that stores the publish time of a +// version from the download of its .pom, and whether the cooldown check is +// to run after the download. It does so only for the .pom of a version +// whose time is not stored yet; for anything else the check runs before. +func (h *MavenHandler) cooldownFromDownload(ctx context.Context, source mavenSource, urlPath, version string) (context.Context, bool) { + if !h.cooldownEnabled() || strings.HasSuffix(version, mavenSnapshot) { + return ctx, false + } + artifactDir := path.Dir(path.Dir(urlPath)) + group, artifact, ok := mavenCoordinates(artifactDir) + if !ok || !usableMavenVersion(version) || path.Base(urlPath) != artifact+"-"+version+".pom" { + return ctx, false + } + versionPURL, packagePURL, store := h.mavenTimeKey(source, group, artifact, version) + if !store { + return ctx, false + } + if ver, err := h.proxy.DB.GetVersionByPURL(versionPURL); err == nil && ver != nil && ver.PublishedAt.Valid { + return ctx, false + } + return WithUpstreamLastModified(ctx, func(published time.Time) { + if err := h.proxy.DB.SetVersionPublishedAt(versionPURL, packagePURL, published); err != nil { + h.proxy.Logger.Warn("cooldown: could not store maven publish time", + "group", group, "artifact", artifact, "version", version, "error", err) + } + }), true +} + // pomLastModified sends a HEAD request for a .pom and returns its // Last-Modified. found is false when upstream doesn't have the file. Only a // 2xx response's header is used: an error page's date says nothing about the @@ -146,6 +174,12 @@ func (h *MavenHandler) pomLastModified(ctx context.Context, pomURL string) (publ switch { case resp.StatusCode == http.StatusNotFound || resp.StatusCode == http.StatusGone: return time.Time{}, false, nil + case resp.StatusCode == http.StatusTooManyRequests && h.proxy.RateLimits != nil: + // The gate has just noted the upstream's Retry-After; pass it on. + if err := h.proxy.RateLimits.Held(pomURL); err != nil { + return time.Time{}, false, err + } + return time.Time{}, false, fmt.Errorf("reading pom time: upstream returned %d", resp.StatusCode) case resp.StatusCode < 200 || resp.StatusCode > 299: return time.Time{}, false, fmt.Errorf("reading pom time: upstream returned %d", resp.StatusCode) } diff --git a/internal/handler/maven_cooldown_test.go b/internal/handler/maven_cooldown_test.go index 74c4268..9a8e271 100644 --- a/internal/handler/maven_cooldown_test.go +++ b/internal/handler/maven_cooldown_test.go @@ -15,6 +15,7 @@ import ( "time" "github.com/git-pkgs/cooldown" + registryclient "github.com/git-pkgs/registries/client" "github.com/git-pkgs/registries/fetch" ) @@ -32,8 +33,9 @@ type mavenCooldownUpstream struct { times map[string]time.Time pomStatus map[string]int - mu sync.Mutex - poms map[string]int + mu sync.Mutex + poms map[string]int + heads int } func newMavenCooldownUpstream(t *testing.T) *mavenCooldownUpstream { @@ -84,11 +86,17 @@ func (u *mavenCooldownUpstream) serve(w http.ResponseWriter, r *http.Request) { if strings.HasSuffix(file, ".pom") { u.mu.Lock() u.poms[version]++ + if r.Method == http.MethodHead { + u.heads++ + } u.mu.Unlock() w.Header().Set(headerLastModified, published.UTC().Format(http.TimeFormat)) if status := u.pomStatus[version]; status != 0 { // An error page dated long ago must not count as an old version. w.Header().Set(headerLastModified, time.Unix(0, 0).UTC().Format(http.TimeFormat)) + if status == http.StatusTooManyRequests { + w.Header().Set("Retry-After", "60") + } w.WriteHeader(status) return } @@ -265,6 +273,42 @@ func TestMavenCooldownRefusesFiles(t *testing.T) { } } +// TestMavenCooldownPOMDownloadCarriesItsTime checks that a build fetching +// pinned versions, which never reads maven-metadata.xml, costs no requests +// beyond its downloads: the .pom's own Last-Modified gives the version's +// time, and Maven fetches the .pom before the version's other files. +func TestMavenCooldownPOMDownloadCarriesItsTime(t *testing.T) { + u := newMavenCooldownUpstream(t) + h, proxy := newMavenCooldownHandler(t, u) + + for _, c := range []struct { + file string + want int + }{ + {"2.1/lib-2.1.pom", http.StatusForbidden}, + {"2.1/lib-2.1.jar", http.StatusForbidden}, + {"1.0/lib-1.0.pom", http.StatusOK}, + {"1.0/lib-1.0.jar", http.StatusOK}, + } { + if w := getMaven(h, mavenTestDir+c.file); w.Code != c.want { + t.Errorf("%s: status = %d, want %d", c.file, w.Code, c.want) + } + } + u.mu.Lock() + heads := u.heads + u.mu.Unlock() + if heads != 0 { + t.Errorf("HEAD requests = %d, want 0", heads) + } + if got := u.pomRequests(); got["2.1"] != 1 || got["1.0"] != 1 { + t.Errorf("pom requests = %v, want one GET each", got) + } + ver, _ := proxy.DB.GetVersionByPURL(canonicalVersionPURL("maven", "com.example:lib", "2.1")) + if ver == nil || !ver.PublishedAt.Valid || !ver.PublishedAt.Time.Equal(u.times["2.1"].Truncate(time.Second)) { + t.Errorf("stored time of 2.1 = %v, want %v", ver, u.times["2.1"]) + } +} + func TestMavenCooldownOldLastUpdatedSkipsPOMs(t *testing.T) { u := newMavenCooldownUpstream(t) u.lastUpdated = time.Now().Add(-4 * 24 * time.Hour) @@ -307,6 +351,24 @@ func TestMavenCooldownIgnoresNon2xxPOMResponses(t *testing.T) { } } +// TestMavenCooldownRateLimitedPOMIsHeld checks that a 429 on a .pom time +// request is answered like any held upstream request, with 503 and the +// upstream's Retry-After, so clients wait instead of failing the build. +func TestMavenCooldownRateLimitedPOMIsHeld(t *testing.T) { + u := newMavenCooldownUpstream(t) + u.pomStatus["2.0"] = http.StatusTooManyRequests + h, proxy := newMavenCooldownHandler(t, u) + gate := registryclient.NewRateLimitGate(u.server.Client().Transport) + proxy.HTTPClient = &http.Client{Transport: gate} + proxy.RateLimits = gate + + // Not the .pom, so its time is read with a HEAD. + w := getMaven(h, mavenTestDir+"2.0/lib-2.0.jar") + if w.Code != http.StatusServiceUnavailable || w.Header().Get("Retry-After") == "" { + t.Errorf("status = %d, Retry-After %q; want 503 with Retry-After", w.Code, w.Header().Get("Retry-After")) + } +} + func TestMavenCooldownDisabledPassesThrough(t *testing.T) { u := newMavenCooldownUpstream(t) h, proxy := newMavenCooldownHandler(t, u) From f81274b499234d5149c490b894f7ceb1d77ceb9e Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Fri, 23 Oct 2026 20:07:53 +0200 Subject: [PATCH 383/633] Describe how Maven publish times are read --- docs/configuration.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/configuration.md b/docs/configuration.md index d932292..d052ff9 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -984,7 +984,7 @@ Currently supported for npm, PyPI, pub.dev, Composer, Cargo, NuGet, Conda, RubyG Go has no publish time. The module proxy's `.info` gives the commit time of the revision a version points at, so a version tagged on an old commit counts as old. The proxy reads each listed version's `.info` once, stores the time, and leaves versions in cooldown out of `@v/list` and `@latest`. When upstream's `@latest` is in cooldown, it serves the newest release out of cooldown instead. Requests for a version in cooldown, including its `.info`, `.mod` and `.zip`, and a branch or commit query that resolves to one, get 403 rather than 404, because the go command moves on to the next `GOPROXY` entry, such as `direct`, after a 404. If a version's `.info` can't be fetched, the list or download fails with 502 instead of letting the version through. -Maven metadata has no publish time per version. The proxy uses the `Last-Modified` header of each version's `.pom`, which on Maven Central is the time the version was deployed. It reads the header once per version with a `HEAD` request and stores the time, for each repository under `upstream.maven_repositories` separately. To keep requests to the upstream down, it only checks versions that can still be in cooldown: none when the metadata's `` is older than the cooldown, and otherwise the versions from the end of the list back to the first one that is old enough. Versions in cooldown are left out of `maven-metadata.xml`, `` and `` are set to the last versions left, and the metadata's `.md5`, `.sha1`, `.sha256` and `.sha512` files are computed from what is served. Requests for any file of a version in cooldown get 403, as for a version on the denylist. If a `.pom` can't be checked, the metadata or download fails with 502 instead of letting the version through; a version without a `.pom` is let through. +Maven metadata has no publish time per version. The proxy uses the `Last-Modified` header of each version's `.pom`, which on Maven Central is the time the version was deployed. It reads the header once per version and stores the time, for each repository under `upstream.maven_repositories` separately: from the response when the `.pom` itself is downloaded, which Maven does before a version's other files, and otherwise with a `HEAD` request. To keep requests to the upstream down, it only checks versions that can still be in cooldown: none when the metadata's `` is older than the cooldown, and otherwise the versions from the end of the list back to the first one that is old enough. Versions in cooldown are left out of `maven-metadata.xml`, `` and `` are set to the last versions left, and the metadata's `.md5`, `.sha1`, `.sha256` and `.sha512` files are computed from what is served. Requests for any file of a version in cooldown get 403, as for a version on the denylist. If a `.pom` can't be checked, the metadata or download fails with 502 instead of letting the version through, or with 503 and `Retry-After` when the upstream rate-limits the proxy; a version without a `.pom` is let through. Maven caveats: From faecb56c851709d0fcdfcdb38a7f818305a09f22 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Fri, 23 Oct 2026 20:53:06 +0200 Subject: [PATCH 384/633] Polish Maven cooldown after review - Name the proxy in the User-Agent of publish-time HEAD requests, as its artifact downloads do, rather than Go's default, which Maven Central asks tools not to send. - Take a version's time from its Gradle .module as well as its .pom, so either one fetched first carries it. - Serve filtered metadata through the shared rewrite path, with gzip and the stale warning like the other ecosystems. - Say in the docs that a refused .pom or .module is still cached. --- docs/configuration.md | 2 +- internal/handler/maven_cooldown.go | 38 +++++------ internal/handler/maven_cooldown_test.go | 91 ++++++++++++++++++++++++- internal/handler/rewrite_cache.go | 8 ++- 4 files changed, 114 insertions(+), 25 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index d052ff9..226d23e 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -984,7 +984,7 @@ Currently supported for npm, PyPI, pub.dev, Composer, Cargo, NuGet, Conda, RubyG Go has no publish time. The module proxy's `.info` gives the commit time of the revision a version points at, so a version tagged on an old commit counts as old. The proxy reads each listed version's `.info` once, stores the time, and leaves versions in cooldown out of `@v/list` and `@latest`. When upstream's `@latest` is in cooldown, it serves the newest release out of cooldown instead. Requests for a version in cooldown, including its `.info`, `.mod` and `.zip`, and a branch or commit query that resolves to one, get 403 rather than 404, because the go command moves on to the next `GOPROXY` entry, such as `direct`, after a 404. If a version's `.info` can't be fetched, the list or download fails with 502 instead of letting the version through. -Maven metadata has no publish time per version. The proxy uses the `Last-Modified` header of each version's `.pom`, which on Maven Central is the time the version was deployed. It reads the header once per version and stores the time, for each repository under `upstream.maven_repositories` separately: from the response when the `.pom` itself is downloaded, which Maven does before a version's other files, and otherwise with a `HEAD` request. To keep requests to the upstream down, it only checks versions that can still be in cooldown: none when the metadata's `` is older than the cooldown, and otherwise the versions from the end of the list back to the first one that is old enough. Versions in cooldown are left out of `maven-metadata.xml`, `` and `` are set to the last versions left, and the metadata's `.md5`, `.sha1`, `.sha256` and `.sha512` files are computed from what is served. Requests for any file of a version in cooldown get 403, as for a version on the denylist. If a `.pom` can't be checked, the metadata or download fails with 502 instead of letting the version through, or with 503 and `Retry-After` when the upstream rate-limits the proxy; a version without a `.pom` is let through. +Maven metadata has no publish time per version. The proxy uses the `Last-Modified` header of each version's `.pom`, which on Maven Central is the time the version was deployed. It reads the header once per version and stores the time, for each repository under `upstream.maven_repositories` separately: from the response when the `.pom` or Gradle `.module` itself is downloaded, which clients do before a version's other files, and otherwise with a `HEAD` request. A `.pom` or `.module` refused this way is still cached, so it counts toward `max_size` before the version leaves cooldown. To keep requests to the upstream down, it only checks versions that can still be in cooldown: none when the metadata's `` is older than the cooldown, and otherwise the versions from the end of the list back to the first one that is old enough. Versions in cooldown are left out of `maven-metadata.xml`, `` and `` are set to the last versions left, and the metadata's `.md5`, `.sha1`, `.sha256` and `.sha512` files are computed from what is served. Requests for any file of a version in cooldown get 403, as for a version on the denylist. If a `.pom` can't be checked, the metadata or download fails with 502 instead of letting the version through, or with 503 and `Retry-After` when the upstream rate-limits the proxy; a version without a `.pom` is let through. Maven caveats: diff --git a/internal/handler/maven_cooldown.go b/internal/handler/maven_cooldown.go index d3388c7..d67d9c9 100644 --- a/internal/handler/maven_cooldown.go +++ b/internal/handler/maven_cooldown.go @@ -37,6 +37,10 @@ import ( const errMavenCooldown = "version is in cooldown" +// mavenTimeUserAgent is the User-Agent of publish-time requests, the one +// the artifact fetcher sends. +const mavenTimeUserAgent = "git-pkgs-proxy/1.0" + const ( mavenMetadataFile = "maven-metadata.xml" mavenSnapshot = "-SNAPSHOT" @@ -130,16 +134,19 @@ func (h *MavenHandler) publishTime(ctx context.Context, source mavenSource, arti } // cooldownFromDownload returns a context that stores the publish time of a -// version from the download of its .pom, and whether the cooldown check is -// to run after the download. It does so only for the .pom of a version -// whose time is not stored yet; for anything else the check runs before. +// version from the download of its .pom or Gradle .module, and whether the +// cooldown check is to run after the download. It does so only for those +// files of a version whose time is not stored yet; for anything else the +// check runs before. Both files are deployed with the version, and +// whichever a client fetches first gives its time. func (h *MavenHandler) cooldownFromDownload(ctx context.Context, source mavenSource, urlPath, version string) (context.Context, bool) { if !h.cooldownEnabled() || strings.HasSuffix(version, mavenSnapshot) { return ctx, false } artifactDir := path.Dir(path.Dir(urlPath)) group, artifact, ok := mavenCoordinates(artifactDir) - if !ok || !usableMavenVersion(version) || path.Base(urlPath) != artifact+"-"+version+".pom" { + if base := artifact + "-" + version; !ok || !usableMavenVersion(version) || + (path.Base(urlPath) != base+".pom" && path.Base(urlPath) != base+".module") { return ctx, false } versionPURL, packagePURL, store := h.mavenTimeKey(source, group, artifact, version) @@ -166,6 +173,9 @@ func (h *MavenHandler) pomLastModified(ctx context.Context, pomURL string) (publ if err != nil { return time.Time{}, false, err } + // Name the proxy as its artifact downloads do; Maven Central asks + // tools not to send an HTTP library's default. + req.Header.Set("User-Agent", mavenTimeUserAgent) resp, err := h.proxy.HTTPClient.Do(req) if err != nil { return time.Time{}, false, fmt.Errorf("reading pom time: %w", err) @@ -258,10 +268,13 @@ func (h *MavenHandler) serveCooldownMetadata(w http.ResponseWriter, r *http.Requ switch { case suffix == "": + // The metadata cache records JSON when an upstream sent no type, + // and maven-metadata.xml is always XML. if contentType == "" || contentType == contentTypeJSON { contentType = "text/xml" } - serveMavenMetadata(w, r, m, contentType) + markStale(w, h.proxy.lookupCachedMeta("maven", cacheKey).stale) + serveRewrittenMetadataAs(w, r, m, contentType) case mavenChecksums[suffix] != nil: sum := mavenChecksums[suffix]() _, _ = sum.Write(m.body) @@ -276,21 +289,6 @@ func (h *MavenHandler) serveCooldownMetadata(w http.ResponseWriter, r *http.Requ return true } -// serveMavenMetadata writes filtered metadata with its validators, or 304 -// when they show the client's copy is current. -func serveMavenMetadata(w http.ResponseWriter, r *http.Request, m rewrittenMetadata, contentType string) { - w.Header().Set(headerETag, m.etag) - if !m.lastModified.IsZero() { - w.Header().Set(headerLastModified, m.lastModified.UTC().Format(http.TimeFormat)) - } - if notModified(r, m.etag, m.lastModified) { - w.WriteHeader(http.StatusNotModified) - return - } - w.Header().Set(headerContentType, contentType) - _, _ = w.Write(m.body) -} - // filterMetadata removes versions in cooldown from an artifact's // maven-metadata.xml and points and at the newest // versions left. Metadata without a version list, such as a group's plugin diff --git a/internal/handler/maven_cooldown_test.go b/internal/handler/maven_cooldown_test.go index 9a8e271..828d1ff 100644 --- a/internal/handler/maven_cooldown_test.go +++ b/internal/handler/maven_cooldown_test.go @@ -1,14 +1,18 @@ package handler import ( + "compress/gzip" "crypto/md5" "crypto/sha1" "crypto/sha256" "crypto/sha512" "encoding/hex" + "fmt" "hash" + "io" "net/http" "net/http/httptest" + "slices" "strings" "sync" "testing" @@ -33,9 +37,10 @@ type mavenCooldownUpstream struct { times map[string]time.Time pomStatus map[string]int - mu sync.Mutex - poms map[string]int - heads int + mu sync.Mutex + poms map[string]int + heads int + userAgents []string } func newMavenCooldownUpstream(t *testing.T) *mavenCooldownUpstream { @@ -83,11 +88,15 @@ func (u *mavenCooldownUpstream) serve(w http.ResponseWriter, r *http.Request) { http.NotFound(w, r) return } + if strings.HasSuffix(file, ".module") { + w.Header().Set(headerLastModified, published.UTC().Format(http.TimeFormat)) + } if strings.HasSuffix(file, ".pom") { u.mu.Lock() u.poms[version]++ if r.Method == http.MethodHead { u.heads++ + u.userAgents = append(u.userAgents, r.UserAgent()) } u.mu.Unlock() w.Header().Set(headerLastModified, published.UTC().Format(http.TimeFormat)) @@ -309,6 +318,82 @@ func TestMavenCooldownPOMDownloadCarriesItsTime(t *testing.T) { } } +// TestMavenCooldownModuleDownloadCarriesItsTime checks that a Gradle +// .module fetched before the .pom gives the version's time as the .pom +// would. +func TestMavenCooldownModuleDownloadCarriesItsTime(t *testing.T) { + u := newMavenCooldownUpstream(t) + h, _ := newMavenCooldownHandler(t, u) + + if w := getMaven(h, mavenTestDir+"2.1/lib-2.1.module"); w.Code != http.StatusForbidden { + t.Errorf("2.1 .module: status = %d, want 403", w.Code) + } + if w := getMaven(h, mavenTestDir+"1.0/lib-1.0.module"); w.Code != http.StatusOK { + t.Errorf("1.0 .module: status = %d, want 200", w.Code) + } + if w := getMaven(h, mavenTestDir+"1.0/lib-1.0.jar"); w.Code != http.StatusOK { + t.Errorf("1.0 .jar: status = %d, want 200", w.Code) + } + if n := u.totalPOMRequests(); n != 0 { + t.Errorf("pom requests = %d, want 0", n) + } +} + +// TestMavenCooldownTimeRequestsNameTheProxy checks the User-Agent of the +// HEAD requests for publish times: Maven Central asks tools to name +// themselves rather than send an HTTP library's default. +func TestMavenCooldownTimeRequestsNameTheProxy(t *testing.T) { + u := newMavenCooldownUpstream(t) + h, _ := newMavenCooldownHandler(t, u) + + getMaven(h, mavenTestDir+"maven-metadata.xml") + u.mu.Lock() + defer u.mu.Unlock() + if len(u.userAgents) == 0 { + t.Fatal("no HEAD requests") + } + for _, ua := range u.userAgents { + if ua != mavenTimeUserAgent { + t.Errorf("User-Agent = %q, want %q", ua, mavenTimeUserAgent) + } + } +} + +// TestMavenCooldownMetadataServedLikeOtherRewrites checks that filtered +// metadata is gzip-compressed for clients that accept it, as the other +// ecosystems' rewritten metadata is. +func TestMavenCooldownMetadataServedLikeOtherRewrites(t *testing.T) { + u := newMavenCooldownUpstream(t) + // Enough versions that the document compresses. + for i := range 50 { + v := fmt.Sprintf("0.%d", i) + u.versions = append([]string{v}, u.versions...) + u.times[v] = time.Now().Add(-100 * 24 * time.Hour) + } + h, proxy := newMavenCooldownHandler(t, u) + // The server always sets up the rewrite cache, which keeps the gzip copy. + proxy.SetMetadataRewriteCacheSize(1 << 20) + + req := httptest.NewRequest(http.MethodGet, mavenTestDir+"maven-metadata.xml", nil) + req.Header.Set(headerAcceptEncoding, "gzip") + w := httptest.NewRecorder() + h.ServeHTTP(w, req) + if w.Code != http.StatusOK || w.Header().Get(headerContentEncoding) != "gzip" { + t.Fatalf("status = %d, Content-Encoding %q; want gzip", w.Code, w.Header().Get(headerContentEncoding)) + } + if ct := w.Header().Get(headerContentType); !strings.Contains(ct, "xml") { + t.Errorf("Content-Type = %q, want XML", ct) + } + zr, err := gzip.NewReader(w.Body) + if err != nil { + t.Fatal(err) + } + body, _ := io.ReadAll(zr) + if versions := mavenMetadataVersions(t, string(body)); slices.Contains(versions, "2.1") { + t.Errorf("2.1 listed in the gzip body: %v", versions) + } +} + func TestMavenCooldownOldLastUpdatedSkipsPOMs(t *testing.T) { u := newMavenCooldownUpstream(t) u.lastUpdated = time.Now().Add(-4 * 24 * time.Hour) diff --git a/internal/handler/rewrite_cache.go b/internal/handler/rewrite_cache.go index 0569a9c..991443a 100644 --- a/internal/handler/rewrite_cache.go +++ b/internal/handler/rewrite_cache.go @@ -402,6 +402,12 @@ func (p *Proxy) storedRewrite(ecosystem, cacheKey, proxyURL, name string) (rewri // accepting gzip gets the compressed copy when there is one. Being another // representation, it has an ETag of its own. func serveRewrittenMetadata(w http.ResponseWriter, r *http.Request, m rewrittenMetadata) { + serveRewrittenMetadataAs(w, r, m, contentTypeJSON) +} + +// serveRewrittenMetadataAs is serveRewrittenMetadata for a document of +// another content type. +func serveRewrittenMetadataAs(w http.ResponseWriter, r *http.Request, m rewrittenMetadata, contentType string) { body, etag := m.body, m.etag compressed := m.gzipped != nil && acceptsGzip(r) if compressed { @@ -416,7 +422,7 @@ func serveRewrittenMetadata(w http.ResponseWriter, r *http.Request, m rewrittenM w.WriteHeader(http.StatusNotModified) return } - w.Header().Set(headerContentType, contentTypeJSON) + w.Header().Set(headerContentType, contentType) if compressed { w.Header().Set(headerContentEncoding, "gzip") } From 003c76049fec3939d70cf727637e6cf09ab3b4bb Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sat, 24 Oct 2026 07:48:08 +0200 Subject: [PATCH 385/633] Require a JSON content type to start mirror jobs A page on another site could make a visitor's browser POST a text/plain body that parses as JSON and start mirror jobs. Browsers only send application/json cross-origin after a CORS preflight, which the proxy never answers, so HandleCreate now refuses other content types with 415. --- docs/configuration.md | 2 +- internal/server/errors.go | 2 ++ internal/server/mirror_api.go | 14 ++++++++++++ internal/server/mirror_api_test.go | 36 ++++++++++++++++++++++++++++++ 4 files changed, 53 insertions(+), 1 deletion(-) diff --git a/docs/configuration.md b/docs/configuration.md index fa77ec6..d594627 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1399,7 +1399,7 @@ Or via environment variable: `PROXY_MIRROR_API=true`. When disabled, the endpoints are not registered and return 404. -Start a mirror job with either PURLs or an inline CycloneDX or SPDX JSON document: +Start a mirror job with either PURLs or an inline CycloneDX or SPDX JSON document. Requests without `Content-Type: application/json` get 415, so other websites cannot start jobs through a visitor's browser: ```bash curl -X POST http://localhost:8080/api/mirror \ diff --git a/internal/server/errors.go b/internal/server/errors.go index c856c0e..88fa325 100644 --- a/internal/server/errors.go +++ b/internal/server/errors.go @@ -15,6 +15,8 @@ const ( ErrCodeUpstream = "UPSTREAM_ERROR" ErrCodeInternal = "INTERNAL_ERROR" ErrCodeBusy = "BUSY" + + ErrCodeUnsupportedMediaType = "UNSUPPORTED_MEDIA_TYPE" ) // ErrorResponse is the JSON body returned for API errors. diff --git a/internal/server/mirror_api.go b/internal/server/mirror_api.go index 028d4e0..89de3d5 100644 --- a/internal/server/mirror_api.go +++ b/internal/server/mirror_api.go @@ -2,6 +2,7 @@ package server import ( "encoding/json" + "mime" "net/http" "github.com/git-pkgs/proxy/internal/mirror" @@ -20,6 +21,14 @@ func NewMirrorAPIHandler(jobs *mirror.JobStore) *MirrorAPIHandler { // HandleCreate starts a new mirror job. func (h *MirrorAPIHandler) HandleCreate(w http.ResponseWriter, r *http.Request) { + // A browser can only send a JSON content type cross-origin after a CORS + // preflight, which the proxy never answers, so requiring it stops other + // sites from starting mirror jobs through a visitor's browser. + if !isJSONRequest(r) { + writeError(w, http.StatusUnsupportedMediaType, ErrCodeUnsupportedMediaType, + "Content-Type must be application/json") + return + } r.Body = http.MaxBytesReader(w, r.Body, maxBodySize) var req mirror.JobRequest if err := json.NewDecoder(r.Body).Decode(&req); err != nil { @@ -59,3 +68,8 @@ func (h *MirrorAPIHandler) HandleCancel(w http.ResponseWriter, r *http.Request) notFound(w, "job not found or not running") } } + +func isJSONRequest(r *http.Request) bool { + mediaType, _, err := mime.ParseMediaType(r.Header.Get("Content-Type")) + return err == nil && mediaType == "application/json" +} diff --git a/internal/server/mirror_api_test.go b/internal/server/mirror_api_test.go index c960348..4567ae1 100644 --- a/internal/server/mirror_api_test.go +++ b/internal/server/mirror_api_test.go @@ -8,6 +8,7 @@ import ( "net/http" "net/http/httptest" "os" + "strings" "testing" "github.com/git-pkgs/proxy/internal/database" @@ -55,6 +56,7 @@ func TestMirrorAPICreateJob(t *testing.T) { }) req := httptest.NewRequest("POST", "/api/mirror", bytes.NewReader(body)) + req.Header.Set("Content-Type", "application/json") w := httptest.NewRecorder() h.HandleCreate(w, req) @@ -91,6 +93,7 @@ func TestMirrorAPICreateJobFromInlineSBOM(t *testing.T) { } req := httptest.NewRequest(http.MethodPost, "/api/mirror", bytes.NewReader(body)) + req.Header.Set("Content-Type", "application/json") w := httptest.NewRecorder() h.HandleCreate(w, req) @@ -119,6 +122,7 @@ func TestMirrorAPICreateJobRejectsPURLsAndSBOM(t *testing.T) { } req := httptest.NewRequest(http.MethodPost, "/api/mirror", bytes.NewReader(body)) + req.Header.Set("Content-Type", "application/json") w := httptest.NewRecorder() h.HandleCreate(w, req) @@ -132,6 +136,7 @@ func TestMirrorAPICreateOversizedBody(t *testing.T) { body := bytes.Repeat([]byte("x"), int(maxBodySize)+1) req := httptest.NewRequest("POST", "/api/mirror", bytes.NewReader(body)) + req.Header.Set("Content-Type", "application/json") w := httptest.NewRecorder() h.HandleCreate(w, req) @@ -144,6 +149,7 @@ func TestMirrorAPICreateInvalidBody(t *testing.T) { h := setupMirrorAPI(t) req := httptest.NewRequest("POST", "/api/mirror", bytes.NewReader([]byte("not json"))) + req.Header.Set("Content-Type", "application/json") w := httptest.NewRecorder() h.HandleCreate(w, req) @@ -157,6 +163,7 @@ func TestMirrorAPICreateEmptyRequest(t *testing.T) { body, _ := json.Marshal(mirror.JobRequest{}) req := httptest.NewRequest("POST", "/api/mirror", bytes.NewReader(body)) + req.Header.Set("Content-Type", "application/json") w := httptest.NewRecorder() h.HandleCreate(w, req) @@ -203,6 +210,7 @@ func TestMirrorAPICreateAndGetJob(t *testing.T) { PURLs: []string{"pkg:npm/lodash@4.17.21"}, }) createReq := httptest.NewRequest("POST", "/api/mirror", bytes.NewReader(body)) + createReq.Header.Set("Content-Type", "application/json") createW := httptest.NewRecorder() h.HandleCreate(createW, createReq) @@ -230,3 +238,31 @@ func TestMirrorAPICreateAndGetJob(t *testing.T) { t.Errorf("job ID = %q, want %q", job.ID, jobID) } } + +// TestMirrorAPICreateRequiresJSON checks that only JSON requests start jobs. +// A page on another site can make a browser POST text/plain or form data +// without a CORS preflight, but not application/json. +func TestMirrorAPICreateRequiresJSON(t *testing.T) { + h := setupMirrorAPI(t) + body := `{"purls":["pkg:npm/lodash@4.17.21"],"x":"="}` + + for _, contentType := range []string{"", "text/plain", "application/x-www-form-urlencoded", "multipart/form-data; boundary=x"} { + req := httptest.NewRequest(http.MethodPost, "/api/mirror", strings.NewReader(body)) + if contentType != "" { + req.Header.Set("Content-Type", contentType) + } + w := httptest.NewRecorder() + h.HandleCreate(w, req) + if w.Code != http.StatusUnsupportedMediaType { + t.Errorf("Content-Type %q: status = %d, want %d", contentType, w.Code, http.StatusUnsupportedMediaType) + } + } + + req := httptest.NewRequest(http.MethodPost, "/api/mirror", strings.NewReader(body)) + req.Header.Set("Content-Type", "application/json; charset=utf-8") + w := httptest.NewRecorder() + h.HandleCreate(w, req) + if w.Code != http.StatusAccepted { + t.Errorf("application/json with charset: status = %d, want %d", w.Code, http.StatusAccepted) + } +} From 9a788958ec7822dd318c9375336f78c3caea3ee5 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sat, 24 Oct 2026 07:48:08 +0200 Subject: [PATCH 386/633] Refuse framing of the web UI Without X-Frame-Options or a frame-ancestors policy another site could frame a version page and trick a signed-in user into clicking Delete. Every /ui response now sends both; browsed archive files keep their sandbox policy alongside frame-ancestors. --- internal/server/browse.go | 2 +- internal/server/browse_test.go | 7 +++++-- internal/server/middleware.go | 12 ++++++++++++ internal/server/server.go | 1 + internal/server/server_test.go | 19 +++++++++++++++++++ 5 files changed, 38 insertions(+), 3 deletions(-) diff --git a/internal/server/browse.go b/internal/server/browse.go index d172ace..1184f2e 100644 --- a/internal/server/browse.go +++ b/internal/server/browse.go @@ -424,7 +424,7 @@ func (s *Server) writeBrowseFile(w http.ResponseWriter, file io.Reader, filePath contentType = detectContentTypeFromPrefix(head[:min(len(head), browseSniffSize)]) } w.Header().Set("Content-Type", contentType) - w.Header().Set("Content-Security-Policy", "sandbox") + w.Header().Set("Content-Security-Policy", "sandbox; "+frameAncestorsNone) w.Header().Set("X-Content-Type-Options", "nosniff") _, filename := path.Split(filePath) diff --git a/internal/server/browse_test.go b/internal/server/browse_test.go index c3352a4..34c864c 100644 --- a/internal/server/browse_test.go +++ b/internal/server/browse_test.go @@ -163,8 +163,11 @@ func TestHandleBrowseFile(t *testing.T) { if got := w.Header().Get("Content-Type"); got != file.contentType { t.Errorf("Content-Type = %q, want %q", got, file.contentType) } - if got := w.Header().Get("Content-Security-Policy"); got != "sandbox" { - t.Errorf("Content-Security-Policy = %q, want sandbox", got) + if got, want := w.Header().Get("Content-Security-Policy"), "sandbox; frame-ancestors 'none'"; got != want { + t.Errorf("Content-Security-Policy = %q, want %q", got, want) + } + if got := w.Header().Get("X-Frame-Options"); got != "DENY" { + t.Errorf("X-Frame-Options = %q, want DENY", got) } if got := w.Header().Get("X-Content-Type-Options"); got != "nosniff" { t.Errorf("X-Content-Type-Options = %q, want nosniff", got) diff --git a/internal/server/middleware.go b/internal/server/middleware.go index ad0fff5..5470d76 100644 --- a/internal/server/middleware.go +++ b/internal/server/middleware.go @@ -174,3 +174,15 @@ func sandboxUpstreamContent(next http.Handler) http.Handler { next.ServeHTTP(w, r) }) } + +// frameAncestorsNone stops other sites from framing the web UI, where a +// framed page could trick a signed-in user into clicking Delete. +const frameAncestorsNone = "frame-ancestors 'none'" + +func denyFraming(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("X-Frame-Options", "DENY") + w.Header().Set("Content-Security-Policy", frameAncestorsNone) + next.ServeHTTP(w, r) + }) +} diff --git a/internal/server/server.go b/internal/server/server.go index 5b0ebf7..c19d303 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -343,6 +343,7 @@ func (s *Server) serve(listener net.Listener) error { // Web UI. Mounted under /ui so a reverse proxy can apply different // access rules to it than to the package endpoints above (#123). r.Route("/ui", func(ui chi.Router) { + ui.Use(denyFraming) ui.Mount("/static", http.StripPrefix("/ui/static/", staticHandler())) ui.Get("/", s.handleRoot) ui.Get("/analytics", s.handleAnalytics) diff --git a/internal/server/server_test.go b/internal/server/server_test.go index 6d937dc..afd0ca9 100644 --- a/internal/server/server_test.go +++ b/internal/server/server_test.go @@ -121,6 +121,7 @@ func newTestServer(t *testing.T) *testServer { r.Get("/stats", s.handleStats) r.Get("/openapi.json", s.handleOpenAPIJSON) r.Route("/ui", func(ui chi.Router) { + ui.Use(denyFraming) ui.Mount("/static", http.StripPrefix("/ui/static/", staticHandler())) ui.Get("/", s.handleRoot) ui.Get("/analytics", s.handleAnalytics) @@ -1695,3 +1696,21 @@ func TestStatsEndpoint_StorageURL(t *testing.T) { t.Errorf("unexpected JSON key storage_path in response (should be storage_url)") } } + +// TestUIRefusesFraming checks that other sites cannot frame UI pages, where +// a framed Delete button could be clicked by a signed-in user. +func TestUIRefusesFraming(t *testing.T) { + ts := newTestServer(t) + defer ts.close() + + for _, target := range []string{"/ui/", "/ui/install", "/ui/static/style.css"} { + w := httptest.NewRecorder() + ts.handler.ServeHTTP(w, httptest.NewRequest(http.MethodGet, target, nil)) + if got := w.Header().Get("X-Frame-Options"); got != "DENY" { + t.Errorf("%s (status %d): X-Frame-Options = %q, want DENY", target, w.Code, got) + } + if got := w.Header().Get("Content-Security-Policy"); got != frameAncestorsNone { + t.Errorf("%s (status %d): Content-Security-Policy = %q, want %q", target, w.Code, got, frameAncestorsNone) + } + } +} From f27fe1193d35eae6c833ec7d02bdf2679b83ead6 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sat, 24 Oct 2026 07:59:32 +0200 Subject: [PATCH 387/633] Refuse overlong gem filenames parseGemFilename tries every "-" in the filename as the start of the version and matches the rest against two patterns each time, so its cost grows with the square of the filename's length. A path of a few hundred kilobytes kept a request busy for minutes, and the parse doesn't watch the request context. Gem filenames longer than 512 bytes are now answered with 400 before parsing. Real ones are far shorter. --- internal/handler/gem.go | 10 ++++++++++ internal/handler/gem_test.go | 25 +++++++++++++++++++++++++ 2 files changed, 35 insertions(+) diff --git a/internal/handler/gem.go b/internal/handler/gem.go index e1acb12..ce75510 100644 --- a/internal/handler/gem.go +++ b/internal/handler/gem.go @@ -99,7 +99,13 @@ func (h *GemHandler) handleDownload(w http.ResponseWriter, r *http.Request) { // starts at the first "-" that leaves such a version and platform, // so a name with a "-" segment of its own, such as devise-2fa, is // kept whole. A filename no split fits is split at the first "-". +// +// Trying every "-" costs time quadratic in the filename's length, so +// a filename longer than maxGemFilenameLength is refused. func (h *GemHandler) parseGemFilename(filename string) (name, version string) { + if len(filename) > maxGemFilenameLength { + return "", "" + } base := strings.TrimSuffix(filename, ".gem") first := -1 for i := 0; i+1 < len(base); i++ { @@ -119,6 +125,10 @@ func (h *GemHandler) parseGemFilename(filename string) (name, version string) { return base[:first], base[first+1:] } +// maxGemFilenameLength bounds the gem filenames parseGemFilename accepts. +// Real gem filenames are a small fraction of this. +const maxGemFilenameLength = 512 + var ( gemVersionPattern = regexp.MustCompile(`^[0-9]+(\.[0-9A-Za-z]+)*$`) gemPlatformPattern = regexp.MustCompile(`^[A-Za-z][0-9A-Za-z_]*(-[0-9A-Za-z_]+)*$`) diff --git a/internal/handler/gem_test.go b/internal/handler/gem_test.go index 9b71074..f4ec9ee 100644 --- a/internal/handler/gem_test.go +++ b/internal/handler/gem_test.go @@ -49,6 +49,31 @@ func TestGemParseFilename(t *testing.T) { } } +func TestGemDownloadRefusesLongFilename(t *testing.T) { + // Every "-1-" could start a version, and each try reads to the "!" at + // the end before failing, so parsing this used to take minutes. + filename := strings.Repeat("-1-aaaaaa", 100_000) + "!.gem" + + h := &GemHandler{proxy: testProxy(), upstreamURL: "http://upstream.invalid"} + req := httptest.NewRequest(http.MethodGet, "/gems/x", nil) + req.SetPathValue("filename", "x"+filename) + w := httptest.NewRecorder() + + done := make(chan struct{}) + go func() { + h.handleDownload(w, req) + close(done) + }() + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatal("parsing a long gem filename did not finish within 5s") + } + if w.Code != http.StatusBadRequest { + t.Errorf("status = %d, want %d", w.Code, http.StatusBadRequest) + } +} + func TestGemCompactIndexCooldown(t *testing.T) { now := time.Now() oldTime := now.Add(-7 * 24 * time.Hour).Format(time.RFC3339) From 351fe7c0b55c0a1f0e32e31ddfd73d9712ec0167 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sat, 24 Oct 2026 08:00:48 +0200 Subject: [PATCH 388/633] Send registry tokens only to the token service's own registry After a 401 the transport fetches a token from the challenge's realm, using credentials configured for that realm, and sends it to the host that issued the challenge. It trusted that host if it had any configured credentials, even unrelated ones, or shared the realm's registrable domain. The second rule makes every tenant on a shared domain such as jfrog.io, azurecr.io or pkg.dev the same site, and the first hands a token minted from one registry's credentials to another registry the proxy happens to have a key for. A credentialed token now goes only to a registry on the realm's own host and port, to one configured with the same credentials as the realm, or to Docker Hub and GitLab.com, whose registries use a token service on another host. Anonymous tokens are unaffected. The scope in the token request also came from the challenge, so a registry could ask for push access or for other repositories. The proxy only reads, so it now always asks for pull access to the repository in the request path. --- docs/configuration.md | 17 ++++- internal/httpclient/token_scope_test.go | 94 +++++++++++++++++++++---- internal/httpclient/transport.go | 65 +++++++++++------ 3 files changed, 139 insertions(+), 37 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index fa77ec6..e2a0513 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -839,9 +839,22 @@ This field has no environment variable, as with the other named upstream maps. Configure authentication for private upstream registries. The same authentication-aware client is used for metadata and artifact downloads, and credentials can reference environment variables using `${VAR_NAME}` syntax. -OCI registries that return a Bearer challenge from a `/v2/{repository}/…` endpoint are handled automatically. The proxy discovers the token realm from `WWW-Authenticate`, applies any configured credentials for the token URL, and reuses the scoped token until shortly before it expires. +OCI registries that return a Bearer challenge from a `/v2/{repository}/…` endpoint are handled automatically. The proxy discovers the token realm from `WWW-Authenticate`, applies any configured credentials for the token URL, and reuses the token until shortly before it expires. The token is always requested for pull access to the repository being fetched (`repository:{repository}:pull`), whatever scope the challenge names. -A token fetched with credentials configured for the token URL is only fetched for, and sent to, a registry that is trusted with it: one with configured credentials of its own, or one on the same site (registrable domain) as the token service, as `registry-1.docker.io` is with `auth.docker.io`. Any other host that answers with a challenge naming that token service gets its 401 back, so it cannot obtain a token minted from your credentials. Tokens from a token service with no configured credentials are anonymous and are fetched for any registry, as before. If your registry and its token service are on different sites, configure credentials for the registry URL as well. +A token fetched with credentials configured for the token URL is only fetched for, and sent to, a registry that is trusted with it: one on the same host and port as the token service, one configured with the same credentials as the token URL, or Docker Hub (`registry-1.docker.io` and `index.docker.io` with `auth.docker.io`) and GitLab.com (`registry.gitlab.com` with `gitlab.com`). Sharing a parent domain is not enough, so `team-a.jfrog.io` is not trusted with tokens from `team-b.jfrog.io`. Any other host that answers with a challenge naming that token service gets its 401 back, so it cannot obtain a token minted from your credentials. Tokens from a token service with no configured credentials are anonymous and are fetched for any registry, as before. If your registry and its token service are on different hosts, such as a self-hosted GitLab with its registry on its own hostname, configure the same credentials for both URLs: + +```yaml +upstream: + auth: + "https://registry.gitlab.example.com": + type: basic + username: "deploy-token" + password: "${GITLAB_DEPLOY_TOKEN}" + "https://gitlab.example.com/jwt/auth": + type: basic + username: "deploy-token" + password: "${GITLAB_DEPLOY_TOKEN}" +``` ### Bearer Token diff --git a/internal/httpclient/token_scope_test.go b/internal/httpclient/token_scope_test.go index 97d8154..563d939 100644 --- a/internal/httpclient/token_scope_test.go +++ b/internal/httpclient/token_scope_test.go @@ -141,24 +141,92 @@ func TestTransportFollowsAnonymousTokenServiceOnAnotherHost(t *testing.T) { } } -func TestSameSite(t *testing.T) { +// TestTransportDoesNotSendCredentialedTokenToHostWithOtherCredentials checks +// that a registry with credentials of its own, but not the realm's, is not +// handed a token minted from the realm's credentials. +func TestTransportDoesNotSendCredentialedTokenToHostWithOtherCredentials(t *testing.T) { + var issued atomic.Int32 + realm := tokenService(t, &issued) + var received []string + vendor := challengingRegistry(t, realm.URL, &received) + + auth := func(rawURL string) (string, string) { + switch { + case strings.HasPrefix(rawURL, realm.URL): + return "Authorization", "Basic Y29ycDpwYXNz" + case strings.HasPrefix(rawURL, vendor.URL): + return "Authorization", "Basic dmVuZG9yOmtleQ==" + } + return "", "" + } + client := &http.Client{Transport: NewTransport(http.DefaultTransport, auth)} + if status := get(t, client, vendor.URL+"/v2/corp/private/blobs/sha256:abc"); status != http.StatusUnauthorized { + t.Errorf("status = %d, want the challenge passed back as 401", status) + } + for _, got := range received { + if got != "Basic dmVuZG9yOmtleQ==" { + t.Errorf("the vendor registry received Authorization %q, want only its own credentials", got) + } + } + if n := issued.Load(); n != 0 { + t.Errorf("token service was asked %d times on the vendor registry's behalf, want 0", n) + } +} + +// TestTransportRequestsPullScopeForRequestedRepository checks that the token +// is requested for pull access to the repository in the request path, not +// whatever scope the challenge names. +func TestTransportRequestsPullScopeForRequestedRepository(t *testing.T) { + var scopes [][]string + var srv *httptest.Server + srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/token" { + scopes = append(scopes, r.URL.Query()["scope"]) + _, _ = io.WriteString(w, `{"token":"t","expires_in":3600}`) + return + } + if r.Header.Get("Authorization") != "Bearer t" { + w.Header().Set("WWW-Authenticate", `Bearer realm="`+srv.URL+`/token",service="registry.corp",scope="repository:corp/other:pull,push"`) + http.Error(w, "authentication required", http.StatusUnauthorized) + return + } + _, _ = io.WriteString(w, "blob") + })) + t.Cleanup(srv.Close) + + client := &http.Client{Transport: NewTransport(http.DefaultTransport, credentialsFor(srv.URL))} + if status := get(t, client, srv.URL+"/v2/corp/app/manifests/latest"); status != http.StatusOK { + t.Fatalf("status = %d, want 200", status) + } + if len(scopes) != 1 || len(scopes[0]) != 1 || scopes[0][0] != "repository:corp/app:pull" { + t.Errorf("token requested with scopes %q, want [[repository:corp/app:pull]]", scopes) + } +} + +func TestSameTokenService(t *testing.T) { for _, tt := range []struct { registry, realm string want bool }{ - {"https://registry-1.docker.io", "https://auth.docker.io", true}, - {"https://registry.gitlab.com", "https://gitlab.com", true}, - {"https://ghcr.io", "https://ghcr.io", true}, - {"https://registry.corp.example", "https://auth.corp.example:8443", true}, - {"https://evil.example", "https://auth.docker.io", false}, - {"https://docker.io.evil.example", "https://auth.docker.io", false}, - {"https://a.github.io", "https://b.github.io", false}, // public suffix: separate sites - {"http://127.0.0.1:5000", "http://127.0.0.1:5000", true}, - {"http://127.0.0.1:5000", "http://127.0.0.1:5001", false}, - {"http://localhost:5000", "http://localhost:5001", false}, + {"https://registry-1.docker.io", "https://auth.docker.io/token", true}, + {"https://index.docker.io", "https://auth.docker.io/token", true}, + {"https://registry.gitlab.com", "https://gitlab.com/jwt/auth", true}, + {"https://ghcr.io", "https://ghcr.io/token", true}, + {"https://harbor.corp.example", "https://harbor.corp.example/service/token", true}, + {"https://registry.corp.example", "https://auth.corp.example", false}, + {"https://evil.example", "https://auth.docker.io/token", false}, + {"https://docker.io.evil.example", "https://auth.docker.io/token", false}, + {"https://evil.jfrog.io", "https://myco.jfrog.io/artifactory/api/docker/token", false}, + {"https://evil.azurecr.io", "https://myco.azurecr.io/oauth2/token", false}, + {"https://gitlab.com", "https://registry.gitlab.com", false}, + {"http://registry-1.docker.io", "https://auth.docker.io/token", false}, + {"https://registry-1.docker.io", "https://auth.docker.io:8443/token", false}, + {"http://127.0.0.1:5000", "http://127.0.0.1:5000/token", true}, + {"http://127.0.0.1:5000", "http://127.0.0.1:5001/token", false}, + {"http://localhost:5000", "https://localhost:5000/token", false}, } { - if got := sameSite(mustParse(t, tt.registry), mustParse(t, tt.realm)); got != tt.want { - t.Errorf("sameSite(%s, %s) = %v, want %v", tt.registry, tt.realm, got, tt.want) + if got := sameTokenService(mustParse(t, tt.registry), mustParse(t, tt.realm)); got != tt.want { + t.Errorf("sameTokenService(%s, %s) = %v, want %v", tt.registry, tt.realm, got, tt.want) } } } diff --git a/internal/httpclient/transport.go b/internal/httpclient/transport.go index 56c0f48..ba78ee5 100644 --- a/internal/httpclient/transport.go +++ b/internal/httpclient/transport.go @@ -13,8 +13,6 @@ import ( "strings" "sync" "time" - - "golang.org/x/net/publicsuffix" ) const ( @@ -90,6 +88,9 @@ func (t *Transport) RoundTrip(req *http.Request) (*http.Response, error) { } challenge, ok := parseBearerChallenge(resp.Header.Values("WWW-Authenticate")) + // Ask only for pull access to the repository being fetched, whatever + // scope the challenge names: the proxy only reads. + challenge.scopes = []string{"repository:" + registryRepository(req.URL) + ":pull"} if !ok || !canReplay(req) || !t.trustedWithToken(req.URL, challenge) { return resp, nil } @@ -115,37 +116,46 @@ func (t *Transport) RoundTrip(req *http.Request) (*http.Response, error) { // from that registry, which can be any host the proxy fetches from, such as // one named in a Composer dist URL or a Helm index. Credentials configured for // the realm would otherwise be turned into a token handed to that host. A -// registry is trusted with such a token when it has configured credentials of -// its own, or is on the same site as the realm, as Docker Hub's -// registry-1.docker.io is with auth.docker.io. A token from a realm with no -// configured credentials is anonymous, and any registry may have it. +// token from a realm with no configured credentials is anonymous, and any +// registry may have it. A credentialed token only goes to the realm's own +// host, to a registry configured with the same credentials as the realm, or +// to a registry in knownTokenServices. func (t *Transport) trustedWithToken(requestURL *url.URL, challenge bearerChallenge) bool { if t.authForURL == nil { return true } - if name, value := t.authForURL(challenge.realm); name == "" || value == "" { + realmName, realmValue := t.authForURL(challenge.realm) + if realmName == "" || realmValue == "" { return true } - if name, value := t.authForURL(requestURL.String()); name != "" && value != "" { + if name, value := t.authForURL(requestURL.String()); name == realmName && value == realmValue { return true } realm, err := url.Parse(challenge.realm) - return err == nil && sameSite(requestURL, realm) + return err == nil && sameTokenService(requestURL, realm) } -// sameSite reports whether two URLs are on the same site: the same registrable -// domain (public suffix plus one label), or, for IP addresses and names -// without one, such as localhost, the same host and port. -func sameSite(a, b *url.URL) bool { - hostA, hostB := strings.ToLower(a.Hostname()), strings.ToLower(b.Hostname()) - if net.ParseIP(hostA) == nil && net.ParseIP(hostB) == nil { - siteA, errA := publicsuffix.EffectiveTLDPlusOne(hostA) - siteB, errB := publicsuffix.EffectiveTLDPlusOne(hostB) - if errA == nil && errB == nil { - return siteA == siteB - } +// knownTokenServices maps registry hosts to the token service host they +// use where the two differ and the registry cannot be configured with the +// token service's credentials under the same upstream.auth key. +var knownTokenServices = map[string]string{ + "registry-1.docker.io": "auth.docker.io", + "index.docker.io": "auth.docker.io", + "registry.gitlab.com": "gitlab.com", +} + +// sameTokenService reports whether a token service at realm belongs to the +// registry at registry: both on the same scheme, host and port, or a known +// pair of registry and token service hosts. +func sameTokenService(registry, realm *url.URL) bool { + if strings.EqualFold(registry.Scheme, realm.Scheme) && strings.EqualFold(registry.Host, realm.Host) { + return true } - return strings.EqualFold(a.Host, b.Host) + if registry.Scheme != "https" || realm.Scheme != "https" || registry.Port() != "" || realm.Port() != "" { + return false + } + want, ok := knownTokenServices[strings.ToLower(registry.Hostname())] + return ok && strings.EqualFold(realm.Hostname(), want) } func (t *Transport) applyAuthentication(req *http.Request, hasExplicitAuthorization bool) { @@ -379,6 +389,17 @@ func (c bearerChallenge) key() string { } func registryProtectionSpace(u *url.URL) string { + repository := registryRepository(u) + if repository == "" { + return "" + } + return u.Scheme + "://" + u.Host + "/v2/" + repository +} + +// registryRepository returns the repository name from an OCI distribution +// API path, such as library/alpine from /v2/library/alpine/manifests/latest, +// or "" if the path is not one. +func registryRepository(u *url.URL) string { const registryPrefix = "/v2/" if u == nil || !strings.HasPrefix(u.Path, registryPrefix) { return "" @@ -393,7 +414,7 @@ func registryProtectionSpace(u *url.URL) string { if end == len(rest) || end == 0 { return "" } - return u.Scheme + "://" + u.Host + registryPrefix + rest[:end] + return rest[:end] } func parseBearerChallenge(values []string) (bearerChallenge, bool) { From aa3adbfa63ced63cd2e416689b0a0c769aea7e5d Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sat, 24 Oct 2026 08:02:10 +0200 Subject: [PATCH 389/633] Bound the registry token and challenge caches The transport remembered a challenge for every /v2/ it was ever challenged on and never forgot one, so a long-running proxy, or a metadata document naming many registries, grew the map without limit. Tokens were only pruned of expired entries, and the expiry came from the token service. Challenges are now kept per registry host, holding only the realm and service, and the pull scope for a request is built from its path, so pulling many repositories from one registry keeps one entry. Both caches have a size limit, evicting an arbitrary challenge or the token that expires soonest, and a token is reused for at most an hour, whatever lifetime the token service claims. --- internal/httpclient/transport.go | 65 +++++++++++++++++++-------- internal/httpclient/transport_test.go | 49 ++++++++++++++++++++ 2 files changed, 96 insertions(+), 18 deletions(-) diff --git a/internal/httpclient/transport.go b/internal/httpclient/transport.go index ba78ee5..a007c8d 100644 --- a/internal/httpclient/transport.go +++ b/internal/httpclient/transport.go @@ -22,6 +22,15 @@ const ( shortTokenSkewDivisor = 10 tokenMaxRetries = 3 tokenRetryBaseDelay = 500 * time.Millisecond + + // maxTokenLifetime caps how long a token is reused, whatever lifetime + // the token service claims. + maxTokenLifetime = time.Hour + // maxCachedTokens and maxCachedChallenges bound the token and + // challenge caches, which grow with every repository and registry + // host the proxy pulls from. + maxCachedTokens = 1024 + maxCachedChallenges = 256 ) // AuthFunc returns a configured authentication header for a URL. @@ -35,7 +44,7 @@ type Transport struct { mu sync.Mutex tokens map[string]cachedToken - challenges map[string]bearerChallenge + challenges map[string]bearerChallenge // by registry origin, without scopes } type cachedToken struct { @@ -83,14 +92,15 @@ func (t *Transport) RoundTrip(req *http.Request) (*http.Response, error) { if hasExplicitAuthorization { return resp, nil } - if registryProtectionSpace(req.URL) == "" { + repository := registryRepository(req.URL) + if repository == "" { return resp, nil } challenge, ok := parseBearerChallenge(resp.Header.Values("WWW-Authenticate")) // Ask only for pull access to the repository being fetched, whatever // scope the challenge names: the proxy only reads. - challenge.scopes = []string{"repository:" + registryRepository(req.URL) + ":pull"} + challenge.scopes = []string{pullScope(repository)} if !ok || !canReplay(req) || !t.trustedWithToken(req.URL, challenge) { return resp, nil } @@ -203,6 +213,17 @@ func (t *Transport) cacheToken(key string, token cachedToken) { delete(t.tokens, cachedKey) } } + if _, ok := t.tokens[key]; !ok && len(t.tokens) >= maxCachedTokens { + // Drop the token that expires soonest. + var oldestKey string + var oldest time.Time + for cachedKey, cached := range t.tokens { + if oldestKey == "" || cached.expiresAt.Before(oldest) { + oldestKey, oldest = cachedKey, cached.expiresAt + } + } + delete(t.tokens, oldestKey) + } t.tokens[key] = token } @@ -282,6 +303,7 @@ func decodeTokenResponse(resp *http.Response) (string, time.Time, error) { if lifetime <= 0 { lifetime = defaultTokenLifetime } + lifetime = min(lifetime, maxTokenLifetime) expiresAt := issuedAt.Add(lifetime).Add(-expirySkew(lifetime)) return token, expiresAt, nil } @@ -344,17 +366,18 @@ func (t configuredTransport) RoundTrip(req *http.Request) (*http.Response, error } func (t *Transport) cachedTokenForRequest(requestURL *url.URL) string { - space := registryProtectionSpace(requestURL) - if space == "" { + repository := registryRepository(requestURL) + if repository == "" { return "" } t.mu.Lock() - challenge, ok := t.challenges[space] + challenge, ok := t.challenges[registryOrigin(requestURL)] t.mu.Unlock() if !ok { return "" } + challenge.scopes = []string{pullScope(repository)} return t.cachedToken(challenge.key()) } @@ -374,26 +397,32 @@ func (t *Transport) cachedToken(key string) string { return token.value } +// rememberChallenge records the token service a registry named, so later +// requests to any of its repositories can send a cached token up front. func (t *Transport) rememberChallenge(requestURL *url.URL, challenge bearerChallenge) { - space := registryProtectionSpace(requestURL) - if space == "" { - return - } + origin := registryOrigin(requestURL) + challenge.scopes = nil t.mu.Lock() - t.challenges[space] = challenge - t.mu.Unlock() + defer t.mu.Unlock() + if _, ok := t.challenges[origin]; !ok && len(t.challenges) >= maxCachedChallenges { + for evict := range t.challenges { + delete(t.challenges, evict) + break + } + } + t.challenges[origin] = challenge } func (c bearerChallenge) key() string { return c.realm + "\x00" + c.service + "\x00" + strings.Join(c.scopes, "\x00") } -func registryProtectionSpace(u *url.URL) string { - repository := registryRepository(u) - if repository == "" { - return "" - } - return u.Scheme + "://" + u.Host + "/v2/" + repository +func registryOrigin(u *url.URL) string { + return strings.ToLower(u.Scheme + "://" + u.Host) +} + +func pullScope(repository string) string { + return "repository:" + repository + ":pull" } // registryRepository returns the repository name from an OCI distribution diff --git a/internal/httpclient/transport_test.go b/internal/httpclient/transport_test.go index 01f3e8b..46a31b6 100644 --- a/internal/httpclient/transport_test.go +++ b/internal/httpclient/transport_test.go @@ -3,6 +3,7 @@ package httpclient import ( "context" "errors" + "fmt" "io" "net" "net/http" @@ -412,3 +413,51 @@ func TestTransportDoesNotFollowBearerChallengeOutsideOCIRegistry(t *testing.T) { t.Errorf("token requests = %d, want 0", tokenRequests) } } + +func TestTransportBoundsTokenCache(t *testing.T) { + transport := NewTransport(http.DefaultTransport, nil) + expiry := time.Now().Add(time.Hour) + for i := range maxCachedTokens + 10 { + transport.cacheToken(fmt.Sprintf("key-%d", i), cachedToken{ + value: "token", + expiresAt: expiry.Add(time.Duration(i) * time.Second), + }) + } + if got := len(transport.tokens); got != maxCachedTokens { + t.Errorf("cached tokens = %d, want %d", got, maxCachedTokens) + } + if got := transport.cachedToken(fmt.Sprintf("key-%d", maxCachedTokens+9)); got != "token" { + t.Error("the newest token was evicted") + } + if got := transport.cachedToken("key-0"); got != "" { + t.Error("the token expiring soonest was kept") + } +} + +func TestTransportBoundsChallengeCache(t *testing.T) { + transport := NewTransport(http.DefaultTransport, nil) + challenge := bearerChallenge{realm: "https://auth.example.test/token", service: "registry"} + for i := range 50 { + transport.rememberChallenge(mustParse(t, fmt.Sprintf("https://registry.example.test/v2/repo-%d/blobs/sha256:abc", i)), challenge) + } + if got := len(transport.challenges); got != 1 { + t.Errorf("challenges for one registry host = %d, want 1", got) + } + for i := range maxCachedChallenges + 10 { + transport.rememberChallenge(mustParse(t, fmt.Sprintf("https://registry-%d.example.test/v2/repo/blobs/sha256:abc", i)), challenge) + } + if got := len(transport.challenges); got != maxCachedChallenges { + t.Errorf("challenges = %d, want %d", got, maxCachedChallenges) + } +} + +func TestTransportCapsTokenLifetime(t *testing.T) { + resp := &http.Response{Body: io.NopCloser(strings.NewReader(`{"token":"t","expires_in":31536000}`))} + _, expiresAt, err := decodeTokenResponse(resp) + if err != nil { + t.Fatal(err) + } + if limit := time.Now().Add(maxTokenLifetime); expiresAt.After(limit) { + t.Errorf("token expires at %v, want no later than %v", expiresAt, limit) + } +} From 112e9c89ec0ef37ba98a771114060fbaa1a36350 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sat, 24 Oct 2026 08:02:22 +0200 Subject: [PATCH 390/633] Require generic_paths for generic upstreams with credentials A generic upstream forwards any path below /generic/{name}/. The documented mise setup puts a GITHUB_TOKEN on api.github.com to raise the rate limit, so anyone who could reach the proxy could make any GET request with that token, including for private repositories it can read, and the answers went into the shared metadata cache. upstream.git already refuses an upstream with credentials unless it lists its repos. upstream.generic_paths now lists, per generic upstream, the paths it may serve, with the same pattern syntax as git repos. Other paths answer 404 without reaching the upstream, and so does a path holding '%', '?', '#' or a backslash, which the upstream could read as a different path than the one that matched. Config validation refuses a generic upstream that upstream.auth has credentials for unless it has generic_paths, which also covers a github.com token reaching the generic github upstream. Upstreams without credentials keep serving every path. The docs and example config show the paths mise needs and recommend a fine-grained token limited to public repositories. --- README.md | 22 +++++- config.example.yaml | 12 +++ docs/configuration.md | 29 ++++++-- internal/config/config.go | 58 +++++++++++++++ internal/config/config_test.go | 123 +++++++++++++++++++++++++++++++ internal/handler/generic.go | 12 ++- internal/handler/generic_test.go | 44 +++++++++++ internal/server/server.go | 3 +- 8 files changed, 294 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 3421dff..70130a6 100644 --- a/README.md +++ b/README.md @@ -859,8 +859,26 @@ Release assets are cached permanently after the first download and keep installing while GitHub is down. Tag lookups through `api.github.com` are cached for `metadata_ttl` and served stale during an outage or rate limit. Commit a `mise.lock` and install with `mise install --locked` so pinned -installs need no API call at all. Add a bearer token for `https://api.github.com` -under `upstream.auth` if the fleet exceeds GitHub's anonymous rate limit. +installs need no API call at all. + +If the fleet exceeds GitHub's anonymous rate limit, add a bearer token for +`https://api.github.com` under `upstream.auth`. Everyone who can reach the +proxy can then make requests with that token, so the proxy requires +`generic_paths` listing the API paths it may serve, and the token should be a +fine-grained one with read-only access to public repositories: + +```yaml +upstream: + generic_paths: + github-api: + - "repos/*/*/releases" + - "repos/*/*/releases/**" + - "repos/*/*/tags" + auth: + "https://api.github.com": + type: bearer + token: "${GITHUB_TOKEN}" +``` Maven, mvnd, Node and Go downloads are cached the same way once their hosts are configured as generic upstreams; their version-pinned paths are built in diff --git a/config.example.yaml b/config.example.yaml index f785232..d3b6da4 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -332,6 +332,18 @@ upstream: # github-api: "https://api.github.com" # apache: "https://archive.apache.org" + # Paths a generic upstream may serve, as path.Match patterns for the path + # after /generic/{name}/; a pattern ending in /** also matches everything + # nested below its prefix. Other paths answer 404. Required for an upstream + # that upstream.auth has credentials for, such as a GitHub token for + # api.github.com; use a fine-grained token with read-only access to public + # repositories. + # generic_paths: + # github-api: + # - "repos/*/*/releases" + # - "repos/*/*/releases/**" + # - "repos/*/*/tags" + # Version-pinned paths of generic upstreams, cached immutably like GitHub # release assets. Each pattern must match the whole path after # /generic/{name}/ and have `version` and `file` named groups; an optional diff --git a/docs/configuration.md b/docs/configuration.md index fa77ec6..fc02e78 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -391,6 +391,13 @@ upstream: generic: github: "https://github.com" github-api: "https://api.github.com" + # Paths each upstream may serve. Required for an upstream that + # upstream.auth has credentials for. + generic_paths: + github-api: + - "repos/*/*/releases" + - "repos/*/*/releases/**" + - "repos/*/*/tags" auth: # Optional: raise the GitHub API rate limit. Scoped to this host only, # so the token is never sent to the object store GitHub redirects to. @@ -400,6 +407,19 @@ upstream: ``` Only configured upstreams are reachable, so this is not an open HTTP proxy. +`generic_paths` limits an upstream to the paths after `/generic/{name}/` that +its patterns match; any other path answers 404. The patterns use the syntax of +`upstream.git` `repos`: `path.Match`, where `*` does not cross a `/`, and a +pattern ending in `/**` also matches every path nested below its prefix. A +path containing `%`, `?`, `#` or a backslash is refused for an upstream with +`generic_paths`. An upstream without `generic_paths` serves every path. + +When `upstream.auth` has credentials for a generic upstream, `generic_paths` +is required and the proxy refuses to start without it: everyone who can reach +the proxy can send requests with those credentials, so list only the paths +they need. For GitHub, use a fine-grained personal access token limited to +public repositories with read-only access; it raises the rate limit like any +other token but cannot read anything private, even through an allowed path. Paths shaped like `{owner}/{repo}/releases/download/{tag}/{asset}` are version-pinned GitHub release assets: they are stored in the artifact cache and served from it without revalidation, including while the upstream is @@ -711,11 +731,10 @@ without `repos` serves every repository on it, and a repository outside `repos` is required: everyone who can reach the proxy can read every repository it serves, so list only those they may read. Per-client permissions are not supported. `upstream.auth` matches credentials by URL, so -they are sent by every route to that host, not only `/git`. With a token for -`https://github.com`, a `generic` upstream at the same URL, such as the one -the mise setup in the README uses, sends it for any path and checks no -`repos`, so `/generic/github/acme/private.git/info/refs?service=git-upload-pack` -would list a private repository's refs to anyone. +they are sent by every route to that host, not only `/git`. A `generic` +upstream at the same URL, such as the one the mise setup in the README uses, +checks `generic_paths` instead of `repos`, and needs them too; keep Git paths +like `{owner}/{repo}.git/info/refs` out of its patterns. Every request for a repository's refs checks the upstream's branches and tags, and fetches only when they changed, so a clone right after a push sees the new diff --git a/internal/config/config.go b/internal/config/config.go index d790416..a660699 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -785,6 +785,16 @@ type UpstreamConfig struct { // Example: {"apache": ["dist/(?Pmaven)/maven-3/(?P[^/]+)/binaries/(?P[^/]+)"]}. GenericArtifacts map[string][]string `json:"generic_artifacts" yaml:"generic_artifacts"` + // GenericPaths maps a Generic upstream name to the paths it may serve, + // as patterns for the request path after /generic/{name}/ with the same + // syntax as GitUpstream.Repos: path.Match, where a pattern ending in + // "/**" also matches every path nested below its prefix. Other paths + // answer 404. Empty serves every path. Required when upstream.auth has + // credentials for the upstream, since everyone who can reach the proxy + // can use them for any path otherwise. + // Example: {"github-api": ["repos/*/*/releases/**"]}. + GenericPaths map[string][]string `json:"generic_paths" yaml:"generic_paths"` + // Git maps names to Git servers, served read-only at /git/{name}/{repo} // from a local cache of bare repositories. See GitUpstream. // Example: {"github": {"url": "https://github.com"}}. @@ -863,6 +873,9 @@ func (u *UpstreamConfig) Validate() error { if err := u.validateGenericArtifacts(); err != nil { return err } + if err := u.validateGenericPaths(); err != nil { + return err + } if err := u.validateGitUpstreams(); err != nil { return err } @@ -902,6 +915,51 @@ func (u *UpstreamConfig) validateGenericArtifacts() error { return nil } +// validateGenericPaths checks upstream.generic_paths patterns. An upstream that +// upstream.auth has credentials for must list its paths, so configuring a +// token does not let everyone who can reach the proxy send any request with +// it, mirroring the repos rule for upstream.git. +func (u *UpstreamConfig) validateGenericPaths() error { + for name, patterns := range u.GenericPaths { + if _, found := u.Generic[name]; !found { + return fmt.Errorf("invalid upstream.generic_paths name %q: no upstream.generic entry", name) + } + for _, pattern := range patterns { + if _, err := path.Match(strings.TrimSuffix(pattern, "/**"), ""); pattern == "" || err != nil { + return fmt.Errorf("invalid upstream.generic_paths.%s pattern %q", name, pattern) + } + } + } + for name, upstreamURL := range u.Generic { + if len(u.GenericPaths[name]) == 0 && u.HasAuthWithin(upstreamURL) { + return fmt.Errorf("upstream.generic.%s needs upstream.generic_paths.%s: upstream.auth has credentials for %s, "+ + "and everyone who can reach the proxy could send any request with them", name, name, upstreamURL) + } + } + return nil +} + +// GenericPathAllowed reports whether p, a request path after +// /generic/{name}/, matches one of an upstream's upstream.generic_paths +// patterns. No patterns allow every path. With patterns, a path holding a +// character the upstream could read as the end of the path or as a different +// path ('%', '?', '#' or a backslash) is refused, so the path that matched is +// the one the upstream sees. +func GenericPathAllowed(patterns []string, p string) bool { + if len(patterns) == 0 { + return true + } + if strings.ContainsAny(p, `%?#\`) { + return false + } + for _, pattern := range patterns { + if matchRepoPattern(pattern, p) { + return true + } + } + return false +} + // validateUpstream checks the upstream block, including the one setting in // it that depends on a top-level option. func (c *Config) validateUpstream() error { diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 41f414c..7590a64 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -1929,3 +1929,126 @@ func TestValidateFreeBSDUpstreams(t *testing.T) { }) } } + +func TestValidateGenericPaths(t *testing.T) { + githubAPI := map[string]string{"github-api": "https://api.github.com"} + token := map[string]AuthConfig{"https://api.github.com": {Type: "bearer", Token: "t"}} + tests := []struct { + name string + upstream UpstreamConfig + wantErr bool + }{ + {name: "public", upstream: UpstreamConfig{Generic: githubAPI}}, + {name: "credentials without paths", upstream: UpstreamConfig{Generic: githubAPI, Auth: token}, wantErr: true}, + { + name: "credentials for a path under the upstream", + upstream: UpstreamConfig{ + Generic: githubAPI, + Auth: map[string]AuthConfig{"https://api.github.com/repos/acme": {Type: "bearer", Token: "t"}}, + }, + wantErr: true, + }, + { + name: "credentials with paths", + upstream: UpstreamConfig{ + Generic: githubAPI, + GenericPaths: map[string][]string{"github-api": {"repos/*/*/releases/**"}}, + Auth: token, + }, + }, + { + name: "credentials for another host", + upstream: UpstreamConfig{ + Generic: githubAPI, + Auth: map[string]AuthConfig{"https://npm.example": {Type: "bearer", Token: "t"}}, + }, + }, + { + name: "paths for an unknown upstream", + upstream: UpstreamConfig{ + Generic: githubAPI, + GenericPaths: map[string][]string{"github": {"*/*/releases/**"}}, + }, + wantErr: true, + }, + { + name: "bad pattern", + upstream: UpstreamConfig{ + Generic: githubAPI, + GenericPaths: map[string][]string{"github-api": {"["}}, + }, + wantErr: true, + }, + { + name: "empty pattern", + upstream: UpstreamConfig{ + Generic: githubAPI, + GenericPaths: map[string][]string{"github-api": {""}}, + }, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if err := tt.upstream.Validate(); (err != nil) != tt.wantErr { + t.Errorf("Validate() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +func TestGenericPathAllowed(t *testing.T) { + patterns := []string{"repos/*/*/releases/**", "rate_limit"} + tests := []struct { + path string + want bool + }{ + {"repos/jqlang/jq/releases/tags/jq-1.7.1", true}, + {"repos/jqlang/jq/releases/latest", true}, + {"rate_limit", true}, + {"repos/jqlang/jq/releases", false}, + {"repos/acme/private/contents/secret.txt", false}, + {"user/repos", false}, + {"repos/jqlang/jq/releases/%2e%2e", false}, + {"repos/jqlang/jq?/releases/x", false}, + {"repos/jqlang/jq/releases/x#y", false}, + {`repos/jqlang/jq/releases/x\y`, false}, + } + for _, tt := range tests { + if got := GenericPathAllowed(patterns, tt.path); got != tt.want { + t.Errorf("GenericPathAllowed(%q) = %v, want %v", tt.path, got, tt.want) + } + } + if !GenericPathAllowed(nil, "user/repos") { + t.Error("GenericPathAllowed without patterns refused a path") + } +} + +func TestLoadGenericPaths(t *testing.T) { + path := filepath.Join(t.TempDir(), "config.yaml") + data := []byte(` +upstream: + generic: + github-api: https://api.github.com + generic_paths: + github-api: + - "repos/*/*/releases/**" + auth: + "https://api.github.com": + type: bearer + token: t +`) + if err := os.WriteFile(path, data, 0o600); err != nil { + t.Fatal(err) + } + cfg, err := Load(path) + if err != nil { + t.Fatalf("Load: %v", err) + } + if err := cfg.Validate(); err != nil { + t.Fatalf("Validate: %v", err) + } + if got := cfg.Upstream.GenericPaths["github-api"]; len(got) != 1 || got[0] != "repos/*/*/releases/**" { + t.Errorf("generic_paths = %v", cfg.Upstream.GenericPaths) + } +} diff --git a/internal/handler/generic.go b/internal/handler/generic.go index 700e53e..0e08140 100644 --- a/internal/handler/generic.go +++ b/internal/handler/generic.go @@ -71,6 +71,9 @@ type GenericHandler struct { proxy *Proxy repositories map[string]string artifactPatterns map[string][]*regexp.Regexp + // paths holds each upstream's upstream.generic_paths patterns; an + // upstream without any serves every path. + paths map[string][]string } // NewGenericHandler creates a generic HTTP download proxy handler. @@ -97,6 +100,13 @@ func NewGenericHandler(proxy *Proxy, repositories map[string]string, artifactPat return h } +// WithPaths limits upstreams to the paths their upstream.generic_paths +// patterns match. Any other path answers 404. +func (h *GenericHandler) WithPaths(paths map[string][]string) *GenericHandler { + h.paths = paths + return h +} + func (h *GenericHandler) addArtifactPatterns(name string, patterns []string) { for _, pattern := range patterns { re, err := config.CompileGenericArtifactPattern(pattern) @@ -131,7 +141,7 @@ func (h *GenericHandler) Routes() http.Handler { return } upstreamURL, found := h.repositories[repository] - if !ok || rest == "" || !found { + if !ok || rest == "" || !found || !config.GenericPathAllowed(h.paths[repository], rest) { http.NotFound(w, r) return } diff --git a/internal/handler/generic_test.go b/internal/handler/generic_test.go index 424eafb..a74be8c 100644 --- a/internal/handler/generic_test.go +++ b/internal/handler/generic_test.go @@ -646,3 +646,47 @@ func TestGenericHandler_UpstreamNamedHostedWhileHostingIsOff(t *testing.T) { t.Errorf("status = %d, body %q, want the upstream response", w.Code, w.Body) } } + +func TestGenericHandler_ServesOnlyConfiguredPaths(t *testing.T) { + var requested []string + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requested = append(requested, r.URL.RequestURI()) + _, _ = io.WriteString(w, "from upstream "+r.URL.Path) + })) + t.Cleanup(upstream.Close) + proxy, _, _, _ := setupTestProxy(t) + proxy.HTTPClient = upstream.Client() + fetcher := fetch.NewFetcher(fetch.WithHTTPClient(upstream.Client()), fetch.WithMaxRetries(0)) + proxy.Fetcher = fetcher + t.Cleanup(func() { _ = fetcher.Close() }) + + h := NewGenericHandler(proxy, map[string]string{"github-api": upstream.URL, "open": upstream.URL}). + WithPaths(map[string][]string{"github-api": {"repos/*/*/releases/**", "rate_limit"}}) + + for _, target := range []string{ + "/github-api/repos/jqlang/jq/releases/tags/jq-1.7.1", + "/github-api/rate_limit", + "/open/user/repos", + } { + if w := serveGenericRequest(h, target); w.Code != http.StatusOK { + t.Errorf("GET %s: status = %d, want 200: %s", target, w.Code, w.Body.String()) + } + } + + served := len(requested) + for _, target := range []string{ + "/github-api/user/repos", + "/github-api/repos/acme/private/contents/secret.txt", + "/github-api/repos/jqlang/jq/releases", + "/github-api/repos/jqlang/jq%3F/releases/x", + "/github-api/repos/jqlang/jq/releases/%2575ser", + "/github-api/jqlang/jq/releases/download/jq-1.7.1/jq-linux-amd64", + } { + if w := serveGenericRequest(h, target); w.Code != http.StatusNotFound { + t.Errorf("GET %s: status = %d, want 404", target, w.Code) + } + } + if len(requested) != served { + t.Errorf("upstream saw %v for paths outside generic_paths", requested[served:]) + } +} diff --git a/internal/server/server.go b/internal/server/server.go index 5b0ebf7..d64e5cf 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -493,7 +493,8 @@ func (s *Server) mountProtocolHandlers(r chi.Router, proxy *handler.Proxy) { s.cfg.Upstream.DebianRepositories, ) rpmHandler := handler.NewRPMHandlerWithUpstream(proxy, s.cfg.BaseURL, s.cfg.Upstream.RPM) - genericHandler := handler.NewGenericHandler(proxy, s.cfg.Upstream.Generic, s.cfg.Upstream.GenericArtifacts) + genericHandler := handler.NewGenericHandler(proxy, s.cfg.Upstream.Generic, s.cfg.Upstream.GenericArtifacts). + WithPaths(s.cfg.Upstream.GenericPaths) r = r.With(sandboxUpstreamContent, rejectQueryInPath, rejectDotSegments) r.Mount("/npm", http.StripPrefix("/npm", npmHandler.Routes())) From 2ea8915e06d019eba2e8ca09562758ce301d9fff Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sat, 24 Oct 2026 08:03:16 +0200 Subject: [PATCH 391/633] Filter gzipped gem compact indexes under cooldown With cooldown on, /info/{name} is fetched from upstream and versions still in cooldown are removed line by line. The fetch passed on the client's Accept-Encoding, and setting that header turns off the transport's transparent decompression, so for any client that accepts gzip, Bundler included, the filter scanned gzip bytes, matched no versions and sent the compressed index on with upstream's Content-Encoding. Cooldown filtered nothing for those clients. The fetch now passes on only Accept, so the transport decompresses the index before it is filtered. If-None-Match and If-Modified-Since are no longer passed on either: the client's validators came from a filtered copy, not from upstream's file. The filtered index no longer carries upstream's ETag, Last-Modified, Digest, Repr-Digest or range headers, which describe the unfiltered file; Bundler checks the digest against the body it gets. The index is now read through ReadMetadata before anything is written. bufio.Scanner stopped at a line over 64 KB and the handler sent what it had so far as a complete 200; a read error or an index over the metadata size limit now answers 502. --- internal/handler/gem.go | 82 +++++++++++++++---------- internal/handler/gem_test.go | 112 +++++++++++++++++++++++++++++++++++ 2 files changed, 163 insertions(+), 31 deletions(-) diff --git a/internal/handler/gem.go b/internal/handler/gem.go index e1acb12..1e76b6b 100644 --- a/internal/handler/gem.go +++ b/internal/handler/gem.go @@ -1,11 +1,13 @@ package handler import ( - "bufio" + "bytes" "encoding/json" "fmt" "net/http" "regexp" + "slices" + "strconv" "strings" "time" @@ -208,48 +210,66 @@ func (h *GemHandler) fetchCompactIndex(r *http.Request, name string) (*http.Resp if err != nil { return nil, err } - for _, hdr := range []string{"Accept", headerAcceptEncoding, "If-None-Match", "If-Modified-Since"} { - if v := r.Header.Get(hdr); v != "" { - req.Header.Set(hdr, v) - } + // Only Accept is passed on. The client's Accept-Encoding would turn off + // the transport's transparent decompression and leave a gzip body to + // filter, and its validators belong to the filtered copy it was served, + // not to upstream's file. + if v := r.Header.Get(headerAccept); v != "" { + req.Header.Set(headerAccept, v) } return h.proxy.HTTPClient.Do(req) } -// writeFilteredIndex writes the compact index response with cooldown-filtered versions removed. +// gemFilteredIndexDroppedHeaders describe upstream's file as a whole, so +// they are wrong for a copy with versions removed. Bundler checks the digest +// headers against the body it receives. +var gemFilteredIndexDroppedHeaders = []string{ + headerContentLength, headerContentEncoding, headerETag, headerLastModified, + "Digest", "Repr-Digest", "Content-Digest", "Accept-Ranges", "Content-Range", +} + +// writeFilteredIndex writes the compact index response with cooldown-filtered +// versions removed. The body is read in full first, so a read error or an +// oversized index answers 502 instead of a truncated 200. func (h *GemHandler) writeFilteredIndex(w http.ResponseWriter, resp *http.Response, name string, filtered map[string]bool) { + body, err := h.proxy.ReadMetadata(resp.Body) + if err != nil { + h.proxy.Logger.Error("failed to read gem compact index", "gem", name, "error", err) + http.Error(w, "failed to read upstream response", http.StatusBadGateway) + return + } + + var out bytes.Buffer + out.Grow(len(body)) + for rest := body; len(rest) > 0; { + line, tail, _ := bytes.Cut(rest, []byte("\n")) + rest = tail + + version := line + if spaceIdx := bytes.IndexByte(line, ' '); spaceIdx > 0 { + version = line[:spaceIdx] + } + if filtered[string(version)] { + h.proxy.Logger.Info("cooldown: filtering gem version", + "gem", name, "version", string(version)) + continue + } + + out.Write(line) + out.WriteByte('\n') + } + for k, vv := range resp.Header { - if strings.EqualFold(k, headerContentLength) { - continue // length will change after filtering + if slices.ContainsFunc(gemFilteredIndexDroppedHeaders, func(d string) bool { return strings.EqualFold(k, d) }) { + continue } for _, v := range vv { w.Header().Add(k, v) } } + w.Header().Set(headerContentLength, strconv.Itoa(out.Len())) w.WriteHeader(http.StatusOK) - - scanner := bufio.NewScanner(resp.Body) - for scanner.Scan() { - line := scanner.Text() - - if line == "---" { - _, _ = fmt.Fprintln(w, line) - continue - } - - version := line - if spaceIdx := strings.IndexByte(line, ' '); spaceIdx > 0 { - version = line[:spaceIdx] - } - - if filtered[version] { - h.proxy.Logger.Info("cooldown: filtering gem version", - "gem", name, "version", version) - continue - } - - _, _ = fmt.Fprintln(w, line) - } + _, _ = w.Write(out.Bytes()) } // gemVersion represents a version entry from the RubyGems versions API. diff --git a/internal/handler/gem_test.go b/internal/handler/gem_test.go index 9b71074..5491340 100644 --- a/internal/handler/gem_test.go +++ b/internal/handler/gem_test.go @@ -1,6 +1,7 @@ package handler import ( + "compress/gzip" "encoding/json" "fmt" "log/slog" @@ -107,6 +108,117 @@ func TestGemCompactIndexCooldown(t *testing.T) { } } +// gemCooldownUpstream serves index as /info/testgem, gzipped when the +// request accepts gzip as rubygems.org does, with 1.0.0 published a week ago +// and 2.0.0 an hour ago. +func gemCooldownUpstream(t *testing.T, index string, headers http.Header) *httptest.Server { + t.Helper() + now := time.Now() + versionsJSON, _ := json.Marshal([]gemVersion{ + {Number: "1.0.0", Platform: "ruby", CreatedAt: now.Add(-7 * 24 * time.Hour).Format(time.RFC3339)}, + {Number: "2.0.0", Platform: "ruby", CreatedAt: now.Add(-time.Hour).Format(time.RFC3339)}, + }) + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/info/testgem": + for k, vv := range headers { + w.Header()[k] = vv + } + w.Header().Set("Content-Type", "text/plain; charset=utf-8") + if !strings.Contains(r.Header.Get("Accept-Encoding"), "gzip") { + _, _ = w.Write([]byte(index)) + return + } + w.Header().Set("Content-Encoding", "gzip") + gz := gzip.NewWriter(w) + _, _ = gz.Write([]byte(index)) + _ = gz.Close() + case "/api/v1/versions/testgem.json": + _, _ = w.Write(versionsJSON) + default: + w.WriteHeader(http.StatusNotFound) + } + })) + t.Cleanup(upstream.Close) + return upstream +} + +func TestGemCompactIndexCooldownClientAcceptsGzip(t *testing.T) { + index := "---\n1.0.0 dep1:>= 1.0|checksum:abc123\n2.0.0 dep1:>= 1.0|checksum:def456\n" + upstream := gemCooldownUpstream(t, index, http.Header{ + "Etag": {`"upstream"`}, + "Repr-Digest": {"sha-256=:upstream:"}, + }) + + proxy := testProxy() + proxy.Cooldown = &cooldown.Config{Default: "3d"} + h := &GemHandler{proxy: proxy, upstreamURL: upstream.URL} + + // Bundler sends Accept-Encoding: gzip by default. + req := httptest.NewRequest(http.MethodGet, "/info/testgem", nil) + req.Header.Set("Accept-Encoding", "gzip;q=1.0,deflate;q=0.6,identity;q=0.3") + req.SetPathValue("name", "testgem") + w := httptest.NewRecorder() + h.handleCompactIndex(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want %d", w.Code, http.StatusOK) + } + if ce := w.Header().Get("Content-Encoding"); ce != "" { + t.Errorf("Content-Encoding = %q, want none", ce) + } + want := "---\n1.0.0 dep1:>= 1.0|checksum:abc123\n" + if got := w.Body.String(); got != want { + t.Errorf("body = %q, want %q", got, want) + } + for _, hdr := range []string{"ETag", "Repr-Digest"} { + if v := w.Header().Get(hdr); v != "" { + t.Errorf("%s = %q, upstream's value must not describe the filtered index", hdr, v) + } + } +} + +func TestGemCompactIndexCooldownLongLine(t *testing.T) { + // A version line longer than bufio.Scanner's 64 KB default used to end + // the response early with a 200. + long := "1.0.0 " + strings.Repeat("dep:>= 1.0,", 10_000) + "|checksum:abc" + index := "---\n" + long + "\n2.0.0 |checksum:def\n3.0.0 |checksum:ghi\n" + upstream := gemCooldownUpstream(t, index, nil) + + proxy := testProxy() + proxy.Cooldown = &cooldown.Config{Default: "3d"} + h := &GemHandler{proxy: proxy, upstreamURL: upstream.URL} + + req := httptest.NewRequest(http.MethodGet, "/info/testgem", nil) + req.SetPathValue("name", "testgem") + w := httptest.NewRecorder() + h.handleCompactIndex(w, req) + + want := "---\n" + long + "\n3.0.0 |checksum:ghi\n" + if w.Code != http.StatusOK || w.Body.String() != want { + t.Fatalf("status = %d, body of %d bytes; want 200 with %d bytes", w.Code, w.Body.Len(), len(want)) + } +} + +func TestGemCompactIndexCooldownOversizedIndex(t *testing.T) { + index := "---\n1.0.0 |checksum:abc\n" + strings.Repeat("9.9.9 |checksum:zzz\n", 100) + upstream := gemCooldownUpstream(t, index, nil) + + proxy := testProxy() + proxy.Cooldown = &cooldown.Config{Default: "3d"} + proxy.MetadataMaxSize = 256 + h := &GemHandler{proxy: proxy, upstreamURL: upstream.URL} + + req := httptest.NewRequest(http.MethodGet, "/info/testgem", nil) + req.SetPathValue("name", "testgem") + w := httptest.NewRecorder() + h.handleCompactIndex(w, req) + + if w.Code != http.StatusBadGateway { + t.Fatalf("status = %d, want %d", w.Code, http.StatusBadGateway) + } +} + func TestGemCompactIndexCooldownWithPlatformVersion(t *testing.T) { now := time.Now() recentTime := now.Add(-1 * time.Hour).Format(time.RFC3339) From b6d7575b88960355de4c6e34a6197336025f1364 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sat, 24 Oct 2026 08:06:10 +0200 Subject: [PATCH 392/633] Don't fetch npm tarballs whose packument can't be read An uncached tarball is checked against the integrity its packument declares. When the packument couldn't be fetched or was refused, for example because it is larger than metadata_max_size or the registry answered 503, the download went ahead with no integrity at all, and the unverified tarball was cached and served to every later client. Without the packument the download now fails like the Composer one does: 502, or 404 when the registry says the package doesn't exist. Nothing is fetched or cached. Tarballs already in the cache are still served, since they were verified when they were stored, and a packument that lists the version without a usable checksum still falls back to an unverified fetch as before. --- internal/handler/npm.go | 23 ++++---- internal/handler/npm_routes_test.go | 11 +++- internal/handler/npm_tarball_test.go | 67 ++++++++++++++++++++--- internal/handler/npm_test.go | 4 ++ internal/handler/stream_artifacts_test.go | 2 +- 5 files changed, 84 insertions(+), 23 deletions(-) diff --git a/internal/handler/npm.go b/internal/handler/npm.go index 0e2d8f2..ba1e2b1 100644 --- a/internal/handler/npm.go +++ b/internal/handler/npm.go @@ -955,18 +955,21 @@ func (h *NPMHandler) getTarball(r *http.Request, registryURL, packageName, versi if cached, err := h.proxy.GetCachedArtifact(r.Context(), "npm", packageName, version, cacheFilename); err != nil || cached != nil { return cached, err } - downloadURL := fmt.Sprintf("%s/%s/-/%s", registryURL, escapeNPMDownloadPackage(packageName), url.PathEscape(filename)) - var sri string + // The packument says where the tarball is and what it must hash to. + // Without it the tarball could not be verified, and an unverified copy + // would be cached and served to everyone, so the download fails instead. body, err := metadata() - if err == nil { - if tarball := npmVersionTarball(body, version); tarball != "" { - downloadURL, err = validateTarballURL(tarball, registryURL) - if err != nil { - return nil, err - } - } - sri = npmVersionIntegrity(body, version) + if err != nil { + return nil, fmt.Errorf("fetching npm metadata to verify tarball: %w", err) } + downloadURL := fmt.Sprintf("%s/%s/-/%s", registryURL, escapeNPMDownloadPackage(packageName), url.PathEscape(filename)) + if tarball := npmVersionTarball(body, version); tarball != "" { + downloadURL, err = validateTarballURL(tarball, registryURL) + if err != nil { + return nil, err + } + } + sri := npmVersionIntegrity(body, version) return h.proxy.GetOrFetchArtifactFromURLWithIntegrity(r.Context(), "npm", packageName, version, cacheFilename, downloadURL, sri) } diff --git a/internal/handler/npm_routes_test.go b/internal/handler/npm_routes_test.go index a2ffb38..56fc952 100644 --- a/internal/handler/npm_routes_test.go +++ b/internal/handler/npm_routes_test.go @@ -190,10 +190,15 @@ func TestNPMRoutedDownloadWithoutMetadataStaysOnRoute(t *testing.T) { } h := newRoutedNPMHandler(proxy, public, private) - serveNPM(t, h, http.MethodGet, "/@example/widgets/-/widgets-1.0.0.tgz", nil, nil) + w := serveNPM(t, h, http.MethodGet, "/@example/widgets/-/widgets-1.0.0.tgz", nil, nil) - if !strings.HasPrefix(artifactFetcher.fetchedURL, private.URL+"/registry/") { - t.Errorf("fetched URL = %q, want one on the route %q", artifactFetcher.fetchedURL, private.URL+"/registry/") + // Without the route's metadata the tarball can't be verified, so it is + // not fetched at all, and in particular not from the default registry. + if w.Code != http.StatusBadGateway { + t.Errorf("status = %d, want 502", w.Code) + } + if artifactFetcher.fetchedURL != "" { + t.Errorf("fetched URL = %q, want none", artifactFetcher.fetchedURL) } } diff --git a/internal/handler/npm_tarball_test.go b/internal/handler/npm_tarball_test.go index 303f218..fe1508d 100644 --- a/internal/handler/npm_tarball_test.go +++ b/internal/handler/npm_tarball_test.go @@ -92,12 +92,9 @@ func npmContentAddressedRegistry(t *testing.T, metadataCalls, tarballCalls *atom } func TestNPMTarballMetadataFallback(t *testing.T) { - for _, body := range []string{`{"versions":{}}`, `{"versions":{"1.0.0":{"dist":{}}}}`, `not json`, "unavailable"} { + for _, body := range []string{`{"versions":{}}`, `{"versions":{"1.0.0":{"dist":{}}}}`} { t.Run(body, func(t *testing.T) { upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - if body == "unavailable" { - w.WriteHeader(http.StatusServiceUnavailable) - } _, _ = io.WriteString(w, body) })) t.Cleanup(upstream.Close) @@ -106,12 +103,10 @@ func TestNPMTarballMetadataFallback(t *testing.T) { p.MetadataTTL = time.Hour fetcher.artifact = &fetch.Artifact{Body: io.NopCloser(strings.NewReader("package"))} h := NewNPMHandler(p, "http://proxy.test", upstream.URL) - if body != "unavailable" { - if _, _, err := p.FetchOrCacheMetadata(t.Context(), "npm", "@example/widget", upstream.URL); err != nil { - t.Fatal(err) - } - upstream.Close() + if _, _, err := p.FetchOrCacheMetadata(t.Context(), "npm", "@example/widget", upstream.URL); err != nil { + t.Fatal(err) } + upstream.Close() response := httptest.NewRecorder() h.Routes().ServeHTTP(response, httptest.NewRequest(http.MethodGet, "/@example/widget/-/widget-1.0.0.tgz", nil)) if response.Code != http.StatusOK { @@ -124,6 +119,60 @@ func TestNPMTarballMetadataFallback(t *testing.T) { } } +// TestNPMTarballWithoutMetadataIsNotFetched checks that a tarball whose +// packument can't be read, so its integrity is unknown, is neither fetched +// nor cached. +func TestNPMTarballWithoutMetadataIsNotFetched(t *testing.T) { + for _, tc := range []struct { + name string + status int + body string + limit int64 + }{ + {name: "unavailable", status: http.StatusServiceUnavailable, body: "unavailable"}, + {name: "over metadata_max_size", status: http.StatusOK, body: `{"versions":{"1.0.0":{"dist":{"integrity":"` + sha512SRI("package") + `"}}}}`, limit: 16}, + } { + t.Run(tc.name, func(t *testing.T) { + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(tc.status) + _, _ = io.WriteString(w, tc.body) + })) + t.Cleanup(upstream.Close) + p, _, _, fetcher := setupTestProxy(t) + if tc.limit > 0 { + p.MetadataMaxSize = tc.limit + } + fetcher.artifact = &fetch.Artifact{Body: io.NopCloser(strings.NewReader("tampered"))} + h := NewNPMHandler(p, "http://proxy.test", upstream.URL) + + response := httptest.NewRecorder() + h.Routes().ServeHTTP(response, httptest.NewRequest(http.MethodGet, "/@example/widget/-/widget-1.0.0.tgz", nil)) + if response.Code != http.StatusBadGateway { + t.Fatalf("status = %d, want 502: %s", response.Code, response.Body.String()) + } + if fetcher.fetchedURL != "" { + t.Errorf("fetched %q without the metadata to verify it", fetcher.fetchedURL) + } + cached, err := p.GetCachedArtifact(t.Context(), "npm", "@example/widget", "1.0.0", "widget-1.0.0.tgz") + if err != nil { + t.Fatal(err) + } + if cached != nil { + _ = cached.Reader.Close() + t.Error("tarball was cached without verification") + } + }) + } +} + +// serveEmptyPackuments answers every metadata request with a packument that +// lists no versions, for tests about the tarball download itself. +func serveEmptyPackuments(p *Proxy) { + p.HTTPClient = &http.Client{Transport: pypiRoundTripFunc(func(r *http.Request) (*http.Response, error) { + return pypiHTTPResponse(r, "application/json", `{"versions":{}}`), nil + })} +} + func TestNPMTarballRejectsUnsafeMetadataURL(t *testing.T) { for _, target := range []string{ "https://outside.invalid/package.tgz", "//outside.invalid/package.tgz", "/registry/package.tgz", diff --git a/internal/handler/npm_test.go b/internal/handler/npm_test.go index 23e223f..bbec4c6 100644 --- a/internal/handler/npm_test.go +++ b/internal/handler/npm_test.go @@ -96,6 +96,7 @@ func TestNPMHandlerUsesConfiguredUpstream(t *testing.T) { Body: io.NopCloser(strings.NewReader("package")), ContentType: "application/gzip", } + serveEmptyPackuments(proxy) h := NewNPMHandler(proxy, "http://proxy.test", "https://npm.example.test/root/") req := httptest.NewRequest(http.MethodGet, "/testpkg/-/testpkg-1.0.0.tgz", nil) @@ -117,6 +118,7 @@ func TestNPMHandlerUsesConfiguredUpstream(t *testing.T) { Body: io.NopCloser(strings.NewReader("package")), ContentType: "application/gzip", } + serveEmptyPackuments(proxy) h := NewNPMHandler(proxy, "http://proxy.test", "https://npm.example.test/root/") req := httptest.NewRequest(http.MethodGet, "/@scope/name/-/name-1.0.0.tgz", nil) @@ -693,6 +695,7 @@ func TestNPMDownloadErrorResponsesAreJSON(t *testing.T) { ContentType: "application/octet-stream", } + serveEmptyPackuments(proxy) h := NewNPMHandler(proxy, "http://proxy.test", "http://upstream.invalid") srv := httptest.NewServer(h.Routes()) defer srv.Close() @@ -1053,6 +1056,7 @@ func TestNPMTarballStillRoutesToDownload(t *testing.T) { Body: io.NopCloser(strings.NewReader("package")), ContentType: "application/gzip", } + serveEmptyPackuments(proxy) h := NewNPMHandler(proxy, "http://proxy.test", "https://npm.example.test") w := serveNPM(t, h, http.MethodGet, "/lodash/-/lodash-4.17.21.tgz", nil, nil) diff --git a/internal/handler/stream_artifacts_test.go b/internal/handler/stream_artifacts_test.go index 3a5929b..af0e3f1 100644 --- a/internal/handler/stream_artifacts_test.go +++ b/internal/handler/stream_artifacts_test.go @@ -317,7 +317,7 @@ func TestStreamArtifactsNPMTarballWithoutFinalChunkIsIncomplete(t *testing.T) { upstream := truncatingUpstream(t, func(r *http.Request) bool { return strings.HasSuffix(r.URL.Path, ".tgz") }, "HTTP/1.1 200 OK\r\nContent-Type: application/octet-stream\r\nTransfer-Encoding: chunked\r\n\r\n5\r\nshort\r\n", - http.NotFound) + func(w http.ResponseWriter, _ *http.Request) { _, _ = io.WriteString(w, `{"versions":{}}`) }) proxy, _, store, _ := setupTestProxy(t) proxy.StreamArtifacts = true From 9500c5b5a44f7be47274da1f20a09a35739d8932 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sat, 24 Oct 2026 08:07:13 +0200 Subject: [PATCH 393/633] Check upstream destinations sent through a forward proxy The upstream client refuses loopback, private and link-local addresses when it dials, but with HTTP_PROXY or HTTPS_PROXY set it dials the forward proxy, so only the forward proxy's address was checked. A URL taken from upstream metadata, such as a Helm chart URL, a Composer dist URL, a token realm or a redirect, could then reach 169.254.169.254 or an internal host through the forward proxy. The transport's proxy function now resolves the destination of every request it sends to a forward proxy and applies the same rules, including upstream.allow_loopback and upstream.allow_private_hosts, before handing it over. A name that cannot be resolved locally is still passed on, since some networks only resolve public names at the forward proxy. Direct requests are unchanged. Git fetches and token requests use the same transport, so they are covered too. --- docs/configuration.md | 2 + internal/server/forward_proxy.go | 61 +++++++++++++++ internal/server/forward_proxy_test.go | 105 ++++++++++++++++++++++++++ internal/server/server.go | 9 ++- 4 files changed, 174 insertions(+), 3 deletions(-) create mode 100644 internal/server/forward_proxy.go create mode 100644 internal/server/forward_proxy_test.go diff --git a/docs/configuration.md b/docs/configuration.md index fa77ec6..bc97ae9 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -293,6 +293,8 @@ Each upstream used by a built-in package route can be set in YAML or JSON under Private, ULA, CGNAT, and loopback addresses are rejected by default. Add each private upstream hostname or IP address to `upstream.allow_private_hosts`. The matching environment variable accepts a comma-separated list. Loopback upstreams also require `upstream.allow_loopback: true`. That setting permits upstream requests and redirects to reach any loopback address. +Upstream requests honour `HTTP_PROXY`, `HTTPS_PROXY` and `NO_PROXY`. When a request goes through a forward proxy, these rules apply twice: to the forward proxy's own address, which therefore has to be listed in `upstream.allow_private_hosts` if it is private, and to the request's destination, which the proxy resolves before handing the request over. Link-local addresses such as `169.254.169.254` are always refused. A destination the proxy cannot resolve itself is passed to the forward proxy, since some networks only resolve public names there; the forward proxy's own policy decides what such a name may reach. + ```yaml upstream: allow_private_hosts: diff --git a/internal/server/forward_proxy.go b/internal/server/forward_proxy.go new file mode 100644 index 0000000..66117a6 --- /dev/null +++ b/internal/server/forward_proxy.go @@ -0,0 +1,61 @@ +package server + +import ( + "context" + "net" + "net/http" + "net/url" + + "github.com/git-pkgs/registries/safehttp" +) + +// upstreamForwardProxy picks the forward proxy for an upstream request. It is +// a variable so tests can route through a test proxy without changing the +// environment, which net/http reads only once. +var upstreamForwardProxy = http.ProxyFromEnvironment + +// lookupUpstreamHost resolves a destination host for the forward proxy check. +var lookupUpstreamHost = net.DefaultResolver.LookupIPAddr + +// guardForwardProxy wraps proxy so that a request sent through a forward +// proxy is held to the same address rules as a direct one. The safehttp dial +// gate only sees the address it connects to, which for a proxied request is +// the forward proxy, so without this a URL taken from upstream metadata could +// reach a link-local or private address through the forward proxy. +// +// The destination is resolved here and every address must pass. A host that +// cannot be resolved locally is passed on, since setups that route all +// traffic through a forward proxy often cannot resolve public names +// themselves; the forward proxy resolves it. +func guardForwardProxy(proxy func(*http.Request) (*url.URL, error), opts safehttp.Options) func(*http.Request) (*url.URL, error) { + if proxy == nil { + return nil + } + checker := safehttp.NewHostIPChecker(opts) + return func(req *http.Request) (*url.URL, error) { + proxyURL, err := proxy(req) + if err != nil || proxyURL == nil { + return proxyURL, err + } + if err := checkProxiedDestination(req.Context(), checker, req.URL.Hostname()); err != nil { + return nil, err + } + return proxyURL, nil + } +} + +func checkProxiedDestination(ctx context.Context, checker *safehttp.HostIPChecker, host string) error { + if ip := net.ParseIP(host); ip != nil { + return checker.Check(host, ip) + } + addrs, err := lookupUpstreamHost(ctx, host) + if err != nil { + return nil //nolint:nilerr // the forward proxy resolves names this host cannot + } + for _, addr := range addrs { + if err := checker.Check(host, addr.IP); err != nil { + return err + } + } + return nil +} diff --git a/internal/server/forward_proxy_test.go b/internal/server/forward_proxy_test.go new file mode 100644 index 0000000..b5287f4 --- /dev/null +++ b/internal/server/forward_proxy_test.go @@ -0,0 +1,105 @@ +package server + +import ( + "context" + "errors" + "net" + "net/http" + "net/http/httptest" + "net/url" + "sync/atomic" + "testing" + + "github.com/git-pkgs/proxy/internal/config" +) + +// routeThroughTestProxy starts a forward proxy that answers every request +// itself, points upstream clients at it, and resolves hostnames from hosts. +func routeThroughTestProxy(t *testing.T, hosts map[string]string) *atomic.Int32 { + t.Helper() + var received atomic.Int32 + forward := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + received.Add(1) + _, _ = w.Write([]byte("via proxy")) + })) + t.Cleanup(forward.Close) + forwardURL, err := url.Parse(forward.URL) + if err != nil { + t.Fatal(err) + } + + oldProxy, oldLookup := upstreamForwardProxy, lookupUpstreamHost + upstreamForwardProxy = http.ProxyURL(forwardURL) + lookupUpstreamHost = func(_ context.Context, host string) ([]net.IPAddr, error) { + if ip, ok := hosts[host]; ok { + return []net.IPAddr{{IP: net.ParseIP(ip)}}, nil + } + return nil, &net.DNSError{Err: "no such host", Name: host, IsNotFound: true} + } + t.Cleanup(func() { upstreamForwardProxy, lookupUpstreamHost = oldProxy, oldLookup }) + return &received +} + +func TestUpstreamClientChecksDestinationBehindForwardProxy(t *testing.T) { + received := routeThroughTestProxy(t, map[string]string{ + "metadata.example.test": "169.254.169.254", + "intranet.example.test": "10.1.2.3", + "allowed.example.test": "10.1.2.4", + "public.example.test": "203.0.113.10", + }) + // The forward proxy listens on loopback, so the client has to be allowed + // to dial it. + client := newUpstreamClient(config.UpstreamConfig{ + AllowLoopback: true, + AllowPrivateHosts: []string{"allowed.example.test", "10.9.9.9"}, + }) + + for _, tt := range []struct { + url string + blocked bool + }{ + {"http://169.254.169.254/latest/meta-data/", true}, + {"https://169.254.169.254/latest/meta-data/", true}, + {"http://[fe80::1]/", true}, + {"http://metadata.example.test/", true}, + {"http://10.1.2.3/", true}, + {"http://intranet.example.test/", true}, + {"http://allowed.example.test/", false}, + {"http://10.9.9.9/", false}, + {"http://203.0.113.10/", false}, + {"http://public.example.test/", false}, + {"http://unresolvable.example.test/", false}, + } { + before := received.Load() + resp, err := client.Get(tt.url) + if err == nil { + _ = resp.Body.Close() + } + reached := received.Load() > before + if tt.blocked { + if err == nil || reached { + t.Errorf("GET %s: err = %v, reached forward proxy = %v; want refused before the forward proxy", tt.url, err, reached) + } + continue + } + if err != nil || !reached { + t.Errorf("GET %s: err = %v, reached forward proxy = %v; want it passed to the forward proxy", tt.url, err, reached) + } + } +} + +func TestGuardForwardProxyKeepsDirectRequests(t *testing.T) { + if guardForwardProxy(nil, upstreamSafeHTTPOptions(config.UpstreamConfig{})) != nil { + t.Error("a transport without a proxy func got one") + } + proxyErr := errors.New("bad proxy config") + guarded := guardForwardProxy(func(*http.Request) (*url.URL, error) { return nil, proxyErr }, upstreamSafeHTTPOptions(config.UpstreamConfig{})) + req := httptest.NewRequest(http.MethodGet, "http://169.254.169.254/", nil) + if _, err := guarded(req); !errors.Is(err, proxyErr) { + t.Errorf("err = %v, want the proxy func's own error", err) + } + direct := guardForwardProxy(func(*http.Request) (*url.URL, error) { return nil, nil }, upstreamSafeHTTPOptions(config.UpstreamConfig{})) + if proxyURL, err := direct(req); proxyURL != nil || err != nil { + t.Errorf("direct request: proxy = %v, err = %v; want nil, nil so the dial gate checks it", proxyURL, err) + } +} diff --git a/internal/server/server.go b/internal/server/server.go index 5b0ebf7..394a699 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -611,14 +611,17 @@ func configureScanning(proxy *handler.Proxy, cfg config.ScanningConfig, baseURL } // newUpstreamClient builds the shared upstream client: a safehttp client whose -// transport keeps upstreamMaxIdleConnsPerHost idle connections per host and +// transport keeps upstreamMaxIdleConnsPerHost idle connections per host, // gives up after upstreamResponseHeaderTimeout when an upstream accepts a -// request but stalls before sending headers. +// request but stalls before sending headers, and checks the destination of +// requests sent through a forward proxy. func newUpstreamClient(upstream config.UpstreamConfig) *http.Client { - client := safehttp.New(nil, upstreamSafeHTTPOptions(upstream)) + opts := upstreamSafeHTTPOptions(upstream) + client := safehttp.New(nil, opts) if transport, ok := client.Transport.(*http.Transport); ok { transport.MaxIdleConnsPerHost = upstreamMaxIdleConnsPerHost transport.ResponseHeaderTimeout = upstreamResponseHeaderTimeout + transport.Proxy = guardForwardProxy(upstreamForwardProxy, opts) } return client } From 13cb62c2360e433e7eb4c228609a5c9788944635 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sat, 24 Oct 2026 08:09:25 +0200 Subject: [PATCH 394/633] Store metadata under a hash of its cache key Cached metadata was stored at _metadata///metadata. Keys are case-sensitive, but on a case-insensitive filesystem such as APFS or NTFS, file:// storage put keys that differ only in case in one file: github.com/Sirupsen/logrus and github.com/sirupsen/logrus each had a row, and whichever was cached last was served for both. Keys may also hold names Windows reserves or end in a dot. The path now carries a SHA-256 of the exact key. Keys that already are SHA-256 hex digests, as most hashed keys are, keep their path. A row still pointing at an unhashed path may hold another key's bytes, so it is neither served nor revalidated; the document is fetched again and stored at the new path, and the old blob is queued for deletion. --- internal/handler/handler.go | 6 +- internal/handler/metadata_path.go | 61 ++++++++++ internal/handler/metadata_path_test.go | 149 +++++++++++++++++++++++++ 3 files changed, 211 insertions(+), 5 deletions(-) create mode 100644 internal/handler/metadata_path.go create mode 100644 internal/handler/metadata_path_test.go diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 8bcac24..9ff0d5a 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -1296,11 +1296,6 @@ func (p *Proxy) serveArtifactError(w http.ResponseWriter, err error, clientMsg s // errStale304 is returned when upstream sends 304 but the cached file is missing. var errStale304 = fmt.Errorf("upstream returned 304 but cached file is missing") -// metadataStoragePath builds a storage path for cached metadata. -func metadataStoragePath(ecosystem, cacheKey string) string { - return "_metadata/" + ecosystem + "/" + cacheKey + "/metadata" -} - // FetchOrCacheMetadata fetches metadata from upstream with caching. // On success it returns the raw response bytes and content type. // If upstream fails and a cached copy exists, the cached version is returned. @@ -1392,6 +1387,7 @@ func (p *Proxy) cachedMetadataState(ctx context.Context, ecosystem, cacheKey str return nil, nil } entry, _ := p.DB.GetMetadataCache(ecosystem, cacheKey) + entry = p.dropLegacyMetadataEntry(ecosystem, cacheKey, entry) if entry != nil && p.MetadataTTL > 0 && entry.FetchedAt.Valid && time.Since(entry.FetchedAt.Time) < p.MetadataTTL { data, ct, err := p.readCachedMetadata(ctx, entry, validate) if err == nil { diff --git a/internal/handler/metadata_path.go b/internal/handler/metadata_path.go new file mode 100644 index 0000000..ca13a59 --- /dev/null +++ b/internal/handler/metadata_path.go @@ -0,0 +1,61 @@ +package handler + +import ( + "crypto/sha256" + "encoding/hex" + + "github.com/git-pkgs/proxy/internal/database" +) + +// metadataStoragePath builds the storage path for cached metadata. Cache +// keys are case-sensitive, but on a case-insensitive filesystem (APFS, NTFS) +// keys that differ only in case, such as Go's github.com/Sirupsen/logrus and +// github.com/sirupsen/logrus, would share one file. Keys may also hold names +// Windows reserves or end in a dot. So the path carries a hash of the exact +// key. Keys that already are lowercase SHA-256 hex digests are used as they +// are, so their existing entries stay valid. +func metadataStoragePath(ecosystem, cacheKey string) string { + if !isSHA256Hex(cacheKey) { + digest := sha256.Sum256([]byte(cacheKey)) + cacheKey = hex.EncodeToString(digest[:]) + } + return legacyMetadataStoragePath(ecosystem, cacheKey) +} + +// legacyMetadataStoragePath is where metadata was stored before paths were +// hashed: under the cache key itself. +func legacyMetadataStoragePath(ecosystem, cacheKey string) string { + return "_metadata/" + ecosystem + "/" + cacheKey + "/metadata" +} + +func isSHA256Hex(s string) bool { + if len(s) != 2*sha256.Size { + return false + } + for i := range len(s) { + c := s[i] + if (c < '0' || c > '9') && (c < 'a' || c > 'f') { + return false + } + } + return true +} + +// dropLegacyMetadataEntry ignores a cache row whose blob is still at its +// legacy, unhashed path. On a case-insensitive filesystem that blob may hold +// another key's document, so it is neither served nor revalidated; the fetch +// that follows stores the document at the hashed path. The old blob is +// queued for deletion, which waits until no row points at it. +func (p *Proxy) dropLegacyMetadataEntry(ecosystem, cacheKey string, entry *database.MetadataCacheEntry) *database.MetadataCacheEntry { + if entry == nil { + return nil + } + legacy := legacyMetadataStoragePath(ecosystem, cacheKey) + if entry.StoragePath != legacy || legacy == metadataStoragePath(ecosystem, cacheKey) { + return entry + } + if err := p.DB.QueuePendingDelete(legacy); err != nil { + p.Logger.Warn("failed to queue legacy metadata blob for deletion", "ecosystem", ecosystem, "key", cacheKey, "error", err) + } + return nil +} diff --git a/internal/handler/metadata_path_test.go b/internal/handler/metadata_path_test.go new file mode 100644 index 0000000..34e73bc --- /dev/null +++ b/internal/handler/metadata_path_test.go @@ -0,0 +1,149 @@ +package handler + +import ( + "bytes" + "context" + "crypto/sha256" + "database/sql" + "encoding/hex" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "path/filepath" + "slices" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/git-pkgs/proxy/internal/database" + "github.com/git-pkgs/proxy/internal/storage" + "github.com/git-pkgs/registries/fetch" +) + +func TestMetadataStoragePathKeepsCaseApart(t *testing.T) { + for _, pair := range [][2]string{ + {"github.com/Sirupsen/logrus", "github.com/sirupsen/logrus"}, + {"JSONStream", "jsonstream"}, + } { + a, b := metadataStoragePath("go", pair[0]), metadataStoragePath("go", pair[1]) + if strings.EqualFold(a, b) { + t.Errorf("%q and %q map to paths %q and %q, which a case-insensitive filesystem treats as one", pair[0], pair[1], a, b) + } + } + for _, key := range []string{"CON", "nul", "trailing.", "a/../b"} { + p := metadataStoragePath("npm", key) + if strings.Contains(p, key) { + t.Errorf("path %q contains the raw key %q", p, key) + } + } + // Keys that already are digests keep their path, so existing entries + // stay valid. + sum := sha256.Sum256([]byte("generic")) + digest := hex.EncodeToString(sum[:]) + if got, want := metadataStoragePath("generic", digest), "_metadata/generic/"+digest+"/metadata"; got != want { + t.Errorf("digest key path = %q, want %q", got, want) + } +} + +// fileStorageProxy returns a proxy whose metadata is cached on the local +// filesystem, as with file:// storage. +func fileStorageProxy(t *testing.T, client *http.Client) (*Proxy, *database.DB) { + t.Helper() + dir := t.TempDir() + db, err := database.Create(filepath.Join(dir, "test.db")) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = db.Close() }) + store, err := storage.OpenBucket(context.Background(), "file://"+filepath.Join(dir, "cache")) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = store.Close() }) + proxy := NewProxy(db, store, &mockFetcher{}, fetch.NewResolver(), slog.New(slog.NewTextHandler(io.Discard, nil))) + proxy.HTTPClient = client + proxy.CacheMetadata = true + proxy.MetadataTTL = time.Hour + return proxy, db +} + +// TestMetadataKeysDifferingInCaseCachedApart caches metadata for two keys +// that differ only in case on file storage. On a case-insensitive filesystem +// they shared one file, and the first key was served the second's document. +func TestMetadataKeysDifferingInCaseCachedApart(t *testing.T) { + var requests atomic.Int32 + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests.Add(1) + _, _ = io.WriteString(w, "document for "+r.URL.Path) + })) + defer upstream.Close() + proxy, _ := fileStorageProxy(t, upstream.Client()) + + keys := []string{"github.com/Sirupsen/logrus", "github.com/sirupsen/logrus"} + for range 2 { + for _, key := range keys { + body, _, err := proxy.FetchOrCacheMetadata(context.Background(), "go", key, upstream.URL+"/"+key) + if err != nil { + t.Fatal(err) + } + if want := "document for /" + key; string(body) != want { + t.Errorf("%s: served %q, want %q", key, body, want) + } + } + } + if got := requests.Load(); got != 2 { + t.Errorf("upstream requests = %d, want 2", got) + } +} + +// TestLegacyMetadataPathRefetched checks that a row still pointing at the +// unhashed path is fetched again, moved to the hashed path, and its old blob +// queued for deletion. +func TestLegacyMetadataPathRefetched(t *testing.T) { + var requests atomic.Int32 + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests.Add(1) + _, _ = io.WriteString(w, "fresh") + })) + defer upstream.Close() + proxy, db := fileStorageProxy(t, upstream.Client()) + + const key = "github.com/Sirupsen/logrus" + legacy := legacyMetadataStoragePath("go", key) + if _, _, err := proxy.Storage.Store(context.Background(), legacy, bytes.NewReader([]byte("old"))); err != nil { + t.Fatal(err) + } + if err := db.UpsertMetadataCache(&database.MetadataCacheEntry{ + Ecosystem: "go", + Name: key, + StoragePath: legacy, + Size: sql.NullInt64{Int64: 3, Valid: true}, + FetchedAt: sql.NullTime{Time: time.Now(), Valid: true}, + }); err != nil { + t.Fatal(err) + } + + body, _, err := proxy.FetchOrCacheMetadata(context.Background(), "go", key, upstream.URL+"/x") + if err != nil { + t.Fatal(err) + } + if string(body) != "fresh" || requests.Load() != 1 { + t.Fatalf("served %q after %d upstream requests, want a fresh fetch", body, requests.Load()) + } + entry, err := db.GetMetadataCache("go", key) + if err != nil || entry == nil { + t.Fatalf("cache row: %v %v", entry, err) + } + if entry.StoragePath != metadataStoragePath("go", key) { + t.Errorf("storage path = %q, want %q", entry.StoragePath, metadataStoragePath("go", key)) + } + due, err := db.GetDuePendingDeletes(time.Now().Add(time.Hour), 10) + if err != nil { + t.Fatal(err) + } + if !slices.Contains(due, legacy) { + t.Errorf("pending deletes = %v, want %q", due, legacy) + } +} From ff10ab3491322e3f2a6d1ce3ab7867b7a2dc2760 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sat, 24 Oct 2026 08:12:59 +0200 Subject: [PATCH 395/633] Verify sha512 OCI digests and refuse ones that can't be checked Manifests and digest-addressed artifacts were only checked against sha256 digests. A sha512 digest, which the OCI spec also allows, or a digest with any other algorithm was fetched and cached unchecked: a manifest requested as sha512:, or a tag whose Docker-Content-Digest header named one, was stored under that digest whatever the body was. Manifests are now checked against any digest in the reference or the Docker-Content-Digest header with the digest's own algorithm, and one the proxy can't compute is answered with DIGEST_INVALID and not cached. GetOrFetchArtifactFromURLWithDigest verifies sha512 (and sha384) digests through the declared-integrity check, so a mismatch is never committed, and refuses other algorithms with ErrUnsupportedDigest instead of fetching unverified. Blob routes still only accept sha256. --- internal/handler/container_manifest.go | 13 ++-- internal/handler/container_test.go | 83 ++++++++++++++++++++------ internal/handler/handler.go | 71 +++++++++++++++++++--- internal/handler/integrity_test.go | 63 +++++++++++++++++++ 4 files changed, 200 insertions(+), 30 deletions(-) diff --git a/internal/handler/container_manifest.go b/internal/handler/container_manifest.go index 82eed45..af69967 100644 --- a/internal/handler/container_manifest.go +++ b/internal/handler/container_manifest.go @@ -114,18 +114,23 @@ func (h *ContainerHandler) serveManifest(w http.ResponseWriter, r *http.Request, h.serveStaleManifestOrError(w, r, cacheName, reference, cached, fmt.Errorf("reading manifest: %w", err)) return } - computedDigest := sha256Digest(body) contentDigest := resp.Header.Get("Docker-Content-Digest") + // A digest reference, and the digest the upstream says the manifest + // has, are both cache keys, so each must be the manifest's own digest. + // One the proxy can't compute is refused rather than cached unchecked. for _, expected := range []string{reference, contentDigest} { - if strings.HasPrefix(expected, "sha256:") && expected != computedDigest { + if !strings.Contains(expected, ":") { + continue + } + if err := verifyDigest(expected, body); err != nil { h.proxy.Logger.Error("upstream manifest failed digest verification", - "name", name, "reference", reference, "expected", expected, "actual", computedDigest) + "name", name, "reference", reference, "expected", expected, "error", err) h.containerError(w, http.StatusBadGateway, "DIGEST_INVALID", "manifest digest verification failed") return } } if contentDigest == "" { - contentDigest = computedDigest + contentDigest = sha256Digest(body) } manifest := &cachedContainerManifest{ diff --git a/internal/handler/container_test.go b/internal/handler/container_test.go index f7b7076..d546725 100644 --- a/internal/handler/container_test.go +++ b/internal/handler/container_test.go @@ -1,6 +1,8 @@ package handler import ( + "crypto/sha512" + "encoding/hex" "encoding/json" "io" "net/http" @@ -10,6 +12,7 @@ import ( "testing" "time" + "github.com/git-pkgs/proxy/internal/database" upstreamhttp "github.com/git-pkgs/proxy/internal/httpclient" "github.com/git-pkgs/purl" "github.com/git-pkgs/registries/fetch" @@ -900,38 +903,82 @@ func TestContainerHandler_ManifestDigestMismatchIsNotCached(t *testing.T) { } } -func TestContainerHandler_ManifestNonSHA256DigestReferenceIsProxied(t *testing.T) { +// TestContainerHandler_ManifestDigestsAreVerified checks that a sha512 +// digest, in the reference or in Docker-Content-Digest, is verified like a +// sha256 one, and that a digest the proxy can't compute is refused rather +// than cached unverified. +func TestContainerHandler_ManifestDigestsAreVerified(t *testing.T) { manifest := `{"schemaVersion":2}` - sha512Reference := "sha512:" + strings.Repeat("a", 128) - sha512Header := "sha512:" + strings.Repeat("b", 128) - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - w.Header().Set("Content-Type", "application/vnd.oci.image.manifest.v1+json") - w.Header().Set("Docker-Content-Digest", sha512Header) - _, _ = io.WriteString(w, manifest) - })) - defer upstream.Close() + sum := sha512.Sum512([]byte(manifest)) + sha512Digest := "sha512:" + hex.EncodeToString(sum[:]) + otherSHA512 := "sha512:" + strings.Repeat("b", 128) + unsupported := "sha1:" + strings.Repeat("c", 40) - proxy, _, _, _ := setupTestProxy(t) - proxy.HTTPClient = upstream.Client() - h := &ContainerHandler{proxy: proxy, registryURL: upstream.URL} + tests := []struct { + name, reference, header string + wantOK bool + }{ + {"sha512 reference", sha512Digest, sha512Digest, true}, + {"sha512 reference without header", sha512Digest, "", true}, + {"tag with sha512 header", "latest", sha512Digest, true}, + {"mismatched sha512 reference", otherSHA512, "", false}, + {"tag with mismatched sha512 header", "latest", otherSHA512, false}, + {"unsupported reference", unsupported, "", false}, + {"tag with unsupported header", "latest", unsupported, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/vnd.oci.image.manifest.v1+json") + if tt.header != "" { + w.Header().Set("Docker-Content-Digest", tt.header) + } + _, _ = io.WriteString(w, manifest) + })) + defer upstream.Close() - for _, reference := range []string{sha512Reference, "latest"} { - t.Run(reference, func(t *testing.T) { + proxy, db, _, _ := setupTestProxy(t) + proxy.HTTPClient = upstream.Client() + h := &ContainerHandler{proxy: proxy, registryURL: upstream.URL} + req := httptest.NewRequest(http.MethodGet, "/library/nginx/manifests/"+tt.reference, nil) w := httptest.NewRecorder() - h.Routes().ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/library/nginx/manifests/"+reference, nil)) + h.Routes().ServeHTTP(w, req) + + if !tt.wantOK { + if w.Code != http.StatusBadGateway { + t.Fatalf("status = %d, want %d; body: %s", w.Code, http.StatusBadGateway, w.Body.String()) + } + requireManifestNotCached(t, h, db, upstream.URL, req, tt.reference, tt.header) + return + } if w.Code != http.StatusOK { t.Fatalf("status = %d, want %d; body: %s", w.Code, http.StatusOK, w.Body.String()) } if got := w.Body.String(); got != manifest { t.Errorf("body = %q, want %q", got, manifest) } - if got := w.Header().Get("Docker-Content-Digest"); got != sha512Header { - t.Errorf("Docker-Content-Digest = %q, want %q", got, sha512Header) - } }) } } +func requireManifestNotCached(t *testing.T, h *ContainerHandler, db *database.DB, upstreamURL string, req *http.Request, keys ...string) { + t.Helper() + accept := normalizeContainerManifestAccept(containerManifestAccept(req)) + for _, key := range keys { + if key == "" { + continue + } + entry, err := db.GetMetadataCache(containerManifestCacheEcosystem, + h.containerManifestSharedCacheKey(upstreamURL, "library/nginx", key, accept)) + if err != nil { + t.Fatalf("checking manifest cache: %v", err) + } + if entry != nil { + t.Errorf("manifest was cached under unverified %s", key) + } + } +} + func TestContainerHandler_ManifestTagWithInvalidDigestIsNotAliased(t *testing.T) { manifest := `{"schemaVersion":2}` invalidDigest := sha256Digest([]byte("different manifest")) diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 8bcac24..e5ab565 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -18,6 +18,7 @@ import ( "github.com/git-pkgs/artifacts" "github.com/git-pkgs/cooldown" + "github.com/git-pkgs/integrity" "github.com/git-pkgs/proxy/internal/database" "github.com/git-pkgs/proxy/internal/denylist" "github.com/git-pkgs/proxy/internal/metrics" @@ -1921,15 +1922,69 @@ func (p *Proxy) GetOrFetchArtifactFromURLWithHeaders(ctx context.Context, ecosys return p.getOrFetchArtifactFromURL(ctx, ecosystem, name, version, filename, downloadURL, headers, "", "") } -// GetOrFetchArtifactFromURLWithDigest retrieves an artifact and verifies its -// SHA-256 digest before adding a newly fetched response to the cache. -// Non-sha256 digests are proxied without verification. -func (p *Proxy) GetOrFetchArtifactFromURLWithDigest(ctx context.Context, ecosystem, name, version, filename, downloadURL, digest string) (*CacheResult, error) { - upstreamHash, _ := strings.CutPrefix(digest, "sha256:") - if upstreamHash == digest { - upstreamHash = "" +// GetOrFetchArtifactFromURLWithDigest retrieves an artifact and verifies a +// newly fetched response against expected, an OCI-style digest such as +// "sha256:" or "sha512:", before serving or caching it. A digest +// whose algorithm can't be verified is refused with ErrUnsupportedDigest +// rather than fetched unverified. An empty expected fetches without +// verification. +func (p *Proxy) GetOrFetchArtifactFromURLWithDigest(ctx context.Context, ecosystem, name, version, filename, downloadURL, expected string) (*CacheResult, error) { + if expected == "" { + return p.getOrFetchArtifactFromURL(ctx, ecosystem, name, version, filename, downloadURL, nil, "", "") } - return p.getOrFetchArtifactFromURL(ctx, ecosystem, name, version, filename, downloadURL, nil, upstreamHash, "") + upstreamHash, declared, err := digestChecks(expected) + if err != nil { + return nil, err + } + return p.getOrFetchArtifactFromURL(ctx, ecosystem, name, version, filename, downloadURL, nil, upstreamHash, declared) +} + +// ErrUnsupportedDigest is returned for a digest whose algorithm or encoding +// the proxy can't verify. +var ErrUnsupportedDigest = errors.New("unsupported digest") + +// digestChecks turns an OCI-style digest into the checks the artifact fetch +// runs: the hex of a sha256 digest, which is also how cached artifacts are +// keyed, or for another algorithm the integrity string verifyIntegrity +// checks. +func digestChecks(expected string) (upstreamHash, declared string, _ error) { + // The sha256 hex is compared as given, ignoring case; a malformed one + // fails as a mismatch once the body has been hashed. + if hash, ok := strings.CutPrefix(expected, "sha256:"); ok { + return hash, "", nil + } + d, err := digest.Parse(expected) + if err != nil { + return "", "", fmt.Errorf("%w %q: %w", ErrUnsupportedDigest, expected, err) + } + var algorithm integrity.Algorithm + switch d.Algorithm() { + case digest.SHA384: + algorithm = integrity.SHA384 + case digest.SHA512: + algorithm = integrity.SHA512 + default: + return "", "", fmt.Errorf("%w %q", ErrUnsupportedDigest, expected) + } + sum, err := integrity.ParseHex(algorithm, d.Encoded()) + if err != nil { + return "", "", fmt.Errorf("%w %q: %w", ErrUnsupportedDigest, expected, err) + } + return "", sum.SRI(), nil +} + +// verifyDigest checks body against expected, an OCI-style digest. A digest +// whose algorithm can't be computed is an ErrUnsupportedDigest, and one that +// doesn't match is an ErrArtifactDigestMismatch. +func verifyDigest(expected string, body []byte) error { + d, err := digest.Parse(expected) + if err != nil { + return fmt.Errorf("%w %q: %w", ErrUnsupportedDigest, expected, err) + } + if actual := d.Algorithm().FromBytes(body); actual != d { + return fmt.Errorf("%w: expected %s, got %s", ErrArtifactDigestMismatch, d, actual) + } + return nil } // GetOrFetchArtifactFromURLWithIntegrity retrieves an artifact and verifies a diff --git a/internal/handler/integrity_test.go b/internal/handler/integrity_test.go index cb84b14..f5dcae4 100644 --- a/internal/handler/integrity_test.go +++ b/internal/handler/integrity_test.go @@ -7,6 +7,7 @@ import ( "encoding/base64" "encoding/hex" "errors" + "fmt" "io" "net/http" "net/http/httptest" @@ -14,6 +15,8 @@ import ( "strings" "testing" "testing/iotest" + + "github.com/git-pkgs/registries/fetch" ) func sha256Hex(data string) string { @@ -451,3 +454,63 @@ func TestFailOnMismatchHoldsBackFinalChunk(t *testing.T) { }) } } + +// TestFetchWithDigestVerifiesEveryAlgorithm checks that a sha512 digest is +// verified before the artifact is cached, like a sha256 one, and that a digest +// whose algorithm can't be verified is refused without fetching. +func TestFetchWithDigestVerifiesEveryAlgorithm(t *testing.T) { + const content = "blob content" + sum512 := sha512.Sum512([]byte(content)) + tests := []struct { + name, digest string + wantErr error + }{ + {"sha256", "sha256:" + sha256Hex(content), nil}, + {"sha512", "sha512:" + hex.EncodeToString(sum512[:]), nil}, + {"mismatched sha256", "sha256:" + sha256Hex("other"), ErrArtifactDigestMismatch}, + {"mismatched sha512", "sha512:" + strings.Repeat("a", 128), ErrArtifactDigestMismatch}, + {"unsupported algorithm", "sha1:" + strings.Repeat("a", 40), ErrUnsupportedDigest}, + {"malformed sha512", "sha512:abc", ErrUnsupportedDigest}, + } + for _, stream := range []bool{false, true} { + for _, tt := range tests { + t.Run(fmt.Sprintf("%s/stream=%t", tt.name, stream), func(t *testing.T) { + checkFetchWithDigest(t, stream, content, tt.digest, tt.wantErr) + }) + } + } +} + +func checkFetchWithDigest(t *testing.T, stream bool, content, digest string, wantErr error) { + t.Helper() + proxy, _, _, fetcher := setupTestProxy(t) + proxy.StreamArtifacts = stream + fetcher.artifact = &fetch.Artifact{Body: io.NopCloser(strings.NewReader(content))} + + result, err := proxy.GetOrFetchArtifactFromURLWithDigest(t.Context(), "oci", "library/demo", digest, digest, + "https://registry.example/v2/library/demo/blobs/"+digest, digest) + if err == nil { + var body []byte + body, err = io.ReadAll(result.Reader) + _ = result.Reader.Close() + if err == nil && string(body) != content { + t.Errorf("body = %q, want %q", body, content) + } + } + if !errors.Is(err, wantErr) { + t.Fatalf("error = %v, want %v", err, wantErr) + } + if errors.Is(wantErr, ErrUnsupportedDigest) && fetcher.fetchCalled { + t.Error("fetched an artifact whose digest can't be verified") + } + cached, err := proxy.GetCachedArtifact(t.Context(), "oci", "library/demo", digest, digest) + if err != nil { + t.Fatal(err) + } + if cached != nil { + _ = cached.Reader.Close() + } + if wantCached := wantErr == nil && !stream; (cached != nil) != wantCached { + t.Errorf("cached = %v, want %v", cached != nil, wantCached) + } +} From ce3958378439236b89fb9ac41eaa1dccc457e249 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sat, 24 Oct 2026 08:12:59 +0200 Subject: [PATCH 396/633] Verify Helm charts before they are cached Without artifact streaming, a chart was fetched with no expected digest, committed to the cache, and only then compared with the index's digest by serveChart, which cleared it again. Until then the unverified chart was a cache entry like any other. The index digest is now always passed to the fetch, so a mismatching chart is discarded before it is committed, as it already was when streaming. serveChart's check stays for charts that were cached before. --- internal/handler/helm.go | 10 +++------- internal/handler/helm_test.go | 4 ++-- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/internal/handler/helm.go b/internal/handler/helm.go index 6183d97..bb6b623 100644 --- a/internal/handler/helm.go +++ b/internal/handler/helm.go @@ -126,14 +126,10 @@ func (h *HelmHandler) handleChart(w http.ResponseWriter, r *http.Request) { return } - // A streamed fetch is never stored, so serveChart's digest check has - // nothing to compare against: verify the stream itself instead. - expectedDigest := "" - if h.proxy.StreamArtifacts { - expectedDigest = "sha256:" + digest - } + // Verify the chart against the index digest while it is fetched, so a + // mismatch is never committed to the cache, nor streamed out complete. result, err := h.proxy.GetOrFetchArtifactFromURLWithDigest( - r.Context(), helmMetadataEcosystem, repository, digest, filename, downloadURL, expectedDigest) + r.Context(), helmMetadataEcosystem, repository, digest, filename, downloadURL, "sha256:"+digest) if err != nil { h.proxy.serveArtifactError(w, err, "failed to fetch chart") return diff --git a/internal/handler/helm_test.go b/internal/handler/helm_test.go index c50533d..26c0877 100644 --- a/internal/handler/helm_test.go +++ b/internal/handler/helm_test.go @@ -179,8 +179,8 @@ func TestHelmHandler_RejectsChartDigestMismatch(t *testing.T) { t.Errorf("rejected chart at %q is not queued for deletion", path) } } - if stored != requests { - t.Errorf("found %d stored charts, want one per request (%d)", stored, requests) + if stored != 0 { + t.Errorf("found %d stored charts, want none: a mismatch is discarded before it is committed", stored) } } From d2c65879965a4fa82972029ee413d00174f5af47 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sat, 24 Oct 2026 08:02:07 +0200 Subject: [PATCH 397/633] Hash path-based metadata cache keys Conan, CRAN, Conda, RPM, the main Debian archive and the default Maven repository keyed cached metadata by the request path with '/' replaced by '_'. Names and versions may contain '_', so foo/bar_1.0 and foo_bar/1.0 shared an entry, and with cache_metadata one reference's recipe files were served for the other. Conan also appended the query with '_', so a path without a query could collide with a shorter path plus one. Hash the upstream URL, the path and, for Conan, the query instead, as the APK, Arch, generic and Helm handlers already do. Each part is length-prefixed so no two inputs share a key. --- internal/handler/cache_key.go | 22 ++++++++ internal/handler/cache_key_test.go | 74 +++++++++++++++++++++++++ internal/handler/conan.go | 6 +- internal/handler/conda.go | 7 ++- internal/handler/conda_encoding_test.go | 10 ++-- internal/handler/conda_shards.go | 8 +-- internal/handler/cran.go | 9 ++- internal/handler/debian.go | 6 +- internal/handler/debian_test.go | 17 +++--- internal/handler/maven.go | 8 +-- internal/handler/rpm.go | 8 ++- 11 files changed, 137 insertions(+), 38 deletions(-) create mode 100644 internal/handler/cache_key.go create mode 100644 internal/handler/cache_key_test.go diff --git a/internal/handler/cache_key.go b/internal/handler/cache_key.go new file mode 100644 index 0000000..684c5d7 --- /dev/null +++ b/internal/handler/cache_key.go @@ -0,0 +1,22 @@ +package handler + +import ( + "crypto/sha256" + "encoding/hex" + "strconv" +) + +// hashedCacheKey derives a metadata cache key from parts such as the +// upstream URL, the request path and the query. Each part is length-prefixed +// before hashing, so no two different part lists share a key, whatever bytes +// the parts contain. Keys built by replacing '/' with '_' were ambiguous: +// foo/bar_1.0 and foo_bar/1.0 mapped to the same entry. +func hashedCacheKey(parts ...string) string { + h := sha256.New() + for _, part := range parts { + _, _ = h.Write([]byte(strconv.Itoa(len(part)))) + _, _ = h.Write([]byte{':'}) + _, _ = h.Write([]byte(part)) + } + return hex.EncodeToString(h.Sum(nil)) +} diff --git a/internal/handler/cache_key_test.go b/internal/handler/cache_key_test.go new file mode 100644 index 0000000..8af47a2 --- /dev/null +++ b/internal/handler/cache_key_test.go @@ -0,0 +1,74 @@ +package handler + +import ( + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" +) + +// TestConanMetadataKeysKeepReferencesApart checks that Conan references whose +// paths only differ in where '/' and '_' fall, or in a query, are cached +// apart. Names and versions may contain '_'. +func TestConanMetadataKeysKeepReferencesApart(t *testing.T) { + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = io.WriteString(w, r.URL.Path+"?"+r.URL.RawQuery) + })) + defer upstream.Close() + + proxy, _, _, _ := setupTestProxy(t) + proxy.CacheMetadata = true + proxy.MetadataTTL = time.Hour + proxy.HTTPClient = upstream.Client() + srv := httptest.NewServer(NewConanHandlerWithUpstream(proxy, testProxyURL, upstream.URL).Routes()) + defer srv.Close() + + for _, pair := range [][2]string{ + {"/v2/conans/foo/bar_1.0/_/_/latest", "/v2/conans/foo_bar/1.0/_/_/latest"}, + {"/v2/conans/foo/bar_1.0/_/_/revisions/r1/files/conanfile.py", "/v2/conans/foo_bar/1.0/_/_/revisions/r1/files/conanfile.py"}, + {"/v2/conans/search?q=zlib", "/v2/conans/search_q=zlib"}, + } { + for _, target := range pair { + resp, err := http.Get(srv.URL + target) + if err != nil { + t.Fatal(err) + } + body, _ := io.ReadAll(resp.Body) + _ = resp.Body.Close() + want := target + if !strings.Contains(want, "?") { + want += "?" + } + if string(body) != want { + t.Errorf("GET %s served %q, want %q", target, body, want) + } + } + } +} + +// TestMetadataCacheKeysUnambiguous checks that the path-based metadata cache +// keys of the other ecosystems keep paths apart that differ only in where +// '/' and '_' fall. +func TestMetadataCacheKeysUnambiguous(t *testing.T) { + const a, b = "x/foo/bar_1.0/file", "x/foo_bar/1.0/file" + + maven := mavenSource{upstreamURL: "https://repo.test"} + debian := NewDebianHandler(nil, "http://proxy.test", "https://deb.test", nil) + rpm := NewRPMHandlerWithUpstream(nil, "http://proxy.test", "https://rpm.test") + cran := NewCRANHandlerWithUpstream(nil, "http://proxy.test", "https://cran.test") + conda := NewCondaHandlerWithUpstream(nil, "http://proxy.test", "https://conda.test") + + for name, key := range map[string]func(string) string{ + "maven": maven.metadataCacheKey, + "debian": func(p string) string { return debian.metadataCacheKeyFor("", debian.upstreamURL, p) }, + "rpm": func(p string) string { return rpm.metadataCacheKey(p) }, + "cran": func(p string) string { return cran.metadataCacheKey("/" + p) }, + "conda": func(p string) string { return conda.metadataCacheKey("/" + p) }, + } { + if key(a) == key(b) { + t.Errorf("%s: %s and %s share metadata cache key %q", name, a, b, key(a)) + } + } +} diff --git a/internal/handler/conan.go b/internal/handler/conan.go index 4208be1..dddd79f 100644 --- a/internal/handler/conan.go +++ b/internal/handler/conan.go @@ -162,11 +162,7 @@ func (h *ConanHandler) shouldCacheFile(filename string) bool { // proxyCached forwards a request with metadata caching. func (h *ConanHandler) proxyCached(w http.ResponseWriter, r *http.Request) { - cacheKey := strings.TrimPrefix(r.URL.Path, "/") - cacheKey = strings.ReplaceAll(cacheKey, "/", "_") - if r.URL.RawQuery != "" { - cacheKey += "_" + r.URL.RawQuery - } + cacheKey := hashedCacheKey(h.upstreamURL, r.URL.Path, r.URL.RawQuery) upstreamURL := h.upstreamURL + r.URL.Path if r.URL.RawQuery != "" { upstreamURL += "?" + r.URL.RawQuery diff --git a/internal/handler/conda.go b/internal/handler/conda.go index 0f229fc..1142ccf 100644 --- a/internal/handler/conda.go +++ b/internal/handler/conda.go @@ -320,9 +320,14 @@ func (h *CondaHandler) recordInCooldown(filename, name, version string, timestam return true } +// metadataCacheKey is the metadata cache key of a request path. +func (h *CondaHandler) metadataCacheKey(urlPath string) string { + return hashedCacheKey(h.upstreamURL, urlPath) +} + // proxyCached forwards a metadata request with caching. func (h *CondaHandler) proxyCached(w http.ResponseWriter, r *http.Request) { - cacheKey := condaCacheKey(r.URL.Path) + cacheKey := h.metadataCacheKey(r.URL.Path) // Large JSON indexes can exceed the metadata limit uncompressed. Conda // clients decode Content-Encoding, so cache and relay gzip verbatim. The // already-compressed repodata.json.bz2 route must stay on identity. diff --git a/internal/handler/conda_encoding_test.go b/internal/handler/conda_encoding_test.go index 4aa5951..a8f3a79 100644 --- a/internal/handler/conda_encoding_test.go +++ b/internal/handler/conda_encoding_test.go @@ -40,7 +40,8 @@ func TestCondaRepodataGzip(t *testing.T) { if tt.mode == "cached" { proxy.MetadataTTL = time.Hour } - handler := NewCondaHandlerWithUpstream(proxy, "http://proxy.local", upstream.URL).Routes() + h := NewCondaHandlerWithUpstream(proxy, "http://proxy.local", upstream.URL) + handler := h.Routes() serve := func() *httptest.ResponseRecorder { w := httptest.NewRecorder() handler.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/conda-forge/linux-64/"+tt.filename, nil)) @@ -53,7 +54,7 @@ func TestCondaRepodataGzip(t *testing.T) { if tt.mode == "stream" { return } - entry, err := db.GetMetadataCache("conda", "conda-forge_linux-64_"+tt.filename) + entry, err := db.GetMetadataCache("conda", h.metadataCacheKey("/conda-forge/linux-64/"+tt.filename)) if err != nil { t.Fatal(err) } @@ -122,7 +123,8 @@ func TestCondaRepodataCacheSizeLimit(t *testing.T) { proxy.HTTPClient = upstream.Client() proxy.CacheMetadata = true proxy.MetadataMaxSize = tt.limit - handler := NewCondaHandlerWithUpstream(proxy, "http://proxy.local", upstream.URL).Routes() + h := NewCondaHandlerWithUpstream(proxy, "http://proxy.local", upstream.URL) + handler := h.Routes() w := httptest.NewRecorder() handler.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/conda-forge/linux-64/repodata.json", nil)) if w.Code != tt.status { @@ -132,7 +134,7 @@ func TestCondaRepodataCacheSizeLimit(t *testing.T) { if !bytes.Equal(w.Body.Bytes(), plain) || w.Header().Get(headerContentEncoding) != "" { t.Fatal("identity response was changed or mislabeled as gzip") } - } else if entry, _ := db.GetMetadataCache("conda", "conda-forge_linux-64_repodata.json"); entry != nil { + } else if entry, _ := db.GetMetadataCache("conda", h.metadataCacheKey("/conda-forge/linux-64/repodata.json")); entry != nil { t.Fatal("oversized response was cached") } }) diff --git a/internal/handler/conda_shards.go b/internal/handler/conda_shards.go index f2de475..c7c7a00 100644 --- a/internal/handler/conda_shards.go +++ b/internal/handler/conda_shards.go @@ -56,7 +56,7 @@ func (h *CondaHandler) cooldownEnabled() bool { func (h *CondaHandler) handleShardIndex(w http.ResponseWriter, r *http.Request) { channel, arch := r.PathValue("channel"), r.PathValue("arch") upstreamURL := h.upstreamURL + r.URL.Path - cacheKey := condaCacheKey(r.URL.Path) + cacheKey := h.metadataCacheKey(r.URL.Path) body, contentType, contentEncoding, err := h.proxy.fetchOrCacheMetadata(r.Context(), "conda", cacheKey, upstreamURL, "identity", nil, "*/*") if err != nil { @@ -121,7 +121,7 @@ func (h *CondaHandler) handleCooldownShard(w http.ResponseWriter, r *http.Reques } upstreamPath := "/" + channel + "/" + arch + "/" + dir + hex.EncodeToString(masked) + condaShardSuffix - body, _, _, err := h.proxy.fetchOrCacheMetadata(r.Context(), "conda", condaCacheKey(upstreamPath), h.upstreamURL+upstreamPath, "identity", nil, "*/*") + body, _, _, err := h.proxy.fetchOrCacheMetadata(r.Context(), "conda", h.metadataCacheKey(upstreamPath), h.upstreamURL+upstreamPath, "identity", nil, "*/*") if err != nil { h.proxy.writeMetadataFetchError(w, err) return @@ -455,7 +455,3 @@ func msgpackNumber(v any) float64 { } } -// condaCacheKey is the metadata cache key for a request path. -func condaCacheKey(urlPath string) string { - return strings.ReplaceAll(strings.TrimPrefix(urlPath, "/"), "/", "_") -} diff --git a/internal/handler/cran.go b/internal/handler/cran.go index d0dbd52..803bd4b 100644 --- a/internal/handler/cran.go +++ b/internal/handler/cran.go @@ -165,11 +165,14 @@ func (h *CRANHandler) isBinaryPackage(filename string) bool { return strings.HasSuffix(filename, ".zip") || strings.HasSuffix(filename, ".tgz") } +// metadataCacheKey is the metadata cache key of a request path. +func (h *CRANHandler) metadataCacheKey(urlPath string) string { + return hashedCacheKey(h.upstreamURL, urlPath) +} + // proxyCached forwards a metadata request with caching. func (h *CRANHandler) proxyCached(w http.ResponseWriter, r *http.Request) { - cacheKey := strings.TrimPrefix(r.URL.Path, "/") - cacheKey = strings.ReplaceAll(cacheKey, "/", "_") - h.proxy.ProxyCached(w, r, h.upstreamURL+r.URL.Path, "cran", cacheKey, "*/*") + h.proxy.ProxyCached(w, r, h.upstreamURL+r.URL.Path, "cran", h.metadataCacheKey(r.URL.Path), "*/*") } // proxyUpstream forwards a request to CRAN without caching. diff --git a/internal/handler/debian.go b/internal/handler/debian.go index dccb823..a196547 100644 --- a/internal/handler/debian.go +++ b/internal/handler/debian.go @@ -203,11 +203,11 @@ func (h *DebianHandler) artifactCacheFilename(repository, filename string) strin } // metadataCacheKeyFor returns the metadata cache key for a request. The main -// archive keeps its separator-based key so existing cache entries stay valid; -// a named repository hashes its identity as APKHandler.metadataCacheKey does. +// archive hashes its upstream URL and the path; a named repository hashes its +// identity as APKHandler.metadataCacheKey does. func (h *DebianHandler) metadataCacheKeyFor(repository, upstreamURL, path string) string { if repository == "" { - return strings.ReplaceAll(path, "/", "_") + return hashedCacheKey(upstreamURL, path) } identity := repository + "\x00" + upstreamURL + "\x00" + path digest := sha256.Sum256([]byte(identity)) diff --git a/internal/handler/debian_test.go b/internal/handler/debian_test.go index b94919f..f31202f 100644 --- a/internal/handler/debian_test.go +++ b/internal/handler/debian_test.go @@ -35,9 +35,11 @@ func TestDebianHandler_Routes(t *testing.T) { assertRoutesBasics(t, h.Routes(), "/dists/stable/Release", "/pool/../../../etc/passwd") } -// TestDebianHandler_LegacyCacheKeysUnchanged pins the cache identities the -// main archive used before named repositories existed. Deployments carry warm -// caches across upgrades, so a changed key here silently discards them. +// TestDebianHandler_LegacyCacheKeysUnchanged pins the artifact cache identity +// the main archive used before named repositories existed. Deployments carry +// warm caches across upgrades, so a changed key here silently discards them. +// Metadata is cheap to refetch; its keys are covered by +// TestMetadataCacheKeysUnambiguous. func TestDebianHandler_LegacyCacheKeysUnchanged(t *testing.T) { const ( poolPath = "pool/main/h/hello/hello_2.10-3_amd64.deb" @@ -54,17 +56,12 @@ func TestDebianHandler_LegacyCacheKeysUnchanged(t *testing.T) { t.Errorf("legacy artifact cache filename = %q, want %q", got, "hello_2.10-3_amd64.deb") } - // Metadata cache: path separators replaced with underscores. - if got := h.metadataCacheKeyFor("", h.upstreamURL, distPath); got != "dists_trixie_InRelease" { - t.Errorf("legacy metadata cache key = %q, want %q", got, "dists_trixie_InRelease") - } - // A named repository must not reuse either identity. if got := h.artifactCacheFilename("security", filename); got == "hello_2.10-3_amd64.deb" { t.Error("named repository reused the legacy artifact cache filename") } - if got := h.metadataCacheKeyFor("security", "https://security.test", distPath); got == "dists_trixie_InRelease" { - t.Error("named repository reused the legacy metadata cache key") + if h.metadataCacheKeyFor("security", "https://security.test", distPath) == h.metadataCacheKeyFor("", h.upstreamURL, distPath) { + t.Error("named repository reused the main archive's metadata cache key") } } diff --git a/internal/handler/maven.go b/internal/handler/maven.go index c201e74..90c4ce5 100644 --- a/internal/handler/maven.go +++ b/internal/handler/maven.go @@ -146,12 +146,12 @@ func (h *MavenHandler) handleRequest(w http.ResponseWriter, r *http.Request) { } // metadataCacheKey is the metadata cache key of urlPath. The default -// repository keeps its original keys. A named repository's keys are hashed -// from its name, upstream URL and the path, so repositories never share -// entries and repointing one drops what it cached. +// repository's keys are hashed from its upstream URLs and the path. A named +// repository's keys are hashed from its name, upstream URL and the path, so +// repositories never share entries and repointing one drops what it cached. func (s mavenSource) metadataCacheKey(urlPath string) string { if s.name == "" { - return strings.ReplaceAll(urlPath, "/", "_") + return hashedCacheKey(s.upstreamURL, s.fallbackURL, urlPath) } digest := sha256.Sum256([]byte(s.name + "\x00" + s.upstreamURL + "\x00" + urlPath)) return "repo_" + hex.EncodeToString(digest[:]) diff --git a/internal/handler/rpm.go b/internal/handler/rpm.go index 3321ccc..8a3a762 100644 --- a/internal/handler/rpm.go +++ b/internal/handler/rpm.go @@ -122,8 +122,12 @@ func (h *RPMHandler) handlePackageDownload(w http.ResponseWriter, r *http.Reques // handleMetadata proxies repository metadata files (repomd.xml, primary.xml.gz, etc.). // These change frequently so we don't cache them. func (h *RPMHandler) handleMetadata(w http.ResponseWriter, r *http.Request, path string) { - cacheKey := strings.ReplaceAll(path, "/", "_") - h.proxy.ProxyCached(w, r, fmt.Sprintf("%s/%s", h.upstreamURL, path), "rpm", cacheKey, "*/*") + h.proxy.ProxyCached(w, r, fmt.Sprintf("%s/%s", h.upstreamURL, path), "rpm", h.metadataCacheKey(path), "*/*") +} + +// metadataCacheKey is the metadata cache key of a repository path. +func (h *RPMHandler) metadataCacheKey(path string) string { + return hashedCacheKey(h.upstreamURL, path) } // proxyFile proxies any file directly without caching. From 77bd66acabff4691a35e6e9926c6138cd5978351 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sat, 24 Oct 2026 08:09:47 +0200 Subject: [PATCH 398/633] Bound Git requests by their body and response progress upload-pack was started with its stdin attached to the request body, after the read deadline had been extended to two hours, and ls-refs requests took no slot. A client that sent part of a request and then nothing kept an upload-pack process waiting, and any number of them could. Stateless requests are complete once sent, so the handler now reads the body in full first, within the server's read timeout, keeping bodies past 1 MiB in a temporary file. Only then are the deadlines extended for the response. info/refs and ls-refs share their own slots, four per CPU, so they still never wait behind clones. Each response write is bounded to two minutes, so a client that stops reading a pack frees its slot instead of holding it until the request timeout. --- internal/gitcache/cache.go | 18 +++- internal/gitcache/helpers_test.go | 11 ++- internal/gitcache/serve.go | 135 +++++++++++++++++++++++------- internal/gitcache/serve_test.go | 101 ++++++++++++++++++++++ internal/handler/git.go | 79 +++++++++++++---- internal/handler/git_test.go | 91 +++++++++++++++++++- 6 files changed, 384 insertions(+), 51 deletions(-) diff --git a/internal/gitcache/cache.go b/internal/gitcache/cache.go index 73d0529..eb1578f 100644 --- a/internal/gitcache/cache.go +++ b/internal/gitcache/cache.go @@ -69,14 +69,15 @@ type Options struct { // else in CacheDir is touched, so it may hold other data. Upstreams map[string]string - // refCheckTimeout, lowSpeedTime, maxRequestBody and maxUploadPacks - // override their defaults in tests, and refreshWaitOver, when set, - // replaces the RefreshWait timer. + // refCheckTimeout, lowSpeedTime, maxRequestBody, maxUploadPacks and + // maxRefListings override their defaults in tests, and refreshWaitOver, + // when set, replaces the RefreshWait timer. refCheckTimeout time.Duration refreshWaitOver <-chan time.Time lowSpeedTime int maxRequestBody int64 maxUploadPacks int + maxRefListings int } // Cache is the Git repository cache. internal/handler serves it at /git. @@ -90,6 +91,7 @@ type Cache struct { relay *relay fetches chan struct{} uploadPacks chan struct{} + refListings chan struct{} refCheckTimeout time.Duration lowSpeedTime int maxRequestBody int64 @@ -149,6 +151,11 @@ const maxConcurrentFetches = 16 // some packs wait on slow clients. const uploadPacksPerCPU = 2 +// refListingsPerCPU caps the upload-pack processes listing refs at once, for +// info/refs and ls-refs. They build no pack and so do not wait behind +// clones, but each is still a process. +const refListingsPerCPU = 4 + // New creates the cache, loading repositories already in CacheDir. func New(opts Options) (*Cache, error) { gitPath, err := exec.LookPath("git") @@ -186,6 +193,10 @@ func New(opts Options) (*Cache, error) { if opts.maxUploadPacks > 0 { uploadPacks = opts.maxUploadPacks } + refListings := refListingsPerCPU * runtime.GOMAXPROCS(0) + if opts.maxRefListings > 0 { + refListings = opts.maxRefListings + } c := &Cache{ opts: opts, client: &client, @@ -195,6 +206,7 @@ func New(opts Options) (*Cache, error) { lock: lock, fetches: make(chan struct{}, maxConcurrentFetches), uploadPacks: make(chan struct{}, uploadPacks), + refListings: make(chan struct{}, refListings), refCheckTimeout: defaultRefCheckTimeout, lowSpeedTime: defaultLowSpeedTime, maxRequestBody: defaultMaxRequestBody, diff --git a/internal/gitcache/helpers_test.go b/internal/gitcache/helpers_test.go index d5d82b7..55b7333 100644 --- a/internal/gitcache/helpers_test.go +++ b/internal/gitcache/helpers_test.go @@ -353,6 +353,15 @@ func testFront(c *Cache, upstreamURL string, fetchWants bool) http.Handler { http.Error(w, "not served", http.StatusForbidden) return } + var req *Request + if !infoRefs { + var err error + if req, err = c.ReadRequest(r); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + defer req.Close() + } lease, err := c.Open(r.Context(), "up/"+strings.TrimSuffix(repo, ".git")+".git", upstreamURL+"/"+repo, OpenOptions{Refresh: infoRefs, FetchWants: fetchWants}) switch { @@ -370,7 +379,7 @@ func testFront(c *Cache, upstreamURL string, fetchWants bool) http.Handler { if infoRefs { lease.ServeInfoRefs(w, r) } else { - lease.ServeUploadPack(w, r) + lease.ServeUploadPack(w, r, req) } }) } diff --git a/internal/gitcache/serve.go b/internal/gitcache/serve.go index 6d88df4..216edc1 100644 --- a/internal/gitcache/serve.go +++ b/internal/gitcache/serve.go @@ -7,6 +7,7 @@ import ( "errors" "io" "net/http" + "os" "path/filepath" "regexp" "slices" @@ -73,6 +74,11 @@ func (l *Lease) Release() { // ServeInfoRefs answers GET info/refs?service=git-upload-pack. func (l *Lease) ServeInfoRefs(w http.ResponseWriter, r *http.Request) { + c := l.cache + if !takeSlot(r.Context(), c.refListings) { + return + } + defer func() { <-c.refListings }() protocol := gitProtocol(r) setNoCache(w) w.Header().Set("Content-Type", advertisementType) @@ -81,34 +87,32 @@ func (l *Lease) ServeInfoRefs(w http.ResponseWriter, r *http.Request) { if !isProtocolV2(protocol) { _, _ = io.WriteString(w, pktLine("# service=git-upload-pack\n")+pktFlush) } - l.cache.uploadPack(w, r, l.repo, protocol, nil, "--advertise-refs") + c.uploadPack(w, r, l.repo, protocol, nil, "--advertise-refs") } -// ServeUploadPack answers POST git-upload-pack. With OpenOptions.FetchWants, -// commits the request wants that the cache lacks are fetched from the -// upstream first. -func (l *Lease) ServeUploadPack(w http.ResponseWriter, r *http.Request) { +// ServeUploadPack answers POST git-upload-pack with req, the request's body +// as ReadRequest read it. With OpenOptions.FetchWants, commits the request +// wants that the cache lacks are fetched from the upstream first. +func (l *Lease) ServeUploadPack(w http.ResponseWriter, r *http.Request, req *Request) { c := l.cache - // The raw body is limited too: a gzip stream of any length can decode - // to almost nothing. - raw := &bodyLimit{r: r.Body, remaining: c.maxRequestBody} - decoded := raw - if r.Header.Get("Content-Encoding") == "gzip" { - gz, err := gzip.NewReader(raw) + body := req.reader() + if req.gzip { + gz, err := gzip.NewReader(body) if err != nil { http.Error(w, "invalid gzip body", http.StatusBadRequest) return } defer func() { _ = gz.Close() }() - decoded = &bodyLimit{r: gz, remaining: c.maxRequestBody} + // A gzip stream of any length can decode to almost nothing. + decoded := &bodyLimit{r: gz, remaining: c.maxRequestBody} + defer func() { + if decoded.exceeded.Load() { + c.logger.Warn("git cache: request body passed the limit", + "repository", l.repo.key, "limit", c.maxRequestBody) + } + }() + body = decoded } - defer func() { - if raw.exceeded.Load() || decoded.exceeded.Load() { - c.logger.Warn("git cache: request body passed the limit", - "repository", l.repo.key, "limit", c.maxRequestBody) - } - }() - body := io.Reader(decoded) if l.fetchWants { var wants []string wants, body = readWants(body) @@ -119,20 +123,31 @@ func (l *Lease) ServeUploadPack(w http.ResponseWriter, r *http.Request) { } var command string command, body = requestCommand(body) - // ls-refs builds no pack, so it does not wait behind clones. - if command != "ls-refs" { - select { - case c.uploadPacks <- struct{}{}: - defer func() { <-c.uploadPacks }() - case <-r.Context().Done(): - return - } + // ls-refs builds no pack, so it waits for a ref listing slot rather than + // behind clones. + slots := c.uploadPacks + if command == "ls-refs" { + slots = c.refListings } + if !takeSlot(r.Context(), slots) { + return + } + defer func() { <-slots }() setNoCache(w) w.Header().Set("Content-Type", resultType) c.uploadPack(w, r, l.repo, gitProtocol(r), body) } +// takeSlot takes a slot from slots, or reports false once ctx is done. +func takeSlot(ctx context.Context, slots chan struct{}) bool { + select { + case slots <- struct{}{}: + return true + case <-ctx.Done(): + return false + } +} + // requestCommand returns the protocol v2 command an upload-pack request // starts with, such as "fetch", or "" for an earlier protocol, with a reader // that replays the whole body. @@ -175,7 +190,67 @@ func (c *Cache) uploadPack(w http.ResponseWriter, r *http.Request, rp *repo, pro // the request may run. const defaultMaxRequestBody = 100 << 20 -var errRequestTooLarge = errors.New("request body too large") +// maxMemoryRequest is the most of a request body kept in memory; the rest of +// a larger one goes to a temporary file. +const maxMemoryRequest = 1 << 20 + +// ErrRequestTooLarge is returned by ReadRequest for a body past the limit. +var ErrRequestTooLarge = errors.New("request body too large") + +// Request is the body of a POST git-upload-pack request, read in full +// before anything is started for it. Stateless requests are complete once +// sent, so a client that sends one slowly, or not at all, holds no +// upload-pack process and is bound by the server's read timeout. +type Request struct { + gzip bool + data []byte + file *os.File +} + +// ReadRequest reads the body of a POST git-upload-pack request, still +// encoded. A body past the limit fails with ErrRequestTooLarge. Close the +// request once it is served. +func (c *Cache) ReadRequest(r *http.Request) (*Request, error) { + req := &Request{gzip: r.Header.Get("Content-Encoding") == "gzip"} + body := &bodyLimit{r: r.Body, remaining: c.maxRequestBody} + var buf bytes.Buffer + if _, err := buf.ReadFrom(io.LimitReader(body, maxMemoryRequest)); err != nil { + return nil, err + } + if buf.Len() < maxMemoryRequest { + req.data = buf.Bytes() + return req, nil + } + file, err := os.CreateTemp("", "git-upload-pack-*") + if err != nil { + return nil, err + } + req.file = file + if _, err := buf.WriteTo(file); err != nil { + req.Close() + return nil, err + } + if _, err := io.Copy(file, body); err != nil { + req.Close() + return nil, err + } + return req, nil +} + +// Close removes any temporary file holding the request. +func (req *Request) Close() { + if req.file != nil { + _ = req.file.Close() + _ = os.Remove(req.file.Name()) + } +} + +func (req *Request) reader() io.Reader { + if req.file != nil { + return io.NewSectionReader(req.file, 0, 1<<63-1) + } + return bytes.NewReader(req.data) +} // bodyLimit fails reads past remaining bytes, where io.LimitReader would end // the body quietly and leave upload-pack a request cut short. @@ -187,7 +262,7 @@ type bodyLimit struct { func (b *bodyLimit) Read(p []byte) (int, error) { if b.exceeded.Load() { - return 0, errRequestTooLarge + return 0, ErrRequestTooLarge } // One byte past the limit tells a longer body from one that ends at it. if int64(len(p)) > b.remaining+1 { @@ -201,7 +276,7 @@ func (b *bodyLimit) Read(p []byte) (int, error) { n = int(b.remaining) b.remaining = 0 b.exceeded.Store(true) - return n, errRequestTooLarge + return n, ErrRequestTooLarge } func gitProtocol(r *http.Request) string { diff --git a/internal/gitcache/serve_test.go b/internal/gitcache/serve_test.go index b5779ae..50b8f75 100644 --- a/internal/gitcache/serve_test.go +++ b/internal/gitcache/serve_test.go @@ -3,9 +3,11 @@ package gitcache import ( "bytes" "compress/gzip" + "errors" "fmt" "io" "net/http" + "os" "os/exec" "path/filepath" "slices" @@ -169,6 +171,105 @@ func TestUploadPackSlots(t *testing.T) { } } +// TestRefListingSlots holds the only slot for listing refs: info/refs and +// ls-refs wait for it. +func TestRefListingSlots(t *testing.T) { + upstream := newFakeUpstream(t) + upstream.newRepo("repo") + tc := newTestCache(t, upstream.server.URL, func(o *Options) { o.maxRefListings = 1 }) + tc.clone(t, "repo") + + post := func(body string) <-chan error { + done := make(chan error, 1) + go func() { + req, _ := http.NewRequest(http.MethodPost, tc.url("repo")+"/git-upload-pack", strings.NewReader(body)) + req.Header.Set("Content-Type", requestType) + req.Header.Set("Git-Protocol", "version=2") + resp, err := http.DefaultClient.Do(req) + if err != nil { + done <- err + return + } + out, _ := io.ReadAll(resp.Body) + _ = resp.Body.Close() + if !strings.Contains(string(out), "refs/heads/main") { + err = fmt.Errorf("status %d, body %q", resp.StatusCode, out) + } + done <- err + }() + return done + } + get := func() <-chan error { + done := make(chan error, 1) + go func() { + resp, err := http.Get(tc.url("repo") + "/info/refs?service=git-upload-pack") + if err != nil { + done <- err + return + } + out, _ := io.ReadAll(resp.Body) + _ = resp.Body.Close() + if !strings.Contains(string(out), "refs/heads/main") { + err = fmt.Errorf("status %d, body %q", resp.StatusCode, out) + } + done <- err + }() + return done + } + + tc.refListings <- struct{}{} + held := true + release := func() { + if held { + held = false + <-tc.refListings + } + } + defer release() + lsRefs := post(pktLine("command=ls-refs\n") + pktDelim + pktLine("ref-prefix refs/heads/\n") + pktFlush) + infoRefs := get() + select { + case err := <-lsRefs: + t.Fatalf("ls-refs finished while the slot was held: %v", err) + case err := <-infoRefs: + t.Fatalf("info/refs finished while the slot was held: %v", err) + case <-time.After(time.Second): + } + release() + for name, done := range map[string]<-chan error{"ls-refs": lsRefs, "info/refs": infoRefs} { + if err := <-done; err != nil { + t.Errorf("%s: %v", name, err) + } + } +} + +// TestReadRequest reads bodies kept in memory and, past maxMemoryRequest, in +// a temporary file removed on Close. +func TestReadRequest(t *testing.T) { + c := &Cache{maxRequestBody: 4 << 20} + for _, size := range []int{0, 100, maxMemoryRequest, 3 << 20} { + body := strings.Repeat("x", size) + r, _ := http.NewRequest(http.MethodPost, "/", strings.NewReader(body)) + req, err := c.ReadRequest(r) + if err != nil { + t.Fatalf("%d bytes: %v", size, err) + } + if got, _ := io.ReadAll(req.reader()); string(got) != body { + t.Errorf("%d bytes: read back %d", size, len(got)) + } + req.Close() + if req.file != nil { + if _, err := os.Stat(req.file.Name()); !errors.Is(err, os.ErrNotExist) { + t.Errorf("%d bytes: temporary file left: %v", size, err) + } + } + } + r, _ := http.NewRequest(http.MethodPost, "/", strings.NewReader(strings.Repeat("x", 5<<20))) + if _, err := c.ReadRequest(r); !errors.Is(err, ErrRequestTooLarge) { + t.Errorf("body past the limit: %v, want ErrRequestTooLarge", err) + } +} + func TestRequestCommand(t *testing.T) { for _, tt := range []struct{ body, want string }{ {pktLine("command=ls-refs\n") + pktDelim + pktFlush, "ls-refs"}, diff --git a/internal/handler/git.go b/internal/handler/git.go index 3fbacfd..895be44 100644 --- a/internal/handler/git.go +++ b/internal/handler/git.go @@ -21,10 +21,14 @@ const ( gitRequestType = "application/x-git-upload-pack-request" // gitRequestTimeout replaces the server's read and write timeouts for Git - // requests: a cold clone of a large repository keeps the client waiting - // before the first byte, a large pack takes a while to send, and the rest - // of a request body is read only after any fetch it needs. + // requests once any request body has been read: a cold clone of a large + // repository keeps the client waiting before the first byte, and a large + // pack takes a while to send. gitRequestTimeout = 2 * time.Hour + // gitWriteStallTimeout bounds each write of a Git response, so a client + // that stops reading does not hold an upload-pack process for + // gitRequestTimeout. + gitWriteStallTimeout = 2 * time.Minute ) // GitHandler serves Git repositories read-only over smart HTTP at @@ -35,6 +39,8 @@ type GitHandler struct { proxy *Proxy cache *gitcache.Cache upstreams map[string]gitUpstream + // writeStall is gitWriteStallTimeout, shorter in tests. + writeStall time.Duration } type gitUpstream struct { @@ -49,7 +55,12 @@ type gitUpstream struct { // NewGitHandler creates a Git handler serving upstream.git from cache. func NewGitHandler(proxy *Proxy, cache *gitcache.Cache, upstream config.UpstreamConfig) *GitHandler { - h := &GitHandler{proxy: proxy, cache: cache, upstreams: make(map[string]gitUpstream, len(upstream.Git))} + h := &GitHandler{ + proxy: proxy, + cache: cache, + upstreams: make(map[string]gitUpstream, len(upstream.Git)), + writeStall: gitWriteStallTimeout, + } for name, git := range upstream.Git { git.URL = strings.TrimSuffix(git.URL, "/") h.upstreams[name] = gitUpstream{ @@ -84,19 +95,28 @@ func (h *GitHandler) serve(w http.ResponseWriter, r *http.Request) { http.Error(w, msg, status) return } - deadline := time.Now().Add(gitRequestTimeout) + // An upload-pack request is read in full first, within the server's read + // timeout, so nothing waits on a client that is slow to send one. + var body *gitcache.Request + if !req.infoRefs { + var err error + if body, err = h.cache.ReadRequest(r); err != nil { + h.serveRequestError(w, req, err) + return + } + defer body.Close() + } + // The read deadline is extended too: once the body is read, an HTTP/1 + // server watches the connection, and a read timeout there would cancel + // the request. rc := http.NewResponseController(w) + deadline := time.Now().Add(gitRequestTimeout) if err := errors.Join(rc.SetReadDeadline(deadline), rc.SetWriteDeadline(deadline)); err != nil { h.proxy.Logger.Warn("git: cannot extend the request deadlines", "error", err) } - // upload-pack writes its response while its stdin is still attached to - // the request body. Without full duplex, an HTTP/1 server reads off and - // closes whatever body is left once the response is flushed. - if !req.infoRefs { - if err := rc.EnableFullDuplex(); err != nil { - h.proxy.Logger.Warn("git: cannot enable full duplex", "error", err) - } - } + w = &stallWriter{ResponseWriter: w, rc: rc, stall: h.writeStall, deadline: deadline} + // What is left buffered once the handler returns is bounded as well. + defer func() { _ = rc.SetWriteDeadline(time.Now().Add(h.writeStall)) }() lease, err := h.cache.Open(r.Context(), req.key, req.repoURL, gitcache.OpenOptions{Refresh: req.infoRefs, FetchWants: req.fetchWants}) @@ -106,7 +126,7 @@ func (h *GitHandler) serve(w http.ResponseWriter, r *http.Request) { } defer lease.Release() if !req.infoRefs { - lease.ServeUploadPack(w, r) + lease.ServeUploadPack(w, r, body) return } if lease.Hit() { @@ -194,6 +214,37 @@ func validGitSegment(segment string, last bool) bool { return true } +// serveRequestError answers an upload-pack request whose body could not be +// read. +func (h *GitHandler) serveRequestError(w http.ResponseWriter, req gitRequest, err error) { + if errors.Is(err, gitcache.ErrRequestTooLarge) { + h.proxy.Logger.Warn("git: request body passed the limit", "repository", req.key) + http.Error(w, "request body too large", http.StatusRequestEntityTooLarge) + return + } + http.Error(w, "cannot read the request body", http.StatusBadRequest) +} + +// stallWriter bounds each write of a response by stall, where the response +// as a whole may take until deadline. +type stallWriter struct { + http.ResponseWriter + rc *http.ResponseController + stall time.Duration + deadline time.Time +} + +func (w *stallWriter) Write(p []byte) (int, error) { + _ = w.rc.SetWriteDeadline(time.Now().Add(w.stall)) + n, err := w.ResponseWriter.Write(p) + _ = w.rc.SetWriteDeadline(w.deadline) + return n, err +} + +func (w *stallWriter) Unwrap() http.ResponseWriter { + return w.ResponseWriter +} + // serveError answers a request whose repository could not be opened. The // cache has already logged why. func (h *GitHandler) serveError(w http.ResponseWriter, r *http.Request, err error) { diff --git a/internal/handler/git_test.go b/internal/handler/git_test.go index ec0b630..0aeff4b 100644 --- a/internal/handler/git_test.go +++ b/internal/handler/git_test.go @@ -1,9 +1,12 @@ package handler import ( + "bufio" "fmt" "io" "log/slog" + "math/rand/v2" + "net" "net/http" "net/http/cgi" "net/http/httptest" @@ -131,9 +134,13 @@ type gitTestSetup struct { logs *syncLog // slow delays every upstream request by half a second. slow atomic.Bool + // active counts the requests the handler is serving. + active atomic.Int32 } -func newGitTestSetup(t *testing.T, readTimeout time.Duration) *gitTestSetup { +// newGitTestSetup serves the cache with the given server read timeout and, +// unless zero, GitHandler.writeStall. +func newGitTestSetup(t *testing.T, readTimeout, writeStall time.Duration) *gitTestSetup { t.Helper() execPath := runGitCommand(t, "", "--exec-path") backendPath := "" @@ -180,7 +187,15 @@ func newGitTestSetup(t *testing.T, readTimeout time.Duration) *gitTestSetup { } t.Cleanup(func() { _ = cache.Close() }) h := NewGitHandler(&Proxy{Logger: logger}, cache, config.UpstreamConfig{Git: map[string]config.GitUpstream{"up": {URL: upstream.URL}}}) - g.server = httptest.NewUnstartedServer(http.StripPrefix("/git", h.Routes())) + if writeStall > 0 { + h.writeStall = writeStall + } + routes := http.StripPrefix("/git", h.Routes()) + g.server = httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + g.active.Add(1) + defer g.active.Add(-1) + routes.ServeHTTP(w, r) + })) g.server.Config.ReadTimeout = readTimeout g.server.Start() t.Cleanup(g.server.Close) @@ -212,7 +227,7 @@ func (l *syncLog) String() string { // whose haves come after a want the cache must fetch first, so upload-pack // reads them after the server's read timeout has passed. func TestGitRequestBodyOutlastsServerReadTimeout(t *testing.T) { - g := newGitTestSetup(t, 200*time.Millisecond) + g := newGitTestSetup(t, 200*time.Millisecond, 0) runGitCommand(t, g.work, "commit", "--quiet", "--allow-empty", "-m", "pull request") runGitCommand(t, g.work, "push", "--quiet", "origin", "HEAD:refs/pull/1/head") sha := runGitCommand(t, g.work, "rev-parse", "HEAD") @@ -237,6 +252,76 @@ func TestGitRequestBodyOutlastsServerReadTimeout(t *testing.T) { } } +// TestGitSlowRequestBodyStartsNothing sends part of a request body and then +// nothing. The request ends at the server's read timeout, without an +// upload-pack waiting for the rest. +func TestGitSlowRequestBodyStartsNothing(t *testing.T) { + g := newGitTestSetup(t, 200*time.Millisecond, 0) + runGitCommand(t, "", "clone", "--quiet", g.url(), filepath.Join(t.TempDir(), "clone")) + + conn, err := net.Dial("tcp", g.server.Listener.Addr().String()) + if err != nil { + t.Fatal(err) + } + defer func() { _ = conn.Close() }() + part := pktLine("command=ls-refs\n") + _, err = fmt.Fprintf(conn, "POST /git/up/repo/git-upload-pack HTTP/1.1\r\nHost: proxy\r\n"+ + "Content-Type: %s\r\nGit-Protocol: version=2\r\nContent-Length: 100\r\n\r\n%s", gitRequestType, part) + if err != nil { + t.Fatal(err) + } + _ = conn.SetReadDeadline(time.Now().Add(10 * time.Second)) + resp, err := http.ReadResponse(bufio.NewReader(conn), nil) + if err != nil { + t.Fatalf("no answer to a request whose body stopped: %v", err) + } + _ = resp.Body.Close() + if resp.StatusCode != http.StatusBadRequest { + t.Errorf("status %d, want %d", resp.StatusCode, http.StatusBadRequest) + } +} + +// TestGitStalledClientIsDropped fetches a pack larger than the connection +// buffers and never reads it. The request ends once a write has waited +// writeStall, rather than holding upload-pack. +func TestGitStalledClientIsDropped(t *testing.T) { + g := newGitTestSetup(t, 0, 200*time.Millisecond) + data := make([]byte, 16<<20) + _, _ = rand.NewChaCha8([32]byte{}).Read(data) + if err := os.WriteFile(filepath.Join(g.work, "large"), data, 0o600); err != nil { + t.Fatal(err) + } + runGitCommand(t, g.work, "add", "large") + runGitCommand(t, g.work, "commit", "--quiet", "-m", "large") + runGitCommand(t, g.work, "push", "--quiet", "origin", "HEAD:main") + sha := runGitCommand(t, g.work, "rev-parse", "HEAD") + runGitCommand(t, "", "clone", "--quiet", g.url(), filepath.Join(t.TempDir(), "clone")) + + conn, err := net.Dial("tcp", g.server.Listener.Addr().String()) + if err != nil { + t.Fatal(err) + } + defer func() { _ = conn.Close() }() + body := pktLine("want "+sha+" side-band-64k\n") + "0000" + pktLine("done\n") + _, err = fmt.Fprintf(conn, "POST /git/up/repo/git-upload-pack HTTP/1.1\r\nHost: proxy\r\n"+ + "Content-Type: %s\r\nContent-Length: %d\r\n\r\n%s", gitRequestType, len(body), body) + if err != nil { + t.Fatal(err) + } + for deadline := time.Now().Add(10 * time.Second); g.active.Load() == 0; { + if time.Now().After(deadline) { + t.Fatal("request never reached the handler") + } + time.Sleep(10 * time.Millisecond) + } + for deadline := time.Now().Add(10 * time.Second); g.active.Load() != 0; { + if time.Now().After(deadline) { + t.Fatal("the handler still serves a client that stopped reading") + } + time.Sleep(10 * time.Millisecond) + } +} + func pktLine(payload string) string { return fmt.Sprintf("%04x%s", len(payload)+4, payload) } From 09e41d0272c34e9433d22cf265e46995960041ba Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sat, 24 Oct 2026 08:11:25 +0200 Subject: [PATCH 399/633] Fail closed in gem, Hex and PyPI cooldown RubyGems, Hex and PyPI keep publish times outside the metadata clients read, so cooldown fetches them from a second API. When that fetch failed, was rate limited, or returned something that didn't parse, the compact index, Hex package or simple page was served unfiltered, and a Hex package that couldn't be re-encoded was passed on as upstream sent it. A flaky or rate-limited timestamp API let every version in cooldown through. These now fail with 502, or with 503 and Retry-After when the rate limit gate held the request, as the Go handler already does. PyPI file downloads check the same way. A PyPI package the JSON API answers 404 for still has no times and is served unfiltered, as before. npm, Composer, pub.dev and PyPI JSON metadata that couldn't be rewritten was also passed on unfiltered unless the package had denied versions. It now fails with 502 whenever cooldown applies too. npm tarball downloads whose packument couldn't be fetched skipped the cooldown check and downloaded straight from upstream; they now fail. All of this only applies when cooldown covers the package: packages exempted with a duration of 0 skip the timestamp lookups, and the gem and Hex handlers serve them through the metadata cache as when cooldown is off. --- docs/configuration.md | 2 + internal/handler/composer.go | 5 + internal/handler/cooldown_fail_closed.go | 34 +++ internal/handler/cooldown_fail_closed_test.go | 271 ++++++++++++++++++ internal/handler/gem.go | 33 +-- internal/handler/gem_test.go | 12 +- internal/handler/hex.go | 39 ++- internal/handler/npm.go | 46 +-- internal/handler/npm_test.go | 4 +- internal/handler/pub.go | 5 + internal/handler/pypi.go | 65 +++-- 11 files changed, 424 insertions(+), 92 deletions(-) create mode 100644 internal/handler/cooldown_fail_closed.go create mode 100644 internal/handler/cooldown_fail_closed_test.go diff --git a/docs/configuration.md b/docs/configuration.md index 4915649..4712d69 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1042,6 +1042,8 @@ Maven caveats: - Maven Central lists versions in version order, not in the order they were published. A patch to an older release line published after a newer release that is out of cooldown stays listed while it is in cooldown, but its files are still refused. - SNAPSHOT versions are not held back. +RubyGems, Hex and PyPI keep their publish times outside the metadata clients read: the RubyGems versions API, the Hex HTTP API (`upstream.hex_api`) and the PyPI JSON API. The proxy fetches them alongside each compact index, package or simple page it filters, and for PyPI downloads. Cooldown fails closed: when it applies to a package and those times can't be fetched or parsed, or the metadata can't be rewritten, the request fails with 502, or with 503 and `Retry-After` while the upstream is rate limiting the proxy, instead of serving the metadata unfiltered. The same goes for npm, Composer and pub.dev metadata that can't be rewritten, and for npm tarball downloads whose publish time can't be read. Packages exempted with a duration of `0` are served as before and don't need the times. A PyPI package the JSON API returns 404 for has no times to go by and is served unfiltered. + With cooldown on, the proxy asks npm for full packuments, since the abbreviated ones it uses otherwise carry no publish times. Abbreviated packuments cached before cooldown was turned on are fetched again in full rather than served, including as a stale copy when the upstream is down. Conda cooldown covers `repodata.json` and sharded repodata (CEP 16). Clients cache shards by the hash in the shard index, so with cooldown on the proxy serves a rewritten index whose shard hashes change every hour and filters each shard as it is fetched. Versions leaving cooldown show up within the hour, and clients fetch the shards they need again once an hour. If a channel's shard index points packages or shards somewhere the proxy doesn't route, the proxy answers 404 for it and clients fall back to `repodata.json`. diff --git a/internal/handler/composer.go b/internal/handler/composer.go index d0c59ff..92eae0a 100644 --- a/internal/handler/composer.go +++ b/internal/handler/composer.go @@ -188,6 +188,11 @@ func (h *ComposerHandler) handlePackageMetadata(w http.ResponseWriter, r *http.R if r.Context().Err() != nil { return // the client left while waiting on a shared rewrite } + if h.proxy.mustFilterMetadata("composer", basePackage) { + h.proxy.Logger.Error("failed to filter composer metadata", "package", basePackage, "error", err) + http.Error(w, "failed to filter package metadata", http.StatusBadGateway) + return + } h.proxy.Logger.Warn("failed to rewrite metadata, proxying original", "error", err) markStale(w, upstream.stale) w.Header().Set(headerContentType, "application/json") diff --git a/internal/handler/cooldown_fail_closed.go b/internal/handler/cooldown_fail_closed.go new file mode 100644 index 0000000..3a0a7f4 --- /dev/null +++ b/internal/handler/cooldown_fail_closed.go @@ -0,0 +1,34 @@ +package handler + +import "net/http" + +// Metadata for a package that cooldown or the denylist applies to is never +// served unfiltered. When the publish times can't be read, or the document +// can't be rewritten, the request fails instead, so a rate-limited or +// failing upstream can't let withheld versions through. + +// cooldownApplies reports whether cooldown withholds new versions of the +// package, taking package overrides and exemptions into account. +func (p *Proxy) cooldownApplies(ecosystem, name string) bool { + return p.cooldownEnabled() && p.Cooldown.For(ecosystem, canonicalPackagePURL(ecosystem, name)) > 0 +} + +// mustFilterMetadata reports whether the package's metadata may have +// versions to withhold, by cooldown or by the denylist, and so must not be +// served as upstream sent it. +func (p *Proxy) mustFilterMetadata(ecosystem, name string) bool { + return p.cooldownApplies(ecosystem, name) || + len(p.Denylist.Versions(canonicalPackagePURL(ecosystem, name))) != 0 +} + +// writeCooldownLookupError answers a metadata request whose publish times +// couldn't be read: 503 with Retry-After when the rate limit gate held the +// request, 502 otherwise. +func (p *Proxy) writeCooldownLookupError(w http.ResponseWriter, ecosystem, name string, err error) { + if writeUpstreamHeld(w, err) { + return + } + p.Logger.Warn("cooldown: could not read publish times", + "ecosystem", ecosystem, "package", name, "error", err) + http.Error(w, "failed to check cooldown", http.StatusBadGateway) +} diff --git a/internal/handler/cooldown_fail_closed_test.go b/internal/handler/cooldown_fail_closed_test.go new file mode 100644 index 0000000..ae3ae63 --- /dev/null +++ b/internal/handler/cooldown_fail_closed_test.go @@ -0,0 +1,271 @@ +package handler + +import ( + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/git-pkgs/cooldown" + registryclient "github.com/git-pkgs/registries/client" + "github.com/git-pkgs/registries/fetch" +) + +// holdPaths returns a client that answers requests for the given paths with +// the rate limit gate's HeldError and sends the rest to upstream. +func holdPaths(paths ...string) *http.Client { + held := ®istryclient.HeldError{Host: "upstream.example", Until: time.Now().Add(30 * time.Second), RetryAfter: 30 * time.Second} + return &http.Client{Transport: roundTripperFunc(func(r *http.Request) (*http.Response, error) { + for _, p := range paths { + if r.URL.Path == p { + return nil, held + } + } + return http.DefaultTransport.RoundTrip(r) + })} +} + +func serveGet(t *testing.T, h http.Handler, path string) *httptest.ResponseRecorder { + t.Helper() + w := httptest.NewRecorder() + h.ServeHTTP(w, httptest.NewRequest(http.MethodGet, path, nil)) + return w +} + +func wantStatus(t *testing.T, w *httptest.ResponseRecorder, status int, leaked string) { + t.Helper() + if w.Code != status { + t.Errorf("status = %d, want %d: %s", w.Code, status, w.Body.String()) + } + if status == http.StatusServiceUnavailable && w.Header().Get("Retry-After") == "" { + t.Error("503 without Retry-After") + } + if leaked != "" && strings.Contains(w.Body.String(), leaked) { + t.Errorf("response lists %q, which cooldown should have withheld", leaked) + } +} + +func TestGemCooldownFailsClosed(t *testing.T) { + index := "---\n1.0.0 |checksum:abc\n2.0.0 |checksum:def\n" + versionsStatus := http.StatusInternalServerError + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if strings.HasPrefix(r.URL.Path, "/api/v1/versions/") { + w.WriteHeader(versionsStatus) + return + } + _, _ = io.WriteString(w, index) + })) + defer upstream.Close() + + t.Run("versions API fails", func(t *testing.T) { + proxy, _, _, _ := setupTestProxy(t) + proxy.Cooldown = &cooldown.Config{Default: "3d"} + h := NewGemHandlerWithUpstream(proxy, "http://proxy.test", upstream.URL) + wantStatus(t, serveGet(t, h.Routes(), "/info/testgem"), http.StatusBadGateway, "2.0.0") + }) + + t.Run("versions API held by rate limit", func(t *testing.T) { + proxy, _, _, _ := setupTestProxy(t) + proxy.Cooldown = &cooldown.Config{Default: "3d"} + proxy.HTTPClient = holdPaths("/api/v1/versions/testgem.json") + h := NewGemHandlerWithUpstream(proxy, "http://proxy.test", upstream.URL) + wantStatus(t, serveGet(t, h.Routes(), "/info/testgem"), http.StatusServiceUnavailable, "2.0.0") + }) + + t.Run("exempt package needs no times", func(t *testing.T) { + proxy, _, _, _ := setupTestProxy(t) + proxy.Cooldown = &cooldown.Config{Default: "3d", Packages: map[string]string{"pkg:gem/testgem": "0"}} + h := NewGemHandlerWithUpstream(proxy, "http://proxy.test", upstream.URL) + w := serveGet(t, h.Routes(), "/info/testgem") + if w.Code != http.StatusOK || w.Body.String() != index { + t.Errorf("status = %d, body = %q; want the index unchanged", w.Code, w.Body.String()) + } + }) +} + +func TestHexCooldownFailsClosed(t *testing.T) { + now := time.Now() + proto := buildHexSigned(buildHexPackage("testpkg", []string{testVersion100, "2.0.0"})) + apiJSON, _ := json.Marshal(hexPackageAPI{Releases: []hexRelease{ + {Version: testVersion100, InsertedAt: now.Add(-7 * 24 * time.Hour).Format(time.RFC3339Nano)}, + {Version: "2.0.0", InsertedAt: now.Add(-time.Hour).Format(time.RFC3339Nano)}, + }}) + + newUpstream := func(t *testing.T, packageBody []byte, apiStatus int) *httptest.Server { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/packages/testpkg": + _, _ = w.Write(packageBody) + case "/api/packages/testpkg": + w.WriteHeader(apiStatus) + _, _ = w.Write(apiJSON) + default: + w.WriteHeader(http.StatusNotFound) + } + })) + t.Cleanup(srv.Close) + return srv + } + handler := func(t *testing.T, srv *httptest.Server, client *http.Client) http.Handler { + proxy, _, _, _ := setupTestProxy(t) + proxy.Cooldown = &cooldown.Config{Default: "3d"} + if client != nil { + proxy.HTTPClient = client + } + return NewHexHandlerWithUpstreams(proxy, "http://proxy.test", srv.URL, srv.URL).Routes() + } + + t.Run("API fails", func(t *testing.T) { + srv := newUpstream(t, proto, http.StatusInternalServerError) + wantStatus(t, serveGet(t, handler(t, srv, nil), "/packages/testpkg"), http.StatusBadGateway, "") + }) + + t.Run("API held by rate limit", func(t *testing.T) { + srv := newUpstream(t, proto, http.StatusOK) + w := serveGet(t, handler(t, srv, holdPaths("/api/packages/testpkg")), "/packages/testpkg") + wantStatus(t, w, http.StatusServiceUnavailable, "") + }) + + t.Run("package can't be filtered", func(t *testing.T) { + srv := newUpstream(t, []byte("not gzip 2.0.0"), http.StatusOK) + wantStatus(t, serveGet(t, handler(t, srv, nil), "/packages/testpkg"), http.StatusBadGateway, "2.0.0") + }) + + t.Run("filtered", func(t *testing.T) { + srv := newUpstream(t, proto, http.StatusOK) + w := serveGet(t, handler(t, srv, nil), "/packages/testpkg") + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want 200", w.Code) + } + }) +} + +func TestPyPICooldownFailsClosed(t *testing.T) { + simple := `{"meta":{"api-version":"1.4"},"name":"ruff","files":[` + + `{"filename":"ruff-1.0.0.tar.gz","url":"https://files.pythonhosted.org/packages/ab/ruff-1.0.0.tar.gz"},` + + `{"filename":"ruff-2.0.0.tar.gz","url":"https://files.pythonhosted.org/packages/cd/ruff-2.0.0.tar.gz"}]}` + + tests := []struct { + name string + json func(*http.Request) (*http.Response, error) + status int + }{ + {"JSON API fails", func(r *http.Request) (*http.Response, error) { + return &http.Response{StatusCode: http.StatusInternalServerError, Body: http.NoBody, Request: r}, nil + }, http.StatusBadGateway}, + {"JSON API held by rate limit", func(*http.Request) (*http.Response, error) { + return nil, ®istryclient.HeldError{Host: "pypi.test", RetryAfter: 30 * time.Second} + }, http.StatusServiceUnavailable}, + {"JSON has no releases", func(r *http.Request) (*http.Response, error) { + return pypiHTTPResponse(r, "application/json", `{"info":{}}`), nil + }, http.StatusBadGateway}, + {"JSON API doesn't know the package", func(r *http.Request) (*http.Response, error) { + return &http.Response{StatusCode: http.StatusNotFound, Body: http.NoBody, Request: r}, nil + }, http.StatusOK}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + h, proxy := setupPyPIHandler(t, func(r *http.Request) (*http.Response, error) { + if r.URL.Path == "/pypi/ruff/json" { + return tt.json(r) + } + return pypiHTTPResponse(r, pypiSimpleJSON, simple), nil + }) + proxy.Cooldown = &cooldown.Config{Default: "7d"} + + req := httptest.NewRequest(http.MethodGet, "/simple/ruff/", nil) + req.Header.Set("Accept", uvPyPIAccept) + w := httptest.NewRecorder() + h.Routes().ServeHTTP(w, req) + leaked := "ruff-2.0.0" + if tt.status == http.StatusOK { + leaked = "" + } + wantStatus(t, w, tt.status, leaked) + }) + } +} + +func TestPyPIDownloadCooldownFailsClosed(t *testing.T) { + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + })) + defer upstream.Close() + + proxy, _, _, fetcher := setupTestProxy(t) + proxy.Cooldown = &cooldown.Config{Default: "7d"} + fetcher.artifact = &fetch.Artifact{Body: io.NopCloser(strings.NewReader("sdist")), ContentType: "application/octet-stream"} + h := &PyPIHandler{proxy: proxy, upstreamURL: upstream.URL, proxyURL: "http://localhost"} + + w := serveGet(t, h.Routes(), "/packages/packages/ab/cd/ef0123456789/newpkg-2.0.0.tar.gz") + wantStatus(t, w, http.StatusBadGateway, "") + if fetcher.fetchCalled { + t.Error("fetched a file whose publish time couldn't be checked") + } +} + +func TestNPMDownloadCooldownFailsClosed(t *testing.T) { + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + })) + defer upstream.Close() + + proxy, _, _, fetcher := setupTestProxy(t) + proxy.Cooldown = &cooldown.Config{Default: "7d"} + fetcher.artifact = &fetch.Artifact{Body: io.NopCloser(strings.NewReader("tarball")), ContentType: "application/octet-stream"} + h := NewNPMHandler(proxy, "http://proxy.test", upstream.URL) + + w := serveGet(t, h.Routes(), "/leftpad/-/leftpad-2.0.0.tgz") + wantStatus(t, w, http.StatusBadGateway, "") + if fetcher.fetchCalled { + t.Error("fetched a tarball whose publish time couldn't be checked") + } +} + +// TestCooldownRewriteFailureFailsClosed checks that a metadata document the +// cooldown rewrite can't parse is refused rather than passed on, and still +// passed on when cooldown doesn't apply. +func TestCooldownRewriteFailureFailsClosed(t *testing.T) { + const broken = `{"name": "pkg", "versions": [ 2.0.0` + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set(headerContentType, contentTypeJSON) + _, _ = io.WriteString(w, broken) + })) + defer upstream.Close() + + tests := []struct { + name string + purl string + handler func(*Proxy) http.Handler + path string + }{ + {"npm", "pkg:npm/pkg", func(p *Proxy) http.Handler { return NewNPMHandler(p, "http://proxy.test", upstream.URL).Routes() }, "/pkg"}, + {"composer", "pkg:composer/vendor/pkg", func(p *Proxy) http.Handler { + return NewComposerHandlerWithUpstreams(p, "http://proxy.test", upstream.URL, upstream.URL).Routes() + }, "/p2/vendor/pkg.json"}, + {"pub", "pkg:pub/pkg", func(p *Proxy) http.Handler { + return NewPubHandlerWithUpstream(p, "http://proxy.test", upstream.URL).Routes() + }, "/api/packages/pkg"}, + {"pypi json", "pkg:pypi/pkg", func(p *Proxy) http.Handler { + return NewPyPIHandlerWithUpstreams(p, "http://proxy.test", upstream.URL, "").Routes() + }, "/pypi/pkg/json"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + proxy, _, _, _ := setupTestProxy(t) + proxy.Cooldown = &cooldown.Config{Default: "7d"} + wantStatus(t, serveGet(t, tt.handler(proxy), tt.path), http.StatusBadGateway, "2.0.0") + }) + t.Run(tt.name+" exempt", func(t *testing.T) { + proxy, _, _, _ := setupTestProxy(t) + proxy.Cooldown = &cooldown.Config{Default: "7d", Packages: map[string]string{tt.purl: "0"}} + w := serveGet(t, tt.handler(proxy), tt.path) + if w.Code != http.StatusOK { + t.Errorf("status = %d, want 200 for a package cooldown doesn't apply to", w.Code) + } + }) + } +} diff --git a/internal/handler/gem.go b/internal/handler/gem.go index 13ea528..b2ff638 100644 --- a/internal/handler/gem.go +++ b/internal/handler/gem.go @@ -145,20 +145,19 @@ func gemVersionAndPlatform(s string) bool { // handleCompactIndex serves the compact index for a gem, filtering versions // based on cooldown when enabled. func (h *GemHandler) handleCompactIndex(w http.ResponseWriter, r *http.Request) { - if h.proxy.Cooldown == nil || !h.proxy.Cooldown.Enabled() { - h.proxyCached(w, r) - return - } - name := r.PathValue("name") if name == "" { http.Error(w, "invalid gem name", http.StatusBadRequest) return } + if !h.proxy.cooldownApplies("gem", name) { + h.proxyCached(w, r) + return + } h.proxy.Logger.Info("gem compact index request with cooldown", "name", name) - indexResp, filteredVersions, err := h.fetchIndexAndVersions(r, name) + indexResp, filteredVersions, versionsErr, err := h.fetchIndexAndVersions(r, name) if err != nil { if writeUpstreamHeld(w, err) { return @@ -174,19 +173,18 @@ func (h *GemHandler) handleCompactIndex(w http.ResponseWriter, r *http.Request) return } - if filteredVersions == nil { - h.proxy.Logger.Warn("failed to fetch version timestamps, proxying unfiltered", "name", name) - h.proxy.relayResponse(w, r, indexResp, nil) + if versionsErr != nil { + h.proxy.writeCooldownLookupError(w, "gem", name, versionsErr) return } h.writeFilteredIndex(w, indexResp, name, filteredVersions) } -// fetchIndexAndVersions fetches the compact index and versions API concurrently. -// Returns the index response, a set of versions to filter (nil if versions API failed), -// and an error if the index fetch itself failed. -func (h *GemHandler) fetchIndexAndVersions(r *http.Request, name string) (*http.Response, map[string]bool, error) { +// fetchIndexAndVersions fetches the compact index and versions API +// concurrently. It returns the index response and the set of versions to +// filter, the versions API's error, and the index fetch's error. +func (h *GemHandler) fetchIndexAndVersions(r *http.Request, name string) (*http.Response, map[string]bool, error, error) { type versionsResult struct { filtered map[string]bool err error @@ -203,14 +201,9 @@ func (h *GemHandler) fetchIndexAndVersions(r *http.Request, name string) (*http. versionsRes := <-versionsCh if err != nil { - return nil, nil, err + return nil, nil, nil, err } - - if versionsRes.err != nil { - return indexResp, nil, nil - } - - return indexResp, versionsRes.filtered, nil + return indexResp, versionsRes.filtered, versionsRes.err, nil } // fetchCompactIndex fetches the compact index from upstream. diff --git a/internal/handler/gem_test.go b/internal/handler/gem_test.go index a615fad..30470bc 100644 --- a/internal/handler/gem_test.go +++ b/internal/handler/gem_test.go @@ -342,14 +342,12 @@ func TestGemCompactIndexVersionsAPIFails(t *testing.T) { w := httptest.NewRecorder() h.handleCompactIndex(w, req) - // Should still return OK with unfiltered content - if w.Code != http.StatusOK { - t.Fatalf("status = %d, want %d", w.Code, http.StatusOK) + // Without publish times the index can't be filtered, so it isn't served. + if w.Code != http.StatusBadGateway { + t.Fatalf("status = %d, want %d", w.Code, http.StatusBadGateway) } - - body := w.Body.String() - if !strings.Contains(body, "1.0.0") { - t.Error("expected unfiltered content when versions API fails") + if strings.Contains(w.Body.String(), "1.0.0") { + t.Error("served the unfiltered index when the versions API failed") } } diff --git a/internal/handler/hex.go b/internal/handler/hex.go index 847f955..5b8e91e 100644 --- a/internal/handler/hex.go +++ b/internal/handler/hex.go @@ -103,20 +103,15 @@ const hexAPIURL = "https://hex.pm" // when enabled. Since the protobuf format has no timestamps, we fetch them from the // Hex HTTP API concurrently. func (h *HexHandler) handlePackages(w http.ResponseWriter, r *http.Request) { - if h.proxy.Cooldown == nil || !h.proxy.Cooldown.Enabled() { - h.proxyCached(w, r) - return - } - name := r.PathValue("name") - if name == "" { + if name == "" || !h.proxy.cooldownApplies("hex", name) { h.proxyCached(w, r) return } h.proxy.Logger.Info("hex package request with cooldown", "name", name) - protoResp, filteredVersions, err := h.fetchPackageAndVersions(r, name) + protoResp, filteredVersions, versionsErr, err := h.fetchPackageAndVersions(r, name) if err != nil { if writeUpstreamHeld(w, err) { return @@ -132,6 +127,11 @@ func (h *HexHandler) handlePackages(w http.ResponseWriter, r *http.Request) { return } + if versionsErr != nil { + h.proxy.writeCooldownLookupError(w, "hex", name, versionsErr) + return + } + body, err := io.ReadAll(protoResp.Body) if err != nil { http.Error(w, "failed to read response", http.StatusInternalServerError) @@ -139,7 +139,7 @@ func (h *HexHandler) handlePackages(w http.ResponseWriter, r *http.Request) { } if len(filteredVersions) == 0 { - // No versions to filter or couldn't get timestamps, pass through + // No versions in cooldown, pass through w.Header().Set(headerContentType, protoResp.Header.Get(headerContentType)) w.Header().Set("Content-Encoding", "gzip") _, _ = w.Write(body) @@ -148,10 +148,8 @@ func (h *HexHandler) handlePackages(w http.ResponseWriter, r *http.Request) { filtered, err := h.filterSignedPackage(body, filteredVersions) if err != nil { - h.proxy.Logger.Warn("failed to filter hex package, proxying original", "error", err) - w.Header().Set(headerContentType, protoResp.Header.Get(headerContentType)) - w.Header().Set("Content-Encoding", "gzip") - _, _ = w.Write(body) + h.proxy.Logger.Error("failed to filter hex package", "name", name, "error", err) + http.Error(w, "failed to filter package metadata", http.StatusBadGateway) return } @@ -160,8 +158,10 @@ func (h *HexHandler) handlePackages(w http.ResponseWriter, r *http.Request) { _, _ = w.Write(filtered) } -// fetchPackageAndVersions fetches the protobuf package and version timestamps concurrently. -func (h *HexHandler) fetchPackageAndVersions(r *http.Request, name string) (*http.Response, map[string]bool, error) { +// fetchPackageAndVersions fetches the protobuf package and version timestamps +// concurrently. It returns the package response and the set of versions to +// filter, the Hex API's error, and the package fetch's error. +func (h *HexHandler) fetchPackageAndVersions(r *http.Request, name string) (*http.Response, map[string]bool, error, error) { type versionsResult struct { filtered map[string]bool err error @@ -178,16 +178,9 @@ func (h *HexHandler) fetchPackageAndVersions(r *http.Request, name string) (*htt versionsRes := <-versionsCh if err != nil { - return nil, nil, err + return nil, nil, nil, err } - - if versionsRes.err != nil { - h.proxy.Logger.Warn("failed to fetch hex version timestamps, proxying unfiltered", - "name", name, "error", versionsRes.err) - return protoResp, nil, nil - } - - return protoResp, versionsRes.filtered, nil + return protoResp, versionsRes.filtered, versionsRes.err, nil } // fetchUpstreamPackage fetches the protobuf package from upstream. diff --git a/internal/handler/npm.go b/internal/handler/npm.go index ba1e2b1..c2256d2 100644 --- a/internal/handler/npm.go +++ b/internal/handler/npm.go @@ -277,7 +277,8 @@ func (h *NPMHandler) handlePackageMetadata(w http.ResponseWriter, r *http.Reques JSONError(w, http.StatusNotFound, ErrVersionNotFound.Error()) return } - if len(h.proxy.Denylist.Versions(canonicalPackagePURL("npm", packageName))) != 0 { + if h.proxy.mustFilterMetadata("npm", packageName) { + h.proxy.Logger.Error("failed to filter npm metadata", "package", packageName, "error", err) JSONError(w, http.StatusBadGateway, "failed to filter package metadata") return } @@ -387,7 +388,8 @@ func (h *NPMHandler) handleFilteredNPMVersionMetadata(w http.ResponseWriter, r * JSONError(w, http.StatusNotFound, ErrVersionNotFound.Error()) return } - if len(h.proxy.Denylist.Versions(canonicalPackagePURL("npm", packageName))) != 0 { + if h.proxy.mustFilterMetadata("npm", packageName) { + h.proxy.Logger.Error("failed to filter npm metadata", "package", packageName, "error", err) JSONError(w, http.StatusBadGateway, "failed to filter package metadata") return } @@ -923,7 +925,17 @@ func (h *NPMHandler) handleDownload(w http.ResponseWriter, r *http.Request) { body, _, _, err := h.proxy.fetchOrCacheMetadata(r.Context(), "npm", cacheKey, upstreamURL, "", h.packumentValidator(), h.packumentAccept()) return body, err }) - if h.versionInCooldown(packageName, version, metadata) { + inCooldown, err := h.versionInCooldown(packageName, version, metadata) + if err != nil { + if writeUpstreamHeld(w, err) { + return + } + h.proxy.Logger.Warn("cooldown: could not read npm publish time", + "package", packageName, "version", version, "error", err) + JSONError(w, http.StatusBadGateway, "failed to check cooldown") + return + } + if inCooldown { h.proxy.Logger.Info("cooldown: withholding npm tarball", "package", packageName, "version", version) JSONError(w, http.StatusNotFound, ErrVersionNotFound.Error()) @@ -1031,7 +1043,8 @@ func validateTarballURL(raw, registryURL string) (string, error) { // proxy has never seen, persisting the parsed time so the packument is // fetched and parsed at most once per version. A version with no usable // publish time is allowed through, matching how applyCooldownFiltering -// treats it. +// treats it. A packument that can't be fetched or parsed is an error, so a +// failing upstream can't let a version through. // // The versions row is keyed by the plain version PURL, which does not say // which registry the version came from, so routed packages leave it alone: @@ -1040,39 +1053,38 @@ func validateTarballURL(raw, registryURL string) (string, error) { // for a private version of the same name, a private time would hold back the // public version, and moving a route to another upstream would keep the old // one's times. -func (h *NPMHandler) versionInCooldown(packageName, version string, metadata func() ([]byte, error)) bool { - if h.proxy.Cooldown == nil || !h.proxy.Cooldown.Enabled() { - return false +func (h *NPMHandler) versionInCooldown(packageName, version string, metadata func() ([]byte, error)) (bool, error) { + if !h.proxy.cooldownApplies("npm", packageName) { + return false, nil } _, routed := h.routes.match(packageName) versionPURL := canonicalVersionPURL("npm", packageName, version) if !routed { if ver, err := h.proxy.DB.GetVersionByPURL(versionPURL); err == nil && ver != nil && ver.PublishedAt.Valid { - return !h.proxy.Cooldown.IsAllowed("npm", canonicalPackagePURL("npm", packageName), ver.PublishedAt.Time) + return !h.proxy.Cooldown.IsAllowed("npm", canonicalPackagePURL("npm", packageName), ver.PublishedAt.Time), nil } } body, err := metadata() + if errors.Is(err, ErrUpstreamNotFound) { + return false, nil + } if err != nil { - h.proxy.Logger.Warn("cooldown: could not fetch npm metadata for download check", - "package", packageName, "version", version, "error", err) - return false + return false, err } published, ok, err := lookupJSONString(body, "time", version) if err != nil { - h.proxy.Logger.Warn("cooldown: could not parse npm metadata for download check", - "package", packageName, "version", version, "error", err) - return false + return false, fmt.Errorf("parsing npm metadata: %w", err) } if !ok { - return false + return false, nil } publishedAt, err := time.Parse(time.RFC3339, published) if err != nil { - return false + return false, nil } if !routed { @@ -1082,7 +1094,7 @@ func (h *NPMHandler) versionInCooldown(packageName, version string, metadata fun } } - return !h.proxy.Cooldown.IsAllowed("npm", canonicalPackagePURL("npm", packageName), publishedAt) + return !h.proxy.Cooldown.IsAllowed("npm", canonicalPackagePURL("npm", packageName), publishedAt), nil } // npmMetadataPath returns the package name and the encoded registry path. diff --git a/internal/handler/npm_test.go b/internal/handler/npm_test.go index bbec4c6..0867776 100644 --- a/internal/handler/npm_test.go +++ b/internal/handler/npm_test.go @@ -561,10 +561,10 @@ func TestNPMDownloadCooldownDisabled(t *testing.T) { proxy, _, _, _ := setupTestProxy(t) h := NewNPMHandler(proxy, "http://proxy.test", "") - if h.versionInCooldown("leftpad", testVersion100, func() ([]byte, error) { + if inCooldown, _ := h.versionInCooldown("leftpad", testVersion100, func() ([]byte, error) { t.Fatal("cooldown must not request metadata when disabled") return nil, nil - }) { + }); inCooldown { t.Error("versionInCooldown = true, want false when cooldown is not configured") } } diff --git a/internal/handler/pub.go b/internal/handler/pub.go index 1fc6997..05670f6 100644 --- a/internal/handler/pub.go +++ b/internal/handler/pub.go @@ -162,6 +162,11 @@ func (h *PubHandler) handlePackageMetadata(w http.ResponseWriter, r *http.Reques rewritten, err := h.rewriteMetadata(name, body) if err != nil { + if h.proxy.mustFilterMetadata("pub", name) { + h.proxy.Logger.Error("failed to filter pub metadata", "package", name, "error", err) + http.Error(w, "failed to filter package metadata", http.StatusBadGateway) + return + } h.proxy.Logger.Warn("failed to rewrite metadata, proxying original", "error", err) w.Header().Set(headerContentType, "application/json") w.WriteHeader(http.StatusOK) diff --git a/internal/handler/pypi.go b/internal/handler/pypi.go index e2ca4db..b61bb3b 100644 --- a/internal/handler/pypi.go +++ b/internal/handler/pypi.go @@ -121,10 +121,14 @@ func (h *PyPIHandler) handleSimplePackage(w http.ResponseWriter, r *http.Request return } - // When cooldown is enabled, fetch JSON metadata to get version timestamps + // When cooldown applies, fetch JSON metadata to get version timestamps var filteredVersions map[string]bool - if h.proxy.Cooldown != nil && h.proxy.Cooldown.Enabled() { - filteredVersions = h.fetchFilteredVersions(r, name) + if h.proxy.cooldownApplies("pypi", name) { + filteredVersions, err = h.fetchFilteredVersions(r, name) + if err != nil { + h.proxy.writeCooldownLookupError(w, "pypi", name, err) + return + } } for version := range h.proxy.Denylist.Versions(canonicalPackagePURL("pypi", name)) { if filteredVersions == nil { @@ -137,7 +141,8 @@ func (h *PyPIHandler) handleSimplePackage(w http.ResponseWriter, r *http.Request if isJSONMediaType(contentType) { rewritten, err = h.rewriteSimpleJSON(body, filteredVersions) if err != nil { - if len(h.proxy.Denylist.Versions(canonicalPackagePURL("pypi", name))) != 0 { + if len(filteredVersions) != 0 { + h.proxy.Logger.Error("failed to filter pypi simple json", "package", name, "error", err) http.Error(w, "failed to filter package metadata", http.StatusBadGateway) return } @@ -263,24 +268,29 @@ func ensureVaryAccept(header http.Header) { header.Add("Vary", "Accept") } -// fetchFilteredVersions fetches JSON metadata and returns a set of version strings -// that should be filtered out due to cooldown. -func (h *PyPIHandler) fetchFilteredVersions(r *http.Request, name string) map[string]bool { +// fetchFilteredVersions fetches JSON metadata and returns a set of version +// strings that should be filtered out due to cooldown. A package the JSON API +// doesn't know has no publish times to go by and gets an empty set; any other +// failure to read them is an error. +func (h *PyPIHandler) fetchFilteredVersions(r *http.Request, name string) (map[string]bool, error) { jsonURL := fmt.Sprintf("%s/pypi/%s/json", h.upstreamURL, name) body, _, err := h.proxy.FetchOrCacheMetadata(r.Context(), "pypi", name+"/json", jsonURL) + if errors.Is(err, ErrUpstreamNotFound) { + return nil, nil + } if err != nil { - return nil + return nil, err } var metadata map[string]any if err := json.Unmarshal(body, &metadata); err != nil { - return nil + return nil, fmt.Errorf("parsing pypi json metadata: %w", err) } releases, ok := metadata["releases"].(map[string]any) if !ok { - return nil + return nil, errors.New("pypi json metadata has no releases") } packagePURL := canonicalPackagePURL("pypi", name) @@ -298,9 +308,9 @@ func (h *PyPIHandler) fetchFilteredVersions(r *http.Request, name string) map[st } if len(filtered) == 0 { - return nil + return nil, nil } - return filtered + return filtered, nil } // rewriteSimpleHTML rewrites package URLs in simple API HTML to point at this proxy. @@ -426,7 +436,8 @@ func (h *PyPIHandler) proxyAndRewriteJSON(w http.ResponseWriter, r *http.Request rewritten, err := h.rewriteJSONMetadata(body) if err != nil { - if len(h.proxy.Denylist.Versions(canonicalPackagePURL("pypi", r.PathValue("name")))) != 0 { + if h.proxy.mustFilterMetadata("pypi", r.PathValue("name")) { + h.proxy.Logger.Error("failed to filter pypi metadata", "url", upstreamURL, "error", err) http.Error(w, "failed to filter package metadata", http.StatusBadGateway) return } @@ -501,13 +512,14 @@ func (h *PyPIHandler) shouldFilterRelease(packagePURL string, files any) bool { // path without ever reading the index. // // A release whose upload time cannot be determined is allowed through, matching -// how fetchFilteredVersions treats it. -func (h *PyPIHandler) versionInCooldown(r *http.Request, name, version string) bool { - if h.proxy.Cooldown == nil || !h.proxy.Cooldown.Enabled() { - return false +// how fetchFilteredVersions treats it. Failing to read the times is an error. +func (h *PyPIHandler) versionInCooldown(r *http.Request, name, version string) (bool, error) { + if !h.proxy.cooldownApplies("pypi", name) { + return false, nil } - return h.fetchFilteredVersions(r, name)[version] + filtered, err := h.fetchFilteredVersions(r, name) + return filtered[version], err } // rewriteFileEntries rewrites URLs in a list of file entries. @@ -612,11 +624,18 @@ func (h *PyPIHandler) handleDownload(w http.ResponseWriter, r *http.Request) { filename := parts[len(parts)-1] name, version := h.parseFilename(filename) - if name != "" && h.versionInCooldown(r, name, version) { - h.proxy.Logger.Info("cooldown: withholding pypi file", - "name", name, "version", version, "filename", filename) - http.Error(w, "not found", http.StatusNotFound) - return + if name != "" { + inCooldown, err := h.versionInCooldown(r, name, version) + if err != nil { + h.proxy.writeCooldownLookupError(w, "pypi", name, err) + return + } + if inCooldown { + h.proxy.Logger.Info("cooldown: withholding pypi file", + "name", name, "version", version, "filename", filename) + http.Error(w, "not found", http.StatusNotFound) + return + } } if name == "" { From aa0460a15b478942490aa70082fa1b04bf1d7693 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sat, 24 Oct 2026 08:13:58 +0200 Subject: [PATCH 400/633] Bound Hex and gem cooldown reads by the metadata size limit With cooldown on, the Hex handler read the /packages response, its decompressed payload and the Hex API's JSON without a limit, and the gem handler decoded the versions API's JSON the same way. A misbehaving upstream could make the proxy hold any amount in memory, and a small gzip payload could expand without bound. These reads now go through ReadMetadata, so metadata_max_size applies to each, and to the Hex payload after decompression as the NuGet handler does. A response over the limit fails the request with 502. --- internal/handler/gem.go | 6 +++- internal/handler/gem_test.go | 29 ++++++++++++++++ internal/handler/hex.go | 16 +++++---- internal/handler/hex_test.go | 65 ++++++++++++++++++++++++++++++++++++ 4 files changed, 109 insertions(+), 7 deletions(-) diff --git a/internal/handler/gem.go b/internal/handler/gem.go index b2ff638..dbb6e08 100644 --- a/internal/handler/gem.go +++ b/internal/handler/gem.go @@ -301,8 +301,12 @@ func (h *GemHandler) fetchFilteredVersions(r *http.Request, name string) (map[st return nil, fmt.Errorf("versions API returned %d", resp.StatusCode) } + body, err := h.proxy.ReadMetadata(resp.Body) + if err != nil { + return nil, err + } var versions []gemVersion - if err := json.NewDecoder(resp.Body).Decode(&versions); err != nil { + if err := json.Unmarshal(body, &versions); err != nil { return nil, err } diff --git a/internal/handler/gem_test.go b/internal/handler/gem_test.go index 30470bc..a13d819 100644 --- a/internal/handler/gem_test.go +++ b/internal/handler/gem_test.go @@ -397,3 +397,32 @@ func TestGemFetchFilteredVersions(t *testing.T) { _ = fmt.Sprintf // silence unused import } + +func TestGemCooldownVersionsSizeLimit(t *testing.T) { + versionsJSON, _ := json.Marshal([]gemVersion{ + {Number: "2.0.0", Platform: "ruby", CreatedAt: time.Now().Add(-time.Hour).Format(time.RFC3339)}, + }) + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if strings.HasPrefix(r.URL.Path, "/api/v1/versions/") { + _, _ = w.Write(versionsJSON) + _, _ = w.Write([]byte(strings.Repeat(" ", 4096))) + return + } + _, _ = w.Write([]byte("---\n2.0.0 |checksum:abc\n")) + })) + defer upstream.Close() + + proxy := testProxy() + proxy.Cooldown = &cooldown.Config{Default: "3d"} + proxy.MetadataMaxSize = 1024 + h := &GemHandler{proxy: proxy, upstreamURL: upstream.URL} + + req := httptest.NewRequest(http.MethodGet, "/info/testgem", nil) + req.SetPathValue("name", "testgem") + w := httptest.NewRecorder() + h.handleCompactIndex(w, req) + + if w.Code != http.StatusBadGateway { + t.Errorf("status = %d, want %d", w.Code, http.StatusBadGateway) + } +} diff --git a/internal/handler/hex.go b/internal/handler/hex.go index 5b8e91e..2d86686 100644 --- a/internal/handler/hex.go +++ b/internal/handler/hex.go @@ -5,7 +5,6 @@ import ( "compress/gzip" "encoding/json" "fmt" - "io" "net/http" "strings" "time" @@ -132,9 +131,10 @@ func (h *HexHandler) handlePackages(w http.ResponseWriter, r *http.Request) { return } - body, err := io.ReadAll(protoResp.Body) + body, err := h.proxy.ReadMetadata(protoResp.Body) if err != nil { - http.Error(w, "failed to read response", http.StatusInternalServerError) + h.proxy.Logger.Error("failed to read hex package", "name", name, "error", err) + http.Error(w, "failed to read response", http.StatusBadGateway) return } @@ -224,8 +224,12 @@ func (h *HexHandler) fetchFilteredVersions(r *http.Request, name string) (map[st return nil, fmt.Errorf("hex API returned %d", resp.StatusCode) } + body, err := h.proxy.ReadMetadata(resp.Body) + if err != nil { + return nil, err + } var pkg hexPackageAPI - if err := json.NewDecoder(resp.Body).Decode(&pkg); err != nil { + if err := json.Unmarshal(body, &pkg); err != nil { return nil, err } @@ -253,12 +257,12 @@ func (h *HexHandler) fetchFilteredVersions(r *http.Request, name string) (map[st // filters releases from the Package payload, and re-encodes as gzipped protobuf // (without the original signature since the payload has changed). func (h *HexHandler) filterSignedPackage(gzippedData []byte, filteredVersions map[string]bool) ([]byte, error) { - // Decompress gzip + // Decompress gzip, with the metadata limit applied to the decompressed size gr, err := gzip.NewReader(bytes.NewReader(gzippedData)) if err != nil { return nil, err } - signed, err := io.ReadAll(gr) + signed, err := h.proxy.ReadMetadata(gr) if err != nil { return nil, err } diff --git a/internal/handler/hex_test.go b/internal/handler/hex_test.go index 92120bd..d448b76 100644 --- a/internal/handler/hex_test.go +++ b/internal/handler/hex_test.go @@ -3,6 +3,7 @@ package handler import ( "bytes" "compress/gzip" + cryptorand "crypto/rand" "encoding/json" "io" "log/slog" @@ -329,3 +330,67 @@ func TestHexHandlePackagesWithoutCooldown(t *testing.T) { t.Fatalf("status = %d, want %d", w.Code, http.StatusOK) } } + +func TestHexCooldownMetadataSizeLimit(t *testing.T) { + const limit = 1024 + now := time.Now() + releases := []hexRelease{ + {Version: testVersion100, InsertedAt: now.Add(-7 * 24 * time.Hour).Format(time.RFC3339Nano)}, + {Version: "2.0.0", InsertedAt: now.Add(-time.Hour).Format(time.RFC3339Nano)}, + } + apiJSON, _ := json.Marshal(hexPackageAPI{Releases: releases}) + pkg := buildHexPackage("testpkg", []string{testVersion100, "2.0.0"}) + proto := buildHexSigned(pkg) + + // An unknown bytes field pads the payload: it compresses to a few bytes + // but decompresses past the limit. + padded := protowire.AppendTag(append([]byte(nil), pkg...), 4, protowire.BytesType) + padded = protowire.AppendBytes(padded, make([]byte, 64*limit)) + bomb := buildHexSigned(padded) + if len(bomb) >= limit { + t.Fatalf("compressed package is %d bytes, want under %d", len(bomb), limit) + } + // Padding that doesn't compress makes the response itself too large. + noise := make([]byte, 2*limit) + _, _ = cryptorand.Read(noise) + large := protowire.AppendTag(append([]byte(nil), pkg...), 4, protowire.BytesType) + large = buildHexSigned(protowire.AppendBytes(large, noise)) + if len(large) <= limit { + t.Fatalf("compressed package is %d bytes, want over %d", len(large), limit) + } + + tests := []struct { + name string + pkg, api []byte + }{ + {"package over limit", large, apiJSON}, + {"package decompresses past limit", bomb, apiJSON}, + {"API response over limit", proto, append(append([]byte(nil), apiJSON...), bytes.Repeat([]byte(" "), 2*limit)...)}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/packages/testpkg": + _, _ = w.Write(tt.pkg) + case "/api/packages/testpkg": + _, _ = w.Write(tt.api) + default: + w.WriteHeader(http.StatusNotFound) + } + })) + defer upstream.Close() + + proxy, _, _, _ := setupTestProxy(t) + proxy.Cooldown = &cooldown.Config{Default: "3d"} + proxy.MetadataMaxSize = limit + h := NewHexHandlerWithUpstreams(proxy, "http://proxy.test", upstream.URL, upstream.URL) + + w := httptest.NewRecorder() + h.Routes().ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/packages/testpkg", nil)) + if w.Code != http.StatusBadGateway { + t.Errorf("status = %d, want %d", w.Code, http.StatusBadGateway) + } + }) + } +} From d64bfc6d365a710bf26bee15642bd3ae863db405 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sat, 24 Oct 2026 08:03:45 +0200 Subject: [PATCH 401/633] Refuse '?' and '#' in paths in more handlers Debian, RPM, APK, PyPI, CRAN and Maven build upstream URLs from the decoded request path, where a '?' or '#' would end the path and cache one file under another's name. Only the server middleware refused them; Arch and FreeBSD also check in the handler. Check in these handlers too, so they stay safe when mounted without the middleware. --- internal/handler/apk.go | 4 +- internal/handler/conda_shards.go | 1 - internal/handler/cran.go | 2 +- internal/handler/debian.go | 4 +- internal/handler/handler.go | 15 ++++++ internal/handler/maven.go | 2 +- internal/handler/pypi.go | 2 +- internal/handler/query_in_path_test.go | 71 ++++++++++++++++++++++++++ internal/handler/rpm.go | 4 +- 9 files changed, 98 insertions(+), 7 deletions(-) create mode 100644 internal/handler/query_in_path_test.go diff --git a/internal/handler/apk.go b/internal/handler/apk.go index 6e18e74..f915c30 100644 --- a/internal/handler/apk.go +++ b/internal/handler/apk.go @@ -82,7 +82,9 @@ func (h *APKHandler) Routes() http.Handler { path := strings.TrimPrefix(r.URL.Path, "/") - if containsPathTraversal(path) { + // A decoded '?' or '#' would cut the upstream path short and cache one + // file under another file's name. + if containsPathTraversal(path) || strings.ContainsAny(path, "?#") { http.Error(w, "invalid path", http.StatusBadRequest) return } diff --git a/internal/handler/conda_shards.go b/internal/handler/conda_shards.go index c7c7a00..01434e4 100644 --- a/internal/handler/conda_shards.go +++ b/internal/handler/conda_shards.go @@ -454,4 +454,3 @@ func msgpackNumber(v any) float64 { return 0 } } - diff --git a/internal/handler/cran.go b/internal/handler/cran.go index 803bd4b..7cba9f4 100644 --- a/internal/handler/cran.go +++ b/internal/handler/cran.go @@ -63,7 +63,7 @@ func (h *CRANHandler) Routes() http.Handler { // Binary package downloads mux.HandleFunc("GET /bin/{platform}/contrib/{rversion}/{filename}", h.handleBinaryDownload) - return mux + return rejectQueryChars(mux) } // handleSourceDownload serves a source package, fetching and caching from upstream. diff --git a/internal/handler/debian.go b/internal/handler/debian.go index a196547..94c423c 100644 --- a/internal/handler/debian.go +++ b/internal/handler/debian.go @@ -83,7 +83,9 @@ func (h *DebianHandler) Routes() http.Handler { path := strings.TrimPrefix(r.URL.Path, "/") - if containsPathTraversal(path) { + // A decoded '?' or '#' would cut the upstream path short and cache one + // file under another file's name. + if containsPathTraversal(path) || strings.ContainsAny(path, "?#") { http.Error(w, "invalid path", http.StatusBadRequest) return } diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 060474c..0abd777 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -44,6 +44,21 @@ func containsPathTraversal(path string) bool { return false } +// rejectQueryChars refuses requests whose decoded path contains '?' or '#', +// for handlers that build upstream URLs from the decoded path: there either +// would cut the upstream path short and cache one file under another file's +// name. The server refuses these paths too; this keeps the handlers safe on +// their own. +func rejectQueryChars(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if strings.ContainsAny(r.URL.Path, "?#") { + http.Error(w, "invalid path", http.StatusBadRequest) + return + } + next.ServeHTTP(w, r) + }) +} + func hasDotDotSegment(path string) bool { path = strings.ReplaceAll(path, "\\", "/") for segment := range strings.SplitSeq(path, "/") { diff --git a/internal/handler/maven.go b/internal/handler/maven.go index 90c4ce5..7bb035e 100644 --- a/internal/handler/maven.go +++ b/internal/handler/maven.go @@ -103,7 +103,7 @@ func (h *MavenHandler) Routes() http.Handler { // e.g., /com/google/guava/guava/32.1.3-jre/guava-32.1.3-jre.jar mux.HandleFunc("GET /", h.handleRequest) - return mux + return rejectQueryChars(mux) } // handleRequest routes Maven requests based on the path. diff --git a/internal/handler/pypi.go b/internal/handler/pypi.go index e2ca4db..021150f 100644 --- a/internal/handler/pypi.go +++ b/internal/handler/pypi.go @@ -84,7 +84,7 @@ func (h *PyPIHandler) Routes() http.Handler { // Package downloads (cache these) mux.HandleFunc("GET /packages/{path...}", h.handleDownload) - return mux + return rejectQueryChars(mux) } // handleSimpleIndex serves the simple API index. diff --git a/internal/handler/query_in_path_test.go b/internal/handler/query_in_path_test.go new file mode 100644 index 0000000..d9f412f --- /dev/null +++ b/internal/handler/query_in_path_test.go @@ -0,0 +1,71 @@ +package handler + +import ( + "errors" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" +) + +// TestHandlersRejectQueryCharsInPath checks that handlers which build upstream +// URLs from the decoded path refuse an encoded '?' or '#' themselves, without +// relying on the server middleware, and never contact the upstream or fetch +// an artifact. +func TestHandlersRejectQueryCharsInPath(t *testing.T) { + var requests atomic.Int32 + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests.Add(1) + _, _ = w.Write([]byte("upstream")) + })) + defer upstream.Close() + + proxy, _, _, fetcher := setupTestProxy(t) + proxy.HTTPClient = upstream.Client() + fetcher.fetchErr = errors.New("artifact fetched") + + handlers := map[string]struct { + routes http.Handler + paths []string + }{ + "debian": {NewDebianHandler(proxy, testProxyURL, upstream.URL, nil).Routes(), []string{ + "/pool/main/h/hello/hello_2.10-3_amd64.deb%3F/foo_9.9_amd64.deb", + "/dists/trixie/InRelease%23x", + }}, + "rpm": {NewRPMHandlerWithUpstream(proxy, testProxyURL, upstream.URL).Routes(), []string{ + "/Packages/n/nginx-1.24.0-2.fc39.x86_64.rpm%3F/foo-9.9-1.fc39.x86_64.rpm", + "/repodata/repomd.xml%3Fx", + }}, + "apk": {NewAPKHandler(proxy, testProxyURL, map[string]string{"alpine": upstream.URL}).Routes(), []string{ + "/alpine/v3.20/main/x86_64/busybox-1.36.1-r29.apk%3F/foo-9.9-r0.apk", + "/alpine/v3.20/main/x86_64/APKINDEX.tar.gz%23x", + }}, + "pypi": {NewPyPIHandlerWithUpstreams(proxy, testProxyURL, upstream.URL, upstream.URL).Routes(), []string{ + "/packages/aa/bb/requests-2.31.0.tar.gz%3F/foo-9.9.tar.gz", + "/simple/requests%3Fx/", + }}, + "cran": {NewCRANHandlerWithUpstream(proxy, testProxyURL, upstream.URL).Routes(), []string{ + "/src/contrib/ggplot2_3.4.4.tar.gz%3Ffoo_9.9.tar.gz", + "/bin/macosx%23x/contrib/4.3/PACKAGES", + }}, + "maven": {NewMavenHandler(proxy, testProxyURL, upstream.URL, upstream.URL).Routes(), []string{ + "/com/example/lib/1.0/lib-1.0.jar%3F/com/example/foo/9.9/foo-9.9.jar", + "/com/example/lib/maven-metadata.xml%23x", + }}, + } + for name, tc := range handlers { + for _, target := range tc.paths { + w := httptest.NewRecorder() + tc.routes.ServeHTTP(w, httptest.NewRequest(http.MethodGet, target, nil)) + if w.Code != http.StatusBadRequest { + t.Errorf("%s: GET %s = %d, want %d", name, target, w.Code, http.StatusBadRequest) + } + } + } + if got := requests.Load(); got != 0 { + t.Errorf("upstream requests = %d, want 0", got) + } + if fetcher.fetchCalled { + t.Errorf("artifact fetched from %s", fetcher.fetchedURL) + } +} diff --git a/internal/handler/rpm.go b/internal/handler/rpm.go index 8a3a762..3794692 100644 --- a/internal/handler/rpm.go +++ b/internal/handler/rpm.go @@ -69,7 +69,9 @@ func (h *RPMHandler) Routes() http.Handler { path := strings.TrimPrefix(r.URL.Path, "/") - if containsPathTraversal(path) { + // A decoded '?' or '#' would cut the upstream path short and cache one + // file under another file's name. + if containsPathTraversal(path) || strings.ContainsAny(path, "?#") { http.Error(w, "invalid path", http.StatusBadRequest) return } From fd4f067a1b98db41ec89a2a30846b5a3ea541dff Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sat, 24 Oct 2026 13:10:40 +0200 Subject: [PATCH 402/633] Keep legacy metadata whose blob matches its digest Ignoring every row still at its unhashed path dropped the whole metadata cache on upgrade, including on Linux and S3 where keys never collided, and left nothing to serve while the upstream was down. A legacy row's blob is now hashed once and compared with the row's content_digest. When it matches, the blob is this key's document: it is copied to the hashed path and the row moved there, so it is served fresh or as a stale fallback as before. Only a mismatch, where another key's document overwrote the shared file, or a row without a digest is dropped and fetched again. The legacy blobs are queued for deletion once no key differing only in case still reads its legacy path, which on a case-insensitive filesystem is the same file. --- internal/database/queries.go | 47 +++++++ internal/handler/handler.go | 2 +- internal/handler/metadata_path.go | 144 ++++++++++++++++++-- internal/handler/metadata_path_test.go | 175 ++++++++++++++++++++++++- 4 files changed, 350 insertions(+), 18 deletions(-) diff --git a/internal/database/queries.go b/internal/database/queries.go index 3afb134..6d2bc18 100644 --- a/internal/database/queries.go +++ b/internal/database/queries.go @@ -1290,6 +1290,53 @@ func (db *DB) RefreshMetadataCache(ecosystem, name, digest string, fetchedAt tim return n > 0, err } +// MoveMetadataCache points the metadata cache row for ecosystem and name at +// to, provided it is still at from and records digest, and reports whether +// it did. The caller has copied the document from from to to. +func (db *DB) MoveMetadataCache(ecosystem, name, from, to, digest string) (bool, error) { + res, err := db.Exec(db.Rebind(` + UPDATE metadata_cache SET storage_path = ?, updated_at = ? + WHERE ecosystem = ? AND name = ? AND storage_path = ? AND content_digest = ? + `), to, time.Now(), ecosystem, name, from, digest) + if err != nil { + return false, err + } + n, err := res.RowsAffected() + return n > 0, err +} + +// ForgetMetadataDocument points the metadata cache row for ecosystem and name +// at to, provided it is still at from, and clears what described the document +// at from: its digest, validators and fetch time. The row stays, so the next +// lookup fetches the document again rather than serving or revalidating it. +func (db *DB) ForgetMetadataDocument(ecosystem, name, from, to string) error { + _, err := db.Exec(db.Rebind(` + UPDATE metadata_cache + SET storage_path = ?, content_digest = NULL, etag = NULL, last_modified = NULL, + fetched_at = NULL, updated_at = ? + WHERE ecosystem = ? AND name = ? AND storage_path = ? + `), to, time.Now(), ecosystem, name, from) + return err +} + +// MetadataPath is a metadata cache row's name and storage path. +type MetadataPath struct { + Name string `db:"name"` + StoragePath string `db:"storage_path"` +} + +// MetadataCaseVariants returns the name and storage path of every metadata +// cache row in ecosystem whose name equals name ignoring case, name's own +// row included. lower folds ASCII only on SQLite. +func (db *DB) MetadataCaseVariants(ecosystem, name string) ([]MetadataPath, error) { + var rows []MetadataPath + err := db.Select(&rows, db.Rebind(` + SELECT name, storage_path FROM metadata_cache + WHERE ecosystem = ? AND lower(name) = lower(?) + `), ecosystem, name) + return rows, err +} + // UpsertMetadataCache writes entry and marks it accessed now, since a // document is stored because a request asked for it. func (db *DB) UpsertMetadataCache(entry *MetadataCacheEntry) error { diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 9ff0d5a..162a316 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -1387,7 +1387,7 @@ func (p *Proxy) cachedMetadataState(ctx context.Context, ecosystem, cacheKey str return nil, nil } entry, _ := p.DB.GetMetadataCache(ecosystem, cacheKey) - entry = p.dropLegacyMetadataEntry(ecosystem, cacheKey, entry) + entry = p.migrateLegacyMetadataEntry(ctx, ecosystem, cacheKey, entry) if entry != nil && p.MetadataTTL > 0 && entry.FetchedAt.Valid && time.Since(entry.FetchedAt.Time) < p.MetadataTTL { data, ct, err := p.readCachedMetadata(ctx, entry, validate) if err == nil { diff --git a/internal/handler/metadata_path.go b/internal/handler/metadata_path.go index ca13a59..337f2f8 100644 --- a/internal/handler/metadata_path.go +++ b/internal/handler/metadata_path.go @@ -1,8 +1,13 @@ package handler import ( + "context" "crypto/sha256" "encoding/hex" + "fmt" + "io" + "slices" + "strings" "github.com/git-pkgs/proxy/internal/database" ) @@ -41,21 +46,132 @@ func isSHA256Hex(s string) bool { return true } -// dropLegacyMetadataEntry ignores a cache row whose blob is still at its -// legacy, unhashed path. On a case-insensitive filesystem that blob may hold -// another key's document, so it is neither served nor revalidated; the fetch -// that follows stores the document at the hashed path. The old blob is -// queued for deletion, which waits until no row points at it. -func (p *Proxy) dropLegacyMetadataEntry(ecosystem, cacheKey string, entry *database.MetadataCacheEntry) *database.MetadataCacheEntry { - if entry == nil { - return nil - } - legacy := legacyMetadataStoragePath(ecosystem, cacheKey) - if entry.StoragePath != legacy || legacy == metadataStoragePath(ecosystem, cacheKey) { +// migrateLegacyMetadataEntry handles a cache row whose blob is still at its +// legacy, unhashed path, and returns the row to use, or nil when there is +// none. On a case-insensitive filesystem keys differing only in case shared +// that blob, so it may hold another key's document. The blob is hashed and +// compared with the row's digest, once per row: +// +// - When it matches, the blob is this key's document. It is copied to the +// hashed path and the row moved there, so it keeps being served, as a +// stale fallback too. If the copy fails the row is served from where it +// is and checked again next time. +// - When it does not, or the row has no digest to check against, the row +// is pointed at the hashed path with its digest and validators cleared, +// so the document is fetched again and the blob never served. +// - When the blob cannot be read, nothing changes and nil is returned. +// +// Once no row's current path is a legacy blob of this key or a case variant +// of it, those blobs are queued for deletion. +func (p *Proxy) migrateLegacyMetadataEntry(ctx context.Context, ecosystem, cacheKey string, entry *database.MetadataCacheEntry) *database.MetadataCacheEntry { + if entry == nil || p.Storage == nil { return entry } - if err := p.DB.QueuePendingDelete(legacy); err != nil { - p.Logger.Warn("failed to queue legacy metadata blob for deletion", "ecosystem", ecosystem, "key", cacheKey, "error", err) + legacy := legacyMetadataStoragePath(ecosystem, cacheKey) + hashed := metadataStoragePath(ecosystem, cacheKey) + if entry.StoragePath != legacy || legacy == hashed { + return entry + } + var matches bool + if entry.ContentDigest.Valid { + digest, err := p.storedDigest(ctx, legacy) + if err != nil { + p.Logger.Warn("failed to read legacy metadata blob", "ecosystem", ecosystem, "key", cacheKey, "error", err) + return nil + } + matches = digest == entry.ContentDigest.String + } + if !matches { + if err := p.DB.ForgetMetadataDocument(ecosystem, cacheKey, legacy, hashed); err != nil { + p.Logger.Warn("failed to drop legacy metadata row", "ecosystem", ecosystem, "key", cacheKey, "error", err) + return nil + } + p.queueLegacyMetadataBlobs(ecosystem, cacheKey) + return nil + } + if !p.moveLegacyMetadataBlob(ctx, entry, legacy, hashed) { + return entry + } + p.queueLegacyMetadataBlobs(ecosystem, cacheKey) + moved := *entry + moved.StoragePath = hashed + return &moved +} + +// storedDigest hashes the blob at path as cacheMetadataBlob records it. +func (p *Proxy) storedDigest(ctx context.Context, path string) (string, error) { + r, err := p.Storage.Open(ctx, path) + if err != nil { + return "", err + } + defer func() { _ = r.Close() }() + h := sha256.New() + if _, err := io.Copy(h, r); err != nil { + return "", err + } + return "sha256:" + hex.EncodeToString(h.Sum(nil)), nil +} + +// moveLegacyMetadataBlob copies entry's verified blob from legacy to hashed +// and points the row there, reporting whether it did. +func (p *Proxy) moveLegacyMetadataBlob(ctx context.Context, entry *database.MetadataCacheEntry, legacy, hashed string) bool { + logFailure := func(err error) { + p.Logger.Warn("failed to move legacy metadata blob", "ecosystem", entry.Ecosystem, "key", entry.Name, "error", err) + } + r, err := p.Storage.Open(ctx, legacy) + if err != nil { + logFailure(err) + return false + } + defer func() { _ = r.Close() }() + _, hash, err := p.Storage.Store(ctx, hashed, r) + if err != nil { + logFailure(err) + return false + } + if "sha256:"+hash != entry.ContentDigest.String { + logFailure(fmt.Errorf("blob changed while being copied")) + return false + } + moved, err := p.DB.MoveMetadataCache(entry.Ecosystem, entry.Name, legacy, hashed, entry.ContentDigest.String) + if err != nil { + logFailure(err) + return false + } + return moved +} + +// queueLegacyMetadataBlobs queues the legacy blobs of cacheKey and its case +// variants for deletion, unless a row still reads one of them: on a +// case-insensitive filesystem a variant still at its legacy path is reading +// the same file. The last variant to move off its legacy path queues them +// all, so on a case-sensitive filesystem none is left behind. Reclaim also +// skips a path a row points at again. +func (p *Proxy) queueLegacyMetadataBlobs(ecosystem, cacheKey string) { + rows, err := p.DB.MetadataCaseVariants(ecosystem, cacheKey) + if err != nil { + p.Logger.Warn("failed to look up metadata case variants", "ecosystem", ecosystem, "key", cacheKey, "error", err) + return + } + for _, row := range rows { + legacy := legacyMetadataStoragePath(ecosystem, row.Name) + if legacy == metadataStoragePath(ecosystem, row.Name) { + continue + } + inUse := slices.ContainsFunc(rows, func(r database.MetadataPath) bool { + return strings.EqualFold(r.StoragePath, legacy) + }) + if inUse { + return + } + } + for _, row := range rows { + legacy := legacyMetadataStoragePath(ecosystem, row.Name) + if legacy == metadataStoragePath(ecosystem, row.Name) { + continue + } + if err := p.DB.QueuePendingDelete(legacy); err != nil { + p.Logger.Warn("failed to queue legacy metadata blob for deletion", "ecosystem", ecosystem, "key", row.Name, "error", err) + } } - return nil } diff --git a/internal/handler/metadata_path_test.go b/internal/handler/metadata_path_test.go index 34e73bc..9eba90c 100644 --- a/internal/handler/metadata_path_test.go +++ b/internal/handler/metadata_path_test.go @@ -98,9 +98,178 @@ func TestMetadataKeysDifferingInCaseCachedApart(t *testing.T) { } } -// TestLegacyMetadataPathRefetched checks that a row still pointing at the -// unhashed path is fetched again, moved to the hashed path, and its old blob -// queued for deletion. +// seedLegacyMetadata stores blob at key's legacy path and a row pointing at +// it, fetched at fetchedAt. The row's digest is that of document, or unset +// when document is empty. +func seedLegacyMetadata(t *testing.T, proxy *Proxy, db *database.DB, key, blob, document string, fetchedAt time.Time) string { + t.Helper() + legacy := legacyMetadataStoragePath("go", key) + if _, _, err := proxy.Storage.Store(context.Background(), legacy, strings.NewReader(blob)); err != nil { + t.Fatal(err) + } + entry := &database.MetadataCacheEntry{ + Ecosystem: "go", + Name: key, + StoragePath: legacy, + ETag: sql.NullString{String: `"v1"`, Valid: true}, + Size: sql.NullInt64{Int64: int64(len(blob)), Valid: true}, + FetchedAt: sql.NullTime{Time: fetchedAt, Valid: true}, + } + if document != "" { + sum := sha256.Sum256([]byte(document)) + entry.ContentDigest = sql.NullString{String: "sha256:" + hex.EncodeToString(sum[:]), Valid: true} + } + if err := db.UpsertMetadataCache(entry); err != nil { + t.Fatal(err) + } + return legacy +} + +func pendingDeletes(t *testing.T, db *database.DB) []string { + t.Helper() + due, err := db.GetDuePendingDeletes(time.Now().Add(time.Hour), 100) + if err != nil { + t.Fatal(err) + } + return due +} + +func metadataRowPath(t *testing.T, db *database.DB, key string) string { + t.Helper() + entry, err := db.GetMetadataCache("go", key) + if err != nil || entry == nil { + t.Fatalf("cache row for %s: %v %v", key, entry, err) + } + return entry.StoragePath +} + +// TestLegacyMetadataMatchingBlobServedWhenUpstreamDown checks that a row at +// its legacy path whose blob matches its digest is still served as a stale +// fallback, and is moved to the hashed path. +func TestLegacyMetadataMatchingBlobServedWhenUpstreamDown(t *testing.T) { + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusBadGateway) + })) + defer upstream.Close() + proxy, db := fileStorageProxy(t, upstream.Client()) + + const key = "github.com/Sirupsen/logrus" + legacy := seedLegacyMetadata(t, proxy, db, key, "old", "old", time.Now().Add(-2*time.Hour)) + + for range 2 { + body, _, err := proxy.FetchOrCacheMetadata(context.Background(), "go", key, upstream.URL+"/x") + if err != nil { + t.Fatalf("legacy document not served while upstream is down: %v", err) + } + if string(body) != "old" { + t.Fatalf("served %q, want %q", body, "old") + } + } + if got, want := metadataRowPath(t, db, key), metadataStoragePath("go", key); got != want { + t.Errorf("storage path = %q, want %q", got, want) + } + if due := pendingDeletes(t, db); !slices.Contains(due, legacy) { + t.Errorf("pending deletes = %v, want %q", due, legacy) + } +} + +// TestLegacyMetadataMatchingBlobServedFromCache checks that a fresh row at +// its legacy path whose blob matches is served without asking upstream. +func TestLegacyMetadataMatchingBlobServedFromCache(t *testing.T) { + var requests atomic.Int32 + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests.Add(1) + _, _ = io.WriteString(w, "fresh") + })) + defer upstream.Close() + proxy, db := fileStorageProxy(t, upstream.Client()) + + const key = "github.com/Sirupsen/logrus" + seedLegacyMetadata(t, proxy, db, key, "old", "old", time.Now()) + + body, _, err := proxy.FetchOrCacheMetadata(context.Background(), "go", key, upstream.URL+"/x") + if err != nil { + t.Fatal(err) + } + if string(body) != "old" || requests.Load() != 0 { + t.Errorf("served %q after %d upstream requests, want the cached document", body, requests.Load()) + } +} + +// TestLegacyMetadataMismatchedBlobNotServed checks that a legacy blob that +// does not match the row's digest, as when another key's document +// overwrote it, is neither served nor used as a fallback. +func TestLegacyMetadataMismatchedBlobNotServed(t *testing.T) { + var down atomic.Bool + var requests atomic.Int32 + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests.Add(1) + if down.Load() { + w.WriteHeader(http.StatusBadGateway) + return + } + if r.Header.Get("If-None-Match") != "" { + t.Errorf("revalidated a document that was dropped") + } + _, _ = io.WriteString(w, "fresh") + })) + defer upstream.Close() + proxy, db := fileStorageProxy(t, upstream.Client()) + + const key = "github.com/Sirupsen/logrus" + seedLegacyMetadata(t, proxy, db, key, "other key's document", "old", time.Now()) + down.Store(true) + if body, _, err := proxy.FetchOrCacheMetadata(context.Background(), "go", key, upstream.URL+"/x"); err == nil { + t.Fatalf("served %q, want an error while upstream is down", body) + } + + down.Store(false) + body, _, err := proxy.FetchOrCacheMetadata(context.Background(), "go", key, upstream.URL+"/x") + if err != nil { + t.Fatal(err) + } + if string(body) != "fresh" { + t.Errorf("served %q, want %q", body, "fresh") + } + if got, want := metadataRowPath(t, db, key), metadataStoragePath("go", key); got != want { + t.Errorf("storage path = %q, want %q", got, want) + } +} + +// TestLegacyMetadataSharedPathKeptForCaseVariant checks that a legacy blob is +// not queued for deletion while a key differing only in case still reads its +// own legacy path, which on a case-insensitive filesystem is the same file, +// and that both are queued once neither does. +func TestLegacyMetadataSharedPathKeptForCaseVariant(t *testing.T) { + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusBadGateway) + })) + defer upstream.Close() + proxy, db := fileStorageProxy(t, upstream.Client()) + + // Both rows hold the same document, so the test runs alike whether or + // not the filesystem folds case. + upper := seedLegacyMetadata(t, proxy, db, "github.com/Sirupsen/logrus", "doc", "doc", time.Now()) + lower := seedLegacyMetadata(t, proxy, db, "github.com/sirupsen/logrus", "doc", "doc", time.Now()) + + if _, _, err := proxy.FetchOrCacheMetadata(context.Background(), "go", "github.com/Sirupsen/logrus", upstream.URL+"/x"); err != nil { + t.Fatal(err) + } + if due := pendingDeletes(t, db); len(due) != 0 { + t.Fatalf("pending deletes = %v while a case variant still reads its legacy blob", due) + } + if _, _, err := proxy.FetchOrCacheMetadata(context.Background(), "go", "github.com/sirupsen/logrus", upstream.URL+"/x"); err != nil { + t.Fatal(err) + } + due := pendingDeletes(t, db) + if !slices.Contains(due, upper) || !slices.Contains(due, lower) { + t.Errorf("pending deletes = %v, want %q and %q", due, upper, lower) + } +} + +// TestLegacyMetadataPathRefetched checks that a row at its legacy path with +// no digest to check its blob against is fetched again, moved to the hashed +// path, and its old blob queued for deletion. func TestLegacyMetadataPathRefetched(t *testing.T) { var requests atomic.Int32 upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { From a29f9fc23437258daf14a0934587bd1af91bb132 Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Sat, 24 Oct 2026 13:06:12 +0100 Subject: [PATCH 403/633] Preserve Docker registries and SBOM versions when mirroring --- cmd/proxy/mirror_test.go | 16 ++++++ internal/mirror/image.go | 9 +++- internal/mirror/image_test.go | 16 ++++++ internal/mirror/sbom_version_test.go | 73 ++++++++++++++++++++++++++++ internal/mirror/source.go | 15 +++++- 5 files changed, 125 insertions(+), 4 deletions(-) create mode 100644 internal/mirror/sbom_version_test.go diff --git a/cmd/proxy/mirror_test.go b/cmd/proxy/mirror_test.go index e0f7e6a..bd39de9 100644 --- a/cmd/proxy/mirror_test.go +++ b/cmd/proxy/mirror_test.go @@ -113,6 +113,22 @@ func TestMirrorCommandDryRunWhileScanning(t *testing.T) { } } +func TestMirrorCommandSBOMComponentVersion(t *testing.T) { + dir := t.TempDir() + configPath := writeScanningMirrorConfig(t, dir) + sbomPath := filepath.Join(dir, "bom.json") + data := []byte(`{"components":[{"name":"app","version":"1.0","purl":"pkg:docker/ghcr.io/team/app"}]}`) + if err := os.WriteFile(sbomPath, data, 0600); err != nil { + t.Fatal(err) + } + output := runMirrorProcess(t, 0, "-config", configPath, "-dry-run", "-sbom", sbomPath) + for _, want := range []string{"Would mirror 1 package versions", "pkg:docker/ghcr.io/team/app@1.0"} { + if !strings.Contains(output, want) { + t.Errorf("dry run missing %q:\n%s", want, output) + } + } +} + func TestRefuseUnscannedMirroring(t *testing.T) { m := mirror.New(nil, nil, nil, nil, 1) refuseUnscannedMirroring(m) diff --git a/internal/mirror/image.go b/internal/mirror/image.go index bc21937..8b16893 100644 --- a/internal/mirror/image.go +++ b/internal/mirror/image.go @@ -64,6 +64,13 @@ func imageRef(pv PackageVersion) (handler.ImageRef, []handler.Platform, error) { ref.References = []string{pv.Version} } repositoryURL := pv.Qualifiers["repository_url"] + if pv.Ecosystem == "docker" && repositoryURL == "" { + host, repository, found := strings.Cut(pv.Name, "/") + if found && (strings.ContainsAny(host, ".:") || host == "localhost") { + ref.Registry = host + ref.Name = repository + } + } if pv.Ecosystem == "oci" { if repositoryURL == "" { return ref, nil, errors.New("pkg:oci needs a repository_url qualifier") @@ -96,8 +103,6 @@ func (m *Mirror) mirrorImage(ctx context.Context, pv PackageVersion, tracker *pr tracker.addError(pv.Ecosystem, pv.Name, pv.Version, err.Error()) m.logger.Warn("mirror failed", "ecosystem", pv.Ecosystem, "name", pv.Name, "version", pv.Version, "error", err) } - // pv.String keeps the namespace separator, which MakeString escapes for - // docker names. if m.proxy != nil && m.proxy.Denylist.Denied(pv.String()) { fail(fmt.Errorf("%w: %s", handler.ErrVersionDenied, pv)) return diff --git a/internal/mirror/image_test.go b/internal/mirror/image_test.go index 339e86c..c38ddf7 100644 --- a/internal/mirror/image_test.go +++ b/internal/mirror/image_test.go @@ -25,6 +25,22 @@ func TestImageRefFromPURL(t *testing.T) { purl: "pkg:docker/library/alpine@3.20", wantRef: handler.ImageRef{Name: "library/alpine", References: []string{"3.20"}}, }, + { + purl: "pkg:docker/ghcr.io/team/app@1.0", + wantRef: handler.ImageRef{Registry: "ghcr.io", Name: "team/app", References: []string{"1.0"}}, + }, + { + purl: "pkg:docker/registry.example:5000/team/app@" + digest, + wantRef: handler.ImageRef{Registry: "registry.example:5000", Name: "team/app", References: []string{digest}}, + }, + { + purl: "pkg:docker/localhost/app@1.0", + wantRef: handler.ImageRef{Registry: "localhost", Name: "app", References: []string{"1.0"}}, + }, + { + purl: "pkg:docker/localhost:5000/team/app@1.0", + wantRef: handler.ImageRef{Registry: "localhost:5000", Name: "team/app", References: []string{"1.0"}}, + }, { purl: "pkg:docker/alpine@3.20?repository_url=ghcr.io", platforms: []string{"linux/arm64/v8"}, diff --git a/internal/mirror/sbom_version_test.go b/internal/mirror/sbom_version_test.go new file mode 100644 index 0000000..190cc34 --- /dev/null +++ b/internal/mirror/sbom_version_test.go @@ -0,0 +1,73 @@ +package mirror + +import ( + "context" + "net/http" + "reflect" + "testing" + + "github.com/git-pkgs/proxy/internal/handler" + "github.com/git-pkgs/registries" +) + +func TestSBOMSourceUsesComponentVersion(t *testing.T) { + for name, data := range map[string]string{ + "CycloneDX JSON": `{"components":[{"name":"app","version":"1.0","purl":"pkg:docker/ghcr.io/team/app"}]}`, + "CycloneDX XML": `app1.0pkg:docker/ghcr.io/team/app`, + "SPDX JSON": `{"spdxVersion":"SPDX-2.3","packages":[{"name":"app","versionInfo":"1.0","externalRefs":[{"referenceType":"purl","referenceLocator":"pkg:docker/ghcr.io/team/app"}]}]}`, + "SPDX tag-value": "DataLicense: CC0-1.0\nSPDXVersion: SPDX-2.3\nSPDXID: SPDXRef-DOCUMENT\nDocumentName: example\nDocumentNamespace: https://example.com/sbom\nPackageName: app\nSPDXID: SPDXRef-app\nPackageVersion: 1.0\nExternalRef: PACKAGE-MANAGER purl pkg:docker/ghcr.io/team/app\n", + } { + t.Run(name, func(t *testing.T) { + source := &SBOMSource{Data: []byte(data)} + var got []PackageVersion + if err := source.Enumerate(context.Background(), func(pv PackageVersion) error { got = append(got, pv); return nil }); err != nil { + t.Fatal(err) + } + if len(got) != 1 || got[0].Version != "1.0" { + t.Fatalf("Enumerate = %+v, want one image at 1.0", got) + } + }) + } +} + +func TestSBOMSourceVersionPrecedence(t *testing.T) { + for _, test := range []struct{ name, componentVersion, imagePURL, wantVersion string }{ + {"PURL version wins", "2.0", "pkg:docker/team/app@1.0", "1.0"}, + {"digest from component", "sha256:abc123", "pkg:docker/team/app", "sha256:abc123"}, + {"unknown version", "", "pkg:docker/team/app", ""}, + {"package version", "1.0", "pkg:npm/example", "1.0"}, + } { + t.Run(test.name, func(t *testing.T) { + data := marshalJSON(t, map[string]any{"components": []map[string]any{{"name": "app", "version": test.componentVersion, "purl": test.imagePURL}}}) + var got []PackageVersion + source := &SBOMSource{Data: data, RegClient: registries.NewClient(registries.WithHTTPClient(&http.Client{Transport: rejectingMirrorTransport{}}))} + if err := source.Enumerate(context.Background(), func(pv PackageVersion) error { got = append(got, pv); return nil }); err != nil { + t.Fatal(err) + } + if len(got) != 1 || got[0].Version != test.wantVersion { + t.Fatalf("Enumerate = %+v, want version %q", got, test.wantVersion) + } + }) + } +} + +func TestMirrorRunSBOMDockerReference(t *testing.T) { + m := setupTestMirror(t, 1) + images := &fakeImages{result: handler.ImageResult{Fetched: 1}} + m.Images = images + source := &SBOMSource{ + Data: []byte(`{"components":[{"name":"app","version":"1.0","purl":"pkg:docker/registry.example:5000/team/app?arch=arm64"}]}`), + Platforms: []string{"linux/arm64"}, + } + progress, err := m.Run(context.Background(), source) + if err != nil { + t.Fatal(err) + } + if progress.Completed != 1 || progress.Failed != 0 { + t.Fatalf("progress = %+v", progress) + } + want := []handler.ImageRef{{Registry: "registry.example:5000", Name: "team/app", References: []string{"1.0"}}} + if !reflect.DeepEqual(images.refs, want) { + t.Errorf("image refs = %+v, want %+v", images.refs, want) + } +} diff --git a/internal/mirror/source.go b/internal/mirror/source.go index 8a4100f..c479289 100644 --- a/internal/mirror/source.go +++ b/internal/mirror/source.go @@ -144,9 +144,20 @@ func (s *SBOMSource) extractPURLs() ([]string, error) { var purls []string for _, p := range doc.Packages { for _, ref := range p.ExternalRefs { - if ref.Type == "purl" && ref.Locator != "" { - purls = append(purls, ref.Locator) + if ref.Type != "purl" || ref.Locator == "" { + continue } + locator := ref.Locator + if p.Version != "" { + parsed, err := purl.Parse(locator) + if err != nil { + return nil, fmt.Errorf("parsing PURL %q: %w", locator, err) + } + if parsed.Version == "" { + locator = parsed.WithVersion(p.Version).String() + } + } + purls = append(purls, locator) } } if len(purls) == 0 { From 94668c917ba8f9d7e21d76f8bbf33505cee29dd0 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sat, 24 Oct 2026 15:55:37 +0200 Subject: [PATCH 404/633] Use git-pkgs/enrichment for the enrichment API internal/enrichment now adapts the shared module instead of calling registries and OSV itself. License categories in the API and the web UI come from enrichment.CategorizeLicense, which normalizes the license first. Vulnerability fixed versions come from the affected range of the queried package. --- docs/architecture.md | 2 +- internal/enrichment/enrichment.go | 289 +++++-------------------- internal/enrichment/enrichment_test.go | 67 ++---- internal/server/api.go | 16 +- internal/server/server.go | 18 +- internal/server/server_test.go | 3 + 6 files changed, 92 insertions(+), 303 deletions(-) diff --git a/docs/architecture.md b/docs/architecture.md index f9325b2..76b2156 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -338,7 +338,7 @@ Version age filtering for supply chain attack mitigation, provided by [github.co ### `internal/enrichment` -Package metadata enrichment. Fetches license, description, homepage, repository URL, and vulnerability data from upstream registries. Powers the `/api/` endpoints and the web UI's package detail pages. +Package metadata enrichment, built on [github.com/git-pkgs/enrichment](https://github.com/git-pkgs/enrichment). Fetches license, description, homepage, repository URL, and vulnerability data from upstream registries. Powers the `/api/` endpoints and the web UI's package detail pages. ### `internal/mirror` diff --git a/internal/enrichment/enrichment.go b/internal/enrichment/enrichment.go index db8cc13..9521ba1 100644 --- a/internal/enrichment/enrichment.go +++ b/internal/enrichment/enrichment.go @@ -1,106 +1,54 @@ -// Package enrichment provides package metadata enrichment using external data sources. -// It fetches license information, vulnerability data, and version information -// from package registries and vulnerability databases. +// Package enrichment adapts github.com/git-pkgs/enrichment to the proxy's +// enrichment API. It builds package URLs from the proxy's ecosystem names, +// normalizes licenses for display and treats retracted versions as yanked. package enrichment import ( "context" "log/slog" "sync" - "time" + shared "github.com/git-pkgs/enrichment" "github.com/git-pkgs/proxy/internal/packageurl" - "github.com/git-pkgs/purl" "github.com/git-pkgs/registries" - _ "github.com/git-pkgs/registries/all" // Import all registry implementations "github.com/git-pkgs/spdx" - "github.com/git-pkgs/vers" - "github.com/git-pkgs/vulns" - "github.com/git-pkgs/vulns/osv" ) -// Service provides package enrichment capabilities. +// Service looks up package, version and vulnerability data. type Service struct { logger *slog.Logger - regClient *registries.Client - vulnSource vulns.Source + registries *shared.RegistriesClient + vulns *shared.VulnerabilityClient } // New creates a new enrichment service. func New(logger *slog.Logger) *Service { return &Service{ logger: logger, - regClient: registries.DefaultClient(), - vulnSource: osv.New(), + registries: shared.NewRegistriesClient(), + vulns: shared.NewVulnerabilityClient(), } } -// PackageInfo contains enriched package metadata. -type PackageInfo struct { - Ecosystem string - Name string - LatestVersion string - License string - Description string - Homepage string - Repository string - RegistryURL string -} +type ( + PackageInfo = shared.PackageInfo + VersionInfo = shared.VersionInfo + VulnInfo = shared.VulnInfo +) -// VersionInfo contains enriched version metadata. -type VersionInfo struct { - Number string - License string - PublishedAt time.Time - Integrity string - Yanked bool -} - -// VulnInfo contains vulnerability information for a package version. -type VulnInfo struct { - ID string - Summary string - Severity string - CVSSScore float64 - FixedVersion string - References []string -} - -// EnrichPackage fetches metadata for a package from registry APIs. +// EnrichPackage fetches metadata for a package from its registry. It returns +// nil without an error when the name can't be expressed as a PURL. func (s *Service) EnrichPackage(ctx context.Context, ecosystem, name string) (*PackageInfo, error) { purlStr := packageurl.MakeString(ecosystem, name, "") if purlStr == "" { return nil, nil } - - pkg, err := registries.FetchPackageFromPURL(ctx, purlStr, s.regClient) - if err != nil { + info, err := s.registries.GetPackage(ctx, purlStr) + if err != nil || info == nil { return nil, err } - - if pkg == nil { - return nil, nil - } - - info := &PackageInfo{ - Ecosystem: ecosystem, - Name: pkg.Name, - LatestVersion: pkg.LatestVersion, - Description: pkg.Description, - Homepage: pkg.Homepage, - Repository: pkg.Repository, - RegistryURL: registries.DefaultURL(ecosystem), - } - - // Normalize license to SPDX format - if pkg.Licenses != "" { - if normalized, err := spdx.NormalizeExpressionLax(pkg.Licenses); err == nil { - info.License = normalized - } else { - info.License = pkg.Licenses - } - } - + info.Ecosystem = ecosystem + info.License = normalizeLicense(info.License) return info, nil } @@ -110,36 +58,24 @@ func (s *Service) EnrichVersion(ctx context.Context, ecosystem, name, version st if purlStr == "" { return nil, nil } - - ver, err := registries.FetchVersionFromPURL(ctx, purlStr, s.regClient) - if err != nil { + info, err := s.registries.GetVersion(ctx, purlStr) + if err != nil || info == nil { return nil, err } - - if ver == nil { - return nil, nil - } - - info := &VersionInfo{ - Number: ver.Number, - PublishedAt: ver.PublishedAt, - Integrity: ver.Integrity, - Yanked: ver.Status == registries.StatusYanked || ver.Status == registries.StatusRetracted, - } - - // Normalize license - if ver.Licenses != "" { - if normalized, err := spdx.NormalizeExpressionLax(ver.Licenses); err == nil { - info.License = normalized - } else { - info.License = ver.Licenses - } - } - + adaptVersion(info) return info, nil } -// BulkEnrichPackages fetches metadata for multiple packages in parallel. +// adaptVersion reports retracted versions as yanked and normalizes the license. +func adaptVersion(info *VersionInfo) { + if info.Status == string(registries.StatusRetracted) { + info.Yanked = true + } + info.License = normalizeLicense(info.License) +} + +// BulkEnrichPackages fetches metadata for multiple packages in parallel, +// keyed by PURL. Packages that can't be fetched are left out. func (s *Service) BulkEnrichPackages(ctx context.Context, packages []struct{ Ecosystem, Name string }) map[string]*PackageInfo { purls := make([]string, 0, len(packages)) for _, pkg := range packages { @@ -150,126 +86,40 @@ func (s *Service) BulkEnrichPackages(ctx context.Context, packages []struct{ Eco if len(purls) == 0 { return map[string]*PackageInfo{} } - - pkgData := registries.BulkFetchPackages(ctx, purls, s.regClient) - - result := make(map[string]*PackageInfo, len(pkgData)) - for purlStr, pkg := range pkgData { - if pkg == nil { - continue - } - - p, err := purl.Parse(purlStr) - if err != nil { - continue - } - info := &PackageInfo{ - Ecosystem: p.Type, - Name: pkg.Name, - LatestVersion: pkg.LatestVersion, - Description: pkg.Description, - Homepage: pkg.Homepage, - Repository: pkg.Repository, - RegistryURL: registries.DefaultURL(p.Type), - } - - if pkg.Licenses != "" { - if normalized, err := spdx.NormalizeExpressionLax(pkg.Licenses); err == nil { - info.License = normalized - } else { - info.License = pkg.Licenses - } - } - - result[purlStr] = info + result, _ := s.registries.BulkLookup(ctx, purls) + for _, info := range result { + info.License = normalizeLicense(info.License) } - return result } // CheckVulnerabilities queries for vulnerabilities affecting a package version. func (s *Service) CheckVulnerabilities(ctx context.Context, ecosystem, name, version string) ([]VulnInfo, error) { - p := packageurl.Make(ecosystem, name, version) - if p == nil { + if packageurl.Make(ecosystem, name, version) == nil { return nil, nil } - - vulnList, err := s.vulnSource.Query(ctx, p) - if err != nil { - return nil, err - } - - results := make([]VulnInfo, 0, len(vulnList)) - for _, v := range vulnList { - info := VulnInfo{ - ID: v.ID, - Summary: v.Summary, - Severity: v.SeverityLevel(), - CVSSScore: v.CVSSScore(), - FixedVersion: v.FixedVersion(ecosystem, name), - } - - for _, ref := range v.References { - info.References = append(info.References, ref.URL) - } - - results = append(results, info) - } - - return results, nil -} - -// IsOutdated checks if a version is older than the latest version. -func (s *Service) IsOutdated(currentVersion, latestVersion string) bool { - if latestVersion == "" || currentVersion == "" { - return false - } - return vers.Compare(currentVersion, latestVersion) < 0 + return s.vulns.Check(ctx, ecosystem, name, version) } // GetLatestVersion fetches the latest version for a package. func (s *Service) GetLatestVersion(ctx context.Context, ecosystem, name string) (string, error) { - purlStr := packageurl.MakeString(ecosystem, name, "") - if purlStr == "" { - return "", nil - } - - latest, err := registries.FetchLatestVersionFromPURL(ctx, purlStr, s.regClient) - if err != nil { + info, err := s.EnrichPackage(ctx, ecosystem, name) + if err != nil || info == nil { return "", err } - - if latest == nil { - return "", nil - } - - return latest.Number, nil + return info.LatestVersion, nil } -// LicenseCategory represents the category of a license. -type LicenseCategory string - -const ( - LicensePermissive LicenseCategory = "permissive" - LicenseCopyleft LicenseCategory = "copyleft" - LicenseUnknown LicenseCategory = "unknown" -) - -// CategorizeLicense returns the category of a license. -func (s *Service) CategorizeLicense(license string) LicenseCategory { - if license == "" || license == "Unknown" { - return LicenseUnknown +// normalizeLicense returns license as an SPDX expression when it can be +// parsed as one, and unchanged otherwise. +func normalizeLicense(license string) string { + if license == "" { + return "" } - - if spdx.HasCopyleft(license) { - return LicenseCopyleft + if normalized, err := spdx.NormalizeExpressionLax(license); err == nil { + return normalized } - - if spdx.IsFullyPermissive(license) { - return LicensePermissive - } - - return LicenseUnknown + return license } // EnrichmentResult contains all enrichment data for a package version. @@ -278,40 +128,21 @@ type EnrichmentResult struct { Version *VersionInfo Vulnerabilities []VulnInfo IsOutdated bool - LicenseCategory LicenseCategory + LicenseCategory shared.LicenseCategory } -// EnrichFull performs full enrichment for a package version. +// EnrichFull fetches package, version and vulnerability data for a package +// version concurrently. Failed lookups are logged and left empty. func (s *Service) EnrichFull(ctx context.Context, ecosystem, name, version string) (*EnrichmentResult, error) { result := &EnrichmentResult{} var wg sync.WaitGroup var pkgErr, verErr, vulnErr error - - // Fetch package info - wg.Add(1) - go func() { - defer wg.Done() - result.Package, pkgErr = s.EnrichPackage(ctx, ecosystem, name) - }() - - // Fetch version info - wg.Add(1) - go func() { - defer wg.Done() - result.Version, verErr = s.EnrichVersion(ctx, ecosystem, name, version) - }() - - // Check vulnerabilities - wg.Add(1) - go func() { - defer wg.Done() - result.Vulnerabilities, vulnErr = s.CheckVulnerabilities(ctx, ecosystem, name, version) - }() - + wg.Go(func() { result.Package, pkgErr = s.EnrichPackage(ctx, ecosystem, name) }) + wg.Go(func() { result.Version, verErr = s.EnrichVersion(ctx, ecosystem, name, version) }) + wg.Go(func() { result.Vulnerabilities, vulnErr = s.CheckVulnerabilities(ctx, ecosystem, name, version) }) wg.Wait() - // Log errors but don't fail if pkgErr != nil { s.logger.Debug("failed to enrich package", "ecosystem", ecosystem, "name", name, "error", pkgErr) } @@ -322,19 +153,17 @@ func (s *Service) EnrichFull(ctx context.Context, ecosystem, name, version strin s.logger.Debug("failed to check vulnerabilities", "ecosystem", ecosystem, "name", name, "version", version, "error", vulnErr) } - // Determine outdated status - if result.Package != nil && result.Package.LatestVersion != "" { - result.IsOutdated = s.IsOutdated(version, result.Package.LatestVersion) + if result.Package != nil { + result.IsOutdated = shared.IsOutdated(version, result.Package.LatestVersion) } - // Categorize license license := "" if result.Version != nil && result.Version.License != "" { license = result.Version.License - } else if result.Package != nil && result.Package.License != "" { + } else if result.Package != nil { license = result.Package.License } - result.LicenseCategory = s.CategorizeLicense(license) + result.LicenseCategory = shared.CategorizeLicense(license) return result, nil } diff --git a/internal/enrichment/enrichment_test.go b/internal/enrichment/enrichment_test.go index 3952eb2..a854e5a 100644 --- a/internal/enrichment/enrichment_test.go +++ b/internal/enrichment/enrichment_test.go @@ -5,6 +5,8 @@ import ( "log/slog" "os" "testing" + + "github.com/git-pkgs/registries" ) func TestNew(t *testing.T) { @@ -15,12 +17,12 @@ func TestNew(t *testing.T) { t.Fatal("New() returned nil") } - if svc.regClient == nil { - t.Error("regClient is nil") + if svc.registries == nil { + t.Error("registries is nil") } - if svc.vulnSource == nil { - t.Error("vulnSource is nil") + if svc.vulns == nil { + t.Error("vulns is nil") } } @@ -54,54 +56,23 @@ func TestSwiftRegistryIdentitySkipsPURLDependentLookups(t *testing.T) { } } -func TestIsOutdated(t *testing.T) { - logger := slog.New(slog.NewTextHandler(os.Stdout, nil)) - svc := New(logger) - +func TestAdaptVersion(t *testing.T) { tests := []struct { - current string - latest string - expected bool + status registries.VersionStatus + yanked bool + license string + wantYanked bool + wantLicense string }{ - {"1.0.0", "2.0.0", true}, - {"2.0.0", "2.0.0", false}, - {"2.0.0", "1.0.0", false}, - {"1.0.0", "", false}, - {"", "2.0.0", false}, - {"1.2.3", "1.2.4", true}, - {"1.2.4", "1.2.3", false}, + {status: registries.StatusRetracted, license: "mit", wantYanked: true, wantLicense: "MIT"}, + {status: registries.StatusYanked, yanked: true, wantYanked: true}, + {status: registries.StatusDeprecated, license: "Custom terms", wantLicense: "Custom terms"}, } - for _, tc := range tests { - result := svc.IsOutdated(tc.current, tc.latest) - if result != tc.expected { - t.Errorf("IsOutdated(%q, %q) = %v, want %v", tc.current, tc.latest, result, tc.expected) - } - } -} - -func TestCategorizeLicense(t *testing.T) { - logger := slog.New(slog.NewTextHandler(os.Stdout, nil)) - svc := New(logger) - - tests := []struct { - license string - expected LicenseCategory - }{ - {"MIT", LicensePermissive}, - {"Apache-2.0", LicensePermissive}, - {"BSD-3-Clause", LicensePermissive}, - {"GPL-3.0", LicenseCopyleft}, - {"AGPL-3.0", LicenseCopyleft}, - {"LGPL-2.1", LicenseCopyleft}, - {"", LicenseUnknown}, - {"Unknown", LicenseUnknown}, - } - - for _, tc := range tests { - result := svc.CategorizeLicense(tc.license) - if result != tc.expected { - t.Errorf("CategorizeLicense(%q) = %v, want %v", tc.license, result, tc.expected) + info := &VersionInfo{Status: string(tc.status), Yanked: tc.yanked, License: tc.license} + adaptVersion(info) + if info.Yanked != tc.wantYanked || info.License != tc.wantLicense { + t.Errorf("adaptVersion(%s, %q) = yanked %v, license %q; want %v, %q", tc.status, tc.license, info.Yanked, info.License, tc.wantYanked, tc.wantLicense) } } } diff --git a/internal/server/api.go b/internal/server/api.go index 992d736..b204367 100644 --- a/internal/server/api.go +++ b/internal/server/api.go @@ -174,7 +174,7 @@ func (h *APIHandler) HandlePackagePath(w http.ResponseWriter, r *http.Request) { Name: info.Name, LatestVersion: info.LatestVersion, License: info.License, - LicenseCategory: string(h.enrichment.CategorizeLicense(info.License)), + LicenseCategory: string(shared.CategorizeLicense(info.License)), Description: info.Description, Homepage: info.Homepage, Repository: info.Repository, @@ -207,7 +207,7 @@ func (h *APIHandler) getPackage(w http.ResponseWriter, r *http.Request, ecosyste Name: info.Name, LatestVersion: info.LatestVersion, License: info.License, - LicenseCategory: string(h.enrichment.CategorizeLicense(info.License)), + LicenseCategory: string(shared.CategorizeLicense(info.License)), Description: info.Description, Homepage: info.Homepage, Repository: info.Repository, @@ -235,7 +235,7 @@ func (h *APIHandler) getVersion(w http.ResponseWriter, r *http.Request, ecosyste Name: result.Package.Name, LatestVersion: result.Package.LatestVersion, License: result.Package.License, - LicenseCategory: string(h.enrichment.CategorizeLicense(result.Package.License)), + LicenseCategory: string(shared.CategorizeLicense(result.Package.License)), Description: result.Package.Description, Homepage: result.Package.Homepage, Repository: result.Package.Repository, @@ -366,7 +366,7 @@ func (h *APIHandler) HandleOutdated(w http.ResponseWriter, r *http.Request) { latest, err := h.enrichment.GetLatestVersion(r.Context(), pkg.Ecosystem, pkg.Name) if err == nil && latest != "" { result.LatestVersion = latest - result.IsOutdated = h.enrichment.IsOutdated(pkg.Version, latest) + result.IsOutdated = shared.IsOutdated(pkg.Version, latest) } resp.Results = append(resp.Results, result) @@ -413,7 +413,7 @@ func (h *APIHandler) HandleBulkLookup(w http.ResponseWriter, r *http.Request) { Name: info.Name, LatestVersion: info.LatestVersion, License: info.License, - LicenseCategory: string(h.enrichment.CategorizeLicense(info.License)), + LicenseCategory: string(shared.CategorizeLicense(info.License)), Description: info.Description, Homepage: info.Homepage, Repository: info.Repository, @@ -444,7 +444,7 @@ func (h *APIHandler) HandleBulkLookup(w http.ResponseWriter, r *http.Request) { Name: info.Name, LatestVersion: info.LatestVersion, License: info.License, - LicenseCategory: string(h.enrichment.CategorizeLicense(info.License)), + LicenseCategory: string(shared.CategorizeLicense(info.License)), Description: info.Description, Homepage: info.Homepage, Repository: info.Repository, @@ -635,9 +635,7 @@ func (h *APIHandler) HandlePackagesList(w http.ResponseWriter, r *http.Request) licenseCategory := licenseCategoryUnknown if pkg.License.Valid { license = pkg.License.String - if h.enrichment != nil { - licenseCategory = string(h.enrichment.CategorizeLicense(license)) - } + licenseCategory = string(shared.CategorizeLicense(license)) } cachedAt := "" if pkg.CachedAt.Valid { diff --git a/internal/server/server.go b/internal/server/server.go index b332f84..bd2f287 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -70,6 +70,7 @@ import ( "github.com/git-pkgs/archives" "github.com/git-pkgs/cooldown" + shared "github.com/git-pkgs/enrichment" swaggerdoc "github.com/git-pkgs/proxy/docs/swagger" "github.com/git-pkgs/proxy/internal/accesslog" "github.com/git-pkgs/proxy/internal/config" @@ -87,7 +88,6 @@ import ( registryclient "github.com/git-pkgs/registries/client" "github.com/git-pkgs/registries/fetch" "github.com/git-pkgs/registries/safehttp" - "github.com/git-pkgs/spdx" "github.com/go-chi/chi/v5" "github.com/go-chi/chi/v5/middleware" ) @@ -1470,21 +1470,9 @@ func formatTimeAgo(t time.Time) string { } } -// categorizeLicenseCSS returns the CSS class suffix for a license category using the spdx module. +// categorizeLicenseCSS returns the CSS class suffix for a license category. func categorizeLicenseCSS(license string) string { - if license == "" { - return licenseCategoryUnknown - } - - if spdx.HasCopyleft(license) { - return "copyleft" - } - - if spdx.IsFullyPermissive(license) { - return "permissive" - } - - return licenseCategoryUnknown + return string(shared.CategorizeLicense(license)) } // categorizeLicense is a helper that handles sql.NullString. diff --git a/internal/server/server_test.go b/internal/server/server_test.go index afd0ca9..292643c 100644 --- a/internal/server/server_test.go +++ b/internal/server/server_test.go @@ -806,6 +806,9 @@ func TestCategorizeLicenseCSS(t *testing.T) { {"MPL-2.0", "copyleft"}, {"", "unknown"}, {"Proprietary", "unknown"}, + {"mit", "permissive"}, + {"Apache 2.0", "permissive"}, + {"GPLv3", "copyleft"}, } for _, tc := range tests { From fc299593ac4c7fb199001f94a92161708996c5b8 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sat, 24 Oct 2026 17:08:22 +0200 Subject: [PATCH 405/633] Answer 404 for packages the registry doesn't have --- go.mod | 6 +++--- go.sum | 4 ++-- internal/enrichment/enrichment.go | 7 ++++++- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 7d17bfd..dd0cdc7 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/git-pkgs/archives v0.9.0 github.com/git-pkgs/artifacts v0.2.1 github.com/git-pkgs/cooldown v0.3.0 - github.com/git-pkgs/enrichment v0.7.3 + github.com/git-pkgs/enrichment v0.7.4-0.20261024153145-1e2684f97ead github.com/git-pkgs/gcs v0.1.0 github.com/git-pkgs/integrity v0.2.0 github.com/git-pkgs/magic v0.5.0 @@ -20,8 +20,6 @@ require ( github.com/git-pkgs/registries v0.11.0 github.com/git-pkgs/sbom v0.1.7 github.com/git-pkgs/spdx v0.3.3 - github.com/git-pkgs/vers v0.7.2 - github.com/git-pkgs/vulns v0.2.4 github.com/go-chi/chi/v5 v5.5.0 github.com/jmoiron/sqlx v1.4.0 github.com/klauspost/compress v1.20.1 @@ -78,6 +76,8 @@ require ( github.com/ecosyste-ms/ecosystems-go v0.5.0 // indirect github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a // indirect github.com/git-pkgs/pom v0.1.8 // indirect + github.com/git-pkgs/vers v0.7.2 // indirect + github.com/git-pkgs/vulns v0.2.4 // indirect github.com/github/go-spdx/v2 v2.7.0 // indirect github.com/go-logr/logr v1.4.4 // indirect github.com/go-logr/stdr v1.2.2 // indirect diff --git a/go.sum b/go.sum index ee37652..4aeff4e 100644 --- a/go.sum +++ b/go.sum @@ -123,8 +123,8 @@ github.com/git-pkgs/artifacts v0.2.1 h1:VwdxR4yTDaqBZ34h0slxQBVyr2Xfa+QGOKCKviMx github.com/git-pkgs/artifacts v0.2.1/go.mod h1:Otosgq52pXT5UNN7lh6s/lszpWKVDO8XrOjN1M70/IA= github.com/git-pkgs/cooldown v0.3.0 h1:eAl2CQFwqXxluoVP9hHmropAf1VoV3gSxLMkN+NEu8A= github.com/git-pkgs/cooldown v0.3.0/go.mod h1:FnTg8wufEALATC9Sm/9AHjLdA+u9eVn96e9BqXXjGB0= -github.com/git-pkgs/enrichment v0.7.3 h1:l8bGVUP2Ncg3uyoZz7Gve+pyojUVcIQ3oJOFbYhWSOM= -github.com/git-pkgs/enrichment v0.7.3/go.mod h1:ZN/cH4PnmzsapEVPvmAwREP7+ox+RX0yfR6KvoB8xKE= +github.com/git-pkgs/enrichment v0.7.4-0.20261024153145-1e2684f97ead h1:EeR1jvscVwtJCYMaJXoalBLFFDizOBnrnFllx/hLSKg= +github.com/git-pkgs/enrichment v0.7.4-0.20261024153145-1e2684f97ead/go.mod h1:ZN/cH4PnmzsapEVPvmAwREP7+ox+RX0yfR6KvoB8xKE= github.com/git-pkgs/gcs v0.1.0 h1:E3awGtsO0xZyHT9FUfEwMHjMkRxw41Bh+c7lgTmPvBo= github.com/git-pkgs/gcs v0.1.0/go.mod h1:bdkCFD66ryaWnU8MBhokVA3WkJfyAEchm9qec5woBpE= github.com/git-pkgs/integrity v0.2.0 h1:sxpAk7EUllZnTYttOoPTW/Yfe5J2imglpuJiuBUZOxM= diff --git a/internal/enrichment/enrichment.go b/internal/enrichment/enrichment.go index 9521ba1..4d847af 100644 --- a/internal/enrichment/enrichment.go +++ b/internal/enrichment/enrichment.go @@ -5,6 +5,7 @@ package enrichment import ( "context" + "errors" "log/slog" "sync" @@ -37,13 +38,17 @@ type ( ) // EnrichPackage fetches metadata for a package from its registry. It returns -// nil without an error when the name can't be expressed as a PURL. +// nil without an error when the registry doesn't have the package or the name +// can't be expressed as a PURL. func (s *Service) EnrichPackage(ctx context.Context, ecosystem, name string) (*PackageInfo, error) { purlStr := packageurl.MakeString(ecosystem, name, "") if purlStr == "" { return nil, nil } info, err := s.registries.GetPackage(ctx, purlStr) + if errors.Is(err, registries.ErrNotFound) { + return nil, nil + } if err != nil || info == nil { return nil, err } From de308e48e03cc433bebe9f9410701c43f7801325 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Fri, 23 Oct 2026 19:29:00 +0200 Subject: [PATCH 406/633] Use purl's ComparisonForm for denylist and NuGet retention keys The per-ecosystem rules for when two spellings name the same package (PEP 503, NuGet case, Composer's leading v, conda channel, OCI paths, Swift registry case) now live in purl. The denylist keeps only the proxy's own pkg:generic/swift-registry spelling and the gem platform match. Keys, cache paths and stored PURLs are unchanged. --- go.mod | 2 +- go.sum | 4 +- internal/denylist/denylist.go | 77 +++++++++-------------------------- internal/handler/nuget.go | 6 ++- 4 files changed, 27 insertions(+), 62 deletions(-) diff --git a/go.mod b/go.mod index 7d17bfd..dbbfd49 100644 --- a/go.mod +++ b/go.mod @@ -16,7 +16,7 @@ require ( github.com/git-pkgs/gcs v0.1.0 github.com/git-pkgs/integrity v0.2.0 github.com/git-pkgs/magic v0.5.0 - github.com/git-pkgs/purl v0.1.23 + github.com/git-pkgs/purl v0.1.24-0.20261024154844-2a1e0a9803d3 github.com/git-pkgs/registries v0.11.0 github.com/git-pkgs/sbom v0.1.7 github.com/git-pkgs/spdx v0.3.3 diff --git a/go.sum b/go.sum index ee37652..dcf8942 100644 --- a/go.sum +++ b/go.sum @@ -133,8 +133,8 @@ github.com/git-pkgs/magic v0.5.0 h1:Pm+4fkHR+K6kg9pwJM4hnwWUuBlkYdLmF1yIAI8lU9Q= github.com/git-pkgs/magic v0.5.0/go.mod h1:SXOqcsNmbmpZjJZHEEWnwxprbsFvpwWgTAZrgXp4Jm4= github.com/git-pkgs/pom v0.1.8 h1:mXCuWw6XkrKKZb6IurRZyIznYLPhV5AYg5ZO38Uj/l8= github.com/git-pkgs/pom v0.1.8/go.mod h1:sQBeYRzjCUn77fcABPn6I/a1aBHFxm55jTmkNxYtGtw= -github.com/git-pkgs/purl v0.1.23 h1:RG7F73S4ZnbAnB/posWf2WS4OhdG0xUbyX8UIkEUhPw= -github.com/git-pkgs/purl v0.1.23/go.mod h1:l8rCfvt5iXVrFIcoQ8+vCLOaHhrtEW2id0LENP+IzMw= +github.com/git-pkgs/purl v0.1.24-0.20261024154844-2a1e0a9803d3 h1:Mi8EkmagjRVQ9ee6oBbmyVHOs9gSIuj3Vjq4cBMrwNs= +github.com/git-pkgs/purl v0.1.24-0.20261024154844-2a1e0a9803d3/go.mod h1:l8rCfvt5iXVrFIcoQ8+vCLOaHhrtEW2id0LENP+IzMw= github.com/git-pkgs/registries v0.11.0 h1:kkB80NjO9UncyyxGfKmdPXWqUMNE/CkNw5Z/6gqW7Rs= github.com/git-pkgs/registries v0.11.0/go.mod h1:bKxgdy/PwnzQXL2FnOktmSQHTKk6B85MA4VtvvYItPc= github.com/git-pkgs/sbom v0.1.7 h1:4eboy/Yed/SC9ZcC/xT50iywHPzWbmCU7azDb5T1RbA= diff --git a/internal/denylist/denylist.go b/internal/denylist/denylist.go index 77b076a..8e84811 100644 --- a/internal/denylist/denylist.go +++ b/internal/denylist/denylist.go @@ -3,7 +3,6 @@ package denylist import ( "fmt" - "regexp" "strings" "github.com/git-pkgs/purl" @@ -116,65 +115,29 @@ func (id identity) channelKey() string { return id.key + "?channel=" + id.channel } -var pypiSeparators = regexp.MustCompile(`[-_.]+`) - -// swiftRegistryIdentity returns the registry scope and name of a Swift -// package. A source coordinate such as pkg:swift/github.com/apple/swift-nio -// names no registry identity. -func swiftRegistryIdentity(pkg *purl.PURL) (scope, name string, ok bool) { - switch { - case pkg.Type == "swift" && pkg.Namespace != "" && !strings.ContainsAny(pkg.Namespace, "./"): - return pkg.Namespace, pkg.Name, true - case pkg.Type == "generic" && strings.EqualFold(pkg.Namespace, "swift-registry"): - scope, name, found := strings.Cut(pkg.Name, ".") - return scope, name, found && scope != "" && name != "" +// genericSwiftRegistry returns the scope and name of a Swift registry archive, +// which the proxy stores as pkg:generic/swift-registry/{scope}.{name}. +func genericSwiftRegistry(pkg *purl.PURL) (scope, name string, ok bool) { + if pkg.Type != "generic" || !strings.EqualFold(pkg.Namespace, "swift-registry") { + return "", "", false } - return "", "", false + scope, name, found := strings.Cut(pkg.Name, ".") + return scope, name, found && scope != "" && name != "" } +// identify compares PURLs in purl's comparison form. The generic +// swift-registry spelling is the proxy's own, so it is mapped here first. func identify(pkg *purl.PURL) identity { - namespace, name, version := pkg.Namespace, pkg.Name, pkg.Version - var channel string - switch pkg.Type { - case "pypi": - // PEP 503: case-insensitive, runs of -, _ and . are equivalent. - name = pypiSeparators.ReplaceAllString(strings.ToLower(name), "-") - case "nuget": - // NuGet ids and versions are case-insensitive. - name, version = strings.ToLower(name), strings.ToLower(version) - case "conda": - // The conda handler names packages "channel/name"; entries may give the - // channel as a qualifier or namespace. An entry without one matches - // every channel. - channel = pkg.Qualifier("channel") - if namespace != "" { - channel, namespace = namespace, "" - } - if i := strings.LastIndex(name, "/"); i >= 0 { - channel, name = name[:i], name[i+1:] - } - case "composer": - // Packagist keeps a tag's leading v ("v6.1.7"); Composer treats it - // as the same version. - if len(version) > 1 && (version[0] == 'v' || version[0] == 'V') && version[1] >= '0' && version[1] <= '9' { - version = version[1:] - } - case "oci": - // Handlers name the repository as one name, library%2Falpine; - // entries write it as a path, library/alpine. - if namespace != "" { - name, namespace = namespace+"/"+name, "" - } - case "swift", "generic": - // Swift registry archives are stored as - // pkg:generic/swift-registry/{scope}.{name}; entries may write - // pkg:swift/{scope}/{name}. Registry identities ignore case. - if scope, pkgName, ok := swiftRegistryIdentity(pkg); ok { - namespace, name = strings.ToLower(scope), strings.ToLower(pkgName) - key := purl.New("swift", namespace, name, "", nil).String() - return identity{key: key, version: version} - } + if scope, name, ok := genericSwiftRegistry(pkg); ok { + // Registry identities ignore case. + pkg = purl.New("swift", strings.ToLower(scope), strings.ToLower(name), pkg.Version, nil) } - key := purl.New(pkg.Type, namespace, name, "", nil).String() - return identity{key: key, channel: channel, version: version} + c := pkg.ComparisonForm() + var channel string + if c.Type == "conda" { + // An entry without a channel matches every channel. + channel = c.Qualifier("channel") + } + key := purl.New(c.Type, c.Namespace, c.Name, "", nil).String() + return identity{key: key, channel: channel, version: c.Version} } diff --git a/internal/handler/nuget.go b/internal/handler/nuget.go index 3d9f727..2989298 100644 --- a/internal/handler/nuget.go +++ b/internal/handler/nuget.go @@ -18,7 +18,8 @@ const ( // Packages are stored under pkg:nuget/{id} with the id lowercased, since // NuGet ids are case-insensitive. purl keeps nuget names as written, so -// package rules are lowercased the same way before the default mapping. +// package rules go through its comparison form, which lowercases them the +// same way, before the default mapping. func init() { retention.Default.Register(retention.Spec{Key: "nuget", CanonicalPackage: nugetRetentionKey}) } @@ -29,7 +30,8 @@ func nugetRetentionKey(p *purl.PURL) (string, bool) { if p == nil { return "", false } - return retention.DefaultCanonical("nuget")(purl.New(p.Type, p.Namespace, strings.ToLower(p.Name), "", nil)) + c := p.ComparisonForm() + return retention.DefaultCanonical("nuget")(purl.New(c.Type, c.Namespace, c.Name, "", nil)) } // NuGetHandler handles NuGet V3 API protocol requests. From 0b2fa835903be0d3567c2ea3feda9dd594139a3c Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sat, 24 Oct 2026 19:00:57 +0200 Subject: [PATCH 407/633] Use registries' forward proxy check registries' safehttp now checks the destination of requests sent through a forward proxy (git-pkgs/registries#116), so the proxy's own guard is removed and newUpstreamClient is back to its earlier form. go.mod points at a pseudo-version of registries main until #116 is released. --- go.mod | 2 +- go.sum | 4 +- internal/server/forward_proxy.go | 61 --------------- internal/server/forward_proxy_test.go | 105 -------------------------- internal/server/server.go | 9 +-- 5 files changed, 6 insertions(+), 175 deletions(-) delete mode 100644 internal/server/forward_proxy.go delete mode 100644 internal/server/forward_proxy_test.go diff --git a/go.mod b/go.mod index e2f860e..3b8425e 100644 --- a/go.mod +++ b/go.mod @@ -17,7 +17,7 @@ require ( github.com/git-pkgs/integrity v0.2.0 github.com/git-pkgs/magic v0.5.0 github.com/git-pkgs/purl v0.1.23 - github.com/git-pkgs/registries v0.11.0 + github.com/git-pkgs/registries v0.11.1-0.20261024160437-36fa54623922 github.com/git-pkgs/sbom v0.1.7 github.com/git-pkgs/spdx v0.3.3 github.com/git-pkgs/vers v0.7.2 diff --git a/go.sum b/go.sum index f298c28..ed30d81 100644 --- a/go.sum +++ b/go.sum @@ -135,8 +135,8 @@ github.com/git-pkgs/pom v0.1.8 h1:mXCuWw6XkrKKZb6IurRZyIznYLPhV5AYg5ZO38Uj/l8= github.com/git-pkgs/pom v0.1.8/go.mod h1:sQBeYRzjCUn77fcABPn6I/a1aBHFxm55jTmkNxYtGtw= github.com/git-pkgs/purl v0.1.23 h1:RG7F73S4ZnbAnB/posWf2WS4OhdG0xUbyX8UIkEUhPw= github.com/git-pkgs/purl v0.1.23/go.mod h1:l8rCfvt5iXVrFIcoQ8+vCLOaHhrtEW2id0LENP+IzMw= -github.com/git-pkgs/registries v0.11.0 h1:kkB80NjO9UncyyxGfKmdPXWqUMNE/CkNw5Z/6gqW7Rs= -github.com/git-pkgs/registries v0.11.0/go.mod h1:bKxgdy/PwnzQXL2FnOktmSQHTKk6B85MA4VtvvYItPc= +github.com/git-pkgs/registries v0.11.1-0.20261024160437-36fa54623922 h1:sg69ptO+sEegScnAmS+iiEVOXYtz5Tb5H2AjZBPV4Vo= +github.com/git-pkgs/registries v0.11.1-0.20261024160437-36fa54623922/go.mod h1:bKxgdy/PwnzQXL2FnOktmSQHTKk6B85MA4VtvvYItPc= github.com/git-pkgs/sbom v0.1.7 h1:4eboy/Yed/SC9ZcC/xT50iywHPzWbmCU7azDb5T1RbA= github.com/git-pkgs/sbom v0.1.7/go.mod h1:KIkZX0v31l62SeFsi2qsQXcZRVrAXvIdPQ4x4en+G2M= github.com/git-pkgs/spdx v0.3.3 h1:xXlUoZzjDSkwxC6mWHfaW+kc8a0+LH6d+gGHuZWftqI= diff --git a/internal/server/forward_proxy.go b/internal/server/forward_proxy.go deleted file mode 100644 index 66117a6..0000000 --- a/internal/server/forward_proxy.go +++ /dev/null @@ -1,61 +0,0 @@ -package server - -import ( - "context" - "net" - "net/http" - "net/url" - - "github.com/git-pkgs/registries/safehttp" -) - -// upstreamForwardProxy picks the forward proxy for an upstream request. It is -// a variable so tests can route through a test proxy without changing the -// environment, which net/http reads only once. -var upstreamForwardProxy = http.ProxyFromEnvironment - -// lookupUpstreamHost resolves a destination host for the forward proxy check. -var lookupUpstreamHost = net.DefaultResolver.LookupIPAddr - -// guardForwardProxy wraps proxy so that a request sent through a forward -// proxy is held to the same address rules as a direct one. The safehttp dial -// gate only sees the address it connects to, which for a proxied request is -// the forward proxy, so without this a URL taken from upstream metadata could -// reach a link-local or private address through the forward proxy. -// -// The destination is resolved here and every address must pass. A host that -// cannot be resolved locally is passed on, since setups that route all -// traffic through a forward proxy often cannot resolve public names -// themselves; the forward proxy resolves it. -func guardForwardProxy(proxy func(*http.Request) (*url.URL, error), opts safehttp.Options) func(*http.Request) (*url.URL, error) { - if proxy == nil { - return nil - } - checker := safehttp.NewHostIPChecker(opts) - return func(req *http.Request) (*url.URL, error) { - proxyURL, err := proxy(req) - if err != nil || proxyURL == nil { - return proxyURL, err - } - if err := checkProxiedDestination(req.Context(), checker, req.URL.Hostname()); err != nil { - return nil, err - } - return proxyURL, nil - } -} - -func checkProxiedDestination(ctx context.Context, checker *safehttp.HostIPChecker, host string) error { - if ip := net.ParseIP(host); ip != nil { - return checker.Check(host, ip) - } - addrs, err := lookupUpstreamHost(ctx, host) - if err != nil { - return nil //nolint:nilerr // the forward proxy resolves names this host cannot - } - for _, addr := range addrs { - if err := checker.Check(host, addr.IP); err != nil { - return err - } - } - return nil -} diff --git a/internal/server/forward_proxy_test.go b/internal/server/forward_proxy_test.go deleted file mode 100644 index b5287f4..0000000 --- a/internal/server/forward_proxy_test.go +++ /dev/null @@ -1,105 +0,0 @@ -package server - -import ( - "context" - "errors" - "net" - "net/http" - "net/http/httptest" - "net/url" - "sync/atomic" - "testing" - - "github.com/git-pkgs/proxy/internal/config" -) - -// routeThroughTestProxy starts a forward proxy that answers every request -// itself, points upstream clients at it, and resolves hostnames from hosts. -func routeThroughTestProxy(t *testing.T, hosts map[string]string) *atomic.Int32 { - t.Helper() - var received atomic.Int32 - forward := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - received.Add(1) - _, _ = w.Write([]byte("via proxy")) - })) - t.Cleanup(forward.Close) - forwardURL, err := url.Parse(forward.URL) - if err != nil { - t.Fatal(err) - } - - oldProxy, oldLookup := upstreamForwardProxy, lookupUpstreamHost - upstreamForwardProxy = http.ProxyURL(forwardURL) - lookupUpstreamHost = func(_ context.Context, host string) ([]net.IPAddr, error) { - if ip, ok := hosts[host]; ok { - return []net.IPAddr{{IP: net.ParseIP(ip)}}, nil - } - return nil, &net.DNSError{Err: "no such host", Name: host, IsNotFound: true} - } - t.Cleanup(func() { upstreamForwardProxy, lookupUpstreamHost = oldProxy, oldLookup }) - return &received -} - -func TestUpstreamClientChecksDestinationBehindForwardProxy(t *testing.T) { - received := routeThroughTestProxy(t, map[string]string{ - "metadata.example.test": "169.254.169.254", - "intranet.example.test": "10.1.2.3", - "allowed.example.test": "10.1.2.4", - "public.example.test": "203.0.113.10", - }) - // The forward proxy listens on loopback, so the client has to be allowed - // to dial it. - client := newUpstreamClient(config.UpstreamConfig{ - AllowLoopback: true, - AllowPrivateHosts: []string{"allowed.example.test", "10.9.9.9"}, - }) - - for _, tt := range []struct { - url string - blocked bool - }{ - {"http://169.254.169.254/latest/meta-data/", true}, - {"https://169.254.169.254/latest/meta-data/", true}, - {"http://[fe80::1]/", true}, - {"http://metadata.example.test/", true}, - {"http://10.1.2.3/", true}, - {"http://intranet.example.test/", true}, - {"http://allowed.example.test/", false}, - {"http://10.9.9.9/", false}, - {"http://203.0.113.10/", false}, - {"http://public.example.test/", false}, - {"http://unresolvable.example.test/", false}, - } { - before := received.Load() - resp, err := client.Get(tt.url) - if err == nil { - _ = resp.Body.Close() - } - reached := received.Load() > before - if tt.blocked { - if err == nil || reached { - t.Errorf("GET %s: err = %v, reached forward proxy = %v; want refused before the forward proxy", tt.url, err, reached) - } - continue - } - if err != nil || !reached { - t.Errorf("GET %s: err = %v, reached forward proxy = %v; want it passed to the forward proxy", tt.url, err, reached) - } - } -} - -func TestGuardForwardProxyKeepsDirectRequests(t *testing.T) { - if guardForwardProxy(nil, upstreamSafeHTTPOptions(config.UpstreamConfig{})) != nil { - t.Error("a transport without a proxy func got one") - } - proxyErr := errors.New("bad proxy config") - guarded := guardForwardProxy(func(*http.Request) (*url.URL, error) { return nil, proxyErr }, upstreamSafeHTTPOptions(config.UpstreamConfig{})) - req := httptest.NewRequest(http.MethodGet, "http://169.254.169.254/", nil) - if _, err := guarded(req); !errors.Is(err, proxyErr) { - t.Errorf("err = %v, want the proxy func's own error", err) - } - direct := guardForwardProxy(func(*http.Request) (*url.URL, error) { return nil, nil }, upstreamSafeHTTPOptions(config.UpstreamConfig{})) - if proxyURL, err := direct(req); proxyURL != nil || err != nil { - t.Errorf("direct request: proxy = %v, err = %v; want nil, nil so the dial gate checks it", proxyURL, err) - } -} diff --git a/internal/server/server.go b/internal/server/server.go index 394a699..5b0ebf7 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -611,17 +611,14 @@ func configureScanning(proxy *handler.Proxy, cfg config.ScanningConfig, baseURL } // newUpstreamClient builds the shared upstream client: a safehttp client whose -// transport keeps upstreamMaxIdleConnsPerHost idle connections per host, +// transport keeps upstreamMaxIdleConnsPerHost idle connections per host and // gives up after upstreamResponseHeaderTimeout when an upstream accepts a -// request but stalls before sending headers, and checks the destination of -// requests sent through a forward proxy. +// request but stalls before sending headers. func newUpstreamClient(upstream config.UpstreamConfig) *http.Client { - opts := upstreamSafeHTTPOptions(upstream) - client := safehttp.New(nil, opts) + client := safehttp.New(nil, upstreamSafeHTTPOptions(upstream)) if transport, ok := client.Transport.(*http.Transport); ok { transport.MaxIdleConnsPerHost = upstreamMaxIdleConnsPerHost transport.ResponseHeaderTimeout = upstreamResponseHeaderTimeout - transport.Proxy = guardForwardProxy(upstreamForwardProxy, opts) } return client } From c3d0448e6ffe1af15d383c31fc0761b32ba75e9b Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sat, 24 Oct 2026 19:03:38 +0200 Subject: [PATCH 408/633] Bump registries for streamed npm packuments registries main streams npm packuments (git-pkgs/registries#109), so listing the versions of packages whose packument is over 10 MiB, such as typescript, no longer fails with "unexpected end of JSON input". It also reads Maven versions from maven-metadata.xml (#114). go.mod points at a pseudo-version of registries main until those are released. --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 3c1d4e2..586ff0c 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,7 @@ require ( github.com/git-pkgs/integrity v0.2.0 github.com/git-pkgs/magic v0.5.0 github.com/git-pkgs/purl v0.1.22 - github.com/git-pkgs/registries v0.11.0 + github.com/git-pkgs/registries v0.11.1-0.20261024160437-36fa54623922 github.com/git-pkgs/sbom v0.1.7 github.com/git-pkgs/spdx v0.3.3 github.com/git-pkgs/vers v0.7.2 diff --git a/go.sum b/go.sum index f486b50..b5be5d0 100644 --- a/go.sum +++ b/go.sum @@ -135,8 +135,8 @@ github.com/git-pkgs/pom v0.1.8 h1:mXCuWw6XkrKKZb6IurRZyIznYLPhV5AYg5ZO38Uj/l8= github.com/git-pkgs/pom v0.1.8/go.mod h1:sQBeYRzjCUn77fcABPn6I/a1aBHFxm55jTmkNxYtGtw= github.com/git-pkgs/purl v0.1.22 h1:qT9FNXT/3/RbWnQCxMHn1CFeuruMsq0RBH154oLt1Xs= github.com/git-pkgs/purl v0.1.22/go.mod h1:l8rCfvt5iXVrFIcoQ8+vCLOaHhrtEW2id0LENP+IzMw= -github.com/git-pkgs/registries v0.11.0 h1:kkB80NjO9UncyyxGfKmdPXWqUMNE/CkNw5Z/6gqW7Rs= -github.com/git-pkgs/registries v0.11.0/go.mod h1:bKxgdy/PwnzQXL2FnOktmSQHTKk6B85MA4VtvvYItPc= +github.com/git-pkgs/registries v0.11.1-0.20261024160437-36fa54623922 h1:sg69ptO+sEegScnAmS+iiEVOXYtz5Tb5H2AjZBPV4Vo= +github.com/git-pkgs/registries v0.11.1-0.20261024160437-36fa54623922/go.mod h1:bKxgdy/PwnzQXL2FnOktmSQHTKk6B85MA4VtvvYItPc= github.com/git-pkgs/sbom v0.1.7 h1:4eboy/Yed/SC9ZcC/xT50iywHPzWbmCU7azDb5T1RbA= github.com/git-pkgs/sbom v0.1.7/go.mod h1:KIkZX0v31l62SeFsi2qsQXcZRVrAXvIdPQ4x4en+G2M= github.com/git-pkgs/spdx v0.3.3 h1:xXlUoZzjDSkwxC6mWHfaW+kc8a0+LH6d+gGHuZWftqI= From 4d0d4a20ae15d955f976eb41f059f703b971235c Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Sun, 25 Oct 2026 09:41:05 +0000 Subject: [PATCH 409/633] Spill large Git request bodies into the cache's temporary directory --- internal/gitcache/serve.go | 2 +- internal/gitcache/serve_test.go | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/internal/gitcache/serve.go b/internal/gitcache/serve.go index 216edc1..bb5cfb0 100644 --- a/internal/gitcache/serve.go +++ b/internal/gitcache/serve.go @@ -221,7 +221,7 @@ func (c *Cache) ReadRequest(r *http.Request) (*Request, error) { req.data = buf.Bytes() return req, nil } - file, err := os.CreateTemp("", "git-upload-pack-*") + file, err := os.CreateTemp(c.tmpDir(), "git-upload-pack-*") if err != nil { return nil, err } diff --git a/internal/gitcache/serve_test.go b/internal/gitcache/serve_test.go index 50b8f75..ca53e40 100644 --- a/internal/gitcache/serve_test.go +++ b/internal/gitcache/serve_test.go @@ -246,7 +246,10 @@ func TestRefListingSlots(t *testing.T) { // TestReadRequest reads bodies kept in memory and, past maxMemoryRequest, in // a temporary file removed on Close. func TestReadRequest(t *testing.T) { - c := &Cache{maxRequestBody: 4 << 20} + c := &Cache{dir: t.TempDir(), maxRequestBody: 4 << 20} + if err := os.Mkdir(c.tmpDir(), dirPermissions); err != nil { + t.Fatal(err) + } for _, size := range []int{0, 100, maxMemoryRequest, 3 << 20} { body := strings.Repeat("x", size) r, _ := http.NewRequest(http.MethodPost, "/", strings.NewReader(body)) From fcac1d5faee8eb53d31dad3acda93882ad127703 Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Sun, 25 Oct 2026 09:55:14 +0000 Subject: [PATCH 410/633] Refuse '?' and '#' through rejectQueryChars in APK, Debian and RPM --- internal/handler/apk.go | 8 +++----- internal/handler/cache_key.go | 3 +-- internal/handler/debian.go | 8 +++----- internal/handler/handler.go | 7 ++----- internal/handler/rpm.go | 8 +++----- 5 files changed, 12 insertions(+), 22 deletions(-) diff --git a/internal/handler/apk.go b/internal/handler/apk.go index f915c30..9024fe0 100644 --- a/internal/handler/apk.go +++ b/internal/handler/apk.go @@ -74,7 +74,7 @@ func NewAPKHandler(proxy *Proxy, proxyURL string, repositories map[string]string // Routes returns the HTTP handler for APK requests. // Mount this at /apk on your router. func (h *APKHandler) Routes() http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + return rejectQueryChars(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.Method != http.MethodGet && r.Method != http.MethodHead { http.Error(w, "method not allowed", http.StatusMethodNotAllowed) return @@ -82,9 +82,7 @@ func (h *APKHandler) Routes() http.Handler { path := strings.TrimPrefix(r.URL.Path, "/") - // A decoded '?' or '#' would cut the upstream path short and cache one - // file under another file's name. - if containsPathTraversal(path) || strings.ContainsAny(path, "?#") { + if containsPathTraversal(path) { http.Error(w, "invalid path", http.StatusBadRequest) return } @@ -109,7 +107,7 @@ func (h *APKHandler) Routes() http.Handler { // Other files - proxy directly. h.proxyFile(w, r, upstreamURL, rest) } - }) + })) } // isAPKIndex reports whether the path names a repository index: diff --git a/internal/handler/cache_key.go b/internal/handler/cache_key.go index 684c5d7..6c2e8a5 100644 --- a/internal/handler/cache_key.go +++ b/internal/handler/cache_key.go @@ -9,8 +9,7 @@ import ( // hashedCacheKey derives a metadata cache key from parts such as the // upstream URL, the request path and the query. Each part is length-prefixed // before hashing, so no two different part lists share a key, whatever bytes -// the parts contain. Keys built by replacing '/' with '_' were ambiguous: -// foo/bar_1.0 and foo_bar/1.0 mapped to the same entry. +// the parts contain. func hashedCacheKey(parts ...string) string { h := sha256.New() for _, part := range parts { diff --git a/internal/handler/debian.go b/internal/handler/debian.go index 94c423c..b2419f6 100644 --- a/internal/handler/debian.go +++ b/internal/handler/debian.go @@ -75,7 +75,7 @@ func NewDebianHandler( // Routes returns the HTTP handler for Debian requests. // Mount this at /debian on your router. func (h *DebianHandler) Routes() http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + return rejectQueryChars(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.Method != http.MethodGet && r.Method != http.MethodHead { http.Error(w, "method not allowed", http.StatusMethodNotAllowed) return @@ -83,9 +83,7 @@ func (h *DebianHandler) Routes() http.Handler { path := strings.TrimPrefix(r.URL.Path, "/") - // A decoded '?' or '#' would cut the upstream path short and cache one - // file under another file's name. - if containsPathTraversal(path) || strings.ContainsAny(path, "?#") { + if containsPathTraversal(path) { http.Error(w, "invalid path", http.StatusBadRequest) return } @@ -122,7 +120,7 @@ func (h *DebianHandler) Routes() http.Handler { // Other files (like README, etc.) - proxy directly h.proxyFile(w, r, upstreamURL, rest) } - }) + })) } // repositoryNotFound answers a request addressed to a repository that is not diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 0abd777..c06ec2d 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -44,11 +44,8 @@ func containsPathTraversal(path string) bool { return false } -// rejectQueryChars refuses requests whose decoded path contains '?' or '#', -// for handlers that build upstream URLs from the decoded path: there either -// would cut the upstream path short and cache one file under another file's -// name. The server refuses these paths too; this keeps the handlers safe on -// their own. +// rejectQueryChars refuses a decoded path containing '?' or '#', which would cut +// the upstream URL short and cache one file under another's name. func rejectQueryChars(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if strings.ContainsAny(r.URL.Path, "?#") { diff --git a/internal/handler/rpm.go b/internal/handler/rpm.go index 3794692..aaac6f5 100644 --- a/internal/handler/rpm.go +++ b/internal/handler/rpm.go @@ -61,7 +61,7 @@ func NewRPMHandlerWithUpstream(proxy *Proxy, proxyURL, upstreamURL string) *RPMH // Routes returns the HTTP handler for RPM requests. // Mount this at /rpm on your router. func (h *RPMHandler) Routes() http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + return rejectQueryChars(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.Method != http.MethodGet && r.Method != http.MethodHead { http.Error(w, "method not allowed", http.StatusMethodNotAllowed) return @@ -69,9 +69,7 @@ func (h *RPMHandler) Routes() http.Handler { path := strings.TrimPrefix(r.URL.Path, "/") - // A decoded '?' or '#' would cut the upstream path short and cache one - // file under another file's name. - if containsPathTraversal(path) || strings.ContainsAny(path, "?#") { + if containsPathTraversal(path) { http.Error(w, "invalid path", http.StatusBadRequest) return } @@ -88,7 +86,7 @@ func (h *RPMHandler) Routes() http.Handler { // Other files - proxy directly h.proxyFile(w, r, path) } - }) + })) } // handlePackageDownload fetches and caches .rpm packages. From 2a4ce1224879630309249c0bfde14f38ab637c12 Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Sun, 25 Oct 2026 10:14:03 +0000 Subject: [PATCH 411/633] Bump purl to v0.1.24 and drop redundant Swift lowercasing --- go.mod | 2 +- go.sum | 4 ++-- internal/denylist/denylist.go | 3 +-- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index dbbfd49..a86e990 100644 --- a/go.mod +++ b/go.mod @@ -16,7 +16,7 @@ require ( github.com/git-pkgs/gcs v0.1.0 github.com/git-pkgs/integrity v0.2.0 github.com/git-pkgs/magic v0.5.0 - github.com/git-pkgs/purl v0.1.24-0.20261024154844-2a1e0a9803d3 + github.com/git-pkgs/purl v0.1.24 github.com/git-pkgs/registries v0.11.0 github.com/git-pkgs/sbom v0.1.7 github.com/git-pkgs/spdx v0.3.3 diff --git a/go.sum b/go.sum index dcf8942..a78c08e 100644 --- a/go.sum +++ b/go.sum @@ -133,8 +133,8 @@ github.com/git-pkgs/magic v0.5.0 h1:Pm+4fkHR+K6kg9pwJM4hnwWUuBlkYdLmF1yIAI8lU9Q= github.com/git-pkgs/magic v0.5.0/go.mod h1:SXOqcsNmbmpZjJZHEEWnwxprbsFvpwWgTAZrgXp4Jm4= github.com/git-pkgs/pom v0.1.8 h1:mXCuWw6XkrKKZb6IurRZyIznYLPhV5AYg5ZO38Uj/l8= github.com/git-pkgs/pom v0.1.8/go.mod h1:sQBeYRzjCUn77fcABPn6I/a1aBHFxm55jTmkNxYtGtw= -github.com/git-pkgs/purl v0.1.24-0.20261024154844-2a1e0a9803d3 h1:Mi8EkmagjRVQ9ee6oBbmyVHOs9gSIuj3Vjq4cBMrwNs= -github.com/git-pkgs/purl v0.1.24-0.20261024154844-2a1e0a9803d3/go.mod h1:l8rCfvt5iXVrFIcoQ8+vCLOaHhrtEW2id0LENP+IzMw= +github.com/git-pkgs/purl v0.1.24 h1:JMEB1uNWI+baEpl6AeVx+Kfq1c+xkou+THxa5KJ3MrU= +github.com/git-pkgs/purl v0.1.24/go.mod h1:l8rCfvt5iXVrFIcoQ8+vCLOaHhrtEW2id0LENP+IzMw= github.com/git-pkgs/registries v0.11.0 h1:kkB80NjO9UncyyxGfKmdPXWqUMNE/CkNw5Z/6gqW7Rs= github.com/git-pkgs/registries v0.11.0/go.mod h1:bKxgdy/PwnzQXL2FnOktmSQHTKk6B85MA4VtvvYItPc= github.com/git-pkgs/sbom v0.1.7 h1:4eboy/Yed/SC9ZcC/xT50iywHPzWbmCU7azDb5T1RbA= diff --git a/internal/denylist/denylist.go b/internal/denylist/denylist.go index 8e84811..15bb9e5 100644 --- a/internal/denylist/denylist.go +++ b/internal/denylist/denylist.go @@ -129,8 +129,7 @@ func genericSwiftRegistry(pkg *purl.PURL) (scope, name string, ok bool) { // swift-registry spelling is the proxy's own, so it is mapped here first. func identify(pkg *purl.PURL) identity { if scope, name, ok := genericSwiftRegistry(pkg); ok { - // Registry identities ignore case. - pkg = purl.New("swift", strings.ToLower(scope), strings.ToLower(name), pkg.Version, nil) + pkg = purl.New("swift", scope, name, pkg.Version, nil) } c := pkg.ComparisonForm() var channel string From 1b641b6e90c41e81bf1e2d077934a0eff3d32878 Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Sun, 25 Oct 2026 10:15:58 +0000 Subject: [PATCH 412/633] Bump enrichment to v0.7.4 --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index dd0cdc7..bf5d552 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/git-pkgs/archives v0.9.0 github.com/git-pkgs/artifacts v0.2.1 github.com/git-pkgs/cooldown v0.3.0 - github.com/git-pkgs/enrichment v0.7.4-0.20261024153145-1e2684f97ead + github.com/git-pkgs/enrichment v0.7.4 github.com/git-pkgs/gcs v0.1.0 github.com/git-pkgs/integrity v0.2.0 github.com/git-pkgs/magic v0.5.0 diff --git a/go.sum b/go.sum index 4aeff4e..6fb0b93 100644 --- a/go.sum +++ b/go.sum @@ -123,8 +123,8 @@ github.com/git-pkgs/artifacts v0.2.1 h1:VwdxR4yTDaqBZ34h0slxQBVyr2Xfa+QGOKCKviMx github.com/git-pkgs/artifacts v0.2.1/go.mod h1:Otosgq52pXT5UNN7lh6s/lszpWKVDO8XrOjN1M70/IA= github.com/git-pkgs/cooldown v0.3.0 h1:eAl2CQFwqXxluoVP9hHmropAf1VoV3gSxLMkN+NEu8A= github.com/git-pkgs/cooldown v0.3.0/go.mod h1:FnTg8wufEALATC9Sm/9AHjLdA+u9eVn96e9BqXXjGB0= -github.com/git-pkgs/enrichment v0.7.4-0.20261024153145-1e2684f97ead h1:EeR1jvscVwtJCYMaJXoalBLFFDizOBnrnFllx/hLSKg= -github.com/git-pkgs/enrichment v0.7.4-0.20261024153145-1e2684f97ead/go.mod h1:ZN/cH4PnmzsapEVPvmAwREP7+ox+RX0yfR6KvoB8xKE= +github.com/git-pkgs/enrichment v0.7.4 h1:owiCKjITG8tNxBikcyUCDuDf1i3C/+Yoa5yJseuqIjA= +github.com/git-pkgs/enrichment v0.7.4/go.mod h1:ZN/cH4PnmzsapEVPvmAwREP7+ox+RX0yfR6KvoB8xKE= github.com/git-pkgs/gcs v0.1.0 h1:E3awGtsO0xZyHT9FUfEwMHjMkRxw41Bh+c7lgTmPvBo= github.com/git-pkgs/gcs v0.1.0/go.mod h1:bdkCFD66ryaWnU8MBhokVA3WkJfyAEchm9qec5woBpE= github.com/git-pkgs/integrity v0.2.0 h1:sxpAk7EUllZnTYttOoPTW/Yfe5J2imglpuJiuBUZOxM= From 910dadc220f8868057dc60a37e3f521acaa4273f Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Sun, 25 Oct 2026 10:17:37 +0000 Subject: [PATCH 413/633] Bump registries to v0.11.1 --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 3b8425e..d7a3d34 100644 --- a/go.mod +++ b/go.mod @@ -17,7 +17,7 @@ require ( github.com/git-pkgs/integrity v0.2.0 github.com/git-pkgs/magic v0.5.0 github.com/git-pkgs/purl v0.1.23 - github.com/git-pkgs/registries v0.11.1-0.20261024160437-36fa54623922 + github.com/git-pkgs/registries v0.11.1 github.com/git-pkgs/sbom v0.1.7 github.com/git-pkgs/spdx v0.3.3 github.com/git-pkgs/vers v0.7.2 diff --git a/go.sum b/go.sum index ed30d81..77e0444 100644 --- a/go.sum +++ b/go.sum @@ -135,8 +135,8 @@ github.com/git-pkgs/pom v0.1.8 h1:mXCuWw6XkrKKZb6IurRZyIznYLPhV5AYg5ZO38Uj/l8= github.com/git-pkgs/pom v0.1.8/go.mod h1:sQBeYRzjCUn77fcABPn6I/a1aBHFxm55jTmkNxYtGtw= github.com/git-pkgs/purl v0.1.23 h1:RG7F73S4ZnbAnB/posWf2WS4OhdG0xUbyX8UIkEUhPw= github.com/git-pkgs/purl v0.1.23/go.mod h1:l8rCfvt5iXVrFIcoQ8+vCLOaHhrtEW2id0LENP+IzMw= -github.com/git-pkgs/registries v0.11.1-0.20261024160437-36fa54623922 h1:sg69ptO+sEegScnAmS+iiEVOXYtz5Tb5H2AjZBPV4Vo= -github.com/git-pkgs/registries v0.11.1-0.20261024160437-36fa54623922/go.mod h1:bKxgdy/PwnzQXL2FnOktmSQHTKk6B85MA4VtvvYItPc= +github.com/git-pkgs/registries v0.11.1 h1:AkVQ15Rnrho4tbTnocYuHQuzNQ1m9jEjWKTA8P3xnPI= +github.com/git-pkgs/registries v0.11.1/go.mod h1:bKxgdy/PwnzQXL2FnOktmSQHTKk6B85MA4VtvvYItPc= github.com/git-pkgs/sbom v0.1.7 h1:4eboy/Yed/SC9ZcC/xT50iywHPzWbmCU7azDb5T1RbA= github.com/git-pkgs/sbom v0.1.7/go.mod h1:KIkZX0v31l62SeFsi2qsQXcZRVrAXvIdPQ4x4en+G2M= github.com/git-pkgs/spdx v0.3.3 h1:xXlUoZzjDSkwxC6mWHfaW+kc8a0+LH6d+gGHuZWftqI= From 42584c384522507cd019dfc80870be1723251d72 Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Sun, 25 Oct 2026 10:29:20 +0000 Subject: [PATCH 414/633] Bump registries to v0.11.1 --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 9f27556..5cf652b 100644 --- a/go.mod +++ b/go.mod @@ -17,7 +17,7 @@ require ( github.com/git-pkgs/integrity v0.2.0 github.com/git-pkgs/magic v0.5.0 github.com/git-pkgs/purl v0.1.23 - github.com/git-pkgs/registries v0.11.1-0.20261024160437-36fa54623922 + github.com/git-pkgs/registries v0.11.1 github.com/git-pkgs/sbom v0.1.7 github.com/git-pkgs/spdx v0.3.3 github.com/git-pkgs/vers v0.7.2 diff --git a/go.sum b/go.sum index c947090..52bda77 100644 --- a/go.sum +++ b/go.sum @@ -135,8 +135,8 @@ github.com/git-pkgs/pom v0.1.8 h1:mXCuWw6XkrKKZb6IurRZyIznYLPhV5AYg5ZO38Uj/l8= github.com/git-pkgs/pom v0.1.8/go.mod h1:sQBeYRzjCUn77fcABPn6I/a1aBHFxm55jTmkNxYtGtw= github.com/git-pkgs/purl v0.1.23 h1:RG7F73S4ZnbAnB/posWf2WS4OhdG0xUbyX8UIkEUhPw= github.com/git-pkgs/purl v0.1.23/go.mod h1:l8rCfvt5iXVrFIcoQ8+vCLOaHhrtEW2id0LENP+IzMw= -github.com/git-pkgs/registries v0.11.1-0.20261024160437-36fa54623922 h1:sg69ptO+sEegScnAmS+iiEVOXYtz5Tb5H2AjZBPV4Vo= -github.com/git-pkgs/registries v0.11.1-0.20261024160437-36fa54623922/go.mod h1:bKxgdy/PwnzQXL2FnOktmSQHTKk6B85MA4VtvvYItPc= +github.com/git-pkgs/registries v0.11.1 h1:AkVQ15Rnrho4tbTnocYuHQuzNQ1m9jEjWKTA8P3xnPI= +github.com/git-pkgs/registries v0.11.1/go.mod h1:bKxgdy/PwnzQXL2FnOktmSQHTKk6B85MA4VtvvYItPc= github.com/git-pkgs/sbom v0.1.7 h1:4eboy/Yed/SC9ZcC/xT50iywHPzWbmCU7azDb5T1RbA= github.com/git-pkgs/sbom v0.1.7/go.mod h1:KIkZX0v31l62SeFsi2qsQXcZRVrAXvIdPQ4x4en+G2M= github.com/git-pkgs/spdx v0.3.3 h1:xXlUoZzjDSkwxC6mWHfaW+kc8a0+LH6d+gGHuZWftqI= From f240d3023fccf8057ff3d80b4e67ce388a280155 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sat, 24 Oct 2026 13:13:12 +0200 Subject: [PATCH 415/633] Stream metadata cache hits for ProxyCached With cache_metadata on, every metadata cache hit read the whole stored document into its own buffer before any coalescing, since coalesceMetadata only covers upstream misses. metadata_max_size bounds one request, not the sum: 30 concurrent hits on a 20 MB document held about 820 MB. The handlers that go through ProxyCached serve the stored document unchanged, so a hit there no longer needs the bytes in memory. serveCachedMetadata streams the blob from storage to the response instead, with the row's Content-Type, Content-Encoding, size, ETag and Last-Modified, and answers If-None-Match and If-Modified-Since with 304 as before. The bytes are checked against the row's content digest as they are sent, and a mismatch aborts the response like a corrupt cached artifact. A row without a digest or size, a document over the size limit, or a blob that can't be opened takes the buffered path as before. Handlers that rewrite or filter metadata keep reading it into memory, and the rewrite cache stays in front of them. writeMetadataValidators is split out of writeMetadataCachedResponseWithEncoding so both paths share the conditional request handling. --- internal/handler/handler.go | 51 ++++-- internal/handler/metadata_stream.go | 77 +++++++++ internal/handler/metadata_stream_test.go | 202 +++++++++++++++++++++++ 3 files changed, 313 insertions(+), 17 deletions(-) create mode 100644 internal/handler/metadata_stream.go create mode 100644 internal/handler/metadata_stream_test.go diff --git a/internal/handler/handler.go b/internal/handler/handler.go index c06ec2d..164ad26 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -1861,6 +1861,11 @@ func (p *Proxy) lookupCachedMeta(ecosystem, cacheKey string) cachedMeta { if err != nil || entry == nil { return cachedMeta{} } + return p.cachedMetaFor(entry) +} + +// cachedMetaFor returns the cache validators and freshness state of entry. +func (p *Proxy) cachedMetaFor(entry *database.MetadataCacheEntry) cachedMeta { var cm cachedMeta if entry.ETag.Valid { cm.etag = entry.ETag.String @@ -1896,6 +1901,10 @@ func (p *Proxy) proxyCachedWithEncoding(w http.ResponseWriter, r *http.Request, return } + if p.serveCachedMetadata(w, r, ecosystem, cacheKey) { + return + } + body, contentType, contentEncoding, err := p.fetchOrCacheMetadata(r.Context(), ecosystem, cacheKey, upstreamURL, acceptEncoding, nil, acceptHeaders...) if err != nil { p.writeMetadataFetchError(w, err) @@ -1931,25 +1940,9 @@ func (p *Proxy) writeMetadataCachedResponse(w http.ResponseWriter, r *http.Reque // mislabel the bytes. func (p *Proxy) writeMetadataCachedResponseWithEncoding(w http.ResponseWriter, r *http.Request, ecosystem, cacheKey string, body []byte, contentType, contentEncoding string) { cm := p.lookupCachedMeta(ecosystem, cacheKey) - - if cm.etag != "" { - w.Header().Set(headerETag, cm.etag) - } - if !cm.lastModified.IsZero() { - w.Header().Set(headerLastModified, cm.lastModified.UTC().Format(http.TimeFormat)) - } - if ifNoneMatchHits(r.Header.Get("If-None-Match"), cm.etag) { - w.WriteHeader(http.StatusNotModified) + if writeMetadataValidators(w, r, cm) { return } - if !cm.lastModified.IsZero() { - if ims := r.Header.Get("If-Modified-Since"); ims != "" { - if t, err := http.ParseTime(ims); err == nil && !cm.lastModified.After(t) { - w.WriteHeader(http.StatusNotModified) - return - } - } - } w.Header().Set(headerContentType, contentType) w.Header().Set(headerContentLength, strconv.Itoa(len(body))) @@ -1963,6 +1956,30 @@ func (p *Proxy) writeMetadataCachedResponseWithEncoding(w http.ResponseWriter, r } } +// writeMetadataValidators sets cm's ETag and Last-Modified on w and answers +// a conditional request they satisfy with 304, reporting whether it did. +func writeMetadataValidators(w http.ResponseWriter, r *http.Request, cm cachedMeta) bool { + if cm.etag != "" { + w.Header().Set(headerETag, cm.etag) + } + if !cm.lastModified.IsZero() { + w.Header().Set(headerLastModified, cm.lastModified.UTC().Format(http.TimeFormat)) + } + if ifNoneMatchHits(r.Header.Get("If-None-Match"), cm.etag) { + w.WriteHeader(http.StatusNotModified) + return true + } + if !cm.lastModified.IsZero() { + if ims := r.Header.Get("If-Modified-Since"); ims != "" { + if t, err := http.ParseTime(ims); err == nil && !cm.lastModified.After(t) { + w.WriteHeader(http.StatusNotModified) + return true + } + } + } + return false +} + // proxyMetadataStream forwards an upstream metadata response by streaming it to the client // without buffering the full body in memory. func (p *Proxy) proxyMetadataStream(w http.ResponseWriter, r *http.Request, upstreamURL, acceptEncoding string, acceptHeaders ...string) { diff --git a/internal/handler/metadata_stream.go b/internal/handler/metadata_stream.go new file mode 100644 index 0000000..2fb994a --- /dev/null +++ b/internal/handler/metadata_stream.go @@ -0,0 +1,77 @@ +package handler + +import ( + "net/http" + "strconv" + "strings" + "time" + + "github.com/git-pkgs/proxy/internal/metrics" +) + +// serveCachedMetadata answers a ProxyCached request from a fresh metadata +// cache row by streaming the stored document, rather than reading it into +// memory first, so concurrent hits on a large document don't each hold a copy +// of it. The bytes are checked against the row's digest as they are sent, and +// a mismatch aborts the response the way a corrupt cached artifact does. +// +// It reports false, having written nothing, when the request should take the +// buffered path instead: no fresh row, a row without a digest or size, a +// document over the metadata size limit, or one that can't be opened. +func (p *Proxy) serveCachedMetadata(w http.ResponseWriter, r *http.Request, ecosystem, cacheKey string) bool { + if p.DB == nil || p.Storage == nil || p.MetadataTTL <= 0 || containsPathTraversal(cacheKey) { + return false + } + entry, err := p.DB.GetMetadataCache(ecosystem, cacheKey) + if err != nil || entry == nil || !entry.FetchedAt.Valid || time.Since(entry.FetchedAt.Time) >= p.MetadataTTL { + return false + } + limit := p.MetadataMaxSize + if limit <= 0 { + limit = defaultMetadataMaxSize + } + digest, ok := strings.CutPrefix(entry.ContentDigest.String, "sha256:") + if !entry.ContentDigest.Valid || !ok || !entry.Size.Valid || entry.Size.Int64 > limit { + return false + } + checks, err := newIntegrityChecks(digest, "") + if err != nil { + return false + } + stored, err := p.Storage.Open(r.Context(), entry.StoragePath) + if err != nil { + return false + } + body, err := checks.wrapFailOnMismatch(stored, func(reason string) { + p.Logger.Error("cached metadata failed integrity check", + "ecosystem", ecosystem, "key", cacheKey, "path", entry.StoragePath, "reason", reason) + }) + if err != nil { + _ = stored.Close() + return false + } + defer func() { _ = body.Close() }() + + p.recordMetadataRead(entry) + metrics.RecordMetadataCacheHit(ecosystem, int(entry.Size.Int64)) + + cm := p.cachedMetaFor(entry) + if writeMetadataValidators(w, r, cm) { + return true + } + contentType := contentTypeJSON + if entry.ContentType.Valid { + contentType = entry.ContentType.String + } + w.Header().Set(headerContentType, contentType) + w.Header().Set(headerContentLength, strconv.FormatInt(entry.Size.Int64, 10)) + if entry.ContentEncoding.String != "" { + w.Header().Set(headerContentEncoding, entry.ContentEncoding.String) + } + markStale(w, cm.stale) + w.WriteHeader(http.StatusOK) + if r.Method != http.MethodHead { + copyArtifactBody(w, body, entry.Size.Int64, false) + } + return true +} diff --git a/internal/handler/metadata_stream_test.go b/internal/handler/metadata_stream_test.go new file mode 100644 index 0000000..2874d9f --- /dev/null +++ b/internal/handler/metadata_stream_test.go @@ -0,0 +1,202 @@ +package handler + +import ( + "bytes" + "context" + "io" + "net/http" + "net/http/httptest" + "slices" + "strconv" + "strings" + "sync" + "testing" + "time" + + "github.com/git-pkgs/proxy/internal/storage" +) + +// stallingStorage hands out cached metadata readers that return the first +// head bytes and then block until release closes, so a test can see whether a +// response starts before the whole document has been read. +type stallingStorage struct { + storage.Storage + head int + release chan struct{} +} + +func (s *stallingStorage) Open(ctx context.Context, path string) (io.ReadCloser, error) { + rc, err := s.Storage.Open(ctx, path) + if err != nil || !strings.HasPrefix(path, "_metadata/") { + return rc, err + } + return &stallingReader{ReadCloser: rc, left: s.head, release: s.release}, nil +} + +type stallingReader struct { + io.ReadCloser + left int + release chan struct{} +} + +func (r *stallingReader) Read(p []byte) (int, error) { + if r.left == 0 { + <-r.release + return r.ReadCloser.Read(p) + } + if len(p) > r.left { + p = p[:r.left] + } + n, err := r.ReadCloser.Read(p) + r.left -= n + return n, err +} + +// cachedMetadataServer serves ProxyCached for one document, cached on the +// first request. +func cachedMetadataServer(t *testing.T, doc []byte) (*Proxy, *mockStorage, *httptest.Server) { + t.Helper() + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set(headerContentType, "application/x-test") + w.Header().Set(headerETag, `"v1"`) + _, _ = w.Write(doc) + })) + t.Cleanup(upstream.Close) + + proxy, _, store, _ := setupTestProxy(t) + proxy.CacheMetadata = true + proxy.MetadataTTL = time.Hour + proxy.HTTPClient = upstream.Client() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + proxy.ProxyCached(w, r, upstream.URL+"/doc", "test", "doc", "*/*") + })) + t.Cleanup(srv.Close) + + resp, err := http.Get(srv.URL) + if err != nil { + t.Fatalf("populating cache: %v", err) + } + _, _ = io.Copy(io.Discard, resp.Body) + _ = resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Fatalf("populating cache: status %d", resp.StatusCode) + } + return proxy, store, srv +} + +func TestConcurrentMetadataCacheHitsStream(t *testing.T) { + const callers = 10 + doc := bytes.Repeat([]byte("0123456789abcdef"), 1<<18) // 4 MB + proxy, store, srv := cachedMetadataServer(t, doc) + + gate := &stallingStorage{Storage: store, head: 64 << 10, release: make(chan struct{})} + proxy.Storage = gate + var releaseOnce sync.Once + release := func() { releaseOnce.Do(func() { close(gate.release) }) } + t.Cleanup(release) + + // Every hit has to start its response while its reader is stalled after + // the first 64 KB. A hit that reads the document into memory first can't + // send anything until release. + type result struct { + resp *http.Response + first []byte + err error + } + started := make(chan result, callers) + for range callers { + go func() { + resp, err := http.Get(srv.URL) + if err != nil { + started <- result{err: err} + return + } + first := make([]byte, 1024) + _, err = io.ReadFull(resp.Body, first) + started <- result{resp: resp, first: first, err: err} + }() + } + results := make([]result, 0, callers) + timeout := time.After(5 * time.Second) + streamed := 0 + for len(results) < callers { + select { + case res := <-started: + results = append(results, res) + case <-timeout: + // Let the stalled hits finish, so the server can shut down. + release() + timeout = nil + streamed = len(results) + } + } + release() + t.Cleanup(func() { + for _, res := range results { + if res.resp != nil { + _ = res.resp.Body.Close() + } + } + }) + if timeout == nil { + t.Fatalf("%d of %d concurrent cache hits started their response before the document was read to its end", streamed, callers) + } + + for _, res := range results { + if res.err != nil { + t.Fatalf("cache hit: %v", res.err) + } + rest, err := io.ReadAll(res.resp.Body) + _ = res.resp.Body.Close() + if err != nil { + t.Fatalf("reading cache hit: %v", err) + } + if body := slices.Concat(res.first, rest); !bytes.Equal(body, doc) { + t.Fatalf("cache hit returned %d bytes, want the %d-byte document", len(body), len(doc)) + } + if got := res.resp.Header.Get(headerContentType); got != "application/x-test" { + t.Errorf("Content-Type = %q, want application/x-test", got) + } + if got := res.resp.Header.Get(headerContentLength); got != strconv.Itoa(len(doc)) { + t.Errorf("Content-Length = %q, want %d", got, len(doc)) + } + if got := res.resp.Header.Get(headerETag); got != `"v1"` { + t.Errorf("ETag = %q, want %q", got, `"v1"`) + } + } + + req, _ := http.NewRequest(http.MethodGet, srv.URL, nil) + req.Header.Set("If-None-Match", `"v1"`) + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatalf("conditional cache hit: %v", err) + } + _ = resp.Body.Close() + if resp.StatusCode != http.StatusNotModified { + t.Errorf("conditional cache hit: status %d, want 304", resp.StatusCode) + } +} + +func TestCachedMetadataHitAbortsOnDigestMismatch(t *testing.T) { + doc := bytes.Repeat([]byte("metadata "), 1<<14) + _, store, srv := cachedMetadataServer(t, doc) + + corrupt := bytes.Clone(doc) + corrupt[len(corrupt)-1] ^= 0xff + store.mu.Lock() + store.files[metadataStoragePath("test", "doc")] = corrupt + store.mu.Unlock() + + resp, err := http.Get(srv.URL) + if err != nil { + t.Fatalf("cache hit: %v", err) + } + defer func() { _ = resp.Body.Close() }() + body, err := io.ReadAll(resp.Body) + if err == nil { + t.Fatalf("cache hit completed with %d bytes of a document that doesn't match its digest, want the response aborted", len(body)) + } + if len(body) >= len(doc) { + t.Errorf("aborted response still delivered %d of %d bytes", len(body), len(doc)) + } +} From 25f62da32211430c0b037b09e7adde75a2904973 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 25 Oct 2026 12:31:10 +0100 Subject: [PATCH 416/633] Queue only a row's own legacy metadata blob The case-variant lookup filtered on lower(name), which no index covers, so a legacy row's first lookup scanned every metadata_cache row in its ecosystem. Reclaim already skips a queued path a row still points at, so each row now queues just its own legacy blob once it has moved off it. On case-insensitive storage a case variant still at its legacy path can lose its blob that way, and is then fetched again. --- internal/database/queries.go | 18 ------- internal/handler/metadata_path.go | 61 ++++------------------ internal/handler/metadata_path_test.go | 71 +++++++++++++++++++------- 3 files changed, 61 insertions(+), 89 deletions(-) diff --git a/internal/database/queries.go b/internal/database/queries.go index 6d2bc18..4b039d8 100644 --- a/internal/database/queries.go +++ b/internal/database/queries.go @@ -1319,24 +1319,6 @@ func (db *DB) ForgetMetadataDocument(ecosystem, name, from, to string) error { return err } -// MetadataPath is a metadata cache row's name and storage path. -type MetadataPath struct { - Name string `db:"name"` - StoragePath string `db:"storage_path"` -} - -// MetadataCaseVariants returns the name and storage path of every metadata -// cache row in ecosystem whose name equals name ignoring case, name's own -// row included. lower folds ASCII only on SQLite. -func (db *DB) MetadataCaseVariants(ecosystem, name string) ([]MetadataPath, error) { - var rows []MetadataPath - err := db.Select(&rows, db.Rebind(` - SELECT name, storage_path FROM metadata_cache - WHERE ecosystem = ? AND lower(name) = lower(?) - `), ecosystem, name) - return rows, err -} - // UpsertMetadataCache writes entry and marks it accessed now, since a // document is stored because a request asked for it. func (db *DB) UpsertMetadataCache(entry *MetadataCacheEntry) error { diff --git a/internal/handler/metadata_path.go b/internal/handler/metadata_path.go index 337f2f8..4637909 100644 --- a/internal/handler/metadata_path.go +++ b/internal/handler/metadata_path.go @@ -6,8 +6,6 @@ import ( "encoding/hex" "fmt" "io" - "slices" - "strings" "github.com/git-pkgs/proxy/internal/database" ) @@ -47,22 +45,8 @@ func isSHA256Hex(s string) bool { } // migrateLegacyMetadataEntry handles a cache row whose blob is still at its -// legacy, unhashed path, and returns the row to use, or nil when there is -// none. On a case-insensitive filesystem keys differing only in case shared -// that blob, so it may hold another key's document. The blob is hashed and -// compared with the row's digest, once per row: -// -// - When it matches, the blob is this key's document. It is copied to the -// hashed path and the row moved there, so it keeps being served, as a -// stale fallback too. If the copy fails the row is served from where it -// is and checked again next time. -// - When it does not, or the row has no digest to check against, the row -// is pointed at the hashed path with its digest and validators cleared, -// so the document is fetched again and the blob never served. -// - When the blob cannot be read, nothing changes and nil is returned. -// -// Once no row's current path is a legacy blob of this key or a case variant -// of it, those blobs are queued for deletion. +// legacy, unhashed path. On a case-insensitive filesystem keys differing only +// in case shared that blob, so it is served only if it matches the row's digest. func (p *Proxy) migrateLegacyMetadataEntry(ctx context.Context, ecosystem, cacheKey string, entry *database.MetadataCacheEntry) *database.MetadataCacheEntry { if entry == nil || p.Storage == nil { return entry @@ -86,13 +70,13 @@ func (p *Proxy) migrateLegacyMetadataEntry(ctx context.Context, ecosystem, cache p.Logger.Warn("failed to drop legacy metadata row", "ecosystem", ecosystem, "key", cacheKey, "error", err) return nil } - p.queueLegacyMetadataBlobs(ecosystem, cacheKey) + p.queueLegacyMetadataBlob(ecosystem, cacheKey, legacy) return nil } if !p.moveLegacyMetadataBlob(ctx, entry, legacy, hashed) { return entry } - p.queueLegacyMetadataBlobs(ecosystem, cacheKey) + p.queueLegacyMetadataBlob(ecosystem, cacheKey, legacy) moved := *entry moved.StoragePath = hashed return &moved @@ -141,37 +125,10 @@ func (p *Proxy) moveLegacyMetadataBlob(ctx context.Context, entry *database.Meta return moved } -// queueLegacyMetadataBlobs queues the legacy blobs of cacheKey and its case -// variants for deletion, unless a row still reads one of them: on a -// case-insensitive filesystem a variant still at its legacy path is reading -// the same file. The last variant to move off its legacy path queues them -// all, so on a case-sensitive filesystem none is left behind. Reclaim also -// skips a path a row points at again. -func (p *Proxy) queueLegacyMetadataBlobs(ecosystem, cacheKey string) { - rows, err := p.DB.MetadataCaseVariants(ecosystem, cacheKey) - if err != nil { - p.Logger.Warn("failed to look up metadata case variants", "ecosystem", ecosystem, "key", cacheKey, "error", err) - return - } - for _, row := range rows { - legacy := legacyMetadataStoragePath(ecosystem, row.Name) - if legacy == metadataStoragePath(ecosystem, row.Name) { - continue - } - inUse := slices.ContainsFunc(rows, func(r database.MetadataPath) bool { - return strings.EqualFold(r.StoragePath, legacy) - }) - if inUse { - return - } - } - for _, row := range rows { - legacy := legacyMetadataStoragePath(ecosystem, row.Name) - if legacy == metadataStoragePath(ecosystem, row.Name) { - continue - } - if err := p.DB.QueuePendingDelete(legacy); err != nil { - p.Logger.Warn("failed to queue legacy metadata blob for deletion", "ecosystem", ecosystem, "key", row.Name, "error", err) - } +// queueLegacyMetadataBlob queues a row's legacy blob for deletion once the row +// no longer reads it. Reclaim skips it while another row still points there. +func (p *Proxy) queueLegacyMetadataBlob(ecosystem, cacheKey, legacy string) { + if err := p.DB.QueuePendingDelete(legacy); err != nil { + p.Logger.Warn("failed to queue legacy metadata blob for deletion", "ecosystem", ecosystem, "key", cacheKey, "error", err) } } diff --git a/internal/handler/metadata_path_test.go b/internal/handler/metadata_path_test.go index 9eba90c..b1f040c 100644 --- a/internal/handler/metadata_path_test.go +++ b/internal/handler/metadata_path_test.go @@ -236,34 +236,67 @@ func TestLegacyMetadataMismatchedBlobNotServed(t *testing.T) { } } -// TestLegacyMetadataSharedPathKeptForCaseVariant checks that a legacy blob is -// not queued for deletion while a key differing only in case still reads its -// own legacy path, which on a case-insensitive filesystem is the same file, -// and that both are queued once neither does. -func TestLegacyMetadataSharedPathKeptForCaseVariant(t *testing.T) { +// caseFoldingStorage stores paths differing only in case as one file, as a +// case-insensitive filesystem does. +type caseFoldingStorage struct{ storage.Storage } + +func (s caseFoldingStorage) Store(ctx context.Context, path string, r io.Reader) (int64, string, error) { + return s.Storage.Store(ctx, strings.ToLower(path), r) +} + +func (s caseFoldingStorage) Open(ctx context.Context, path string) (io.ReadCloser, error) { + return s.Storage.Open(ctx, strings.ToLower(path)) +} + +func (s caseFoldingStorage) Exists(ctx context.Context, path string) (bool, error) { + return s.Storage.Exists(ctx, strings.ToLower(path)) +} + +func (s caseFoldingStorage) Delete(ctx context.Context, path string) error { + return s.Storage.Delete(ctx, strings.ToLower(path)) +} + +func (s caseFoldingStorage) Size(ctx context.Context, path string) (int64, error) { + return s.Storage.Size(ctx, strings.ToLower(path)) +} + +// TestLegacyMetadataCaseVariantRefetchedAfterReclaim checks that on +// case-insensitive storage, a key whose legacy blob was shared with a case +// variant and deleted after that variant moved is fetched again rather than +// served. +func TestLegacyMetadataCaseVariantRefetchedAfterReclaim(t *testing.T) { + var requests atomic.Int32 upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(http.StatusBadGateway) + requests.Add(1) + _, _ = io.WriteString(w, "fresh") })) defer upstream.Close() proxy, db := fileStorageProxy(t, upstream.Client()) + proxy.Storage = caseFoldingStorage{proxy.Storage} - // Both rows hold the same document, so the test runs alike whether or - // not the filesystem folds case. - upper := seedLegacyMetadata(t, proxy, db, "github.com/Sirupsen/logrus", "doc", "doc", time.Now()) - lower := seedLegacyMetadata(t, proxy, db, "github.com/sirupsen/logrus", "doc", "doc", time.Now()) + const upper, lower = "github.com/Sirupsen/logrus", "github.com/sirupsen/logrus" + seedLegacyMetadata(t, proxy, db, upper, "doc", "doc", time.Now()) + seedLegacyMetadata(t, proxy, db, lower, "doc", "doc", time.Now()) - if _, _, err := proxy.FetchOrCacheMetadata(context.Background(), "go", "github.com/Sirupsen/logrus", upstream.URL+"/x"); err != nil { + if body, _, err := proxy.FetchOrCacheMetadata(context.Background(), "go", upper, upstream.URL+"/x"); err != nil || string(body) != "doc" { + t.Fatalf("FetchOrCacheMetadata(%s) = %q, %v; want the migrated document", upper, body, err) + } + // Reclaim deletes the queued legacy blob, which is also lower's. + for _, path := range pendingDeletes(t, db) { + if err := proxy.Storage.Delete(context.Background(), path); err != nil { + t.Fatal(err) + } + } + + body, _, err := proxy.FetchOrCacheMetadata(context.Background(), "go", lower, upstream.URL+"/x") + if err != nil { t.Fatal(err) } - if due := pendingDeletes(t, db); len(due) != 0 { - t.Fatalf("pending deletes = %v while a case variant still reads its legacy blob", due) + if string(body) != "fresh" || requests.Load() != 1 { + t.Errorf("served %q after %d upstream requests, want a fresh fetch", body, requests.Load()) } - if _, _, err := proxy.FetchOrCacheMetadata(context.Background(), "go", "github.com/sirupsen/logrus", upstream.URL+"/x"); err != nil { - t.Fatal(err) - } - due := pendingDeletes(t, db) - if !slices.Contains(due, upper) || !slices.Contains(due, lower) { - t.Errorf("pending deletes = %v, want %q and %q", due, upper, lower) + if got, want := metadataRowPath(t, db, lower), metadataStoragePath("go", lower); got != want { + t.Errorf("storage path = %q, want %q", got, want) } } From 224bce8f51cccdffde12a34604cd5fdde399ce3d Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 25 Oct 2026 12:31:33 +0100 Subject: [PATCH 417/633] Serve cached downloads when cooldown can't read publish times PyPI and npm downloads look up the publish time before the cache. When that lookup fails, a file that is already cached is now served, and only a cache miss fails with 502 or 503. The proxy keeps serving what it has while the upstream is unreachable, and nothing new is fetched unchecked. Also drops the unattached comment block in cooldown_fail_closed.go. --- internal/handler/cooldown_fail_closed.go | 23 +++++++--- internal/handler/cooldown_fail_closed_test.go | 42 +++++++++++++++++++ internal/handler/npm.go | 3 ++ internal/handler/pypi.go | 4 +- 4 files changed, 66 insertions(+), 6 deletions(-) diff --git a/internal/handler/cooldown_fail_closed.go b/internal/handler/cooldown_fail_closed.go index 3a0a7f4..da600c1 100644 --- a/internal/handler/cooldown_fail_closed.go +++ b/internal/handler/cooldown_fail_closed.go @@ -2,11 +2,6 @@ package handler import "net/http" -// Metadata for a package that cooldown or the denylist applies to is never -// served unfiltered. When the publish times can't be read, or the document -// can't be rewritten, the request fails instead, so a rate-limited or -// failing upstream can't let withheld versions through. - // cooldownApplies reports whether cooldown withholds new versions of the // package, taking package overrides and exemptions into account. func (p *Proxy) cooldownApplies(ecosystem, name string) bool { @@ -32,3 +27,21 @@ func (p *Proxy) writeCooldownLookupError(w http.ResponseWriter, ecosystem, name "ecosystem", ecosystem, "package", name, "error", err) http.Error(w, "failed to check cooldown", http.StatusBadGateway) } + +// serveCachedDespiteCooldownError serves a download from the cache when its +// publish time couldn't be read, and reports whether it did. A cached file was +// already let through once, so the proxy keeps serving it while the upstream +// is unreachable; only a cache miss fails. +func (p *Proxy) serveCachedDespiteCooldownError(w http.ResponseWriter, r *http.Request, ecosystem, name, version, filename string, lookupErr error) bool { + if p.versionDenied(ecosystem, name, version) { + return false + } + cached, err := p.GetCachedArtifact(r.Context(), ecosystem, name, version, filename) + if err != nil || cached == nil { + return false + } + p.Logger.Warn("cooldown: could not read publish times, serving cached file", + "ecosystem", ecosystem, "package", name, "version", version, "error", lookupErr) + ServeArtifactRequest(w, r, cached) + return true +} diff --git a/internal/handler/cooldown_fail_closed_test.go b/internal/handler/cooldown_fail_closed_test.go index ae3ae63..dc7cb98 100644 --- a/internal/handler/cooldown_fail_closed_test.go +++ b/internal/handler/cooldown_fail_closed_test.go @@ -225,6 +225,48 @@ func TestNPMDownloadCooldownFailsClosed(t *testing.T) { } } +// TestDownloadCooldownErrorServesCachedFile checks that a file already in the +// cache is still served when its publish time can't be read, while an uncached +// one fails. +func TestDownloadCooldownErrorServesCachedFile(t *testing.T) { + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + })) + defer upstream.Close() + + tests := []struct { + ecosystem, name, version, filename, path string + handler func(*Proxy) http.Handler + }{ + {"pypi", "newpkg", "2.0.0", "newpkg-2.0.0.tar.gz", "/packages/packages/ab/cd/ef0123456789/newpkg-2.0.0.tar.gz", + func(p *Proxy) http.Handler { + return (&PyPIHandler{proxy: p, upstreamURL: upstream.URL, downloadURL: upstream.URL, proxyURL: "http://localhost"}).Routes() + }}, + {"npm", "leftpad", "2.0.0", "leftpad-2.0.0.tgz", "/leftpad/-/leftpad-2.0.0.tgz", + func(p *Proxy) http.Handler { return NewNPMHandler(p, "http://proxy.test", upstream.URL).Routes() }}, + } + for _, tt := range tests { + t.Run(tt.ecosystem, func(t *testing.T) { + proxy, db, store, fetcher := setupTestProxy(t) + proxy.Cooldown = &cooldown.Config{Default: "7d"} + h := tt.handler(proxy) + + seedPackage(t, db, store, tt.ecosystem, tt.name, tt.version, tt.filename, "cached file") + w := serveGet(t, h, tt.path) + wantStatus(t, w, http.StatusOK, "") + if w.Body.String() != "cached file" { + t.Errorf("body = %q, want the cached file", w.Body.String()) + } + + other := strings.ReplaceAll(tt.path, "2.0.0", "3.0.0") + wantStatus(t, serveGet(t, h, other), http.StatusBadGateway, "") + if fetcher.fetchCalled { + t.Error("fetched a file whose publish time couldn't be checked") + } + }) + } +} + // TestCooldownRewriteFailureFailsClosed checks that a metadata document the // cooldown rewrite can't parse is refused rather than passed on, and still // passed on when cooldown doesn't apply. diff --git a/internal/handler/npm.go b/internal/handler/npm.go index c2256d2..fbeb7cf 100644 --- a/internal/handler/npm.go +++ b/internal/handler/npm.go @@ -927,6 +927,9 @@ func (h *NPMHandler) handleDownload(w http.ResponseWriter, r *http.Request) { }) inCooldown, err := h.versionInCooldown(packageName, version, metadata) if err != nil { + if h.proxy.serveCachedDespiteCooldownError(w, r, "npm", packageName, version, h.routes.cacheFilename(packageName, filename), err) { + return + } if writeUpstreamHeld(w, err) { return } diff --git a/internal/handler/pypi.go b/internal/handler/pypi.go index b61bb3b..e22471c 100644 --- a/internal/handler/pypi.go +++ b/internal/handler/pypi.go @@ -627,7 +627,9 @@ func (h *PyPIHandler) handleDownload(w http.ResponseWriter, r *http.Request) { if name != "" { inCooldown, err := h.versionInCooldown(r, name, version) if err != nil { - h.proxy.writeCooldownLookupError(w, "pypi", name, err) + if !h.proxy.serveCachedDespiteCooldownError(w, r, "pypi", name, version, filename, err) { + h.proxy.writeCooldownLookupError(w, "pypi", name, err) + } return } if inCooldown { From 4402075916696722f75f12b5f7c00bc91d0d9175 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 25 Oct 2026 12:32:17 +0100 Subject: [PATCH 418/633] Document serving cached downloads when publish times can't be read --- docs/configuration.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/configuration.md b/docs/configuration.md index 4712d69..d0e4455 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1042,7 +1042,7 @@ Maven caveats: - Maven Central lists versions in version order, not in the order they were published. A patch to an older release line published after a newer release that is out of cooldown stays listed while it is in cooldown, but its files are still refused. - SNAPSHOT versions are not held back. -RubyGems, Hex and PyPI keep their publish times outside the metadata clients read: the RubyGems versions API, the Hex HTTP API (`upstream.hex_api`) and the PyPI JSON API. The proxy fetches them alongside each compact index, package or simple page it filters, and for PyPI downloads. Cooldown fails closed: when it applies to a package and those times can't be fetched or parsed, or the metadata can't be rewritten, the request fails with 502, or with 503 and `Retry-After` while the upstream is rate limiting the proxy, instead of serving the metadata unfiltered. The same goes for npm, Composer and pub.dev metadata that can't be rewritten, and for npm tarball downloads whose publish time can't be read. Packages exempted with a duration of `0` are served as before and don't need the times. A PyPI package the JSON API returns 404 for has no times to go by and is served unfiltered. +RubyGems, Hex and PyPI keep their publish times outside the metadata clients read: the RubyGems versions API, the Hex HTTP API (`upstream.hex_api`) and the PyPI JSON API. The proxy fetches them alongside each compact index, package or simple page it filters, and for PyPI downloads. Cooldown fails closed: when it applies to a package and those times can't be fetched or parsed, or the metadata can't be rewritten, the request fails with 502, or with 503 and `Retry-After` while the upstream is rate limiting the proxy, instead of serving the metadata unfiltered. The same goes for npm, Composer and pub.dev metadata that can't be rewritten, and for npm tarball downloads whose publish time can't be read. A PyPI file or npm tarball that is already cached is still served when its publish time can't be read; only a download that would have to be fetched fails. Packages exempted with a duration of `0` are served as before and don't need the times. A PyPI package the JSON API returns 404 for has no times to go by and is served unfiltered. With cooldown on, the proxy asks npm for full packuments, since the abbreviated ones it uses otherwise carry no publish times. Abbreviated packuments cached before cooldown was turned on are fetched again in full rather than served, including as a stale copy when the upstream is down. From 607a55e7464901c8644db56569cbe1d04f2803f6 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 25 Oct 2026 12:36:36 +0100 Subject: [PATCH 419/633] Mirror PyPI packages proxy mirror and mirror jobs failed for pypi with "unsupported ecosystem", since the resolver they use has no registry for it. A PyPI version has several files, so the mirror now lists them from upstream.pypi's JSON API, through the same client as version listing, and fetches each one, verified against its sha256. The file_name qualifier limits it to one file. Files are cached under the name and version the download handler reads from the filename, which can differ from the PURL (zope.interface's wheels are zope_interface), so pip is served them from the cache. --- docs/configuration.md | 2 + internal/handler/pypi.go | 12 ++- internal/mirror/mirror.go | 72 ++++++++++++++ internal/mirror/source.go | 4 +- internal/mirror/versions.go | 43 ++++++-- internal/server/mirror_fetch_test.go | 4 +- internal/server/mirror_pypi_test.go | 140 +++++++++++++++++++++++++++ 7 files changed, 263 insertions(+), 14 deletions(-) create mode 100644 internal/server/mirror_pypi_test.go diff --git a/docs/configuration.md b/docs/configuration.md index 70a1861..c64a196 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1506,6 +1506,8 @@ The `proxy mirror` command pre-populates the cache from various sources. It acce Packages are downloaded from the configured upstreams: npm, cargo, gem, golang, hex, pub, maven and nuget from `upstream.npm`, `upstream.cargo_download`, `upstream.gem`, `upstream.go`, `upstream.hex`, `upstream.pub`, `upstream.maven` and `upstream.nuget`. npm packages in `upstream.npm_routes` come from their route and are cached under the route's keys, so clients get them from the cache as they would after an install. Mirror API jobs do the same. +A PyPI version is mirrored with every file `upstream.pypi` lists for it: the sdist and each wheel, checked against the listed sha256 and cached under the name and version the download handler reads from the filename. Add the `file_name` qualifier to mirror one file, for example `pkg:pypi/zope.interface@7.2?file_name=zope.interface-7.2-cp312-cp312-macosx_11_0_arm64.whl`. + | Flag | Default | Description | |------|---------|-------------| | `--sbom` | | Path to CycloneDX or SPDX SBOM file | diff --git a/internal/handler/pypi.go b/internal/handler/pypi.go index 021150f..3c89b73 100644 --- a/internal/handler/pypi.go +++ b/internal/handler/pypi.go @@ -649,13 +649,19 @@ var archiveExtensions = []string{".tar.gz", ".tar.bz2", ".tar.xz", ".tar.Z", ".t // bdist_msi formats, which share a filename layout. var windowsInstallerExtensions = []string{".exe", ".msi"} -// parseFilename extracts package name and version from a PyPI filename. -// Handles wheels, sdists and legacy bdist formats: +func (h *PyPIHandler) parseFilename(filename string) (name, version string) { + return PyPIFilenameIdentity(filename) +} + +// PyPIFilenameIdentity extracts the package name and version a PyPI file is +// cached under from its filename, so proxy mirror stores files where the +// download handler looks for them. Handles wheels, sdists and legacy bdist +// formats: // - requests-2.31.0-py3-none-any.whl // - requests-2.31.0.tar.gz // - numpy-1.8.0-py2.7-macosx-10.9-x86_64.egg // - numpy-1.8.0.win32-py2.7.exe -func (h *PyPIHandler) parseFilename(filename string) (name, version string) { +func PyPIFilenameIdentity(filename string) (name, version string) { // PEP 658/714 core-metadata sidecars are the distribution filename plus // ".metadata"; they describe the same name and version. Without this, pip's // metadata-only fetches fall back to a hash-derived package identifier. diff --git a/internal/mirror/mirror.go b/internal/mirror/mirror.go index 4b79ce9..ddf9736 100644 --- a/internal/mirror/mirror.go +++ b/internal/mirror/mirror.go @@ -3,6 +3,7 @@ package mirror import ( "context" + "errors" "fmt" "log/slog" "sync" @@ -226,6 +227,10 @@ func (m *Mirror) mirrorOne(ctx context.Context, pv PackageVersion, tracker *prog "ecosystem", pv.Ecosystem, "name", pv.Name, "version", pv.Version, "reason", m.RefusePackages) return } + if pv.Ecosystem == "pypi" { + m.mirrorFiles(ctx, pv, tracker) + return + } result, err := m.proxy.GetOrFetchArtifact(ctx, pv.Ecosystem, pv.Name, pv.Version, "") if err != nil { tracker.failed.Add(1) @@ -251,3 +256,70 @@ func (m *Mirror) mirrorOne(ctx context.Context, pv PackageVersion, tracker *prog "size", result.Artifact.Size) } } + +// mirrorFiles mirrors every file of a PyPI version, or only the one named by +// the file_name qualifier. Each file is cached under the name and version +// the download handler reads from its filename, so clients are served it. +func (m *Mirror) mirrorFiles(ctx context.Context, pv PackageVersion, tracker *progressTracker) { + fail := func(err error) { + tracker.failed.Add(1) + tracker.addError(pv.Ecosystem, pv.Name, pv.Version, err.Error()) + m.logger.Warn("mirror failed", + "ecosystem", pv.Ecosystem, "name", pv.Name, "version", pv.Version, "error", err) + } + files, err := m.fileLister().ListFiles(ctx, pv.Ecosystem, pv.Name, pv.Version) + if err != nil { + fail(fmt.Errorf("listing files: %w", err)) + return + } + wantFile := pv.Qualifiers["file_name"] + var matched, fetched int + var size int64 + for _, file := range files { + if wantFile != "" && file.Filename != wantFile { + continue + } + matched++ + name, version := handler.PyPIFilenameIdentity(file.Filename) + if name == "" { + name, version = pv.Name, pv.Version + } + result, err := m.proxy.GetOrFetchArtifactFromURLWithIntegrity(ctx, pv.Ecosystem, name, version, file.Filename, file.URL, file.Integrity) + if err != nil { + fail(fmt.Errorf("%s: %w", file.Filename, err)) + return + } + if result.Reader != nil { + _ = result.Reader.Close() + } + if !result.Cached { + fetched++ + size += result.Artifact.Size + } + } + switch { + case matched == 0 && wantFile != "": + fail(fmt.Errorf("no file named %q", wantFile)) + case matched == 0: + fail(errors.New("version has no files")) + case fetched == 0: + tracker.skipped.Add(1) + m.logger.Debug("already cached", + "ecosystem", pv.Ecosystem, "name", pv.Name, "version", pv.Version) + default: + tracker.completed.Add(1) + tracker.bytes.Add(size) + m.logger.Info("mirrored", + "ecosystem", pv.Ecosystem, "name", pv.Name, "version", pv.Version, + "files", fetched, "size", size) + } +} + +// fileLister lists files from where Versions lists versions, so both come +// from the configured upstream. +func (m *Mirror) fileLister() FileLister { //nolint:ireturn // any lister will do + if lister, ok := m.Versions.(FileLister); ok { + return lister + } + return RegistryVersions{} +} diff --git a/internal/mirror/source.go b/internal/mirror/source.go index a88a3a8..89596ad 100644 --- a/internal/mirror/source.go +++ b/internal/mirror/source.go @@ -14,7 +14,7 @@ type PackageVersion struct { Name string Version string // Qualifiers are the PURL qualifiers. Images use repository_url, tag and - // arch. + // arch; PyPI uses file_name to mirror one file of the version. Qualifiers map[string]string // Platforms limit which platforms of an image index are mirrored. Platforms []string @@ -78,6 +78,8 @@ func (s *PURLSource) Enumerate(ctx context.Context, fn func(PackageVersion) erro if isImageEcosystem(ecosystem) { pv.Qualifiers = p.Qualifiers.Map() pv.Platforms = s.Platforms + } else if fileName := p.Qualifier("file_name"); fileName != "" && ecosystem == "pypi" { + pv.Qualifiers = map[string]string{"file_name": fileName} } if err := fn(pv); err != nil { return err diff --git a/internal/mirror/versions.go b/internal/mirror/versions.go index f62102b..8eab11f 100644 --- a/internal/mirror/versions.go +++ b/internal/mirror/versions.go @@ -2,6 +2,7 @@ package mirror import ( "context" + "fmt" "github.com/git-pkgs/purl" "github.com/git-pkgs/registries" @@ -22,8 +23,40 @@ type RegistryVersions struct { URL func(purlType, name string) (string, error) } +// FileLister lists the files of one package version, for ecosystems that +// publish several files per version. +type FileLister interface { + ListFiles(ctx context.Context, ecosystem, name, version string) ([]registries.Artifact, error) +} + // ListVersions implements VersionLister. func (r RegistryVersions) ListVersions(ctx context.Context, ecosystem, name string) ([]string, error) { + versions, err := r.fetchVersions(ctx, ecosystem, name) + if err != nil { + return nil, err + } + result := make([]string, len(versions)) + for i, v := range versions { + result[i] = v.Number + } + return result, nil +} + +// ListFiles implements FileLister. +func (r RegistryVersions) ListFiles(ctx context.Context, ecosystem, name, version string) ([]registries.Artifact, error) { + versions, err := r.fetchVersions(ctx, ecosystem, name) + if err != nil { + return nil, err + } + for _, v := range versions { + if v.Number == version { + return v.Artifacts, nil + } + } + return nil, fmt.Errorf("%w: %s %s@%s", registries.ErrNotFound, ecosystem, name, version) +} + +func (r RegistryVersions) fetchVersions(ctx context.Context, ecosystem, name string) ([]registries.Version, error) { purlType := purl.EcosystemToPURLType(ecosystem) var baseURL string if r.URL != nil { @@ -40,13 +73,5 @@ func (r RegistryVersions) ListVersions(ctx context.Context, ecosystem, name stri if err != nil { return nil, err } - versions, err := reg.FetchVersions(ctx, name) - if err != nil { - return nil, err - } - result := make([]string, len(versions)) - for i, v := range versions { - result[i] = v.Number - } - return result, nil + return reg.FetchVersions(ctx, name) } diff --git a/internal/server/mirror_fetch_test.go b/internal/server/mirror_fetch_test.go index 2b3970d..66f8d57 100644 --- a/internal/server/mirror_fetch_test.go +++ b/internal/server/mirror_fetch_test.go @@ -52,7 +52,9 @@ func mirrorWithConfig(t *testing.T, cfg *config.Config, purls ...string) (*mirro } logger := slog.New(slog.NewTextHandler(io.Discard, nil)) proxy, _ := NewMirrorProxy(cfg, db, store, logger) - progress, err := mirror.New(proxy, db, store, logger, 1).Run(context.Background(), &mirror.PURLSource{PURLs: purls}) + m := mirror.New(proxy, db, store, logger, 1) + m.Versions = NewMirrorVersions(cfg.Upstream, proxy.HTTPClient) + progress, err := m.Run(context.Background(), &mirror.PURLSource{PURLs: purls}) if err != nil { t.Fatalf("Run() error = %v", err) } diff --git a/internal/server/mirror_pypi_test.go b/internal/server/mirror_pypi_test.go new file mode 100644 index 0000000..103e0f5 --- /dev/null +++ b/internal/server/mirror_pypi_test.go @@ -0,0 +1,140 @@ +package server + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + + "github.com/git-pkgs/proxy/internal/config" + "github.com/git-pkgs/proxy/internal/handler" +) + +const ( + mirrorWheelPath = "/packages/ab/cd/example_pkg-1.0-py3-none-any.whl" + mirrorSdistPath = "/packages/ef/01/example.pkg-1.0.tar.gz" +) + +// pypiUpstream serves the JSON API for example-pkg 1.0 with a wheel and an +// sdist, whose filenames spell the name differently, and counts file +// downloads by path. +type pypiUpstream struct { + *httptest.Server + mu sync.Mutex + files map[string]string + hits map[string]int +} + +func newPyPIUpstream(t *testing.T, digests map[string]string) *pypiUpstream { + t.Helper() + u := &pypiUpstream{ + files: map[string]string{mirrorWheelPath: "wheel", mirrorSdistPath: "sdist"}, + hits: map[string]int{}, + } + u.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/pypi/example-pkg/json" { + var files []map[string]any + for _, path := range []string{mirrorWheelPath, mirrorSdistPath} { + sum := sha256.Sum256([]byte(u.files[path])) + digest := hex.EncodeToString(sum[:]) + if d, ok := digests[path]; ok { + digest = d + } + files = append(files, map[string]any{ + "url": u.URL + path, + "digests": map[string]string{"sha256": digest}, + "size": len(u.files[path]), + }) + } + _ = json.NewEncoder(w).Encode(map[string]any{ + "info": map[string]any{"name": "example-pkg"}, + "releases": map[string]any{"1.0": files}, + }) + return + } + body, ok := u.files[r.URL.Path] + if !ok { + http.NotFound(w, r) + return + } + u.mu.Lock() + u.hits[r.URL.Path]++ + u.mu.Unlock() + _, _ = io.WriteString(w, body) + })) + t.Cleanup(u.Close) + return u +} + +func (u *pypiUpstream) hitsFor(path string) int { + u.mu.Lock() + defer u.mu.Unlock() + return u.hits[path] +} + +func pypiMirrorConfig(upstream *pypiUpstream) *config.Config { + cfg := config.Default() + cfg.Upstream.PyPI = upstream.URL + cfg.Upstream.PyPIDownload = upstream.URL + return cfg +} + +func TestMirrorPyPIFetchesEveryFile(t *testing.T) { + upstream := newPyPIUpstream(t, nil) + cfg := pypiMirrorConfig(upstream) + + progress, proxy := mirrorWithConfig(t, cfg, "pkg:pypi/example-pkg@1.0") + + if progress.Completed != 1 || progress.Failed != 0 { + t.Fatalf("completed = %d, failed = %d, errors = %v", progress.Completed, progress.Failed, progress.Errors) + } + for _, path := range []string{mirrorWheelPath, mirrorSdistPath} { + if got := upstream.hitsFor(path); got != 1 { + t.Errorf("%s fetched %d times, want 1", path, got) + } + } + + // pip gets both files from the cache once the upstream is down. + upstream.Close() + pypi := handler.NewPyPIHandlerWithUpstreams(proxy, "http://proxy.test", cfg.Upstream.PyPI, cfg.Upstream.PyPIDownload) + for path, want := range map[string]string{mirrorWheelPath: "wheel", mirrorSdistPath: "sdist"} { + rec := httptest.NewRecorder() + pypi.Routes().ServeHTTP(rec, httptest.NewRequest(http.MethodGet, path, nil)) + if rec.Code != http.StatusOK || rec.Body.String() != want { + t.Errorf("%s from cache: status = %d, body = %q, want 200 and %q", path, rec.Code, rec.Body, want) + } + } +} + +func TestMirrorPyPIFileNameQualifier(t *testing.T) { + upstream := newPyPIUpstream(t, nil) + + progress, _ := mirrorWithConfig(t, pypiMirrorConfig(upstream), "pkg:pypi/example-pkg@1.0?file_name=example.pkg-1.0.tar.gz") + + if progress.Completed != 1 || upstream.hitsFor(mirrorSdistPath) != 1 { + t.Errorf("completed = %d, sdist hits = %d, errors = %v; want the sdist", progress.Completed, upstream.hitsFor(mirrorSdistPath), progress.Errors) + } + if upstream.hitsFor(mirrorWheelPath) != 0 { + t.Error("the wheel was mirrored although file_name names the sdist") + } + + progress, _ = mirrorWithConfig(t, pypiMirrorConfig(upstream), "pkg:pypi/example-pkg@1.0?file_name=missing-1.0.tar.gz") + if progress.Failed != 1 || len(progress.Errors) != 1 || !strings.Contains(progress.Errors[0].Error, "missing-1.0.tar.gz") { + t.Errorf("failed = %d, errors = %v; want a failure naming the missing file", progress.Failed, progress.Errors) + } +} + +func TestMirrorPyPIRefusesDigestMismatch(t *testing.T) { + upstream := newPyPIUpstream(t, map[string]string{mirrorWheelPath: strings.Repeat("0", 64)}) + + progress, _ := mirrorWithConfig(t, pypiMirrorConfig(upstream), "pkg:pypi/example-pkg@1.0?file_name=example_pkg-1.0-py3-none-any.whl") + + if progress.Failed != 1 || progress.Completed != 0 { + t.Errorf("completed = %d, failed = %d, errors = %v; want the mismatching wheel refused", progress.Completed, progress.Failed, progress.Errors) + } +} From 775b92f1b81b302cd01d5913fb88672e2393a9c7 Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Sun, 25 Oct 2026 11:42:21 +0000 Subject: [PATCH 420/633] Mount an emptyDir at /tmp in the Helm chart (#590) The chart's root filesystem is read-only, so hosted Composer pushes failed writing their temporary zip. --- README.md | 2 ++ deploy/charts/proxy/templates/deployment.yaml | 9 +++++++++ deploy/charts/proxy/values.yaml | 4 ++++ 3 files changed, 15 insertions(+) diff --git a/README.md b/README.md index ebf9321..f977234 100644 --- a/README.md +++ b/README.md @@ -1683,6 +1683,8 @@ docker build -t proxy . docker run -p 8080:8080 -v proxy-data:/data proxy ``` +With `--read-only`, also pass `--tmpfs /tmp`: the proxy writes some uploads to a temporary file before reading them. + With Postgres and S3: ```bash diff --git a/deploy/charts/proxy/templates/deployment.yaml b/deploy/charts/proxy/templates/deployment.yaml index 0c8efb3..7a1027b 100644 --- a/deploy/charts/proxy/templates/deployment.yaml +++ b/deploy/charts/proxy/templates/deployment.yaml @@ -70,6 +70,8 @@ spec: readOnly: true - name: data mountPath: {{ .Values.persistence.mountPath }} + - name: tmp + mountPath: /tmp volumes: - name: config configMap: @@ -84,6 +86,13 @@ spec: {{- else }} emptyDir: {} {{- end }} + - name: tmp + emptyDir: + {{- with .Values.tmpDir.sizeLimit }} + sizeLimit: {{ . }} + {{- else }} + {} + {{- end }} {{- with .Values.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} diff --git a/deploy/charts/proxy/values.yaml b/deploy/charts/proxy/values.yaml index cb7a0bf..bdff879 100644 --- a/deploy/charts/proxy/values.yaml +++ b/deploy/charts/proxy/values.yaml @@ -103,6 +103,10 @@ containerSecurityContext: drop: - ALL +# /tmp is an emptyDir, since the root filesystem is read-only. +tmpDir: + sizeLimit: "" + resources: {} # requests: # cpu: 100m From 6c9505f3220c39d97fdd02acce000ad0ea788efc Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 25 Oct 2026 12:32:20 +0100 Subject: [PATCH 421/633] Drop cached metadata that fails its integrity check A streamed cache hit whose bytes don't match the row's digest now records an integrity failure and deletes the row, queueing its blob for deletion, as a corrupt cached artifact is handled. The next request fetches the document from upstream instead of aborting until the TTL runs out. --- internal/database/cache_eviction.go | 26 +++++++++++++++++ internal/handler/metadata_stream.go | 10 ++++++- internal/handler/metadata_stream_test.go | 37 +++++++++++++++++++----- 3 files changed, 64 insertions(+), 9 deletions(-) diff --git a/internal/database/cache_eviction.go b/internal/database/cache_eviction.go index e14df8d..33bcdae 100644 --- a/internal/database/cache_eviction.go +++ b/internal/database/cache_eviction.go @@ -145,6 +145,32 @@ func (db *DB) EvictMetadataCache(ecosystem, name, storagePath string, lastAccess return true, tx.Commit() } +// DiscardMetadataCache deletes a metadata cache row whose stored document +// failed its integrity check, if it still points at storagePath with digest, +// and queues the path for deletion in the same transaction. A row a newer +// fetch has rewritten is left alone. +func (db *DB) DiscardMetadataCache(ecosystem, name, storagePath, digest string) error { + tx, err := db.Beginx() + if err != nil { + return err + } + defer func() { _ = tx.Rollback() }() + res, err := tx.Exec(db.Rebind(` + DELETE FROM metadata_cache + WHERE ecosystem = ? AND name = ? AND storage_path = ? AND content_digest = ?`), + ecosystem, name, storagePath, digest) + if err != nil { + return err + } + if n, err := res.RowsAffected(); err != nil || n == 0 { + return err + } + if _, err := tx.Exec(db.Rebind(queuePendingDeleteQuery), storagePath, time.Now().UTC()); err != nil { + return err + } + return tx.Commit() +} + // RecordMetadataHit marks a cached metadata document as served now. Like // RecordArtifactHit it is buffered when BatchHits is on. func (db *DB) RecordMetadataHit(ecosystem, name string) error { diff --git a/internal/handler/metadata_stream.go b/internal/handler/metadata_stream.go index 2fb994a..8bb319d 100644 --- a/internal/handler/metadata_stream.go +++ b/internal/handler/metadata_stream.go @@ -1,6 +1,7 @@ package handler import ( + "errors" "net/http" "strconv" "strings" @@ -13,7 +14,8 @@ import ( // cache row by streaming the stored document, rather than reading it into // memory first, so concurrent hits on a large document don't each hold a copy // of it. The bytes are checked against the row's digest as they are sent, and -// a mismatch aborts the response the way a corrupt cached artifact does. +// a mismatch aborts the response and drops the row, the way a corrupt cached +// artifact is handled. // // It reports false, having written nothing, when the request should take the // buffered path instead: no fresh row, a row without a digest or size, a @@ -45,6 +47,12 @@ func (p *Proxy) serveCachedMetadata(w http.ResponseWriter, r *http.Request, ecos body, err := checks.wrapFailOnMismatch(stored, func(reason string) { p.Logger.Error("cached metadata failed integrity check", "ecosystem", ecosystem, "key", cacheKey, "path", entry.StoragePath, "reason", reason) + metrics.RecordIntegrityFailure(ecosystem, metrics.FailureDetail{ + Path: entry.StoragePath, Err: errors.New(reason)}) + // Drop the row so the next request fetches the document again. + if err := p.DB.DiscardMetadataCache(ecosystem, cacheKey, entry.StoragePath, entry.ContentDigest.String); err != nil { + p.Logger.Warn("failed to clear corrupt metadata from cache", "error", err) + } }) if err != nil { _ = stored.Close() diff --git a/internal/handler/metadata_stream_test.go b/internal/handler/metadata_stream_test.go index 2874d9f..702eea8 100644 --- a/internal/handler/metadata_stream_test.go +++ b/internal/handler/metadata_stream_test.go @@ -10,6 +10,7 @@ import ( "strconv" "strings" "sync" + "sync/atomic" "testing" "time" @@ -52,22 +53,24 @@ func (r *stallingReader) Read(p []byte) (int, error) { return n, err } -// cachedMetadataServer serves ProxyCached for one document, cached on the -// first request. -func cachedMetadataServer(t *testing.T, doc []byte) (*Proxy, *mockStorage, *httptest.Server) { +// cachedMetadataServer serves doc through ProxyCached and caches it with one +// request. upstreamHits counts the requests that reached the upstream. +func cachedMetadataServer(t *testing.T, doc []byte) (proxy *Proxy, store *mockStorage, srv *httptest.Server, upstreamHits *atomic.Int32) { t.Helper() + upstreamHits = &atomic.Int32{} upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + upstreamHits.Add(1) w.Header().Set(headerContentType, "application/x-test") w.Header().Set(headerETag, `"v1"`) _, _ = w.Write(doc) })) t.Cleanup(upstream.Close) - proxy, _, store, _ := setupTestProxy(t) + proxy, _, store, _ = setupTestProxy(t) proxy.CacheMetadata = true proxy.MetadataTTL = time.Hour proxy.HTTPClient = upstream.Client() - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { proxy.ProxyCached(w, r, upstream.URL+"/doc", "test", "doc", "*/*") })) t.Cleanup(srv.Close) @@ -81,13 +84,13 @@ func cachedMetadataServer(t *testing.T, doc []byte) (*Proxy, *mockStorage, *http if resp.StatusCode != http.StatusOK { t.Fatalf("populating cache: status %d", resp.StatusCode) } - return proxy, store, srv + return proxy, store, srv, upstreamHits } func TestConcurrentMetadataCacheHitsStream(t *testing.T) { const callers = 10 doc := bytes.Repeat([]byte("0123456789abcdef"), 1<<18) // 4 MB - proxy, store, srv := cachedMetadataServer(t, doc) + proxy, store, srv, _ := cachedMetadataServer(t, doc) gate := &stallingStorage{Storage: store, head: 64 << 10, release: make(chan struct{})} proxy.Storage = gate @@ -179,7 +182,7 @@ func TestConcurrentMetadataCacheHitsStream(t *testing.T) { func TestCachedMetadataHitAbortsOnDigestMismatch(t *testing.T) { doc := bytes.Repeat([]byte("metadata "), 1<<14) - _, store, srv := cachedMetadataServer(t, doc) + _, store, srv, upstreamHits := cachedMetadataServer(t, doc) corrupt := bytes.Clone(doc) corrupt[len(corrupt)-1] ^= 0xff @@ -199,4 +202,22 @@ func TestCachedMetadataHitAbortsOnDigestMismatch(t *testing.T) { if len(body) >= len(doc) { t.Errorf("aborted response still delivered %d of %d bytes", len(body), len(doc)) } + + // The corrupt row is dropped, so the next request fetches the document + // from upstream again instead of aborting until the TTL runs out. + resp, err = http.Get(srv.URL) + if err != nil { + t.Fatalf("request after abort: %v", err) + } + defer func() { _ = resp.Body.Close() }() + body, err = io.ReadAll(resp.Body) + if err != nil { + t.Fatalf("request after abort: %v", err) + } + if !bytes.Equal(body, doc) { + t.Errorf("request after abort got %d bytes, want the upstream document (%d bytes)", len(body), len(doc)) + } + if got := upstreamHits.Load(); got != 2 { + t.Errorf("upstream requests = %d, want 2 (populate, then refetch after the corrupt hit)", got) + } } From 16ae3deb1c491dc91dd8cb25f6555fb343692516 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 25 Oct 2026 14:25:35 +0100 Subject: [PATCH 422/633] Check mirrored files against their own SHA-256 Mirroring a PyPI version passed each file's sha256 as the integrity GetOrFetchArtifactFromURLWithIntegrity records with the version. Wheels of one version share the name and version they are cached under, so the last file's checksum replaced the others', and serving the other wheels from the cache failed the integrity check. The mirror now passes the sha256 as a digest, which is checked against that file's content hash alone. --- internal/mirror/mirror.go | 15 ++++++++++++++- internal/server/mirror_pypi_test.go | 21 +++++++++++---------- 2 files changed, 25 insertions(+), 11 deletions(-) diff --git a/internal/mirror/mirror.go b/internal/mirror/mirror.go index ddf9736..24d98ff 100644 --- a/internal/mirror/mirror.go +++ b/internal/mirror/mirror.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "log/slog" + "strings" "sync" "sync/atomic" "time" @@ -284,7 +285,7 @@ func (m *Mirror) mirrorFiles(ctx context.Context, pv PackageVersion, tracker *pr if name == "" { name, version = pv.Name, pv.Version } - result, err := m.proxy.GetOrFetchArtifactFromURLWithIntegrity(ctx, pv.Ecosystem, name, version, file.Filename, file.URL, file.Integrity) + result, err := m.proxy.GetOrFetchArtifactFromURLWithDigest(ctx, pv.Ecosystem, name, version, file.Filename, file.URL, fileDigest(file.Integrity)) if err != nil { fail(fmt.Errorf("%s: %w", file.Filename, err)) return @@ -315,6 +316,18 @@ func (m *Mirror) mirrorFiles(ctx context.Context, pv PackageVersion, tracker *pr } } +// fileDigest returns the SHA-256 of one file as a digest, which is checked +// against that file alone. The integrity GetOrFetchArtifactFromURLWithIntegrity +// takes is kept with the version instead, and the files of a version that +// share a name and version would overwrite each other's. Other checksums are +// not checked, as the download handlers don't check them either. +func fileDigest(integrity string) string { + if sum, ok := strings.CutPrefix(integrity, "sha256-"); ok { + return "sha256:" + sum + } + return "" +} + // fileLister lists files from where Versions lists versions, so both come // from the configured upstream. func (m *Mirror) fileLister() FileLister { //nolint:ireturn // any lister will do diff --git a/internal/server/mirror_pypi_test.go b/internal/server/mirror_pypi_test.go index 103e0f5..a591957 100644 --- a/internal/server/mirror_pypi_test.go +++ b/internal/server/mirror_pypi_test.go @@ -16,13 +16,14 @@ import ( ) const ( - mirrorWheelPath = "/packages/ab/cd/example_pkg-1.0-py3-none-any.whl" - mirrorSdistPath = "/packages/ef/01/example.pkg-1.0.tar.gz" + mirrorWheelPath = "/packages/ab/cd/example_pkg-1.0-py3-none-any.whl" + mirrorLinuxWheelPath = "/packages/23/45/example_pkg-1.0-cp312-cp312-manylinux_2_17_x86_64.whl" + mirrorSdistPath = "/packages/ef/01/example.pkg-1.0.tar.gz" ) -// pypiUpstream serves the JSON API for example-pkg 1.0 with a wheel and an -// sdist, whose filenames spell the name differently, and counts file -// downloads by path. +// pypiUpstream serves the JSON API for example-pkg 1.0 with two wheels, which +// are cached under the same name and version, and an sdist, whose filename +// spells the name differently. It counts file downloads by path. type pypiUpstream struct { *httptest.Server mu sync.Mutex @@ -33,13 +34,13 @@ type pypiUpstream struct { func newPyPIUpstream(t *testing.T, digests map[string]string) *pypiUpstream { t.Helper() u := &pypiUpstream{ - files: map[string]string{mirrorWheelPath: "wheel", mirrorSdistPath: "sdist"}, + files: map[string]string{mirrorWheelPath: "wheel", mirrorLinuxWheelPath: "linux wheel", mirrorSdistPath: "sdist"}, hits: map[string]int{}, } u.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.URL.Path == "/pypi/example-pkg/json" { var files []map[string]any - for _, path := range []string{mirrorWheelPath, mirrorSdistPath} { + for _, path := range []string{mirrorWheelPath, mirrorLinuxWheelPath, mirrorSdistPath} { sum := sha256.Sum256([]byte(u.files[path])) digest := hex.EncodeToString(sum[:]) if d, ok := digests[path]; ok { @@ -93,16 +94,16 @@ func TestMirrorPyPIFetchesEveryFile(t *testing.T) { if progress.Completed != 1 || progress.Failed != 0 { t.Fatalf("completed = %d, failed = %d, errors = %v", progress.Completed, progress.Failed, progress.Errors) } - for _, path := range []string{mirrorWheelPath, mirrorSdistPath} { + for path := range upstream.files { if got := upstream.hitsFor(path); got != 1 { t.Errorf("%s fetched %d times, want 1", path, got) } } - // pip gets both files from the cache once the upstream is down. + // pip gets every file from the cache once the upstream is down. upstream.Close() pypi := handler.NewPyPIHandlerWithUpstreams(proxy, "http://proxy.test", cfg.Upstream.PyPI, cfg.Upstream.PyPIDownload) - for path, want := range map[string]string{mirrorWheelPath: "wheel", mirrorSdistPath: "sdist"} { + for path, want := range upstream.files { rec := httptest.NewRecorder() pypi.Routes().ServeHTTP(rec, httptest.NewRequest(http.MethodGet, path, nil)) if rec.Code != http.StatusOK || rec.Body.String() != want { From 749934eb6c161f1f43905b1b00c14edfb7d5d118 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 25 Oct 2026 14:25:40 +0100 Subject: [PATCH 423/633] Mirror conda packages A conda version is mirrored with every build anaconda.org lists for it, each cached under the channel, name, version and subdir/filename the conda download handler reads from its path. The channel comes from the namespace or the channel qualifier; a package without one is refused, as retention keys without one are. The subdir, build and type qualifiers select builds. Which files a version has, which qualifiers select them and where the handler caches them is now described per ecosystem, so PyPI and conda share mirrorFiles. Part of #414. --- docs/configuration.md | 2 + internal/handler/conda.go | 8 +- internal/mirror/files.go | 132 ++++++++++++++++++++++ internal/mirror/files_test.go | 38 +++++++ internal/mirror/mirror.go | 37 +++--- internal/mirror/source.go | 21 ++-- internal/server/mirror_conda_test.go | 161 +++++++++++++++++++++++++++ internal/server/mirror_fetch_test.go | 12 +- internal/server/mirror_versions.go | 8 +- 9 files changed, 391 insertions(+), 28 deletions(-) create mode 100644 internal/mirror/files.go create mode 100644 internal/mirror/files_test.go create mode 100644 internal/server/mirror_conda_test.go diff --git a/docs/configuration.md b/docs/configuration.md index 5a877e3..49994ad 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1510,6 +1510,8 @@ Packages are downloaded from the configured upstreams: npm, cargo, gem, golang, A PyPI version is mirrored with every file `upstream.pypi` lists for it: the sdist and each wheel, checked against the listed sha256 and cached under the name and version the download handler reads from the filename. Add the `file_name` qualifier to mirror one file, for example `pkg:pypi/zope.interface@7.2?file_name=zope.interface-7.2-cp312-cp312-macosx_11_0_arm64.whl`. +A conda version is mirrored with every build anaconda.org lists for it, across all subdirs, and each build is cached the way `/conda/{channel}/{subdir}/{file}` serves it. Give the channel as the namespace or the `channel` qualifier, for example `pkg:conda/conda-forge/numpy@2.1.0`; a package without one is refused. The `subdir`, `build` and `type` qualifiers select builds, for example `pkg:conda/conda-forge/numpy@2.1.0?subdir=linux-64&type=conda`. Builds with a listed sha256 are checked against it. Files are listed from anaconda.org, so mirroring conda fails while `upstream.conda` points somewhere else. + | Flag | Default | Description | |------|---------|-------------| | `--sbom` | | Path to CycloneDX or SPDX SBOM file | diff --git a/internal/handler/conda.go b/internal/handler/conda.go index 1142ccf..b372468 100644 --- a/internal/handler/conda.go +++ b/internal/handler/conda.go @@ -142,9 +142,15 @@ func (h *CondaHandler) isPackageFile(filename string) bool { } // parseFilename extracts name and version from a Conda package filename. +func (h *CondaHandler) parseFilename(filename string) (name, version string) { + return CondaFilenameIdentity(filename) +} + +// CondaFilenameIdentity returns the name and version a download of filename +// is cached under, or empty strings when it is not a package file name. // Conda filenames are: {name}-{version}-{build}.{ext} // e.g., "numpy-1.24.0-py311h64a7726_0.conda" -func (h *CondaHandler) parseFilename(filename string) (name, version string) { +func CondaFilenameIdentity(filename string) (name, version string) { // Remove extension base := filename for _, ext := range []string{".conda", ".tar.bz2"} { diff --git a/internal/mirror/files.go b/internal/mirror/files.go new file mode 100644 index 0000000..8cf61fe --- /dev/null +++ b/internal/mirror/files.go @@ -0,0 +1,132 @@ +package mirror + +import ( + "fmt" + "slices" + "strings" + + "github.com/git-pkgs/proxy/internal/handler" + "github.com/git-pkgs/registries" +) + +// fileEcosystem describes an ecosystem whose versions publish several files, +// each mirrored on its own. +type fileEcosystem struct { + // qualifiers are the PURL qualifiers that select files. A file is + // mirrored when each one given matches the value attrs reports for it. + qualifiers []string + attrs func(file registries.Artifact) map[string]string + // cacheKey returns the name, version and filename the download handler + // caches file under, or false when the handler would not cache it. + cacheKey func(pv PackageVersion, file registries.Artifact) (name, version, filename string, ok bool) + // check refuses a package the ecosystem can't mirror, before its files + // are listed. + check func(pv PackageVersion) error +} + +var fileEcosystems = map[string]fileEcosystem{ + "pypi": { + qualifiers: []string{"file_name"}, + attrs: func(file registries.Artifact) map[string]string { + return map[string]string{"file_name": file.Filename} + }, + cacheKey: func(pv PackageVersion, file registries.Artifact) (string, string, string, bool) { + name, version := handler.PyPIFilenameIdentity(file.Filename) + if name == "" { + name, version = pv.Name, pv.Version + } + return name, version, file.Filename, true + }, + }, + "conda": { + qualifiers: []string{"subdir", "build", "type"}, + attrs: func(file registries.Artifact) map[string]string { + subdir, _ := file.Metadata["subdir"].(string) + build, _ := file.Metadata["build"].(string) + return map[string]string{"subdir": subdir, "build": build, "type": condaFileType(file.Filename)} + }, + cacheKey: func(pv PackageVersion, file registries.Artifact) (string, string, string, bool) { + channel, _, _ := strings.Cut(pv.Name, "/") + subdir, _ := file.Metadata["subdir"].(string) + name, version := handler.CondaFilenameIdentity(file.Filename) + if name == "" || subdir == "" || condaFileType(file.Filename) == "" { + return "", "", "", false + } + return channel + "/" + name, version, subdir + "/" + file.Filename, true + }, + check: func(pv PackageVersion) error { + if !strings.Contains(pv.Name, "/") { + return fmt.Errorf("conda packages need a channel, such as pkg:conda/conda-forge/%s", pv.Name) + } + return nil + }, + }, +} + +// condaFileType returns the PURL type qualifier of a conda package file. +func condaFileType(filename string) string { + switch { + case strings.HasSuffix(filename, ".conda"): + return "conda" + case strings.HasSuffix(filename, ".tar.bz2"): + return "tar.bz2" + default: + return "" + } +} + +// selection returns the qualifiers of pv that select files, as +// "key=value" pairs, for error messages. +func (e fileEcosystem) selection(pv PackageVersion) string { + var pairs []string + for _, key := range e.qualifiers { + if value := pv.Qualifiers[key]; value != "" { + pairs = append(pairs, key+"="+value) + } + } + return strings.Join(pairs, " ") +} + +// selects reports whether file matches every selecting qualifier of pv. +func (e fileEcosystem) selects(pv PackageVersion, file registries.Artifact) bool { + attrs := e.attrs(file) + for _, key := range e.qualifiers { + if want := pv.Qualifiers[key]; want != "" && attrs[key] != want { + return false + } + } + return true +} + +// fileQualifiers keeps the qualifiers of a PURL that select files of its +// ecosystem, or returns nil. +func fileQualifiers(ecosystem string, qualifiers map[string]string) map[string]string { + e, ok := fileEcosystems[ecosystem] + if !ok { + return nil + } + var kept map[string]string + for key, value := range qualifiers { + if value != "" && slices.Contains(e.qualifiers, key) { + if kept == nil { + kept = map[string]string{} + } + kept[key] = value + } + } + return kept +} + +// condaName returns a conda package name with its channel, taken from the +// namespace or the channel qualifier. Without either it stays bare, and +// mirroring refuses it. +func condaName(namespace, name, channel string) string { + switch { + case namespace != "": + return namespace + "/" + name + case channel != "": + return channel + "/" + name + default: + return name + } +} diff --git a/internal/mirror/files_test.go b/internal/mirror/files_test.go new file mode 100644 index 0000000..d84af72 --- /dev/null +++ b/internal/mirror/files_test.go @@ -0,0 +1,38 @@ +package mirror + +import ( + "reflect" + "testing" +) + +func TestPURLSourceCondaChannelAndQualifiers(t *testing.T) { + versions := &fakeVersions{versions: []string{"1.0", "1.1"}} + source := &PURLSource{ + PURLs: []string{ + "pkg:conda/conda-forge/numpy@2.1.0?subdir=linux-64&build=py312h_0&type=conda", + "pkg:conda/numpy@2.1.0?channel=conda-forge&repository_url=https://example.com", + "pkg:conda/numpy@2.1.0", + "pkg:conda/numpy", + "pkg:conda/numpy?channel=bioconda&subdir=noarch", + }, + Versions: versions, + } + + got := enumerate(t, source) + + want := []PackageVersion{ + {Ecosystem: "conda", Name: "conda-forge/numpy", Version: "2.1.0", + Qualifiers: map[string]string{"subdir": "linux-64", "build": "py312h_0", "type": "conda"}}, + {Ecosystem: "conda", Name: "conda-forge/numpy", Version: "2.1.0"}, + {Ecosystem: "conda", Name: "numpy", Version: "2.1.0"}, + {Ecosystem: "conda", Name: "numpy"}, + {Ecosystem: "conda", Name: "bioconda/numpy", Version: "1.0", Qualifiers: map[string]string{"subdir": "noarch"}}, + {Ecosystem: "conda", Name: "bioconda/numpy", Version: "1.1", Qualifiers: map[string]string{"subdir": "noarch"}}, + } + if !reflect.DeepEqual(got, want) { + t.Errorf("items = %+v\nwant %+v", got, want) + } + if !reflect.DeepEqual(versions.asked, []string{"conda/bioconda/numpy"}) { + t.Errorf("listed versions of %v, want only the package with a channel", versions.asked) + } +} diff --git a/internal/mirror/mirror.go b/internal/mirror/mirror.go index 24d98ff..2769161 100644 --- a/internal/mirror/mirror.go +++ b/internal/mirror/mirror.go @@ -228,8 +228,8 @@ func (m *Mirror) mirrorOne(ctx context.Context, pv PackageVersion, tracker *prog "ecosystem", pv.Ecosystem, "name", pv.Name, "version", pv.Version, "reason", m.RefusePackages) return } - if pv.Ecosystem == "pypi" { - m.mirrorFiles(ctx, pv, tracker) + if eco, ok := fileEcosystems[pv.Ecosystem]; ok { + m.mirrorFiles(ctx, pv, eco, tracker) return } result, err := m.proxy.GetOrFetchArtifact(ctx, pv.Ecosystem, pv.Name, pv.Version, "") @@ -258,34 +258,39 @@ func (m *Mirror) mirrorOne(ctx context.Context, pv PackageVersion, tracker *prog } } -// mirrorFiles mirrors every file of a PyPI version, or only the one named by -// the file_name qualifier. Each file is cached under the name and version -// the download handler reads from its filename, so clients are served it. -func (m *Mirror) mirrorFiles(ctx context.Context, pv PackageVersion, tracker *progressTracker) { +// mirrorFiles mirrors every file of a version, or those its qualifiers +// select. Each file is cached under the name and version the download +// handler reads from its filename, so clients are served it. +func (m *Mirror) mirrorFiles(ctx context.Context, pv PackageVersion, eco fileEcosystem, tracker *progressTracker) { fail := func(err error) { tracker.failed.Add(1) tracker.addError(pv.Ecosystem, pv.Name, pv.Version, err.Error()) m.logger.Warn("mirror failed", "ecosystem", pv.Ecosystem, "name", pv.Name, "version", pv.Version, "error", err) } + if eco.check != nil { + if err := eco.check(pv); err != nil { + fail(err) + return + } + } files, err := m.fileLister().ListFiles(ctx, pv.Ecosystem, pv.Name, pv.Version) if err != nil { fail(fmt.Errorf("listing files: %w", err)) return } - wantFile := pv.Qualifiers["file_name"] var matched, fetched int var size int64 for _, file := range files { - if wantFile != "" && file.Filename != wantFile { + if !eco.selects(pv, file) { + continue + } + name, version, filename, ok := eco.cacheKey(pv, file) + if !ok { continue } matched++ - name, version := handler.PyPIFilenameIdentity(file.Filename) - if name == "" { - name, version = pv.Name, pv.Version - } - result, err := m.proxy.GetOrFetchArtifactFromURLWithDigest(ctx, pv.Ecosystem, name, version, file.Filename, file.URL, fileDigest(file.Integrity)) + result, err := m.proxy.GetOrFetchArtifactFromURLWithDigest(ctx, pv.Ecosystem, name, version, filename, file.URL, fileDigest(file.Integrity)) if err != nil { fail(fmt.Errorf("%s: %w", file.Filename, err)) return @@ -298,9 +303,9 @@ func (m *Mirror) mirrorFiles(ctx context.Context, pv PackageVersion, tracker *pr size += result.Artifact.Size } } - switch { - case matched == 0 && wantFile != "": - fail(fmt.Errorf("no file named %q", wantFile)) + switch selection := eco.selection(pv); { + case matched == 0 && selection != "": + fail(fmt.Errorf("no file matches %s", selection)) case matched == 0: fail(errors.New("version has no files")) case fetched == 0: diff --git a/internal/mirror/source.go b/internal/mirror/source.go index 89596ad..d8778b5 100644 --- a/internal/mirror/source.go +++ b/internal/mirror/source.go @@ -3,6 +3,7 @@ package mirror import ( "context" "fmt" + "strings" "github.com/git-pkgs/purl" "github.com/git-pkgs/sbom" @@ -14,7 +15,8 @@ type PackageVersion struct { Name string Version string // Qualifiers are the PURL qualifiers. Images use repository_url, tag and - // arch; PyPI uses file_name to mirror one file of the version. + // arch; PyPI uses file_name, and conda subdir, build and type, to select + // which files of the version are mirrored. Qualifiers map[string]string // Platforms limit which platforms of an image index are mirrored. Platforms []string @@ -67,19 +69,20 @@ func (s *PURLSource) Enumerate(ctx context.Context, fn func(PackageVersion) erro ecosystem := handlerEcosystem(p.Type) name := p.Name - if p.Namespace != "" { + if ecosystem == "conda" { + name = condaName(p.Namespace, p.Name, p.Qualifier("channel")) + } else if p.Namespace != "" { name = p.Namespace + "/" + p.Name } + qualifiers := fileQualifiers(ecosystem, p.Qualifiers.Map()) - if p.Version != "" || isImageEcosystem(ecosystem) { - // Images are never expanded to all tags; an unversioned image - // fails when it is mirrored. - pv := PackageVersion{Ecosystem: ecosystem, Name: name, Version: p.Version} + // Images are never expanded to all tags, and conda packages without + // a channel aren't listed; both fail when they are mirrored. + if p.Version != "" || isImageEcosystem(ecosystem) || (ecosystem == "conda" && !strings.Contains(name, "/")) { + pv := PackageVersion{Ecosystem: ecosystem, Name: name, Version: p.Version, Qualifiers: qualifiers} if isImageEcosystem(ecosystem) { pv.Qualifiers = p.Qualifiers.Map() pv.Platforms = s.Platforms - } else if fileName := p.Qualifier("file_name"); fileName != "" && ecosystem == "pypi" { - pv.Qualifiers = map[string]string{"file_name": fileName} } if err := fn(pv); err != nil { return err @@ -93,7 +96,7 @@ func (s *PURLSource) Enumerate(ctx context.Context, fn func(PackageVersion) erro return fmt.Errorf("fetching versions for %s/%s: %w", ecosystem, name, err) } for _, v := range listed { - if err := fn(PackageVersion{Ecosystem: ecosystem, Name: name, Version: v}); err != nil { + if err := fn(PackageVersion{Ecosystem: ecosystem, Name: name, Version: v, Qualifiers: qualifiers}); err != nil { return err } } diff --git a/internal/server/mirror_conda_test.go b/internal/server/mirror_conda_test.go new file mode 100644 index 0000000..e75ac9c --- /dev/null +++ b/internal/server/mirror_conda_test.go @@ -0,0 +1,161 @@ +package server + +import ( + "crypto/md5" //nolint:gosec // anaconda.org gives older files only an MD5 + "crypto/sha256" + "encoding/hex" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + + "github.com/git-pkgs/proxy/internal/config" + "github.com/git-pkgs/proxy/internal/handler" + "github.com/git-pkgs/proxy/internal/mirror" + "github.com/git-pkgs/registries" +) + +const ( + condaLinuxFile = "linux-64/example-pkg-1.0-h123_0.conda" + condaNoarchFile = "noarch/example-pkg-1.0-pyh_0.tar.bz2" +) + +// condaUpstream serves the anaconda.org package API for +// conda-forge/example-pkg 1.0, with a linux-64 build that has a SHA-256 and +// a noarch build that, like older uploads, has only an MD5. It counts file +// downloads by subdir and filename. +type condaUpstream struct { + *httptest.Server + mu sync.Mutex + files map[string]string + hits map[string]int +} + +func newCondaUpstream(t *testing.T) *condaUpstream { + t.Helper() + u := &condaUpstream{ + files: map[string]string{condaLinuxFile: "linux build", condaNoarchFile: "noarch build"}, + hits: map[string]int{}, + } + u.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/package/conda-forge/example-pkg" { + linuxSum := sha256.Sum256([]byte(u.files[condaLinuxFile])) + noarchSum := md5.Sum([]byte(u.files[condaNoarchFile])) //nolint:gosec // see import + file := func(basename, build string) map[string]any { + subdir, _, _ := strings.Cut(basename, "/") + return map[string]any{ + "version": "1.0", + "basename": basename, + "download_url": u.URL + "/download/conda-forge/example-pkg/1.0/" + basename, + "attrs": map[string]any{"subdir": subdir, "build": build}, + "size": len(u.files[basename]), + } + } + linux := file(condaLinuxFile, "h123_0") + linux["sha256"] = hex.EncodeToString(linuxSum[:]) + noarch := file(condaNoarchFile, "pyh_0") + noarch["md5"] = hex.EncodeToString(noarchSum[:]) + _ = json.NewEncoder(w).Encode(map[string]any{ + "name": "example-pkg", + "versions": []string{"1.0"}, + "files": []any{linux, noarch}, + }) + return + } + basename := strings.TrimPrefix(r.URL.Path, "/download/conda-forge/example-pkg/1.0/") + body, ok := u.files[basename] + if !ok { + http.NotFound(w, r) + return + } + u.mu.Lock() + u.hits[basename]++ + u.mu.Unlock() + _, _ = io.WriteString(w, body) + })) + t.Cleanup(u.Close) + return u +} + +func (u *condaUpstream) hitsFor(basename string) int { + u.mu.Lock() + defer u.mu.Unlock() + return u.hits[basename] +} + +// mirrorConda mirrors purls with files listed from upstream. The proxy only +// lists conda files from anaconda.org itself, so the test points the lister +// at the fake directly. +func mirrorConda(t *testing.T, upstream *condaUpstream, purls ...string) (*mirror.Progress, *handler.Proxy) { + t.Helper() + versions := mirror.RegistryVersions{ + Client: registries.NewClient(registries.WithHTTPClient(upstream.Client())), + URL: func(string, string) (string, error) { return upstream.URL, nil }, + } + return mirrorWithVersions(t, config.Default(), versions, purls...) +} + +func TestMirrorCondaFetchesEveryFile(t *testing.T) { + upstream := newCondaUpstream(t) + + progress, proxy := mirrorConda(t, upstream, "pkg:conda/conda-forge/example-pkg@1.0") + + if progress.Completed != 1 || progress.Failed != 0 { + t.Fatalf("completed = %d, failed = %d, errors = %v", progress.Completed, progress.Failed, progress.Errors) + } + for _, basename := range []string{condaLinuxFile, condaNoarchFile} { + if got := upstream.hitsFor(basename); got != 1 { + t.Errorf("%s fetched %d times, want 1", basename, got) + } + } + + // conda gets both files from the cache once the upstream is down. + upstream.Close() + conda := handler.NewCondaHandlerWithUpstream(proxy, "http://proxy.test", upstream.URL) + for basename, want := range upstream.files { + rec := httptest.NewRecorder() + conda.Routes().ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/conda-forge/"+basename, nil)) + if rec.Code != http.StatusOK || rec.Body.String() != want { + t.Errorf("%s from cache: status = %d, body = %q, want 200 and %q", basename, rec.Code, rec.Body, want) + } + } + + progress, _ = mirrorConda(t, upstream, "pkg:conda/conda-forge/example-pkg@1.0") + if progress.Failed != 1 { + t.Errorf("failed = %d after the upstream closed, want 1", progress.Failed) + } +} + +func TestMirrorCondaQualifiersSelectFiles(t *testing.T) { + upstream := newCondaUpstream(t) + + progress, _ := mirrorConda(t, upstream, "pkg:conda/example-pkg@1.0?channel=conda-forge&subdir=noarch") + + if progress.Completed != 1 || upstream.hitsFor(condaNoarchFile) != 1 { + t.Errorf("completed = %d, noarch hits = %d, errors = %v; want the noarch build", progress.Completed, upstream.hitsFor(condaNoarchFile), progress.Errors) + } + if upstream.hitsFor(condaLinuxFile) != 0 { + t.Error("the linux-64 build was mirrored although subdir names noarch") + } + + progress, _ = mirrorConda(t, upstream, "pkg:conda/conda-forge/example-pkg@1.0?subdir=linux-64&type=tar.bz2") + if progress.Failed != 1 || len(progress.Errors) != 1 || !strings.Contains(progress.Errors[0].Error, "subdir=linux-64 type=tar.bz2") { + t.Errorf("failed = %d, errors = %v; want a failure naming the selection", progress.Failed, progress.Errors) + } +} + +func TestMirrorCondaNeedsChannel(t *testing.T) { + upstream := newCondaUpstream(t) + + progress, _ := mirrorConda(t, upstream, "pkg:conda/example-pkg@1.0") + + if progress.Failed != 1 || len(progress.Errors) != 1 || !strings.Contains(progress.Errors[0].Error, "channel") { + t.Errorf("failed = %d, errors = %v; want a failure asking for a channel", progress.Failed, progress.Errors) + } + if upstream.hitsFor(condaLinuxFile)+upstream.hitsFor(condaNoarchFile) != 0 { + t.Error("files were fetched for a package without a channel") + } +} diff --git a/internal/server/mirror_fetch_test.go b/internal/server/mirror_fetch_test.go index 66f8d57..142594c 100644 --- a/internal/server/mirror_fetch_test.go +++ b/internal/server/mirror_fetch_test.go @@ -36,6 +36,13 @@ func newTarballUpstream(t *testing.T, path, body string) (*httptest.Server, *ato // mirrorWithConfig mirrors purls with the proxy proxy mirror builds for cfg. func mirrorWithConfig(t *testing.T, cfg *config.Config, purls ...string) (*mirror.Progress, *handler.Proxy) { + t.Helper() + return mirrorWithVersions(t, cfg, nil, purls...) +} + +// mirrorWithVersions mirrors purls, listing versions and files with versions, +// or from the configured upstreams when it is nil. +func mirrorWithVersions(t *testing.T, cfg *config.Config, versions mirror.VersionLister, purls ...string) (*mirror.Progress, *handler.Proxy) { t.Helper() cfg.Upstream.AllowLoopback = true db, err := database.Create(t.TempDir() + "/test.db") @@ -53,7 +60,10 @@ func mirrorWithConfig(t *testing.T, cfg *config.Config, purls ...string) (*mirro logger := slog.New(slog.NewTextHandler(io.Discard, nil)) proxy, _ := NewMirrorProxy(cfg, db, store, logger) m := mirror.New(proxy, db, store, logger, 1) - m.Versions = NewMirrorVersions(cfg.Upstream, proxy.HTTPClient) + m.Versions = versions + if versions == nil { + m.Versions = NewMirrorVersions(cfg.Upstream, proxy.HTTPClient) + } progress, err := m.Run(context.Background(), &mirror.PURLSource{PURLs: purls}) if err != nil { t.Fatalf("Run() error = %v", err) diff --git a/internal/server/mirror_versions.go b/internal/server/mirror_versions.go index 9b981fd..a7b0689 100644 --- a/internal/server/mirror_versions.go +++ b/internal/server/mirror_versions.go @@ -2,6 +2,7 @@ package server import ( "cmp" + "errors" "fmt" "net/http" "strings" @@ -71,7 +72,12 @@ func mirrorVersionsURL(u config.UpstreamConfig, purlType, name string) (string, case "cargo": return publicOnly("upstream.cargo", u.Cargo, d.Cargo) case "conda": - return publicOnly("upstream.conda", u.Conda, d.Conda) + // Files are listed from anaconda.org even for a given version, so a + // version doesn't help here. + if upstreamOr(u.Conda, d.Conda) != strings.TrimSuffix(d.Conda, "/") { + return "", errors.New("mirroring conda packages from a custom upstream.conda isn't supported") + } + return "", nil case "julia": return publicOnly("upstream.julia", u.Julia, d.Julia) default: From ae9cf36f2623837d3d23718cce68b8e4d9f25465 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 25 Oct 2026 14:36:43 +0100 Subject: [PATCH 424/633] Mirror Composer releases Mirroring pkg:composer failed with unsupported ecosystem. A tagged release is now mirrored with the dist archive upstream.composer lists, under the file name the Composer handler serves it as and checked against the listed shasum. The release's publish time is stored as the handler stores it, so the archive is served from the cache without the metadata, with cooldown on as well. Branches are refused, since the handler reads the metadata for every branch download, and so are archive URLs with a query and packages in upstream.composer_routes. The mirror now finds the whole version rather than its files, for the dist type and publish time. Part of #414. --- docs/configuration.md | 2 + internal/handler/composer.go | 30 ++++++ internal/mirror/mirror.go | 78 +++++++++++---- internal/mirror/versions.go | 18 ++-- internal/server/mirror_composer_test.go | 126 ++++++++++++++++++++++++ internal/server/mirror_versions.go | 2 +- 6 files changed, 227 insertions(+), 29 deletions(-) create mode 100644 internal/server/mirror_composer_test.go diff --git a/docs/configuration.md b/docs/configuration.md index 49994ad..217d4f8 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1512,6 +1512,8 @@ A PyPI version is mirrored with every file `upstream.pypi` lists for it: the sdi A conda version is mirrored with every build anaconda.org lists for it, across all subdirs, and each build is cached the way `/conda/{channel}/{subdir}/{file}` serves it. Give the channel as the namespace or the `channel` qualifier, for example `pkg:conda/conda-forge/numpy@2.1.0`; a package without one is refused. The `subdir`, `build` and `type` qualifiers select builds, for example `pkg:conda/conda-forge/numpy@2.1.0?subdir=linux-64&type=conda`. Builds with a listed sha256 are checked against it. Files are listed from anaconda.org, so mirroring conda fails while `upstream.conda` points somewhere else. +A Composer release is mirrored with the dist archive `upstream.composer` lists for it, checked against its shasum when one is listed, and cached under the name `/composer/files/...` serves it as. Give the version as the repository lists it, for example `pkg:composer/monolog/monolog@3.8.1`. Branches such as `dev-main` aren't mirrored, since their archive changes with every push, and neither are packages in `upstream.composer_routes`. + | Flag | Default | Description | |------|---------|-------------| | `--sbom` | | Path to CycloneDX or SPDX SBOM file | diff --git a/internal/handler/composer.go b/internal/handler/composer.go index 92eae0a..bc3e17c 100644 --- a/internal/handler/composer.go +++ b/internal/handler/composer.go @@ -1011,6 +1011,36 @@ func composerIntegrity(shasum, version string, listed bool) string { return sha1Checksum(shasum) } +// ComposerReleaseArchive returns the file name the Composer handler caches a +// tagged release's archive under, and the integrity it checks it against, +// given the dist the release's metadata lists. It refuses versions the +// handler only serves after reading the metadata again: branches, whose +// archive changes with every push, and archives whose URL has a query, which +// clients ask for with one. +func ComposerReleaseArchive(version, distURL, distType, shasum string) (filename, declared string, err error) { + switch { + case isDevVersion(version): + return "", "", fmt.Errorf("%s is a branch; Composer branches are fetched when requested, not mirrored", version) + case distURL == "": + return "", "", fmt.Errorf("%s has no dist archive", version) + case strings.ContainsAny(distURL, "?#"): + return "", "", fmt.Errorf("the archive URL of %s has a query, which isn't mirrored", version) + } + return distFilename(distURL, distType), composerIntegrity(shasum, version, true), nil +} + +// RecordComposerPublishTime stores the publish time of a tagged release from +// the default repository, as the Composer handler does when it resolves the +// release's dist, so the archive is served from the cache without the +// metadata, with or without cooldown. +func (p *Proxy) RecordComposerPublishTime(packageName, version string, publishedAt time.Time) error { + if p.DB == nil || isDevVersion(version) || publishedAt.IsZero() { + return nil + } + return p.DB.SetVersionPublishedAt(canonicalVersionPURL("composer", packageName, version), + canonicalPackagePURL("composer", packageName), publishedAt) +} + // proxyUpstream forwards a request to packagist.org without caching. func (h *ComposerHandler) proxyUpstream(w http.ResponseWriter, r *http.Request) { upstreamURL := h.upstreamURL + r.URL.Path diff --git a/internal/mirror/mirror.go b/internal/mirror/mirror.go index 2769161..6bd8d18 100644 --- a/internal/mirror/mirror.go +++ b/internal/mirror/mirror.go @@ -232,15 +232,29 @@ func (m *Mirror) mirrorOne(ctx context.Context, pv PackageVersion, tracker *prog m.mirrorFiles(ctx, pv, eco, tracker) return } - result, err := m.proxy.GetOrFetchArtifact(ctx, pv.Ecosystem, pv.Name, pv.Version, "") - if err != nil { - tracker.failed.Add(1) - tracker.addError(pv.Ecosystem, pv.Name, pv.Version, err.Error()) - m.logger.Warn("mirror failed", - "ecosystem", pv.Ecosystem, "name", pv.Name, "version", pv.Version, "error", err) + if pv.Ecosystem == "composer" { + m.mirrorComposer(ctx, pv, tracker) return } + result, err := m.proxy.GetOrFetchArtifact(ctx, pv.Ecosystem, pv.Name, pv.Version, "") + if err != nil { + m.recordFailure(tracker, pv, err) + return + } + m.recordResult(tracker, pv, result) +} +// recordFailure counts and logs a package that could not be mirrored. +func (m *Mirror) recordFailure(tracker *progressTracker, pv PackageVersion, err error) { + tracker.failed.Add(1) + tracker.addError(pv.Ecosystem, pv.Name, pv.Version, err.Error()) + m.logger.Warn("mirror failed", + "ecosystem", pv.Ecosystem, "name", pv.Name, "version", pv.Version, "error", err) +} + +// recordResult closes the fetched artifact and counts it as mirrored, or as +// skipped when it was already cached. +func (m *Mirror) recordResult(tracker *progressTracker, pv PackageVersion, result *handler.CacheResult) { if result.Reader != nil { _ = result.Reader.Close() } @@ -258,30 +272,56 @@ func (m *Mirror) mirrorOne(ctx context.Context, pv PackageVersion, tracker *prog } } +// mirrorComposer mirrors the archive of a tagged Composer release under the +// file name the Composer handler serves it as, checked against the listed +// shasum, and records the release's publish time, which the handler needs to +// serve the archive from the cache without the metadata. +func (m *Mirror) mirrorComposer(ctx context.Context, pv PackageVersion, tracker *progressTracker) { + // Packagist names are lowercase, and so are the paths Composer requests. + name := strings.ToLower(pv.Name) + v, err := m.versionFinder().FindVersion(ctx, pv.Ecosystem, name, pv.Version) + if err != nil { + m.recordFailure(tracker, pv, err) + return + } + distURL, _ := v.Metadata["dist_url"].(string) + distType, _ := v.Metadata["dist_type"].(string) + filename, declared, err := handler.ComposerReleaseArchive(v.Number, distURL, distType, strings.TrimPrefix(v.Integrity, "sha1-")) + if err != nil { + m.recordFailure(tracker, pv, err) + return + } + result, err := m.proxy.GetOrFetchArtifactFromURLWithIntegrity(ctx, pv.Ecosystem, name, v.Number, filename, distURL, declared) + if err != nil { + m.recordFailure(tracker, pv, err) + return + } + if err := m.proxy.RecordComposerPublishTime(name, v.Number, v.PublishedAt); err != nil { + m.logger.Warn("failed to store composer publish time", + "name", name, "version", v.Number, "error", err) + } + m.recordResult(tracker, pv, result) +} + // mirrorFiles mirrors every file of a version, or those its qualifiers // select. Each file is cached under the name and version the download // handler reads from its filename, so clients are served it. func (m *Mirror) mirrorFiles(ctx context.Context, pv PackageVersion, eco fileEcosystem, tracker *progressTracker) { - fail := func(err error) { - tracker.failed.Add(1) - tracker.addError(pv.Ecosystem, pv.Name, pv.Version, err.Error()) - m.logger.Warn("mirror failed", - "ecosystem", pv.Ecosystem, "name", pv.Name, "version", pv.Version, "error", err) - } + fail := func(err error) { m.recordFailure(tracker, pv, err) } if eco.check != nil { if err := eco.check(pv); err != nil { fail(err) return } } - files, err := m.fileLister().ListFiles(ctx, pv.Ecosystem, pv.Name, pv.Version) + v, err := m.versionFinder().FindVersion(ctx, pv.Ecosystem, pv.Name, pv.Version) if err != nil { fail(fmt.Errorf("listing files: %w", err)) return } var matched, fetched int var size int64 - for _, file := range files { + for _, file := range v.Artifacts { if !eco.selects(pv, file) { continue } @@ -333,11 +373,11 @@ func fileDigest(integrity string) string { return "" } -// fileLister lists files from where Versions lists versions, so both come -// from the configured upstream. -func (m *Mirror) fileLister() FileLister { //nolint:ireturn // any lister will do - if lister, ok := m.Versions.(FileLister); ok { - return lister +// versionFinder finds versions where Versions lists them, so both come from +// the configured upstream. +func (m *Mirror) versionFinder() VersionFinder { //nolint:ireturn // any finder will do + if finder, ok := m.Versions.(VersionFinder); ok { + return finder } return RegistryVersions{} } diff --git a/internal/mirror/versions.go b/internal/mirror/versions.go index 8eab11f..1b68a6d 100644 --- a/internal/mirror/versions.go +++ b/internal/mirror/versions.go @@ -23,10 +23,10 @@ type RegistryVersions struct { URL func(purlType, name string) (string, error) } -// FileLister lists the files of one package version, for ecosystems that -// publish several files per version. -type FileLister interface { - ListFiles(ctx context.Context, ecosystem, name, version string) ([]registries.Artifact, error) +// VersionFinder finds one version of a package, with the files it +// publishes, for ecosystems the mirror fetches from the version's metadata. +type VersionFinder interface { + FindVersion(ctx context.Context, ecosystem, name, version string) (registries.Version, error) } // ListVersions implements VersionLister. @@ -42,18 +42,18 @@ func (r RegistryVersions) ListVersions(ctx context.Context, ecosystem, name stri return result, nil } -// ListFiles implements FileLister. -func (r RegistryVersions) ListFiles(ctx context.Context, ecosystem, name, version string) ([]registries.Artifact, error) { +// FindVersion implements VersionFinder. +func (r RegistryVersions) FindVersion(ctx context.Context, ecosystem, name, version string) (registries.Version, error) { versions, err := r.fetchVersions(ctx, ecosystem, name) if err != nil { - return nil, err + return registries.Version{}, err } for _, v := range versions { if v.Number == version { - return v.Artifacts, nil + return v, nil } } - return nil, fmt.Errorf("%w: %s %s@%s", registries.ErrNotFound, ecosystem, name, version) + return registries.Version{}, fmt.Errorf("%w: %s %s@%s", registries.ErrNotFound, ecosystem, name, version) } func (r RegistryVersions) fetchVersions(ctx context.Context, ecosystem, name string) ([]registries.Version, error) { diff --git a/internal/server/mirror_composer_test.go b/internal/server/mirror_composer_test.go new file mode 100644 index 0000000..b51aadf --- /dev/null +++ b/internal/server/mirror_composer_test.go @@ -0,0 +1,126 @@ +package server + +import ( + "crypto/sha1" //nolint:gosec // Packagist lists SHA-1 shasums + "encoding/hex" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + + "github.com/git-pkgs/cooldown" + "github.com/git-pkgs/proxy/internal/config" + "github.com/git-pkgs/proxy/internal/handler" +) + +const composerCommit = "0123456789abcdef0123456789abcdef01234567" + +// composerUpstream serves the Packagist API for acme/widget with release +// 1.0.0, published in 2020, and branch dev-main, and the 1.0.0 archive. +type composerUpstream struct { + *httptest.Server + archive string + shasum string + apiHits atomic.Int32 + distHits atomic.Int32 +} + +func newComposerUpstream(t *testing.T) *composerUpstream { + t.Helper() + u := &composerUpstream{archive: composerZip(`{"name":"acme/widget"}`)} + sum := sha1.Sum([]byte(u.archive)) //nolint:gosec // see import + u.shasum = hex.EncodeToString(sum[:]) + u.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/packages/acme/widget.json": + u.apiHits.Add(1) + dist := func(shasum string) map[string]string { + return map[string]string{"type": "zip", "url": u.URL + "/zipball/" + composerCommit, "shasum": shasum} + } + _ = json.NewEncoder(w).Encode(map[string]any{"package": map[string]any{ + "name": "acme/widget", + "versions": map[string]any{ + "1.0.0": map[string]any{"version": "1.0.0", "time": "2020-01-02T03:04:05+00:00", "dist": dist(u.shasum)}, + "dev-main": map[string]any{"version": "dev-main", "time": "2026-10-01T00:00:00+00:00", "dist": dist("")}, + }, + }}) + case "/zipball/" + composerCommit: + u.distHits.Add(1) + _, _ = io.WriteString(w, u.archive) + default: + http.NotFound(w, r) + } + })) + t.Cleanup(u.Close) + return u +} + +func composerMirrorConfig(upstream *composerUpstream) *config.Config { + cfg := config.Default() + cfg.Upstream.Composer = upstream.URL + cfg.Upstream.ComposerRepository = upstream.URL + return cfg +} + +func TestMirrorComposerReleaseIsServedFromCache(t *testing.T) { + upstream := newComposerUpstream(t) + cfg := composerMirrorConfig(upstream) + + progress, proxy := mirrorWithConfig(t, cfg, "pkg:composer/Acme/Widget@1.0.0") + + if progress.Completed != 1 || upstream.distHits.Load() != 1 { + t.Fatalf("completed = %d, dist hits = %d, errors = %v", progress.Completed, upstream.distHits.Load(), progress.Errors) + } + + // Composer gets the archive from the cache once the upstream is down, + // with cooldown on as well: the mirror recorded the publish time. + upstream.Close() + proxy.Cooldown = &cooldown.Config{Default: "3d"} + composer := handler.NewComposerHandlerWithUpstreams(proxy, "http://proxy.test", cfg.Upstream.Composer, cfg.Upstream.ComposerRepository) + rec := httptest.NewRecorder() + composer.Routes().ServeHTTP(rec, httptest.NewRequest(http.MethodGet, + "/files/acme/widget/1.0.0/"+composerCommit+".zip", nil)) + if rec.Code != http.StatusOK || rec.Body.String() != upstream.archive { + t.Errorf("archive from cache: status = %d, body %q; want 200 and the mirrored archive", rec.Code, strings.TrimSpace(rec.Body.String())) + } +} + +func TestMirrorComposerRefusesShasumMismatch(t *testing.T) { + upstream := newComposerUpstream(t) + upstream.shasum = strings.Repeat("0", 40) + + progress, _ := mirrorWithConfig(t, composerMirrorConfig(upstream), "pkg:composer/acme/widget@1.0.0") + + if progress.Failed != 1 || progress.Completed != 0 { + t.Errorf("completed = %d, failed = %d, errors = %v; want the archive refused", progress.Completed, progress.Failed, progress.Errors) + } +} + +func TestMirrorComposerRefusesBranches(t *testing.T) { + upstream := newComposerUpstream(t) + + progress, _ := mirrorWithConfig(t, composerMirrorConfig(upstream), "pkg:composer/acme/widget@dev-main") + + if progress.Failed != 1 || len(progress.Errors) != 1 || !strings.Contains(progress.Errors[0].Error, "branch") { + t.Errorf("failed = %d, errors = %v; want the branch refused", progress.Failed, progress.Errors) + } + if upstream.distHits.Load() != 0 { + t.Error("the branch archive was fetched") + } +} + +func TestMirrorComposerRefusesRoutedPackages(t *testing.T) { + upstream := newComposerUpstream(t) + cfg := composerMirrorConfig(upstream) + cfg.Upstream.ComposerRoutes = map[string]string{"acme/*": "https://composer.example.com"} + + progress, _ := mirrorWithConfig(t, cfg, "pkg:composer/acme/widget@1.0.0") + + if progress.Failed != 1 || upstream.apiHits.Load() != 0 || upstream.distHits.Load() != 0 { + t.Errorf("failed = %d, api hits = %d, dist hits = %d; want a routed package refused without asking Packagist", + progress.Failed, upstream.apiHits.Load(), upstream.distHits.Load()) + } +} diff --git a/internal/server/mirror_versions.go b/internal/server/mirror_versions.go index a7b0689..404388d 100644 --- a/internal/server/mirror_versions.go +++ b/internal/server/mirror_versions.go @@ -62,7 +62,7 @@ func mirrorVersionsURL(u config.UpstreamConfig, purlType, name string) (string, return upstreamOr(u.NuGet, d.NuGet) + "/v3", nil case "composer": if _, ok := handler.RoutedPackageURL(u.ComposerRoutes, name); ok { - return "", fmt.Errorf("listing versions of packages in upstream.composer_routes isn't supported; give a version") + return "", errors.New("mirroring packages in upstream.composer_routes isn't supported") } return upstreamOr(u.Composer, d.Composer), nil case "cran": From a2889d40a01adba435fd392fbac6712b9577957d Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 25 Oct 2026 17:21:11 +0100 Subject: [PATCH 425/633] Log per-handler request lines at debug level Each download and metadata handler logged its own request line at info, next to the request line LoggerMiddleware already writes for every request with path, status and duration. That doubled the info log volume on the hot path. The handler lines only repeat values parsed from the path, so they move to debug. Kept at info: the npm audit line (it carries the request body size), the julia package line (it carries the name resolved from the UUID) and the gem and hex cooldown lines. --- internal/handler/cargo.go | 4 ++-- internal/handler/composer.go | 4 ++-- internal/handler/conda.go | 2 +- internal/handler/container.go | 4 ++-- internal/handler/container_referrers.go | 2 +- internal/handler/filename_download.go | 2 +- internal/handler/go.go | 2 +- internal/handler/julia.go | 4 ++-- internal/handler/maven.go | 2 +- internal/handler/npm.go | 4 ++-- internal/handler/nuget.go | 4 ++-- internal/handler/pub.go | 4 ++-- internal/handler/pypi.go | 8 ++++---- 13 files changed, 23 insertions(+), 23 deletions(-) diff --git a/internal/handler/cargo.go b/internal/handler/cargo.go index d30d2e6..578c9ec 100644 --- a/internal/handler/cargo.go +++ b/internal/handler/cargo.go @@ -102,7 +102,7 @@ func (h *CargoHandler) handleIndex(w http.ResponseWriter, r *http.Request) { return } - h.proxy.Logger.Info("cargo index request", "crate", name) + h.proxy.Logger.Debug("cargo index request", "crate", name) indexPath := h.buildIndexPath(name) upstreamURL := fmt.Sprintf("%s/%s", h.indexURL, indexPath) @@ -214,7 +214,7 @@ func (h *CargoHandler) handleDownload(w http.ResponseWriter, r *http.Request) { filename := fmt.Sprintf("%s-%s.crate", name, version) - h.proxy.Logger.Info("cargo download request", + h.proxy.Logger.Debug("cargo download request", "crate", name, "version", version, "filename", filename) downloadURL := fmt.Sprintf( diff --git a/internal/handler/composer.go b/internal/handler/composer.go index 92eae0a..c2970e8 100644 --- a/internal/handler/composer.go +++ b/internal/handler/composer.go @@ -143,7 +143,7 @@ func (h *ComposerHandler) handlePackageMetadata(w http.ResponseWriter, r *http.R pkg := parts[1] packageName := vendor + "/" + pkg - h.proxy.Logger.Info("composer metadata request", "package", packageName) + h.proxy.Logger.Debug("composer metadata request", "package", packageName) repoURL, cacheKey := h.sourceFor(packageName) upstreamURL := fmt.Sprintf("%s/p2/%s/%s.json", repoURL, vendor, pkg) @@ -621,7 +621,7 @@ func (h *ComposerHandler) handleDownload(w http.ResponseWriter, r *http.Request) packageName := vendor + "/" + pkg - h.proxy.Logger.Info("composer download request", + h.proxy.Logger.Debug("composer download request", "package", packageName, "version", version, "filename", filename) local, err := h.proxy.GetLocalArtifact(r.Context(), "composer", packageName, version, filename) diff --git a/internal/handler/conda.go b/internal/handler/conda.go index 1142ccf..3b25275 100644 --- a/internal/handler/conda.go +++ b/internal/handler/conda.go @@ -119,7 +119,7 @@ func (h *CondaHandler) handleDownload(w http.ResponseWriter, r *http.Request) { // Include channel in package name packageName := channel + "/" + name - h.proxy.Logger.Info("conda download request", + h.proxy.Logger.Debug("conda download request", "channel", channel, "arch", arch, "name", name, "version", version) upstreamURL := h.upstreamURL + r.URL.Path diff --git a/internal/handler/container.go b/internal/handler/container.go index 57c30d8..394d98d 100644 --- a/internal/handler/container.go +++ b/internal/handler/container.go @@ -393,7 +393,7 @@ func (h *ContainerHandler) handleBlobDownload(w http.ResponseWriter, r *http.Req return } - h.proxy.Logger.Info("container blob request", "name", upstreamName, "digest", digest) + h.proxy.Logger.Debug("container blob request", "name", upstreamName, "digest", digest) filename := digest cached, err := h.proxy.GetCachedArtifact(r.Context(), "oci", cacheName, digest, filename) @@ -481,7 +481,7 @@ func (h *ContainerHandler) handleManifest(w http.ResponseWriter, r *http.Request return } - h.proxy.Logger.Info("container manifest request", "name", upstreamName, "reference", reference) + h.proxy.Logger.Debug("container manifest request", "name", upstreamName, "reference", reference) if h.manifestDenied(w, cacheName, reference) { return } diff --git a/internal/handler/container_referrers.go b/internal/handler/container_referrers.go index 37261c5..94904f8 100644 --- a/internal/handler/container_referrers.go +++ b/internal/handler/container_referrers.go @@ -55,7 +55,7 @@ func (h *ContainerHandler) handleReferrers(w http.ResponseWriter, r *http.Reques return } - h.proxy.Logger.Info("container referrers request", "name", upstreamName, "digest", digest) + h.proxy.Logger.Debug("container referrers request", "name", upstreamName, "digest", digest) h.serveReferrers(w, r, registryURL, upstreamName, digest) } diff --git a/internal/handler/filename_download.go b/internal/handler/filename_download.go index fc96586..153c0e6 100644 --- a/internal/handler/filename_download.go +++ b/internal/handler/filename_download.go @@ -27,7 +27,7 @@ func (p *Proxy) handleFilenameDownload(w http.ResponseWriter, r *http.Request, d return } - p.Logger.Info(d.ecosystem+" download request", + p.Logger.Debug(d.ecosystem+" download request", "name", name, "version", version, "filename", filename) downloadURL := d.upstreamURL + r.URL.Path diff --git a/internal/handler/go.go b/internal/handler/go.go index e544cd7..44311bf 100644 --- a/internal/handler/go.go +++ b/internal/handler/go.go @@ -143,7 +143,7 @@ func (h *GoHandler) handleDownload(w http.ResponseWriter, r *http.Request, modul decodedModule := decodeGoModule(module) filename := goZipFilename(decodedModule, version) - h.proxy.Logger.Info("go module download request", + h.proxy.Logger.Debug("go module download request", "module", decodedModule, "version", version) // The URL escapes uppercase letters in the version (v1.0.0-!r!c1 for diff --git a/internal/handler/julia.go b/internal/handler/julia.go index aaac731..67693b2 100644 --- a/internal/handler/julia.go +++ b/internal/handler/julia.go @@ -107,7 +107,7 @@ func (h *JuliaHandler) handleRegistry(w http.ResponseWriter, r *http.Request) { return } - h.proxy.Logger.Info("julia registry request", "uuid", uuid, "hash", hash) + h.proxy.Logger.Debug("julia registry request", "uuid", uuid, "hash", hash) upstreamURL := h.upstreamURL + r.URL.Path result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "julia", juliaRegistryName, hash, hash+".tar.gz", upstreamURL) @@ -156,7 +156,7 @@ func (h *JuliaHandler) handleArtifact(w http.ResponseWriter, r *http.Request) { return } - h.proxy.Logger.Info("julia artifact request", "hash", hash) + h.proxy.Logger.Debug("julia artifact request", "hash", hash) upstreamURL := h.upstreamURL + r.URL.Path result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "julia", juliaArtifactName, hash, hash+".tar.gz", upstreamURL) diff --git a/internal/handler/maven.go b/internal/handler/maven.go index 7bb035e..6c6a647 100644 --- a/internal/handler/maven.go +++ b/internal/handler/maven.go @@ -236,7 +236,7 @@ func (h *MavenHandler) handleDownload(w http.ResponseWriter, r *http.Request, so // Maven uses group:artifact as the package name name := fmt.Sprintf("%s:%s", group, artifact) - h.proxy.Logger.Info("maven download request", "repository", source.name, + h.proxy.Logger.Debug("maven download request", "repository", source.name, "group", group, "artifact", artifact, "version", version, "filename", filename) upstreamURL := fmt.Sprintf("%s/%s", source.upstreamURL, urlPath) diff --git a/internal/handler/npm.go b/internal/handler/npm.go index fbeb7cf..b749a40 100644 --- a/internal/handler/npm.go +++ b/internal/handler/npm.go @@ -214,7 +214,7 @@ func (h *NPMHandler) handlePackageMetadata(w http.ResponseWriter, r *http.Reques return } - h.proxy.Logger.Info("npm metadata request", "package", packageName, "path", packagePath) + h.proxy.Logger.Debug("npm metadata request", "package", packageName, "path", packagePath) registryURL, cacheKey := h.sourceFor(packageName) // A version or dist-tag is its own path segment. Escaping the whole path @@ -897,7 +897,7 @@ func (h *NPMHandler) handleDownload(w http.ResponseWriter, r *http.Request) { return } - h.proxy.Logger.Info("npm download request", + h.proxy.Logger.Debug("npm download request", "package", packageName, "version", version, "filename", filename) if h.proxy.versionDenied("npm", packageName, version) { diff --git a/internal/handler/nuget.go b/internal/handler/nuget.go index 2989298..7a5b1b2 100644 --- a/internal/handler/nuget.go +++ b/internal/handler/nuget.go @@ -88,7 +88,7 @@ func (h *NuGetHandler) Routes() http.Handler { // handleServiceIndex serves the NuGet V3 service index with rewritten URLs. func (h *NuGetHandler) handleServiceIndex(w http.ResponseWriter, r *http.Request) { - h.proxy.Logger.Info("nuget service index request") + h.proxy.Logger.Debug("nuget service index request") upstreamURL := h.upstreamURL + "/v3/index.json" @@ -203,7 +203,7 @@ func (h *NuGetHandler) handleDownload(w http.ResponseWriter, r *http.Request) { return } - h.proxy.Logger.Info("nuget download request", + h.proxy.Logger.Debug("nuget download request", "id", id, "version", version, "filename", filename) // NuGet package IDs are case-insensitive, lowercase for storage diff --git a/internal/handler/pub.go b/internal/handler/pub.go index 05670f6..3c81298 100644 --- a/internal/handler/pub.go +++ b/internal/handler/pub.go @@ -91,7 +91,7 @@ func (h *PubHandler) handleDownload(w http.ResponseWriter, r *http.Request) { filename := name + "-" + version + pubArchiveExt - h.proxy.Logger.Info("pub download request", + h.proxy.Logger.Debug("pub download request", "name", name, "version", version) downloadURL := fmt.Sprintf("%s/packages/%s/versions/%s%s", @@ -142,7 +142,7 @@ func (h *PubHandler) handlePackageMetadata(w http.ResponseWriter, r *http.Reques return } - h.proxy.Logger.Info("pub metadata request", "package", name) + h.proxy.Logger.Debug("pub metadata request", "package", name) upstreamURL := fmt.Sprintf("%s/api/packages/%s", h.upstreamURL, name) diff --git a/internal/handler/pypi.go b/internal/handler/pypi.go index 5667501..bcd4aa9 100644 --- a/internal/handler/pypi.go +++ b/internal/handler/pypi.go @@ -101,7 +101,7 @@ func (h *PyPIHandler) handleSimplePackage(w http.ResponseWriter, r *http.Request return } - h.proxy.Logger.Info("pypi simple request", "package", name) + h.proxy.Logger.Debug("pypi simple request", "package", name) upstreamURL := fmt.Sprintf("%s/simple/%s/", h.upstreamURL, name) accept := selectPyPISimpleRepresentation(r.Header.Get("Accept")) @@ -392,7 +392,7 @@ func (h *PyPIHandler) handleJSON(w http.ResponseWriter, r *http.Request) { return } - h.proxy.Logger.Info("pypi json request", "package", name) + h.proxy.Logger.Debug("pypi json request", "package", name) upstreamURL := fmt.Sprintf("%s/pypi/%s/json", h.upstreamURL, name) h.proxyAndRewriteJSON(w, r, upstreamURL, name+"/json") @@ -412,7 +412,7 @@ func (h *PyPIHandler) handleVersionJSON(w http.ResponseWriter, r *http.Request) return } - h.proxy.Logger.Info("pypi version json request", "package", name, "version", version) + h.proxy.Logger.Debug("pypi version json request", "package", name, "version", version) upstreamURL := fmt.Sprintf("%s/pypi/%s/%s/json", h.upstreamURL, name, version) h.proxyAndRewriteJSON(w, r, upstreamURL, name+"/"+version) @@ -646,7 +646,7 @@ func (h *PyPIHandler) handleDownload(w http.ResponseWriter, r *http.Request) { version = "0" } - h.proxy.Logger.Info("pypi download request", + h.proxy.Logger.Debug("pypi download request", "name", name, "version", version, "filename", filename) // The path value starts with 'packages/' (no leading slash), so add From 77bec54bb5c1a310a9a7a978bffec5b40e122735 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 25 Oct 2026 17:19:53 +0100 Subject: [PATCH 426/633] Verify cache hits against the content hash only A cache hit hashed the stored object with SHA-256 for content_hash and again with every native algorithm in versions.integrity, SHA-512 for npm. The native integrity was already checked when the bytes were stored and content_hash was computed from the same bytes, so the second hash adds CPU per byte served without catching anything more. Records without a content hash still fall back to the native integrity. --- docs/architecture.md | 4 +- internal/handler/handler.go | 2 +- internal/handler/handler_test.go | 63 ++++++++++++++++++++++++++++++ internal/handler/integrity.go | 13 ++++++ internal/handler/integrity_test.go | 34 ++++++++++++++++ 5 files changed, 114 insertions(+), 2 deletions(-) diff --git a/docs/architecture.md b/docs/architecture.md index 76b2156..b5f4ba9 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -51,7 +51,9 @@ Metadata is not cached - always fetched fresh. This ensures clients see new vers - Open file from storage - Record hit (increment counter, update last_accessed_at) - Return reader to handler - - Handler streams file to client + - Handler streams file to client, checking it against the recorded + SHA-256 `content_hash` (the registry's own integrity was checked when the + file was stored); a mismatch aborts the response and clears the record **Cache Miss:** - Resolve download URL using Resolver diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 2e0d688..af4d329 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -485,7 +485,7 @@ func (p *Proxy) readStored(ctx context.Context, pkgPURL, versionPURL, filename s return nil, nil } - result.Reader, err = checks.wrapFailOnMismatch(reader, + result.Reader, err = checks.forStoredObject().wrapFailOnMismatch(reader, func(reason string) { p.Logger.Error("cached artifact failed integrity check", "purl", versionPURL, "filename", filename, diff --git a/internal/handler/handler_test.go b/internal/handler/handler_test.go index ff5a080..cf3b007 100644 --- a/internal/handler/handler_test.go +++ b/internal/handler/handler_test.go @@ -442,6 +442,69 @@ func TestCorruptCachedArtifactIsQueuedForDeletion(t *testing.T) { assertQueuedForDeletion(t, db, storagePath) } +// TestCachedArtifactVerifiedByContentHash checks that a cache hit is verified +// against the recorded content hash only. The version's native integrity is +// one value per version and was checked when the bytes were stored; a stale +// or foreign value there must not discard an artifact whose bytes still +// match their SHA-256. +func TestCachedArtifactVerifiedByContentHash(t *testing.T) { + proxy, db, store, _ := setupTestProxy(t) + seedPackage(t, db, store, "npm", "hashed", "1.0.0", "hashed-1.0.0.tgz", "cached content") + setVersionIntegrity(t, db, "npm", "hashed", "1.0.0", sha512SRI("other bytes")) + + result, err := proxy.GetCachedArtifact(context.Background(), "npm", "hashed", "1.0.0", "hashed-1.0.0.tgz") + if err != nil || result == nil { + t.Fatalf("GetCachedArtifact = %v, %v", result, err) + } + body, err := io.ReadAll(result.Reader) + _ = result.Reader.Close() + if err != nil || string(body) != "cached content" { + t.Fatalf("read %q, %v; want the cached bytes", body, err) + } + artifact, err := db.GetArtifact("pkg:npm/hashed@1.0.0", "hashed-1.0.0.tgz") + if err != nil || artifact == nil || !artifact.StoragePath.Valid { + t.Errorf("record = %+v (err %v), want it kept", artifact, err) + } +} + +// TestCachedArtifactWithoutContentHashChecksNativeIntegrity checks that a +// record without a content hash is still verified against the native +// integrity, which is then the only digest there is. +func TestCachedArtifactWithoutContentHashChecksNativeIntegrity(t *testing.T) { + proxy, db, store, _ := setupTestProxy(t) + seedPackage(t, db, store, "npm", "nohash", "1.0.0", "nohash-1.0.0.tgz", "cached content") + setVersionIntegrity(t, db, "npm", "nohash", "1.0.0", sha512SRI("other bytes")) + artifact, err := db.GetArtifact("pkg:npm/nohash@1.0.0", "nohash-1.0.0.tgz") + if err != nil { + t.Fatal(err) + } + artifact.ContentHash = sql.NullString{} + if err := db.UpsertArtifact(artifact); err != nil { + t.Fatal(err) + } + + result, err := proxy.GetCachedArtifact(context.Background(), "npm", "nohash", "1.0.0", "nohash-1.0.0.tgz") + if err != nil || result == nil { + t.Fatalf("GetCachedArtifact = %v, %v", result, err) + } + _, err = io.ReadAll(result.Reader) + _ = result.Reader.Close() + if !errors.Is(err, ErrArtifactDigestMismatch) { + t.Fatalf("read error = %v, want ErrArtifactDigestMismatch", err) + } +} + +func setVersionIntegrity(t *testing.T, db *database.DB, ecosystem, name, version, integrity string) { + t.Helper() + if err := db.UpsertVersion(&database.Version{ + PURL: purl.MakePURLString(ecosystem, name, version), + PackagePURL: purl.MakePURLString(ecosystem, name, ""), + Integrity: sql.NullString{String: integrity, Valid: true}, + }); err != nil { + t.Fatal(err) + } +} + // assertQueuedForDeletion fails unless path waits in the pending delete queue. func assertQueuedForDeletion(t *testing.T, db *database.DB, path string) { t.Helper() diff --git a/internal/handler/integrity.go b/internal/handler/integrity.go index a04555f..1bbd406 100644 --- a/internal/handler/integrity.go +++ b/internal/handler/integrity.go @@ -49,6 +49,19 @@ func newIntegrityChecks(contentHash, native string) (integrityChecks, error) { return checks, nil } +// forStoredObject returns the checks to run when serving a stored object. +// When the content hash is recorded it alone is checked. It was computed +// from the bytes that were verified against the native integrity before +// they were stored, so a match binds the bytes just as well, and serving a +// cache hit costs one hash instead of two (SHA-256 plus SHA-512 for npm). +// Without a content hash the native digests are all there is to check. +func (c integrityChecks) forStoredObject() integrityChecks { + if len(c.contentHash) == 0 { + return c + } + return integrityChecks{contentHash: c.contentHash, algorithms: []integrity.Algorithm{integrity.SHA256}} +} + // verifyIntegrity wraps body to check it against the declared integrity as // it is read. verified, called once body has been read, reports a mismatch as // ErrArtifactDigestMismatch, and also a body that was not read to its end, diff --git a/internal/handler/integrity_test.go b/internal/handler/integrity_test.go index f5dcae4..84ce34d 100644 --- a/internal/handler/integrity_test.go +++ b/internal/handler/integrity_test.go @@ -514,3 +514,37 @@ func checkFetchWithDigest(t *testing.T, stream bool, content, digest string, wan t.Errorf("cached = %v, want %v", cached != nil, wantCached) } } + +// BenchmarkStoredObjectVerification compares verifying a cache hit with +// every recorded digest (SHA-256 content hash plus npm's SHA-512) against +// the content hash alone. +func BenchmarkStoredObjectVerification(b *testing.B) { + data := strings.Repeat("x", 8<<20) + checks, err := newIntegrityChecks(sha256Hex(data), sha512SRI(data)) + if err != nil { + b.Fatal(err) + } + for _, bench := range []struct { + name string + checks integrityChecks + }{ + {"all digests", checks}, + {"content hash", checks.forStoredObject()}, + } { + b.Run(bench.name, func(b *testing.B) { + b.SetBytes(int64(len(data))) + buf := make([]byte, 32<<10) + for b.Loop() { + reader, err := bench.checks.wrapFailOnMismatch(io.NopCloser(strings.NewReader(data)), func(reason string) { + b.Fatal(reason) + }) + if err != nil { + b.Fatal(err) + } + if _, err := io.CopyBuffer(io.Discard, reader, buf); err != nil { + b.Fatal(err) + } + } + }) + } +} From 486768ba195a4343ecea6dde568fc09d93e5b4b6 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 25 Oct 2026 17:22:23 +0100 Subject: [PATCH 427/633] Use smaller upload buffers for S3 and Azure Store passed empty writer options, so the S3 driver allocated six 8 MiB part buffers up front for every object over 16 MiB, on top of reading the first 16 MiB whole, and Azure used five 8 MiB blocks. Use 5 MiB parts, the S3 minimum, with two uploads in flight. A 32 MiB store against a fake S3 endpoint allocates 54 MB instead of 88 MB. --- docs/configuration.md | 2 + internal/storage/blob.go | 15 +++- internal/storage/s3_upload_test.go | 120 +++++++++++++++++++++++++++++ 3 files changed, 136 insertions(+), 1 deletion(-) create mode 100644 internal/storage/s3_upload_test.go diff --git a/docs/configuration.md b/docs/configuration.md index 5a877e3..61a9f63 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -161,6 +161,8 @@ export AWS_SECRET_ACCESS_KEY=your-secret export AWS_REGION=us-east-1 ``` +Objects up to 16 MiB are buffered in memory and sent in one request. Larger ones are uploaded in 5 MiB parts, two at a time, so a large artifact being cached holds about 30 MiB of upload buffers. Azure Blob Storage uploads use the same part size and concurrency. + ### S3-Compatible (MinIO, etc.) ```yaml diff --git a/internal/storage/blob.go b/internal/storage/blob.go index 7a34d7d..f6f022b 100644 --- a/internal/storage/blob.go +++ b/internal/storage/blob.go @@ -22,6 +22,16 @@ import ( const osWindows = "windows" +// Upload buffering for cloud backends. The S3 driver otherwise allocates +// (5+1) x 8 MiB part buffers up front for every object over 16 MiB, and the +// Azure driver uses 5 x 8 MiB blocks; with several large artifacts being +// cached at once that adds up to hundreds of MiB. 5 MiB is the smallest part +// S3 accepts. The file backend ignores both. +const ( + uploadBufferSize = 5 << 20 + uploadConcurrency = 2 +) + // attrsExt is fileblob's sidecar suffix, kept only to clear sidecars an // earlier version wrote. const attrsExt = ".attrs" @@ -212,7 +222,10 @@ func (b *Blob) Store(ctx context.Context, path string, r io.Reader) (int64, stri writeCtx, abort := context.WithCancel(ctx) defer abort() - opts := &blob.WriterOptions{} + opts := &blob.WriterOptions{ + BufferSize: uploadBufferSize, + MaxConcurrency: uploadConcurrency, + } w, err := b.bucket.NewWriter(writeCtx, path, opts) if err != nil { return 0, "", fmt.Errorf("creating writer: %w", err) diff --git a/internal/storage/s3_upload_test.go b/internal/storage/s3_upload_test.go new file mode 100644 index 0000000..898b1f7 --- /dev/null +++ b/internal/storage/s3_upload_test.go @@ -0,0 +1,120 @@ +package storage + +import ( + "bytes" + "context" + "fmt" + "io" + "maps" + "net/http" + "net/http/httptest" + "slices" + "strconv" + "strings" + "sync" + "testing" +) + +// fakeS3 implements just enough of the S3 API for PutObject and multipart +// uploads, recording the size of each part it receives. +type fakeS3 struct { + mu sync.Mutex + objects map[string][]byte + parts map[int][]byte + partSizes []int + // discard drops bodies so a benchmark measures only the client. + discard bool +} + +func newFakeS3(t testing.TB) (*fakeS3, string) { + t.Helper() + f := &fakeS3{objects: map[string][]byte{}, parts: map[int][]byte{}} + server := httptest.NewServer(f) + t.Cleanup(server.Close) + url := "s3://bucket?endpoint=" + server.URL + "®ion=us-east-1&use_path_style=true&anonymous=true&request_checksum_calculation=when_required" + return f, url +} + +func (f *fakeS3) ServeHTTP(w http.ResponseWriter, r *http.Request) { + key := strings.TrimPrefix(r.URL.Path, "/bucket/") + q := r.URL.Query() + if f.discard { + _, _ = io.Copy(io.Discard, r.Body) + r.Body = http.NoBody + } + body, err := io.ReadAll(r.Body) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + f.mu.Lock() + defer f.mu.Unlock() + switch { + case r.Method == http.MethodPost && q.Has("uploads"): + f.parts = map[int][]byte{} + _, _ = fmt.Fprintf(w, `bucket%supload`, key) + case r.Method == http.MethodPut && q.Has("partNumber"): + n, _ := strconv.Atoi(q.Get("partNumber")) + f.parts[n] = body + f.partSizes = append(f.partSizes, len(body)) + w.Header().Set("ETag", fmt.Sprintf(`"part%d"`, n)) + case r.Method == http.MethodPost && q.Has("uploadId"): + var object []byte + for _, n := range slices.Sorted(maps.Keys(f.parts)) { + object = append(object, f.parts[n]...) + } + f.objects[key] = object + _, _ = fmt.Fprintf(w, `bucket%s"object"`, key) + case r.Method == http.MethodPut: + f.objects[key] = body + w.Header().Set("ETag", `"object"`) + default: + http.Error(w, "unsupported", http.StatusNotImplemented) + } +} + +func TestS3StoreUsesSmallUploadParts(t *testing.T) { + fake, url := newFakeS3(t) + store, err := OpenBucket(context.Background(), url) + if err != nil { + t.Fatal(err) + } + defer func() { _ = store.Close() }() + + small := []byte("small object") + large := bytes.Repeat([]byte("0123456789abcdef"), 20<<20/16) + for name, data := range map[string][]byte{"small": small, "large": large} { + size, _, err := store.Store(context.Background(), name, bytes.NewReader(data)) + if err != nil { + t.Fatalf("Store(%s): %v", name, err) + } + if size != int64(len(data)) || !bytes.Equal(fake.objects[name], data) { + t.Errorf("Store(%s) wrote %d bytes, stored %d; want %d", name, size, len(fake.objects[name]), len(data)) + } + } + if len(fake.partSizes) == 0 { + t.Fatal("large object was not uploaded in parts") + } + for _, n := range fake.partSizes { + if n > uploadBufferSize { + t.Errorf("part of %d bytes, want at most %d", n, uploadBufferSize) + } + } +} + +func BenchmarkS3Store(b *testing.B) { + fake, url := newFakeS3(b) + fake.discard = true + store, err := OpenBucket(context.Background(), url) + if err != nil { + b.Fatal(err) + } + defer func() { _ = store.Close() }() + data := bytes.Repeat([]byte("0123456789abcdef"), 32<<20/16) + b.SetBytes(int64(len(data))) + for b.Loop() { + if _, _, err := store.Store(context.Background(), "object", bytes.NewReader(data)); err != nil { + b.Fatal(err) + } + } +} From a5868f1e911749a05f04e59f85588b56da3d3a9b Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 25 Oct 2026 17:24:24 +0100 Subject: [PATCH 428/633] Read metadata into a buffer of its known size ReadMetadata grew its buffer with io.ReadAll, which allocates about 2.5 times the document size in total. Cached documents have a recorded size and most upstream responses a Content-Length, so read those into one buffer of that size. The size is a hint: a missing, wrong or over-limit one still reads the body correctly and enforces the limit. --- internal/handler/handler.go | 57 ++++++++++++++++++-- internal/handler/handler_bench_test.go | 26 +++++++++ internal/handler/read_metadata_test.go | 73 ++++++++++++++++++++++++++ 3 files changed, 152 insertions(+), 4 deletions(-) diff --git a/internal/handler/handler.go b/internal/handler/handler.go index af4d329..e3eb65a 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -149,14 +149,52 @@ var ErrMetadataTooLarge = errors.New("metadata response exceeds size limit") // from unexpectedly large responses. Returns ErrMetadataTooLarge if the response // is truncated by the limit. func (p *Proxy) ReadMetadata(r io.Reader) ([]byte, error) { + return p.ReadMetadataSized(r, -1) +} + +// ReadMetadataSized is ReadMetadata for a body expected to be size bytes +// long, such as a cached document or a response with a Content-Length. +// Reading into a buffer of that size avoids io.ReadAll growing its buffer +// step by step, which peaks at more than twice the document size. size is +// only a hint: a negative one, or one above the limit, reads as ReadMetadata +// does, and a body of a different length is still read correctly. +func (p *Proxy) ReadMetadataSized(r io.Reader, size int64) ([]byte, error) { limit := p.MetadataMaxSize if limit <= 0 { limit = defaultMetadataMaxSize } - data, err := io.ReadAll(io.LimitReader(r, limit+1)) + if size < 0 || size > limit { + return readLimited(r, nil, limit) + } + // One byte more than expected, so a body of exactly size bytes ends in + // the buffer and a longer one is noticed. + buf := make([]byte, size+1) + n, err := io.ReadFull(r, buf) + switch { + case errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF): + if int64(n) < size/2 { + // The hint was far off; don't keep the oversized buffer alive. + return bytes.Clone(buf[:n]), nil + } + return buf[:n], nil + case err != nil: + return nil, err + } + return readLimited(r, buf, limit) +} + +// readLimited appends the rest of r to data, failing with +// ErrMetadataTooLarge once the total exceeds limit. +func readLimited(r io.Reader, data []byte, limit int64) ([]byte, error) { + rest, err := io.ReadAll(io.LimitReader(r, limit+1-int64(len(data)))) if err != nil { return nil, err } + if data == nil { + data = rest + } else { + data = append(data, rest...) + } if int64(len(data)) > limit { return nil, ErrMetadataTooLarge } @@ -1640,7 +1678,7 @@ func (p *Proxy) readCachedMetadata(ctx context.Context, entry *database.Metadata return nil, "", err } defer func() { _ = cached.Close() }() - data, err := p.ReadMetadata(cached) + data, err := p.ReadMetadataSized(cached, metadataEntrySize(entry)) if err == nil && validate != nil { err = validate(data) } @@ -1655,6 +1693,15 @@ func (p *Proxy) readCachedMetadata(ctx context.Context, entry *database.Metadata return data, ct, nil } +// metadataEntrySize is the recorded size of a cached metadata document, or +// -1 when none is recorded. +func metadataEntrySize(entry *database.MetadataCacheEntry) int64 { + if !entry.Size.Valid { + return -1 + } + return entry.Size.Int64 +} + // recordMetadataRead marks a cached metadata document as used, so size // eviction drops the documents nobody asks for first. func (p *Proxy) recordMetadataRead(entry *database.MetadataCacheEntry) { @@ -1716,7 +1763,7 @@ func (p *Proxy) fetchUpstreamMetadata(ctx context.Context, upstreamURL string, e return nil, errStale304 } defer func() { _ = cached.Close() }() - data, readErr := p.ReadMetadata(cached) + data, readErr := p.ReadMetadataSized(cached, metadataEntrySize(entry)) if readErr != nil { return nil, errStale304 } @@ -1741,7 +1788,9 @@ func (p *Proxy) fetchUpstreamMetadata(ctx context.Context, upstreamURL string, e return nil, fmt.Errorf("upstream returned %d", resp.StatusCode) } - body, err := p.ReadMetadata(resp.Body) + // ContentLength is -1 when the transport decompresses the body, and + // otherwise the length of the bytes read here. + body, err := p.ReadMetadataSized(resp.Body, resp.ContentLength) if err != nil { return nil, fmt.Errorf("reading response: %w", err) } diff --git a/internal/handler/handler_bench_test.go b/internal/handler/handler_bench_test.go index 9dbdb74..fdcc935 100644 --- a/internal/handler/handler_bench_test.go +++ b/internal/handler/handler_bench_test.go @@ -136,6 +136,32 @@ func BenchmarkReadMetadata(b *testing.B) { } } +// BenchmarkReadMetadataSized reads a large document with and without its +// size known, as for a cached document or a response with Content-Length. +func BenchmarkReadMetadataSized(b *testing.B) { + const size = 64 << 20 + payload := bytes.Repeat([]byte("x"), size) + proxy := &Proxy{MetadataMaxSize: 100 << 20} + for _, bench := range []struct { + name string + hint int64 + }{ + {"unknown size", -1}, + {"known size", size}, + } { + b.Run(bench.name, func(b *testing.B) { + b.SetBytes(size) + b.ReportAllocs() + for b.Loop() { + data, err := proxy.ReadMetadataSized(bytes.NewReader(payload), bench.hint) + if err != nil || len(data) != size { + b.Fatalf("read %d bytes, %v", len(data), err) + } + } + }) + } +} + func BenchmarkArtifactPURLConstruction(b *testing.B) { for _, tc := range []struct { name string diff --git a/internal/handler/read_metadata_test.go b/internal/handler/read_metadata_test.go index b13bddb..639405c 100644 --- a/internal/handler/read_metadata_test.go +++ b/internal/handler/read_metadata_test.go @@ -3,7 +3,11 @@ package handler import ( "bytes" "errors" + "fmt" + "io" + "runtime" "testing" + "testing/iotest" ) func TestReadMetadata(t *testing.T) { @@ -58,3 +62,72 @@ func TestReadMetadata(t *testing.T) { } }) } + +func TestReadMetadataSized(t *testing.T) { + const limit = 1024 + p := &Proxy{MetadataMaxSize: limit} + body := bytes.Repeat([]byte("x"), 100) + + for _, tt := range []struct { + name string + body []byte + size int64 + }{ + {"exact size", body, 100}, + {"unknown size", body, -1}, + {"size too small", body, 10}, + {"size too large", body, 900}, + {"size above limit", body, limit * 4}, + {"empty body", nil, 0}, + {"empty body with size", nil, 50}, + {"body at limit", bytes.Repeat([]byte("x"), limit), limit}, + {"body at limit, size too small", bytes.Repeat([]byte("x"), limit), 1}, + } { + t.Run(tt.name, func(t *testing.T) { + // OneByteReader makes sure short reads are not taken for EOF. + got, err := p.ReadMetadataSized(iotest.OneByteReader(bytes.NewReader(tt.body)), tt.size) + if err != nil { + t.Fatalf("ReadMetadataSized: %v", err) + } + if !bytes.Equal(got, tt.body) { + t.Errorf("got %d bytes, want %d", len(got), len(tt.body)) + } + }) + } + + for _, size := range []int64{-1, 10, limit, limit + 1, limit * 4} { + t.Run(fmt.Sprintf("over limit with size %d", size), func(t *testing.T) { + _, err := p.ReadMetadataSized(bytes.NewReader(make([]byte, limit+1)), size) + if !errors.Is(err, ErrMetadataTooLarge) { + t.Errorf("got error %v, want ErrMetadataTooLarge", err) + } + }) + } + + t.Run("read error", func(t *testing.T) { + errRead := errors.New("read failed") + r := io.MultiReader(bytes.NewReader(body), iotest.ErrReader(errRead)) + if _, err := p.ReadMetadataSized(r, 100); !errors.Is(err, errRead) { + t.Errorf("got error %v, want %v", err, errRead) + } + }) +} + +// TestReadMetadataSizedAllocatesOnce checks that a correct size hint reads +// into a single buffer of that size rather than a growing one. +func TestReadMetadataSizedAllocatesOnce(t *testing.T) { + const size = 8 << 20 + p := &Proxy{MetadataMaxSize: 64 << 20} + payload := bytes.Repeat([]byte("x"), size) + + var before, after runtime.MemStats + runtime.ReadMemStats(&before) + data, err := p.ReadMetadataSized(bytes.NewReader(payload), size) + runtime.ReadMemStats(&after) + if err != nil || len(data) != size { + t.Fatalf("read %d bytes, %v", len(data), err) + } + if allocated := after.TotalAlloc - before.TotalAlloc; allocated > size*3/2 { + t.Errorf("allocated %d bytes reading %d", allocated, size) + } +} From 23f48c5563c9baaa0fd34387419d269d56c8e147 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 25 Oct 2026 17:27:05 +0100 Subject: [PATCH 429/633] Answer HEAD on a cached artifact without opening it A HEAD on a cache hit opened the stored object and closed it unread. On S3 and Azure that is a GET whose body is thrown away, which can also drop the pooled connection; on GCS it is a GET as well. The server now marks HEAD requests in the request context, and readStored answers a marked cache hit from the database record. Size, content type and ETag already came from the record. Accept-Ranges depended on the opened reader being seekable, so backends now say whether their readers are (Blob does, GCS does not) and the HEAD response matches the GET. A HEAD still counts as a cache hit, as before. DirectServe redirects are unchanged since they are decided before the object would be opened. --- internal/handler/handler.go | 35 +++++++++- internal/handler/handler_test.go | 5 ++ internal/handler/head_cache_hit_test.go | 88 +++++++++++++++++++++++ internal/server/head_request_test.go | 93 +++++++++++++++++++++++++ internal/server/middleware.go | 14 ++++ internal/server/server.go | 6 +- internal/storage/blob.go | 5 ++ internal/storage/blob_test.go | 8 +++ internal/storage/storage.go | 7 ++ 9 files changed, 254 insertions(+), 7 deletions(-) create mode 100644 internal/handler/head_cache_hit_test.go create mode 100644 internal/server/head_request_test.go diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 2e0d688..094bdc0 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -279,6 +279,9 @@ type CacheResult struct { Artifact artifacts.Artifact Cached bool storagePath string + // rangeable is set on a HEAD cache hit, which has no Reader, when the + // stored object would open as a seekable reader. + rangeable bool } // GetOrFetchArtifact retrieves an artifact from cache or fetches from upstream. @@ -474,6 +477,12 @@ func (p *Proxy) readStored(ctx context.Context, pkgPURL, versionPURL, filename s } } + if servingHead(ctx) { + result.rangeable = readsSeekable(p.Storage) + p.recordCacheHit(artifact.Ecosystem, versionPURL, filename) + return result, nil + } + start := time.Now() reader, err := p.Storage.Open(ctx, artifact.StoragePath) metrics.RecordStorageOperation("read", time.Since(start)) @@ -892,6 +901,28 @@ func (p *Proxy) runSharedFetch(ctx context.Context, key string, f *inflightFetch return p.openStoredArtifact(ctx, stored, path) } +type servingHeadKey struct{} + +// ServingHead marks ctx as serving a HEAD request. A cache hit under it is +// answered from the artifact's database record without opening the stored +// object: on S3 or GCS opening it issues a GET whose unread body would only +// be thrown away. The server sets it on every HEAD request. +func ServingHead(ctx context.Context) context.Context { + return context.WithValue(ctx, servingHeadKey{}, true) +} + +func servingHead(ctx context.Context) bool { + head, _ := ctx.Value(servingHeadKey{}).(bool) + return head +} + +// readsSeekable reports whether store opens objects as seekable readers, so a +// HEAD response can advertise byte ranges exactly as the GET would. +func readsSeekable(store storage.Storage) bool { + s, ok := store.(storage.SeekableReads) + return ok && s.ReadsSeekable() +} + type detachUpstreamFetchKey struct{} // DetachUpstreamFetch marks ctx so that an artifact fetch started under it @@ -1067,9 +1098,9 @@ func serveArtifactResponse(w http.ResponseWriter, method string, request *http.R if result.Reader != nil { seeker, _ = result.Reader.(io.Seeker) } - if seeker != nil && result.Artifact.Size >= 0 { + if (seeker != nil || result.rangeable) && result.Artifact.Size >= 0 { w.Header().Set("Accept-Ranges", "bytes") - if request != nil && serveArtifactRange(w, request, result, seeker) { + if seeker != nil && request != nil && serveArtifactRange(w, request, result, seeker) { return } } diff --git a/internal/handler/handler_test.go b/internal/handler/handler_test.go index ff5a080..77945af 100644 --- a/internal/handler/handler_test.go +++ b/internal/handler/handler_test.go @@ -39,6 +39,7 @@ type mockStorage struct { signErr error seekable bool stores int + opens int } func newMockStorage() *mockStorage { @@ -63,6 +64,7 @@ func (s *mockStorage) Store(_ context.Context, path string, r io.Reader) (int64, func (s *mockStorage) Open(_ context.Context, path string) (io.ReadCloser, error) { s.mu.Lock() defer s.mu.Unlock() + s.opens++ if s.openErr != nil { return nil, s.openErr } @@ -76,6 +78,9 @@ func (s *mockStorage) Open(_ context.Context, path string) (io.ReadCloser, error return io.NopCloser(bytes.NewReader(data)), nil } +// ReadsSeekable reports what Open returns, as storage.Blob does. +func (s *mockStorage) ReadsSeekable() bool { return s.seekable } + type mockSeekableReadCloser struct { *bytes.Reader } diff --git a/internal/handler/head_cache_hit_test.go b/internal/handler/head_cache_hit_test.go new file mode 100644 index 0000000..3bcdf6c --- /dev/null +++ b/internal/handler/head_cache_hit_test.go @@ -0,0 +1,88 @@ +package handler + +import ( + "context" + "fmt" + "maps" + "net/http" + "net/http/httptest" + "testing" + + "github.com/git-pkgs/purl" +) + +// A HEAD on a cache hit is answered from the artifact's database record. It +// must not open the stored object (a GET on S3 or GCS), and its headers must +// be the ones the GET sends. +func TestHeadCacheHitSkipsStorageOpen(t *testing.T) { + const digest = "sha256:abc123def456abc123def456abc123def456abc123def456abc123def456abcd" + for _, seekable := range []bool{false, true} { + t.Run(fmt.Sprintf("seekable=%t", seekable), func(t *testing.T) { + proxy, db, store, fetcher := setupTestProxy(t) + store.seekable = seekable + seedPackage(t, db, store, "oci", "library/nginx", digest, digest, "cached blob") + h := &ContainerHandler{ + proxy: proxy, + registryURL: "https://registry.invalid", + proxyURL: "http://localhost:8080", + } + target := "/library/nginx/blobs/" + digest + + get := httptest.NewRecorder() + h.Routes().ServeHTTP(get, httptest.NewRequest(http.MethodGet, target, nil)) + if get.Code != http.StatusOK || get.Body.String() != "cached blob" { + t.Fatalf("GET: status %d, body %q", get.Code, get.Body.String()) + } + opensAfterGet := store.opens + + req := httptest.NewRequest(http.MethodHead, target, nil) + req = req.WithContext(ServingHead(req.Context())) + head := httptest.NewRecorder() + h.Routes().ServeHTTP(head, req) + + if head.Code != http.StatusOK { + t.Fatalf("HEAD: status %d, body %q", head.Code, head.Body.String()) + } + if got := store.opens - opensAfterGet; got != 0 { + t.Errorf("HEAD opened the stored object %d times, want 0", got) + } + if head.Body.Len() != 0 { + t.Errorf("HEAD body length = %d, want 0", head.Body.Len()) + } + if !maps.EqualFunc(get.Header(), head.Header(), func(a, b []string) bool { + return fmt.Sprint(a) == fmt.Sprint(b) + }) { + t.Errorf("HEAD headers differ from GET:\n GET %v\n HEAD %v", get.Header(), head.Header()) + } + if got := get.Header().Get("Accept-Ranges") == "bytes"; got != seekable { + t.Errorf("GET Accept-Ranges set = %t, want %t", got, seekable) + } + if fetcher.fetchCalled { + t.Error("fetcher called on a cache hit") + } + + // A HEAD still counts as a hit, as it did when it opened the object. + art, err := db.GetArtifact(purl.MakePURLString("oci", "library/nginx", digest), digest) + if err != nil || art == nil { + t.Fatalf("artifact record: %v", err) + } + if art.HitCount != 2 { + t.Errorf("hit count = %d, want 2 (GET and HEAD)", art.HitCount) + } + }) + } +} + +// A request that is not marked as HEAD reads the object as before. +func TestCacheHitWithoutHeadMarkOpensObject(t *testing.T) { + proxy, db, store, _ := setupTestProxy(t) + seedPackage(t, db, store, "npm", "demo", "1.0.0", "demo.tgz", "content") + result, err := proxy.GetCachedArtifact(context.Background(), "npm", "demo", "1.0.0", "demo.tgz") + if err != nil || result == nil || result.Reader == nil { + t.Fatalf("result=%+v err=%v; want an open reader", result, err) + } + _ = result.Reader.Close() + if store.opens != 1 { + t.Errorf("opens = %d, want 1", store.opens) + } +} diff --git a/internal/server/head_request_test.go b/internal/server/head_request_test.go new file mode 100644 index 0000000..c33db9c --- /dev/null +++ b/internal/server/head_request_test.go @@ -0,0 +1,93 @@ +package server + +import ( + "context" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "path/filepath" + "sync/atomic" + "testing" + + "github.com/git-pkgs/proxy/internal/database" + "github.com/git-pkgs/proxy/internal/handler" + "github.com/git-pkgs/proxy/internal/storage" + "github.com/git-pkgs/registries/fetch" +) + +// openCountingBlob counts the objects a request opens. +type openCountingBlob struct { + *storage.Blob + opens atomic.Int32 +} + +func (b *openCountingBlob) Open(ctx context.Context, path string) (io.ReadCloser, error) { + b.opens.Add(1) + return b.Blob.Open(ctx, path) +} + +// The server marks HEAD requests, so a HEAD on a cached artifact is answered +// without opening the stored object and with the headers the GET sends. +func TestMarkRequestContextHeadSkipsStorageOpen(t *testing.T) { + const pkgPath = "/core/os/x86_64/demo-1.0-1-x86_64.pkg.tar.zst" + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != pkgPath { + http.NotFound(w, r) + return + } + _, _ = io.WriteString(w, "demo package") + })) + defer upstream.Close() + + dir := t.TempDir() + db, err := database.Create(filepath.Join(dir, "proxy.db")) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = db.Close() }) + opened, err := storage.OpenBucket(context.Background(), "file://"+filepath.Join(dir, "artifacts")) + if err != nil { + t.Fatal(err) + } + blob, ok := opened.(*storage.Blob) + if !ok { + t.Fatalf("OpenBucket returned %T, want *storage.Blob", opened) + } + store := &openCountingBlob{Blob: blob} + fetcher := fetch.NewFetcher(fetch.WithHTTPClient(upstream.Client()), fetch.WithMaxRetries(0)) + t.Cleanup(func() { _ = fetcher.Close() }) + proxy := handler.NewProxy(db, store, fetcher, fetch.NewResolver(), slog.New(slog.NewTextHandler(io.Discard, nil))) + routes := markRequestContext(handler.NewArchHandler(proxy, "http://proxy.example", + map[string]string{"archlinux": upstream.URL}).Routes()) + + serve := func(method string) *httptest.ResponseRecorder { + w := httptest.NewRecorder() + routes.ServeHTTP(w, httptest.NewRequest(method, "/archlinux"+pkgPath, nil)) + return w + } + if w := serve(http.MethodGet); w.Code != http.StatusOK { + t.Fatalf("seeding GET: status %d", w.Code) + } + get := serve(http.MethodGet) + if get.Code != http.StatusOK || get.Body.String() != "demo package" { + t.Fatalf("cached GET: status %d, body %q", get.Code, get.Body.String()) + } + + before := store.opens.Load() + head := serve(http.MethodHead) + if head.Code != http.StatusOK || head.Body.Len() != 0 { + t.Fatalf("HEAD: status %d, body %q", head.Code, head.Body.String()) + } + if got := store.opens.Load() - before; got != 0 { + t.Errorf("HEAD opened the stored object %d times, want 0", got) + } + for _, name := range []string{"Content-Type", "Content-Length", "ETag", "Accept-Ranges"} { + if g, h := get.Header().Get(name), head.Header().Get(name); g != h { + t.Errorf("%s: GET %q, HEAD %q", name, g, h) + } + } + if head.Header().Get("Accept-Ranges") != "bytes" { + t.Errorf("Accept-Ranges = %q, want bytes", head.Header().Get("Accept-Ranges")) + } +} diff --git a/internal/server/middleware.go b/internal/server/middleware.go index 5470d76..f0dd831 100644 --- a/internal/server/middleware.go +++ b/internal/server/middleware.go @@ -9,6 +9,7 @@ import ( "github.com/git-pkgs/proxy/internal/accesslog" "github.com/git-pkgs/proxy/internal/config" + "github.com/git-pkgs/proxy/internal/handler" "github.com/git-pkgs/proxy/internal/metrics" "github.com/go-chi/chi/v5/middleware" ) @@ -37,6 +38,19 @@ func GetRequestID(ctx context.Context) string { return accesslog.RequestID(ctx) } +// markRequestContext marks each client request so upstream fetches it starts +// outlive a client that hangs up, and so a HEAD answered from the cache does +// not open the stored object. +func markRequestContext(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + ctx := handler.DetachUpstreamFetch(r.Context()) + if r.Method == http.MethodHead { + ctx = handler.ServingHead(ctx) + } + next.ServeHTTP(w, r.WithContext(ctx)) + }) +} + // LoggerMiddleware logs HTTP requests with request ID correlation. func (s *Server) LoggerMiddleware(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { diff --git a/internal/server/server.go b/internal/server/server.go index 3dc2f9b..9a3ab61 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -316,11 +316,7 @@ func (s *Server) serve(listener net.Listener) error { next.ServeHTTP(w, r) }) }) - r.Use(func(next http.Handler) http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - next.ServeHTTP(w, r.WithContext(handler.DetachUpstreamFetch(r.Context()))) - }) - }) + r.Use(markRequestContext) // Mount protocol handlers s.mountProtocolHandlers(r, proxy) diff --git a/internal/storage/blob.go b/internal/storage/blob.go index 7a34d7d..f2a57db 100644 --- a/internal/storage/blob.go +++ b/internal/storage/blob.go @@ -255,6 +255,11 @@ func (b *Blob) Open(ctx context.Context, path string) (io.ReadCloser, error) { return r, nil } +// ReadsSeekable reports that Open returns seekable readers: an *os.File for +// file:// buckets and a *blob.Reader, which seeks with a ranged read, +// otherwise. +func (b *Blob) ReadsSeekable() bool { return true } + func (b *Blob) Exists(ctx context.Context, path string) (bool, error) { exists, err := b.bucket.Exists(ctx, path) if err != nil { diff --git a/internal/storage/blob_test.go b/internal/storage/blob_test.go index 37b9ac6..1a764de 100644 --- a/internal/storage/blob_test.go +++ b/internal/storage/blob_test.go @@ -84,6 +84,14 @@ func TestBlobOpen(t *testing.T) { } } +// Blob.ReadsSeekable promises seekable readers for every bucket: file:// +// objects open as *os.File (checked in TestBlobOpen) and remote ones as +// *blob.Reader. +var ( + _ io.Seeker = (*blob.Reader)(nil) + _ SeekableReads = (*Blob)(nil) +) + func TestBlobOpenNotFound(t *testing.T) { b := createTestBlob(t) ctx := context.Background() diff --git a/internal/storage/storage.go b/internal/storage/storage.go index dfce286..edca684 100644 --- a/internal/storage/storage.go +++ b/internal/storage/storage.go @@ -77,6 +77,13 @@ type Storage interface { Close() error } +// SeekableReads is implemented by backends whose Open always returns a +// reader that implements io.Seeker. It tells a caller that byte ranges can be +// served from an object without opening it first. +type SeekableReads interface { + ReadsSeekable() bool +} + // FetchPath builds the storage path for one fetch of an artifact: // {ecosystem}/{name}/{version}/{fetchID}/{filename}. Each fetch writes its own // object, so no fetch overwrites or deletes another's. From 6d39461d3b4ae808ffde9c365ea5e8d4449d3bd4 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 25 Oct 2026 17:25:56 +0100 Subject: [PATCH 430/633] Set the Go memory limit from the cgroup limit Nothing set GOMEMLIMIT, so in a container the heap could grow until the container was OOM-killed while much of it was garbage, for example while large metadata documents were being rewritten. At startup on Linux, read the cgroup v2 or v1 memory limit and set the runtime's soft limit to 80% of it, leaving room for git subprocesses. An explicit GOMEMLIMIT wins. --- cmd/proxy/main.go | 2 + docs/architecture.md | 4 ++ docs/configuration.md | 4 ++ internal/memlimit/memlimit.go | 111 +++++++++++++++++++++++++++++ internal/memlimit/memlimit_test.go | 66 +++++++++++++++++ 5 files changed, 187 insertions(+) create mode 100644 internal/memlimit/memlimit.go create mode 100644 internal/memlimit/memlimit_test.go diff --git a/cmd/proxy/main.go b/cmd/proxy/main.go index 55f4836..a61534a 100644 --- a/cmd/proxy/main.go +++ b/cmd/proxy/main.go @@ -125,6 +125,7 @@ import ( "github.com/git-pkgs/proxy/internal/config" "github.com/git-pkgs/proxy/internal/database" "github.com/git-pkgs/proxy/internal/denylist" + "github.com/git-pkgs/proxy/internal/memlimit" "github.com/git-pkgs/proxy/internal/mirror" "github.com/git-pkgs/proxy/internal/server" "github.com/git-pkgs/proxy/internal/storage" @@ -323,6 +324,7 @@ func runServe() { // Setup logger logger := setupLogger(cfg.Log.Level, cfg.Log.Format) + memlimit.Apply(logger) // Create and start server srv, err := server.New(cfg, logger, server.BuildInfo{ diff --git a/docs/architecture.md b/docs/architecture.md index b5f4ba9..64a4bef 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -330,6 +330,10 @@ HTTP server setup, web UI, and API handlers. - Enrichment API for package metadata, vulnerability scanning, and outdated detection - Health, stats, and Prometheus metrics endpoints. `/health` runs an active write → size-check → read → verify → delete probe against the storage backend and returns a structured JSON response (`HealthResponse`) with `"ok"` / `"error"` status per subsystem. Probe results are cached (default 30 s, configurable via `health.storage_probe_interval`) to avoid overwhelming remote backends. The response also carries a `circuit_breakers` map reporting each upstream's artifact-fetch breaker as `"open"` or `"closed"`, keyed by the host fetched from (or an opaque placeholder where the fetch URL has no host to read); the same state is published as the `proxy_circuit_breaker_state` gauge on each `/metrics` scrape. An open breaker leaves the overall status `"ok"` — it describes an upstream, not this proxy. +### `internal/memlimit` + +Sets Go's soft memory limit to 80% of the cgroup memory limit at startup on Linux, unless `GOMEMLIMIT` is set. See "Memory limit" in the configuration docs. + ### `internal/metrics` Prometheus metrics for cache performance, upstream latency, storage operations, and active requests. See the Monitoring section of the README for the full metric list. diff --git a/docs/configuration.md b/docs/configuration.md index 61a9f63..f9cb68c 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -39,6 +39,10 @@ scanning: These are independent limits, not reserved time slices: a download approaching 10 minutes followed by a 16-minute scan would still exceed an 18-minute response deadline. Any reverse proxy in front of the server also needs sufficient timeout allowance. The upstream transport's existing response-header timeout remains unchanged. +### Memory limit + +On Linux, when the process runs under a cgroup memory limit, such as a container's `--memory` or a Kubernetes memory limit, `proxy serve` sets Go's soft memory limit to 80% of it and log the value at startup (`set Go memory limit from cgroup`). The garbage collector then runs more often as memory use approaches the limit, rather than letting the heap grow until the container is killed. The remaining 20% is left for memory Go doesn't manage, such as `git` processes the proxy runs. Set the `GOMEMLIMIT` environment variable to choose the value yourself, or `GOMEMLIMIT=off` to disable it. + ## Storage The proxy stores cached artifacts using gocloud.dev/blob, supporting local filesystem and S3-compatible storage. diff --git a/internal/memlimit/memlimit.go b/internal/memlimit/memlimit.go new file mode 100644 index 0000000..ff753a1 --- /dev/null +++ b/internal/memlimit/memlimit.go @@ -0,0 +1,111 @@ +// Package memlimit sets the Go runtime's soft memory limit from the +// container's memory limit. +// +// Without a limit the garbage collector only runs when the heap doubles, so +// a process holding a large live heap, such as a big metadata document +// being rewritten, can grow past a container limit and be OOM-killed while +// most of that memory is garbage. A soft limit just below the container's +// makes the collector work harder as it gets close instead. +package memlimit + +import ( + "bufio" + "io/fs" + "log/slog" + "os" + "path" + "runtime" + "runtime/debug" + "strconv" + "strings" +) + +// ratio is the share of the container limit given to the Go runtime. The +// rest is left for memory the runtime does not account for, such as the git +// processes the proxy runs, which share the container's cgroup. +const ratio = 0.8 + +// noLimit is the value cgroup v1 reports, rounded to a page, when no limit +// is set. Anything this large is treated as unlimited. +const noLimit = 1 << 62 + +// Apply sets the runtime memory limit to ratio of the cgroup memory limit, +// unless GOMEMLIMIT is set, which the runtime has applied already, or no +// limit can be found. It only looks for one on Linux. +func Apply(logger *slog.Logger) { + if os.Getenv("GOMEMLIMIT") != "" || runtime.GOOS != "linux" { + return + } + limit, ok := cgroupLimit(os.DirFS("/")) + if !ok { + return + } + set := int64(float64(limit) * ratio) + debug.SetMemoryLimit(set) + logger.Info("set Go memory limit from cgroup", "memory_limit", set, "cgroup_limit", limit) +} + +// cgroupLimit returns the memory limit of the cgroup the process is in, +// read from fsys, the root filesystem. It reads cgroup v2's memory.max, or +// v1's memory.limit_in_bytes, first in the process's own cgroup and then at +// the hierarchy root, which is what a container sees as its own. +func cgroupLimit(fsys fs.FS) (int64, bool) { + v2, v1 := ownCgroups(fsys) + candidates := []string{ + path.Join("sys/fs/cgroup", v2, "memory.max"), + "sys/fs/cgroup/memory.max", + path.Join("sys/fs/cgroup/memory", v1, "memory.limit_in_bytes"), + "sys/fs/cgroup/memory/memory.limit_in_bytes", + } + for _, name := range candidates { + data, err := fs.ReadFile(fsys, name) + if err != nil { + continue + } + // "max" means no limit at this level; a later candidate may still + // have one. + limit, err := strconv.ParseInt(strings.TrimSpace(string(data)), 10, 64) + if err == nil && limit > 0 && limit < noLimit { + return limit, true + } + } + return 0, false +} + +// ownCgroups returns the process's cgroup v2 path and its cgroup v1 memory +// controller path from /proc/self/cgroup, each "" when not listed. +func ownCgroups(fsys fs.FS) (v2, v1 string) { + f, err := fsys.Open("proc/self/cgroup") + if err != nil { + return "", "" + } + defer func() { _ = f.Close() }() + scanner := bufio.NewScanner(f) + for scanner.Scan() { + // Lines are "hierarchy-ID:controller-list:cgroup-path". + id, rest, ok := strings.Cut(scanner.Text(), ":") + if !ok { + continue + } + controllers, cgroup, ok := strings.Cut(rest, ":") + if !ok { + continue + } + switch { + case id == "0" && controllers == "": + v2 = cgroup + case hasController(controllers, "memory"): + v1 = cgroup + } + } + return v2, v1 +} + +func hasController(list, name string) bool { + for controller := range strings.SplitSeq(list, ",") { + if controller == name { + return true + } + } + return false +} diff --git a/internal/memlimit/memlimit_test.go b/internal/memlimit/memlimit_test.go new file mode 100644 index 0000000..bf8474b --- /dev/null +++ b/internal/memlimit/memlimit_test.go @@ -0,0 +1,66 @@ +package memlimit + +import ( + "testing" + "testing/fstest" +) + +func TestCgroupLimit(t *testing.T) { + const gib = 1 << 30 + for _, tt := range []struct { + name string + files map[string]string + want int64 + }{ + { + name: "cgroup v2 container", + files: map[string]string{"proc/self/cgroup": "0::/\n", "sys/fs/cgroup/memory.max": "2147483648\n"}, + want: 2 * gib, + }, + { + name: "cgroup v2 own cgroup", + files: map[string]string{ + "proc/self/cgroup": "0::/system.slice/proxy.service\n", + "sys/fs/cgroup/system.slice/proxy.service/memory.max": "1073741824\n", + }, + want: gib, + }, + { + name: "cgroup v2 unlimited", + files: map[string]string{"proc/self/cgroup": "0::/\n", "sys/fs/cgroup/memory.max": "max\n"}, + }, + { + name: "cgroup v1", + files: map[string]string{ + "proc/self/cgroup": "12:cpu,cpuacct:/\n4:memory:/\n", + "sys/fs/cgroup/memory/memory.limit_in_bytes": "536870912\n", + }, + want: gib / 2, + }, + { + name: "cgroup v1 unlimited", + files: map[string]string{ + "proc/self/cgroup": "4:memory:/\n", + "sys/fs/cgroup/memory/memory.limit_in_bytes": "9223372036854771712\n", + }, + }, + { + name: "malformed", + files: map[string]string{"sys/fs/cgroup/memory.max": "lots\n"}, + }, + { + name: "no cgroup files", + }, + } { + t.Run(tt.name, func(t *testing.T) { + fsys := fstest.MapFS{} + for name, data := range tt.files { + fsys[name] = &fstest.MapFile{Data: []byte(data)} + } + got, ok := cgroupLimit(fsys) + if got != tt.want || ok != (tt.want != 0) { + t.Errorf("cgroupLimit = %d, %v; want %d", got, ok, tt.want) + } + }) + } +} From 2b620378346582124767c46c970068ceb89eaa36 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 25 Oct 2026 17:29:22 +0100 Subject: [PATCH 431/633] Serve immutable Go and RPM metadata without revalidation A Go module's .mod and .info for a canonical version, and RPM repodata files named after their checksum, never change once published. They went through the metadata TTL, so each cost a revalidation every TTL with cache_metadata on and a full fetch on every request with it off. They are now kept in the metadata cache with or without cache_metadata and served whatever their age. A stored copy is checked against its digest and fetched again if it doesn't match. Go queries, @v/list, @latest and repomd.xml stay on the TTL. --- docs/configuration.md | 2 + internal/handler/go.go | 27 +- internal/handler/go_cooldown.go | 10 +- internal/handler/handler.go | 7 +- internal/handler/metadata_immutable.go | 116 +++++++++ internal/handler/metadata_immutable_test.go | 270 ++++++++++++++++++++ internal/handler/metadata_stream.go | 17 +- internal/handler/rpm.go | 28 +- 8 files changed, 467 insertions(+), 10 deletions(-) create mode 100644 internal/handler/metadata_immutable.go create mode 100644 internal/handler/metadata_immutable_test.go diff --git a/docs/configuration.md b/docs/configuration.md index 5a877e3..a4935aa 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1277,6 +1277,8 @@ cache_metadata: true Or via environment variable: `PROXY_CACHE_METADATA=true`. +Some metadata never changes once published, and is cached like an artifact, with or without `cache_metadata`, and served without revalidation whatever `metadata_ttl` says: a Go module's `.mod` and `.info` for a canonical version (a release, pre-release, pseudo-version or `+incompatible` version; the checksum database fixes their content), and RPM repodata files named after their own checksum (`repodata/-primary.xml.gz`). A stored copy is checked against the digest recorded for it and fetched again if it doesn't match. Go branch and commit queries, `@v/list` and `@latest`, and RPM `repomd.xml` and plainly named repodata files follow `metadata_ttl` as before. A checksum-named repodata file over `metadata_max_size` is streamed from upstream uncached. + Composer archive downloads look up their upstream URL in the same cached metadata, so an archive is resolved without asking Packagist again and stays installable while it is down. Only a version or file the cached metadata does not list yet is looked up upstream. With or without `cache_metadata`, a cached archive of a tagged release from the default repository is served without any metadata lookup; with cooldown enabled it is checked against the publish time recorded when the archive was first resolved. Branch (dev) versions, routed packages and archives picked by a query string such as `?sha=` are always resolved from the metadata. The `proxy mirror` command always enables metadata caching regardless of this setting. diff --git a/internal/handler/go.go b/internal/handler/go.go index e544cd7..dc38ed0 100644 --- a/internal/handler/go.go +++ b/internal/handler/go.go @@ -98,7 +98,7 @@ func (h *GoHandler) handleRequest(w http.ResponseWriter, r *http.Request) { case strings.HasSuffix(rest, ".info"): // GET /{module}/@v/{version}.info - version metadata - h.proxyCached(w, r, decodedMod+"/@v/"+rest) + h.proxyVersionFile(w, r, decodedMod+"/@v/"+rest, strings.TrimSuffix(rest, ".info")) case strings.HasSuffix(rest, ".mod"): // GET /{module}/@v/{version}.mod - go.mod file @@ -110,7 +110,7 @@ func (h *GoHandler) handleRequest(w http.ResponseWriter, r *http.Request) { if h.checkCooldown(w, r, module, decodedMod, version) { return } - h.proxyCached(w, r, decodedMod+"/@v/"+rest) + h.proxyVersionFile(w, r, decodedMod+"/@v/"+rest, strings.TrimSuffix(rest, ".mod")) case strings.HasSuffix(rest, ".zip"): // GET /{module}/@v/{version}.zip - source archive (cache this) @@ -242,6 +242,29 @@ func (h *GoHandler) proxyCached(w http.ResponseWriter, r *http.Request, cacheKey h.proxy.ProxyCached(w, r, h.upstreamURL+r.URL.Path, "golang", cacheKey, "*/*") } +// proxyVersionFile serves a version's .info or .mod. For a canonical version +// it never changes, so it is served from the cache without revalidation. +func (h *GoHandler) proxyVersionFile(w http.ResponseWriter, r *http.Request, cacheKey, escapedVersion string) { + if goVersionImmutable(escapedVersion) { + h.proxy.proxyImmutable(w, r, h.upstreamURL+r.URL.Path, "golang", cacheKey, "*/*") + return + } + h.proxyCached(w, r, cacheKey) +} + +// goVersionImmutable reports whether escapedVersion, as it appears in a +// proxy URL, names a canonical version, whose .info and .mod the module +// proxy protocol and the checksum database fix for good. This includes +// pseudo-versions, which name a commit by its hash, and +incompatible +// versions. Anything else, such as a branch name or v1.2 for v1.2.0, is a +// query that can resolve to another version later and stays on the TTL. +// The .zip of a canonical version is cached for good already, and a .mod +// that changed upstream would no longer match it. +func goVersionImmutable(escapedVersion string) bool { + version, err := gomodule.UnescapeVersion(escapedVersion) + return err == nil && version != "" && gomodule.CanonicalVersion(version) == version +} + // decodeGoModule decodes an encoded module path. // In the encoding, uppercase letters are represented as "!" followed by lowercase. func decodeGoModule(encoded string) string { diff --git a/internal/handler/go_cooldown.go b/internal/handler/go_cooldown.go index c3aa0cb..90189d4 100644 --- a/internal/handler/go_cooldown.go +++ b/internal/handler/go_cooldown.go @@ -58,8 +58,16 @@ func (h *GoHandler) writeGoMetadataError(w http.ResponseWriter, err error) { http.Error(w, "failed to fetch from upstream", http.StatusBadGateway) } +// fetchGoMetadata reads a module's @v/list, @latest, or a version's .info. +// The .info of a canonical version comes from the cache whatever its age, +// like proxyVersionFile serves it. func (h *GoHandler) fetchGoMetadata(ctx context.Context, escapedModule, module, rest string) ([]byte, error) { - body, _, err := h.proxy.FetchOrCacheMetadata(ctx, "golang", module+"/"+rest, h.upstreamURL+"/"+escapedModule+"/"+rest, "*/*") + cacheKey, upstreamURL := module+"/"+rest, h.upstreamURL+"/"+escapedModule+"/"+rest + if v, ok := strings.CutPrefix(rest, "@v/"); ok && strings.HasSuffix(v, ".info") && goVersionImmutable(strings.TrimSuffix(v, ".info")) { + res := h.proxy.fetchImmutableMetadata(ctx, "golang", cacheKey, upstreamURL, "*/*") + return res.body, res.err + } + body, _, err := h.proxy.FetchOrCacheMetadata(ctx, "golang", cacheKey, upstreamURL, "*/*") return body, err } diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 2e0d688..d71b1e8 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -1935,7 +1935,12 @@ func (p *Proxy) writeMetadataCachedResponse(w http.ResponseWriter, r *http.Reque // missing or stale when the metadata cache write failed and would otherwise // mislabel the bytes. func (p *Proxy) writeMetadataCachedResponseWithEncoding(w http.ResponseWriter, r *http.Request, ecosystem, cacheKey string, body []byte, contentType, contentEncoding string) { - cm := p.lookupCachedMeta(ecosystem, cacheKey) + writeMetadataBody(w, r, p.lookupCachedMeta(ecosystem, cacheKey), body, contentType, contentEncoding) +} + +// writeMetadataBody writes a metadata document with the validators and +// staleness in cm. +func writeMetadataBody(w http.ResponseWriter, r *http.Request, cm cachedMeta, body []byte, contentType, contentEncoding string) { if writeMetadataValidators(w, r, cm) { return } diff --git a/internal/handler/metadata_immutable.go b/internal/handler/metadata_immutable.go new file mode 100644 index 0000000..691fbc1 --- /dev/null +++ b/internal/handler/metadata_immutable.go @@ -0,0 +1,116 @@ +package handler + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "net/http" + + "github.com/git-pkgs/proxy/internal/metrics" +) + +// Some metadata documents never change once published: a Go module's .mod +// and .info for a canonical version, or an RPM repodata file named after its +// own checksum. Revalidating them every metadata TTL, or fetching them again +// on every request without cache_metadata, only costs upstream requests. They +// are kept in the metadata cache like other metadata, but served whatever +// their age and stored whether or not cache_metadata is set, the way +// artifacts are. + +// errImmutableDigest is returned when a stored immutable document doesn't +// match the digest recorded for it. +var errImmutableDigest = errors.New("stored metadata does not match its digest") + +// proxyImmutable answers a request for an immutable metadata document: from +// the metadata cache when it is stored, otherwise fetched once and stored. A +// document over the metadata size limit is streamed from upstream instead, +// uncached. +func (p *Proxy) proxyImmutable(w http.ResponseWriter, r *http.Request, upstreamURL, ecosystem, cacheKey, accept string) { + if p.serveStoredMetadata(w, r, ecosystem, cacheKey, true) { + return + } + res := p.fetchImmutableMetadata(r.Context(), ecosystem, cacheKey, upstreamURL, accept) + if errors.Is(res.err, ErrMetadataTooLarge) { + p.proxyMetadataStream(w, r, upstreamURL, "identity", accept) + return + } + if res.err != nil { + p.writeMetadataFetchError(w, res.err) + return + } + cm := p.lookupCachedMeta(ecosystem, cacheKey) + cm.stale = false + writeMetadataBody(w, r, cm, res.body, res.contentType, res.contentEncoding) +} + +// fetchImmutableMetadata returns an immutable metadata document from the +// metadata cache whatever its age, or fetches and stores it. Upstream's bytes +// are kept as sent (Accept-Encoding: identity). Without a database or storage +// it falls back to the TTL lookup. +func (p *Proxy) fetchImmutableMetadata(ctx context.Context, ecosystem, cacheKey, upstreamURL, accept string) metadataResult { + if p.DB == nil || p.Storage == nil { + return p.lookupMetadata(ctx, ecosystem, cacheKey, upstreamURL, accept, "identity", nil) + } + if containsPathTraversal(cacheKey) { + return metadataResult{err: fmt.Errorf("invalid cache key: %q", cacheKey)} + } + if hit := p.storedImmutable(ctx, ecosystem, cacheKey); hit != nil { + metrics.RecordMetadataCacheHit(ecosystem, len(hit.body)) + return *hit + } + metrics.RecordMetadataCacheMiss(ecosystem) + + // Kept apart from TTL lookups of the same key, which don't store the + // document without cache_metadata. + key := metadataCoalesceKey(ecosystem, cacheKey, upstreamURL, accept, "identity", false) + "\x00immutable" + res, _ := p.coalesceMetadata(ctx, key, func(fetchCtx context.Context) metadataResult { + if hit := p.storedImmutable(fetchCtx, ecosystem, cacheKey); hit != nil { + return *hit + } + meta, err := p.fetchUpstreamMetadata(fetchCtx, upstreamURL, nil, accept, "identity") + if err != nil { + return metadataResult{err: err} + } + digest := p.cacheMetadataBlob(fetchCtx, ecosystem, cacheKey, metadataStoragePath(ecosystem, cacheKey), meta) + return metadataResult{body: meta.body, contentType: meta.contentType, contentEncoding: meta.contentEncoding, digest: digest} + }) + return res +} + +// storedImmutable reads an immutable document from the metadata cache, or +// returns nil. Since the copy is never revalidated, it is checked against the +// digest recorded when it was stored; a row without one, or whose bytes don't +// match, is a miss, and the fetch that follows replaces it. +func (p *Proxy) storedImmutable(ctx context.Context, ecosystem, cacheKey string) *metadataResult { + entry, err := p.DB.GetMetadataCache(ecosystem, cacheKey) + if err != nil || entry == nil { + return nil + } + entry = p.migrateLegacyMetadataEntry(ctx, ecosystem, cacheKey, entry) + if entry == nil || !entry.ContentDigest.Valid { + return nil + } + want := entry.ContentDigest.String + body, contentType, err := p.readCachedMetadata(ctx, entry, func(b []byte) error { + sum := sha256.Sum256(b) + if "sha256:"+hex.EncodeToString(sum[:]) != want { + return errImmutableDigest + } + return nil + }) + if errors.Is(err, errImmutableDigest) { + p.Logger.Error("cached metadata failed integrity check", + "ecosystem", ecosystem, "key", cacheKey, "path", entry.StoragePath) + metrics.RecordIntegrityFailure(ecosystem, metrics.FailureDetail{Path: entry.StoragePath, Err: err}) + if err := p.DB.DiscardMetadataCache(ecosystem, cacheKey, entry.StoragePath, want); err != nil { + p.Logger.Warn("failed to clear corrupt metadata from cache", "error", err) + } + } + if err != nil { + return nil + } + res := metadataResult{body: body, contentType: contentType, contentEncoding: entry.ContentEncoding.String}.describedBy(entry) + return &res +} diff --git a/internal/handler/metadata_immutable_test.go b/internal/handler/metadata_immutable_test.go new file mode 100644 index 0000000..95b1a17 --- /dev/null +++ b/internal/handler/metadata_immutable_test.go @@ -0,0 +1,270 @@ +package handler + +import ( + "context" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + "time" + + "github.com/git-pkgs/cooldown" +) + +// countingUpstream answers every path with body and counts requests per path. +type countingUpstream struct { + server *httptest.Server + mu sync.Mutex + hits map[string]int + body func(path string) (string, bool) +} + +func newCountingUpstream(t *testing.T, body func(path string) (string, bool)) *countingUpstream { + t.Helper() + u := &countingUpstream{hits: map[string]int{}, body: body} + u.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + u.mu.Lock() + u.hits[r.URL.Path]++ + u.mu.Unlock() + b, ok := u.body(r.URL.Path) + if !ok { + http.NotFound(w, r) + return + } + _, _ = w.Write([]byte(b)) + })) + t.Cleanup(u.server.Close) + return u +} + +func (u *countingUpstream) count(path string) int { + u.mu.Lock() + defer u.mu.Unlock() + return u.hits[path] +} + +const goPseudoVersion = "v0.0.0-20240102150405-abcdefabcdef" + +func goImmutableUpstream(t *testing.T) *countingUpstream { + return newCountingUpstream(t, func(path string) (string, bool) { + rest, ok := strings.CutPrefix(path, "/example.com/!mod/") + if !ok { + return "", false + } + switch { + case rest == "@v/list": + return "v1.0.0\n", true + case rest == "@latest", rest == "@v/main.info": + return `{"Version":"v1.0.0","Time":"2020-01-01T00:00:00Z"}`, true + case rest == "@v/v1.0.0.info": + return `{"Version":"v1.0.0","Time":"2020-01-01T00:00:00Z"}`, true + case rest == "@v/"+goPseudoVersion+".info": + return `{"Version":"` + goPseudoVersion + `","Time":"2024-01-02T15:04:05Z"}`, true + case strings.HasSuffix(rest, ".mod"): + return "module example.com/Mod\n", true + } + return "", false + }) +} + +// TestGoVersionFilesAreNotRevalidated checks that a canonical version's .mod +// and .info are fetched once and then served from the cache, with or without +// cache_metadata and whatever the metadata TTL, while queries, @v/list and +// @latest still go upstream. +func TestGoVersionFilesAreNotRevalidated(t *testing.T) { + for _, tt := range []struct { + name string + cacheMetadata, cooldown bool + }{ + {"cache_metadata off", false, false}, + {"cache_metadata on", true, false}, + {"cache_metadata off, cooldown", false, true}, + {"cache_metadata on, cooldown", true, true}, + } { + t.Run(tt.name, func(t *testing.T) { + testGoVersionFilesAreNotRevalidated(t, tt.cacheMetadata, tt.cooldown) + }) + } +} + +func testGoVersionFilesAreNotRevalidated(t *testing.T, cacheMetadata, withCooldown bool) { + u := goImmutableUpstream(t) + proxy, _, _, _ := setupTestProxy(t) + proxy.CacheMetadata = cacheMetadata + proxy.MetadataTTL = time.Nanosecond // every TTL-bound entry is already expired + if withCooldown { + proxy.Cooldown = &cooldown.Config{Default: "3d"} + } + h := NewGoHandlerWithUpstream(proxy, "http://localhost:8080", u.server.URL).Routes() + + once := []string{ + "/example.com/!mod/@v/v1.0.0.mod", + "/example.com/!mod/@v/v1.0.0.info", + "/example.com/!mod/@v/" + goPseudoVersion + ".mod", + "/example.com/!mod/@v/" + goPseudoVersion + ".info", + } + every := []string{ + "/example.com/!mod/@v/main.info", + "/example.com/!mod/@v/list", + "/example.com/!mod/@latest", + } + for range 3 { + for _, p := range append(append([]string{}, once...), every...) { + if w := goGet(h, p); w.Code != http.StatusOK { + t.Fatalf("GET %s = %d: %s", p, w.Code, w.Body.String()) + } + } + } + for _, p := range once { + if w := goGet(h, p); w.Header().Get("Warning") != "" { + t.Errorf("GET %s marked stale", p) + } + if got := u.count(p); got != 1 { + t.Errorf("upstream requests for %s = %d, want 1", p, got) + } + } + for _, p := range every { + if got := u.count(p); got < 2 { + t.Errorf("upstream requests for %s = %d, want one per request", p, got) + } + } +} + +func TestGoVersionImmutable(t *testing.T) { + tests := map[string]bool{ + "v1.0.0": true, + "v1.0.0-!r!c1": true, // v1.0.0-RC1 + "v2.0.0+incompatible": true, + goPseudoVersion: true, + "v1.2": false, // resolves to v1.2.0 + "v1.0.0+build": false, + "main": false, + "latest": false, + "v1.0.0-RC1": false, // not a valid escaped version + "": false, + "master-20240102150405-x": false, + } + for v, want := range tests { + if got := goVersionImmutable(v); got != want { + t.Errorf("goVersionImmutable(%q) = %v, want %v", v, got, want) + } + } +} + +// TestImmutableMetadataRefetchedWhenCorrupt checks that a stored immutable +// document whose bytes no longer match their digest is fetched again rather +// than returned for good. (A streamed hit of such a document aborts, as +// TestCachedMetadataHitAbortsOnDigestMismatch checks.) +func TestImmutableMetadataRefetchedWhenCorrupt(t *testing.T) { + u := goImmutableUpstream(t) + proxy, db, store, _ := setupTestProxy(t) + const ( + path = "/example.com/!mod/@v/v1.0.0.info" + key = "example.com/Mod/@v/v1.0.0.info" + want = `{"Version":"v1.0.0","Time":"2020-01-01T00:00:00Z"}` + ) + fetch := func() string { + t.Helper() + res := proxy.fetchImmutableMetadata(context.Background(), "golang", key, u.server.URL+path, "*/*") + if res.err != nil { + t.Fatalf("fetch: %v", res.err) + } + return string(res.body) + } + + fetch() + entry, err := db.GetMetadataCache("golang", key) + if err != nil || entry == nil { + t.Fatalf("document not stored: %v", err) + } + if _, _, err := store.Store(context.Background(), entry.StoragePath, strings.NewReader(`{"Version":"v6.6.6"}`)); err != nil { + t.Fatal(err) + } + + if got := fetch(); got != want { + t.Fatalf("got %q, want the upstream document", got) + } + if got := u.count(path); got != 2 { + t.Errorf("upstream requests = %d, want 2", got) + } + if got := fetch(); got != want { + t.Fatalf("got %q after refetch, want the upstream document", got) + } + if got := u.count(path); got != 2 { + t.Errorf("upstream requests after refetch = %d, want 2", got) + } +} + +// TestRPMContentAddressedRepodataIsNotRevalidated checks that repodata files +// named after their checksum are fetched once, while repomd.xml and plainly +// named files follow the TTL. +func TestRPMContentAddressedRepodataIsNotRevalidated(t *testing.T) { + const ( + sum = "2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae" + primary = "/releases/40/Everything/x86_64/os/repodata/" + sum + "-primary.xml.zst" + repomd = "/releases/40/Everything/x86_64/os/repodata/repomd.xml" + plain = "/releases/40/Everything/x86_64/os/repodata/primary.xml.gz" + ) + for _, cacheMetadata := range []bool{false, true} { + u := newCountingUpstream(t, func(string) (string, bool) { return "data", true }) + proxy, _, _, _ := setupTestProxy(t) + proxy.CacheMetadata = cacheMetadata + proxy.MetadataTTL = time.Nanosecond + h := NewRPMHandlerWithUpstream(proxy, "http://localhost:8080", u.server.URL).Routes() + + for range 3 { + for _, p := range []string{primary, repomd, plain} { + if w := goGet(h, p); w.Code != http.StatusOK || w.Body.String() != "data" { + t.Fatalf("GET %s = %d %q", p, w.Code, w.Body.String()) + } + } + } + if got := u.count(primary); got != 1 { + t.Errorf("cache_metadata=%v: upstream requests for checksum-named file = %d, want 1", cacheMetadata, got) + } + for _, p := range []string{repomd, plain} { + if got := u.count(p); got != 3 { + t.Errorf("cache_metadata=%v: upstream requests for %s = %d, want 3", cacheMetadata, p, got) + } + } + } +} + +func TestRPMContentAddressed(t *testing.T) { + sha256 := strings.Repeat("a", 64) + tests := map[string]bool{ + "os/repodata/" + sha256 + "-primary.xml.gz": true, + "os/repodata/" + strings.Repeat("0", 40) + "-other.xml.gz": true, + "os/repodata/" + strings.Repeat("f", 128) + "-comps.xml": true, + "os/repodata/repomd.xml": false, + "os/repodata/repomd.xml.asc": false, + "os/repodata/primary.xml.gz": false, + "os/repodata/" + strings.Repeat("a", 63) + "-primary.xml.gz": false, + "os/repodata/" + strings.Repeat("A", 64) + "-primary.xml.gz": false, + "os/repodata/" + sha256: false, + } + for p, want := range tests { + if got := rpmContentAddressed(p); got != want { + t.Errorf("rpmContentAddressed(%q) = %v, want %v", p, got, want) + } + } +} + +// TestImmutableMetadataOverSizeLimitIsStreamed checks that a checksum-named +// repodata file too large to buffer is streamed from upstream rather than +// refused. +func TestImmutableMetadataOverSizeLimitIsStreamed(t *testing.T) { + const path = "/os/repodata/" + "2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae" + "-filelists.xml.gz" + u := newCountingUpstream(t, func(string) (string, bool) { return "large document", true }) + proxy, db, _, _ := setupTestProxy(t) + proxy.MetadataMaxSize = 4 + h := NewRPMHandlerWithUpstream(proxy, "http://localhost:8080", u.server.URL).Routes() + + if w := goGet(h, path); w.Code != http.StatusOK || w.Body.String() != "large document" { + t.Fatalf("got %d %q, want the upstream document", w.Code, w.Body.String()) + } + if entry, _ := db.GetMetadataCache("rpm", hashedCacheKey(u.server.URL, strings.TrimPrefix(path, "/"))); entry != nil { + t.Error("stored a document over the size limit") + } +} diff --git a/internal/handler/metadata_stream.go b/internal/handler/metadata_stream.go index 8bb319d..2101ed1 100644 --- a/internal/handler/metadata_stream.go +++ b/internal/handler/metadata_stream.go @@ -21,11 +21,21 @@ import ( // buffered path instead: no fresh row, a row without a digest or size, a // document over the metadata size limit, or one that can't be opened. func (p *Proxy) serveCachedMetadata(w http.ResponseWriter, r *http.Request, ecosystem, cacheKey string) bool { - if p.DB == nil || p.Storage == nil || p.MetadataTTL <= 0 || containsPathTraversal(cacheKey) { + return p.serveStoredMetadata(w, r, ecosystem, cacheKey, false) +} + +// serveStoredMetadata is serveCachedMetadata. With immutable set the row is +// served whatever its age and never marked stale, for documents that can't +// change upstream (see proxyImmutable). +func (p *Proxy) serveStoredMetadata(w http.ResponseWriter, r *http.Request, ecosystem, cacheKey string, immutable bool) bool { + if p.DB == nil || p.Storage == nil || (!immutable && p.MetadataTTL <= 0) || containsPathTraversal(cacheKey) { return false } entry, err := p.DB.GetMetadataCache(ecosystem, cacheKey) - if err != nil || entry == nil || !entry.FetchedAt.Valid || time.Since(entry.FetchedAt.Time) >= p.MetadataTTL { + if err != nil || entry == nil || !entry.FetchedAt.Valid { + return false + } + if !immutable && time.Since(entry.FetchedAt.Time) >= p.MetadataTTL { return false } limit := p.MetadataMaxSize @@ -64,6 +74,9 @@ func (p *Proxy) serveCachedMetadata(w http.ResponseWriter, r *http.Request, ecos metrics.RecordMetadataCacheHit(ecosystem, int(entry.Size.Int64)) cm := p.cachedMetaFor(entry) + if immutable { + cm.stale = false + } if writeMetadataValidators(w, r, cm) { return true } diff --git a/internal/handler/rpm.go b/internal/handler/rpm.go index aaac6f5..95163b9 100644 --- a/internal/handler/rpm.go +++ b/internal/handler/rpm.go @@ -80,7 +80,7 @@ func (h *RPMHandler) Routes() http.Handler { // Package downloads - cache these h.handlePackageDownload(w, r, path) case strings.Contains(path, "/repodata/"): - // Repository metadata - proxy without caching (changes frequently) + // Repository metadata h.handleMetadata(w, r, path) default: // Other files - proxy directly @@ -119,10 +119,30 @@ func (h *RPMHandler) handlePackageDownload(w http.ResponseWriter, r *http.Reques ServeArtifactRequest(w, r, result) } -// handleMetadata proxies repository metadata files (repomd.xml, primary.xml.gz, etc.). -// These change frequently so we don't cache them. +// handleMetadata proxies repository metadata files (repomd.xml, +// primary.xml.gz, etc.). repomd.xml changes with every repository update and +// follows the metadata TTL. The files it points at are usually named after +// their own checksum and so never change; those are kept for good. func (h *RPMHandler) handleMetadata(w http.ResponseWriter, r *http.Request, path string) { - h.proxy.ProxyCached(w, r, fmt.Sprintf("%s/%s", h.upstreamURL, path), "rpm", h.metadataCacheKey(path), "*/*") + upstreamURL := fmt.Sprintf("%s/%s", h.upstreamURL, path) + if rpmContentAddressed(path) { + h.proxy.proxyImmutable(w, r, upstreamURL, "rpm", h.metadataCacheKey(path), "*/*") + return + } + h.proxy.ProxyCached(w, r, upstreamURL, "rpm", h.metadataCacheKey(path), "*/*") +} + +// rpmChecksumPrefix matches a repodata filename that createrepo prefixed +// with the file's checksum: -primary.xml.gz. The hex lengths are those +// of the checksum types createrepo_c offers: SHA-1, SHA-224, SHA-256, +// SHA-384 and SHA-512. Repositories built with --simple-md-filenames use +// plain names, which can be replaced in place and follow the TTL. +var rpmChecksumPrefix = regexp.MustCompile(`^(?:[0-9a-f]{40}|[0-9a-f]{56}|[0-9a-f]{64}|[0-9a-f]{96}|[0-9a-f]{128})-[^/]+$`) + +// rpmContentAddressed reports whether a repodata path names a file after +// its checksum, so its content never changes. +func rpmContentAddressed(path string) bool { + return rpmChecksumPrefix.MatchString(path[strings.LastIndex(path, "/")+1:]) } // metadataCacheKey is the metadata cache key of a repository path. From fd0d9a5df21d4792c881fd95ac7c917f4abeb610 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 25 Oct 2026 17:29:22 +0100 Subject: [PATCH 432/633] Answer PyPI download cooldown from stored upload times Every PyPI download with cooldown on read the JSON API first, cached files included. Without cache_metadata that was a full /pypi//json per file. A download that looks up the JSON API now stores the release's newest upload time on its versions row. A cached file is then checked against the stored time without asking upstream. A file that isn't cached is still looked up, since files can be added to a release later, and the stored time is refreshed. A stored time inside the window withholds the release without a lookup. --- docs/configuration.md | 2 +- internal/handler/pypi.go | 101 +++++++++++++++++--- internal/handler/pypi_upload_time_test.go | 107 ++++++++++++++++++++++ 3 files changed, 194 insertions(+), 16 deletions(-) create mode 100644 internal/handler/pypi_upload_time_test.go diff --git a/docs/configuration.md b/docs/configuration.md index a4935aa..f83efc1 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1044,7 +1044,7 @@ Maven caveats: - Maven Central lists versions in version order, not in the order they were published. A patch to an older release line published after a newer release that is out of cooldown stays listed while it is in cooldown, but its files are still refused. - SNAPSHOT versions are not held back. -RubyGems, Hex and PyPI keep their publish times outside the metadata clients read: the RubyGems versions API, the Hex HTTP API (`upstream.hex_api`) and the PyPI JSON API. The proxy fetches them alongside each compact index, package or simple page it filters, and for PyPI downloads. Cooldown fails closed: when it applies to a package and those times can't be fetched or parsed, or the metadata can't be rewritten, the request fails with 502, or with 503 and `Retry-After` while the upstream is rate limiting the proxy, instead of serving the metadata unfiltered. The same goes for npm, Composer and pub.dev metadata that can't be rewritten, and for npm tarball downloads whose publish time can't be read. A PyPI file or npm tarball that is already cached is still served when its publish time can't be read; only a download that would have to be fetched fails. Packages exempted with a duration of `0` are served as before and don't need the times. A PyPI package the JSON API returns 404 for has no times to go by and is served unfiltered. +RubyGems, Hex and PyPI keep their publish times outside the metadata clients read: the RubyGems versions API, the Hex HTTP API (`upstream.hex_api`) and the PyPI JSON API. The proxy fetches them alongside each compact index, package or simple page it filters, and for PyPI downloads. Cooldown fails closed: when it applies to a package and those times can't be fetched or parsed, or the metadata can't be rewritten, the request fails with 502, or with 503 and `Retry-After` while the upstream is rate limiting the proxy, instead of serving the metadata unfiltered. The same goes for npm, Composer and pub.dev metadata that can't be rewritten, and for npm tarball downloads whose publish time can't be read. A PyPI file or npm tarball that is already cached is still served when its publish time can't be read; only a download that would have to be fetched fails. A PyPI release's time is the upload time of its newest file. The proxy stores it when a download looks it up, and serves files of that release that are already cached against the stored time without reading the JSON API again. A file that isn't cached yet is always looked up, since it may have been added to the release since, and the newer time is stored. Packages exempted with a duration of `0` are served as before and don't need the times. A PyPI package the JSON API returns 404 for has no times to go by and is served unfiltered. With cooldown on, the proxy asks npm for full packuments, since the abbreviated ones it uses otherwise carry no publish times. Abbreviated packuments cached before cooldown was turned on are fetched again in full rather than served, including as a stale copy when the upstream is down. diff --git a/internal/handler/pypi.go b/internal/handler/pypi.go index 5667501..717e9ec 100644 --- a/internal/handler/pypi.go +++ b/internal/handler/pypi.go @@ -273,6 +273,30 @@ func ensureVaryAccept(header http.Header) { // doesn't know has no publish times to go by and gets an empty set; any other // failure to read them is an error. func (h *PyPIHandler) fetchFilteredVersions(r *http.Request, name string) (map[string]bool, error) { + times, err := h.fetchReleaseTimes(r, name) + if err != nil { + return nil, err + } + + packagePURL := canonicalPackagePURL("pypi", name) + filtered := make(map[string]bool) + + for version, publishedAt := range times { + if !publishedAt.IsZero() && !h.proxy.Cooldown.IsAllowed("pypi", packagePURL, publishedAt) { + filtered[version] = true + } + } + + if len(filtered) == 0 { + return nil, nil + } + return filtered, nil +} + +// fetchReleaseTimes reads the JSON API and returns each release's newest +// upload time, zero for a release without one. A package the JSON API +// doesn't know gets nil; any other failure is an error. +func (h *PyPIHandler) fetchReleaseTimes(r *http.Request, name string) (map[string]time.Time, error) { jsonURL := fmt.Sprintf("%s/pypi/%s/json", h.upstreamURL, name) body, _, err := h.proxy.FetchOrCacheMetadata(r.Context(), "pypi", name+"/json", jsonURL) @@ -293,24 +317,15 @@ func (h *PyPIHandler) fetchFilteredVersions(r *http.Request, name string) (map[s return nil, errors.New("pypi json metadata has no releases") } - packagePURL := canonicalPackagePURL("pypi", name) - filtered := make(map[string]bool) - + times := make(map[string]time.Time, len(releases)) for version, files := range releases { filesArr, ok := files.([]any) if !ok { continue } - publishedAt := h.newestUploadTime(filesArr) - if !publishedAt.IsZero() && !h.proxy.Cooldown.IsAllowed("pypi", packagePURL, publishedAt) { - filtered[version] = true - } + times[version] = h.newestUploadTime(filesArr) } - - if len(filtered) == 0 { - return nil, nil - } - return filtered, nil + return times, nil } // rewriteSimpleHTML rewrites package URLs in simple API HTML to point at this proxy. @@ -513,13 +528,65 @@ func (h *PyPIHandler) shouldFilterRelease(packagePURL string, files any) bool { // // A release whose upload time cannot be determined is allowed through, matching // how fetchFilteredVersions treats it. Failing to read the times is an error. +// The time read is stored for storedCooldown. func (h *PyPIHandler) versionInCooldown(r *http.Request, name, version string) (bool, error) { if !h.proxy.cooldownApplies("pypi", name) { return false, nil } - filtered, err := h.fetchFilteredVersions(r, name) - return filtered[version], err + times, err := h.fetchReleaseTimes(r, name) + if err != nil { + return false, err + } + publishedAt := times[version] + if publishedAt.IsZero() { + return false, nil + } + h.storeUploadTime(name, version, publishedAt) + return !h.proxy.Cooldown.IsAllowed("pypi", canonicalPackagePURL("pypi", name), publishedAt), nil +} + +// storeUploadTime records a release's newest upload time under the version +// PURL the file is cached under, so storedCooldown finds it. +func (h *PyPIHandler) storeUploadTime(name, version string, publishedAt time.Time) { + if h.proxy.DB == nil { + return + } + packagePURL, versionPURL, err := packagePURLStrings("pypi", name, version) + if err != nil { + return + } + if err := h.proxy.DB.SetVersionPublishedAt(versionPURL, packagePURL, publishedAt); err != nil { + h.proxy.Logger.Warn("cooldown: could not store pypi upload time", + "package", name, "version", version, "error", err) + } +} + +// storedCooldown answers the cooldown check for a download from the upload +// time stored for its release, without reading the JSON API, and reports +// whether it could. A release's time is its newest file's, and files can be +// added to a release later, so the stored time only ever moves forward: +// - a stored time inside the window withholds every file of the release; +// - a stored time outside it only lets through a file that is already +// cached. A file that is not cached may be one uploaded since, so it goes +// to the JSON API, which also stores the release's current time. +func (h *PyPIHandler) storedCooldown(r *http.Request, name, version, filename string) (inCooldown, answered bool) { + if h.proxy.DB == nil || !h.proxy.cooldownApplies("pypi", name) { + return false, false + } + packagePURL, versionPURL, err := packagePURLStrings("pypi", name, version) + if err != nil { + return false, false + } + ver, err := h.proxy.DB.GetVersionByPURL(versionPURL) + if err != nil || ver == nil || !ver.PublishedAt.Valid { + return false, false + } + if !h.proxy.Cooldown.IsAllowed("pypi", packagePURL, ver.PublishedAt.Time) { + return true, true + } + cached, err := h.proxy.HasCachedArtifact(r.Context(), "pypi", name, version, filename) + return false, err == nil && cached } // rewriteFileEntries rewrites URLs in a list of file entries. @@ -625,7 +692,11 @@ func (h *PyPIHandler) handleDownload(w http.ResponseWriter, r *http.Request) { name, version := h.parseFilename(filename) if name != "" { - inCooldown, err := h.versionInCooldown(r, name, version) + inCooldown, answered := h.storedCooldown(r, name, version, filename) + var err error + if !answered { + inCooldown, err = h.versionInCooldown(r, name, version) + } if err != nil { if !h.proxy.serveCachedDespiteCooldownError(w, r, "pypi", name, version, filename, err) { h.proxy.writeCooldownLookupError(w, "pypi", name, err) diff --git a/internal/handler/pypi_upload_time_test.go b/internal/handler/pypi_upload_time_test.go new file mode 100644 index 0000000..f61a5f0 --- /dev/null +++ b/internal/handler/pypi_upload_time_test.go @@ -0,0 +1,107 @@ +package handler + +import ( + "io" + "net/http" + "net/http/httptest" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/git-pkgs/cooldown" + "github.com/git-pkgs/registries/fetch" +) + +// pypiUploadTimeSetup serves the JSON API for newpkg from releases, which the +// test may change, and counts its requests. cache_metadata is off, so every +// lookup reaches upstream. +func pypiUploadTimeSetup(t *testing.T) (h http.Handler, setReleases func(string), jsonRequests *atomic.Int64, fetcher *mockFetcher) { + t.Helper() + var mu sync.Mutex + releases := "" + jsonRequests = &atomic.Int64{} + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/pypi/newpkg/json" { + http.NotFound(w, r) + return + } + jsonRequests.Add(1) + mu.Lock() + defer mu.Unlock() + w.Header().Set("Content-Type", contentTypeJSON) + _, _ = io.WriteString(w, releases) + })) + t.Cleanup(upstream.Close) + + proxy, _, _, fetcher := setupTestProxy(t) + proxy.HTTPClient = upstream.Client() + proxy.Cooldown = &cooldown.Config{Default: "7d"} + setReleases = func(s string) { + mu.Lock() + defer mu.Unlock() + releases = s + } + return (&PyPIHandler{proxy: proxy, upstreamURL: upstream.URL, downloadURL: upstream.URL, proxyURL: "http://localhost"}).Routes(), + setReleases, jsonRequests, fetcher +} + +func uploaded(ago time.Duration) string { + return time.Now().Add(-ago).Format(time.RFC3339) +} + +// TestPyPICachedDownloadUsesStoredUploadTime checks that once a release's +// upload time is stored, downloads of its cached files don't read the JSON +// API again. +func TestPyPICachedDownloadUsesStoredUploadTime(t *testing.T) { + h, setReleases, jsonRequests, fetcher := pypiUploadTimeSetup(t) + setReleases(`{"releases": {"1.0.0": [{"upload_time_iso_8601": "` + uploaded(30*24*time.Hour) + `"}]}}`) + fetcher.artifact = &fetch.Artifact{Body: io.NopCloser(strings.NewReader("sdist")), ContentType: "application/octet-stream"} + + for range 3 { + w := serveGet(t, h, "/packages/packages/ab/cd/ef0123456789/newpkg-1.0.0.tar.gz") + wantStatus(t, w, http.StatusOK, "") + if w.Body.String() != "sdist" { + t.Fatalf("body = %q", w.Body.String()) + } + } + if got := jsonRequests.Load(); got != 1 { + t.Errorf("JSON API requests = %d, want 1", got) + } +} + +// TestPyPIFileAddedToReleaseIsChecked checks that a file uploaded to a +// release after its time was stored is not let through on the stored time: +// it isn't cached, so the JSON API is read, the newer time is stored, and +// from then on the whole release is withheld until it is out of cooldown. +func TestPyPIFileAddedToReleaseIsChecked(t *testing.T) { + h, setReleases, jsonRequests, fetcher := pypiUploadTimeSetup(t) + sdist := `{"upload_time_iso_8601": "` + uploaded(30*24*time.Hour) + `"}` + setReleases(`{"releases": {"1.0.0": [` + sdist + `]}}`) + fetcher.artifact = &fetch.Artifact{Body: io.NopCloser(strings.NewReader("sdist")), ContentType: "application/octet-stream"} + + const ( + sdistPath = "/packages/packages/ab/cd/ef0123456789/newpkg-1.0.0.tar.gz" + wheelPath = "/packages/packages/ab/cd/ef0123456789/newpkg-1.0.0-py3-none-any.whl" + ) + wantStatus(t, serveGet(t, h, sdistPath), http.StatusOK, "") + + // A wheel is added to the release an hour ago. + setReleases(`{"releases": {"1.0.0": [` + sdist + `, {"upload_time_iso_8601": "` + uploaded(time.Hour) + `"}]}}`) + fetcher.fetchCalled = false + wantStatus(t, serveGet(t, h, wheelPath), http.StatusNotFound, "") + if fetcher.fetchCalled { + t.Error("fetched a file that is inside the cooldown window") + } + if got := jsonRequests.Load(); got != 2 { + t.Errorf("JSON API requests = %d, want 2 (the new file is looked up)", got) + } + + // The stored time is now the wheel's, so the cached sdist is withheld + // too, as the JSON API lookup would do, without asking it. + wantStatus(t, serveGet(t, h, sdistPath), http.StatusNotFound, "") + if got := jsonRequests.Load(); got != 2 { + t.Errorf("JSON API requests = %d, want 2", got) + } +} From 674528a25e317513c7d887cd8f7f5307a328fe90 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 25 Oct 2026 17:32:22 +0100 Subject: [PATCH 433/633] Skip the pushed artifact lookup when hosting is disabled Every npm and Composer download ran GetLocalArtifact, a full artifact row lookup, even with hosted packages disabled. localVersions already returns early in that case, so metadata does not list pushes, and /generic/hosted/ is only served while hosting is enabled. GetLocalArtifact now returns nil without a lookup when hosting is off. Downloads then take the normal proxy path. An archive pushed earlier is still answered from storage by that path and is never replaced by the upstream copy; only cooldown now applies to it, as metadata no longer lists it either. The hosted docs say so. --- docs/configuration.md | 2 ++ internal/handler/hosted.go | 7 +++++++ internal/handler/hosted_test.go | 29 +++++++++++++++++++++++++++++ internal/handler/npm_hosted_test.go | 18 ++++++++++++++++++ 4 files changed, 56 insertions(+) diff --git a/docs/configuration.md b/docs/configuration.md index 5a877e3..a801ca8 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1364,6 +1364,8 @@ hosted: `hosted.enabled` cannot be combined with `storage.cache_artifacts: false`. +With `hosted.enabled` off, archives pushed earlier stay in storage but are not served as hosted packages: package metadata doesn't list them, `/generic/hosted/` isn't served, and npm and Composer downloads take the normal proxy path, cooldown included. An upstream copy still never replaces a pushed archive. + ### Pushing a patched package 1. Build the archive with the original package name and the version you push in its manifest (`package.json` or `composer.json`), e.g. `request-2.88.3.tgz`. diff --git a/internal/handler/hosted.go b/internal/handler/hosted.go index 5ff43ac..33f2191 100644 --- a/internal/handler/hosted.go +++ b/internal/handler/hosted.go @@ -469,7 +469,14 @@ func (p *Proxy) deleteLocal(ecosystem, name, version, filename string, notFound // before serving a cached copy call it first, so pushed versions the upstream // does not list are still served. Pushed artifacts are not a cache, so they // are served even when StreamArtifacts is set. +// +// With hosted packages disabled it returns nil without a lookup, as +// localVersions does: pushes are then left out of metadata, so they are not +// served ahead of the upstream either. func (p *Proxy) GetLocalArtifact(ctx context.Context, ecosystem, name, version, filename string) (*CacheResult, error) { + if !p.HostedEnabled { + return nil, nil + } pkgPURL, versionPURL, err := packagePURLStrings(ecosystem, name, version) if err != nil { return nil, err diff --git a/internal/handler/hosted_test.go b/internal/handler/hosted_test.go index fbe00a5..b0640be 100644 --- a/internal/handler/hosted_test.go +++ b/internal/handler/hosted_test.go @@ -228,6 +228,7 @@ func TestLocalArtifactsAndDeleteLocalVersion(t *testing.T) { func TestGetLocalArtifactIgnoresUpstreamCopies(t *testing.T) { proxy, _, _, fetcher := setupTestProxy(t) + proxy.HostedEnabled = true fetcher.artifact = &fetch.Artifact{Body: io.NopCloser(strings.NewReader("upstream tarball")), ContentType: "application/gzip"} cached, err := proxy.GetOrFetchArtifactFromURL(context.Background(), "npm", "request", "2.88.3", "request-2.88.3.tgz", "https://registry.npmjs.org/request/-/request-2.88.3.tgz") @@ -251,6 +252,34 @@ func TestGetLocalArtifactIgnoresUpstreamCopies(t *testing.T) { } } +// With hosted packages disabled, downloads don't look for a pushed artifact, +// as metadata requests don't. +func TestGetLocalArtifactNeedsHostedEnabled(t *testing.T) { + proxy, db, _, _ := setupTestProxy(t) + proxy.HostedEnabled = true + pushRequest(t, proxy) + if local, err := proxy.GetLocalArtifact(context.Background(), "npm", "request", "2.88.3", "request-2.88.3.tgz"); err != nil || local == nil { + t.Fatalf("GetLocalArtifact with hosted enabled = %+v, %v, want the pushed artifact", local, err) + } else { + _ = local.Reader.Close() + } + + proxy.HostedEnabled = false + if local, err := proxy.GetLocalArtifact(context.Background(), "npm", "request", "2.88.3", "request-2.88.3.tgz"); err != nil || local != nil { + t.Fatalf("GetLocalArtifact with hosted disabled = %+v, %v, want nil, nil", local, err) + } + + // A closed database fails every query, so a nil error shows no lookup ran. + _ = db.Close() + if local, err := proxy.GetLocalArtifact(context.Background(), "npm", "request", "2.88.3", "request-2.88.3.tgz"); err != nil || local != nil { + t.Errorf("GetLocalArtifact with hosted disabled = %+v, %v, want nil, nil without a database lookup", local, err) + } + proxy.HostedEnabled = true + if _, err := proxy.GetLocalArtifact(context.Background(), "npm", "request", "2.88.3", "request-2.88.3.tgz"); err == nil { + t.Error("GetLocalArtifact with hosted enabled did not query the closed database") + } +} + func TestPushLocalArtifactRecordsManifest(t *testing.T) { proxy, _, _, _ := setupTestProxy(t) proxy.HostedEnabled = true diff --git a/internal/handler/npm_hosted_test.go b/internal/handler/npm_hosted_test.go index 75c82a1..083319a 100644 --- a/internal/handler/npm_hosted_test.go +++ b/internal/handler/npm_hosted_test.go @@ -229,6 +229,24 @@ func TestNPMMetadataIgnoresPushesWhenHostedDisabled(t *testing.T) { } } +// With hosted packages disabled, a pushed archive is not served ahead of the +// upstream, but it is still never replaced by the upstream copy: the +// download is answered from its stored record like any cached archive. +func TestNPMDownloadWithHostedDisabledKeepsPush(t *testing.T) { + f := newNPMHostedFixture(t, map[string]string{"/request": requestPackument}) + manifest := `{"name":"request","version":"2.88.3"}` + f.push(t, "request", "2.88.3", manifest) + f.proxy.HostedEnabled = false + + rec := f.get(t, "/request/-/request-2.88.3.tgz") + if rec.Code != http.StatusOK || rec.Body.String() != string(npmTarball(manifest)) { + t.Errorf("download status = %d, want 200 and the pushed tarball", rec.Code) + } + if f.fetcher.fetchCalled { + t.Error("the upstream tarball was fetched for a pushed version") + } +} + func TestNPMDownloadOfPushedPackageWithCooldown(t *testing.T) { f := newNPMHostedFixture(t, nil) f.proxy.Cooldown = &cooldown.Config{Default: "7d"} From 0b76ca248aa01cc2b41d96559a660b1c6bb264e7 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 25 Oct 2026 17:32:22 +0100 Subject: [PATCH 434/633] Revalidate stale OCI tags with HEAD and share the request A stale tag was revalidated by repeating the client's GET with If-None-Match. Docker Hub counts manifest GETs against its pull limit, so every TTL expiry of a tag could cost a pull. The tag is now resolved with a HEAD and the Docker-Content-Digest compared with the cached one; the manifest is fetched by digest only when that digest isn't cached. A HEAD without a digest falls back to the conditional GET. A client HEAD whose digest is cached now points the tag at it, so the GET that follows is a cache hit. Concurrent requests for the same manifest share one upstream exchange, and instances sharing Postgres take turns through the fetch lease. --- docs/configuration.md | 2 + internal/handler/container.go | 5 + internal/handler/container_manifest.go | 114 +---- internal/handler/container_manifest_fetch.go | 353 +++++++++++++++ .../container_manifest_revalidate_test.go | 422 ++++++++++++++++++ 5 files changed, 806 insertions(+), 90 deletions(-) create mode 100644 internal/handler/container_manifest_fetch.go create mode 100644 internal/handler/container_manifest_revalidate_test.go diff --git a/docs/configuration.md b/docs/configuration.md index 5a877e3..08ae4fa 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1271,6 +1271,8 @@ By default the proxy fetches metadata fresh from upstream on every request. Enab OCI manifests and tag lists are always cached because cached image blobs cannot be pulled without their manifests and offline clients may need tag resolution. Digest-addressed manifests are immutable and served directly from cache. Tag-addressed manifests and tag lists follow `metadata_ttl`, revalidate when stale, and fall back to the last cached response when the registry is unavailable. +A stale tag is revalidated with a HEAD request carrying the client's `Accept` header, and the `Docker-Content-Digest` it returns is compared with the cached one. If the tag did not move, or moved to a manifest already cached by digest, nothing else is fetched. Otherwise the new manifest is fetched once by digest and stored under the tag and the digest. Docker Hub counts manifest GETs, but not HEADs, against its pull rate limit, so an unchanged tag costs no pulls however often its TTL runs out. A registry whose HEAD response has no digest gets a conditional GET for the tag instead. A client's HEAD request for a stale or uncached tag is passed on as a HEAD; when the digest it returns is cached, the tag is pointed at it, so the GET that follows is served from cache. Concurrent requests for the same manifest share one upstream request, and instances sharing a Postgres database take turns through the same [fetch lease](#several-instances) as other metadata. + ```yaml cache_metadata: true ``` diff --git a/internal/handler/container.go b/internal/handler/container.go index 57c30d8..1648ccb 100644 --- a/internal/handler/container.go +++ b/internal/handler/container.go @@ -15,6 +15,7 @@ import ( "github.com/git-pkgs/proxy/internal/retention" "github.com/git-pkgs/purl" + "golang.org/x/sync/singleflight" ) const ( @@ -114,6 +115,10 @@ type ContainerHandler struct { // refusalsFull is set once the refusals set reached its limit and that // was logged. refusalsFull bool + + // manifestFlight shares one upstream exchange among concurrent + // requests for the same manifest. + manifestFlight singleflight.Group } type containerRegistry struct { diff --git a/internal/handler/container_manifest.go b/internal/handler/container_manifest.go index af69967..6d129d0 100644 --- a/internal/handler/container_manifest.go +++ b/internal/handler/container_manifest.go @@ -53,100 +53,31 @@ type cachedContainerManifest struct { // resolves to under cacheName, the name blobs are stored under. func (h *ContainerHandler) serveManifest(w http.ResponseWriter, r *http.Request, registryURL, name, cacheName, reference string) { accept := containerManifestAccept(r) - cacheAccept := normalizeContainerManifestAccept(accept) - cached, fromFallback := h.loadContainerManifestForAccept(r.Context(), registryURL, name, reference, accept, cacheAccept) - - immutable := manifestDigestReferencePattern.MatchString(reference) - if cached != nil && (immutable || h.containerManifestFresh(cached)) { + req := manifestRequest{ + registryURL: registryURL, + name: name, + reference: reference, + accept: accept, + cacheAccept: normalizeContainerManifestAccept(accept), + method: r.Method, + } + cached, _ := h.loadContainerManifestForAccept(r.Context(), registryURL, name, reference, req.accept, req.cacheAccept) + if cached != nil && (req.immutable() || h.containerManifestFresh(cached)) { h.writeManifest(w, r, cacheName, cached, false) return } - upstreamURL := fmt.Sprintf("%s/v2/%s/manifests/%s", registryURL, name, reference) - req, err := http.NewRequestWithContext(r.Context(), r.Method, upstreamURL, nil) - if err != nil { - h.containerError(w, http.StatusInternalServerError, "INTERNAL_ERROR", "failed to create request") - return + outcome := h.coalescedManifest(r.Context(), req) + switch { + case outcome.manifest != nil: + h.writeManifest(w, r, cacheName, outcome.manifest, false) + case outcome.relay != nil: + outcome.relay.write(h.proxy, w, r) + case outcome.digestInvalid: + h.containerError(w, http.StatusBadGateway, "DIGEST_INVALID", "manifest digest verification failed") + default: + h.serveStaleManifestOrError(w, r, cacheName, reference, cached, outcome.err) } - req.Header.Set("Accept", accept) - if cached != nil && cached.etag != "" { - req.Header.Set("If-None-Match", cached.etag) - } - - resp, err := h.proxy.HTTPClient.Do(req) - if err != nil { - h.serveStaleManifestOrError(w, r, cacheName, reference, cached, err) - return - } - defer func() { _ = resp.Body.Close() }() - - if resp.StatusCode == http.StatusNotModified && cached != nil { - cached.fetchedAt = time.Now() - if fromFallback { - // The shared entry holds another representation for other - // clients; only this client's own entries are refreshed. - h.storeContainerManifestLegacyEntries(r.Context(), registryURL, name, reference, accept, cacheAccept, cached) - } else { - h.storeContainerManifestForAccept(r.Context(), registryURL, name, reference, accept, cacheAccept, cached) - } - h.writeManifest(w, r, cacheName, cached, false) - return - } - if resp.StatusCode != http.StatusOK { - if shouldServeStaleManifest(resp.StatusCode) && (cached != nil || referrersTagPattern.MatchString(reference)) { - h.serveStaleManifestOrError(w, r, cacheName, reference, cached, fmt.Errorf("upstream status %d", resp.StatusCode)) - return - } - h.proxy.relayResponse(w, r, resp, copyContainerManifestHeaders) - return - } - - if r.Method == http.MethodHead { - if h.manifestDenied(w, cacheName, resp.Header.Get("Docker-Content-Digest")) { - return - } - h.proxy.relayResponse(w, r, resp, copyContainerManifestHeaders) - return - } - - body, err := h.proxy.ReadMetadata(resp.Body) - if err != nil { - h.serveStaleManifestOrError(w, r, cacheName, reference, cached, fmt.Errorf("reading manifest: %w", err)) - return - } - contentDigest := resp.Header.Get("Docker-Content-Digest") - // A digest reference, and the digest the upstream says the manifest - // has, are both cache keys, so each must be the manifest's own digest. - // One the proxy can't compute is refused rather than cached unchecked. - for _, expected := range []string{reference, contentDigest} { - if !strings.Contains(expected, ":") { - continue - } - if err := verifyDigest(expected, body); err != nil { - h.proxy.Logger.Error("upstream manifest failed digest verification", - "name", name, "reference", reference, "expected", expected, "error", err) - h.containerError(w, http.StatusBadGateway, "DIGEST_INVALID", "manifest digest verification failed") - return - } - } - if contentDigest == "" { - contentDigest = sha256Digest(body) - } - - manifest := &cachedContainerManifest{ - body: body, - contentType: resp.Header.Get(headerContentType), - contentDigest: contentDigest, - etag: resp.Header.Get(headerETag), - size: int64(len(body)), - lastModified: parseHTTPTime(resp.Header.Get(headerLastModified)), - fetchedAt: time.Now(), - } - h.storeContainerManifestForAccept(r.Context(), registryURL, name, reference, accept, cacheAccept, manifest) - if manifest.contentDigest != reference && manifestDigestReferencePattern.MatchString(manifest.contentDigest) { - h.storeContainerManifestForAccept(r.Context(), registryURL, name, manifest.contentDigest, accept, cacheAccept, manifest) - } - h.writeManifest(w, r, cacheName, manifest, false) } func (h *ContainerHandler) serveStaleManifestOrError(w http.ResponseWriter, r *http.Request, cacheName, reference string, cached *cachedContainerManifest, err error) { @@ -345,7 +276,10 @@ func writeContainerManifest(w http.ResponseWriter, r *http.Request, manifest *ca if manifest.contentType != "" { w.Header().Set(headerContentType, manifest.contentType) } - w.Header().Set(headerContentLength, strconv.FormatInt(manifest.size, 10)) + // A manifest known only from an upstream HEAD may lack a size. + if manifest.size >= 0 { + w.Header().Set(headerContentLength, strconv.FormatInt(manifest.size, 10)) + } if manifest.contentDigest != "" { w.Header().Set("Docker-Content-Digest", manifest.contentDigest) } diff --git a/internal/handler/container_manifest_fetch.go b/internal/handler/container_manifest_fetch.go new file mode 100644 index 0000000..4b5ef15 --- /dev/null +++ b/internal/handler/container_manifest_fetch.go @@ -0,0 +1,353 @@ +package handler + +import ( + "bytes" + "context" + "fmt" + "io" + "net/http" + "strconv" + "strings" + "time" + + "github.com/git-pkgs/proxy/internal/metrics" +) + +// manifestRequest is one client manifest request as the upstream sees it. +type manifestRequest struct { + registryURL string + name string + reference string + accept string + cacheAccept string + method string +} + +func (req manifestRequest) immutable() bool { + return manifestDigestReferencePattern.MatchString(req.reference) +} + +// coalesceKey identifies requests that can share one upstream exchange. The +// normalized Accept is part of it because registries answer a tag with an +// index or a single manifest depending on it, and the method because a HEAD +// never fetches a manifest body the client did not ask for. +func (req manifestRequest) coalesceKey() string { + return strings.Join([]string{containerManifestCacheEcosystem, req.method, req.registryURL, req.name, req.reference, req.cacheAccept}, "\x00") +} + +func (req manifestRequest) url(reference string) string { + return fmt.Sprintf("%s/v2/%s/manifests/%s", req.registryURL, req.name, reference) +} + +// manifestOutcome is what callers sharing a manifest request write. Exactly +// one of manifest, relay and digestInvalid is set; otherwise err is the +// upstream failure, answered from the caller's stale copy if it has one. +type manifestOutcome struct { + manifest *cachedContainerManifest + relay *capturedResponse + digestInvalid bool + err error +} + +// capturedResponse is an upstream response other than a manifest, such as a +// 404 or 401, read in full so every caller sharing the request can relay it. +// A body that could not be read to the end is relayed up to that point and +// then fails the same way, so clients still see the truncation. +type capturedResponse struct { + status int + header http.Header + body []byte + readErr error + trailer http.Header +} + +func (h *ContainerHandler) captureResponse(resp *http.Response) *capturedResponse { + captured := &capturedResponse{status: resp.StatusCode, header: resp.Header.Clone()} + if resp.Request != nil && resp.Request.Method == http.MethodHead { + return captured + } + limit := h.proxy.MetadataMaxSize + if limit <= 0 { + limit = defaultMetadataMaxSize + } + captured.body, captured.readErr = io.ReadAll(io.LimitReader(resp.Body, limit+1)) + if captured.readErr == nil && int64(len(captured.body)) > limit { + captured.body, captured.readErr = captured.body[:limit], ErrMetadataTooLarge + } + captured.trailer = resp.Trailer.Clone() + return captured +} + +type failingReader struct{ err error } + +func (f failingReader) Read([]byte) (int, error) { return 0, f.err } + +func (c *capturedResponse) write(p *Proxy, w http.ResponseWriter, r *http.Request) { + var body io.Reader = bytes.NewReader(c.body) + if c.readErr != nil { + body = io.MultiReader(body, failingReader{c.readErr}) + } + resp := &http.Response{ + StatusCode: c.status, + Header: c.header.Clone(), + Trailer: c.trailer.Clone(), + Body: io.NopCloser(body), + } + p.relayResponse(w, r, resp, copyContainerManifestHeaders) +} + +// coalescedManifest runs the upstream part of a manifest request once for +// concurrent callers with the same key. Like coalesceMetadata, it runs on a +// context detached from the first caller, and each waiter still leaves when +// its own context ends. The result is a whole manifest with its headers, +// which a metadataResult does not carry, hence the separate group. +func (h *ContainerHandler) coalescedManifest(ctx context.Context, req manifestRequest) manifestOutcome { + results := h.manifestFlight.DoChan(req.coalesceKey(), func() (any, error) { + return h.fetchManifest(context.WithoutCancel(ctx), req), nil + }) + select { + case <-ctx.Done(): + return manifestOutcome{err: ctx.Err()} + case res := <-results: + outcome, ok := res.Val.(manifestOutcome) + if !ok { + return manifestOutcome{err: errSharedFetchAbandoned} + } + return outcome + } +} + +// fetchManifest answers a manifest request that the cache could not. Other +// instances sharing the database take turns through a fetch lease, so a tag +// that expired everywhere at once is revalidated once. +func (h *ContainerHandler) fetchManifest(ctx context.Context, req manifestRequest) (outcome manifestOutcome) { + usable := func(m *cachedContainerManifest) bool { + return m != nil && (req.immutable() || h.containerManifestFresh(m)) + } + // A shared request that finished since the caller looked has refreshed + // the entry already. + cached, fromFallback := h.loadContainerManifestForAccept(ctx, req.registryURL, req.name, req.reference, req.accept, req.cacheAccept) + if usable(cached) { + return manifestOutcome{manifest: cached} + } + + // Without a TTL nothing another instance stores is fresh, so there is + // nothing to wait for. + if req.immutable() || h.proxy.MetadataTTL > 0 { + var stored *cachedContainerManifest + release, lease, err := h.proxy.awaitFetchLease(ctx, metrics.CacheKindMetadata, req.coalesceKey(), func() bool { + stored, _ = h.loadContainerManifestForAccept(ctx, req.registryURL, req.name, req.reference, req.accept, req.cacheAccept) + return usable(stored) + }) + if err != nil { + return manifestOutcome{err: err} + } + switch lease { + case leaseStored: + return manifestOutcome{manifest: stored} + case leaseFailed: + if cached != nil { + return manifestOutcome{err: errFetchFailedElsewhere} + } + case leaseFetch: + } + defer func() { release(outcome.err != nil) }() + } + + switch { + case req.method == http.MethodHead: + return h.headManifest(ctx, req, cached, fromFallback) + case req.immutable(): + return h.getManifest(ctx, req, req.reference, cached, fromFallback) + default: + return h.revalidateTag(ctx, req, cached, fromFallback) + } +} + +// revalidateTag answers a GET for a tag that is stale or not cached. Docker +// Hub counts manifest GETs against its pull limit but not HEADs, so a stale +// tag is resolved with a HEAD first and the manifest fetched by digest only +// when it is not cached yet. A registry whose HEAD gives no digest gets the +// GET for the tag, as before. A tag never seen is fetched with one GET: a +// HEAD first would only add a round trip. +func (h *ContainerHandler) revalidateTag(ctx context.Context, req manifestRequest, cached *cachedContainerManifest, fromFallback bool) manifestOutcome { + if cached == nil { + return h.getManifest(ctx, req, req.reference, nil, false) + } + resp, err := h.requestManifest(ctx, http.MethodHead, req, req.reference, "") + if err != nil { + return manifestOutcome{err: err} + } + _ = resp.Body.Close() + + digest := resp.Header.Get("Docker-Content-Digest") + switch { + case resp.StatusCode == http.StatusOK && manifestDigestReferencePattern.MatchString(digest): + if outcome, ok := h.tagFromCachedDigest(ctx, req, cached, fromFallback, resp, digest); ok { + return outcome + } + if outcome := h.getManifest(ctx, req, digest, nil, false); outcome.manifest != nil { + return outcome + } + // The tag may have moved again, or the registry may not serve + // manifests by digest to this client; ask for the tag itself. + case shouldServeStaleManifest(resp.StatusCode): + return manifestOutcome{err: fmt.Errorf("upstream status %d", resp.StatusCode)} + } + return h.getManifest(ctx, req, req.reference, cached, fromFallback) +} + +// headManifest answers a client HEAD. When the digest upstream names is +// cached, the tag entry is pointed at it, so the next GET is a cache hit. +// Otherwise the upstream headers are passed on and nothing is stored. +func (h *ContainerHandler) headManifest(ctx context.Context, req manifestRequest, cached *cachedContainerManifest, fromFallback bool) manifestOutcome { + resp, err := h.requestManifest(ctx, http.MethodHead, req, req.reference, "") + if err != nil { + return manifestOutcome{err: err} + } + _ = resp.Body.Close() + + digest := resp.Header.Get("Docker-Content-Digest") + switch { + case resp.StatusCode == http.StatusOK && manifestDigestReferencePattern.MatchString(digest): + if !req.immutable() { + if outcome, ok := h.tagFromCachedDigest(ctx, req, cached, fromFallback, resp, digest); ok { + return outcome + } + } + size := int64(-1) + if value := resp.Header.Get(headerContentLength); value != "" { + if parsed, err := strconv.ParseInt(value, 10, 64); err == nil { + size = parsed + } + } + return manifestOutcome{manifest: &cachedContainerManifest{ + contentType: resp.Header.Get(headerContentType), + contentDigest: digest, + etag: resp.Header.Get(headerETag), + size: size, + lastModified: parseHTTPTime(resp.Header.Get(headerLastModified)), + }} + case shouldServeStaleManifest(resp.StatusCode) && (cached != nil || referrersTagPattern.MatchString(req.reference)): + return manifestOutcome{err: fmt.Errorf("upstream status %d", resp.StatusCode)} + } + return manifestOutcome{relay: h.captureResponse(resp)} +} + +// tagFromCachedDigest answers a tag from the cache once a HEAD named the +// digest it points to: the cached tag entry if the tag did not move, or the +// manifest stored under the new digest. It reports false when neither is +// cached. +func (h *ContainerHandler) tagFromCachedDigest(ctx context.Context, req manifestRequest, cached *cachedContainerManifest, fromFallback bool, resp *http.Response, digest string) (manifestOutcome, bool) { + if cached != nil && cached.contentDigest == digest { + cached.fetchedAt = time.Now() + h.refreshManifest(ctx, req, cached, fromFallback) + return manifestOutcome{manifest: cached}, true + } + + stored, err := h.loadContainerManifest(ctx, h.containerManifestSharedCacheKey(req.registryURL, req.name, digest, req.cacheAccept)) + if err != nil || stored == nil || stored.contentDigest != digest || !containerManifestCacheCompatible(req.accept, stored) { + return manifestOutcome{}, false + } + moved := *stored + moved.fetchedAt = time.Now() + if etag := resp.Header.Get(headerETag); etag != "" { + moved.etag = etag + } + if lastModified := parseHTTPTime(resp.Header.Get(headerLastModified)); !lastModified.IsZero() { + moved.lastModified = lastModified + } + h.storeContainerManifestForAccept(ctx, req.registryURL, req.name, req.reference, req.accept, req.cacheAccept, &moved) + return manifestOutcome{manifest: &moved}, true +} + +// refreshManifest stores cached again with its new fetch time. +func (h *ContainerHandler) refreshManifest(ctx context.Context, req manifestRequest, cached *cachedContainerManifest, fromFallback bool) { + if fromFallback { + // The shared entry holds another representation for other + // clients; only this client's own entries are refreshed. + h.storeContainerManifestLegacyEntries(ctx, req.registryURL, req.name, req.reference, req.accept, req.cacheAccept, cached) + return + } + h.storeContainerManifestForAccept(ctx, req.registryURL, req.name, req.reference, req.accept, req.cacheAccept, cached) +} + +// getManifest GETs fetchRef, the requested reference or the digest a tag +// resolved to, and stores the manifest under the requested reference and +// its digest. cached, if set, is revalidated with its ETag. +func (h *ContainerHandler) getManifest(ctx context.Context, req manifestRequest, fetchRef string, cached *cachedContainerManifest, fromFallback bool) manifestOutcome { + etag := "" + if cached != nil { + etag = cached.etag + } + resp, err := h.requestManifest(ctx, http.MethodGet, req, fetchRef, etag) + if err != nil { + return manifestOutcome{err: err} + } + defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode == http.StatusNotModified && cached != nil { + cached.fetchedAt = time.Now() + h.refreshManifest(ctx, req, cached, fromFallback) + return manifestOutcome{manifest: cached} + } + if resp.StatusCode != http.StatusOK { + if shouldServeStaleManifest(resp.StatusCode) && (cached != nil || referrersTagPattern.MatchString(req.reference)) { + return manifestOutcome{err: fmt.Errorf("upstream status %d", resp.StatusCode)} + } + return manifestOutcome{relay: h.captureResponse(resp)} + } + + body, err := h.proxy.ReadMetadata(resp.Body) + if err != nil { + return manifestOutcome{err: fmt.Errorf("reading manifest: %w", err)} + } + contentDigest := resp.Header.Get("Docker-Content-Digest") + // The references fetched and stored under, and the digest the upstream + // says the manifest has, are all cache keys, so each digest among them + // must be the manifest's own. One the proxy can't compute is refused + // rather than cached unchecked. + for _, expected := range []string{req.reference, fetchRef, contentDigest} { + if !strings.Contains(expected, ":") { + continue + } + if err := verifyDigest(expected, body); err != nil { + h.proxy.Logger.Error("upstream manifest failed digest verification", + "name", req.name, "reference", req.reference, "expected", expected, "error", err) + return manifestOutcome{digestInvalid: true} + } + } + if contentDigest == "" { + contentDigest = sha256Digest(body) + } + + manifest := &cachedContainerManifest{ + body: body, + contentType: resp.Header.Get(headerContentType), + contentDigest: contentDigest, + etag: resp.Header.Get(headerETag), + size: int64(len(body)), + lastModified: parseHTTPTime(resp.Header.Get(headerLastModified)), + fetchedAt: time.Now(), + } + h.storeContainerManifestForAccept(ctx, req.registryURL, req.name, req.reference, req.accept, req.cacheAccept, manifest) + if manifest.contentDigest != req.reference && manifestDigestReferencePattern.MatchString(manifest.contentDigest) { + h.storeContainerManifestForAccept(ctx, req.registryURL, req.name, manifest.contentDigest, req.accept, req.cacheAccept, manifest) + } + return manifestOutcome{manifest: manifest} +} + +// requestManifest sends one manifest request upstream with the client's +// Accept header. The registry client adds credentials and follows bearer +// challenges for HEAD as for GET. +func (h *ContainerHandler) requestManifest(ctx context.Context, method string, req manifestRequest, reference, etag string) (*http.Response, error) { + upstream, err := http.NewRequestWithContext(ctx, method, req.url(reference), nil) + if err != nil { + return nil, fmt.Errorf("creating manifest request: %w", err) + } + upstream.Header.Set("Accept", req.accept) + if etag != "" { + upstream.Header.Set("If-None-Match", etag) + } + return h.proxy.HTTPClient.Do(upstream) +} diff --git a/internal/handler/container_manifest_revalidate_test.go b/internal/handler/container_manifest_revalidate_test.go new file mode 100644 index 0000000..abe0fc5 --- /dev/null +++ b/internal/handler/container_manifest_revalidate_test.go @@ -0,0 +1,422 @@ +package handler + +import ( + "io" + "net/http" + "net/http/httptest" + "strconv" + "strings" + "sync" + "testing" + "time" +) + +const revalidateRepo = "/v2/library/app/manifests/" + +// countingRegistry serves manifests by tag and digest and counts GET and HEAD +// requests per reference, since Docker Hub counts only GETs as pulls. +type countingRegistry struct { + server *httptest.Server + + mu sync.Mutex + docs map[string]string // by digest + tags map[string]string // tag -> digest + gets map[string]int + heads map[string]int + headAccept string + // headDigest is false for a registry whose HEAD has no + // Docker-Content-Digest. + headDigest bool + // hold delays each response until holdFor requests arrived, or + // holdTimeout passed, so concurrent callers overlap upstream. + holdFor int + holdTimeout time.Duration + arrived int + release chan struct{} +} + +func newCountingRegistry(t *testing.T) *countingRegistry { + t.Helper() + registry := &countingRegistry{ + docs: make(map[string]string), + tags: make(map[string]string), + gets: make(map[string]int), + heads: make(map[string]int), + headDigest: true, + release: make(chan struct{}), + } + registry.server = httptest.NewServer(http.HandlerFunc(registry.serve)) + t.Cleanup(registry.server.Close) + return registry +} + +func (c *countingRegistry) serve(w http.ResponseWriter, r *http.Request) { + reference, ok := strings.CutPrefix(r.URL.Path, revalidateRepo) + if !ok { + http.NotFound(w, r) + return + } + c.mu.Lock() + if r.Method == http.MethodHead { + c.heads[reference]++ + c.headAccept = r.Header.Get("Accept") + } else { + c.gets[reference]++ + } + hold := c.holdFor > 0 + if hold { + c.arrived++ + if c.arrived == c.holdFor { + close(c.release) + } + } + digest, isTag := c.tags[reference] + if !isTag { + digest = reference + } + body, found := c.docs[digest] + headDigest := c.headDigest + c.mu.Unlock() + + if hold { + select { + case <-c.release: + case <-time.After(c.holdTimeout): + } + } + if !found { + http.NotFound(w, r) + return + } + etag := `"` + digest + `"` + w.Header().Set("Content-Type", ociManifestMediaType) + w.Header().Set("ETag", etag) + if r.Method != http.MethodHead || headDigest { + w.Header().Set("Docker-Content-Digest", digest) + } + if r.Header.Get("If-None-Match") == etag { + w.WriteHeader(http.StatusNotModified) + return + } + w.Header().Set("Content-Length", strconv.Itoa(len(body))) + if r.Method != http.MethodHead { + _, _ = io.WriteString(w, body) + } +} + +// add stores a manifest and returns its digest. +func (c *countingRegistry) add(body string) string { + digest := sha256Digest([]byte(body)) + c.mu.Lock() + c.docs[digest] = body + c.mu.Unlock() + return digest +} + +// pointLatest points the latest tag at digest. +func (c *countingRegistry) pointLatest(digest string) { + c.mu.Lock() + c.tags["latest"] = digest + c.mu.Unlock() +} + +// counts returns the GETs and HEADs seen so far, all references together. +func (c *countingRegistry) counts() (gets, heads int) { + c.mu.Lock() + defer c.mu.Unlock() + for _, n := range c.gets { + gets += n + } + for _, n := range c.heads { + heads += n + } + return gets, heads +} + +func (c *countingRegistry) getsOf(reference string) int { + c.mu.Lock() + defer c.mu.Unlock() + return c.gets[reference] +} + +func newRevalidateTestHandler(t *testing.T, registry *countingRegistry) *ContainerHandler { + t.Helper() + proxy, _, _, _ := setupTestProxy(t) + proxy.HTTPClient = registry.server.Client() + proxy.MetadataTTL = time.Hour + return &ContainerHandler{proxy: proxy, registryURL: registry.server.URL, proxyURL: "http://localhost:8080"} +} + +// ageLatest moves the cached entry of the latest tag past the TTL. +func ageLatest(t *testing.T, h *ContainerHandler) { + t.Helper() + key := h.containerManifestSharedCacheKey(h.registryURL, "library/app", "latest", normalizeContainerManifestAccept(dockerManifestAccept)) + entry, err := h.proxy.DB.GetMetadataCache(containerManifestCacheEcosystem, key) + if err != nil || entry == nil { + t.Fatalf("cached tag row = %v, %v, want one", entry, err) + } + entry.FetchedAt.Time = time.Now().Add(-2 * time.Hour) + if err := h.proxy.DB.UpsertMetadataCache(entry); err != nil { + t.Fatalf("ageing cached tag: %v", err) + } +} + +func pullManifest(t *testing.T, h *ContainerHandler, method, reference string) *httptest.ResponseRecorder { + t.Helper() + request := httptest.NewRequest(method, "/library/app/manifests/"+reference, nil) + request.Header.Set("Accept", dockerManifestAccept) + response := httptest.NewRecorder() + h.Routes().ServeHTTP(response, request) + if response.Code != http.StatusOK { + t.Fatalf("%s %s status = %d: %s", method, reference, response.Code, response.Body.String()) + } + return response +} + +func manifestDoc(n string) string { + return `{"schemaVersion":2,"mediaType":"application/vnd.oci.image.manifest.v1+json","annotations":{"n":"` + n + `"}}` +} + +func TestContainerHandler_StaleTagUnchangedRevalidatesWithHead(t *testing.T) { + registry := newCountingRegistry(t) + digest := registry.add(manifestDoc("a")) + registry.pointLatest(digest) + h := newRevalidateTestHandler(t, registry) + + pullManifest(t, h, http.MethodGet, "latest") + ageLatest(t, h) + response := pullManifest(t, h, http.MethodGet, "latest") + + if response.Body.String() != manifestDoc("a") { + t.Errorf("body = %q, want the cached manifest", response.Body.String()) + } + if gets, heads := registry.counts(); gets != 1 || heads != 1 { + t.Errorf("upstream GETs = %d, HEADs = %d; want 1 GET for the first pull and 1 HEAD to revalidate", gets, heads) + } + registry.mu.Lock() + headAccept := registry.headAccept + registry.mu.Unlock() + if headAccept != dockerManifestAccept { + t.Errorf("HEAD Accept = %q, want the client's %q", headAccept, dockerManifestAccept) + } + // The HEAD refreshed the entry, so the next pull stays local. + pullManifest(t, h, http.MethodGet, "latest") + if gets, heads := registry.counts(); gets != 1 || heads != 1 { + t.Errorf("after a fresh pull: GETs = %d, HEADs = %d, want 1 and 1", gets, heads) + } +} + +func TestContainerHandler_StaleTagMovedToCachedDigestNeedsNoGet(t *testing.T) { + registry := newCountingRegistry(t) + first := registry.add(manifestDoc("a")) + second := registry.add(manifestDoc("b")) + registry.pointLatest(first) + h := newRevalidateTestHandler(t, registry) + + pullManifest(t, h, http.MethodGet, "latest") + pullManifest(t, h, http.MethodGet, second) + registry.pointLatest(second) + ageLatest(t, h) + + response := pullManifest(t, h, http.MethodGet, "latest") + if response.Body.String() != manifestDoc("b") { + t.Errorf("body = %q, want the manifest the tag moved to", response.Body.String()) + } + if got := response.Header().Get("Docker-Content-Digest"); got != second { + t.Errorf("Docker-Content-Digest = %q, want %q", got, second) + } + if gets, heads := registry.counts(); gets != 2 || heads != 1 { + t.Errorf("upstream GETs = %d, HEADs = %d; want only the 2 setup GETs and 1 HEAD", gets, heads) + } +} + +func TestContainerHandler_StaleTagMovedToNewDigestGetsByDigest(t *testing.T) { + registry := newCountingRegistry(t) + first := registry.add(manifestDoc("a")) + registry.pointLatest(first) + h := newRevalidateTestHandler(t, registry) + + pullManifest(t, h, http.MethodGet, "latest") + second := registry.add(manifestDoc("b")) + registry.pointLatest(second) + ageLatest(t, h) + + response := pullManifest(t, h, http.MethodGet, "latest") + if response.Body.String() != manifestDoc("b") { + t.Errorf("body = %q, want the manifest the tag moved to", response.Body.String()) + } + if got := registry.getsOf("latest"); got != 1 { + t.Errorf("GETs of the tag = %d, want only the first pull's", got) + } + if got := registry.getsOf(second); got != 1 { + t.Errorf("GETs of the new digest = %d, want 1", got) + } + if _, heads := registry.counts(); heads != 1 { + t.Errorf("HEADs = %d, want 1", heads) + } + // Both the tag and the digest are now cached. + pullManifest(t, h, http.MethodGet, "latest") + pullManifest(t, h, http.MethodGet, second) + if gets, heads := registry.counts(); gets != 2 || heads != 1 { + t.Errorf("after cached pulls: GETs = %d, HEADs = %d, want 2 and 1", gets, heads) + } +} + +func TestContainerHandler_ConcurrentStaleTagRevalidatesOnce(t *testing.T) { + const pulls = 8 + registry := newCountingRegistry(t) + digest := registry.add(manifestDoc("a")) + registry.pointLatest(digest) + h := newRevalidateTestHandler(t, registry) + + pullManifest(t, h, http.MethodGet, "latest") + ageLatest(t, h) + setupGets, setupHeads := registry.counts() + registry.mu.Lock() + // Hold upstream answers until every pull reached upstream, or long + // enough that a coalesced request is the only one. + registry.holdFor = pulls + registry.holdTimeout = 300 * time.Millisecond + registry.mu.Unlock() + + var wg sync.WaitGroup + for range pulls { + wg.Go(func() { + request := httptest.NewRequest(http.MethodGet, "/library/app/manifests/latest", nil) + request.Header.Set("Accept", dockerManifestAccept) + response := httptest.NewRecorder() + h.Routes().ServeHTTP(response, request) + if response.Code != http.StatusOK || response.Body.String() != manifestDoc("a") { + t.Errorf("concurrent pull = %d %q", response.Code, response.Body.String()) + } + }) + } + wg.Wait() + + gets, heads := registry.counts() + if requests := gets - setupGets + heads - setupHeads; requests != 1 { + t.Errorf("upstream requests for %d concurrent pulls = %d, want 1", pulls, requests) + } +} + +func TestContainerHandler_StaleTagHeadWithoutDigestFallsBackToGet(t *testing.T) { + registry := newCountingRegistry(t) + first := registry.add(manifestDoc("a")) + registry.pointLatest(first) + registry.headDigest = false + h := newRevalidateTestHandler(t, registry) + + pullManifest(t, h, http.MethodGet, "latest") + second := registry.add(manifestDoc("b")) + registry.pointLatest(second) + ageLatest(t, h) + + response := pullManifest(t, h, http.MethodGet, "latest") + if response.Body.String() != manifestDoc("b") { + t.Errorf("body = %q, want the manifest the tag moved to", response.Body.String()) + } + if got := registry.getsOf("latest"); got != 2 { + t.Errorf("GETs of the tag = %d, want 2: the HEAD gave no digest", got) + } +} + +func TestContainerHandler_ClientHeadPointsTagAtCachedDigest(t *testing.T) { + registry := newCountingRegistry(t) + first := registry.add(manifestDoc("a")) + second := registry.add(manifestDoc("b")) + registry.pointLatest(first) + h := newRevalidateTestHandler(t, registry) + + pullManifest(t, h, http.MethodGet, "latest") + pullManifest(t, h, http.MethodGet, second) + registry.pointLatest(second) + ageLatest(t, h) + + head := pullManifest(t, h, http.MethodHead, "latest") + if got := head.Header().Get("Docker-Content-Digest"); got != second { + t.Errorf("HEAD Docker-Content-Digest = %q, want %q", got, second) + } + // The HEAD moved the cached tag, so the GET that follows it in a pull + // is a cache hit. + gets, heads := registry.counts() + response := pullManifest(t, h, http.MethodGet, "latest") + if response.Body.String() != manifestDoc("b") { + t.Errorf("body = %q, want the manifest the tag moved to", response.Body.String()) + } + if afterGets, afterHeads := registry.counts(); afterGets != gets || afterHeads != heads { + t.Errorf("GET after HEAD went upstream: GETs %d -> %d, HEADs %d -> %d", gets, afterGets, heads, afterHeads) + } +} + +func TestContainerHandler_ClientHeadOfUncachedDigestIsNotStored(t *testing.T) { + registry := newCountingRegistry(t) + first := registry.add(manifestDoc("a")) + registry.pointLatest(first) + h := newRevalidateTestHandler(t, registry) + + pullManifest(t, h, http.MethodGet, "latest") + second := registry.add(manifestDoc("b")) + registry.pointLatest(second) + ageLatest(t, h) + + head := pullManifest(t, h, http.MethodHead, "latest") + if got := head.Header().Get("Docker-Content-Digest"); got != second { + t.Errorf("HEAD Docker-Content-Digest = %q, want %q", got, second) + } + if got := head.Header().Get("Content-Length"); got != strconv.Itoa(len(manifestDoc("b"))) { + t.Errorf("HEAD Content-Length = %q, want the upstream manifest size", got) + } + if gets, _ := registry.counts(); gets != 1 { + t.Errorf("GETs = %d, want 1: a client HEAD must not cost a pull", gets) + } +} + +func TestContainerHandler_ConcurrentDigestMissFetchesOnce(t *testing.T) { + const pulls = 8 + registry := newCountingRegistry(t) + digest := registry.add(manifestDoc("a")) + registry.holdFor = pulls + registry.holdTimeout = 300 * time.Millisecond + h := newRevalidateTestHandler(t, registry) + + var wg sync.WaitGroup + for range pulls { + wg.Go(func() { + request := httptest.NewRequest(http.MethodGet, "/library/app/manifests/"+digest, nil) + request.Header.Set("Accept", dockerManifestAccept) + response := httptest.NewRecorder() + h.Routes().ServeHTTP(response, request) + if response.Code != http.StatusOK || response.Body.String() != manifestDoc("a") { + t.Errorf("concurrent pull = %d %q", response.Code, response.Body.String()) + } + }) + } + wg.Wait() + + if got := registry.getsOf(digest); got != 1 { + t.Errorf("upstream GETs for %d concurrent pulls by digest = %d, want 1", pulls, got) + } +} + +func TestContainerHandler_StaleTagDeletedUpstreamRelaysNotFound(t *testing.T) { + registry := newCountingRegistry(t) + registry.pointLatest(registry.add(manifestDoc("a"))) + h := newRevalidateTestHandler(t, registry) + + pullManifest(t, h, http.MethodGet, "latest") + registry.mu.Lock() + delete(registry.tags, "latest") + registry.mu.Unlock() + ageLatest(t, h) + + request := httptest.NewRequest(http.MethodGet, "/library/app/manifests/latest", nil) + request.Header.Set("Accept", dockerManifestAccept) + response := httptest.NewRecorder() + h.Routes().ServeHTTP(response, request) + if response.Code != http.StatusNotFound { + t.Fatalf("status = %d, want the upstream 404", response.Code) + } + if !strings.Contains(response.Body.String(), "not found") { + t.Errorf("body = %q, want the upstream error body", response.Body.String()) + } +} From 3afbf2444e4bfe31f3969f08179af33438fc6868 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 25 Oct 2026 17:32:52 +0100 Subject: [PATCH 435/633] Cache Helm and PyPI metadata rewrites The Helm index.yaml and PyPI simple pages and JSON API documents were parsed and rewritten on every request, cache hits and conditional requests included. They now go through the rewrite cache like npm and Composer, so a repeat request is served from memory with an ETag of its own, a dated Last-Modified, a gzip copy and 304 when it matches. PyPI simple pages are keyed by their content type and the versions filtered out as well as the upstream page. Download hrefs are now rewritten with a literal prefix search that makes the same changes as the old regular expression, and the link filter's expressions are compiled once. A chart cache miss no longer parses the whole Helm index again while the stored index is unchanged: the chart downloads it lists are kept per repository, keyed by the index digest. --- docs/architecture.md | 2 +- docs/configuration.md | 2 +- internal/handler/denylist_test.go | 2 +- internal/handler/helm.go | 143 ++++++++-- internal/handler/helm_oci_test.go | 12 +- internal/handler/helm_pypi_bench_test.go | 184 ++++++++++++ internal/handler/helm_pypi_rewrite_test.go | 307 +++++++++++++++++++++ internal/handler/helm_test.go | 2 +- internal/handler/pypi.go | 221 ++++++++++----- internal/handler/pypi_test.go | 2 +- internal/handler/rewrite_cache.go | 67 ++++- 11 files changed, 836 insertions(+), 108 deletions(-) create mode 100644 internal/handler/helm_pypi_bench_test.go create mode 100644 internal/handler/helm_pypi_rewrite_test.go diff --git a/docs/architecture.md b/docs/architecture.md index 76b2156..b4290ec 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -200,7 +200,7 @@ metadata_dates ( ) ``` -`metadata_dates` holds the `Last-Modified` served with rewritten npm and Composer metadata. A package's date is the second it started serving its current representation, and a new representation is only dated in a later second. Keeping it in the database means replicas sharing one serve the same date, and never give one representation's date to another, so `If-Modified-Since` cannot return 304 for a body the client does not have. +`metadata_dates` holds the `Last-Modified` served with rewritten metadata, such as npm packuments, Composer documents, PyPI simple pages and Helm indexes. A package's date is the second it started serving its current representation, and a new representation is only dated in a later second. Keeping it in the database means replicas sharing one serve the same date, and never give one representation's date to another, so `If-Modified-Since` cannot return 304 for a body the client does not have. On PostgreSQL, `INTEGER PRIMARY KEY` becomes `SERIAL`, `DATETIME` becomes `TIMESTAMP`, `INTEGER DEFAULT 0` booleans become `BOOLEAN DEFAULT FALSE`, and size/count columns use `BIGINT`. diff --git a/docs/configuration.md b/docs/configuration.md index 5a877e3..7699dc9 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1317,7 +1317,7 @@ Or via environment variable: `PROXY_ARTIFACT_MAX_SIZE=2GB`. ### Rewritten metadata cache -The npm and Composer handlers rewrite every metadata document they serve, so that download URLs point at the proxy. That means decoding the whole document and encoding it again, and Composer's minified documents are also expanded, which together cost milliseconds per typical package and far more for very large ones. The proxy keeps rewritten documents in memory, so each distinct upstream document is rewritten once and requests arriving while it is being rewritten wait for that rewrite. New bytes from upstream are rewritten again. With version cooldown enabled, a cached rewrite is kept for one minute only, because cooldown filtering depends on the current time: a version leaving cooldown shows up at most a minute late. Such documents are dated like any other, so clients that revalidate with `If-Modified-Since`, such as Composer, get `304 Not Modified`. +The npm, Composer, PyPI and Helm handlers rewrite every metadata document they serve, so that download URLs point at the proxy. That means decoding the whole document and encoding it again, and Composer's minified documents are also expanded, which together cost milliseconds per typical package and far more for very large ones; a large Helm `index.yaml` takes over a hundred milliseconds. The proxy keeps rewritten documents in memory, so each distinct upstream document is rewritten once and requests arriving while it is being rewritten wait for that rewrite. New bytes from upstream are rewritten again. A PyPI simple page is cached per representation (HTML or JSON) and per set of versions filtered out of it. With version cooldown enabled, a cached rewrite is kept for one minute only, because cooldown filtering depends on the current time: a version leaving cooldown shows up at most a minute late. Cached documents carry an `ETag` of their own, and are dated like any other, so clients that revalidate with `If-None-Match` or `If-Modified-Since`, such as Composer, get `304 Not Modified`. Each cached document is also kept gzip-compressed, when that makes it smaller, and served with `Content-Encoding: gzip` to clients that accept it. A large packument is compressed once rather than on every request, by the proxy or by a reverse proxy in front of it. The compressed copy counts against the cache size and typically adds about an eighth to each entry. Documents rewritten outside the cache are served uncompressed. diff --git a/internal/handler/denylist_test.go b/internal/handler/denylist_test.go index 8e92471..e89f186 100644 --- a/internal/handler/denylist_test.go +++ b/internal/handler/denylist_test.go @@ -147,7 +147,7 @@ func TestPyPIDenylistJSONMetadata(t *testing.T) { setTestDenylist(t, p, "pkg:pypi/demo@1.0.0") h := NewPyPIHandlerWithUpstreams(p, "https://proxy.example", "", "") body := `{"info":{"name":"Demo","version":"1.0.0"},"releases":{"1.0.0":[{"url":"https://files.pythonhosted.org/packages/demo-1.0.0.tar.gz"}],"2.0.0":[]},"urls":[{"url":"https://files.pythonhosted.org/packages/demo-1.0.0.tar.gz"}]}` - out, err := h.rewriteJSONMetadata([]byte(body)) + out, _, err := h.rewriteJSONMetadata([]byte(body)) if err != nil { t.Fatal(err) } diff --git a/internal/handler/helm.go b/internal/handler/helm.go index bb6b623..b800069 100644 --- a/internal/handler/helm.go +++ b/internal/handler/helm.go @@ -9,6 +9,7 @@ import ( "net/url" "path" "strings" + "sync" "time" "gopkg.in/yaml.v3" @@ -38,6 +39,10 @@ type HelmHandler struct { proxyURL string repositories map[string]string ociRegistries []helmOCIRegistry + + lookupMu sync.Mutex + // lookups holds the charts of each repository's last parsed index. + lookups map[string]*helmChartLookup } // NewHelmHandler creates a Helm chart repository protocol handler. @@ -68,20 +73,36 @@ func (h *HelmHandler) handleIndex(w http.ResponseWriter, r *http.Request) { return } - body, contentType, err := h.fetchIndex(r, repository, upstreamURL) - if err != nil { - h.serveIndexError(w, err) + // Parsing and encoding a large index costs far more than serving it, so + // rewrites go through the rewrite cache like npm packuments. The cache + // key holds the repository and upstream URL, both part of cacheKey. + cacheKey := h.indexCacheKey(repository, upstreamURL) + if stored, contentType, ok := h.proxy.storedRewriteVariant(helmMetadataEcosystem, cacheKey, h.proxyURL, cacheKey, nil); ok { + serveRewrittenMetadataAs(w, r, stored, contentType) return } - rewritten, err := h.rewriteIndex(repository, upstreamURL, body) + index := h.fetchIndex(r, repository, upstreamURL) + if index.err != nil { + h.serveIndexError(w, index.err) + return + } + + rewritten, err := h.proxy.cachedRewrite(r.Context(), helmMetadataEcosystem, h.proxyURL, cacheKey, index.body, index.digest, + func(body []byte) ([]byte, []string, error) { + return h.rewriteIndex(repository, upstreamURL, body) + }) if err != nil { + if r.Context().Err() != nil { + return // the client left while waiting on a shared rewrite + } h.proxy.Logger.Warn("failed to rewrite Helm index", "repository", repository, "error", err) http.Error(w, "invalid Helm repository index", http.StatusBadGateway) return } - h.proxy.writeMetadataCachedResponse(w, r, helmMetadataEcosystem, h.indexCacheKey(repository, upstreamURL), rewritten, contentType) + markStale(w, index.stale) + serveRewrittenMetadataAs(w, r, rewritten, index.contentType) } func (h *HelmHandler) handleChart(w http.ResponseWriter, r *http.Request) { @@ -109,13 +130,13 @@ func (h *HelmHandler) handleChart(w http.ResponseWriter, r *http.Request) { return } - body, _, err := h.fetchIndex(r, repository, upstreamURL) - if err != nil { - h.serveIndexError(w, err) + index := h.fetchIndex(r, repository, upstreamURL) + if index.err != nil { + h.serveIndexError(w, index.err) return } - downloadURL, err := h.findChartDownload(upstreamURL, body, digest, filename) + downloadURL, err := h.findChartDownload(repository, upstreamURL, index, digest, filename) if err != nil { if errors.Is(err, errHelmChartNotFound) { http.NotFound(w, r) @@ -161,13 +182,14 @@ func (h *HelmHandler) repositoryForRequest(r *http.Request) (name, upstreamURL s return name, upstreamURL, ok } -func (h *HelmHandler) fetchIndex(r *http.Request, repository, upstreamURL string) ([]byte, string, error) { - return h.proxy.FetchOrCacheMetadata( +func (h *HelmHandler) fetchIndex(r *http.Request, repository, upstreamURL string) metadataResult { + return h.proxy.fetchMetadataDocument( r.Context(), helmMetadataEcosystem, h.indexCacheKey(repository, upstreamURL), upstreamURL+"/"+helmIndexFilename, "application/x-yaml, text/yaml;q=0.9, */*;q=0.1", + nil, ) } @@ -189,17 +211,21 @@ func (h *HelmHandler) serveIndexError(w http.ResponseWriter, err error) { http.Error(w, "failed to fetch Helm repository index", http.StatusBadGateway) } -func (h *HelmHandler) rewriteIndex(repository, upstreamURL string, body []byte) ([]byte, error) { +// rewriteIndex points the index's chart URLs at the proxy and drops releases +// in cooldown. It also returns the releases it kept, by chart name and +// digest, which together with the index fix its output. +func (h *HelmHandler) rewriteIndex(repository, upstreamURL string, body []byte) ([]byte, []string, error) { document, entries, err := parseHelmIndex(body) if err != nil { - return nil, err + return nil, nil, err } + var kept []string for i := 0; i < len(entries.Content); i += 2 { chartName := entries.Content[i].Value releases := entries.Content[i+1] if releases.Kind != yaml.SequenceNode { - return nil, fmt.Errorf("chart %q releases must be a sequence", chartName) + return nil, nil, fmt.Errorf("chart %q releases must be a sequence", chartName) } filtered := make([]*yaml.Node, 0, len(releases.Content)) @@ -221,42 +247,109 @@ func (h *HelmHandler) rewriteIndex(repository, upstreamURL string, body []byte) } } filtered = append(filtered, release) + kept = append(kept, chartName+"@"+chart.digest) } releases.Content = filtered } - return yaml.Marshal(document) + out, err := yaml.Marshal(document) + return out, kept, err } -func (h *HelmHandler) findChartDownload(upstreamURL string, body []byte, digest, filename string) (string, error) { - _, entries, err := parseHelmIndex(body) +// findChartDownload returns the upstream URL of a chart in the index. Parsing +// a large index takes a while, so the charts it lists are kept per +// repository for as long as the stored index stays the same. +func (h *HelmHandler) findChartDownload(repository, upstreamURL string, index metadataResult, digest, filename string) (string, error) { + charts, err := h.chartLookup(repository, upstreamURL, index) if err != nil { return "", err } + for _, chart := range charts.byDigest[digest] { + if h.chartOnCooldown(chart.name, chart.created) { + continue + } + for _, download := range chart.downloads { + if download.filename == filename { + return download.url, nil + } + } + } + return "", errHelmChartNotFound +} +// helmChartLookup holds the HTTP chart downloads an index lists, by chart +// digest, in index order. +type helmChartLookup struct { + indexDigest string + byDigest map[string][]helmChartEntry +} + +type helmChartEntry struct { + name string + created time.Time + downloads []helmChartDownload +} + +// chartLookup returns the charts the index lists, parsing it only when the +// last lookup built for the repository came from other bytes. An index +// without a digest, read with the metadata cache off, is parsed every time. +func (h *HelmHandler) chartLookup(repository, upstreamURL string, index metadataResult) (*helmChartLookup, error) { + key := repository + "\x00" + upstreamURL + if index.digest != "" { + h.lookupMu.Lock() + cached := h.lookups[key] + h.lookupMu.Unlock() + if cached != nil && cached.indexDigest == index.digest { + return cached, nil + } + } + + lookup, err := h.buildChartLookup(upstreamURL, index.body) + if err != nil { + return nil, err + } + if index.digest != "" { + lookup.indexDigest = index.digest + h.lookupMu.Lock() + if h.lookups == nil { + h.lookups = make(map[string]*helmChartLookup) + } + h.lookups[key] = lookup + h.lookupMu.Unlock() + } + return lookup, nil +} + +func (h *HelmHandler) buildChartLookup(upstreamURL string, body []byte) (*helmChartLookup, error) { + _, entries, err := parseHelmIndex(body) + if err != nil { + return nil, err + } + + lookup := &helmChartLookup{byDigest: make(map[string][]helmChartEntry)} for i := 0; i < len(entries.Content); i += 2 { chartName := entries.Content[i].Value releases := entries.Content[i+1] if releases.Kind != yaml.SequenceNode { - return "", fmt.Errorf("chart %q releases must be a sequence", chartName) + return nil, fmt.Errorf("chart %q releases must be a sequence", chartName) } for _, release := range releases.Content { chart, err := h.parseChartRelease(chartName, upstreamURL, release) if err != nil { continue } - if chart.digest != digest || h.chartOnCooldown(chartName, chart.created) { - continue - } + entry := helmChartEntry{name: chartName, created: chart.created} for _, download := range chart.downloads { - if !download.oci && download.filename == filename { - return download.url, nil + if !download.oci { + // Drop the YAML node, so the lookup doesn't keep the + // parsed index alive. + entry.downloads = append(entry.downloads, helmChartDownload{url: download.url, filename: download.filename}) } } + lookup.byDigest[chart.digest] = append(lookup.byDigest[chart.digest], entry) } } - - return "", errHelmChartNotFound + return lookup, nil } func (h *HelmHandler) chartOnCooldown(chartName string, created time.Time) bool { diff --git a/internal/handler/helm_oci_test.go b/internal/handler/helm_oci_test.go index c5163bd..e421e3f 100644 --- a/internal/handler/helm_oci_test.go +++ b/internal/handler/helm_oci_test.go @@ -57,7 +57,7 @@ entries: if !strings.Contains(response.Body.String(), ociURL) || !strings.Contains(response.Body.String(), h.chartProxyURL("mixed", digest, "demo-1.0.0.tgz")) { t.Fatalf("incorrectly rewritten index: %s", response.Body.String()) } - download, err := h.findChartDownload(upstream.URL, []byte(index), digest, "demo-1.0.0.tgz") + download, err := h.findChartDownload("mixed", upstream.URL, metadataResult{body: []byte(index)}, digest, "demo-1.0.0.tgz") if err != nil || download != upstream.URL+"/demo-1.0.0.tgz" { t.Fatalf("HTTP chart lookup = %q, %v", download, err) } @@ -103,7 +103,7 @@ func TestHelmOCIReferenceRewriting(t *testing.T) { } h := NewHelmHandlerWithOCIRegistries(&Proxy{}, tt.proxyURL, nil, tt.defaultRegistry, tt.registries) index := fmt.Sprintf("apiVersion: v1\nentries:\n demo:\n - digest: %s\n urls: [%q]\n", strings.Repeat("a", 64), tt.reference) - rewritten, err := h.rewriteIndex("mixed", "https://charts.example", []byte(index)) + rewritten, _, err := h.rewriteIndex("mixed", "https://charts.example", []byte(index)) if err != nil { t.Fatal(err) } @@ -125,7 +125,7 @@ func TestHelmIndexOmitsMalformedChartReleases(t *testing.T) { t.Run(reference, func(t *testing.T) { h := NewHelmHandler(&Proxy{Logger: slog.New(slog.NewTextHandler(io.Discard, nil))}, "https://proxy.example", nil) index := fmt.Sprintf("apiVersion: v1\nentries:\n demo:\n - digest: %s\n urls: [%q]\n - digest: %s\n urls: [demo-1.0.0.tgz]\n", strings.Repeat("b", 64), reference, goodDigest) - rewritten, err := h.rewriteIndex("mixed", "https://charts.example", []byte(index)) + rewritten, _, err := h.rewriteIndex("mixed", "https://charts.example", []byte(index)) if err != nil { t.Fatal(err) } @@ -135,11 +135,11 @@ func TestHelmIndexOmitsMalformedChartReleases(t *testing.T) { if !strings.Contains(string(rewritten), h.chartProxyURL("mixed", goodDigest, "demo-1.0.0.tgz")) { t.Fatalf("valid release missing from rewritten index: %s", rewritten) } - download, err := h.findChartDownload("https://charts.example", []byte(index), goodDigest, "demo-1.0.0.tgz") + download, err := h.findChartDownload("mixed", "https://charts.example", metadataResult{body: []byte(index)}, goodDigest, "demo-1.0.0.tgz") if err != nil || download != "https://charts.example/demo-1.0.0.tgz" { t.Fatalf("chart lookup = %q, %v", download, err) } - if _, err := h.findChartDownload("https://charts.example", []byte(index), strings.Repeat("b", 64), "demo-1.0.0.tgz"); !errors.Is(err, errHelmChartNotFound) { + if _, err := h.findChartDownload("mixed", "https://charts.example", metadataResult{body: []byte(index)}, strings.Repeat("b", 64), "demo-1.0.0.tgz"); !errors.Is(err, errHelmChartNotFound) { t.Fatalf("omitted release lookup error = %v, want errHelmChartNotFound", err) } }) @@ -160,7 +160,7 @@ entries: urls: [oci://ghcr.io/owner/demo:2.0.0] `, strings.Repeat("a", 64), time.Now().Add(-7*24*time.Hour).Format(time.RFC3339), strings.Repeat("b", 64), time.Now().Add(-time.Hour).Format(time.RFC3339)) - rewritten, err := h.rewriteIndex("mixed", "https://charts.example", []byte(index)) + rewritten, _, err := h.rewriteIndex("mixed", "https://charts.example", []byte(index)) if err != nil { t.Fatal(err) } diff --git a/internal/handler/helm_pypi_bench_test.go b/internal/handler/helm_pypi_bench_test.go new file mode 100644 index 0000000..afda74f --- /dev/null +++ b/internal/handler/helm_pypi_bench_test.go @@ -0,0 +1,184 @@ +package handler + +import ( + "fmt" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" +) + +// benchHelmIndex builds an index.yaml shaped like a large public chart +// repository: many charts, each with many releases. +func benchHelmIndex(charts, releases int) string { + var b strings.Builder + b.WriteString("apiVersion: v1\nentries:\n") + for c := range charts { + fmt.Fprintf(&b, " chart-%d:\n", c) + for r := range releases { + fmt.Fprintf(&b, ` - apiVersion: v2 + appVersion: 1.%d.0 + created: "2021-01-02T03:04:05.123456789Z" + dependencies: + - name: common + repository: oci://registry.example/charts + version: 2.x.x + description: A chart for benchmarking the index rewrite with a realistic amount of text. + digest: %064x + home: https://example.com/chart-%d + keywords: [web, server, benchmark] + maintainers: + - name: Maintainer + url: https://example.com + name: chart-%d + sources: + - https://github.com/example/chart-%d + urls: + - https://charts.example/chart-%d-1.%d.0.tgz + version: 1.%d.0 +`, r, c*releases+r, c, c, c, c, r, r) + } + } + b.WriteString("generated: \"2021-01-02T03:04:05Z\"\n") + return b.String() +} + +func benchServe(b *testing.B, h http.Handler, target, accept string) { + b.Helper() + b.ReportAllocs() + for b.Loop() { + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, target, nil) + if accept != "" { + req.Header.Set("Accept", accept) + } + h.ServeHTTP(rec, req) + if rec.Code != http.StatusOK { + b.Fatalf("status %d: %s", rec.Code, rec.Body.String()) + } + } +} + +// BenchmarkHelmIndexRequest serves a cached 2.5 MB index repeatedly. +func BenchmarkHelmIndexRequest(b *testing.B) { + index := benchHelmIndex(250, 20) + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/x-yaml") + _, _ = w.Write([]byte(index)) + })) + defer upstream.Close() + proxy, _, _, _ := setupTestProxy(b) + proxy.HTTPClient = upstream.Client() + proxy.CacheMetadata = true + proxy.MetadataTTL = time.Hour + proxy.SetMetadataRewriteCacheSize(256 << 20) + h := NewHelmHandler(proxy, "http://proxy.example", map[string]string{"stable": upstream.URL}).Routes() + b.SetBytes(int64(len(index))) + benchServe(b, h, "/stable/index.yaml", "") +} + +// benchPyPIProject builds simple HTML, simple JSON and JSON API documents +// for a project with many files, like botocore. +func benchPyPIProject(versions, filesPerVersion int) (simpleHTML, simpleJSON, jsonAPI string) { + var hb, sb, jb strings.Builder + hb.WriteString("\n\n \n \n Links for big\n \n \n

Links for big

\n") + sb.WriteString(`{"meta":{"api-version":"1.1"},"name":"big","files":[`) + jb.WriteString(`{"info":{"name":"big","version":"1.0.0"},"releases":{`) + n := 0 + for v := range versions { + version := fmt.Sprintf("1.%d.0", v) + if v > 0 { + jb.WriteByte(',') + } + fmt.Fprintf(&jb, "%q:[", version) + for f := range filesPerVersion { + filename := fmt.Sprintf("big-%s-py3-none-any%d.whl", version, f) + if f == 1 { + filename = fmt.Sprintf("big-%s.tar.gz", version) + } + url := fmt.Sprintf("https://files.pythonhosted.org/packages/%02x/%02x/%060x/%s", v%256, f, v*filesPerVersion+f, filename) + fmt.Fprintf(&hb, " %s
\n", url, n, n, n, filename) + if n > 0 { + sb.WriteByte(',') + } + fmt.Fprintf(&sb, `{"filename":%q,"url":%q,"hashes":{"sha256":"%064x"},"requires-python":">=3.8","core-metadata":{"sha256":"%064x"},"size":12345,"upload-time":"2021-01-02T03:04:05.123456Z","yanked":false}`, filename, url, n, n) + if f > 0 { + jb.WriteByte(',') + } + fmt.Fprintf(&jb, `{"filename":%q,"url":%q,"digests":{"sha256":"%064x","md5":"%032x"},"packagetype":"bdist_wheel","python_version":"py3","size":12345,"upload_time_iso_8601":"2021-01-02T03:04:05.123456Z","yanked":false}`, filename, url, n, n) + n++ + } + jb.WriteByte(']') + } + hb.WriteString(" \n\n\n") + sb.WriteString("]}") + jb.WriteString(`},"urls":[]}`) + return hb.String(), sb.String(), jb.String() +} + +func benchPyPIHandler(b *testing.B, deny ...string) http.Handler { + b.Helper() + simpleHTML, simpleJSON, jsonAPI := benchPyPIProject(2000, 2) + h, proxy := setupPyPIHandler(b, func(r *http.Request) (*http.Response, error) { + switch { + case r.URL.Path == "/pypi/big/json": + return pypiHTTPResponse(r, "application/json", jsonAPI), nil + case r.Header.Get("Accept") == pypiSimpleJSON: + return pypiHTTPResponse(r, pypiSimpleJSON, simpleJSON), nil + default: + return pypiHTTPResponse(r, "text/html", simpleHTML), nil + } + }) + proxy.CacheMetadata = true + proxy.MetadataTTL = time.Hour + proxy.SetMetadataRewriteCacheSize(256 << 20) + if len(deny) > 0 { + setTestDenylist(b, proxy, deny...) + } + return h.Routes() +} + +func BenchmarkPyPISimpleHTMLRequest(b *testing.B) { + benchServe(b, benchPyPIHandler(b), "/simple/big/", "") +} + +// BenchmarkPyPISimpleHTMLFilteredRequest has a version denied, so the page +// is filtered as well as rewritten. +func BenchmarkPyPISimpleHTMLFilteredRequest(b *testing.B) { + benchServe(b, benchPyPIHandler(b, "pkg:pypi/big@1.5.0"), "/simple/big/", "") +} + +func BenchmarkPyPISimpleJSONRequest(b *testing.B) { + benchServe(b, benchPyPIHandler(b), "/simple/big/", pypiSimpleJSON) +} + +func BenchmarkPyPIJSONAPIRequest(b *testing.B) { + benchServe(b, benchPyPIHandler(b), "/pypi/big/json", "") +} + +// BenchmarkPyPIRewriteSimpleHTML is the rewrite a cache miss pays. +func BenchmarkPyPIRewriteSimpleHTML(b *testing.B) { + simpleHTML, _, _ := benchPyPIProject(2000, 2) + body := []byte(simpleHTML) + h := NewPyPIHandlerWithUpstreams(testProxy(), "http://proxy.example", "", "") + b.SetBytes(int64(len(body))) + b.ReportAllocs() + for b.Loop() { + _ = h.rewriteSimpleHTML(body, nil) + } +} + +// BenchmarkPyPIRewriteSimpleHTMLFiltered is the same with a version filtered +// out. +func BenchmarkPyPIRewriteSimpleHTMLFiltered(b *testing.B) { + simpleHTML, _, _ := benchPyPIProject(2000, 2) + body := []byte(simpleHTML) + h := NewPyPIHandlerWithUpstreams(testProxy(), "http://proxy.example", "", "") + filtered := map[string]bool{"1.5.0": true} + b.SetBytes(int64(len(body))) + b.ReportAllocs() + for b.Loop() { + _ = h.rewriteSimpleHTML(h.filterSimpleHTMLLinks(body, filtered), filtered) + } +} diff --git a/internal/handler/helm_pypi_rewrite_test.go b/internal/handler/helm_pypi_rewrite_test.go new file mode 100644 index 0000000..8a8dd67 --- /dev/null +++ b/internal/handler/helm_pypi_rewrite_test.go @@ -0,0 +1,307 @@ +package handler + +import ( + "errors" + "fmt" + "net/http" + "net/http/httptest" + "regexp" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/git-pkgs/cooldown" +) + +// getRewritten requests target and checks the response status. +func getRewritten(t *testing.T, h http.Handler, target string, header http.Header, want int) *httptest.ResponseRecorder { + t.Helper() + req := httptest.NewRequest(http.MethodGet, target, nil) + for k, v := range header { + req.Header[k] = v + } + rec := httptest.NewRecorder() + h.ServeHTTP(rec, req) + if rec.Code != want { + t.Fatalf("GET %s: status %d, want %d: %s", target, rec.Code, want, rec.Body.String()) + } + return rec +} + +// checkServedFromRewriteCache requests target twice more after a first +// request, with storage failing, and checks the repeat is the cached rewrite: +// same body and ETag, 304 for the ETag, gzip for a client accepting it. +func checkServedFromRewriteCache(t *testing.T, h http.Handler, store *mockStorage, target string, header http.Header) string { + t.Helper() + first := getRewritten(t, h, target, header, http.StatusOK) + etag := first.Header().Get(headerETag) + if etag == "" { + t.Fatalf("no ETag on rewritten metadata; headers %v", first.Header()) + } + + // The repeat must not read the stored document back. + store.mu.Lock() + store.openErr = errors.New("storage must not be read") + store.mu.Unlock() + t.Cleanup(func() { + store.mu.Lock() + store.openErr = nil + store.mu.Unlock() + }) + + second := getRewritten(t, h, target, header, http.StatusOK) + if second.Body.String() != first.Body.String() || second.Header().Get(headerETag) != etag { + t.Errorf("repeat differs: ETag %q vs %q", second.Header().Get(headerETag), etag) + } + if got, want := second.Header().Get(headerContentType), first.Header().Get(headerContentType); got != want { + t.Errorf("repeat Content-Type = %q, want %q", got, want) + } + + conditional := header.Clone() + if conditional == nil { + conditional = http.Header{} + } + conditional.Set("If-None-Match", etag) + getRewritten(t, h, target, conditional, http.StatusNotModified) + + gzipped := header.Clone() + if gzipped == nil { + gzipped = http.Header{} + } + gzipped.Set(headerAcceptEncoding, "gzip") + compressed := getRewritten(t, h, target, gzipped, http.StatusOK) + if got := compressed.Header().Get(headerContentEncoding); got != "gzip" { + t.Errorf("Content-Encoding = %q, want gzip", got) + } + return first.Body.String() +} + +func TestHelmIndexServedFromRewriteCache(t *testing.T) { + digest := strings.Repeat("a", 64) + var indexRequests atomic.Int32 + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + indexRequests.Add(1) + w.Header().Set("Content-Type", "application/x-yaml") + _, _ = fmt.Fprintf(w, "apiVersion: v1\nentries:\n demo:\n%s", strings.Repeat(fmt.Sprintf( + " - digest: %s\n name: demo\n urls: [demo-1.0.0.tgz]\n version: 1.0.0\n", digest), 20)) + })) + defer upstream.Close() + + proxy, _, store, _ := setupTestProxy(t) + proxy.HTTPClient = upstream.Client() + proxy.CacheMetadata = true + proxy.MetadataTTL = time.Hour + proxy.SetMetadataRewriteCacheSize(1 << 20) + h := NewHelmHandler(proxy, "http://proxy.example", map[string]string{"stable": upstream.URL}).Routes() + + body := checkServedFromRewriteCache(t, h, store, "/stable/index.yaml", nil) + if !strings.Contains(body, "http://proxy.example/helm/stable/charts/"+digest+"/demo-1.0.0.tgz") { + t.Errorf("index not rewritten: %s", body) + } + if got := indexRequests.Load(); got != 1 { + t.Errorf("upstream requests = %d, want 1", got) + } + if got := proxy.rewrites.len(); got != 1 { + t.Errorf("cached rewrites = %d, want 1", got) + } +} + +// TestHelmChartLookupFollowsIndex checks that chart downloads are looked up +// in charts kept from the last parse of the same index, and that a changed +// index is parsed again. +func TestHelmChartLookupFollowsIndex(t *testing.T) { + chartA, chartB := []byte("chart a"), []byte("chart b") + digestA, digestB := helmSHA256Hex(chartA), helmSHA256Hex(chartB) + var withB atomic.Bool + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/index.yaml": + w.Header().Set("Content-Type", "application/x-yaml") + _, _ = fmt.Fprintf(w, "apiVersion: v1\nentries:\n a:\n - digest: %s\n urls: [a-1.0.0.tgz]\n", digestA) + if withB.Load() { + _, _ = fmt.Fprintf(w, " b:\n - digest: %s\n urls: [b-1.0.0.tgz]\n", digestB) + } + case "/a-1.0.0.tgz": + _, _ = w.Write(chartA) + case "/b-1.0.0.tgz": + _, _ = w.Write(chartB) + default: + http.NotFound(w, r) + } + })) + defer upstream.Close() + + proxy, _, _, _ := setupTestProxy(t) + proxy.HTTPClient = upstream.Client() + proxy.CacheMetadata = true + proxy.MetadataTTL = time.Hour + h := NewHelmHandler(proxy, "http://proxy.example", map[string]string{"stable": upstream.URL}) + index := func() metadataResult { + return h.fetchIndex(httptest.NewRequest(http.MethodGet, "/", nil), "stable", upstream.URL) + } + + first, err := h.findChartDownload("stable", upstream.URL, index(), digestA, "a-1.0.0.tgz") + if err != nil || first != upstream.URL+"/a-1.0.0.tgz" { + t.Fatalf("lookup = %q, %v", first, err) + } + lookup := h.lookups["stable\x00"+upstream.URL] + if lookup == nil { + t.Fatal("chart lookup was not kept") + } + if _, err := h.findChartDownload("stable", upstream.URL, index(), digestB, "b-1.0.0.tgz"); !errors.Is(err, errHelmChartNotFound) { + t.Fatalf("lookup of unlisted chart: %v", err) + } + if h.lookups["stable\x00"+upstream.URL] != lookup { + t.Error("same index was parsed again") + } + + // A new index from upstream lists a new chart. + withB.Store(true) + proxy.MetadataTTL = 0 + got, err := h.findChartDownload("stable", upstream.URL, index(), digestB, "b-1.0.0.tgz") + if err != nil || got != upstream.URL+"/b-1.0.0.tgz" { + t.Fatalf("lookup in new index = %q, %v", got, err) + } +} + +func TestPyPIMetadataServedFromRewriteCache(t *testing.T) { + const files = "https://files.pythonhosted.org/packages/ab/cd/" + simpleHTML := `demo-1.0.0.tar.gz
` + + `demo-2.0.0.tar.gz
` + + strings.Repeat("\n", 20) + simpleJSON := `{"meta":{"api-version":"1.1"},"name":"demo","files":[` + + `{"filename":"demo-1.0.0.tar.gz","url":"` + files + `demo-1.0.0.tar.gz"},` + + `{"filename":"demo-2.0.0.tar.gz","url":"` + files + `demo-2.0.0.tar.gz"}],` + + `"padding":"` + strings.Repeat("x", 1000) + `"}` + jsonAPI := `{"info":{"name":"demo","version":"1.0.0","description":"` + strings.Repeat("x", 1000) + `"},` + + `"releases":{"1.0.0":[{"url":"` + files + `demo-1.0.0.tar.gz"}],"2.0.0":[{"url":"` + files + `demo-2.0.0.tar.gz"}]},` + + `"urls":[{"url":"` + files + `demo-1.0.0.tar.gz"}]}` + + cases := []struct { + name, target, accept string + }{ + {"simple html", "/simple/demo/", ""}, + {"simple json", "/simple/demo/", pypiSimpleJSON}, + {"json api", "/pypi/demo/json", ""}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + var upstreamCalls atomic.Int32 + proxy, _, store, _ := setupTestProxy(t) + proxy.HTTPClient = &http.Client{Transport: pypiRoundTripFunc(func(r *http.Request) (*http.Response, error) { + upstreamCalls.Add(1) + switch { + case r.URL.Path == "/pypi/demo/json": + return pypiHTTPResponse(r, "application/json", jsonAPI), nil + case r.Header.Get("Accept") == pypiSimpleJSON: + return pypiHTTPResponse(r, pypiSimpleJSON, simpleJSON), nil + default: + return pypiHTTPResponse(r, "text/html", simpleHTML), nil + } + })} + proxy.CacheMetadata = true + proxy.MetadataTTL = time.Hour + proxy.SetMetadataRewriteCacheSize(1 << 20) + setTestDenylist(t, proxy, "pkg:pypi/demo@2.0.0") + h := NewPyPIHandlerWithUpstreams(proxy, "http://proxy.test", "https://pypi.test", "") + + var header http.Header + if c.accept != "" { + header = http.Header{"Accept": {c.accept}} + } + body := checkServedFromRewriteCache(t, h.Routes(), store, c.target, header) + if !strings.Contains(body, "http://proxy.test/pypi/packages/packages/ab/cd/demo-1.0.0.tar.gz") { + t.Errorf("download URL not rewritten: %s", body) + } + if strings.Contains(body, "demo-2.0.0") { + t.Errorf("denied version served: %s", body) + } + if got := upstreamCalls.Load(); got != 1 { + t.Errorf("upstream calls = %d, want 1", got) + } + + if c.target == "/simple/demo/" { + rec := getRewritten(t, h.Routes(), c.target, header, http.StatusOK) + vary := strings.Join(rec.Header().Values("Vary"), ", ") + if !strings.Contains(vary, "Accept-Encoding") || !strings.Contains(strings.ReplaceAll(vary, "Accept-Encoding", ""), "Accept") { + t.Errorf("Vary = %q, want Accept and Accept-Encoding", vary) + } + } + }) + } +} + +// TestPyPISimpleRewriteFollowsCooldown checks that a cached rewrite of a +// simple page is not served once the versions cooldown filters out change, +// though the page itself is the same. +func TestPyPISimpleRewriteFollowsCooldown(t *testing.T) { + now := time.Now() + old := now.Add(-30 * 24 * time.Hour).Format(time.RFC3339) + var released atomic.Value + released.Store(now.Add(-time.Hour).Format(time.RFC3339)) + + h, proxy := setupPyPIHandler(t, func(r *http.Request) (*http.Response, error) { + switch r.URL.Path { + case "/simple/ruff/": + return pypiHTTPResponse(r, "text/html", + `ruff-1.0.0.tar.gz`+ + `ruff-2.0.0.tar.gz`), nil + case "/pypi/ruff/json": + return pypiHTTPResponse(r, "application/json", `{"releases":{`+ + `"1.0.0":[{"upload_time_iso_8601":"`+old+`"}],`+ + `"2.0.0":[{"upload_time_iso_8601":"`+released.Load().(string)+`"}]}}`), nil + default: + t.Fatalf("unexpected upstream path: %s", r.URL.Path) + return nil, nil + } + }) + proxy.Cooldown = &cooldown.Config{Default: "7d"} + proxy.SetMetadataRewriteCacheSize(1 << 20) + + first := getRewritten(t, h.Routes(), "/simple/ruff/", nil, http.StatusOK).Body.String() + if strings.Contains(first, "ruff-2.0.0") || !strings.Contains(first, "ruff-1.0.0") { + t.Fatalf("first page = %s, want only ruff-1.0.0", first) + } + + released.Store(old) + second := getRewritten(t, h.Routes(), "/simple/ruff/", nil, http.StatusOK).Body.String() + if !strings.Contains(second, "ruff-2.0.0") { + t.Errorf("version out of cooldown still filtered: %s", second) + } +} + +// rewriteDownloadHrefsRegexp is the regular expression rewrite that +// rewriteDownloadHrefs replaces. +func rewriteDownloadHrefsRegexp(h *PyPIHandler, body []byte) []byte { + re := regexp.MustCompile(`href="(` + regexp.QuoteMeta(h.downloadURL) + `/packages/[^"]+)"`) + return re.ReplaceAllFunc(body, func(match []byte) []byte { + submatch := re.FindSubmatch(match) + return []byte(`href="` + h.proxyURL + "/pypi/packages" + strings.TrimPrefix(string(submatch[1]), h.downloadURL) + `"`) + }) +} + +func FuzzPyPIRewriteDownloadHrefs(f *testing.F) { + const dl = "https://files.pythonhosted.org" + for _, seed := range []string{ + `x`, + `empty`, + ``, + ``, + `x`, + `x`, + `x`, + `href="` + dl + `/packages/` + "\n" + `multi line"`, + `href="href="` + dl + `/packages/x"`, + } { + f.Add(seed) + } + h := NewPyPIHandlerWithUpstreams(testProxy(), "http://proxy.test", "", dl) + f.Fuzz(func(t *testing.T, body string) { + want := rewriteDownloadHrefsRegexp(h, []byte(body)) + if got := h.rewriteDownloadHrefs([]byte(body)); string(got) != string(want) { + t.Errorf("rewriteDownloadHrefs(%q)\n = %q\nwant %q", body, got, want) + } + }) +} diff --git a/internal/handler/helm_test.go b/internal/handler/helm_test.go index 26c0877..837934a 100644 --- a/internal/handler/helm_test.go +++ b/internal/handler/helm_test.go @@ -290,7 +290,7 @@ entries: `, time.Now().Add(-10*24*time.Hour).Format(time.RFC3339), oldDigest, time.Now().Add(-time.Hour).Format(time.RFC3339), newDigest) - rewritten, err := h.rewriteIndex("test", "https://charts.example", []byte(body)) + rewritten, _, err := h.rewriteIndex("test", "https://charts.example", []byte(body)) if err != nil { t.Fatalf("rewriteIndex() error = %v", err) } diff --git a/internal/handler/pypi.go b/internal/handler/pypi.go index 5667501..956b959 100644 --- a/internal/handler/pypi.go +++ b/internal/handler/pypi.go @@ -1,14 +1,17 @@ package handler import ( + "bytes" "crypto/sha256" "encoding/hex" "encoding/json" "errors" "fmt" + "maps" "mime" "net/http" "regexp" + "slices" "strconv" "strings" "time" @@ -40,11 +43,10 @@ const ( // PyPIHandler handles PyPI registry protocol requests. type PyPIHandler struct { - proxy *Proxy - upstreamURL string - downloadURL string - downloadHrefRe *regexp.Regexp - proxyURL string + proxy *Proxy + upstreamURL string + downloadURL string + proxyURL string } // PyPI files are stored under pkg:pypi/{name}, the name taken from the @@ -65,7 +67,6 @@ func NewPyPIHandlerWithUpstreams(proxy *Proxy, proxyURL, upstreamURL, downloadUR downloadURL: configuredUpstreamURL(downloadURL, pypiDownloadUpstream), proxyURL: strings.TrimSuffix(proxyURL, "/"), } - h.downloadHrefRe = regexp.MustCompile(`href="(` + regexp.QuoteMeta(h.downloadURL) + `/packages/[^"]+)"`) return h } @@ -107,8 +108,24 @@ func (h *PyPIHandler) handleSimplePackage(w http.ResponseWriter, r *http.Request accept := selectPyPISimpleRepresentation(r.Header.Get("Accept")) cacheKey := pypiSimpleCacheKey(name, accept) - body, contentType, err := h.proxy.FetchOrCacheMetadata(r.Context(), "pypi", cacheKey, upstreamURL, accept) - if err != nil { + // Rewritten pages go through the rewrite cache, keyed by the upstream + // page, its content type and the versions filtered out. Without cooldown + // the filtered versions are known up front, so a repeat request can be + // answered without reading the stored page. + cooldown := h.proxy.cooldownApplies("pypi", name) + if !cooldown { + denied := h.addDeniedVersions(name, nil) + if stored, contentType, ok := h.proxy.storedRewriteVariant("pypi", cacheKey, h.proxyURL, cacheKey, func(contentType string) []string { + return simpleRewriteVariant(contentType, denied) + }); ok { + ensureVaryAccept(w.Header()) + serveRewrittenMetadataAs(w, r, stored, contentType) + return + } + } + + upstream := h.proxy.fetchMetadataDocument(r.Context(), "pypi", cacheKey, upstreamURL, accept, nil) + if err := upstream.err; err != nil { if errors.Is(err, ErrUpstreamNotFound) { http.Error(w, "not found", http.StatusNotFound) return @@ -120,44 +137,70 @@ func (h *PyPIHandler) handleSimplePackage(w http.ResponseWriter, r *http.Request http.Error(w, "upstream request failed", http.StatusBadGateway) return } + contentType := upstream.contentType // When cooldown applies, fetch JSON metadata to get version timestamps var filteredVersions map[string]bool - if h.proxy.cooldownApplies("pypi", name) { + if cooldown { + var err error filteredVersions, err = h.fetchFilteredVersions(r, name) if err != nil { h.proxy.writeCooldownLookupError(w, "pypi", name, err) return } } - for version := range h.proxy.Denylist.Versions(canonicalPackagePURL("pypi", name)) { - if filteredVersions == nil { - filteredVersions = make(map[string]bool) - } - filteredVersions[version] = true - } + filteredVersions = h.addDeniedVersions(name, filteredVersions) - var rewritten []byte - if isJSONMediaType(contentType) { - rewritten, err = h.rewriteSimpleJSON(body, filteredVersions) - if err != nil { - if len(filteredVersions) != 0 { - h.proxy.Logger.Error("failed to filter pypi simple json", "package", name, "error", err) - http.Error(w, "failed to filter package metadata", http.StatusBadGateway) - return - } - h.proxy.Logger.Warn("failed to rewrite pypi simple json, proxying original", "error", err) - rewritten = body + isJSON := isJSONMediaType(contentType) + digest := variantDigest(upstream.digest, upstream.body, simpleRewriteVariant(contentType, filteredVersions)...) + rewritten, err := h.proxy.cachedRewrite(r.Context(), "pypi", h.proxyURL, cacheKey, upstream.body, digest, func(body []byte) ([]byte, []string, error) { + if isJSON { + out, err := h.rewriteSimpleJSON(body, filteredVersions) + return out, nil, err } - } else { body = h.filterSimpleHTMLLinks(body, filteredVersions) - rewritten = h.rewriteSimpleHTML(body, filteredVersions) + return h.rewriteSimpleHTML(body, filteredVersions), nil, nil + }) + if err != nil { + if r.Context().Err() != nil { + return // the client left while waiting on a shared rewrite + } + if len(filteredVersions) != 0 { + h.proxy.Logger.Error("failed to filter pypi simple json", "package", name, "error", err) + http.Error(w, "failed to filter package metadata", http.StatusBadGateway) + return + } + h.proxy.Logger.Warn("failed to rewrite pypi simple json, proxying original", "error", err) + w.Header().Set(headerContentType, contentType) + ensureVaryAccept(w.Header()) + markStale(w, upstream.stale) + w.WriteHeader(http.StatusOK) + _, _ = w.Write(upstream.body) + return } - w.Header().Set(headerContentType, contentType) ensureVaryAccept(w.Header()) - w.WriteHeader(http.StatusOK) - _, _ = w.Write(rewritten) + markStale(w, upstream.stale) + serveRewrittenMetadataAs(w, r, rewritten, contentType) +} + +// addDeniedVersions adds the package's denylisted versions to filtered, +// allocating it when there are any and it is nil. +func (h *PyPIHandler) addDeniedVersions(name string, filtered map[string]bool) map[string]bool { + for version := range h.proxy.Denylist.Versions(canonicalPackagePURL("pypi", name)) { + if filtered == nil { + filtered = make(map[string]bool) + } + filtered[version] = true + } + return filtered +} + +// simpleRewriteVariant returns what a simple page rewrite depends on besides +// the upstream page: its content type, which picks the HTML or JSON rewrite, +// and the versions filtered out. +func simpleRewriteVariant(contentType string, filtered map[string]bool) []string { + return append([]string{contentType}, slices.Sorted(maps.Keys(filtered))...) } func selectPyPISimpleRepresentation(accept string) string { @@ -258,14 +301,7 @@ func isJSONMediaType(contentType string) bool { } func ensureVaryAccept(header http.Header) { - for _, value := range header.Values("Vary") { - for field := range strings.SplitSeq(value, ",") { - if strings.EqualFold(strings.TrimSpace(field), "Accept") { - return - } - } - } - header.Add("Vary", "Accept") + ensureVary(header, "Accept") } // fetchFilteredVersions fetches JSON metadata and returns a set of version @@ -313,17 +349,21 @@ func (h *PyPIHandler) fetchFilteredVersions(r *http.Request, name string) (map[s return filtered, nil } +var ( + // simpleLinkRe matches a whole anchor tag: filename. + simpleLinkRe = regexp.MustCompile(`]+href="[^"]*"[^>]*>[^<]+`) + // simpleLinkTextRe captures an anchor's text. + simpleLinkTextRe = regexp.MustCompile(`>([^<]+)`) +) + // rewriteSimpleHTML rewrites package URLs in simple API HTML to point at this proxy. // If filteredVersions is non-nil, links for those versions are removed entirely. func (h *PyPIHandler) rewriteSimpleHTML(body []byte, filteredVersions map[string]bool) []byte { // If cooldown filtering is active, remove entire tags for filtered versions if len(filteredVersions) > 0 { - // Match full anchor tags: filename - linkRe := regexp.MustCompile(`]+href="[^"]*"[^>]*>[^<]+`) - body = linkRe.ReplaceAllFunc(body, func(match []byte) []byte { + body = simpleLinkRe.ReplaceAllFunc(body, func(match []byte) []byte { // Extract filename from between tags - innerRe := regexp.MustCompile(`>([^<]+)`) - innerMatch := innerRe.FindSubmatch(match) + innerMatch := simpleLinkTextRe.FindSubmatch(match) if len(innerMatch) < minSubmatchParts { return match } @@ -336,18 +376,49 @@ func (h *PyPIHandler) rewriteSimpleHTML(body []byte, filteredVersions map[string }) } - // Match href attributes pointing to packages on the configured download host. - return h.downloadHrefRe.ReplaceAllFunc(body, func(match []byte) []byte { - submatch := h.downloadHrefRe.FindSubmatch(match) - if len(submatch) < minSubmatchParts { - return match + return h.rewriteDownloadHrefs(body) +} + +// rewriteDownloadHrefs points href attributes for files on the configured +// download host at the proxy. It makes the same replacements as the regular +// expression `href="(/packages/[^"]+)"` would: an href +// whose URL starts with the download host's /packages/ and has at least one +// more character before the closing quote. Finding the literal prefix is many +// times faster than running the regular expression over every link. +func (h *PyPIHandler) rewriteDownloadHrefs(body []byte) []byte { + from := []byte(`href="` + h.downloadURL + `/packages/`) + // The download route is /pypi/packages/{path...} and keeps the path + // from the download host, /packages/ included. + to := []byte(`href="` + h.proxyURL + `/pypi/packages/packages/`) + + var out []byte + done := 0 // body[:done] is in out already + for search := 0; ; { + i := bytes.Index(body[search:], from) + if i < 0 { + break } - - origURL := string(submatch[1]) - - newURL := h.proxyURL + "/pypi/packages" + strings.TrimPrefix(origURL, h.downloadURL) - return []byte(fmt.Sprintf(`href="%s"`, newURL)) - }) + start := search + i + rest := start + len(from) + end := bytes.IndexByte(body[rest:], '"') + if end <= 0 { + // No closing quote, or nothing before it: not a match. The + // regular expression would go on looking from the next byte. + search = start + 1 + continue + } + if out == nil { + out = make([]byte, 0, len(body)+len(body)/8) + } + out = append(out, body[done:start]...) + out = append(out, to...) + done = rest + search = rest + end + 1 + } + if out == nil { + return body + } + return append(out, body[done:]...) } func (h *PyPIHandler) rewriteSimpleJSON(body []byte, filteredVersions map[string]bool) ([]byte, error) { @@ -419,9 +490,15 @@ func (h *PyPIHandler) handleVersionJSON(w http.ResponseWriter, r *http.Request) } // proxyAndRewriteJSON fetches JSON metadata and rewrites download URLs. +// Rewrites go through the rewrite cache, as for npm. func (h *PyPIHandler) proxyAndRewriteJSON(w http.ResponseWriter, r *http.Request, upstreamURL, cacheKey string) { - body, _, err := h.proxy.FetchOrCacheMetadata(r.Context(), "pypi", cacheKey, upstreamURL) - if err != nil { + if stored, ok := h.proxy.storedRewrite("pypi", cacheKey, h.proxyURL, cacheKey); ok { + serveRewrittenMetadata(w, r, stored) + return + } + + upstream := h.proxy.fetchMetadataDocument(r.Context(), "pypi", cacheKey, upstreamURL, contentTypeJSON, nil) + if err := upstream.err; err != nil { if errors.Is(err, ErrUpstreamNotFound) { http.Error(w, "not found", http.StatusNotFound) return @@ -434,29 +511,35 @@ func (h *PyPIHandler) proxyAndRewriteJSON(w http.ResponseWriter, r *http.Request return } - rewritten, err := h.rewriteJSONMetadata(body) + rewritten, err := h.proxy.cachedRewrite(r.Context(), "pypi", h.proxyURL, cacheKey, upstream.body, upstream.digest, h.rewriteJSONMetadata) if err != nil { + if r.Context().Err() != nil { + return // the client left while waiting on a shared rewrite + } if h.proxy.mustFilterMetadata("pypi", r.PathValue("name")) { h.proxy.Logger.Error("failed to filter pypi metadata", "url", upstreamURL, "error", err) http.Error(w, "failed to filter package metadata", http.StatusBadGateway) return } h.proxy.Logger.Warn("failed to rewrite metadata, proxying original", "error", err) + markStale(w, upstream.stale) w.Header().Set(headerContentType, "application/json") - _, _ = w.Write(body) + _, _ = w.Write(upstream.body) return } - w.Header().Set(headerContentType, "application/json") - _, _ = w.Write(rewritten) + markStale(w, upstream.stale) + serveRewrittenMetadata(w, r, rewritten) } // rewriteJSONMetadata rewrites download URLs in PyPI JSON metadata. // If cooldown is enabled, versions published too recently are filtered out. -func (h *PyPIHandler) rewriteJSONMetadata(body []byte) ([]byte, error) { +// It also returns what it kept: the releases, and the current version's files +// unless they were dropped. +func (h *PyPIHandler) rewriteJSONMetadata(body []byte) ([]byte, []string, error) { var metadata map[string]any if err := json.Unmarshal(body, &metadata); err != nil { - return nil, err + return nil, nil, err } packageName, _ := extractPyPIName(metadata) @@ -468,7 +551,17 @@ func (h *PyPIHandler) rewriteJSONMetadata(body []byte) ([]byte, error) { h.filterAndRewriteReleases(metadata, packageName, packagePURL) h.filterAndRewriteURLs(metadata, packagePURL) - return json.Marshal(metadata) + var kept []string + if releases, ok := metadata["releases"].(map[string]any); ok { + kept = slices.Collect(maps.Keys(releases)) + } + if urls, ok := metadata["urls"].([]any); ok && len(urls) > 0 { + // Not a valid release key, so it can't be confused with one. + kept = append(kept, "\x00urls") + } + + out, err := json.Marshal(metadata) + return out, kept, err } // filterAndRewriteReleases applies cooldown filtering and URL rewriting to the diff --git a/internal/handler/pypi_test.go b/internal/handler/pypi_test.go index c370719..dd59974 100644 --- a/internal/handler/pypi_test.go +++ b/internal/handler/pypi_test.go @@ -375,7 +375,7 @@ func TestPyPIRewriteJSONMetadataCooldown(t *testing.T) { "urls": [{"url": "https://files.pythonhosted.org/packages/ab/cd/requests-2.31.0.tar.gz", "upload_time_iso_8601": "` + recent + `"}] }` - output, err := h.rewriteJSONMetadata([]byte(input)) + output, _, err := h.rewriteJSONMetadata([]byte(input)) if err != nil { t.Fatalf("rewriteJSONMetadata failed: %v", err) } diff --git a/internal/handler/rewrite_cache.go b/internal/handler/rewrite_cache.go index 991443a..8daf6aa 100644 --- a/internal/handler/rewrite_cache.go +++ b/internal/handler/rewrite_cache.go @@ -374,27 +374,66 @@ func (p *Proxy) cooldownEnabled() bool { // again. When it reports false the caller takes the usual fetch and rewrite // path. func (p *Proxy) storedRewrite(ecosystem, cacheKey, proxyURL, name string) (rewrittenMetadata, bool) { + m, _, ok := p.storedRewriteVariant(ecosystem, cacheKey, proxyURL, name, nil) + return m, ok +} + +// storedRewriteVariant is storedRewrite for a rewrite whose output depends on +// more than the stored document. variant is given the stored content type +// and returns the rest of what the rewrite depends on, as passed to +// variantDigest; nil means nothing else. It also returns the stored content +// type, as a fetch of the document would. +func (p *Proxy) storedRewriteVariant(ecosystem, cacheKey, proxyURL, name string, variant func(contentType string) []string) (rewrittenMetadata, string, bool) { if p.rewrites == nil { - return rewrittenMetadata{}, false + return rewrittenMetadata{}, "", false } if !p.CacheMetadata || p.DB == nil || p.MetadataTTL <= 0 { - return rewrittenMetadata{}, false + return rewrittenMetadata{}, "", false } entry, err := p.DB.GetMetadataCache(ecosystem, cacheKey) if err != nil || entry == nil || !entry.ContentDigest.Valid || entry.ContentEncoding.String != "" { - return rewrittenMetadata{}, false + return rewrittenMetadata{}, "", false } if !entry.FetchedAt.Valid || time.Since(entry.FetchedAt.Time) >= p.MetadataTTL { - return rewrittenMetadata{}, false + return rewrittenMetadata{}, "", false } - m, ok := p.rewrites.get(rewriteCacheKeyForDigest(ecosystem, proxyURL, name, entry.ContentDigest.String)) + contentType := contentTypeJSON + if entry.ContentType.Valid { + contentType = entry.ContentType.String + } + digest := entry.ContentDigest.String + if variant != nil { + digest = variantDigest(digest, nil, variant(contentType)...) + } + m, ok := p.rewrites.get(rewriteCacheKeyForDigest(ecosystem, proxyURL, name, digest)) if !ok { - return rewrittenMetadata{}, false + return rewrittenMetadata{}, "", false } m.lastModified = p.metadataDate(ecosystem, proxyURL, name, m.etag) metrics.RecordMetadataCacheHit(ecosystem, len(m.body)) p.recordMetadataRead(entry) - return m, true + return m, contentType, true +} + +// variantDigest returns a digest for a rewrite of in that also depends on +// parts, such as the content type or the versions a filter removes, so that +// rewrites differing in them are cached apart. digest is in's digest as the +// metadata cache records it, or empty to hash in. Passed to cachedRewrite in +// place of the document's digest, it keeps the metadata date keyed by the +// package alone. +func variantDigest(digest string, in []byte, parts ...string) string { + if digest == "" { + sum := sha256.Sum256(in) + digest = "sha256:" + hex.EncodeToString(sum[:]) + } + h := sha256.New() + var length []byte + for _, part := range parts { + length = binary.AppendUvarint(length[:0], uint64(len(part))) + _, _ = h.Write(length) + _, _ = io.WriteString(h, part) + } + return digest + "+" + hex.EncodeToString(h.Sum(nil)) } // serveRewrittenMetadata writes rewritten metadata with its validators, or @@ -413,7 +452,7 @@ func serveRewrittenMetadataAs(w http.ResponseWriter, r *http.Request, m rewritte if compressed { body, etag = m.gzipped, gzipETag(m.etag) } - w.Header().Set("Vary", headerAcceptEncoding) + ensureVary(w.Header(), headerAcceptEncoding) w.Header().Set(headerETag, etag) if !m.lastModified.IsZero() { w.Header().Set(headerLastModified, m.lastModified.UTC().Format(http.TimeFormat)) @@ -430,6 +469,18 @@ func serveRewrittenMetadataAs(w http.ResponseWriter, r *http.Request, m rewritte _, _ = w.Write(body) } +// ensureVary adds field to the Vary header unless it is listed already. +func ensureVary(header http.Header, field string) { + for _, value := range header.Values("Vary") { + for listed := range strings.SplitSeq(value, ",") { + if strings.EqualFold(strings.TrimSpace(listed), field) { + return + } + } + } + header.Add("Vary", field) +} + // gzipETag returns the ETag of the gzip representation of the document // whose ETag is etag. func gzipETag(etag string) string { From 3935ccd6f762cdd5de2b5351b6424e345bf6ae8b Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 25 Oct 2026 17:34:08 +0100 Subject: [PATCH 436/633] Share upstream rate-limit holds across instances Holds placed by the rate limit gate are written to a new upstream_holds table, and each instance reads that table at most every two seconds and refuses requests to a host any instance holds. Before, every replica had to get its own 429 before it stopped sending. Instances waiting on a fetch lease reread the holds when the fetch fails, so when the holder was rate limited they answer 503 with its Retry-After instead of each sending one more request to the limited host. --- README.md | 2 +- docs/configuration.md | 4 +- internal/database/database_test.go | 2 +- internal/database/schema.go | 28 +++ internal/database/upstream_holds.go | 58 ++++++ internal/database/upstream_holds_test.go | 48 +++++ internal/handler/handler.go | 21 ++- internal/handler/shared_holds_test.go | 170 +++++++++++++++++ internal/handler/upstream_held.go | 26 +++ internal/httpclient/shared_holds.go | 213 ++++++++++++++++++++++ internal/httpclient/shared_holds_test.go | 125 +++++++++++++ internal/server/breakers_test.go | 3 +- internal/server/registry_policies_test.go | 3 +- internal/server/server.go | 12 +- 14 files changed, 700 insertions(+), 15 deletions(-) create mode 100644 internal/database/upstream_holds.go create mode 100644 internal/database/upstream_holds_test.go create mode 100644 internal/handler/shared_holds_test.go create mode 100644 internal/httpclient/shared_holds.go create mode 100644 internal/httpclient/shared_holds_test.go diff --git a/README.md b/README.md index f977234..e8ecec0 100644 --- a/README.md +++ b/README.md @@ -1630,7 +1630,7 @@ Failing checks include an `"error"` field. Storage failures also include a `"ste `circuit_breakers` reports the state of each upstream's artifact-fetch circuit breaker (`"open"` or `"closed"`), keyed by upstream host — or by the `hostless-url-` placeholder described under [Monitoring](#monitoring) where the fetch URL has no host to read. The key is omitted until the proxy has fetched an artifact from at least one upstream, and a host appears only once a breaker has been created for it. Breakers trip after repeated upstream failures and retry the upstream after an exponential backoff. While one is open, artifact downloads for that host return HTTP 502 on a cache miss without contacting the upstream; already-cached artifacts are still served from storage, since the cache is checked before the fetcher. A breaker is reported as `"open"` throughout its backoff, including the half-open window in which it admits one probe request to test recovery. Breaker state is per process and in memory, so a restart clears it, but a restart is not needed for recovery: the backoff keeps retrying for as long as the breaker is open, so it closes on its own once the upstream serves again. -`upstream_holds` lists upstream hosts that answered the proxy with HTTP 429, or 503 with a `Retry-After`, and the time until which the proxy sends them nothing. Every request to a held host, metadata and artifacts alike, fails locally for the duration the upstream asked for (capped at one hour), or for one minute doubling up to fifteen when the upstream gave no `Retry-After`. Cached metadata is served stale during a hold, and anything uncached returns HTTP 503 with the remaining `Retry-After` so a client can pace itself too. Metadata requests do not go through the circuit breaker, so without the hold a rate-limited upstream would keep receiving one request per cache miss, which is exactly the traffic it is asking the proxy to stop sending. The first non-limited response after a hold ends clears it, and `proxy_upstream_rate_limit_holds_total` counts the holds placed. The key is omitted while no host is held. +`upstream_holds` lists upstream hosts that answered the proxy with HTTP 429, or 503 with a `Retry-After`, and the time until which the proxy sends them nothing. Every request to a held host, metadata and artifacts alike, fails locally for the duration the upstream asked for (capped at one hour), or for one minute doubling up to fifteen when the upstream gave no `Retry-After`. Cached metadata is served stale during a hold, and anything uncached returns HTTP 503 with the remaining `Retry-After` so a client can pace itself too. Metadata requests do not go through the circuit breaker, so without the hold a rate-limited upstream would keep receiving one request per cache miss, which is exactly the traffic it is asking the proxy to stop sending. The first non-limited response after a hold ends clears it, and `proxy_upstream_rate_limit_holds_total` counts the holds placed. Holds are recorded in the database, so instances sharing one honour each other's holds and list them here too. The key is omitted while no host is held. An open breaker does **not** set `status` to `"error"` or change the HTTP status code: it reports a specific upstream refusing to serve, not this proxy being unfit to receive traffic, and failing the readiness probe over one unhealthy upstream would pull the pod out of rotation for every other ecosystem too. Use `proxy_circuit_breaker_state` for alerting on it. diff --git a/docs/configuration.md b/docs/configuration.md index 5a877e3..959eb45 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -207,7 +207,9 @@ Instances coordinate through PostgreSQL advisory locks: - Eviction, reclaim, retention and the Gradle build cache sweep run on one instance at a time. An instance that finds a job running elsewhere skips that pass. - A [scheduled mirror](#scheduled-refresh) runs on whichever instance reaches it first in each interval. -Cache misses are fetched from upstream once across instances. The instance that fetches holds a lease on the miss in the `fetch_leases` table and renews it while it downloads. Another instance missing the same file waits until it is stored and serves the stored copy; `proxy_fetch_lease_waits_total` counts those waits. If the fetch fails, the instances waiting on it stop waiting together: each serves its stale metadata copy if it has one, and otherwise fetches itself, so a failing upstream costs every instance one attempt at the same time rather than one after another. A lease of an instance that stopped expires after 30 seconds. Metadata is shared this way only with [`cache_metadata`](#metadata-caching) and a `metadata_ttl` above zero, since without a stored copy there is nothing to wait for. A cache hit costs no extra query; a miss costs one. +Cache misses are fetched from upstream once across instances. The instance that fetches holds a lease on the miss in the `fetch_leases` table and renews it while it downloads. Another instance missing the same file waits until it is stored and serves the stored copy; `proxy_fetch_lease_waits_total` counts those waits. If the fetch fails, the instances waiting on it stop waiting together: each serves its stale metadata copy if it has one, and otherwise fetches itself, so a failing upstream costs every instance one attempt at the same time rather than one after another. When the fetch failed because the upstream rate limited it, the waiters without a stale copy answer 503 with the upstream's `Retry-After` instead of fetching. A lease of an instance that stopped expires after 30 seconds. Metadata is shared this way only with [`cache_metadata`](#metadata-caching) and a `metadata_ttl` above zero, since without a stored copy there is nothing to wait for. A cache hit costs no extra query; a miss costs one. + +Rate-limit holds (see `upstream_holds` under [Health Check](../README.md#health-check)) are shared through the `upstream_holds` table. When one instance gets a 429, or a 503 with `Retry-After`, it records the host and the end of the hold, and the other instances stop sending to that host too instead of each waiting for its own 429. Each instance rereads the table at most every two seconds, so another instance can still send for up to two seconds after a hold starts. A hold is also recorded with SQLite, where it then outlasts a restart. Each instance needs its own [`git.cache_dir`](#git-repositories). diff --git a/internal/database/database_test.go b/internal/database/database_test.go index 8b29ced..f8956a6 100644 --- a/internal/database/database_test.go +++ b/internal/database/database_test.go @@ -613,7 +613,7 @@ func createTestPostgresDB(t *testing.T) *DB { // Drop and recreate every table CreateSchema creates for clean test state; // leftover migration records make the next CreateSchema fail on the // migrations primary key. - tables := []string{"artifacts", "pending_deletes", "versions", "packages", "vulnerabilities", "metadata_cache", "metadata_dates", "job_runs", "fetch_leases", "migrations", "schema_info"} + tables := []string{"artifacts", "pending_deletes", "versions", "packages", "vulnerabilities", "metadata_cache", "metadata_dates", "job_runs", "fetch_leases", "upstream_holds", "migrations", "schema_info"} for _, table := range tables { _, _ = db.Exec("DROP TABLE IF EXISTS " + table + " CASCADE") } diff --git a/internal/database/schema.go b/internal/database/schema.go index 74cc00f..b2fef63 100644 --- a/internal/database/schema.go +++ b/internal/database/schema.go @@ -147,6 +147,11 @@ CREATE TABLE IF NOT EXISTS metadata_dates ( PRIMARY KEY (ecosystem, proxy_url, name) ); +CREATE TABLE IF NOT EXISTS upstream_holds ( + host TEXT NOT NULL PRIMARY KEY, + until_ms INTEGER NOT NULL +); + CREATE TABLE IF NOT EXISTS migrations ( name TEXT NOT NULL PRIMARY KEY, applied_at DATETIME NOT NULL @@ -283,6 +288,11 @@ CREATE TABLE IF NOT EXISTS metadata_dates ( PRIMARY KEY (ecosystem, proxy_url, name) ); +CREATE TABLE IF NOT EXISTS upstream_holds ( + host TEXT NOT NULL PRIMARY KEY, + until_ms BIGINT NOT NULL +); + CREATE TABLE IF NOT EXISTS migrations ( name TEXT NOT NULL PRIMARY KEY, applied_at TIMESTAMP NOT NULL @@ -447,6 +457,7 @@ var migrations = []migration{ {"014_add_metadata_last_accessed", migrateAddMetadataLastAccessed}, {"015_add_job_runs", migrateAddJobRuns}, {"016_add_fetch_leases", migrateAddFetchLeases}, + {"018_add_upstream_holds", migrateAddUpstreamHolds}, } // isTableNotFound returns true if the error indicates a missing table. @@ -782,6 +793,23 @@ func migrateAddFetchLeases(db *DB) error { return nil } +// migrateAddUpstreamHolds creates the table through which instances sharing +// the database honour each other's upstream rate-limit holds. +func migrateAddUpstreamHolds(db *DB) error { + millis := "INTEGER" + if db.dialect == DialectPostgres { + millis = "BIGINT" + } + query := fmt.Sprintf(`CREATE TABLE IF NOT EXISTS upstream_holds ( + host TEXT NOT NULL PRIMARY KEY, + until_ms %s NOT NULL + )`, millis) + if _, err := db.Exec(query); err != nil { + return fmt.Errorf("creating upstream_holds table: %w", err) + } + return nil +} + // migrateAddPendingDeletesQueuedAtIndex indexes the column reclaim orders // the queue by, which grows large after a retention sweep. func migrateAddPendingDeletesQueuedAtIndex(db *DB) error { diff --git a/internal/database/upstream_holds.go b/internal/database/upstream_holds.go new file mode 100644 index 0000000..6f842ca --- /dev/null +++ b/internal/database/upstream_holds.go @@ -0,0 +1,58 @@ +package database + +import ( + "fmt" + "time" +) + +// Upstream holds let instances sharing a database honour a rate-limit hold +// any one of them placed, so each does not need its own 429 before it stops +// sending. Times are the instances' own clocks in Unix milliseconds; a few +// seconds of skew does not matter next to holds of a minute or more. + +type upstreamHoldRow struct { + Host string `db:"host"` + UntilMS int64 `db:"until_ms"` +} + +// PutUpstreamHold records that requests to host are held until until. A hold +// already recorded for longer is kept. +func (db *DB) PutUpstreamHold(host string, until time.Time) error { + _, err := db.Exec(db.Rebind(` + INSERT INTO upstream_holds (host, until_ms) VALUES (?, ?) + ON CONFLICT (host) DO UPDATE SET until_ms = CASE + WHEN excluded.until_ms > upstream_holds.until_ms THEN excluded.until_ms + ELSE upstream_holds.until_ms END + `), host, until.UnixMilli()) + if err != nil { + return fmt.Errorf("recording upstream hold: %w", err) + } + return nil +} + +// UpstreamHolds returns the holds that have not ended by now, and deletes +// those that have. +func (db *DB) UpstreamHolds(now time.Time) (map[string]time.Time, error) { + var rows []upstreamHoldRow + if err := db.Select(&rows, `SELECT host, until_ms FROM upstream_holds`); err != nil { + return nil, fmt.Errorf("reading upstream holds: %w", err) + } + holds := make(map[string]time.Time, len(rows)) + expired := false + for _, row := range rows { + until := time.UnixMilli(row.UntilMS) + if now.Before(until) { + holds[row.Host] = until + } else { + expired = true + } + } + // Only write when there is something to delete, so the frequent reads + // stay reads. + if expired { + if _, err := db.Exec(db.Rebind(`DELETE FROM upstream_holds WHERE until_ms <= ?`), now.UnixMilli()); err != nil { + return holds, fmt.Errorf("deleting expired upstream holds: %w", err) + } + } + return holds, nil +} diff --git a/internal/database/upstream_holds_test.go b/internal/database/upstream_holds_test.go new file mode 100644 index 0000000..5d6617c --- /dev/null +++ b/internal/database/upstream_holds_test.go @@ -0,0 +1,48 @@ +package database + +import ( + "testing" + "time" +) + +func TestUpstreamHolds(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + now := time.UnixMilli(time.Now().UnixMilli()) + if err := db.PutUpstreamHold("registry.npmjs.org", now.Add(time.Minute)); err != nil { + t.Fatal(err) + } + // A shorter hold reported later keeps the longer one. + if err := db.PutUpstreamHold("registry.npmjs.org", now.Add(30*time.Second)); err != nil { + t.Fatal(err) + } + if err := db.PutUpstreamHold("pypi.org", now.Add(10*time.Second)); err != nil { + t.Fatal(err) + } + holds, err := db.UpstreamHolds(now) + if err != nil { + t.Fatal(err) + } + if len(holds) != 2 || !holds["registry.npmjs.org"].Equal(now.Add(time.Minute)) { + t.Fatalf("holds = %v", holds) + } + + // A longer one replaces it. + if err := db.PutUpstreamHold("registry.npmjs.org", now.Add(time.Hour)); err != nil { + t.Fatal(err) + } + holds, err = db.UpstreamHolds(now.Add(20 * time.Second)) + if err != nil { + t.Fatal(err) + } + if len(holds) != 1 || !holds["registry.npmjs.org"].Equal(now.Add(time.Hour)) { + t.Fatalf("holds after pypi.org ended = %v", holds) + } + var rows int + if err := db.Get(&rows, "SELECT COUNT(*) FROM upstream_holds"); err != nil { + t.Fatal(err) + } + if rows != 1 { + t.Errorf("rows = %d, want the ended hold deleted", rows) + } + }) +} diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 2e0d688..513a531 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -26,7 +26,6 @@ import ( "github.com/git-pkgs/proxy/internal/scanner" "github.com/git-pkgs/proxy/internal/storage" "github.com/git-pkgs/purl" - registryclient "github.com/git-pkgs/registries/client" "github.com/git-pkgs/registries/fetch" "github.com/opencontainers/go-digest" ) @@ -195,7 +194,7 @@ type Proxy struct { // RateLimits is the gate under the upstream clients. Artifact fetches // check it before calling Fetcher, so held hosts never reach its breaker. - RateLimits *registryclient.RateLimitGate + RateLimits UpstreamHolds // HostedNPMScopes and HostedComposerVendors name namespaces served from // pushed packages alone, without asking the upstream. @@ -584,7 +583,7 @@ func (p *Proxy) fetchAndCache(ctx context.Context, ecosystem, name, version, fil } } - if err := p.RateLimits.Held(info.URL); err != nil { + if err := p.upstreamHeld(info.URL); err != nil { return artifacts.Artifact{}, "", err } p.Logger.Info("fetching from upstream", @@ -874,7 +873,12 @@ func (p *Proxy) runSharedFetch(ctx context.Context, key string, f *inflightFetch } // With leaseFailed, release is a no-op and this instance fetches on its // own, so a failing upstream costs each instance one attempt in - // parallel. + // parallel. A rate-limited holder has published its hold by now, and + // rereading it makes commit refuse the fetch with the holder's + // Retry-After instead of sending one more request to the limited host. + if outcome == leaseFailed { + p.reloadUpstreamHolds() + } failed := true defer func() { release(failed) }() @@ -1441,7 +1445,10 @@ func (p *Proxy) coalescedMetadataMiss(ctx context.Context, ecosystem, cacheKey, return *hit case leaseFailed: // Serve the stale copy at once, as the holder did, rather - // than trying the failing upstream again. Without one, fetch. + // than trying the failing upstream again. Without one, fetch, + // unless the holder was rate limited: its hold, read here, + // refuses the fetch with its Retry-After. + p.reloadUpstreamHolds() if entry != nil { if res := p.staleMetadata(fetchCtx, ecosystem, cacheKey, validate, entry, errFetchFailedElsewhere); res.err == nil { return res @@ -2202,7 +2209,7 @@ func (p *Proxy) discardStaleArtifact(pkgPURL, versionPURL, filename, upstreamHas } func (p *Proxy) fetchAndCacheFromURL(ctx context.Context, ecosystem, name, version, filename, pkgPURL, versionPURL, downloadURL string, headers http.Header, upstreamHash, declared string) (artifacts.Artifact, string, error) { - if err := p.RateLimits.Held(downloadURL); err != nil { + if err := p.upstreamHeld(downloadURL); err != nil { return artifacts.Artifact{}, "", err } p.Logger.Info("fetching from upstream", @@ -2269,7 +2276,7 @@ func (p *Proxy) fetchUpstream(ctx context.Context, downloadURL string, headers h // after the bytes have been sent, and an unterminated chunked response is the // only way left to make the client reject them (see serveArtifact). func (p *Proxy) streamFromUpstream(ctx context.Context, ecosystem, name, version, filename, versionPURL, upstreamURL, upstreamHash, declared string, fetchArtifact func(context.Context) (*fetch.Artifact, error)) (*CacheResult, error) { - if err := p.RateLimits.Held(upstreamURL); err != nil { + if err := p.upstreamHeld(upstreamURL); err != nil { return nil, err } p.Logger.Info("streaming from upstream", diff --git a/internal/handler/shared_holds_test.go b/internal/handler/shared_holds_test.go new file mode 100644 index 0000000..5536c00 --- /dev/null +++ b/internal/handler/shared_holds_test.go @@ -0,0 +1,170 @@ +package handler + +import ( + "context" + "errors" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "path/filepath" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/git-pkgs/proxy/internal/database" + upstreamhttp "github.com/git-pkgs/proxy/internal/httpclient" + "github.com/git-pkgs/proxy/internal/metrics" + registryclient "github.com/git-pkgs/registries/client" + "github.com/git-pkgs/registries/fetch" + "github.com/prometheus/client_golang/prometheus/testutil" +) + +// limitedUpstream answers every request with 429 and a Retry-After of ten +// minutes. Requests wait for release, which is open unless a test closes it +// later. +type limitedUpstream struct { + server *httptest.Server + calls atomic.Int32 + entered chan struct{} + release chan struct{} + released sync.Once +} + +func newLimitedUpstream(t *testing.T, blocked bool) *limitedUpstream { + u := &limitedUpstream{entered: make(chan struct{}, 8), release: make(chan struct{})} + if !blocked { + u.open() + } + t.Cleanup(u.open) + u.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + u.calls.Add(1) + u.entered <- struct{}{} + <-u.release + w.Header().Set("Retry-After", "600") + w.WriteHeader(http.StatusTooManyRequests) + })) + t.Cleanup(u.server.Close) + return u +} + +func (u *limitedUpstream) open() { u.released.Do(func() { close(u.release) }) } + +func (u *limitedUpstream) artifactURL() string { return u.server.URL + "/left-pad-1.3.0.tgz" } + +// useSharedGate routes p's upstream requests through its own rate limit gate, +// sharing holds through db as a replica does. +func useSharedGate(p *Proxy, db *database.DB, u *limitedUpstream) *upstreamhttp.SharedRateLimitGate { + gate := upstreamhttp.NewSharedRateLimitGate(registryclient.NewRateLimitGate(u.server.Client().Transport), db, nil) + client := &http.Client{Transport: gate} + p.HTTPClient = client + p.Fetcher = fetch.NewFetcher(fetch.WithHTTPClient(client), fetch.WithBaseDelay(time.Millisecond)) + p.RateLimits = gate + return gate +} + +func fetchLimited(ctx context.Context, p *Proxy, u *limitedUpstream) error { + res, err := p.GetOrFetchArtifactFromURL(ctx, "npm", "left-pad", "1.3.0", "left-pad-1.3.0.tgz", u.artifactURL()) + if err == nil { + _ = res.Reader.Close() + } + return err +} + +// requireHeld checks that err is answered like any held upstream request, +// with 503 and a Retry-After. +func requireHeld(t *testing.T, name string, err error) { + t.Helper() + w := httptest.NewRecorder() + if !writeUpstreamHeld(w, err) { + t.Fatalf("%s: err = %v, want a held upstream", name, err) + } + if w.Code != http.StatusServiceUnavailable || w.Header().Get("Retry-After") == "" { + t.Fatalf("%s: status %d, Retry-After %q; want 503 with Retry-After", name, w.Code, w.Header().Get("Retry-After")) + } +} + +// TestSharedHoldStopsOtherInstance checks that once one instance is rate +// limited, another sharing its database refuses requests to that host +// without sending them, until the hold ends. +func TestSharedHoldStopsOtherInstance(t *testing.T) { + path := filepath.Join(t.TempDir(), "shared.db") + dbA, err := database.Create(path) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = dbA.Close() }) + dbB, err := database.Open(path) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = dbB.Close() }) + logger := slog.New(slog.NewTextHandler(io.Discard, nil)) + store := newMockStorage() + a := NewProxy(dbA, store, &mockFetcher{}, fetch.NewResolver(), logger) + b := NewProxy(dbB, store, &mockFetcher{}, fetch.NewResolver(), logger) + + u := newLimitedUpstream(t, false) + useSharedGate(a, dbA, u) + gateB := useSharedGate(b, dbB, u) + gateB.Refresh = 0 + + if err := fetchLimited(context.Background(), a, u); err == nil { + t.Fatal("instance a: want the rate limit error") + } + if got := u.calls.Load(); got != 1 { + t.Fatalf("upstream requests after a = %d, want 1", got) + } + + requireHeld(t, "instance b artifact", fetchLimited(context.Background(), b, u)) + _, _, err = b.FetchOrCacheMetadata(context.Background(), "npm", "left-pad", u.server.URL+"/left-pad", "application/json") + if !errors.Is(err, registryclient.ErrUpstreamHeld) { + t.Errorf("instance b metadata: err = %v, want a held upstream", err) + } + if got := u.calls.Load(); got != 1 { + t.Errorf("upstream requests after b = %d, want 1", got) + } + + // The hold ends when the upstream said, and is then deleted. + holds, err := dbB.UpstreamHolds(time.Now().Add(11 * time.Minute)) + if err != nil || len(holds) != 0 { + t.Fatalf("holds after expiry = %v, %v; want none", holds, err) + } + if err := b.upstreamHeld(u.artifactURL()); err != nil { + t.Errorf("instance b after expiry: %v, want no hold", err) + } +} + +// TestFetchLeaseRateLimitedHolderStopsWaiters checks that instances waiting +// on a fetch that was rate limited answer with the holder's Retry-After +// instead of each sending one more request to the limited host. +func TestFetchLeaseRateLimitedHolderStopsWaiters(t *testing.T) { + a, b, db := sharedInstances(t) + u := newLimitedUpstream(t, true) + useSharedGate(a, db, u) + gateB := useSharedGate(b, db, u) + // b has read the shared holds just before, and would not read them + // again for an hour on its own. + gateB.Refresh = time.Hour + if err := b.upstreamHeld(u.artifactURL()); err != nil { + t.Fatal(err) + } + + waits := testutil.ToFloat64(metrics.FetchLeaseWaits.WithLabelValues(metrics.CacheKindArtifact)) + resA := make(chan error, 1) + go func() { resA <- fetchLimited(context.Background(), a, u) }() + <-u.entered + resB := make(chan error, 1) + go func() { resB <- fetchLimited(context.Background(), b, u) }() + waitForLeaseWaits(t, metrics.CacheKindArtifact, waits) + u.open() + + if err := <-resA; err == nil { + t.Fatal("instance a: want the rate limit error") + } + requireHeld(t, "instance b", <-resB) + if got := u.calls.Load(); got != 1 { + t.Errorf("upstream requests = %d, want 1", got) + } +} diff --git a/internal/handler/upstream_held.go b/internal/handler/upstream_held.go index 1097e7e..57ac84d 100644 --- a/internal/handler/upstream_held.go +++ b/internal/handler/upstream_held.go @@ -25,3 +25,29 @@ func writeUpstreamHeld(w http.ResponseWriter, err error) bool { http.Error(w, held.Error(), http.StatusServiceUnavailable) return true } + +// UpstreamHolds reports whether requests to a URL's host are held by rate +// limiting. *registryclient.RateLimitGate and the shared gate in +// internal/httpclient implement it. +type UpstreamHolds interface { + // Held returns a *registryclient.HeldError while rawURL's host is held. + Held(rawURL string) error +} + +// upstreamHeld returns the hold on rawURL's host, if any. +func (p *Proxy) upstreamHeld(rawURL string) error { + if p.RateLimits == nil { + return nil + } + return p.RateLimits.Held(rawURL) +} + +// reloadUpstreamHolds rereads holds shared with other instances, when the +// gate shares them. It is called after another instance's fetch failed, +// since a 429 there will have placed a hold this instance may not have read +// yet. +func (p *Proxy) reloadUpstreamHolds() { + if r, ok := p.RateLimits.(interface{ Reload() }); ok { + r.Reload() + } +} diff --git a/internal/httpclient/shared_holds.go b/internal/httpclient/shared_holds.go new file mode 100644 index 0000000..175931b --- /dev/null +++ b/internal/httpclient/shared_holds.go @@ -0,0 +1,213 @@ +package httpclient + +import ( + "log/slog" + "net/http" + "net/url" + "strings" + "sync" + "time" + + registryclient "github.com/git-pkgs/registries/client" +) + +// DefaultHoldRefresh is how long a SharedRateLimitGate trusts its copy of +// the shared holds before reading them again. It bounds both the database +// reads, at one per interval per instance, and how long another instance +// may keep sending after a hold is placed. +const DefaultHoldRefresh = 2 * time.Second + +// HoldStore keeps rate-limit holds where every instance sharing it sees them. +type HoldStore interface { + PutUpstreamHold(host string, until time.Time) error + UpstreamHolds(now time.Time) (map[string]time.Time, error) +} + +// SharedRateLimitGate adds holds shared with other instances to a +// RateLimitGate. Holds the gate places are published to the store, and +// requests to a host another instance holds are refused like the gate's own, +// with a HeldError. Without a store it behaves as the gate alone. +type SharedRateLimitGate struct { + gate *registryclient.RateLimitGate + store HoldStore + logger *slog.Logger + now func() time.Time + + // Refresh is how long the shared holds read from the store are used + // before reading them again. Set it before use. + Refresh time.Duration + + mu sync.Mutex + shared map[string]time.Time + // published holds the holds this instance wrote, so a read that started + // before a write does not drop it. + published map[string]time.Time + loaded time.Time + loading bool +} + +// NewSharedRateLimitGate wraps gate, publishing its holds to store and +// honouring those in store. It takes over gate.OnHold, calling the callback +// set before. A nil store shares nothing. +func NewSharedRateLimitGate(gate *registryclient.RateLimitGate, store HoldStore, logger *slog.Logger) *SharedRateLimitGate { + s := &SharedRateLimitGate{ + gate: gate, + store: store, + logger: logger, + now: time.Now, + Refresh: DefaultHoldRefresh, + shared: make(map[string]time.Time), + published: make(map[string]time.Time), + } + if store != nil { + onHold := gate.OnHold + gate.OnHold = func(event registryclient.HoldEvent) { + s.publish(event.Host, event.Until) + if onHold != nil { + onHold(event) + } + } + } + return s +} + +// RoundTrip implements http.RoundTripper. +func (s *SharedRateLimitGate) RoundTrip(req *http.Request) (*http.Response, error) { + if err := s.sharedHeld(hostKey(req.URL)); err != nil { + return nil, err + } + return s.gate.RoundTrip(req) +} + +// Held returns a *registryclient.HeldError when requests to rawURL's host are +// held by this instance or another, and nil otherwise. It is safe to call on +// a nil gate. +func (s *SharedRateLimitGate) Held(rawURL string) error { + if s == nil { + return nil + } + if err := s.gate.Held(rawURL); err != nil { + return err + } + u, err := url.Parse(rawURL) + if err != nil { + return nil + } + return s.sharedHeld(hostKey(u)) +} + +// Holds returns the hosts held by this instance or another, and when each +// hold ends. It is safe to call on a nil gate. +func (s *SharedRateLimitGate) Holds() map[string]time.Time { + if s == nil { + return nil + } + holds := s.gate.Holds() + if s.store == nil { + return holds + } + s.load(false) + now := s.now() + s.mu.Lock() + defer s.mu.Unlock() + for host, until := range s.shared { + if now.Before(until) && until.After(holds[host]) { + if holds == nil { + holds = make(map[string]time.Time) + } + holds[host] = until + } + } + return holds +} + +// Reload reads the shared holds now rather than when the copy runs out. A +// caller that knows another instance has just failed, say on a 429, uses it +// to see that instance's hold at once. +func (s *SharedRateLimitGate) Reload() { + if s == nil || s.store == nil { + return + } + s.load(true) +} + +func (s *SharedRateLimitGate) sharedHeld(host string) error { + if s.store == nil { + return nil + } + s.load(false) + now := s.now() + s.mu.Lock() + until, ok := s.shared[host] + s.mu.Unlock() + if !ok || !now.Before(until) { + return nil + } + return ®istryclient.HeldError{Host: host, Until: until, RetryAfter: until.Sub(now)} +} + +// load refreshes the copy of the shared holds once it is older than Refresh, +// or at once when force is set. One caller reads the store while concurrent +// callers go on with the copy they have; a forced read does not wait for +// one already running, which may have started before the hold it is after. +func (s *SharedRateLimitGate) load(force bool) { + now := s.now() + s.mu.Lock() + if !force && (s.loading || now.Sub(s.loaded) < s.Refresh) { + s.mu.Unlock() + return + } + s.loading = true + s.mu.Unlock() + + holds, err := s.store.UpstreamHolds(now) + + s.mu.Lock() + defer s.mu.Unlock() + s.loading = false + // A failed read keeps the last copy and waits a full interval before + // trying again, so a database outage costs no more queries than usual. + s.loaded = now + if err != nil { + if s.logger != nil { + s.logger.Warn("reading shared upstream holds failed", "error", err) + } + if holds == nil { + return + } + } + for host, until := range s.published { + if !now.Before(until) { + delete(s.published, host) + } else if until.After(holds[host]) { + holds[host] = until + } + } + s.shared = holds +} + +// publish records a hold this instance placed, locally and in the store. The +// responses already in flight when a hold starts each report it again; only a +// later end is written. +func (s *SharedRateLimitGate) publish(host string, until time.Time) { + s.mu.Lock() + if !until.After(s.published[host]) { + s.mu.Unlock() + return + } + s.published[host] = until + if until.After(s.shared[host]) { + s.shared[host] = until + } + s.mu.Unlock() + if err := s.store.PutUpstreamHold(host, until); err != nil && s.logger != nil { + s.logger.Warn("publishing upstream hold failed", "host", host, "error", err) + } +} + +func hostKey(u *url.URL) string { + if u == nil { + return "" + } + return strings.ToLower(u.Host) +} diff --git a/internal/httpclient/shared_holds_test.go b/internal/httpclient/shared_holds_test.go new file mode 100644 index 0000000..31b9d79 --- /dev/null +++ b/internal/httpclient/shared_holds_test.go @@ -0,0 +1,125 @@ +package httpclient + +import ( + "errors" + "net/http" + "net/http/httptest" + "sync" + "sync/atomic" + "testing" + "time" + + registryclient "github.com/git-pkgs/registries/client" +) + +// memoryHoldStore is a HoldStore shared by gates in one test, counting reads. +type memoryHoldStore struct { + mu sync.Mutex + holds map[string]time.Time + reads atomic.Int32 +} + +func (m *memoryHoldStore) PutUpstreamHold(host string, until time.Time) error { + m.mu.Lock() + defer m.mu.Unlock() + if until.After(m.holds[host]) { + m.holds[host] = until + } + return nil +} + +func (m *memoryHoldStore) UpstreamHolds(now time.Time) (map[string]time.Time, error) { + m.reads.Add(1) + m.mu.Lock() + defer m.mu.Unlock() + out := make(map[string]time.Time) + for host, until := range m.holds { + if now.Before(until) { + out[host] = until + } + } + return out, nil +} + +func TestSharedRateLimitGateSharesHolds(t *testing.T) { + var calls atomic.Int32 + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + calls.Add(1) + w.Header().Set("Retry-After", "120") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer upstream.Close() + + store := &memoryHoldStore{holds: make(map[string]time.Time)} + a := NewSharedRateLimitGate(registryclient.NewRateLimitGate(upstream.Client().Transport), store, nil) + b := NewSharedRateLimitGate(registryclient.NewRateLimitGate(upstream.Client().Transport), store, nil) + clock := time.Now() + b.now = func() time.Time { return clock } + + // b reads the store before a is rate limited, and keeps that copy for + // Refresh. + if err := b.Held(upstream.URL); err != nil { + t.Fatalf("b held before any 429: %v", err) + } + resp, err := a.RoundTrip(httptest.NewRequest(http.MethodGet, upstream.URL, nil)) + if err != nil || resp.StatusCode != http.StatusTooManyRequests { + t.Fatalf("a: %v, %v; want the 429", resp, err) + } + _ = resp.Body.Close() + if err := b.Held(upstream.URL); err != nil { + t.Errorf("b refreshed before Refresh ran out: %v", err) + } + + clock = clock.Add(DefaultHoldRefresh) + _, err = b.RoundTrip(httptest.NewRequest(http.MethodGet, upstream.URL, nil)) + var held *registryclient.HeldError + if !errors.As(err, &held) || held.RetryAfter <= 100*time.Second { + t.Fatalf("b after refresh: %v, want a hold of about two minutes", err) + } + if calls.Load() != 1 { + t.Errorf("upstream requests = %d, want 1", calls.Load()) + } + if holds := b.Holds(); len(holds) != 1 { + t.Errorf("b.Holds() = %v, want a's hold", holds) + } + // One read by a before its request, two by b: the first and the one + // after Refresh. + if n := store.reads.Load(); n != 3 { + t.Errorf("store reads = %d, want 3", n) + } + + // Once the hold ends, b sends again. + clock = clock.Add(2 * time.Minute) + if err := b.Held(upstream.URL); err != nil { + t.Errorf("b after the hold ended: %v", err) + } +} + +func TestSharedRateLimitGateReload(t *testing.T) { + store := &memoryHoldStore{holds: make(map[string]time.Time)} + g := NewSharedRateLimitGate(registryclient.NewRateLimitGate(nil), store, nil) + g.Refresh = time.Hour + if err := g.Held("https://registry.example/x"); err != nil { + t.Fatal(err) + } + _ = store.PutUpstreamHold("registry.example", time.Now().Add(time.Minute)) + if err := g.Held("https://registry.example/x"); err != nil { + t.Fatalf("held before reload: %v", err) + } + g.Reload() + if !errors.Is(g.Held("https://registry.example/x"), registryclient.ErrUpstreamHeld) { + t.Error("not held after reload") + } +} + +func TestSharedRateLimitGateWithoutStore(t *testing.T) { + g := NewSharedRateLimitGate(registryclient.NewRateLimitGate(nil), nil, nil) + if err := g.Held("https://registry.example/x"); err != nil { + t.Fatal(err) + } + g.Reload() + var nilGate *SharedRateLimitGate + if nilGate.Held("https://registry.example/x") != nil || nilGate.Holds() != nil { + t.Error("nil gate holds") + } +} diff --git a/internal/server/breakers_test.go b/internal/server/breakers_test.go index 3a9e764..6cfb511 100644 --- a/internal/server/breakers_test.go +++ b/internal/server/breakers_test.go @@ -12,6 +12,7 @@ import ( "testing" "time" + upstreamhttp "github.com/git-pkgs/proxy/internal/httpclient" "github.com/git-pkgs/proxy/internal/metrics" registryclient "github.com/git-pkgs/registries/client" "github.com/git-pkgs/registries/fetch" @@ -364,7 +365,7 @@ func TestHealthEndpoint_ReportsUpstreamHolds(t *testing.T) { if resp, err := gate.RoundTrip(req); err == nil { _ = resp.Body.Close() } - ts.server.rateLimits = gate + ts.server.rateLimits = upstreamhttp.NewSharedRateLimitGate(gate, nil, nil) w := httptest.NewRecorder() ts.handler.ServeHTTP(w, httptest.NewRequest("GET", "/health", nil)) diff --git a/internal/server/registry_policies_test.go b/internal/server/registry_policies_test.go index 881356a..4fb1942 100644 --- a/internal/server/registry_policies_test.go +++ b/internal/server/registry_policies_test.go @@ -13,6 +13,7 @@ import ( "time" "github.com/git-pkgs/proxy/internal/config" + upstreamhttp "github.com/git-pkgs/proxy/internal/httpclient" registryclient "github.com/git-pkgs/registries/client" "github.com/git-pkgs/registries/fetch" ) @@ -59,7 +60,7 @@ func TestMirrorProxySharesHoldAfterBearerRetry(t *testing.T) { t.Fatalf("metadata fetch = %v, upstream calls %d", err, calls.Load()) } var held *registryclient.HeldError - if !errors.As(proxy.RateLimits.Held(upstream.URL), &held) || held.RetryAfter < 59*time.Minute || len(proxy.RateLimits.Holds()) != 1 { + if !errors.As(proxy.RateLimits.Held(upstream.URL), &held) || held.RetryAfter < 59*time.Minute || len(proxy.RateLimits.(*upstreamhttp.SharedRateLimitGate).Holds()) != 1 { t.Fatalf("shared hold = %+v", held) } } diff --git a/internal/server/server.go b/internal/server/server.go index 3dc2f9b..4a7738f 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -120,7 +120,7 @@ type Server struct { accessLog *accesslog.Logger ecr *ecrTokens breakers *breakerMonitor - rateLimits *registryclient.RateLimitGate + rateLimits *upstreamhttp.SharedRateLimitGate ecoStats ecosystemStatsCache metaStats metadataStatsCache sources sourceTracker @@ -571,8 +571,14 @@ func (s *Server) newUpstreamClients(artifactTimeout time.Duration) (metadataClie s.logger.Info("upstream rate limit cleared", "host", host) } } - s.rateLimits = gate - authTransport := upstreamhttp.NewTransport(gate, upstreamhttp.AuthFunc(s.authForURL)) + // Holds go through the database so instances sharing it stop sending to + // a host as soon as any one of them is rate limited. + var holdStore upstreamhttp.HoldStore + if s.db != nil { + holdStore = s.db + } + s.rateLimits = upstreamhttp.NewSharedRateLimitGate(gate, holdStore, s.logger) + authTransport := upstreamhttp.NewTransport(s.rateLimits, upstreamhttp.AuthFunc(s.authForURL)) metadata := *safeClient metadata.Timeout = s.cfg.ParseHTTPTimeout() metadata.Transport = authTransport From 3bf0cda23818087ae7021a23acb95fd0f6617137 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 25 Oct 2026 17:35:49 +0100 Subject: [PATCH 437/633] Read SQLite through a separate read-only pool SQLite used one connection for everything, so request lookups queued behind the per-minute stats queries, dashboard pages and eviction scans. Writes keep the single connection. Get, Select, Query and QueryRow now go to a pool of query_only connections (GOMAXPROCS, 2 to 8), which WAL mode lets run alongside the writer. A write sent to that pool fails. Pragmas move into the DSN so every connection gets them. Postgres and in-memory SQLite keep one pool for both. --- docs/architecture.md | 2 + docs/configuration.md | 4 +- internal/database/database.go | 125 +++++++++- internal/database/hits.go | 2 +- internal/database/read_pool_bench_test.go | 157 ++++++++++++ internal/database/read_pool_test.go | 281 ++++++++++++++++++++++ 6 files changed, 559 insertions(+), 12 deletions(-) create mode 100644 internal/database/read_pool_bench_test.go create mode 100644 internal/database/read_pool_test.go diff --git a/docs/architecture.md b/docs/architecture.md index 76b2156..1afb201 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -96,6 +96,8 @@ the full artifact digest because they read only part of the artifact. SQLite or PostgreSQL database for cache metadata. SQLite uses `modernc.org/sqlite` (pure Go, no CGO). PostgreSQL uses `lib/pq`. +For SQLite, `DB` holds two pools: one connection for writes and a pool of read-only (`query_only`) connections for reads. `Get`, `Select`, `Query` and `QueryRow` go to the reader; `Exec`, transactions and everything else go to the writer, so a read inside a transaction sees the transaction's writes. A write sent to the reader fails instead of running. WAL mode lets the readers run while the writer commits. PostgreSQL and in-memory SQLite use one pool for both. + The schema is compatible with [git-pkgs](https://github.com/git-pkgs) databases. The proxy adds the `artifacts` and `vulnerabilities` tables on top of the shared `packages` and `versions` tables, so both tools can point at the same database. **Tables:** diff --git a/docs/configuration.md b/docs/configuration.md index 5a877e3..d6c8bfa 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -185,6 +185,8 @@ database: | `database.driver` | `PROXY_DATABASE_DRIVER` | `-database-driver` | `sqlite` or `postgres` | | `database.path` | `PROXY_DATABASE_PATH` | `-database-path` | SQLite file path | +The database runs in WAL mode. Writes go through a single connection, since SQLite allows one writer at a time. Reads use a separate pool of read-only connections (one per CPU, between 2 and 8), so request lookups don't wait behind writes, stats queries or eviction scans. An in-memory database (`:memory:`) uses one connection for both. + ### PostgreSQL ```yaml @@ -213,7 +215,7 @@ Each instance needs its own [`git.cache_dir`](#git-repositories). ### Hit counts -Every cache hit updates the artifact's hit count and last-access time, which the stats pages and LRU eviction use. Rather than writing each hit as its own transaction, the proxy counts hits in memory and writes them together every `hit_flush_interval`. SQLite allows one writer at a time, and the proxy uses a single SQLite connection, so with a write per hit, concurrent downloads queue behind each other. +Every cache hit updates the artifact's hit count and last-access time, which the stats pages and LRU eviction use. Rather than writing each hit as its own transaction, the proxy counts hits in memory and writes them together every `hit_flush_interval`. SQLite allows one writer at a time, and the proxy writes through a single SQLite connection, so with a write per hit, concurrent downloads queue behind each other. ```yaml database: diff --git a/internal/database/database.go b/internal/database/database.go index 5a7e127..28a16f3 100644 --- a/internal/database/database.go +++ b/internal/database/database.go @@ -2,9 +2,12 @@ package database import ( "context" + "database/sql" "fmt" "os" "path/filepath" + "runtime" + "strings" "time" "github.com/jmoiron/sqlx" @@ -33,8 +36,33 @@ const ( DialectPostgres Dialect = "postgres" ) +// SQLite DSN parameters. Every connection a pool opens applies them, so a +// connection database/sql reopens gets them too. The reader pool sets +// query_only, which makes a write sent to it fail instead of being run. +const ( + sqliteWriterParams = "?_busy_timeout=5000&_journal_mode=WAL&_synchronous=NORMAL" + sqliteReaderParams = "?_busy_timeout=5000&_query_only=1" +) + +// Size bounds of the SQLite reader pool. At least two, so a slow stats or +// eviction scan does not block request lookups even on one CPU. +const ( + sqliteMinReaders = 2 + sqliteMaxReaders = 8 +) + +// DB is the proxy database. The embedded pool takes every write. For SQLite +// it holds a single connection, as SQLite allows one writer at a time, and +// reads go to a separate read-only pool so they don't queue behind writes or +// behind each other (WAL mode lets readers run alongside the writer). For +// Postgres and in-memory SQLite both are the same pool. +// +// Get, Select, Query and QueryRow (and their Context forms) use the reader. +// Exec, transactions and everything else use the writer, so a read inside a +// transaction goes through the transaction and sees its own writes. type DB struct { *sqlx.DB + reader *sqlx.DB dialect Dialect path string hits *hitBatch @@ -76,24 +104,48 @@ func Open(path string) (*DB, error) { } } - // Add busy_timeout to handle concurrent writes - sqlDB, err := sqlx.Open("sqlite", path+"?_busy_timeout=5000") + writer, err := sqlx.Open("sqlite", path+sqliteWriterParams) if err != nil { return nil, fmt.Errorf("opening database: %w", err) } + // One connection serializes writes, so they never wait on SQLite's + // busy timeout for each other. + writer.SetMaxOpenConns(1) - // Limit connections to 1 for SQLite to serialize writes - sqlDB.SetMaxOpenConns(1) - - db := &DB{DB: sqlDB, dialect: DialectSQLite, path: path} - if err := db.OptimizeForReads(); err != nil { - _ = sqlDB.Close() - return nil, fmt.Errorf("optimizing database: %w", err) + // Connecting applies journal_mode=WAL before any reader opens the file. + if err := writer.Ping(); err != nil { + _ = writer.Close() + return nil, fmt.Errorf("opening database: %w", err) } + db := &DB{DB: writer, reader: writer, dialect: DialectSQLite, path: path} + if isMemoryPath(path) { + // Each connection to an in-memory database gets its own empty + // database, so the reads have to share the writer's connection. + return db, nil + } + + reader, err := sqlx.Open("sqlite", path+sqliteReaderParams) + if err != nil { + _ = writer.Close() + return nil, fmt.Errorf("opening database reader: %w", err) + } + n := sqliteReaderConns() + reader.SetMaxOpenConns(n) + reader.SetMaxIdleConns(n) + db.reader = reader + return db, nil } +func sqliteReaderConns() int { + return min(max(runtime.GOMAXPROCS(0), sqliteMinReaders), sqliteMaxReaders) +} + +func isMemoryPath(path string) bool { + return path == "" || strings.HasPrefix(path, ":memory:") || strings.Contains(path, "mode=memory") +} + func OpenOrCreate(path string) (*DB, error) { if Exists(path) { return Open(path) @@ -117,7 +169,7 @@ func OpenPostgres(url string) (*DB, error) { return nil, fmt.Errorf("connecting to postgres: %w", err) } - return &DB{DB: sqlDB, dialect: DialectPostgres}, nil + return &DB{DB: sqlDB, reader: sqlDB, dialect: DialectPostgres}, nil } func OpenPostgresOrCreate(url string) (*DB, error) { @@ -179,3 +231,56 @@ func (db *DB) Path() string { func (db *DB) Rebind(query string) string { return db.DB.Rebind(query) } + +// Get runs a read on the reader pool. +func (db *DB) Get(dest any, query string, args ...any) error { + return db.reader.Get(dest, query, args...) +} + +// GetContext runs a read on the reader pool. +func (db *DB) GetContext(ctx context.Context, dest any, query string, args ...any) error { + return db.reader.GetContext(ctx, dest, query, args...) +} + +// Select runs a read on the reader pool. +func (db *DB) Select(dest any, query string, args ...any) error { + return db.reader.Select(dest, query, args...) +} + +// SelectContext runs a read on the reader pool. +func (db *DB) SelectContext(ctx context.Context, dest any, query string, args ...any) error { + return db.reader.SelectContext(ctx, dest, query, args...) +} + +// Query runs a read on the reader pool. +func (db *DB) Query(query string, args ...any) (*sql.Rows, error) { + return db.reader.Query(query, args...) +} + +// QueryContext runs a read on the reader pool. +func (db *DB) QueryContext(ctx context.Context, query string, args ...any) (*sql.Rows, error) { + return db.reader.QueryContext(ctx, query, args...) +} + +// QueryRow runs a read on the reader pool. +func (db *DB) QueryRow(query string, args ...any) *sql.Row { + return db.reader.QueryRow(query, args...) +} + +// QueryRowContext runs a read on the reader pool. +func (db *DB) QueryRowContext(ctx context.Context, query string, args ...any) *sql.Row { + return db.reader.QueryRowContext(ctx, query, args...) +} + +// closePools closes the reader pool, if separate, then the writer. The +// writer goes last so that, as the last connection, it checkpoints the WAL. +func (db *DB) closePools() error { + var readerErr error + if db.reader != nil && db.reader != db.DB { + readerErr = db.reader.Close() + } + if err := db.DB.Close(); err != nil { + return err + } + return readerErr +} diff --git a/internal/database/hits.go b/internal/database/hits.go index 3bb19c7..3609b5f 100644 --- a/internal/database/hits.go +++ b/internal/database/hits.go @@ -211,5 +211,5 @@ func (db *DB) Close() error { b.stopOnce.Do(func() { close(b.stop) }) <-b.done } - return db.DB.Close() + return db.closePools() } diff --git a/internal/database/read_pool_bench_test.go b/internal/database/read_pool_bench_test.go new file mode 100644 index 0000000..bae4cc5 --- /dev/null +++ b/internal/database/read_pool_bench_test.go @@ -0,0 +1,157 @@ +package database + +import ( + "fmt" + "path/filepath" + "sync" + "sync/atomic" + "testing" + "time" +) + +// benchArtifacts is the artifact count the read pool benchmarks seed: about +// what a busy cache holds, and enough for the stats queries to take a while. +const ( + benchPackages = 30_000 + benchVersions = 100_000 + benchArtifacts = 300_000 +) + +// seedBenchDB creates a database at path with benchArtifacts artifacts +// spread over three ecosystems, written in one statement per table. +func seedBenchDB(b *testing.B, path string) { + b.Helper() + db, err := Create(path) + if err != nil { + b.Fatalf("Create: %v", err) + } + defer func() { _ = db.Close() }() + stmts := []string{ + fmt.Sprintf(`WITH RECURSIVE n(i) AS (SELECT 0 UNION ALL SELECT i+1 FROM n WHERE i+1 < %d) + INSERT INTO packages (purl, ecosystem, name, created_at, updated_at) + SELECT 'pkg:' || e || '/p' || i, e, 'p' || i, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP + FROM (SELECT i, CASE i %% 3 WHEN 0 THEN 'npm' WHEN 1 THEN 'pypi' ELSE 'cargo' END AS e FROM n)`, benchPackages), + fmt.Sprintf(`WITH RECURSIVE n(i) AS (SELECT 0 UNION ALL SELECT i+1 FROM n WHERE i+1 < %d) + INSERT INTO versions (purl, package_purl, integrity, created_at, updated_at) + SELECT p.purl || '@' || n.i, p.purl, 'sha256-x', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP + FROM n JOIN packages p ON p.id = (n.i %% %d) + 1`, benchVersions, benchPackages), + fmt.Sprintf(`WITH RECURSIVE n(i) AS (SELECT 0 UNION ALL SELECT i+1 FROM n WHERE i+1 < %d) + INSERT INTO artifacts (version_purl, filename, upstream_url, storage_path, content_hash, size, + content_type, fetched_at, hit_count, last_accessed_at, created_at, updated_at) + SELECT v.purl, 'f' || n.i, 'https://example.org/' || n.i, 'blobs/' || n.i, 'h', 1000 + n.i, + 'application/octet-stream', CURRENT_TIMESTAMP, n.i %% 7, CURRENT_TIMESTAMP, + CURRENT_TIMESTAMP, CURRENT_TIMESTAMP + FROM n JOIN versions v ON v.id = (n.i %% %d) + 1`, benchArtifacts, benchVersions), + } + for _, s := range stmts { + if _, err := db.Exec(s); err != nil { + b.Fatalf("seeding: %v", err) + } + } +} + +// openBenchDB opens path. With oneConn it routes reads to the writer, as +// before the read pool existed. +func openBenchDB(b *testing.B, path string, oneConn bool) *DB { + b.Helper() + db, err := Open(path) + if err != nil { + b.Fatalf("Open: %v", err) + } + if oneConn { + _ = db.reader.Close() + db.reader = db.DB + } + b.Cleanup(func() { _ = db.Close() }) + return db +} + +// benchKey returns the lookup arguments for artifact i, as seeded above. +func benchKey(i int) (pkgPURL, versionPURL, filename string) { + v := i % benchVersions + p := v % benchPackages + eco := [...]string{"npm", "pypi", "cargo"}[p%3] + pkgPURL = fmt.Sprintf("pkg:%s/p%d", eco, p) + return pkgPURL, fmt.Sprintf("%s@%d", pkgPURL, v), fmt.Sprintf("f%d", i) +} + +func lookup(b *testing.B, db *DB, i int) { + pkgPURL, versionPURL, filename := benchKey(i % benchArtifacts) + art, err := db.GetCachedArtifact(pkgPURL, versionPURL, filename) + if err != nil || art == nil { + b.Fatalf("GetCachedArtifact(%s, %s) = %v, %v", versionPURL, filename, art, err) + } +} + +var poolModes = []struct { + name string + oneConn bool +}{ + {"one-connection", true}, + {"read-pool", false}, +} + +// BenchmarkSQLiteParallelLookups runs GetCachedArtifact from parallel +// goroutines, as concurrent downloads do. +func BenchmarkSQLiteParallelLookups(b *testing.B) { + path := filepath.Join(b.TempDir(), "bench.db") + seedBenchDB(b, path) + for _, mode := range poolModes { + b.Run(mode.name, func(b *testing.B) { + db := openBenchDB(b, path, mode.oneConn) + var next atomic.Int64 + b.ResetTimer() + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + lookup(b, db, int(next.Add(7919))) + } + }) + }) + } +} + +// BenchmarkSQLiteLookupDuringStats runs GetCachedArtifact one at a time +// while another goroutine keeps running GetEcosystemStats and GetCacheStats, +// as the metrics updater and the dashboard do. It reports the slowest lookup. +func BenchmarkSQLiteLookupDuringStats(b *testing.B) { + path := filepath.Join(b.TempDir(), "bench.db") + seedBenchDB(b, path) + for _, mode := range poolModes { + b.Run(mode.name, func(b *testing.B) { + db := openBenchDB(b, path, mode.oneConn) + stop := make(chan struct{}) + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + for { + select { + case <-stop: + return + default: + } + if _, err := db.GetEcosystemStats(); err != nil { + b.Errorf("GetEcosystemStats: %v", err) + return + } + if _, err := db.GetCacheStats(); err != nil { + b.Errorf("GetCacheStats: %v", err) + return + } + } + }() + + var slowest time.Duration + b.ResetTimer() + for i := range b.N { + start := time.Now() + lookup(b, db, i*7919) + slowest = max(slowest, time.Since(start)) + } + b.StopTimer() + close(stop) + wg.Wait() + b.ReportMetric(float64(slowest.Microseconds())/1000, "max-ms") + }) + } +} diff --git a/internal/database/read_pool_test.go b/internal/database/read_pool_test.go new file mode 100644 index 0000000..a1d6071 --- /dev/null +++ b/internal/database/read_pool_test.go @@ -0,0 +1,281 @@ +package database + +import ( + "context" + "database/sql" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +func seedOnePackage(t *testing.T, db *DB) { + t.Helper() + pkg := &Package{PURL: "pkg:npm/left-pad", Ecosystem: "npm", Name: "left-pad"} + ver := &Version{PURL: "pkg:npm/left-pad@1.3.0", PackagePURL: pkg.PURL} + art := &Artifact{ + VersionPURL: ver.PURL, + Filename: "left-pad-1.3.0.tgz", + UpstreamURL: "https://registry.npmjs.org/left-pad/-/left-pad-1.3.0.tgz", + StoragePath: sql.NullString{String: "npm/left-pad/1.3.0/left-pad-1.3.0.tgz", Valid: true}, + Size: sql.NullInt64{Int64: 10, Valid: true}, + FetchedAt: sql.NullTime{Time: time.Now(), Valid: true}, + } + if err := db.RecordCachedArtifact(pkg, ver, art); err != nil { + t.Fatalf("RecordCachedArtifact: %v", err) + } +} + +func TestSQLiteUsesSeparateReadPool(t *testing.T) { + db := newTestDB(t) + if db.reader == db.DB { + t.Fatal("file database shares one pool for reads and writes") + } + if got, want := db.reader.Stats().MaxOpenConnections, sqliteReaderConns(); got != want { + t.Errorf("reader MaxOpenConnections = %d, want %d", got, want) + } + if got := db.DB.Stats().MaxOpenConnections; got != 1 { + t.Errorf("writer MaxOpenConnections = %d, want 1", got) + } +} + +func TestSQLiteReadPoolRejectsWrites(t *testing.T) { + db := newTestDB(t) + + if _, err := db.reader.Exec(`INSERT INTO pending_deletes (path, queued_at) VALUES ('x', CURRENT_TIMESTAMP)`); err == nil { + t.Fatal("write on the reader pool succeeded") + } else if !strings.Contains(err.Error(), "readonly") { + t.Fatalf("write on the reader pool failed with %v, want a read-only error", err) + } + + // Get goes to the reader, so a write sent through it must fail too + // rather than run. + var path string + err := db.Get(&path, `INSERT INTO pending_deletes (path, queued_at) VALUES ('y', CURRENT_TIMESTAMP) RETURNING path`) + if err == nil { + t.Fatal("INSERT ... RETURNING through Get succeeded") + } + + var n int + if err := db.Get(&n, `SELECT COUNT(*) FROM pending_deletes`); err != nil { + t.Fatalf("count: %v", err) + } + if n != 0 { + t.Fatalf("pending_deletes has %d rows after rejected writes, want 0", n) + } +} + +// pragmaInt reads an integer pragma on conn. +func pragmaInt(t *testing.T, conn *sql.Conn, name string) int { + t.Helper() + var v int + if err := conn.QueryRowContext(context.Background(), "PRAGMA "+name).Scan(&v); err != nil { + t.Fatalf("PRAGMA %s: %v", name, err) + } + return v +} + +func pragmaString(t *testing.T, conn *sql.Conn, name string) string { + t.Helper() + var v string + if err := conn.QueryRowContext(context.Background(), "PRAGMA "+name).Scan(&v); err != nil { + t.Fatalf("PRAGMA %s: %v", name, err) + } + return v +} + +// TestSQLitePragmasOnEveryConnection holds every connection of both pools at +// once, so each one is a separate SQLite connection, and checks that each +// got the settings from the DSN. +func TestSQLitePragmasOnEveryConnection(t *testing.T) { + db := newTestDB(t) + ctx := context.Background() + + writer, err := db.Conn(ctx) + if err != nil { + t.Fatalf("writer conn: %v", err) + } + defer func() { _ = writer.Close() }() + if got := pragmaInt(t, writer, "busy_timeout"); got != 5000 { + t.Errorf("writer busy_timeout = %d, want 5000", got) + } + if got := pragmaString(t, writer, "journal_mode"); got != "wal" { + t.Errorf("writer journal_mode = %q, want wal", got) + } + if got := pragmaInt(t, writer, "synchronous"); got != 1 { + t.Errorf("writer synchronous = %d, want 1 (NORMAL)", got) + } + if got := pragmaInt(t, writer, "query_only"); got != 0 { + t.Errorf("writer query_only = %d, want 0", got) + } + + n := sqliteReaderConns() + conns := make([]*sql.Conn, 0, n) + defer func() { + for _, c := range conns { + _ = c.Close() + } + }() + for i := range n { + c, err := db.reader.Conn(ctx) + if err != nil { + t.Fatalf("reader conn %d: %v", i, err) + } + conns = append(conns, c) + if got := pragmaInt(t, c, "busy_timeout"); got != 5000 { + t.Errorf("reader %d busy_timeout = %d, want 5000", i, got) + } + if got := pragmaString(t, c, "journal_mode"); got != "wal" { + t.Errorf("reader %d journal_mode = %q, want wal", i, got) + } + if got := pragmaInt(t, c, "query_only"); got != 1 { + t.Errorf("reader %d query_only = %d, want 1", i, got) + } + } + if got := db.reader.Stats().OpenConnections; got != n { + t.Errorf("reader pool has %d open connections, want %d", got, n) + } +} + +// TestSQLiteReadNotBlockedBySlowRead holds one reader connection in an open +// read, as a long stats scan does, and a write transaction on the writer. +// A lookup must still complete, and must not see the uncommitted write. +func TestSQLiteReadNotBlockedBySlowRead(t *testing.T) { + db := newTestDB(t) + seedOnePackage(t, db) + + rows, err := db.Query(`SELECT purl FROM packages`) + if err != nil { + t.Fatalf("Query: %v", err) + } + defer func() { _ = rows.Close() }() + if !rows.Next() { + t.Fatal("no package row") + } + + // Bounded, so with a single shared connection the test fails instead + // of waiting forever for the connection the open read holds. + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + tx, err := db.BeginTxx(ctx, nil) + if err != nil { + t.Fatalf("BeginTxx while a read is open: %v", err) + } + defer func() { _ = tx.Rollback() }() + if _, err := tx.Exec(`INSERT INTO pending_deletes (path, queued_at) VALUES ('uncommitted', CURRENT_TIMESTAMP)`); err != nil { + t.Fatalf("insert in tx: %v", err) + } + + type result struct { + art *CachedArtifact + pending int + err error + } + done := make(chan result, 1) + go func() { + var r result + r.art, r.err = db.GetCachedArtifact("pkg:npm/left-pad", "pkg:npm/left-pad@1.3.0", "left-pad-1.3.0.tgz") + if r.err == nil { + r.err = db.GetContext(ctx, &r.pending, `SELECT COUNT(*) FROM pending_deletes`) + } + done <- r + }() + + select { + case r := <-done: + if r.err != nil { + t.Fatalf("lookup: %v", r.err) + } + if r.art == nil { + t.Fatal("lookup found no artifact") + } + if r.pending != 0 { + t.Errorf("reader saw %d uncommitted pending deletes, want 0", r.pending) + } + case <-time.After(5 * time.Second): + t.Fatal("lookup blocked behind an open read and a write transaction") + } + + if err := tx.Commit(); err != nil { + t.Fatalf("Commit: %v", err) + } + var pending int + if err := db.Get(&pending, `SELECT COUNT(*) FROM pending_deletes`); err != nil { + t.Fatalf("count after commit: %v", err) + } + if pending != 1 { + t.Errorf("reader saw %d pending deletes after commit, want 1", pending) + } +} + +// TestSQLiteReadAfterWrite checks that a read right after a write, on the +// other pool, sees it. +func TestSQLiteReadAfterWrite(t *testing.T) { + db := newTestDB(t) + for i := range 50 { + path := fmt.Sprintf("npm/p/%d.tgz", i) + if err := db.QueuePendingDelete(path); err != nil { + t.Fatalf("QueuePendingDelete: %v", err) + } + var n int + if err := db.Get(&n, db.Rebind(`SELECT COUNT(*) FROM pending_deletes WHERE path = ?`), path); err != nil { + t.Fatalf("count: %v", err) + } + if n != 1 { + t.Fatalf("write %d not visible to the reader", i) + } + } +} + +func TestSQLiteMemoryDatabaseSharesOneConnection(t *testing.T) { + db, err := Open(":memory:") + if err != nil { + t.Fatalf("Open: %v", err) + } + defer func() { _ = db.Close() }() + if db.reader != db.DB { + t.Fatal("in-memory database got a separate reader pool") + } + if err := db.CreateSchema(); err != nil { + t.Fatalf("CreateSchema: %v", err) + } + seedOnePackage(t, db) + art, err := db.GetCachedArtifact("pkg:npm/left-pad", "pkg:npm/left-pad@1.3.0", "left-pad-1.3.0.tgz") + if err != nil || art == nil { + t.Fatalf("GetCachedArtifact = %v, %v", art, err) + } +} + +func TestSQLiteCloseClosesBothPools(t *testing.T) { + db, err := Create(filepath.Join(t.TempDir(), "close.db")) + if err != nil { + t.Fatalf("Create: %v", err) + } + reader := db.reader + if err := db.Close(); err != nil { + t.Fatalf("Close: %v", err) + } + if err := reader.Ping(); err == nil { + t.Fatal("reader pool still open after Close") + } + if err := db.Ping(); err == nil { + t.Fatal("writer pool still open after Close") + } +} + +func TestPostgresSharesOnePool(t *testing.T) { + url := os.Getenv("PROXY_DATABASE_URL") + if url == "" { + t.Skip("PROXY_DATABASE_URL not set, skipping postgres test") + } + db, err := OpenPostgres(url) + if err != nil { + t.Fatalf("OpenPostgres: %v", err) + } + defer func() { _ = db.Close() }() + if db.reader != db.DB { + t.Fatal("postgres got a separate reader pool") + } +} From b09ac872646d293b06b3d3a09fd57837f7677f1e Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 25 Oct 2026 17:41:40 +0100 Subject: [PATCH 438/633] Read gem, Hex and conda cooldown metadata through the metadata cache With cooldown on, the RubyGems compact index and versions API, the Hex package and Hex API entry, and conda's repodata.json were fetched with plain HTTP requests on every client request: no TTL, no revalidation, no coalescing and no stale fallback, even with cache_metadata. Gem and Hex now read both documents through the metadata cache and filter the cached copies. The filtered result goes through the rewrite cache and is served with an ETag. When the times document is older than the index and misses a version, it is fetched again; if that fails the version is left out. Conda filters a fresh stored repodata.json as it reads it from storage, uses the metadata cache otherwise, and remembers the ETag of the filtered copy until a withheld entry leaves cooldown. --- docs/configuration.md | 8 +- internal/handler/conda.go | 185 ++++- .../handler/cooldown_metadata_cache_test.go | 644 ++++++++++++++++++ internal/handler/cooldown_times.go | 149 ++++ internal/handler/gem.go | 252 +++---- internal/handler/gem_test.go | 46 +- internal/handler/handler.go | 30 +- internal/handler/hex.go | 223 +++--- internal/handler/hex_test.go | 2 +- internal/handler/metadata_stream.go | 98 ++- internal/handler/relay_test.go | 11 +- internal/handler/upstream_test.go | 8 +- 12 files changed, 1323 insertions(+), 333 deletions(-) create mode 100644 internal/handler/cooldown_metadata_cache_test.go create mode 100644 internal/handler/cooldown_times.go diff --git a/docs/configuration.md b/docs/configuration.md index 5a877e3..564825b 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1046,9 +1046,11 @@ Maven caveats: RubyGems, Hex and PyPI keep their publish times outside the metadata clients read: the RubyGems versions API, the Hex HTTP API (`upstream.hex_api`) and the PyPI JSON API. The proxy fetches them alongside each compact index, package or simple page it filters, and for PyPI downloads. Cooldown fails closed: when it applies to a package and those times can't be fetched or parsed, or the metadata can't be rewritten, the request fails with 502, or with 503 and `Retry-After` while the upstream is rate limiting the proxy, instead of serving the metadata unfiltered. The same goes for npm, Composer and pub.dev metadata that can't be rewritten, and for npm tarball downloads whose publish time can't be read. A PyPI file or npm tarball that is already cached is still served when its publish time can't be read; only a download that would have to be fetched fails. Packages exempted with a duration of `0` are served as before and don't need the times. A PyPI package the JSON API returns 404 for has no times to go by and is served unfiltered. +For RubyGems and Hex, both documents go through the metadata cache. With [`cache_metadata`](#metadata-caching) they are served from it within `metadata_ttl` without asking the upstream, revalidated with their ETags after it, and filtered from the last cached copies, marked stale, while the upstream is down; without it they are still fetched once for concurrent requests. Each document has its own TTL, so the compact index or Hex package can be newer than the cached times and list a version they don't have. The times are then fetched again before filtering, and if that fails, such versions are left out until the times can be read. The filtered index or package carries an ETag of its own, so Bundler and Mix revalidate it with `304 Not Modified`. Publish times are compared with the current time on each request, but the filtered result is kept for up to a minute (see [rewritten metadata cache](#rewritten-metadata-cache)), so a version leaving cooldown can show up a minute late; a newly published version shows up within `metadata_ttl`, as with other cached metadata. Upstream errors are answered as for other cached metadata (404, or 502) rather than relayed. + With cooldown on, the proxy asks npm for full packuments, since the abbreviated ones it uses otherwise carry no publish times. Abbreviated packuments cached before cooldown was turned on are fetched again in full rather than served, including as a stale copy when the upstream is down. -Conda cooldown covers `repodata.json` and sharded repodata (CEP 16). Clients cache shards by the hash in the shard index, so with cooldown on the proxy serves a rewritten index whose shard hashes change every hour and filters each shard as it is fetched. Versions leaving cooldown show up within the hour, and clients fetch the shards they need again once an hour. If a channel's shard index points packages or shards somewhere the proxy doesn't route, the proxy answers 404 for it and clients fall back to `repodata.json`. +Conda cooldown covers `repodata.json` and sharded repodata (CEP 16). With `cache_metadata`, `repodata.json` is read through the metadata cache as it is without cooldown: within `metadata_ttl` the stored copy is filtered as it is read from storage, after it the upstream is asked with its ETag, and while the upstream is down the stale copy is filtered. Once a document has been filtered, the proxy remembers the ETag of the result until the first entry it left out leaves cooldown, so a client revalidating with `If-None-Match` gets `304 Not Modified` without the document being filtered or sent again. The first response for each document has no ETag, as it is sent while it is filtered. Without `cache_metadata`, `repodata.json` is fetched and filtered on every request. Clients cache shards by the hash in the shard index, so with cooldown on the proxy serves a rewritten index whose shard hashes change every hour and filters each shard as it is fetched. Versions leaving cooldown show up within the hour, and clients fetch the shards they need again once an hour. If a channel's shard index points packages or shards somewhere the proxy doesn't route, the proxy answers 404 for it and clients fall back to `repodata.json`. Note: Hex cooldown requires disabling registry signature verification since the proxy re-encodes the protobuf payload without the original signature. Set `HEX_NO_VERIFY_REPO_ORIGIN=1` or configure your repo with `no_verify: true`. @@ -1297,7 +1299,7 @@ When upstream is unreachable and the cached entry is past its TTL, the proxy ser ### Metadata size limit -Upstream metadata responses are buffered in memory before being rewritten and served. `metadata_max_size` caps that buffer to protect against OOM from a misbehaving upstream. Some npm packages with thousands of versions (for example `renovate`) exceed the 100 MB default, so raise this if you see `metadata response exceeds size limit` in the logs. Conda `repodata.json` with cooldown enabled is filtered as it streams and isn't held in memory, so this limit doesn't apply to it. +Upstream metadata responses are buffered in memory before being rewritten and served. `metadata_max_size` caps that buffer to protect against OOM from a misbehaving upstream. Some npm packages with thousands of versions (for example `renovate`) exceed the 100 MB default, so raise this if you see `metadata response exceeds size limit` in the logs. Conda `repodata.json` with cooldown enabled is filtered as it streams and isn't held in memory decompressed. With `cache_metadata` the limit applies to the compressed document as it does without cooldown; one over it is filtered as it streams from the upstream and not cached. ```yaml metadata_max_size: "100MB" # default @@ -1317,7 +1319,7 @@ Or via environment variable: `PROXY_ARTIFACT_MAX_SIZE=2GB`. ### Rewritten metadata cache -The npm and Composer handlers rewrite every metadata document they serve, so that download URLs point at the proxy. That means decoding the whole document and encoding it again, and Composer's minified documents are also expanded, which together cost milliseconds per typical package and far more for very large ones. The proxy keeps rewritten documents in memory, so each distinct upstream document is rewritten once and requests arriving while it is being rewritten wait for that rewrite. New bytes from upstream are rewritten again. With version cooldown enabled, a cached rewrite is kept for one minute only, because cooldown filtering depends on the current time: a version leaving cooldown shows up at most a minute late. Such documents are dated like any other, so clients that revalidate with `If-Modified-Since`, such as Composer, get `304 Not Modified`. +The npm and Composer handlers rewrite every metadata document they serve, so that download URLs point at the proxy. That means decoding the whole document and encoding it again, and Composer's minified documents are also expanded, which together cost milliseconds per typical package and far more for very large ones. The proxy keeps rewritten documents in memory, so each distinct upstream document is rewritten once and requests arriving while it is being rewritten wait for that rewrite. New bytes from upstream are rewritten again. With version cooldown enabled, a cached rewrite is kept for one minute only, because cooldown filtering depends on the current time: a version leaving cooldown shows up at most a minute late. Such documents are dated like any other, so clients that revalidate with `If-Modified-Since`, such as Composer, get `304 Not Modified`. The RubyGems compact index and Hex packages filtered by cooldown are kept the same way; they are never served gzip-compressed. Each cached document is also kept gzip-compressed, when that makes it smaller, and served with `Content-Encoding: gzip` to clients that accept it. A large packument is compressed once rather than on every request, by the proxy or by a reverse proxy in front of it. The compressed copy counts against the cache size and typically adds about an eighth to each entry. Documents rewritten outside the cache are served uncompressed. diff --git a/internal/handler/conda.go b/internal/handler/conda.go index 1142ccf..1a14eb8 100644 --- a/internal/handler/conda.go +++ b/internal/handler/conda.go @@ -2,12 +2,19 @@ package handler import ( "bufio" + "bytes" "compress/gzip" + "crypto/sha256" + "encoding/hex" "encoding/json" + "errors" + "fmt" + "hash" "io" "net/http" "slices" "strings" + "sync" "time" "github.com/git-pkgs/proxy/internal/jsonstream" @@ -55,6 +62,11 @@ type CondaHandler struct { proxy *Proxy upstreamURL string proxyURL string + + // repodataStates holds, per metadata cache key, what is known of the + // last filtered copy of a repodata document. Guarded by repodataMu. + repodataMu sync.Mutex + repodataStates map[string]repodataState } // NewCondaHandler creates a new Conda protocol handler. @@ -179,6 +191,14 @@ func (h *CondaHandler) parseFilename(filename string) (name, version string) { } // handleRepodata proxies repodata.json, applying cooldown filtering when enabled. +// +// With cache_metadata the document is read through the metadata cache, as +// without cooldown, so within metadata_ttl it is filtered from storage +// without asking upstream, and after it revalidated with its ETag. A stored +// copy is streamed through the filter rather than read into memory; a +// document just fetched is held compressed, as the metadata cache holds every +// document it fetches. The decompressed document, hundreds of MB for +// conda-forge, is never held whole. func (h *CondaHandler) handleRepodata(w http.ResponseWriter, r *http.Request) { if h.proxy.Cooldown == nil || !h.proxy.Cooldown.Enabled() { h.proxyCached(w, r) @@ -186,7 +206,49 @@ func (h *CondaHandler) handleRepodata(w http.ResponseWriter, r *http.Request) { } upstreamURL := h.upstreamURL + r.URL.Path + p := h.proxy + if !p.CacheMetadata || p.DB == nil || p.Storage == nil { + h.streamRepodata(w, r, upstreamURL) + return + } + cacheKey := h.metadataCacheKey(r.URL.Path) + now := time.Now() + if entry := p.freshMetadataEntry("conda", cacheKey); entry != nil { + if body, err := p.openMetadataEntry(r.Context(), "conda", cacheKey, entry); err == nil { + defer func() { _ = body.Close() }() + p.recordStreamedMetadataHit("conda", entry) + digest := entry.ContentDigest.String + if !h.repodataNotModified(w, r, cacheKey, digest, now) { + h.writeFilteredRepodata(w, upstreamURL, cacheKey, digest, body, entry.ContentEncoding.String, now) + } + return + } + } + + // The same key and encoding as proxyCached, so with and without cooldown + // the proxy keeps one copy of the document. + res := p.lookupMetadata(r.Context(), "conda", cacheKey, upstreamURL, "*/*", "gzip", nil) + if errors.Is(res.err, ErrMetadataTooLarge) { + // Too large to cache even compressed: stream it as without the cache. + p.Logger.Warn("conda repodata exceeds the metadata size limit, streaming it uncached", "url", upstreamURL) + h.streamRepodata(w, r, upstreamURL) + return + } + if res.err != nil { + p.writeMetadataFetchError(w, res.err) + return + } + markStale(w, res.stale) + if h.repodataNotModified(w, r, cacheKey, res.digest, now) { + return + } + h.writeFilteredRepodata(w, upstreamURL, cacheKey, res.digest, bytes.NewReader(res.body), res.contentEncoding, now) +} + +// streamRepodata filters repodata.json as it streams from upstream, without +// the metadata cache. +func (h *CondaHandler) streamRepodata(w http.ResponseWriter, r *http.Request, upstreamURL string) { h.proxy.Logger.Debug("fetching repodata for cooldown filtering", "url", upstreamURL) req, err := http.NewRequestWithContext(r.Context(), http.MethodGet, upstreamURL, nil) @@ -212,11 +274,21 @@ func (h *CondaHandler) handleRepodata(w http.ResponseWriter, r *http.Request) { return } - // Explicitly requesting gzip disables the transport's automatic decoding. + // Explicitly requesting gzip disables the transport's automatic decoding, + // so writeFilteredRepodata decodes it. + h.writeFilteredRepodata(w, upstreamURL, "", "", resp.Body, resp.Header.Get(headerContentEncoding), time.Now()) +} + +// writeFilteredRepodata writes repodata.json from src, decoded by +// contentEncoding, without the entries in cooldown at now. digest identifies +// the document in the metadata cache under cacheKey, and is empty when it +// isn't cached; when known, the ETag of the filtered copy is remembered for +// the next request. +func (h *CondaHandler) writeFilteredRepodata(w http.ResponseWriter, upstreamURL, cacheKey, digest string, src io.Reader, contentEncoding string, now time.Time) { // Cooldown needs JSON, so decode here and bound the decompressed size too. - var reader io.Reader = resp.Body - if strings.EqualFold(strings.TrimSpace(resp.Header.Get(headerContentEncoding)), "gzip") { - decoded, err := gzip.NewReader(resp.Body) + reader := src + if strings.EqualFold(strings.TrimSpace(contentEncoding), "gzip") { + decoded, err := gzip.NewReader(src) if err != nil { http.Error(w, "failed to decode response", http.StatusBadGateway) return @@ -231,7 +303,8 @@ func (h *CondaHandler) handleRepodata(w http.ResponseWriter, r *http.Request) { out := &countingWriter{w: w} buffered := bufio.NewWriterSize(out, repodataWriteBuffer) w.Header().Set(headerContentType, "application/json") - err = h.filterRepodata(io.LimitReader(reader, maxRepodataSize), buffered) + filter := h.newRepodataFilter(now) + err := h.filterRepodataWith(io.LimitReader(reader, maxRepodataSize), buffered, filter) if err == nil { err = buffered.Flush() } @@ -243,6 +316,61 @@ func (h *CondaHandler) handleRepodata(w http.ResponseWriter, r *http.Request) { } panic(http.ErrAbortHandler) } + if digest != "" { + h.rememberRepodata(cacheKey, digest, filter) + } +} + +// repodataState is what the proxy remembers of the last filtered copy of a +// cached repodata document: which document it was made from, its ETag, and +// until when filtering that document again gives the same copy. +type repodataState struct { + digest string + etag string + // until is when the first entry left out leaves cooldown, or zero when + // none was left out. + until time.Time +} + +// repodataNotModified sets the ETag of the filtered copy of the document +// identified by digest when the proxy knows it, and answers a conditional +// request it matches with 304, reporting whether it did. The ETag is known +// once the document has been filtered, until an entry it left out leaves +// cooldown. Filtering a large document costs seconds, and sending it +// hundreds of MB, so this is what makes revalidating clients cheap. +func (h *CondaHandler) repodataNotModified(w http.ResponseWriter, r *http.Request, cacheKey, digest string, now time.Time) bool { + if digest == "" { + return false + } + h.repodataMu.Lock() + state, ok := h.repodataStates[cacheKey] + h.repodataMu.Unlock() + if !ok || state.digest != digest || (!state.until.IsZero() && !now.Before(state.until)) { + return false + } + w.Header().Set(headerETag, state.etag) + if ifNoneMatchHits(r.Header.Get("If-None-Match"), state.etag) { + w.WriteHeader(http.StatusNotModified) + return true + } + return false +} + +// rememberRepodata records the ETag of a filtered copy of the document +// identified by digest. The copy depends only on the document, the startup +// configuration and which entries were left out, and those stay left out +// until the first of them leaves cooldown. +func (h *CondaHandler) rememberRepodata(cacheKey, digest string, filter *repodataFilter) { + if filter.unknownUntil { + return + } + etag := keptVersionsETag(digest, "", cacheKey, []string{hex.EncodeToString(filter.withheld.Sum(nil))}) + h.repodataMu.Lock() + defer h.repodataMu.Unlock() + if h.repodataStates == nil { + h.repodataStates = make(map[string]repodataState) + } + h.repodataStates[cacheKey] = repodataState{digest: digest, etag: etag, until: filter.until} } // countingWriter records whether anything has reached the response yet. @@ -271,6 +399,11 @@ const ( // packages and packages.conda that are still inside their cooldown. Kept // entries and every other member are written with their upstream bytes. func (h *CondaHandler) filterRepodata(r io.Reader, w io.Writer) error { + return h.filterRepodataWith(r, w, h.newRepodataFilter(time.Now())) +} + +// filterRepodataWith is filterRepodata deciding by filter. +func (h *CondaHandler) filterRepodataWith(r io.Reader, w io.Writer, filter *repodataFilter) error { return jsonstream.CopyObject(json.NewDecoder(r), w, func(m *jsonstream.Member) error { if m.Key != "packages" && m.Key != "packages.conda" { return m.Copy() @@ -280,7 +413,7 @@ func (h *CondaHandler) filterRepodata(r io.Reader, w io.Writer) error { if err != nil { return err } - if h.inCooldown(entry.Key, raw) { + if filter.inCooldown(entry.Key, raw) { return nil } return entry.Write(raw) @@ -294,14 +427,50 @@ type repodataEntry struct { Timestamp float64 `json:"timestamp"` } +// repodataFilter decides which repodata entries cooldown leaves out, all as +// of one time, and records what it left out, which with the document fixes +// the filtered copy. +type repodataFilter struct { + h *CondaHandler + now time.Time + // withheld hashes the filenames left out. + withheld hash.Hash + // until is when the first entry left out leaves cooldown. + until time.Time + // unknownUntil is set when an entry was left out without a time it + // leaves cooldown, so until can't be relied on. + unknownUntil bool +} + +func (h *CondaHandler) newRepodataFilter(now time.Time) *repodataFilter { + return &repodataFilter{h: h, now: now, withheld: sha256.New()} +} + // inCooldown reports whether a repodata entry was published too recently. // Entries without a name or timestamp are kept. -func (h *CondaHandler) inCooldown(filename string, raw []byte) bool { +func (f *repodataFilter) inCooldown(filename string, raw []byte) bool { var entry repodataEntry if err := json.Unmarshal(raw, &entry); err != nil { return false } - return h.recordInCooldown(filename, entry.Name, entry.Version, entry.Timestamp) + if entry.Name == "" || entry.Timestamp == 0 { + return false + } + publishedAt := time.Unix(int64(entry.Timestamp)/condaTimestampDivisor, 0) + decision := f.h.proxy.Cooldown.Evaluate("conda", canonicalPackagePURL("conda", entry.Name), publishedAt, f.now) + if decision.Allowed { + return false + } + f.h.proxy.Logger.Info("cooldown: filtering conda package", + "name", entry.Name, "version", entry.Version, "filename", filename) + _, _ = fmt.Fprintf(f.withheld, "%d:%s", len(filename), filename) + switch { + case decision.AvailableAt.IsZero(): + f.unknownUntil = true + case f.until.IsZero() || decision.AvailableAt.Before(f.until): + f.until = decision.AvailableAt + } + return true } // recordInCooldown reports whether a package record with a millisecond diff --git a/internal/handler/cooldown_metadata_cache_test.go b/internal/handler/cooldown_metadata_cache_test.go new file mode 100644 index 0000000..08ec870 --- /dev/null +++ b/internal/handler/cooldown_metadata_cache_test.go @@ -0,0 +1,644 @@ +package handler + +import ( + "bytes" + "compress/gzip" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + "time" + + "github.com/git-pkgs/cooldown" +) + +// countingUpstream serves fixed documents by path with ETags, answers a +// matching If-None-Match with 304, and counts what it was asked. Setting +// down makes it answer 500, and closing gate holds requests until opened. +type countingUpstream struct { + *httptest.Server + mu sync.Mutex + docs map[string][]byte + gzipped map[string]bool + requests map[string]int + notModified map[string]int + down map[string]bool + gate chan struct{} +} + +func newCountingUpstream(t *testing.T) *countingUpstream { + t.Helper() + u := &countingUpstream{ + docs: map[string][]byte{}, gzipped: map[string]bool{}, + requests: map[string]int{}, notModified: map[string]int{}, down: map[string]bool{}, + } + u.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + u.mu.Lock() + u.requests[r.URL.Path]++ + gate := u.gate + doc, ok := u.docs[r.URL.Path] + down := u.down[r.URL.Path] + gz := u.gzipped[r.URL.Path] + u.mu.Unlock() + if gate != nil { + <-gate + } + switch { + case down: + w.WriteHeader(http.StatusInternalServerError) + return + case !ok: + w.WriteHeader(http.StatusNotFound) + return + } + sum := sha256.Sum256(doc) + etag := `"` + hex.EncodeToString(sum[:8]) + `"` + w.Header().Set("ETag", etag) + if r.Header.Get("If-None-Match") == etag { + u.mu.Lock() + u.notModified[r.URL.Path]++ + u.mu.Unlock() + w.WriteHeader(http.StatusNotModified) + return + } + if gz && strings.Contains(r.Header.Get("Accept-Encoding"), "gzip") { + w.Header().Set("Content-Encoding", "gzip") + zw := gzip.NewWriter(w) + _, _ = zw.Write(doc) + _ = zw.Close() + return + } + _, _ = w.Write(doc) + })) + t.Cleanup(u.Close) + return u +} + +func (u *countingUpstream) set(path string, doc []byte) { + u.mu.Lock() + defer u.mu.Unlock() + u.docs[path] = doc +} + +func (u *countingUpstream) setDown(path string, down bool) { + u.mu.Lock() + defer u.mu.Unlock() + u.down[path] = down +} + +func (u *countingUpstream) counts(path string) (requests, notModified int) { + u.mu.Lock() + defer u.mu.Unlock() + return u.requests[path], u.notModified[path] +} + +// cooldownCacheProxy returns a proxy with cooldown and the metadata cache on. +func cooldownCacheProxy(t *testing.T, u *countingUpstream) *Proxy { + t.Helper() + proxy, _, _, _ := setupTestProxy(t) + proxy.HTTPClient = u.Client() + config := &cooldown.Config{Default: "3d"} + // Config parses itself on first use, which isn't safe to race. + config.Enabled() + proxy.Cooldown = config + proxy.CacheMetadata = true + proxy.MetadataTTL = time.Hour + return proxy +} + +// expireCachedMetadata moves a metadata cache row's fetch time past the TTL. +func expireCachedMetadata(t *testing.T, proxy *Proxy, ecosystem, key string) { + t.Helper() + entry, err := proxy.DB.GetMetadataCache(ecosystem, key) + if err != nil || entry == nil { + t.Fatalf("metadata entry %s/%s: %v", ecosystem, key, err) + } + entry.FetchedAt.Time = time.Now().Add(-2 * proxy.MetadataTTL) + if err := proxy.DB.UpsertMetadataCache(entry); err != nil { + t.Fatalf("expiring metadata entry: %v", err) + } +} + +func getWith(t *testing.T, h http.Handler, path string, header http.Header) *httptest.ResponseRecorder { + t.Helper() + req := httptest.NewRequest(http.MethodGet, path, nil) + for k, vv := range header { + req.Header[k] = vv + } + w := httptest.NewRecorder() + h.ServeHTTP(w, req) + return w +} + +const ( + gemInfoPath = "/info/testgem" + gemVersionsPath = "/api/v1/versions/testgem.json" +) + +func gemVersionsJSON(t *testing.T, versions map[string]time.Duration) []byte { + t.Helper() + var list []gemVersion + for number, age := range versions { + list = append(list, gemVersion{Number: number, Platform: "ruby", CreatedAt: time.Now().Add(-age).Format(time.RFC3339)}) + } + b, err := json.Marshal(list) + if err != nil { + t.Fatal(err) + } + return b +} + +// gemCacheSetup serves testgem with 1.0.0 published a week ago and 2.0.0 an +// hour ago. +func gemCacheSetup(t *testing.T) (*countingUpstream, *Proxy, http.Handler) { + t.Helper() + u := newCountingUpstream(t) + u.set(gemInfoPath, []byte("---\n1.0.0 |checksum:a\n2.0.0 |checksum:b\n")) + u.set(gemVersionsPath, gemVersionsJSON(t, map[string]time.Duration{"1.0.0": 7 * 24 * time.Hour, "2.0.0": time.Hour})) + proxy := cooldownCacheProxy(t, u) + return u, proxy, NewGemHandlerWithUpstream(proxy, "http://proxy.test", u.URL).Routes() +} + +const gemFiltered = "---\n1.0.0 |checksum:a\n" + +func TestGemCooldownServesFromMetadataCache(t *testing.T) { + u, _, h := gemCacheSetup(t) + for i := range 3 { + w := getWith(t, h, gemInfoPath, nil) + if w.Code != http.StatusOK || w.Body.String() != gemFiltered { + t.Fatalf("request %d: status %d, body %q", i, w.Code, w.Body) + } + } + for _, path := range []string{gemInfoPath, gemVersionsPath} { + if n, _ := u.counts(path); n != 1 { + t.Errorf("%s requested %d times, want 1 within the TTL", path, n) + } + } +} + +func TestGemCooldownRevalidatesAfterTTL(t *testing.T) { + u, proxy, h := gemCacheSetup(t) + getWith(t, h, gemInfoPath, nil) + expireCachedMetadata(t, proxy, "gem", "info/testgem") + expireCachedMetadata(t, proxy, "gem", "api/v1/versions/testgem.json") + + w := getWith(t, h, gemInfoPath, nil) + if w.Code != http.StatusOK || w.Body.String() != gemFiltered { + t.Fatalf("status %d, body %q", w.Code, w.Body) + } + for _, path := range []string{gemInfoPath, gemVersionsPath} { + if n, nm := u.counts(path); n != 2 || nm != 1 { + t.Errorf("%s: %d requests, %d answered 304; want 2 and 1", path, n, nm) + } + } +} + +func TestGemCooldownConcurrentRequestsShareFetch(t *testing.T) { + u, proxy, h := gemCacheSetup(t) + u.gate = make(chan struct{}) + var release sync.Once + t.Cleanup(func() { release.Do(func() { close(u.gate) }) }) + + const clients = 5 + var wg sync.WaitGroup + codes := make([]int, clients) + bodies := make([]string, clients) + for i := range clients { + wg.Go(func() { + w := getWith(t, h, gemInfoPath, nil) + codes[i], bodies[i] = w.Code, w.Body.String() + }) + } + waitForMetadataWaiters(t, proxy, metadataCoalesceKey("gem", "info/testgem", u.URL+gemInfoPath, "*/*", "identity", false), clients-1) + waitForMetadataWaiters(t, proxy, metadataCoalesceKey("gem", "api/v1/versions/testgem.json", u.URL+gemVersionsPath, contentTypeJSON, "", true), clients-1) + release.Do(func() { close(u.gate) }) + wg.Wait() + + for i := range clients { + if codes[i] != http.StatusOK || bodies[i] != gemFiltered { + t.Errorf("client %d: status %d, body %q", i, codes[i], bodies[i]) + } + } + for _, path := range []string{gemInfoPath, gemVersionsPath} { + if n, _ := u.counts(path); n != 1 { + t.Errorf("%s requested %d times, want 1", path, n) + } + } +} + +func TestGemCooldownStaleCopyIsFiltered(t *testing.T) { + u, proxy, h := gemCacheSetup(t) + getWith(t, h, gemInfoPath, nil) + expireCachedMetadata(t, proxy, "gem", "info/testgem") + expireCachedMetadata(t, proxy, "gem", "api/v1/versions/testgem.json") + u.setDown(gemInfoPath, true) + u.setDown(gemVersionsPath, true) + + w := getWith(t, h, gemInfoPath, nil) + if w.Code != http.StatusOK || w.Body.String() != gemFiltered { + t.Fatalf("status %d, body %q; want the stale copy, filtered", w.Code, w.Body) + } + if got := w.Header().Get("Warning"); got != staleWarning { + t.Errorf("Warning = %q, want %q", got, staleWarning) + } +} + +func TestGemCooldownFailsClosedWithoutTimes(t *testing.T) { + u, _, h := gemCacheSetup(t) + u.setDown(gemVersionsPath, true) + + w := getWith(t, h, gemInfoPath, nil) + if w.Code != http.StatusBadGateway || strings.Contains(w.Body.String(), "1.0.0") { + t.Fatalf("status %d, body %q; want 502 without the index", w.Code, w.Body) + } + // The index alone was cached; it must not be served unfiltered either. + w = getWith(t, h, gemInfoPath, nil) + if w.Code != http.StatusBadGateway { + t.Fatalf("second request: status %d, want 502", w.Code) + } +} + +func TestGemCooldownClientRevalidates(t *testing.T) { + _, _, h := gemCacheSetup(t) + first := getWith(t, h, gemInfoPath, nil) + etag := first.Header().Get("ETag") + if etag == "" { + t.Fatal("filtered index served without an ETag") + } + w := getWith(t, h, gemInfoPath, http.Header{"If-None-Match": {etag}}) + if w.Code != http.StatusNotModified { + t.Fatalf("status %d, want 304", w.Code) + } +} + +// The index can be fetched again while the versions API response is still +// within its TTL. A version published in between is then in the index but +// has no publish time, and must not be let through for that. +func TestGemCooldownRefreshesTimesOlderThanIndex(t *testing.T) { + for _, timesDown := range []bool{false, true} { + name := "times refreshed" + if timesDown { + name = "times upstream down" + } + t.Run(name, func(t *testing.T) { + u, proxy, h := gemCacheSetup(t) + getWith(t, h, gemInfoPath, nil) + + u.set(gemInfoPath, []byte("---\n1.0.0 |checksum:a\n2.0.0 |checksum:b\n3.0.0 |checksum:c\n")) + u.set(gemVersionsPath, gemVersionsJSON(t, map[string]time.Duration{ + "1.0.0": 7 * 24 * time.Hour, "2.0.0": time.Hour, "3.0.0": time.Minute, + })) + u.setDown(gemVersionsPath, timesDown) + expireCachedMetadata(t, proxy, "gem", "info/testgem") + + w := getWith(t, h, gemInfoPath, nil) + if w.Code != http.StatusOK || w.Body.String() != gemFiltered { + t.Fatalf("status %d, body %q; want %q", w.Code, w.Body, gemFiltered) + } + if n, _ := u.counts(gemVersionsPath); n != 2 { + t.Errorf("versions API requested %d times, want 2", n) + } + if got := w.Header().Get("Warning"); (got != "") != timesDown { + t.Errorf("Warning = %q with times upstream down = %v", got, timesDown) + } + }) + } +} + +const ( + hexPackagePath = "/packages/testpkg" + hexAPIPath = "/api/packages/testpkg" +) + +func hexAPIJSON(t *testing.T, releases map[string]time.Duration) []byte { + t.Helper() + var api hexPackageAPI + for version, age := range releases { + api.Releases = append(api.Releases, hexRelease{Version: version, InsertedAt: time.Now().Add(-age).Format(time.RFC3339Nano)}) + } + b, err := json.Marshal(api) + if err != nil { + t.Fatal(err) + } + return b +} + +func hexCacheSetup(t *testing.T, recent time.Duration) (*countingUpstream, *Proxy, http.Handler, []byte) { + t.Helper() + u := newCountingUpstream(t) + signed := buildHexSigned(buildHexPackage("testpkg", []string{testVersion100, "2.0.0"})) + u.set(hexPackagePath, signed) + u.set(hexAPIPath, hexAPIJSON(t, map[string]time.Duration{testVersion100: 7 * 24 * time.Hour, "2.0.0": recent})) + proxy := cooldownCacheProxy(t, u) + return u, proxy, NewHexHandlerWithUpstreams(proxy, "http://proxy.test", u.URL, u.URL).Routes(), signed +} + +// hexVersions decodes the release versions of a served package. +func hexVersions(t *testing.T, body []byte) []string { + t.Helper() + zr, err := gzip.NewReader(bytes.NewReader(body)) + if err != nil { + t.Fatal(err) + } + signed, err := io.ReadAll(zr) + if err != nil { + t.Fatal(err) + } + payload, err := extractProtobufBytes(signed, 1) + if err != nil { + t.Fatal(err) + } + versions, err := packageReleaseVersions(payload) + if err != nil { + t.Fatal(err) + } + return versions +} + +func TestHexCooldownServesFromMetadataCache(t *testing.T) { + u, proxy, h, _ := hexCacheSetup(t, time.Hour) + for i := range 3 { + w := getWith(t, h, hexPackagePath, nil) + if w.Code != http.StatusOK { + t.Fatalf("request %d: status %d", i, w.Code) + } + if got := hexVersions(t, w.Body.Bytes()); len(got) != 1 || got[0] != testVersion100 { + t.Fatalf("request %d: versions %v, want [%s]", i, got, testVersion100) + } + if ce := w.Header().Get("Content-Encoding"); ce != "gzip" { + t.Errorf("Content-Encoding = %q, want gzip", ce) + } + } + for _, path := range []string{hexPackagePath, hexAPIPath} { + if n, _ := u.counts(path); n != 1 { + t.Errorf("%s requested %d times, want 1 within the TTL", path, n) + } + } + + expireCachedMetadata(t, proxy, "hex", "packages/testpkg") + expireCachedMetadata(t, proxy, "hex", hashedCacheKey(u.URL, "api/packages/testpkg")) + if w := getWith(t, h, hexPackagePath, nil); w.Code != http.StatusOK || len(hexVersions(t, w.Body.Bytes())) != 1 { + t.Fatalf("after TTL: status %d", w.Code) + } + for _, path := range []string{hexPackagePath, hexAPIPath} { + if n, nm := u.counts(path); n != 2 || nm != 1 { + t.Errorf("%s: %d requests, %d answered 304; want 2 and 1", path, n, nm) + } + } +} + +func TestHexCooldownStaleCopyIsFiltered(t *testing.T) { + u, proxy, h, _ := hexCacheSetup(t, time.Hour) + getWith(t, h, hexPackagePath, nil) + expireCachedMetadata(t, proxy, "hex", "packages/testpkg") + expireCachedMetadata(t, proxy, "hex", hashedCacheKey(u.URL, "api/packages/testpkg")) + u.setDown(hexPackagePath, true) + u.setDown(hexAPIPath, true) + + w := getWith(t, h, hexPackagePath, nil) + if w.Code != http.StatusOK { + t.Fatalf("status %d, want the stale copy", w.Code) + } + if got := hexVersions(t, w.Body.Bytes()); len(got) != 1 || got[0] != testVersion100 { + t.Errorf("versions %v, want [%s]", got, testVersion100) + } + if got := w.Header().Get("Warning"); got != staleWarning { + t.Errorf("Warning = %q, want %q", got, staleWarning) + } +} + +func TestHexCooldownFailsClosedWithoutTimes(t *testing.T) { + u, _, h, _ := hexCacheSetup(t, time.Hour) + u.setDown(hexAPIPath, true) + for i := range 2 { + if w := getWith(t, h, hexPackagePath, nil); w.Code != http.StatusBadGateway { + t.Fatalf("request %d: status %d, want 502", i, w.Code) + } + } +} + +func TestHexCooldownUnfilteredKeepsSignature(t *testing.T) { + _, _, h, signed := hexCacheSetup(t, 7*24*time.Hour) + w := getWith(t, h, hexPackagePath, nil) + if w.Code != http.StatusOK || !bytes.Equal(w.Body.Bytes(), signed) { + t.Fatalf("status %d; want upstream's signed bytes when nothing is in cooldown", w.Code) + } + etag := w.Header().Get("ETag") + if etag == "" { + t.Fatal("package served without an ETag") + } + if w := getWith(t, h, hexPackagePath, http.Header{"If-None-Match": {etag}}); w.Code != http.StatusNotModified { + t.Fatalf("revalidation: status %d, want 304", w.Code) + } +} + +const condaRepodataPath = "/conda-forge/noarch/repodata.json" + +func condaCacheSetup(t *testing.T) (*countingUpstream, *Proxy, *CondaHandler, http.Handler) { + t.Helper() + u := newCountingUpstream(t) + repodata, err := json.Marshal(map[string]any{ + "info": map[string]any{"subdir": "noarch"}, + "packages": map[string]any{ + "old-1.0.0-0.tar.bz2": map[string]any{"name": "old", "version": "1.0.0", "timestamp": time.Now().Add(-7 * 24 * time.Hour).UnixMilli()}, + "new-2.0.0-0.tar.bz2": map[string]any{"name": "new", "version": "2.0.0", "timestamp": time.Now().Add(-time.Hour).UnixMilli()}, + }, + "packages.conda": map[string]any{}, + }) + if err != nil { + t.Fatal(err) + } + u.set(condaRepodataPath, repodata) + u.gzipped[condaRepodataPath] = true + proxy := cooldownCacheProxy(t, u) + h := NewCondaHandlerWithUpstream(proxy, "http://proxy.test", u.URL) + return u, proxy, h, h.Routes() +} + +func condaPackages(t *testing.T, w *httptest.ResponseRecorder) []string { + t.Helper() + if w.Code != http.StatusOK { + t.Fatalf("status %d, body %q", w.Code, w.Body) + } + var doc struct { + Packages map[string]json.RawMessage `json:"packages"` + } + if err := json.Unmarshal(w.Body.Bytes(), &doc); err != nil { + t.Fatalf("decoding repodata: %v", err) + } + var names []string + for name := range doc.Packages { + names = append(names, name) + } + return names +} + +func assertCondaFiltered(t *testing.T, w *httptest.ResponseRecorder) { + t.Helper() + if got := condaPackages(t, w); len(got) != 1 || got[0] != "old-1.0.0-0.tar.bz2" { + t.Fatalf("packages %v, want only old-1.0.0-0.tar.bz2", got) + } +} + +func TestCondaCooldownServesFromMetadataCache(t *testing.T) { + u, proxy, h, routes := condaCacheSetup(t) + for range 3 { + assertCondaFiltered(t, getWith(t, routes, condaRepodataPath, nil)) + } + if n, _ := u.counts(condaRepodataPath); n != 1 { + t.Errorf("repodata requested %d times, want 1 within the TTL", n) + } + + expireCachedMetadata(t, proxy, "conda", h.metadataCacheKey(condaRepodataPath)) + assertCondaFiltered(t, getWith(t, routes, condaRepodataPath, nil)) + if n, nm := u.counts(condaRepodataPath); n != 2 || nm != 1 { + t.Errorf("%d requests, %d answered 304; want 2 and 1", n, nm) + } +} + +func TestCondaCooldownStaleCopyIsFiltered(t *testing.T) { + u, proxy, h, routes := condaCacheSetup(t) + assertCondaFiltered(t, getWith(t, routes, condaRepodataPath, nil)) + expireCachedMetadata(t, proxy, "conda", h.metadataCacheKey(condaRepodataPath)) + u.setDown(condaRepodataPath, true) + + w := getWith(t, routes, condaRepodataPath, nil) + assertCondaFiltered(t, w) + if got := w.Header().Get("Warning"); got != staleWarning { + t.Errorf("Warning = %q, want %q", got, staleWarning) + } +} + +func TestCondaCooldownConcurrentRequestsShareFetch(t *testing.T) { + u, proxy, h, routes := condaCacheSetup(t) + u.gate = make(chan struct{}) + var release sync.Once + t.Cleanup(func() { release.Do(func() { close(u.gate) }) }) + + const clients = 5 + var wg sync.WaitGroup + recorders := make([]*httptest.ResponseRecorder, clients) + for i := range clients { + wg.Go(func() { recorders[i] = getWith(t, routes, condaRepodataPath, nil) }) + } + key := metadataCoalesceKey("conda", h.metadataCacheKey(condaRepodataPath), u.URL+condaRepodataPath, "*/*", "gzip", false) + waitForMetadataWaiters(t, proxy, key, clients-1) + release.Do(func() { close(u.gate) }) + wg.Wait() + + for _, w := range recorders { + assertCondaFiltered(t, w) + } + if n, _ := u.counts(condaRepodataPath); n != 1 { + t.Errorf("repodata requested %d times, want 1", n) + } +} + +func TestCondaCooldownClientRevalidates(t *testing.T) { + _, _, h, routes := condaCacheSetup(t) + // The first copy is streamed as it is filtered, before its ETag is known. + assertCondaFiltered(t, getWith(t, routes, condaRepodataPath, nil)) + w := getWith(t, routes, condaRepodataPath, nil) + assertCondaFiltered(t, w) + etag := w.Header().Get("ETag") + if etag == "" { + t.Fatal("filtered repodata served without an ETag") + } + if w := getWith(t, routes, condaRepodataPath, http.Header{"If-None-Match": {etag}}); w.Code != http.StatusNotModified { + t.Fatalf("revalidation: status %d, want 304", w.Code) + } + + // Once the entry left out leaves cooldown, the copy changes and so does + // its ETag. + key := h.metadataCacheKey(condaRepodataPath) + h.repodataMu.Lock() + state := h.repodataStates[key] + h.repodataMu.Unlock() + if want := time.Now().Add(-time.Hour + 3*24*time.Hour); state.until.Sub(want).Abs() > time.Minute { + t.Errorf("ETag kept until %v, want about %v", state.until, want) + } + req := httptest.NewRequest(http.MethodGet, condaRepodataPath, nil) + req.Header.Set("If-None-Match", etag) + if h.repodataNotModified(httptest.NewRecorder(), req, key, state.digest, state.until) { + t.Error("ETag still matched after the withheld entry left cooldown") + } +} + +func TestCondaCooldownWithoutMetadataCacheStreams(t *testing.T) { + u, proxy, _, routes := condaCacheSetup(t) + proxy.CacheMetadata = false + for range 2 { + assertCondaFiltered(t, getWith(t, routes, condaRepodataPath, nil)) + } + if n, _ := u.counts(condaRepodataPath); n != 2 { + t.Errorf("repodata requested %d times, want 2 without cache_metadata", n) + } +} + +// A cached repodata.json is filtered as it is read from storage: the +// response starts while the stored document is still being read. +func TestCondaCooldownStreamsCachedRepodata(t *testing.T) { + u, proxy, _, routes := condaCacheSetup(t) + packages := map[string]any{} + for i := range 20000 { + packages[fmt.Sprintf("pkg%d-1.0.0-0.tar.bz2", i)] = map[string]any{ + "name": fmt.Sprintf("pkg%d", i), "version": "1.0.0", "timestamp": time.Now().Add(-7 * 24 * time.Hour).UnixMilli(), + } + } + repodata, err := json.Marshal(map[string]any{"info": map[string]any{}, "packages": packages}) + if err != nil { + t.Fatal(err) + } + u.set(condaRepodataPath, repodata) + // Stored uncompressed, so the document is larger than the part the + // storage hands out before stalling. + u.gzipped[condaRepodataPath] = false + if w := getWith(t, routes, condaRepodataPath, nil); w.Code != http.StatusOK { + t.Fatalf("populating cache: status %d", w.Code) + } + + gate := &stallingStorage{Storage: proxy.Storage, head: 256 << 10, release: make(chan struct{})} + proxy.Storage = gate + var release sync.Once + t.Cleanup(func() { release.Do(func() { close(gate.release) }) }) + srv := httptest.NewServer(routes) + t.Cleanup(srv.Close) + + type result struct { + resp *http.Response + err error + } + started := make(chan result, 1) + go func() { + resp, err := http.Get(srv.URL + condaRepodataPath) + if err == nil { + _, err = io.ReadFull(resp.Body, make([]byte, 1024)) + } + started <- result{resp, err} + }() + var res result + select { + case res = <-started: + case <-time.After(5 * time.Second): + release.Do(func() { close(gate.release) }) + if res := <-started; res.resp != nil { + _ = res.resp.Body.Close() + } + t.Fatal("response did not start before the cached document was read to its end") + } + release.Do(func() { close(gate.release) }) + if res.err != nil { + t.Fatal(res.err) + } + _, _ = io.Copy(io.Discard, res.resp.Body) + _ = res.resp.Body.Close() + if n, _ := u.counts(condaRepodataPath); n != 1 { + t.Errorf("repodata requested %d times, want 1", n) + } +} diff --git a/internal/handler/cooldown_times.go b/internal/handler/cooldown_times.go new file mode 100644 index 0000000..88532cd --- /dev/null +++ b/internal/handler/cooldown_times.go @@ -0,0 +1,149 @@ +package handler + +import ( + "bytes" + "compress/gzip" + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "strings" + "time" +) + +// publishTimes maps each version a times document lists to its publish +// time. A version whose time didn't parse maps to the zero time and is let +// through. +type publishTimes map[string]time.Time + +// timedMetadata describes metadata whose publish times live in a second +// document, as with the RubyGems compact index and the versions API, or a +// Hex package and the Hex HTTP API. Both are read through the metadata cache. +type timedMetadata struct { + ecosystem string + // docKey and docURL are the cache key and upstream URL of the document + // clients read. They match the plain metadata path's, so both share one + // cached copy. + docKey, docURL string + // timesKey and timesURL are those of the document with publish times. + timesKey, timesURL string + timesAccept string + parseTimes func([]byte) (publishTimes, error) +} + +// timedDocuments is what fetchTimedMetadata hands back. published is read +// from times and is nil when times.err is set. +type timedDocuments struct { + doc metadataResult + times metadataResult + published publishTimes + // withholdUnknown is set when doc lists versions the times document + // doesn't, and that document is a stale copy: those versions are newer + // than the copy, so they are left out rather than let through. + withholdUnknown bool +} + +// fetchTimedMetadata fetches both documents concurrently. The document +// clients read is fetched as the plain metadata path fetches it, with +// identity encoding and Accept */*. +// +// Each document has its own TTL, so a cached times document can be older +// than the document it is used to filter, and miss versions published in +// between. Those versions would be let through as having no publish time. +// listed returns the versions the document lists; when some are missing +// from a times document read from the cache, the times document is fetched +// again before filtering. +func (p *Proxy) fetchTimedMetadata(ctx context.Context, t timedMetadata, listed func(metadataResult) ([]string, error)) (timedDocuments, error) { + var res timedDocuments + var published publishTimes + validate := func(body []byte) error { + parsed, err := t.parseTimes(body) + if err == nil { + published = parsed + } + return err + } + + done := make(chan struct{}) + go func() { + defer close(done) + res.times = p.lookupMetadata(ctx, t.ecosystem, t.timesKey, t.timesURL, t.timesAccept, "", validate) + }() + res.doc = p.lookupMetadata(ctx, t.ecosystem, t.docKey, t.docURL, "*/*", "identity", nil) + <-done + + if res.doc.err != nil || res.times.err != nil { + return res, nil + } + versions, err := listed(res.doc) + if err != nil { + return res, err + } + unknown := func() bool { + for _, v := range versions { + if _, ok := published[v]; !ok { + return true + } + } + return false + } + if unknown() && res.times.fromCache { + p.Logger.Info("cooldown: publish times older than metadata, fetching them again", + "ecosystem", t.ecosystem, "key", t.timesKey) + res.times = p.refreshMetadata(ctx, t.ecosystem, t.timesKey, t.timesURL, t.timesAccept, "", validate) + if res.times.err != nil { + return res, nil + } + } + res.published = published + res.withholdUnknown = res.times.stale && unknown() + return res, nil +} + +// rewriteDigest identifies the inputs of a cooldown rewrite for the rewrite +// cache: both documents and whether unknown versions are withheld. +func (d timedDocuments) rewriteDigest() string { + h := sha256.New() + for _, res := range []metadataResult{d.doc, d.times} { + digest := res.digest + if digest == "" { + sum := sha256.Sum256(res.body) + digest = "sha256:" + hex.EncodeToString(sum[:]) + } + _, _ = fmt.Fprintf(h, "%d:%s", len(digest), digest) + } + _, _ = fmt.Fprintf(h, "%t", d.withholdUnknown) + return "sha256:" + hex.EncodeToString(h.Sum(nil)) +} + +// withheld reports whether cooldown leaves version out, given the +// package's publish times. +func (d timedDocuments) withheld(p *Proxy, ecosystem, packagePURL, version string) bool { + published, ok := d.published[version] + if !ok { + return d.withholdUnknown + } + if published.IsZero() { + return false + } + return !p.Cooldown.IsAllowed(ecosystem, packagePURL, published) +} + +// decodedMetadataBody returns res's body without its Content-Encoding. The +// documents are fetched with identity encoding, so an upstream that sends +// gzip anyway is the only case to handle. +func (p *Proxy) decodedMetadataBody(res metadataResult) ([]byte, error) { + switch strings.ToLower(strings.TrimSpace(res.contentEncoding)) { + case "", "identity": + return res.body, nil + case "gzip", "x-gzip": + zr, err := gzip.NewReader(bytes.NewReader(res.body)) + if err != nil { + return nil, err + } + defer func() { _ = zr.Close() }() + return p.ReadMetadata(zr) + default: + return nil, fmt.Errorf("unsupported content encoding %q", res.contentEncoding) + } +} diff --git a/internal/handler/gem.go b/internal/handler/gem.go index dbb6e08..edae77d 100644 --- a/internal/handler/gem.go +++ b/internal/handler/gem.go @@ -6,8 +6,6 @@ import ( "fmt" "net/http" "regexp" - "slices" - "strconv" "strings" "time" @@ -144,6 +142,12 @@ func gemVersionAndPlatform(s string) bool { // handleCompactIndex serves the compact index for a gem, filtering versions // based on cooldown when enabled. +// +// With cooldown the index and the versions API's publish times are read +// through the metadata cache, so a bundle install within metadata_ttl of the +// last one asks rubygems.org nothing, and one after it revalidates both with +// their ETags. The filtered index is kept in the rewrite cache and served +// with an ETag of its own. func (h *GemHandler) handleCompactIndex(w http.ResponseWriter, r *http.Request) { name := r.PathValue("name") if name == "" { @@ -157,122 +161,118 @@ func (h *GemHandler) handleCompactIndex(w http.ResponseWriter, r *http.Request) h.proxy.Logger.Info("gem compact index request with cooldown", "name", name) - indexResp, filteredVersions, versionsErr, err := h.fetchIndexAndVersions(r, name) - if err != nil { - if writeUpstreamHeld(w, err) { - return - } - h.proxy.Logger.Error("upstream compact index request failed", "error", err) - http.Error(w, "upstream request failed", http.StatusBadGateway) + docs, err := h.proxy.fetchTimedMetadata(r.Context(), h.timedIndex(name), h.indexVersions) + switch { + case docs.doc.err != nil: + h.proxy.writeMetadataFetchError(w, docs.doc.err) return - } - defer func() { _ = indexResp.Body.Close() }() - - if indexResp.StatusCode != http.StatusOK { - h.proxy.relayResponse(w, r, indexResp, nil) + case docs.times.err != nil: + h.proxy.writeCooldownLookupError(w, "gem", name, docs.times.err) return - } - - if versionsErr != nil { - h.proxy.writeCooldownLookupError(w, "gem", name, versionsErr) - return - } - - h.writeFilteredIndex(w, indexResp, name, filteredVersions) -} - -// fetchIndexAndVersions fetches the compact index and versions API -// concurrently. It returns the index response and the set of versions to -// filter, the versions API's error, and the index fetch's error. -func (h *GemHandler) fetchIndexAndVersions(r *http.Request, name string) (*http.Response, map[string]bool, error, error) { - type versionsResult struct { - filtered map[string]bool - err error - } - - versionsCh := make(chan versionsResult, 1) - go func() { - filtered, err := h.fetchFilteredVersions(r, name) - versionsCh <- versionsResult{filtered: filtered, err: err} - }() - - indexResp, err := h.fetchCompactIndex(r, name) - - versionsRes := <-versionsCh - - if err != nil { - return nil, nil, nil, err - } - return indexResp, versionsRes.filtered, versionsRes.err, nil -} - -// fetchCompactIndex fetches the compact index from upstream. -func (h *GemHandler) fetchCompactIndex(r *http.Request, name string) (*http.Response, error) { - indexURL := h.upstreamURL + "/info/" + name - req, err := http.NewRequestWithContext(r.Context(), http.MethodGet, indexURL, nil) - if err != nil { - return nil, err - } - // Only Accept is passed on. The client's Accept-Encoding would turn off - // the transport's transparent decompression and leave a gzip body to - // filter, and its validators belong to the filtered copy it was served, - // not to upstream's file. - if v := r.Header.Get(headerAccept); v != "" { - req.Header.Set(headerAccept, v) - } - return h.proxy.HTTPClient.Do(req) -} - -// gemFilteredIndexDroppedHeaders describe upstream's file as a whole, so -// they are wrong for a copy with versions removed. Bundler checks the digest -// headers against the body it receives. -var gemFilteredIndexDroppedHeaders = []string{ - headerContentLength, headerContentEncoding, headerETag, headerLastModified, - "Digest", "Repr-Digest", "Content-Digest", "Accept-Ranges", "Content-Range", -} - -// writeFilteredIndex writes the compact index response with cooldown-filtered -// versions removed. The body is read in full first, so a read error or an -// oversized index answers 502 instead of a truncated 200. -func (h *GemHandler) writeFilteredIndex(w http.ResponseWriter, resp *http.Response, name string, filtered map[string]bool) { - body, err := h.proxy.ReadMetadata(resp.Body) - if err != nil { + case err != nil: h.proxy.Logger.Error("failed to read gem compact index", "gem", name, "error", err) http.Error(w, "failed to read upstream response", http.StatusBadGateway) return } + packagePURL := canonicalPackagePURL("gem", name) + m, err := h.proxy.cachedRewrite(r.Context(), "gem", h.proxyURL, "info/"+name, docs.doc.body, docs.rewriteDigest(), + func([]byte) ([]byte, []string, error) { + body, err := h.proxy.decodedMetadataBody(docs.doc) + if err != nil { + return nil, nil, err + } + out, kept := h.filterIndex(body, name, func(version string) bool { + return docs.withheld(h.proxy, "gem", packagePURL, version) + }) + return out, kept, nil + }) + if err != nil { + h.proxy.Logger.Error("failed to filter gem compact index", "gem", name, "error", err) + http.Error(w, "failed to read upstream response", http.StatusBadGateway) + return + } + + // Served as identity, as upstream's index was: Bundler checks digest + // headers against the bytes it receives, and asks for ranges of it. + m.gzipped = nil + contentType := docs.doc.contentType + if contentType == "" || contentType == contentTypeJSON { + contentType = "text/plain; charset=utf-8" + } + markStale(w, docs.doc.stale || docs.times.stale) + serveRewrittenMetadataAs(w, r, m, contentType) +} + +// timedIndex describes a gem's compact index and the versions API it takes +// publish times from. +func (h *GemHandler) timedIndex(name string) timedMetadata { + return timedMetadata{ + ecosystem: "gem", + docKey: "info/" + name, + docURL: h.upstreamURL + "/info/" + name, + timesKey: "api/v1/versions/" + name + ".json", + timesURL: fmt.Sprintf("%s/api/v1/versions/%s.json", h.upstreamURL, name), + timesAccept: contentTypeJSON, + parseTimes: parseGemVersions, + } +} + +// indexVersions lists the versions in a compact index, as the versions API +// names them. +func (h *GemHandler) indexVersions(doc metadataResult) ([]string, error) { + body, err := h.proxy.decodedMetadataBody(doc) + if err != nil { + return nil, err + } + var versions []string + for rest := body; len(rest) > 0; { + line, tail, _ := bytes.Cut(rest, []byte("\n")) + rest = tail + if version := gemIndexLineVersion(line); gemIndexListsVersion(version) { + versions = append(versions, string(version)) + } + } + return versions, nil +} + +// gemIndexLineVersion returns the version a compact index line starts with. +func gemIndexLineVersion(line []byte) []byte { + if spaceIdx := bytes.IndexByte(line, ' '); spaceIdx > 0 { + return line[:spaceIdx] + } + return line +} + +// gemIndexListsVersion reports whether a compact index line's first field +// is a version, rather than the "---" that starts the file or an empty line. +func gemIndexListsVersion(field []byte) bool { + return len(field) > 0 && string(field) != "---" +} + +// filterIndex returns the compact index without the lines of versions +// withheld reports, and the versions it kept. +func (h *GemHandler) filterIndex(body []byte, name string, withheld func(string) bool) ([]byte, []string) { + var kept []string var out bytes.Buffer out.Grow(len(body)) for rest := body; len(rest) > 0; { line, tail, _ := bytes.Cut(rest, []byte("\n")) rest = tail - version := line - if spaceIdx := bytes.IndexByte(line, ' '); spaceIdx > 0 { - version = line[:spaceIdx] + version := gemIndexLineVersion(line) + if gemIndexListsVersion(version) { + if withheld(string(version)) { + h.proxy.Logger.Info("cooldown: filtering gem version", + "gem", name, "version", string(version)) + continue + } + kept = append(kept, string(version)) } - if filtered[string(version)] { - h.proxy.Logger.Info("cooldown: filtering gem version", - "gem", name, "version", string(version)) - continue - } - out.Write(line) out.WriteByte('\n') } - - for k, vv := range resp.Header { - if slices.ContainsFunc(gemFilteredIndexDroppedHeaders, func(d string) bool { return strings.EqualFold(k, d) }) { - continue - } - for _, v := range vv { - w.Header().Add(k, v) - } - } - w.Header().Set(headerContentLength, strconv.Itoa(out.Len())) - w.WriteHeader(http.StatusOK) - _, _ = w.Write(out.Bytes()) + return out.Bytes(), kept } // gemVersion represents a version entry from the RubyGems versions API. @@ -282,54 +282,26 @@ type gemVersion struct { CreatedAt string `json:"created_at"` } -// fetchFilteredVersions fetches the versions API and returns a set of version -// strings that should be filtered out by cooldown. -func (h *GemHandler) fetchFilteredVersions(r *http.Request, name string) (map[string]bool, error) { - versionsURL := fmt.Sprintf("%s/api/v1/versions/%s.json", h.upstreamURL, name) - req, err := http.NewRequestWithContext(r.Context(), http.MethodGet, versionsURL, nil) - if err != nil { - return nil, err - } - - resp, err := h.proxy.HTTPClient.Do(req) - if err != nil { - return nil, err - } - defer func() { _ = resp.Body.Close() }() - - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("versions API returned %d", resp.StatusCode) - } - - body, err := h.proxy.ReadMetadata(resp.Body) - if err != nil { - return nil, err - } +// parseGemVersions reads the publish times from a versions API response, +// keyed by version as the compact index writes it. +func parseGemVersions(body []byte) (publishTimes, error) { var versions []gemVersion if err := json.Unmarshal(body, &versions); err != nil { return nil, err } - - packagePURL := canonicalPackagePURL("gem", name) - filtered := make(map[string]bool) - + times := make(publishTimes, len(versions)) for _, v := range versions { + versionStr := v.Number + if v.Platform != "" && v.Platform != "ruby" { + versionStr = v.Number + "-" + v.Platform + } createdAt, err := time.Parse(time.RFC3339, v.CreatedAt) if err != nil { - continue - } - - if !h.proxy.Cooldown.IsAllowed("gem", packagePURL, createdAt) { - // Build version string matching compact index format - versionStr := v.Number - if v.Platform != "" && v.Platform != "ruby" { - versionStr = v.Number + "-" + v.Platform - } - filtered[versionStr] = true + createdAt = time.Time{} } + times[versionStr] = createdAt } - - return filtered, nil + return times, nil } // proxyCached forwards a metadata request with caching. diff --git a/internal/handler/gem_test.go b/internal/handler/gem_test.go index a13d819..ba62ef8 100644 --- a/internal/handler/gem_test.go +++ b/internal/handler/gem_test.go @@ -196,10 +196,11 @@ func TestGemCompactIndexCooldownClientAcceptsGzip(t *testing.T) { if got := w.Body.String(); got != want { t.Errorf("body = %q, want %q", got, want) } - for _, hdr := range []string{"ETag", "Repr-Digest"} { - if v := w.Header().Get(hdr); v != "" { - t.Errorf("%s = %q, upstream's value must not describe the filtered index", hdr, v) - } + if v := w.Header().Get("Repr-Digest"); v != "" { + t.Errorf("Repr-Digest = %q, upstream's value must not describe the filtered index", v) + } + if v := w.Header().Get("ETag"); v == "" || v == `"upstream"` { + t.Errorf("ETag = %q, want one of the filtered index", v) } } @@ -351,7 +352,7 @@ func TestGemCompactIndexVersionsAPIFails(t *testing.T) { } } -func TestGemFetchFilteredVersions(t *testing.T) { +func TestGemParseVersions(t *testing.T) { now := time.Now() oldTime := now.Add(-7 * 24 * time.Hour).Format(time.RFC3339) recentTime := now.Add(-1 * time.Hour).Format(time.RFC3339) @@ -360,39 +361,22 @@ func TestGemFetchFilteredVersions(t *testing.T) { {Number: "1.0.0", Platform: "ruby", CreatedAt: oldTime}, {Number: "2.0.0", Platform: "ruby", CreatedAt: recentTime}, {Number: "2.0.0", Platform: "java", CreatedAt: recentTime}, + {Number: "3.0.0", Platform: "ruby", CreatedAt: "not a time"}, }) - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Content-Type", "application/json") - _, _ = w.Write(versionsJSON) - })) - defer upstream.Close() - proxy := testProxy() - proxy.Cooldown = &cooldown.Config{ - Default: "3d", - } + proxy.Cooldown = &cooldown.Config{Default: "3d"} - h := &GemHandler{ - proxy: proxy, - upstreamURL: upstream.URL, - proxyURL: "http://proxy.local", - } - - req := httptest.NewRequest(http.MethodGet, "/info/testgem", nil) - filtered, err := h.fetchFilteredVersions(req, "testgem") + times, err := parseGemVersions(versionsJSON) if err != nil { t.Fatal(err) } - - if filtered["1.0.0"] { - t.Error("version 1.0.0 should not be filtered (old enough)") - } - if !filtered["2.0.0"] { - t.Error("version 2.0.0 (ruby) should be filtered") - } - if !filtered["2.0.0-java"] { - t.Error("version 2.0.0-java should be filtered") + docs := timedDocuments{published: times} + purl := canonicalPackagePURL("gem", "testgem") + for version, want := range map[string]bool{"1.0.0": false, "2.0.0": true, "2.0.0-java": true, "3.0.0": false, "4.0.0": false} { + if got := docs.withheld(proxy, "gem", purl, version); got != want { + t.Errorf("withheld(%s) = %v, want %v", version, got, want) + } } _ = fmt.Sprintf // silence unused import diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 2e0d688..9c78ad4 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -1479,7 +1479,7 @@ func (p *Proxy) cachedMetadataState(ctx context.Context, ecosystem, cacheKey str if entry != nil && p.MetadataTTL > 0 && entry.FetchedAt.Valid && time.Since(entry.FetchedAt.Time) < p.MetadataTTL { data, ct, err := p.readCachedMetadata(ctx, entry, validate) if err == nil { - hit := metadataResult{body: data, contentType: ct, contentEncoding: entry.ContentEncoding.String}.describedBy(entry) + hit := metadataResult{body: data, contentType: ct, contentEncoding: entry.ContentEncoding.String, fromCache: true}.describedBy(entry) return entry, &hit } if validate != nil { @@ -1490,6 +1490,31 @@ func (p *Proxy) cachedMetadataState(ctx context.Context, ecosystem, cacheKey str return entry, nil } +// refreshMetadata is lookupMetadata for a document whose cached copy is known +// to be out of date although it is within its TTL. It skips the TTL and asks +// upstream, revalidating the cached copy with its validators, and falls back +// to that copy if upstream fails, which the result reports as stale. +// Concurrent refreshes of one document share a fetch. +func (p *Proxy) refreshMetadata(ctx context.Context, ecosystem, cacheKey, upstreamURL, accept, acceptEncoding string, validate func([]byte) error) metadataResult { + if containsPathTraversal(cacheKey) { + return metadataResult{err: fmt.Errorf("invalid cache key: %q", cacheKey)} + } + key := metadataCoalesceKey(ecosystem, cacheKey, upstreamURL, accept, acceptEncoding, validate != nil) + "\x00refresh" + res, shared := p.coalesceMetadata(ctx, key, func(fetchCtx context.Context) metadataResult { + var entry *database.MetadataCacheEntry + if p.CacheMetadata && p.DB != nil { + entry, _ = p.DB.GetMetadataCache(ecosystem, cacheKey) + } + return p.fetchMetadataFromUpstream(fetchCtx, ecosystem, cacheKey, upstreamURL, accept, acceptEncoding, validate, entry) + }) + if shared && res.err == nil && validate != nil { + if err := validate(res.body); err != nil { + return metadataResult{err: err} + } + } + return res +} + // fetchMetadataFromUpstream fetches metadata after a cache miss, caches it, // and falls back to the cached copy if upstream fails. entry is the row to // revalidate against, or nil. @@ -1558,6 +1583,9 @@ type metadataResult struct { // stale is set when the body is the cached copy served because upstream // failed. stale bool + // fromCache is set when the body is a stored copy within its TTL, read + // without asking upstream. + fromCache bool } // describedBy returns res with the digest that entry, the cache row its body diff --git a/internal/handler/hex.go b/internal/handler/hex.go index 2d86686..a46a7e4 100644 --- a/internal/handler/hex.go +++ b/internal/handler/hex.go @@ -101,6 +101,11 @@ const hexAPIURL = "https://hex.pm" // handlePackages proxies the /packages/{name} endpoint, applying cooldown filtering // when enabled. Since the protobuf format has no timestamps, we fetch them from the // Hex HTTP API concurrently. +// +// Both are read through the metadata cache, so within metadata_ttl a package +// costs no request to repo.hex.pm or to the rate limited Hex API, and after +// it both are revalidated with their ETags. The filtered package is kept in +// the rewrite cache and served with an ETag of its own. func (h *HexHandler) handlePackages(w http.ResponseWriter, r *http.Request) { name := r.PathValue("name") if name == "" || !h.proxy.cooldownApplies("hex", name) { @@ -110,87 +115,102 @@ func (h *HexHandler) handlePackages(w http.ResponseWriter, r *http.Request) { h.proxy.Logger.Info("hex package request with cooldown", "name", name) - protoResp, filteredVersions, versionsErr, err := h.fetchPackageAndVersions(r, name) - if err != nil { - if writeUpstreamHeld(w, err) { - return - } - h.proxy.Logger.Error("upstream request failed", "error", err) - http.Error(w, "upstream request failed", http.StatusBadGateway) + docs, err := h.proxy.fetchTimedMetadata(r.Context(), h.timedPackage(name), h.packageVersions) + switch { + case docs.doc.err != nil: + h.proxy.writeMetadataFetchError(w, docs.doc.err) return - } - defer func() { _ = protoResp.Body.Close() }() - - if protoResp.StatusCode != http.StatusOK { - h.proxy.relayResponse(w, r, protoResp, nil) + case docs.times.err != nil: + h.proxy.writeCooldownLookupError(w, "hex", name, docs.times.err) return - } - - if versionsErr != nil { - h.proxy.writeCooldownLookupError(w, "hex", name, versionsErr) - return - } - - body, err := h.proxy.ReadMetadata(protoResp.Body) - if err != nil { + case err != nil: h.proxy.Logger.Error("failed to read hex package", "name", name, "error", err) - http.Error(w, "failed to read response", http.StatusBadGateway) + http.Error(w, "failed to filter package metadata", http.StatusBadGateway) return } - if len(filteredVersions) == 0 { - // No versions in cooldown, pass through - w.Header().Set(headerContentType, protoResp.Header.Get(headerContentType)) - w.Header().Set("Content-Encoding", "gzip") - _, _ = w.Write(body) - return - } - - filtered, err := h.filterSignedPackage(body, filteredVersions) + packagePURL := canonicalPackagePURL("hex", name) + m, err := h.proxy.cachedRewrite(r.Context(), "hex", h.proxyURL, "packages/"+name, docs.doc.body, docs.rewriteDigest(), + func([]byte) ([]byte, []string, error) { + body, err := h.proxy.decodedMetadataBody(docs.doc) + if err != nil { + return nil, nil, err + } + versions, err := h.packageVersions(docs.doc) + if err != nil { + return nil, nil, err + } + filtered := make(map[string]bool) + var kept []string + for _, version := range versions { + if docs.withheld(h.proxy, "hex", packagePURL, version) { + filtered[version] = true + h.proxy.Logger.Info("cooldown: filtering hex version", "package", name, "version", version) + continue + } + kept = append(kept, version) + } + if len(filtered) == 0 { + // Nothing in cooldown: upstream's bytes, signature included. + return body, kept, nil + } + out, err := h.filterSignedPackage(body, filtered) + return out, kept, err + }) if err != nil { h.proxy.Logger.Error("failed to filter hex package", "name", name, "error", err) http.Error(w, "failed to filter package metadata", http.StatusBadGateway) return } + markStale(w, docs.doc.stale || docs.times.stale) + serveHexPackage(w, r, m) +} + +// serveHexPackage writes a package with its validators, or 304 when they +// show the client's copy is current. The registry file is gzip itself, and +// Hex clients gunzip it however it is labelled, so it is never compressed +// again. +func serveHexPackage(w http.ResponseWriter, r *http.Request, m rewrittenMetadata) { + w.Header().Set(headerETag, m.etag) + if !m.lastModified.IsZero() { + w.Header().Set(headerLastModified, m.lastModified.UTC().Format(http.TimeFormat)) + } + if notModified(r, m.etag, m.lastModified) { + w.WriteHeader(http.StatusNotModified) + return + } w.Header().Set(headerContentType, "application/octet-stream") - w.Header().Set("Content-Encoding", "gzip") - _, _ = w.Write(filtered) + w.Header().Set(headerContentEncoding, "gzip") + w.WriteHeader(http.StatusOK) + _, _ = w.Write(m.body) } -// fetchPackageAndVersions fetches the protobuf package and version timestamps -// concurrently. It returns the package response and the set of versions to -// filter, the Hex API's error, and the package fetch's error. -func (h *HexHandler) fetchPackageAndVersions(r *http.Request, name string) (*http.Response, map[string]bool, error, error) { - type versionsResult struct { - filtered map[string]bool - err error +// timedPackage describes a package's registry file and the Hex API entry +// it takes publish times from. +func (h *HexHandler) timedPackage(name string) timedMetadata { + return timedMetadata{ + ecosystem: "hex", + docKey: "packages/" + name, + docURL: h.upstreamURL + "/packages/" + name, + timesKey: hashedCacheKey(h.apiURL, "api/packages/"+name), + timesURL: fmt.Sprintf("%s/api/packages/%s", h.apiURL, name), + timesAccept: contentTypeJSON, + parseTimes: parseHexReleases, } - - versionsCh := make(chan versionsResult, 1) - go func() { - filtered, err := h.fetchFilteredVersions(r, name) - versionsCh <- versionsResult{filtered: filtered, err: err} - }() - - protoResp, err := h.fetchUpstreamPackage(r, name) - - versionsRes := <-versionsCh - - if err != nil { - return nil, nil, nil, err - } - return protoResp, versionsRes.filtered, versionsRes.err, nil } -// fetchUpstreamPackage fetches the protobuf package from upstream. -func (h *HexHandler) fetchUpstreamPackage(r *http.Request, name string) (*http.Response, error) { - upstreamURL := h.upstreamURL + "/packages/" + name - req, err := http.NewRequestWithContext(r.Context(), http.MethodGet, upstreamURL, nil) +// packageVersions lists the releases in a gzipped Signed package. +func (h *HexHandler) packageVersions(doc metadataResult) ([]string, error) { + body, err := h.proxy.decodedMetadataBody(doc) if err != nil { return nil, err } - return h.proxy.HTTPClient.Do(req) + payload, err := h.signedPayload(body) + if err != nil { + return nil, err + } + return packageReleaseVersions(payload) } // hexRelease represents a version entry from the Hex API. @@ -204,60 +224,26 @@ type hexPackageAPI struct { Releases []hexRelease `json:"releases"` } -// fetchFilteredVersions fetches the Hex API and returns a set of version -// strings that should be filtered out by cooldown. -func (h *HexHandler) fetchFilteredVersions(r *http.Request, name string) (map[string]bool, error) { - apiURL := fmt.Sprintf("%s/api/packages/%s", h.apiURL, name) - req, err := http.NewRequestWithContext(r.Context(), http.MethodGet, apiURL, nil) - if err != nil { - return nil, err - } - req.Header.Set("Accept", "application/json") - - resp, err := h.proxy.HTTPClient.Do(req) - if err != nil { - return nil, err - } - defer func() { _ = resp.Body.Close() }() - - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("hex API returned %d", resp.StatusCode) - } - - body, err := h.proxy.ReadMetadata(resp.Body) - if err != nil { - return nil, err - } +// parseHexReleases reads the publish times from a Hex API package response. +func parseHexReleases(body []byte) (publishTimes, error) { var pkg hexPackageAPI if err := json.Unmarshal(body, &pkg); err != nil { return nil, err } - - packagePURL := canonicalPackagePURL("hex", name) - filtered := make(map[string]bool) - + times := make(publishTimes, len(pkg.Releases)) for _, release := range pkg.Releases { insertedAt, err := time.Parse(time.RFC3339Nano, release.InsertedAt) if err != nil { - continue - } - - if !h.proxy.Cooldown.IsAllowed("hex", packagePURL, insertedAt) { - filtered[release.Version] = true - h.proxy.Logger.Info("cooldown: filtering hex version", - "package", name, "version", release.Version, - "published", release.InsertedAt) + insertedAt = time.Time{} } + times[release.Version] = insertedAt } - - return filtered, nil + return times, nil } -// filterSignedPackage decompresses gzipped data, decodes the Signed protobuf wrapper, -// filters releases from the Package payload, and re-encodes as gzipped protobuf -// (without the original signature since the payload has changed). -func (h *HexHandler) filterSignedPackage(gzippedData []byte, filteredVersions map[string]bool) ([]byte, error) { - // Decompress gzip, with the metadata limit applied to the decompressed size +// signedPayload decompresses a gzipped Signed message, with the metadata +// limit applied to the decompressed size, and returns its payload. +func (h *HexHandler) signedPayload(gzippedData []byte) ([]byte, error) { gr, err := gzip.NewReader(bytes.NewReader(gzippedData)) if err != nil { return nil, err @@ -273,9 +259,30 @@ func (h *HexHandler) filterSignedPackage(gzippedData []byte, filteredVersions ma if err != nil { return nil, fmt.Errorf("extracting payload: %w", err) } + return payload, nil +} + +// packageReleaseVersions lists the versions of the releases in a Package +// message. +func packageReleaseVersions(payload []byte) ([]string, error) { + var versions []string + _, err := filterPackageReleases(payload, nil, func(version string) { + versions = append(versions, version) + }) + return versions, err +} + +// filterSignedPackage decompresses gzipped data, decodes the Signed protobuf wrapper, +// filters releases from the Package payload, and re-encodes as gzipped protobuf +// (without the original signature since the payload has changed). +func (h *HexHandler) filterSignedPackage(gzippedData []byte, filteredVersions map[string]bool) ([]byte, error) { + payload, err := h.signedPayload(gzippedData) + if err != nil { + return nil, err + } // Filter releases from the Package message - filteredPayload, err := filterPackageReleases(payload, filteredVersions) + filteredPayload, err := filterPackageReleases(payload, filteredVersions, nil) if err != nil { return nil, fmt.Errorf("filtering releases: %w", err) } @@ -298,9 +305,10 @@ func (h *HexHandler) filterSignedPackage(gzippedData []byte, filteredVersions ma return buf.Bytes(), nil } -// filterPackageReleases filters releases from a Package protobuf message. +// filterPackageReleases filters releases from a Package protobuf message, +// calling seen, when not nil, with the version of each release. // Package: field 1 = releases (repeated), field 2 = name, field 3 = repository -func filterPackageReleases(payload []byte, filteredVersions map[string]bool) ([]byte, error) { +func filterPackageReleases(payload []byte, filteredVersions map[string]bool, seen func(string)) ([]byte, error) { var result []byte data := payload @@ -325,6 +333,9 @@ func filterPackageReleases(payload []byte, filteredVersions map[string]bool) ([] if num == 1 { // releases field version := extractReleaseVersion(v) + if seen != nil { + seen(version) + } if filteredVersions[version] { continue // skip this release } diff --git a/internal/handler/hex_test.go b/internal/handler/hex_test.go index d448b76..a0b0448 100644 --- a/internal/handler/hex_test.go +++ b/internal/handler/hex_test.go @@ -95,7 +95,7 @@ func buildHexSigned(payload []byte) []byte { func TestHexFilterPackageReleases(t *testing.T) { pkg := buildHexPackage("phoenix", []string{testVersion100, "2.0.0", "3.0.0"}) - filtered, err := filterPackageReleases(pkg, map[string]bool{"2.0.0": true}) + filtered, err := filterPackageReleases(pkg, map[string]bool{"2.0.0": true}, nil) if err != nil { t.Fatal(err) } diff --git a/internal/handler/metadata_stream.go b/internal/handler/metadata_stream.go index 8bb319d..fd4fb83 100644 --- a/internal/handler/metadata_stream.go +++ b/internal/handler/metadata_stream.go @@ -1,12 +1,15 @@ package handler import ( + "context" "errors" + "io" "net/http" "strconv" "strings" "time" + "github.com/git-pkgs/proxy/internal/database" "github.com/git-pkgs/proxy/internal/metrics" ) @@ -21,47 +24,17 @@ import ( // buffered path instead: no fresh row, a row without a digest or size, a // document over the metadata size limit, or one that can't be opened. func (p *Proxy) serveCachedMetadata(w http.ResponseWriter, r *http.Request, ecosystem, cacheKey string) bool { - if p.DB == nil || p.Storage == nil || p.MetadataTTL <= 0 || containsPathTraversal(cacheKey) { + entry := p.freshMetadataEntry(ecosystem, cacheKey) + if entry == nil { return false } - entry, err := p.DB.GetMetadataCache(ecosystem, cacheKey) - if err != nil || entry == nil || !entry.FetchedAt.Valid || time.Since(entry.FetchedAt.Time) >= p.MetadataTTL { - return false - } - limit := p.MetadataMaxSize - if limit <= 0 { - limit = defaultMetadataMaxSize - } - digest, ok := strings.CutPrefix(entry.ContentDigest.String, "sha256:") - if !entry.ContentDigest.Valid || !ok || !entry.Size.Valid || entry.Size.Int64 > limit { - return false - } - checks, err := newIntegrityChecks(digest, "") + body, err := p.openMetadataEntry(r.Context(), ecosystem, cacheKey, entry) if err != nil { return false } - stored, err := p.Storage.Open(r.Context(), entry.StoragePath) - if err != nil { - return false - } - body, err := checks.wrapFailOnMismatch(stored, func(reason string) { - p.Logger.Error("cached metadata failed integrity check", - "ecosystem", ecosystem, "key", cacheKey, "path", entry.StoragePath, "reason", reason) - metrics.RecordIntegrityFailure(ecosystem, metrics.FailureDetail{ - Path: entry.StoragePath, Err: errors.New(reason)}) - // Drop the row so the next request fetches the document again. - if err := p.DB.DiscardMetadataCache(ecosystem, cacheKey, entry.StoragePath, entry.ContentDigest.String); err != nil { - p.Logger.Warn("failed to clear corrupt metadata from cache", "error", err) - } - }) - if err != nil { - _ = stored.Close() - return false - } defer func() { _ = body.Close() }() - p.recordMetadataRead(entry) - metrics.RecordMetadataCacheHit(ecosystem, int(entry.Size.Int64)) + p.recordStreamedMetadataHit(ecosystem, entry) cm := p.cachedMetaFor(entry) if writeMetadataValidators(w, r, cm) { @@ -83,3 +56,60 @@ func (p *Proxy) serveCachedMetadata(w http.ResponseWriter, r *http.Request, ecos } return true } + +// freshMetadataEntry returns the metadata cache row for ecosystem and +// cacheKey when its document can be streamed: the row is within its TTL and +// records a sha256 digest and a size within the metadata size limit. It +// returns nil otherwise. +func (p *Proxy) freshMetadataEntry(ecosystem, cacheKey string) *database.MetadataCacheEntry { + if p.DB == nil || p.Storage == nil || p.MetadataTTL <= 0 || containsPathTraversal(cacheKey) { + return nil + } + entry, err := p.DB.GetMetadataCache(ecosystem, cacheKey) + if err != nil || entry == nil || !entry.FetchedAt.Valid || time.Since(entry.FetchedAt.Time) >= p.MetadataTTL { + return nil + } + limit := p.MetadataMaxSize + if limit <= 0 { + limit = defaultMetadataMaxSize + } + if !entry.ContentDigest.Valid || !strings.HasPrefix(entry.ContentDigest.String, "sha256:") || !entry.Size.Valid || entry.Size.Int64 > limit { + return nil + } + return entry +} + +// openMetadataEntry opens the stored document of a row freshMetadataEntry +// returned. Its bytes are checked against the row's digest as they are read: +// a mismatch fails the read at the end and drops the row, so the next +// request fetches the document again. +func (p *Proxy) openMetadataEntry(ctx context.Context, ecosystem, cacheKey string, entry *database.MetadataCacheEntry) (io.ReadCloser, error) { + checks, err := newIntegrityChecks(strings.TrimPrefix(entry.ContentDigest.String, "sha256:"), "") + if err != nil { + return nil, err + } + stored, err := p.Storage.Open(ctx, entry.StoragePath) + if err != nil { + return nil, err + } + body, err := checks.wrapFailOnMismatch(stored, func(reason string) { + p.Logger.Error("cached metadata failed integrity check", + "ecosystem", ecosystem, "key", cacheKey, "path", entry.StoragePath, "reason", reason) + metrics.RecordIntegrityFailure(ecosystem, metrics.FailureDetail{ + Path: entry.StoragePath, Err: errors.New(reason)}) + if err := p.DB.DiscardMetadataCache(ecosystem, cacheKey, entry.StoragePath, entry.ContentDigest.String); err != nil { + p.Logger.Warn("failed to clear corrupt metadata from cache", "error", err) + } + }) + if err != nil { + _ = stored.Close() + return nil, err + } + return body, nil +} + +// recordStreamedMetadataHit counts a hit on a row served by streaming. +func (p *Proxy) recordStreamedMetadataHit(ecosystem string, entry *database.MetadataCacheEntry) { + p.recordMetadataRead(entry) + metrics.RecordMetadataCacheHit(ecosystem, int(entry.Size.Int64)) +} diff --git a/internal/handler/relay_test.go b/internal/handler/relay_test.go index a7a8fb7..72d20f3 100644 --- a/internal/handler/relay_test.go +++ b/internal/handler/relay_test.go @@ -52,8 +52,8 @@ func TestRelayRoutes(t *testing.T) { "/upstream", "/file", "/metadata", "/nuget/query", "/conan/v2/conans/demo/1.0/user/stable/revisions/rev/files/other.txt", "/composer/search.json", "/pypi/simple/", "/gem/api/v1/dependencies", - "/gem/info/demo", "/conda/conda-forge/noarch/repodata.json", - "/hex/packages/demo", "/swift/scope/demo/1.0.0/Package.swift", + "/conda/conda-forge/noarch/repodata.json", + "/swift/scope/demo/1.0.0/Package.swift", "/v2/library/demo/manifests/latest", "/v2/library/demo/tags/list", "/npm/-/npm/v1/keys", } { @@ -82,10 +82,9 @@ func testRelayRoute(t *testing.T, route string, truncated bool) { if !truncated { _, _ = fmt.Fprint(rw, "0\r\nX-Checksum: verified\r\nX-Late: discovered-at-eof\r\nX-Private: still-secret\r\n\r\n") } - // Two of these routes fan out a second upstream request for cooldown - // timestamps and abandon its body on a non-200, so this may be writing to - // a connection the proxy has already dropped. What the proxy made of the - // response under test is asserted downstream. + // The proxy may drop the connection before reading the whole + // response. What it made of the response under test is asserted + // downstream. _ = rw.Flush() })) defer upstream.Close() diff --git a/internal/handler/upstream_test.go b/internal/handler/upstream_test.go index 7839eaa..9d99214 100644 --- a/internal/handler/upstream_test.go +++ b/internal/handler/upstream_test.go @@ -1,6 +1,7 @@ package handler import ( + "context" "io" "net/http" "net/http/httptest" @@ -92,9 +93,10 @@ func TestHexHandlerUsesConfiguredAPIUpstream(t *testing.T) { upstream.URL+"/hex", upstream.URL+"/hex-api", ) - _, err := h.fetchFilteredVersions(httptest.NewRequest(http.MethodGet, "/", nil), "demo") - if err != nil { - t.Fatalf("fetchFilteredVersions failed: %v", err) + timed := h.timedPackage("demo") + res := h.proxy.lookupMetadata(context.Background(), "hex", timed.timesKey, timed.timesURL, timed.timesAccept, "", nil) + if res.err != nil { + t.Fatalf("fetching publish times failed: %v", res.err) } if requestedPath != "/hex-api/api/packages/demo" { t.Errorf("API path = %q, want %q", requestedPath, "/hex-api/api/packages/demo") From 4b3bd6f815d31b7d12c721bbb9dfbe8f835905ec Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 25 Oct 2026 17:50:49 +0100 Subject: [PATCH 439/633] Refuse unfiltered conda repodata while cooldown is on The proxy filters repodata.json and current_repodata.json, but repodata.json.bz2 went through the metadata cache unfiltered, and repodata.json.zst and repodata.jlap reached the download handler and were passed through. conda tries repodata.json.zst first, so with cooldown on it got versions still in cooldown. With cooldown on these now answer 404, and conda falls back to repodata.json. repodata_from_packages.json is filtered like repodata.json. --- docs/configuration.md | 2 +- internal/handler/conda.go | 34 ++++++++- .../handler/conda_repodata_variants_test.go | 76 +++++++++++++++++++ 3 files changed, 109 insertions(+), 3 deletions(-) create mode 100644 internal/handler/conda_repodata_variants_test.go diff --git a/docs/configuration.md b/docs/configuration.md index 5a877e3..ab2a2d1 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1048,7 +1048,7 @@ RubyGems, Hex and PyPI keep their publish times outside the metadata clients rea With cooldown on, the proxy asks npm for full packuments, since the abbreviated ones it uses otherwise carry no publish times. Abbreviated packuments cached before cooldown was turned on are fetched again in full rather than served, including as a stale copy when the upstream is down. -Conda cooldown covers `repodata.json` and sharded repodata (CEP 16). Clients cache shards by the hash in the shard index, so with cooldown on the proxy serves a rewritten index whose shard hashes change every hour and filters each shard as it is fetched. Versions leaving cooldown show up within the hour, and clients fetch the shards they need again once an hour. If a channel's shard index points packages or shards somewhere the proxy doesn't route, the proxy answers 404 for it and clients fall back to `repodata.json`. +Conda cooldown covers `repodata.json`, `current_repodata.json`, `repodata_from_packages.json` and sharded repodata (CEP 16). Other forms of the repodata, such as `repodata.json.zst`, `repodata.json.bz2` and `repodata.jlap` patches, would list versions still in cooldown, so with cooldown on they are answered 404 and conda and mamba fall back to `repodata.json`. Clients cache shards by the hash in the shard index, so with cooldown on the proxy serves a rewritten index whose shard hashes change every hour and filters each shard as it is fetched. Versions leaving cooldown show up within the hour, and clients fetch the shards they need again once an hour. If a channel's shard index points packages or shards somewhere the proxy doesn't route, the proxy answers 404 for it and clients fall back to `repodata.json`. Note: Hex cooldown requires disabling registry signature verification since the proxy re-encodes the protobuf payload without the original signature. Set `HEX_NO_VERIFY_REPO_ORIGIN=1` or configure your repo with `no_verify: true`. diff --git a/internal/handler/conda.go b/internal/handler/conda.go index 1142ccf..09178a1 100644 --- a/internal/handler/conda.go +++ b/internal/handler/conda.go @@ -79,8 +79,9 @@ func (h *CondaHandler) Routes() http.Handler { // Channel index (repodata) mux.HandleFunc("GET /{channel}/{arch}/repodata.json", h.handleRepodata) - mux.HandleFunc("GET /{channel}/{arch}/repodata.json.bz2", h.proxyCached) + mux.HandleFunc("GET /{channel}/{arch}/repodata.json.bz2", h.handleRepodataVariant) mux.HandleFunc("GET /{channel}/{arch}/current_repodata.json", h.handleRepodata) + mux.HandleFunc("GET /{channel}/{arch}/repodata_from_packages.json", h.handleRepodata) // Sharded repodata (CEP 16) mux.HandleFunc("GET /{channel}/{arch}/"+condaShardIndexFile, h.handleShardIndex) @@ -106,6 +107,10 @@ func (h *CondaHandler) handleDownload(w http.ResponseWriter, r *http.Request) { // Only cache actual package files if !h.isPackageFile(filename) { + if isCondaRepodataVariant(filename) && h.cooldownEnabled() { + refuseUnfilteredRepodata(w) + return + } h.proxyUpstream(w, r) return } @@ -178,9 +183,34 @@ func (h *CondaHandler) parseFilename(filename string) (name, version string) { return name, version } +// isCondaRepodataVariant reports whether filename is a form of a subdir's +// repodata other than the plain JSON documents the proxy filters, such as +// repodata.json.zst, repodata.json.bz2 or repodata.jlap patches. Package +// files are told apart by the caller. +func isCondaRepodataVariant(filename string) bool { + return strings.HasPrefix(filename, "repodata") || strings.HasPrefix(filename, "current_repodata") +} + +// handleRepodataVariant serves repodata.json.bz2 from the metadata cache, +// unless cooldown is on: the proxy filters only the JSON, so a compressed +// copy would list versions still in cooldown. +func (h *CondaHandler) handleRepodataVariant(w http.ResponseWriter, r *http.Request) { + if h.cooldownEnabled() { + refuseUnfilteredRepodata(w) + return + } + h.proxyCached(w, r) +} + +// refuseUnfilteredRepodata answers 404 for a repodata variant the proxy +// can't filter. conda and mamba then fall back to repodata.json. +func refuseUnfilteredRepodata(w http.ResponseWriter) { + http.Error(w, "only repodata.json is served while cooldown is enabled", http.StatusNotFound) +} + // handleRepodata proxies repodata.json, applying cooldown filtering when enabled. func (h *CondaHandler) handleRepodata(w http.ResponseWriter, r *http.Request) { - if h.proxy.Cooldown == nil || !h.proxy.Cooldown.Enabled() { + if !h.cooldownEnabled() { h.proxyCached(w, r) return } diff --git a/internal/handler/conda_repodata_variants_test.go b/internal/handler/conda_repodata_variants_test.go new file mode 100644 index 0000000..48f997b --- /dev/null +++ b/internal/handler/conda_repodata_variants_test.go @@ -0,0 +1,76 @@ +package handler + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + "time" + + "github.com/git-pkgs/cooldown" +) + +// The proxy filters only the plain JSON repodata. With cooldown on, any +// other form of it would list versions still in cooldown, so it is refused +// and clients fall back to repodata.json. +func TestCondaRepodataVariantsUnderCooldown(t *testing.T) { + plain, err := json.Marshal(map[string]any{ + "packages": map[string]any{ + "old.tar.bz2": map[string]any{"name": "demo", "timestamp": time.Now().Add(-7 * 24 * time.Hour).UnixMilli()}, + "new.tar.bz2": map[string]any{"name": "demo", "timestamp": time.Now().UnixMilli()}, + }, + }) + if err != nil { + t.Fatal(err) + } + var requests atomic.Int32 + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests.Add(1) + _, _ = w.Write(plain) + })) + defer upstream.Close() + + variants := []string{ + "repodata.json.bz2", + "repodata.json.zst", + "repodata.jlap", + "current_repodata.json.zst", + "repodata_from_packages.json.zst", + } + for _, enabled := range []bool{true, false} { + proxy := testProxy() + proxy.HTTPClient = upstream.Client() + if enabled { + proxy.Cooldown = &cooldown.Config{Default: "3d"} + } + handler := NewCondaHandlerWithUpstream(proxy, "http://proxy.local", upstream.URL).Routes() + for _, filename := range variants { + requests.Store(0) + w := httptest.NewRecorder() + handler.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/conda-forge/linux-64/"+filename, nil)) + switch { + case enabled && (w.Code != http.StatusNotFound || requests.Load() != 0): + t.Errorf("cooldown on, %s: status %d, %d upstream requests; want 404 without asking upstream", filename, w.Code, requests.Load()) + case !enabled && (w.Code != http.StatusOK || requests.Load() != 1): + t.Errorf("cooldown off, %s: status %d, %d upstream requests; want it passed through", filename, w.Code, requests.Load()) + } + } + } + + // repodata_from_packages.json is plain JSON and is filtered like + // repodata.json. + proxy := testProxy() + proxy.HTTPClient = upstream.Client() + proxy.Cooldown = &cooldown.Config{Default: "3d"} + handler := NewCondaHandlerWithUpstream(proxy, "http://proxy.local", upstream.URL).Routes() + w := httptest.NewRecorder() + handler.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/conda-forge/linux-64/repodata_from_packages.json", nil)) + var result map[string]map[string]any + if err := json.Unmarshal(w.Body.Bytes(), &result); err != nil { + t.Fatalf("status %d, body %s: %v", w.Code, w.Body, err) + } + if len(result["packages"]) != 1 || result["packages"]["old.tar.bz2"] == nil { + t.Errorf("repodata_from_packages.json not filtered: %s", w.Body) + } +} From 63477356792224f329becec6c4f830abae9be787 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 25 Oct 2026 18:14:58 +0100 Subject: [PATCH 440/633] Treat file::memory: URIs as in-memory and test more of the read pool isMemoryPath missed URIs such as file::memory:?cache=shared, whose readers would each have got an empty database. New tests open a database left in rollback-journal mode and run writes, hit flushes, lookups and stats scans concurrently. --- internal/database/database.go | 5 +- internal/database/read_pool_test.go | 112 ++++++++++++++++++++++++++++ 2 files changed, 116 insertions(+), 1 deletion(-) diff --git a/internal/database/database.go b/internal/database/database.go index 28a16f3..ac71ac1 100644 --- a/internal/database/database.go +++ b/internal/database/database.go @@ -142,8 +142,11 @@ func sqliteReaderConns() int { return min(max(runtime.GOMAXPROCS(0), sqliteMinReaders), sqliteMaxReaders) } +// isMemoryPath reports whether path names an in-memory database, as +// ":memory:" or a URI such as "file::memory:?cache=shared" or +// "file:x?mode=memory". func isMemoryPath(path string) bool { - return path == "" || strings.HasPrefix(path, ":memory:") || strings.Contains(path, "mode=memory") + return path == "" || strings.Contains(path, ":memory:") || strings.Contains(path, "mode=memory") } func OpenOrCreate(path string) (*DB, error) { diff --git a/internal/database/read_pool_test.go b/internal/database/read_pool_test.go index a1d6071..20436e1 100644 --- a/internal/database/read_pool_test.go +++ b/internal/database/read_pool_test.go @@ -3,10 +3,12 @@ package database import ( "context" "database/sql" + "errors" "fmt" "os" "path/filepath" "strings" + "sync" "testing" "time" ) @@ -279,3 +281,113 @@ func TestPostgresSharesOnePool(t *testing.T) { t.Fatal("postgres got a separate reader pool") } } + +func TestIsMemoryPath(t *testing.T) { + for path, want := range map[string]bool{ + "": true, + ":memory:": true, + "file::memory:": true, + "file::memory:?cache=shared": true, + "file:cache?mode=memory": true, + "/var/lib/proxy/cache.db": false, + "cache.db": false, + } { + if got := isMemoryPath(path); got != want { + t.Errorf("isMemoryPath(%q) = %v, want %v", path, got, want) + } + } +} + +// TestSQLiteExistingRollbackJournalDatabase opens a database file left in +// rollback-journal mode, as an older or foreign tool may leave it. The +// writer must switch it to WAL before readers open it, or reads would block +// on writes. +func TestSQLiteExistingRollbackJournalDatabase(t *testing.T) { + path := filepath.Join(t.TempDir(), "old.db") + db, err := Create(path) + if err != nil { + t.Fatalf("Create: %v", err) + } + seedOnePackage(t, db) + if _, err := db.Exec("PRAGMA journal_mode = DELETE"); err != nil { + t.Fatalf("leaving WAL: %v", err) + } + if err := db.Close(); err != nil { + t.Fatalf("Close: %v", err) + } + + db, err = Open(path) + if err != nil { + t.Fatalf("Open: %v", err) + } + defer func() { _ = db.Close() }() + var mode string + if err := db.reader.Get(&mode, "PRAGMA journal_mode"); err != nil || mode != "wal" { + t.Fatalf("reader journal_mode = %q, %v; want wal", mode, err) + } + art, err := db.GetCachedArtifact("pkg:npm/left-pad", "pkg:npm/left-pad@1.3.0", "left-pad-1.3.0.tgz") + if err != nil || art == nil { + t.Fatalf("GetCachedArtifact = %v, %v", art, err) + } +} + +// TestSQLiteConcurrentReadsAndWrites runs the writes a busy proxy makes +// (recording artifacts and hits) alongside request lookups and stats scans. +// None may fail with a busy, locked or read-only error. +func TestSQLiteConcurrentReadsAndWrites(t *testing.T) { + db := newTestDB(t) + seedOnePackage(t, db) + deadline := time.Now().Add(time.Second) + errs := make(chan error, 64) + var wg sync.WaitGroup + run := func(f func(i int) error) { + wg.Add(1) + go func() { + defer wg.Done() + for i := 0; time.Now().Before(deadline); i++ { + if err := f(i); err != nil { + errs <- err + return + } + } + }() + } + for w := range 2 { + run(func(i int) error { + name := fmt.Sprintf("pkg-%d-%d", w, i) + pkg := &Package{PURL: "pkg:npm/" + name, Ecosystem: "npm", Name: name} + ver := &Version{PURL: pkg.PURL + "@1.0.0", PackagePURL: pkg.PURL} + art := &Artifact{ + VersionPURL: ver.PURL, + Filename: name + ".tgz", + UpstreamURL: "https://registry.npmjs.org/" + name, + StoragePath: sql.NullString{String: "npm/" + name, Valid: true}, + Size: sql.NullInt64{Int64: 10, Valid: true}, + FetchedAt: sql.NullTime{Time: time.Now(), Valid: true}, + } + return db.RecordCachedArtifact(pkg, ver, art) + }) + } + run(func(int) error { return db.RecordArtifactHit("pkg:npm/left-pad@1.3.0", "left-pad-1.3.0.tgz") }) + for range 4 { + run(func(int) error { + art, err := db.GetCachedArtifact("pkg:npm/left-pad", "pkg:npm/left-pad@1.3.0", "left-pad-1.3.0.tgz") + if err == nil && art == nil { + err = errors.New("seeded artifact not found") + } + return err + }) + } + run(func(int) error { + if _, err := db.GetCacheStats(); err != nil { + return err + } + _, err := db.GetEcosystemStats() + return err + }) + wg.Wait() + close(errs) + for err := range errs { + t.Error(err) + } +} From 605e27d69a6ebabae61f2a8959b6dc5cc35c37eb Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 25 Oct 2026 14:57:49 +0100 Subject: [PATCH 441/633] Cache Git LFS objects git-lfs finds the LFS API under the rewritten remote, so clones through /git/ already asked the proxy for LFS objects and got 404s. The proxy now serves the batch API for downloads: it forwards the batch to the upstream, which decides which objects the repository has, and points each download at the proxy. Objects are served from the artifact cache. A miss asks the upstream batch API for a fresh download URL, fetches it and checks it against its sha256 object ID before caching it. Download URLs often name another host than the Git server, such as object storage. The proxy fetches from whatever host the upstream names, with the usual address checks; upstream.auth credentials stay scoped to their hosts. Objects are cached per repository. Uploads and locking are refused. Closes #517. --- README.md | 3 +- docs/architecture.md | 1 + docs/configuration.md | 17 +- internal/handler/git.go | 53 ++++- internal/handler/git_lfs.go | 320 +++++++++++++++++++++++++++++++ internal/handler/git_lfs_test.go | 213 ++++++++++++++++++++ internal/server/server.go | 2 +- 7 files changed, 599 insertions(+), 10 deletions(-) create mode 100644 internal/handler/git_lfs.go create mode 100644 internal/handler/git_lfs_test.go diff --git a/README.md b/README.md index e8ecec0..bcfd327 100644 --- a/README.md +++ b/README.md @@ -925,7 +925,8 @@ git config --global url."http://localhost:8080/git/github/".insteadOf https://gi ``` Each repository is cached as a bare clone and refreshed whenever its branches -or tags change upstream. See [configuration](docs/configuration.md#git-repositories) +or tags change upstream. Git LFS objects are cached in the artifact cache; +git-lfs uses the proxy for them with no further setup. See [configuration](docs/configuration.md#git-repositories) for setting the rewrite through environment variables, size limits, private repositories and outage behavior. diff --git a/docs/architecture.md b/docs/architecture.md index adaaf7b..60da601 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -308,6 +308,7 @@ HTTP protocol handlers for each registry type. **GitHandler:** - Serves `/git/{name}/{repo}` from `internal/gitcache`, read-only over smart HTTP - Validates repository paths and the `repos` allowlist, and maps cache errors to statuses +- Serves the Git LFS API under `{repo}.git/info/lfs/`: forwards download batches upstream and points each object at `objects/{oid}/{size}`, which is served from the artifact cache (ecosystem `git`, name `{name}/{repo}`, version the object ID); a miss asks the upstream batch API for a fresh download URL and checks the object against its sha256 ID ### `internal/gitcache` diff --git a/docs/configuration.md b/docs/configuration.md index 1049f2b..3f4fbc8 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -837,7 +837,22 @@ those yourself. The proxy locks `cache_dir` while it runs, through a `.gitcache-lock` file in it, so a second process given the same directory, such as another replica on a shared volume, fails to start; give each its own. -Pushing, the dumb HTTP protocol and Git LFS are not supported. The settings +Git LFS objects are cached in the artifact cache, so they use the configured +storage backend and retention rather than `cache_dir`. git-lfs finds the LFS +API under the rewritten remote, at `/git/{name}/{repo}.git/info/lfs`, with no +further configuration. A batch request is passed to the upstream, which +decides which objects the repository has, and each download is pointed at +the proxy. An object the cache doesn't have is fetched from the download URL +a fresh batch request returns. That URL is often on another host than the +Git server, such as GitHub's or Hugging Face's object storage; the proxy +fetches from whatever host the upstream names, with the usual address checks, +and `upstream.auth` credentials only go to the hosts they are configured for. +An object is only cached once its content matches its sha256 object ID. +Objects are cached per repository, so one repository's objects are never +served for another. + +Pushing, LFS uploads and locking, and the dumb HTTP protocol are not +supported. The settings under `git` can also be set with `PROXY_GIT_CACHE_DIR`, `PROXY_GIT_MAX_SIZE`, `PROXY_GIT_MAX_REPO_SIZE`, `PROXY_GIT_REFRESH_WAIT` and `PROXY_GIT_STALE_MAX_AGE`. diff --git a/internal/handler/git.go b/internal/handler/git.go index 895be44..0e4b493 100644 --- a/internal/handler/git.go +++ b/internal/handler/git.go @@ -39,6 +39,8 @@ type GitHandler struct { proxy *Proxy cache *gitcache.Cache upstreams map[string]gitUpstream + // baseURL is where clients reach the proxy, for LFS download URLs. + baseURL string // writeStall is gitWriteStallTimeout, shorter in tests. writeStall time.Duration } @@ -90,6 +92,10 @@ type gitRequest struct { } func (h *GitHandler) serve(w http.ResponseWriter, r *http.Request) { + if repoPath, endpoint, ok := strings.Cut(strings.TrimPrefix(r.URL.Path, "/"), gitLFSPath); ok { + h.serveLFS(w, r, repoPath, endpoint) + return + } req, status, msg := h.parse(r) if status != 0 { http.Error(w, msg, status) @@ -168,10 +174,38 @@ func (h *GitHandler) parse(r *http.Request) (gitRequest, int, string) { return req, http.StatusNotFound, "not found" } + repo, status, msg := h.repository(path) + if status != 0 { + return req, status, msg + } + req.key = repo.key + req.repoURL = repo.url + req.fetchWants = repo.upstream.fetchWants + return req, 0, "" +} + +// gitRepository is a repository of an upstream.git host, as a request path +// names it. +type gitRepository struct { + // name is the upstream's name and repo the path under it, without .git. + name, repo string + upstream gitUpstream + // key names the repository in the cache: {name}-{URL hash}/{repo}.git, + // with any .git the client gave removed from repo first, so both + // spellings share one copy. + key string + // url is the upstream repository, spelled as the client spelled it, and + // escapedRepo is repo escaped for a URL path. + url, escapedRepo string +} + +// repository resolves {name}/{repo} to an upstream repository, or returns +// the status and message to refuse it with. +func (h *GitHandler) repository(path string) (gitRepository, int, string) { name, repoPath, _ := strings.Cut(path, "/") upstream, ok := h.upstreams[name] if !ok { - return req, http.StatusNotFound, "unknown upstream" + return gitRepository{}, http.StatusNotFound, "unknown upstream" } segments := strings.Split(repoPath, "/") last := len(segments) - 1 @@ -182,7 +216,7 @@ func (h *GitHandler) parse(r *http.Request) (gitRequest, int, string) { escaped := make([]string, len(segments)) for i, segment := range segments { if !validGitSegment(segment, i == last) { - return req, http.StatusNotFound, "invalid repository path" + return gitRepository{}, http.StatusNotFound, "invalid repository path" } escaped[i] = url.PathEscape(segment) } @@ -190,12 +224,17 @@ func (h *GitHandler) parse(r *http.Request) (gitRequest, int, string) { if !upstream.AllowsRepo(repo) { // As for a missing repository, so a prober learns nothing about // the allowlist. - return req, http.StatusNotFound, "repository not found" + return gitRepository{}, http.StatusNotFound, "repository not found" } - req.key = upstream.dir + "/" + repo + ".git" - req.repoURL = upstream.URL + "/" + strings.Join(escaped, "/") + suffix - req.fetchWants = upstream.fetchWants - return req, 0, "" + escapedRepo := strings.Join(escaped, "/") + return gitRepository{ + name: name, + repo: repo, + upstream: upstream, + key: upstream.dir + "/" + repo + ".git", + url: upstream.URL + "/" + escapedRepo + suffix, + escapedRepo: escapedRepo, + }, 0, "" } // validGitSegment checks one segment of a repository path, which becomes both diff --git a/internal/handler/git_lfs.go b/internal/handler/git_lfs.go new file mode 100644 index 0000000..0ce00e1 --- /dev/null +++ b/internal/handler/git_lfs.go @@ -0,0 +1,320 @@ +package handler + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "slices" + "strconv" + "strings" +) + +const ( + // gitLFSPath separates a repository from the Git LFS API below it, as + // git-lfs derives the API URL from a remote: {remote}.git/info/lfs. + gitLFSPath = "/info/lfs/" + gitLFSMediaType = "application/vnd.git-lfs+json" + gitLFSTransfer = "basic" + gitLFSHashAlgo = "sha256" + + // gitLFSMaxBatchRequest bounds a client's batch request. git-lfs sends + // 100 objects per batch by default, a few kilobytes. + gitLFSMaxBatchRequest = 1 << 20 + // gitLFSMaxBatchResponse bounds an upstream batch response, which holds + // an href and headers per object. + gitLFSMaxBatchResponse = 16 << 20 + + gitLFSOIDLength = 64 +) + +// lfsBatchRequest is a Git LFS batch API request. +type lfsBatchRequest struct { + Operation string `json:"operation"` + Transfers []string `json:"transfers,omitempty"` + Ref json.RawMessage `json:"ref,omitempty"` + Objects []lfsObject `json:"objects"` + HashAlgo string `json:"hash_algo,omitempty"` +} + +type lfsObject struct { + OID string `json:"oid"` + Size int64 `json:"size"` +} + +// lfsBatchResponse is a Git LFS batch API response. +type lfsBatchResponse struct { + Transfer string `json:"transfer,omitempty"` + Objects []lfsObjectResponse `json:"objects"` + HashAlgo string `json:"hash_algo,omitempty"` +} + +type lfsObjectResponse struct { + OID string `json:"oid"` + Size int64 `json:"size"` + Authenticated bool `json:"authenticated,omitempty"` + Actions map[string]lfsAction `json:"actions,omitempty"` + Error *lfsObjectError `json:"error,omitempty"` +} + +type lfsAction struct { + Href string `json:"href"` + Header map[string]string `json:"header,omitempty"` +} + +type lfsObjectError struct { + Code int `json:"code"` + Message string `json:"message"` +} + +// lfsUpstreamError is a batch request the upstream answered with an error +// status. +type lfsUpstreamError struct { + status int +} + +func (e *lfsUpstreamError) Error() string { + return fmt.Sprintf("LFS batch API answered %d", e.status) +} + +// WithBaseURL sets the URL clients reach the proxy at, which LFS batch +// responses send them back to for each object. +func (h *GitHandler) WithBaseURL(baseURL string) *GitHandler { + h.baseURL = strings.TrimSuffix(baseURL, "/") + return h +} + +// serveLFS answers a request to the Git LFS API of a repository: batch +// downloads, and the objects the batch responses point at. Objects are +// stored in the artifact cache, checked against their sha256 object ID. +// Uploads and locking are refused. +func (h *GitHandler) serveLFS(w http.ResponseWriter, r *http.Request, repoPath, endpoint string) { + repo, status, msg := h.repository(repoPath) + if status != 0 { + lfsError(w, status, msg) + return + } + if endpoint == "objects/batch" { + if r.Method != http.MethodPost { + lfsError(w, http.StatusMethodNotAllowed, "method not allowed") + return + } + h.serveLFSBatch(w, r, repo) + return + } + oid, sizeText, ok := strings.Cut(strings.TrimPrefix(endpoint, "objects/"), "/") + size, err := strconv.ParseInt(sizeText, 10, 64) + if !strings.HasPrefix(endpoint, "objects/") || !ok || err != nil || size < 0 || !validLFSOID(oid) { + lfsError(w, http.StatusNotFound, "not found") + return + } + if r.Method != http.MethodGet && r.Method != http.MethodHead { + lfsError(w, http.StatusMethodNotAllowed, "method not allowed") + return + } + h.serveLFSObject(w, r, repo, lfsObject{OID: oid, Size: size}) +} + +// serveLFSBatch forwards a download batch upstream and points every object +// the upstream offers at this proxy. The upstream answers for the +// repository, so a client only gets objects it could get from there. +func (h *GitHandler) serveLFSBatch(w http.ResponseWriter, r *http.Request, repo gitRepository) { + var req lfsBatchRequest + if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, gitLFSMaxBatchRequest)).Decode(&req); err != nil { + var tooLarge *http.MaxBytesError + if errors.As(err, &tooLarge) { + lfsError(w, http.StatusRequestEntityTooLarge, "batch request too large") + return + } + lfsError(w, http.StatusBadRequest, "invalid batch request") + return + } + switch { + case req.Operation != "download": + lfsError(w, http.StatusForbidden, "pushing is not supported") + return + case req.HashAlgo != "" && req.HashAlgo != gitLFSHashAlgo: + lfsError(w, http.StatusConflict, "only sha256 objects are supported") + return + case len(req.Transfers) > 0 && !slices.Contains(req.Transfers, gitLFSTransfer): + lfsError(w, http.StatusUnprocessableEntity, "only the basic transfer is supported") + return + } + requested := make(map[string]int64, len(req.Objects)) + for _, object := range req.Objects { + if !validLFSOID(object.OID) || object.Size < 0 { + lfsError(w, http.StatusUnprocessableEntity, "invalid object") + return + } + requested[object.OID] = object.Size + } + + upstream, err := h.lfsBatch(r.Context(), repo, req.Ref, req.Objects) + if err != nil { + h.serveLFSUpstreamError(w, r, repo, err) + return + } + resp := lfsBatchResponse{Transfer: gitLFSTransfer, Objects: []lfsObjectResponse{}, HashAlgo: gitLFSHashAlgo} + for _, object := range upstream.Objects { + // Only objects the client asked for, at the size it gave, so a + // response can't point the client at anything else. + size, ok := requested[object.OID] + if !ok || size != object.Size { + continue + } + out := lfsObjectResponse{OID: object.OID, Size: object.Size, Error: object.Error} + if download, ok := object.Actions["download"]; ok && object.Error == nil && download.Href != "" { + out.Authenticated = true + out.Actions = map[string]lfsAction{"download": {Href: h.lfsObjectURL(repo, object.OID, object.Size)}} + } + resp.Objects = append(resp.Objects, out) + } + w.Header().Set("Content-Type", gitLFSMediaType) + _ = json.NewEncoder(w).Encode(resp) +} + +// serveLFSObject serves an object from the artifact cache, or on a miss asks +// the upstream batch API for a fresh download URL, fetches it and checks it +// against its object ID before caching it. +func (h *GitHandler) serveLFSObject(w http.ResponseWriter, r *http.Request, repo gitRepository, object lfsObject) { + // Objects are cached per repository: having one cached for another + // repository says nothing about whether this one has it. + name := repo.name + "/" + repo.repo + cached, err := h.proxy.GetCachedArtifact(r.Context(), gitEcosystem, name, object.OID, object.OID) + if err != nil { + h.proxy.serveArtifactError(w, err, "failed to read cached LFS object") + return + } + if cached == nil { + upstream, err := h.lfsBatch(r.Context(), repo, nil, []lfsObject{object}) + if err != nil { + h.serveLFSUpstreamError(w, r, repo, err) + return + } + download, status, msg := lfsDownload(upstream, object) + if status != 0 { + lfsError(w, status, msg) + return + } + header := make(http.Header, len(download.Header)) + for key, value := range download.Header { + header.Set(key, value) + } + cached, err = h.proxy.getOrFetchArtifactFromURL(r.Context(), gitEcosystem, name, object.OID, object.OID, + download.Href, header, object.OID, "") + if err != nil { + h.proxy.serveArtifactError(w, err, "failed to fetch LFS object") + return + } + } + serveGenericArtifact(w, r, cached) +} + +// lfsDownload returns the download action the upstream gave for object, or +// the status and message to answer with when it gave none. +func lfsDownload(resp *lfsBatchResponse, object lfsObject) (lfsAction, int, string) { + for _, got := range resp.Objects { + if got.OID != object.OID { + continue + } + if got.Error != nil { + if got.Error.Code == http.StatusNotFound || got.Error.Code == http.StatusGone { + return lfsAction{}, http.StatusNotFound, "object not found" + } + return lfsAction{}, http.StatusBadGateway, "upstream refused the object" + } + download, ok := got.Actions["download"] + u, err := url.Parse(download.Href) + if !ok || err != nil || (u.Scheme != "https" && u.Scheme != "http") || u.Host == "" { + return lfsAction{}, http.StatusBadGateway, "upstream gave no download for the object" + } + return download, 0, "" + } + return lfsAction{}, http.StatusNotFound, "object not found" +} + +// lfsBatch asks the upstream's LFS batch API for download actions for +// objects. upstream.auth applies to it as to the repository. +func (h *GitHandler) lfsBatch(ctx context.Context, repo gitRepository, ref json.RawMessage, objects []lfsObject) (*lfsBatchResponse, error) { + body, err := json.Marshal(lfsBatchRequest{ + Operation: "download", + Transfers: []string{gitLFSTransfer}, + Ref: ref, + Objects: objects, + HashAlgo: gitLFSHashAlgo, + }) + if err != nil { + return nil, err + } + batchURL := strings.TrimSuffix(repo.url, ".git") + ".git" + gitLFSPath + "objects/batch" + req, err := http.NewRequestWithContext(ctx, http.MethodPost, batchURL, bytes.NewReader(body)) + if err != nil { + return nil, err + } + req.Header.Set("Accept", gitLFSMediaType) + req.Header.Set("Content-Type", gitLFSMediaType) + resp, err := h.proxy.HTTPClient.Do(req) + if err != nil { + return nil, err + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode != http.StatusOK { + return nil, &lfsUpstreamError{status: resp.StatusCode} + } + var batch lfsBatchResponse + if err := json.NewDecoder(io.LimitReader(resp.Body, gitLFSMaxBatchResponse)).Decode(&batch); err != nil { + return nil, fmt.Errorf("reading LFS batch response: %w", err) + } + if batch.Transfer != "" && batch.Transfer != gitLFSTransfer { + return nil, fmt.Errorf("LFS batch response uses the %q transfer", batch.Transfer) + } + return &batch, nil +} + +// serveLFSUpstreamError answers a request whose upstream batch request +// failed. +func (h *GitHandler) serveLFSUpstreamError(w http.ResponseWriter, r *http.Request, repo gitRepository, err error) { + if r.Context().Err() != nil { + return + } + h.proxy.Logger.Warn("git: LFS batch request failed", "repository", repo.key, "error", err) + var upstream *lfsUpstreamError + switch { + case errors.As(err, &upstream) && upstream.status == http.StatusNotFound: + lfsError(w, http.StatusNotFound, "repository not found") + case errors.As(err, &upstream) && upstream.status < http.StatusInternalServerError: + lfsError(w, http.StatusBadGateway, "upstream refused the request") + default: + lfsError(w, http.StatusBadGateway, "upstream unavailable") + } +} + +// lfsObjectURL is where this proxy serves an object of repo. +func (h *GitHandler) lfsObjectURL(repo gitRepository, oid string, size int64) string { + return fmt.Sprintf("%s/git/%s/%s.git%sobjects/%s/%d", + h.baseURL, url.PathEscape(repo.name), repo.escapedRepo, gitLFSPath, oid, size) +} + +// validLFSOID reports whether oid is a sha256 object ID in lowercase hex. +func validLFSOID(oid string) bool { + if len(oid) != gitLFSOIDLength { + return false + } + for _, c := range oid { + if (c < '0' || c > '9') && (c < 'a' || c > 'f') { + return false + } + } + return true +} + +// lfsError answers with a Git LFS error document, which git-lfs shows. +func lfsError(w http.ResponseWriter, status int, msg string) { + w.Header().Set("Content-Type", gitLFSMediaType) + w.WriteHeader(status) + _ = json.NewEncoder(w).Encode(map[string]string{"message": msg}) +} diff --git a/internal/handler/git_lfs_test.go b/internal/handler/git_lfs_test.go new file mode 100644 index 0000000..1104a2b --- /dev/null +++ b/internal/handler/git_lfs_test.go @@ -0,0 +1,213 @@ +package handler + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + + "github.com/git-pkgs/proxy/internal/config" + "github.com/git-pkgs/registries/fetch" +) + +const lfsTestContent = "large file content" + +// lfsUpstream is a Git host with an LFS batch API whose download hrefs point +// at a separate object store, as GitHub's do. The store only serves objects +// to requests carrying the header the batch response gave. +type lfsUpstream struct { + git, store *httptest.Server + oid string + content atomic.Value // string the store serves for oid + batches atomic.Int32 + downloads atomic.Int32 + // lastBatch is the last batch request the upstream got. + lastBatch atomic.Value +} + +func newLFSUpstream(t *testing.T) *lfsUpstream { + t.Helper() + sum := sha256.Sum256([]byte(lfsTestContent)) + u := &lfsUpstream{oid: hex.EncodeToString(sum[:])} + u.content.Store(lfsTestContent) + u.store = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/objects/"+u.oid || r.Header.Get("X-Signed") != "token" { + http.Error(w, "forbidden", http.StatusForbidden) + return + } + u.downloads.Add(1) + _, _ = io.WriteString(w, u.content.Load().(string)) + })) + u.git = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost || r.URL.Path != "/acme/widget.git/info/lfs/objects/batch" { + http.NotFound(w, r) + return + } + u.batches.Add(1) + var req lfsBatchRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + u.lastBatch.Store(req) + resp := lfsBatchResponse{Transfer: "basic"} + for _, object := range req.Objects { + if object.OID != u.oid { + resp.Objects = append(resp.Objects, lfsObjectResponse{OID: object.OID, Size: object.Size, + Error: &lfsObjectError{Code: http.StatusNotFound, Message: "Object does not exist"}}) + continue + } + resp.Objects = append(resp.Objects, lfsObjectResponse{OID: object.OID, Size: object.Size, + Actions: map[string]lfsAction{"download": { + Href: u.store.URL + "/objects/" + u.oid, + Header: map[string]string{"X-Signed": "token"}, + }}}) + } + // An object nobody asked for is not passed on. + resp.Objects = append(resp.Objects, lfsObjectResponse{OID: strings.Repeat("e", 64), Size: 1, + Actions: map[string]lfsAction{"download": {Href: u.store.URL + "/other"}}}) + w.Header().Set("Content-Type", gitLFSMediaType) + _ = json.NewEncoder(w).Encode(resp) + })) + t.Cleanup(u.git.Close) + t.Cleanup(u.store.Close) + return u +} + +func newLFSHandler(t *testing.T, u *lfsUpstream, repos ...string) *GitHandler { + t.Helper() + proxy, _, store, _ := setupTestProxy(t) + store.seekable = true + proxy.HTTPClient = u.git.Client() + fetcher := fetch.NewFetcher(fetch.WithHTTPClient(u.store.Client()), fetch.WithMaxRetries(0)) + proxy.Fetcher = fetcher + t.Cleanup(func() { _ = fetcher.Close() }) + return NewGitHandler(proxy, nil, config.UpstreamConfig{Git: map[string]config.GitUpstream{ + "up": {URL: u.git.URL, Repos: repos}, + }}).WithBaseURL("http://proxy.test/") +} + +func lfsRequest(h *GitHandler, method, target, body string) *httptest.ResponseRecorder { + r := httptest.NewRequest(method, target, strings.NewReader(body)) + r.Header.Set("Content-Type", gitLFSMediaType) + w := httptest.NewRecorder() + h.Routes().ServeHTTP(w, r) + return w +} + +func lfsBatchBody(operation string, objects ...lfsObject) string { + body, _ := json.Marshal(lfsBatchRequest{Operation: operation, Transfers: []string{"basic"}, + Ref: json.RawMessage(`{"name":"refs/heads/main"}`), Objects: objects}) + return string(body) +} + +func TestGitLFSBatchPointsObjectsAtTheProxy(t *testing.T) { + u := newLFSUpstream(t) + h := newLFSHandler(t, u) + missing := strings.Repeat("0", 64) + size := int64(len(lfsTestContent)) + + w := lfsRequest(h, http.MethodPost, "/up/acme/widget.git/info/lfs/objects/batch", + lfsBatchBody("download", lfsObject{OID: u.oid, Size: size}, lfsObject{OID: missing, Size: 3})) + if w.Code != http.StatusOK || w.Header().Get("Content-Type") != gitLFSMediaType { + t.Fatalf("batch: status %d, type %q, body %s", w.Code, w.Header().Get("Content-Type"), w.Body) + } + var resp lfsBatchResponse + if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil { + t.Fatal(err) + } + if len(resp.Objects) != 2 { + t.Fatalf("objects = %+v, want the two requested", resp.Objects) + } + got, notFound := resp.Objects[0], resp.Objects[1] + want := fmt.Sprintf("http://proxy.test/git/up/acme/widget.git/info/lfs/objects/%s/%d", u.oid, size) + if download := got.Actions["download"]; download.Href != want || len(download.Header) != 0 || !got.Authenticated { + t.Errorf("download = %+v, authenticated %v; want %s without the upstream's headers", download, got.Authenticated, want) + } + if notFound.Error == nil || notFound.Error.Code != http.StatusNotFound || notFound.Actions != nil { + t.Errorf("missing object = %+v, want the upstream's 404", notFound) + } + if req, _ := u.lastBatch.Load().(lfsBatchRequest); string(req.Ref) != `{"name":"refs/heads/main"}` { + t.Errorf("upstream got ref %s, want the client's", req.Ref) + } +} + +func TestGitLFSObjectIsCached(t *testing.T) { + u := newLFSUpstream(t) + h := newLFSHandler(t, u) + path := fmt.Sprintf("/up/acme/widget.git/info/lfs/objects/%s/%d", u.oid, len(lfsTestContent)) + + w := lfsRequest(h, http.MethodGet, path, "") + if w.Code != http.StatusOK || w.Body.String() != lfsTestContent { + t.Fatalf("first GET: status %d, body %q", w.Code, w.Body) + } + if u.batches.Load() != 1 || u.downloads.Load() != 1 { + t.Errorf("batches %d, downloads %d; want one of each", u.batches.Load(), u.downloads.Load()) + } + + u.git.Close() + u.store.Close() + w = lfsRequest(h, http.MethodGet, path, "") + if w.Code != http.StatusOK || w.Body.String() != lfsTestContent { + t.Errorf("GET with the upstream down: status %d, body %q; want it from the cache", w.Code, w.Body) + } +} + +func TestGitLFSObjectRefusesWrongContent(t *testing.T) { + u := newLFSUpstream(t) + u.content.Store("tampered content!!") + h := newLFSHandler(t, u) + path := fmt.Sprintf("/up/acme/widget.git/info/lfs/objects/%s/%d", u.oid, len(lfsTestContent)) + + for range 2 { + if w := lfsRequest(h, http.MethodGet, path, ""); w.Code == http.StatusOK && w.Body.String() == "tampered content!!" { + t.Fatal("served content that does not match its object ID") + } + } + if u.downloads.Load() != 2 { + t.Errorf("downloads = %d, want 2: the mismatching object must not be cached", u.downloads.Load()) + } +} + +func TestGitLFSRefusals(t *testing.T) { + u := newLFSUpstream(t) + h := newLFSHandler(t, u, "acme/*") + object := lfsObject{OID: u.oid, Size: int64(len(lfsTestContent))} + tests := []struct { + name, method, target, body string + status int + }{ + {"upload", http.MethodPost, "/up/acme/widget.git/info/lfs/objects/batch", lfsBatchBody("upload", object), http.StatusForbidden}, + {"sha512 objects", http.MethodPost, "/up/acme/widget.git/info/lfs/objects/batch", + `{"operation":"download","objects":[],"hash_algo":"sha512"}`, http.StatusConflict}, + {"other transfers", http.MethodPost, "/up/acme/widget.git/info/lfs/objects/batch", + `{"operation":"download","transfers":["ssh"],"objects":[]}`, http.StatusUnprocessableEntity}, + {"invalid oid", http.MethodPost, "/up/acme/widget.git/info/lfs/objects/batch", + lfsBatchBody("download", lfsObject{OID: "../x", Size: 1}), http.StatusUnprocessableEntity}, + {"batch GET", http.MethodGet, "/up/acme/widget.git/info/lfs/objects/batch", "", http.StatusMethodNotAllowed}, + {"repository outside repos", http.MethodPost, "/up/other/widget.git/info/lfs/objects/batch", lfsBatchBody("download", object), http.StatusNotFound}, + {"unknown repository", http.MethodPost, "/up/acme/missing.git/info/lfs/objects/batch", lfsBatchBody("download", object), http.StatusNotFound}, + {"object without size", http.MethodGet, "/up/acme/widget.git/info/lfs/objects/" + u.oid, "", http.StatusNotFound}, + {"uppercase oid", http.MethodGet, "/up/acme/widget.git/info/lfs/objects/" + strings.ToUpper(u.oid) + "/1", "", http.StatusNotFound}, + {"object PUT", http.MethodPut, "/up/acme/widget.git/info/lfs/objects/" + u.oid + "/1", "", http.StatusMethodNotAllowed}, + {"locks", http.MethodPost, "/up/acme/widget.git/info/lfs/locks/verify", "{}", http.StatusNotFound}, + {"missing object", http.MethodGet, "/up/acme/widget.git/info/lfs/objects/" + strings.Repeat("0", 64) + "/3", "", http.StatusNotFound}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + w := lfsRequest(h, tt.method, tt.target, tt.body) + if w.Code != tt.status || w.Header().Get("Content-Type") != gitLFSMediaType { + t.Errorf("status %d, type %q, body %s; want %d", w.Code, w.Header().Get("Content-Type"), w.Body, tt.status) + } + }) + } + if u.downloads.Load() != 0 { + t.Errorf("downloads = %d, want none", u.downloads.Load()) + } +} diff --git a/internal/server/server.go b/internal/server/server.go index bb99be6..12619a6 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -530,7 +530,7 @@ func (s *Server) mountProtocolHandlers(r chi.Router, proxy *handler.Proxy) { r.Mount("/rpm", http.StripPrefix("/rpm", rpmHandler.Routes())) r.Mount("/generic", http.StripPrefix("/generic", genericHandler.Routes())) if s.git != nil { - gitHandler := handler.NewGitHandler(proxy, s.git, s.cfg.Upstream) + gitHandler := handler.NewGitHandler(proxy, s.git, s.cfg.Upstream).WithBaseURL(s.cfg.BaseURL) r.Mount("/git", http.StripPrefix("/git", gitHandler.Routes())) } } From 371cefa889bf41763fb9b35cf9447b667b9b29f7 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 25 Oct 2026 16:08:15 +0100 Subject: [PATCH 442/633] Harden the Git LFS cache - Keep signed download URLs out of logs and the database: the fetch uses the signed URL, everything else the URL without its query. - Answer a batch from the cache when the upstream can't be reached or fails, so git-lfs can use cached objects during an outage. Refusals are still passed on. - Answer a HEAD for an uncached object from the batch response instead of downloading it. --- docs/architecture.md | 2 +- docs/configuration.md | 12 ++- internal/handler/git_lfs.go | 77 +++++++++++++++++-- internal/handler/git_lfs_test.go | 123 ++++++++++++++++++++++++++++++- internal/handler/handler.go | 54 +++++++++++++- 5 files changed, 253 insertions(+), 15 deletions(-) diff --git a/docs/architecture.md b/docs/architecture.md index 60da601..8de5fa6 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -308,7 +308,7 @@ HTTP protocol handlers for each registry type. **GitHandler:** - Serves `/git/{name}/{repo}` from `internal/gitcache`, read-only over smart HTTP - Validates repository paths and the `repos` allowlist, and maps cache errors to statuses -- Serves the Git LFS API under `{repo}.git/info/lfs/`: forwards download batches upstream and points each object at `objects/{oid}/{size}`, which is served from the artifact cache (ecosystem `git`, name `{name}/{repo}`, version the object ID); a miss asks the upstream batch API for a fresh download URL and checks the object against its sha256 ID +- Serves the Git LFS API under `{repo}.git/info/lfs/`: forwards download batches upstream and points each object at `objects/{oid}/{size}`, which is served from the artifact cache (ecosystem `git`, name `{name}/{repo}`, version the object ID); a miss asks the upstream batch API for a fresh download URL, recorded without its signed query, and checks the object against its sha256 ID; a batch the upstream can't answer is answered from the cache ### `internal/gitcache` diff --git a/docs/configuration.md b/docs/configuration.md index 3f4fbc8..026bc58 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -847,9 +847,15 @@ a fresh batch request returns. That URL is often on another host than the Git server, such as GitHub's or Hugging Face's object storage; the proxy fetches from whatever host the upstream names, with the usual address checks, and `upstream.auth` credentials only go to the hosts they are configured for. -An object is only cached once its content matches its sha256 object ID. -Objects are cached per repository, so one repository's objects are never -served for another. +Download URLs are often signed; only the URL without its query is logged +and recorded with the object. An object is only cached once its content +matches its sha256 object ID. Objects are cached per repository, so one +repository's objects are never served for another. When the upstream can't +be reached or fails, a batch is answered with the objects cached for the +repository, and the others are reported as unavailable; a batch the upstream +refuses is refused. A cached object stays available after it is removed +upstream, until retention removes it or you delete it through the +[cache delete API](#cache-delete-api). Pushing, LFS uploads and locking, and the dumb HTTP protocol are not supported. The settings diff --git a/internal/handler/git_lfs.go b/internal/handler/git_lfs.go index 0ce00e1..5b9be90 100644 --- a/internal/handler/git_lfs.go +++ b/internal/handler/git_lfs.go @@ -21,6 +21,8 @@ const ( gitLFSMediaType = "application/vnd.git-lfs+json" gitLFSTransfer = "basic" gitLFSHashAlgo = "sha256" + // gitLFSObjectMediaType is what objects are served as. + gitLFSObjectMediaType = "application/octet-stream" // gitLFSMaxBatchRequest bounds a client's batch request. git-lfs sends // 100 objects per batch by default, a few kilobytes. @@ -155,10 +157,14 @@ func (h *GitHandler) serveLFSBatch(w http.ResponseWriter, r *http.Request, repo upstream, err := h.lfsBatch(r.Context(), repo, req.Ref, req.Objects) if err != nil { + if resp, ok := h.lfsBatchFromCache(r, repo, req.Objects, err); ok { + writeLFSBatch(w, resp) + return + } h.serveLFSUpstreamError(w, r, repo, err) return } - resp := lfsBatchResponse{Transfer: gitLFSTransfer, Objects: []lfsObjectResponse{}, HashAlgo: gitLFSHashAlgo} + resp := newLFSBatchResponse() for _, object := range upstream.Objects { // Only objects the client asked for, at the size it gave, so a // response can't point the client at anything else. @@ -173,6 +179,49 @@ func (h *GitHandler) serveLFSBatch(w http.ResponseWriter, r *http.Request, repo } resp.Objects = append(resp.Objects, out) } + writeLFSBatch(w, resp) +} + +// lfsBatchFromCache answers a batch the upstream couldn't, because it was +// unreachable or failing, from the objects cached for repo. This is what a +// GET of each object would serve anyway. A refusal by the upstream is not +// overridden, and a batch none of whose objects are cached is left to fail. +func (h *GitHandler) lfsBatchFromCache(r *http.Request, repo gitRepository, objects []lfsObject, err error) (lfsBatchResponse, bool) { + var refused *lfsUpstreamError + if r.Context().Err() != nil || (errors.As(err, &refused) && refused.status < http.StatusInternalServerError) { + return lfsBatchResponse{}, false + } + resp := newLFSBatchResponse() + seen := make(map[string]bool, len(objects)) + cached := 0 + for _, object := range objects { + if seen[object.OID] { + continue + } + seen[object.OID] = true + out := lfsObjectResponse{OID: object.OID, Size: object.Size} + if ok, _ := h.proxy.HasCachedArtifact(r.Context(), gitEcosystem, repo.lfsName(), object.OID, object.OID); ok { + out.Authenticated = true + out.Actions = map[string]lfsAction{"download": {Href: h.lfsObjectURL(repo, object.OID, object.Size)}} + cached++ + } else { + out.Error = &lfsObjectError{Code: http.StatusBadGateway, Message: "upstream unavailable and object not cached"} + } + resp.Objects = append(resp.Objects, out) + } + if cached == 0 { + return lfsBatchResponse{}, false + } + h.proxy.Logger.Warn("git: LFS batch request failed, answering from the cache", + "repository", repo.key, "cached", cached, "objects", len(resp.Objects), "error", err) + return resp, true +} + +func newLFSBatchResponse() lfsBatchResponse { + return lfsBatchResponse{Transfer: gitLFSTransfer, Objects: []lfsObjectResponse{}, HashAlgo: gitLFSHashAlgo} +} + +func writeLFSBatch(w http.ResponseWriter, resp lfsBatchResponse) { w.Header().Set("Content-Type", gitLFSMediaType) _ = json.NewEncoder(w).Encode(resp) } @@ -181,9 +230,7 @@ func (h *GitHandler) serveLFSBatch(w http.ResponseWriter, r *http.Request, repo // the upstream batch API for a fresh download URL, fetches it and checks it // against its object ID before caching it. func (h *GitHandler) serveLFSObject(w http.ResponseWriter, r *http.Request, repo gitRepository, object lfsObject) { - // Objects are cached per repository: having one cached for another - // repository says nothing about whether this one has it. - name := repo.name + "/" + repo.repo + name := repo.lfsName() cached, err := h.proxy.GetCachedArtifact(r.Context(), gitEcosystem, name, object.OID, object.OID) if err != nil { h.proxy.serveArtifactError(w, err, "failed to read cached LFS object") @@ -200,12 +247,23 @@ func (h *GitHandler) serveLFSObject(w http.ResponseWriter, r *http.Request, repo lfsError(w, status, msg) return } + if r.Method == http.MethodHead { + // The upstream has the object, and a HEAD needs no more than + // that: fetching it is left to a GET. + w.Header().Set("Content-Type", gitLFSObjectMediaType) + w.Header().Set("Content-Length", strconv.FormatInt(object.Size, 10)) + return + } header := make(http.Header, len(download.Header)) for key, value := range download.Header { header.Set(key, value) } - cached, err = h.proxy.getOrFetchArtifactFromURL(r.Context(), gitEcosystem, name, object.OID, object.OID, - download.Href, header, object.OID, "") + // Download URLs are often signed, granting access to the object + // until they expire, so only the URL without its query is logged + // and recorded. + ctx, recorded := withSignedDownload(r.Context(), download.Href) + cached, err = h.proxy.getOrFetchArtifactFromURL(ctx, gitEcosystem, name, object.OID, object.OID, + recorded, header, object.OID, "") if err != nil { h.proxy.serveArtifactError(w, err, "failed to fetch LFS object") return @@ -293,6 +351,13 @@ func (h *GitHandler) serveLFSUpstreamError(w http.ResponseWriter, r *http.Reques } } +// lfsName names repo's LFS objects in the artifact cache. Objects are cached +// per repository: having one cached for another repository says nothing +// about whether this one has it. +func (repo gitRepository) lfsName() string { + return repo.name + "/" + repo.repo +} + // lfsObjectURL is where this proxy serves an object of repo. func (h *GitHandler) lfsObjectURL(repo gitRepository, oid string, size int64) string { return fmt.Sprintf("%s/git/%s/%s.git%sobjects/%s/%d", diff --git a/internal/handler/git_lfs_test.go b/internal/handler/git_lfs_test.go index 1104a2b..62cb3aa 100644 --- a/internal/handler/git_lfs_test.go +++ b/internal/handler/git_lfs_test.go @@ -1,11 +1,13 @@ package handler import ( + "bytes" "crypto/sha256" "encoding/hex" "encoding/json" "fmt" "io" + "log/slog" "net/http" "net/http/httptest" "strings" @@ -18,17 +20,25 @@ import ( const lfsTestContent = "large file content" +// lfsSignature is the signature in the upstream's download URLs, which must +// not reach logs or the database. +const lfsSignature = "s3cr3t-signature" + // lfsUpstream is a Git host with an LFS batch API whose download hrefs point // at a separate object store, as GitHub's do. The store only serves objects -// to requests carrying the header the batch response gave. +// to requests carrying the header and signed URL the batch response gave. type lfsUpstream struct { git, store *httptest.Server oid string content atomic.Value // string the store serves for oid batches atomic.Int32 downloads atomic.Int32 + // batchStatus, when set, is the status the batch API fails with. + batchStatus atomic.Int32 // lastBatch is the last batch request the upstream got. lastBatch atomic.Value + // logs is what the proxy logged. + logs bytes.Buffer } func newLFSUpstream(t *testing.T) *lfsUpstream { @@ -37,7 +47,8 @@ func newLFSUpstream(t *testing.T) *lfsUpstream { u := &lfsUpstream{oid: hex.EncodeToString(sum[:])} u.content.Store(lfsTestContent) u.store = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.URL.Path != "/objects/"+u.oid || r.Header.Get("X-Signed") != "token" { + if r.URL.Path != "/objects/"+u.oid || r.Header.Get("X-Signed") != "token" || + r.URL.Query().Get("Signature") != lfsSignature { http.Error(w, "forbidden", http.StatusForbidden) return } @@ -50,6 +61,10 @@ func newLFSUpstream(t *testing.T) *lfsUpstream { return } u.batches.Add(1) + if status := u.batchStatus.Load(); status != 0 { + http.Error(w, "failing", int(status)) + return + } var req lfsBatchRequest if err := json.NewDecoder(r.Body).Decode(&req); err != nil { http.Error(w, err.Error(), http.StatusBadRequest) @@ -65,7 +80,7 @@ func newLFSUpstream(t *testing.T) *lfsUpstream { } resp.Objects = append(resp.Objects, lfsObjectResponse{OID: object.OID, Size: object.Size, Actions: map[string]lfsAction{"download": { - Href: u.store.URL + "/objects/" + u.oid, + Href: u.store.URL + "/objects/" + u.oid + "?Expires=1&Signature=" + lfsSignature, Header: map[string]string{"X-Signed": "token"}, }}}) } @@ -84,6 +99,7 @@ func newLFSHandler(t *testing.T, u *lfsUpstream, repos ...string) *GitHandler { t.Helper() proxy, _, store, _ := setupTestProxy(t) store.seekable = true + proxy.Logger = slog.New(slog.NewTextHandler(&u.logs, nil)) proxy.HTTPClient = u.git.Client() fetcher := fetch.NewFetcher(fetch.WithHTTPClient(u.store.Client()), fetch.WithMaxRetries(0)) proxy.Fetcher = fetcher @@ -211,3 +227,104 @@ func TestGitLFSRefusals(t *testing.T) { t.Errorf("downloads = %d, want none", u.downloads.Load()) } } + +func TestGitLFSKeepsSignedURLsOutOfLogsAndDatabase(t *testing.T) { + u := newLFSUpstream(t) + h := newLFSHandler(t, u) + path := fmt.Sprintf("/up/acme/widget.git/info/lfs/objects/%s/%d", u.oid, len(lfsTestContent)) + + if w := lfsRequest(h, http.MethodGet, path, ""); w.Code != http.StatusOK { + t.Fatalf("GET: status %d, body %s", w.Code, w.Body) + } + _, versionPURL, err := packagePURLStrings(gitEcosystem, "up/acme/widget", u.oid) + if err != nil { + t.Fatal(err) + } + record, err := h.proxy.DB.GetArtifact(versionPURL, u.oid) + if err != nil || record == nil { + t.Fatalf("artifact record: %v, %v", record, err) + } + if want := u.store.URL + "/objects/" + u.oid; record.UpstreamURL != want { + t.Errorf("recorded upstream URL %q, want %q", record.UpstreamURL, want) + } + + // A failed download is logged without the signature too, although the + // connection error names the URL. + other := newLFSUpstream(t) + other.store.Close() + h = newLFSHandler(t, other) + if w := lfsRequest(h, http.MethodGet, path, ""); w.Code == http.StatusOK { + t.Fatalf("GET from an unreachable store: status %d", w.Code) + } + if !strings.Contains(other.logs.String(), other.store.URL+"/objects/"+other.oid) { + t.Fatalf("logs don't name the failed download:\n%s", other.logs.String()) + } + for _, logs := range []string{u.logs.String(), other.logs.String()} { + if strings.Contains(logs, lfsSignature) { + t.Errorf("logs contain the download URL's signature:\n%s", logs) + } + } +} + +func TestGitLFSBatchFromCacheWhenUpstreamFails(t *testing.T) { + u := newLFSUpstream(t) + h := newLFSHandler(t, u) + size := int64(len(lfsTestContent)) + missing := strings.Repeat("0", 64) + batch := lfsBatchBody("download", lfsObject{OID: u.oid, Size: size}, lfsObject{OID: missing, Size: 3}) + const batchPath = "/up/acme/widget.git/info/lfs/objects/batch" + + // Nothing cached yet: a failing upstream fails the batch. + u.batchStatus.Store(http.StatusServiceUnavailable) + if w := lfsRequest(h, http.MethodPost, batchPath, batch); w.Code != http.StatusBadGateway { + t.Fatalf("batch with nothing cached: status %d, body %s; want 502", w.Code, w.Body) + } + u.batchStatus.Store(0) + if w := lfsRequest(h, http.MethodGet, fmt.Sprintf("/up/acme/widget.git/info/lfs/objects/%s/%d", u.oid, size), ""); w.Code != http.StatusOK { + t.Fatalf("GET: status %d, body %s", w.Code, w.Body) + } + + // A refusal is the upstream's answer, not an outage. + u.batchStatus.Store(http.StatusForbidden) + if w := lfsRequest(h, http.MethodPost, batchPath, batch); w.Code != http.StatusBadGateway { + t.Errorf("batch the upstream refused: status %d, body %s; want 502", w.Code, w.Body) + } + + u.git.Close() + w := lfsRequest(h, http.MethodPost, batchPath, batch) + if w.Code != http.StatusOK { + t.Fatalf("batch with the upstream down: status %d, body %s", w.Code, w.Body) + } + var resp lfsBatchResponse + if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil { + t.Fatal(err) + } + if len(resp.Objects) != 2 { + t.Fatalf("objects = %+v, want the two requested", resp.Objects) + } + want := fmt.Sprintf("http://proxy.test/git/up/acme/widget.git/info/lfs/objects/%s/%d", u.oid, size) + if got := resp.Objects[0]; got.Actions["download"].Href != want || !got.Authenticated || got.Error != nil { + t.Errorf("cached object = %+v, want a download from %s", got, want) + } + if got := resp.Objects[1]; got.Actions != nil || got.Error == nil { + t.Errorf("uncached object = %+v, want an error", got) + } +} + +func TestGitLFSHeadDoesNotDownload(t *testing.T) { + u := newLFSUpstream(t) + h := newLFSHandler(t, u) + path := fmt.Sprintf("/up/acme/widget.git/info/lfs/objects/%s/%d", u.oid, len(lfsTestContent)) + + w := lfsRequest(h, http.MethodHead, path, "") + if w.Code != http.StatusOK || w.Header().Get("Content-Length") != fmt.Sprint(len(lfsTestContent)) { + t.Errorf("HEAD: status %d, Content-Length %q", w.Code, w.Header().Get("Content-Length")) + } + if u.downloads.Load() != 0 { + t.Errorf("downloads = %d, want none for a HEAD", u.downloads.Load()) + } + missing := fmt.Sprintf("/up/acme/widget.git/info/lfs/objects/%s/3", strings.Repeat("0", 64)) + if w := lfsRequest(h, http.MethodHead, missing, ""); w.Code != http.StatusNotFound { + t.Errorf("HEAD of a missing object: status %d, want 404", w.Code) + } +} diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 3e5984e..909e76e 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -2332,9 +2332,59 @@ type observedFetcher interface { FetchObservedWithHeaders(ctx context.Context, url string, headers http.Header) (*fetch.ObservedArtifact, error) } -// fetchUpstream fetches downloadURL, reporting its Last-Modified when ctx -// asks for it. +type signedDownloadKey struct{} + +// signedDownload is a download URL carrying a signature, and the URL without +// its query that stands for it everywhere but the request itself. +type signedDownload struct { + signed, recorded string +} + +// withSignedDownload returns a context under which a fetch of the returned +// URL requests signedURL instead. The returned URL, signedURL without its +// query and fragment, is what is logged, recorded with the artifact and used +// to coalesce fetches, so a signature that grants access to the object +// stays out of logs and the database. +func withSignedDownload(ctx context.Context, signedURL string) (context.Context, string) { + u, err := url.Parse(signedURL) + if err != nil { + return ctx, signedURL + } + u.User = nil + u.RawQuery, u.ForceQuery = "", false + u.Fragment, u.RawFragment = "", "" + recorded := u.String() + return context.WithValue(ctx, signedDownloadKey{}, signedDownload{signed: signedURL, recorded: recorded}), recorded +} + +// redactedFetchError is a fetch error whose text names the recorded URL +// rather than the signed one. +type redactedFetchError struct { + err error + download signedDownload +} + +func (e *redactedFetchError) Error() string { + return strings.ReplaceAll(e.err.Error(), e.download.signed, e.download.recorded) +} + +func (e *redactedFetchError) Unwrap() error { return e.err } + +// fetchUpstream fetches downloadURL, or the signed URL ctx gives for it, +// reporting its Last-Modified when ctx asks for it. func (p *Proxy) fetchUpstream(ctx context.Context, downloadURL string, headers http.Header) (*fetch.Artifact, error) { + download, ok := ctx.Value(signedDownloadKey{}).(signedDownload) + if !ok || download.recorded != downloadURL { + return p.fetchUpstreamURL(ctx, downloadURL, headers) + } + artifact, err := p.fetchUpstreamURL(ctx, download.signed, headers) + if err != nil { + return nil, &redactedFetchError{err: err, download: download} + } + return artifact, nil +} + +func (p *Proxy) fetchUpstreamURL(ctx context.Context, downloadURL string, headers http.Header) (*fetch.Artifact, error) { report, _ := ctx.Value(upstreamLastModifiedKey{}).(func(time.Time)) observer, ok := p.Fetcher.(observedFetcher) if report == nil || !ok { From fe4be3ccd61e726c1ab13076eadcab4e6eda30fd Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 25 Oct 2026 16:46:40 +0100 Subject: [PATCH 443/633] Reuse batch download URLs and stored copies for LFS objects - Keep the download URLs of a batch response in memory until shortly before they expire, so the GET that follows needs no second batch request. A kept URL that fails is replaced by a fresh one. - Cut off a download that isn't the object's size. - Copy an object whose content is already stored, such as for a fork, instead of downloading it, once the upstream has confirmed the repository has it. Artifacts are indexed by content hash for this. --- docs/architecture.md | 2 +- docs/configuration.md | 14 +- internal/database/queries.go | 28 ++++ internal/database/schema.go | 15 ++ internal/handler/git.go | 12 +- internal/handler/git_lfs.go | 250 ++++++++++++++++++++++++++----- internal/handler/git_lfs_test.go | 161 +++++++++++++++++++- internal/handler/handler.go | 72 ++++++++- 8 files changed, 499 insertions(+), 55 deletions(-) diff --git a/docs/architecture.md b/docs/architecture.md index 8de5fa6..e7ce8c8 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -308,7 +308,7 @@ HTTP protocol handlers for each registry type. **GitHandler:** - Serves `/git/{name}/{repo}` from `internal/gitcache`, read-only over smart HTTP - Validates repository paths and the `repos` allowlist, and maps cache errors to statuses -- Serves the Git LFS API under `{repo}.git/info/lfs/`: forwards download batches upstream and points each object at `objects/{oid}/{size}`, which is served from the artifact cache (ecosystem `git`, name `{name}/{repo}`, version the object ID); a miss asks the upstream batch API for a fresh download URL, recorded without its signed query, and checks the object against its sha256 ID; a batch the upstream can't answer is answered from the cache +- Serves the Git LFS API under `{repo}.git/info/lfs/`: forwards download batches upstream and points each object at `objects/{oid}/{size}`, which is served from the artifact cache (ecosystem `git`, name `{name}/{repo}`, version the object ID); a miss uses the download URL kept from the batch or asks the upstream batch API for a fresh one, recorded without its signed query, copies the object from another repository when its content is already stored, and checks the object against its sha256 ID; a batch the upstream can't answer is answered from the cache ### `internal/gitcache` diff --git a/docs/configuration.md b/docs/configuration.md index 026bc58..bd53a3e 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -843,10 +843,16 @@ API under the rewritten remote, at `/git/{name}/{repo}.git/info/lfs`, with no further configuration. A batch request is passed to the upstream, which decides which objects the repository has, and each download is pointed at the proxy. An object the cache doesn't have is fetched from the download URL -a fresh batch request returns. That URL is often on another host than the -Git server, such as GitHub's or Hugging Face's object storage; the proxy -fetches from whatever host the upstream names, with the usual address checks, -and `upstream.auth` credentials only go to the hosts they are configured for. +the batch returned, which the proxy keeps in memory until shortly before it +expires, or else from one a fresh batch request returns. That URL is often +on another host than the Git server, such as GitHub's or Hugging Face's +object storage; the proxy fetches from whatever host the upstream names, +with the usual address checks, and `upstream.auth` credentials only go to +the hosts they are configured for. A download that isn't the object's size +is cut off. If the object's content is already stored under another +repository, such as a fork, it is copied from there instead of downloaded, +once the upstream has confirmed this repository has it; the copy takes its +own space. Download URLs are often signed; only the URL without its query is logged and recorded with the object. An object is only cached once its content matches its sha256 object ID. Objects are cached per repository, so one diff --git a/internal/database/queries.go b/internal/database/queries.go index 4b039d8..7af5d07 100644 --- a/internal/database/queries.go +++ b/internal/database/queries.go @@ -265,6 +265,34 @@ func (db *DB) GetCachedArtifact(packagePURL, versionPURL, filename string) (*Cac return row.artifact(versionPURL, filename), nil } +// StoredContent is a stored object with a given content hash. +type StoredContent struct { + StoragePath string `db:"storage_path"` + Size int64 `db:"size"` + ContentType sql.NullString `db:"content_type"` +} + +// FindStoredContent returns a cached artifact whose content has the sha256 +// hash, under any name, or nil if none is stored. Pushed artifacts are left +// out. +func (db *DB) FindStoredContent(hash string) (*StoredContent, error) { + var found StoredContent + query := db.Rebind(` + SELECT storage_path, size, content_type FROM artifacts + WHERE content_hash = ? AND storage_path IS NOT NULL AND fetched_at IS NOT NULL + AND size IS NOT NULL AND source <> 'local' + LIMIT 1 + `) + err := db.Get(&found, query, hash) + if err == sql.ErrNoRows { + return nil, nil + } + if err != nil { + return nil, err + } + return &found, nil +} + type cachedArtifactRow struct { Ecosystem string `db:"ecosystem"` StoragePath string `db:"storage_path"` diff --git a/internal/database/schema.go b/internal/database/schema.go index b2fef63..46a21ec 100644 --- a/internal/database/schema.go +++ b/internal/database/schema.go @@ -79,6 +79,7 @@ CREATE TABLE IF NOT EXISTS artifacts ( ); CREATE UNIQUE INDEX IF NOT EXISTS idx_artifacts_version_filename ON artifacts(version_purl, filename); CREATE INDEX IF NOT EXISTS idx_artifacts_storage_path ON artifacts(storage_path); +CREATE INDEX IF NOT EXISTS idx_artifacts_content_hash ON artifacts(content_hash); CREATE INDEX IF NOT EXISTS idx_artifacts_last_accessed ON artifacts(last_accessed_at); CREATE TABLE IF NOT EXISTS pending_deletes ( @@ -220,6 +221,7 @@ CREATE TABLE IF NOT EXISTS artifacts ( ); CREATE UNIQUE INDEX IF NOT EXISTS idx_artifacts_version_filename ON artifacts(version_purl, filename); CREATE INDEX IF NOT EXISTS idx_artifacts_storage_path ON artifacts(storage_path); +CREATE INDEX IF NOT EXISTS idx_artifacts_content_hash ON artifacts(content_hash); CREATE INDEX IF NOT EXISTS idx_artifacts_last_accessed ON artifacts(last_accessed_at); CREATE TABLE IF NOT EXISTS pending_deletes ( @@ -320,6 +322,7 @@ CREATE TABLE IF NOT EXISTS artifacts ( ); CREATE UNIQUE INDEX IF NOT EXISTS idx_artifacts_version_filename ON artifacts(version_purl, filename); CREATE INDEX IF NOT EXISTS idx_artifacts_storage_path ON artifacts(storage_path); +CREATE INDEX IF NOT EXISTS idx_artifacts_content_hash ON artifacts(content_hash); CREATE INDEX IF NOT EXISTS idx_artifacts_last_accessed ON artifacts(last_accessed_at); ` @@ -343,6 +346,7 @@ CREATE TABLE IF NOT EXISTS artifacts ( ); CREATE UNIQUE INDEX IF NOT EXISTS idx_artifacts_version_filename ON artifacts(version_purl, filename); CREATE INDEX IF NOT EXISTS idx_artifacts_storage_path ON artifacts(storage_path); +CREATE INDEX IF NOT EXISTS idx_artifacts_content_hash ON artifacts(content_hash); CREATE INDEX IF NOT EXISTS idx_artifacts_last_accessed ON artifacts(last_accessed_at); ` @@ -457,6 +461,7 @@ var migrations = []migration{ {"014_add_metadata_last_accessed", migrateAddMetadataLastAccessed}, {"015_add_job_runs", migrateAddJobRuns}, {"016_add_fetch_leases", migrateAddFetchLeases}, + {"017_add_artifact_content_hash_index", migrateAddArtifactContentHashIndex}, {"018_add_upstream_holds", migrateAddUpstreamHolds}, } @@ -820,6 +825,16 @@ func migrateAddPendingDeletesQueuedAtIndex(db *DB) error { return nil } +// migrateAddArtifactContentHashIndex indexes artifacts by content, so an +// artifact already stored under another name can be found by its hash. +func migrateAddArtifactContentHashIndex(db *DB) error { + _, err := db.Exec(`CREATE INDEX IF NOT EXISTS idx_artifacts_content_hash ON artifacts(content_hash)`) + if err != nil { + return fmt.Errorf("creating artifacts content_hash index: %w", err) + } + return nil +} + // migrateAddMetadataDates creates the table of Last-Modified dates for // rewritten metadata, which replicas sharing the database agree on. func migrateAddMetadataDates(db *DB) error { diff --git a/internal/handler/git.go b/internal/handler/git.go index 0e4b493..7243d84 100644 --- a/internal/handler/git.go +++ b/internal/handler/git.go @@ -41,6 +41,9 @@ type GitHandler struct { upstreams map[string]gitUpstream // baseURL is where clients reach the proxy, for LFS download URLs. baseURL string + // lfsDownloads keeps upstream LFS download URLs between a batch and + // the GETs that follow it. + lfsDownloads *lfsDownloads // writeStall is gitWriteStallTimeout, shorter in tests. writeStall time.Duration } @@ -58,10 +61,11 @@ type gitUpstream struct { // NewGitHandler creates a Git handler serving upstream.git from cache. func NewGitHandler(proxy *Proxy, cache *gitcache.Cache, upstream config.UpstreamConfig) *GitHandler { h := &GitHandler{ - proxy: proxy, - cache: cache, - upstreams: make(map[string]gitUpstream, len(upstream.Git)), - writeStall: gitWriteStallTimeout, + proxy: proxy, + cache: cache, + upstreams: make(map[string]gitUpstream, len(upstream.Git)), + writeStall: gitWriteStallTimeout, + lfsDownloads: &lfsDownloads{}, } for name, git := range upstream.Git { git.URL = strings.TrimSuffix(git.URL, "/") diff --git a/internal/handler/git_lfs.go b/internal/handler/git_lfs.go index 5b9be90..d4f6fe6 100644 --- a/internal/handler/git_lfs.go +++ b/internal/handler/git_lfs.go @@ -12,6 +12,8 @@ import ( "slices" "strconv" "strings" + "sync" + "time" ) const ( @@ -64,8 +66,10 @@ type lfsObjectResponse struct { } type lfsAction struct { - Href string `json:"href"` - Header map[string]string `json:"header,omitempty"` + Href string `json:"href"` + Header map[string]string `json:"header,omitempty"` + ExpiresIn int64 `json:"expires_in,omitempty"` + ExpiresAt string `json:"expires_at,omitempty"` } type lfsObjectError struct { @@ -176,6 +180,9 @@ func (h *GitHandler) serveLFSBatch(w http.ResponseWriter, r *http.Request, repo if download, ok := object.Actions["download"]; ok && object.Error == nil && download.Href != "" { out.Authenticated = true out.Actions = map[string]lfsAction{"download": {Href: h.lfsObjectURL(repo, object.OID, object.Size)}} + // Kept so the client's GET for the object needs no second + // batch request. + h.lfsDownloads.put(repo.key, lfsObject{OID: object.OID, Size: object.Size}, download, time.Now()) } resp.Objects = append(resp.Objects, out) } @@ -226,52 +233,116 @@ func writeLFSBatch(w http.ResponseWriter, resp lfsBatchResponse) { _ = json.NewEncoder(w).Encode(resp) } -// serveLFSObject serves an object from the artifact cache, or on a miss asks -// the upstream batch API for a fresh download URL, fetches it and checks it -// against its object ID before caching it. +// serveLFSObject serves an object from the artifact cache. On a miss it +// takes the download URL from the client's batch, or asks the upstream batch +// API for a fresh one, and caches the object from there, or from a copy +// already stored under another repository, once its content matches its +// object ID. func (h *GitHandler) serveLFSObject(w http.ResponseWriter, r *http.Request, repo gitRepository, object lfsObject) { - name := repo.lfsName() - cached, err := h.proxy.GetCachedArtifact(r.Context(), gitEcosystem, name, object.OID, object.OID) + cached, err := h.proxy.GetCachedArtifact(r.Context(), gitEcosystem, repo.lfsName(), object.OID, object.OID) if err != nil { h.proxy.serveArtifactError(w, err, "failed to read cached LFS object") return } if cached == nil { - upstream, err := h.lfsBatch(r.Context(), repo, nil, []lfsObject{object}) - if err != nil { - h.serveLFSUpstreamError(w, r, repo, err) - return - } - download, status, msg := lfsDownload(upstream, object) - if status != 0 { - lfsError(w, status, msg) - return - } - if r.Method == http.MethodHead { - // The upstream has the object, and a HEAD needs no more than - // that: fetching it is left to a GET. - w.Header().Set("Content-Type", gitLFSObjectMediaType) - w.Header().Set("Content-Length", strconv.FormatInt(object.Size, 10)) - return - } - header := make(http.Header, len(download.Header)) - for key, value := range download.Header { - header.Set(key, value) - } - // Download URLs are often signed, granting access to the object - // until they expire, so only the URL without its query is logged - // and recorded. - ctx, recorded := withSignedDownload(r.Context(), download.Href) - cached, err = h.proxy.getOrFetchArtifactFromURL(ctx, gitEcosystem, name, object.OID, object.OID, - recorded, header, object.OID, "") - if err != nil { - h.proxy.serveArtifactError(w, err, "failed to fetch LFS object") + var ok bool + if cached, ok = h.fetchLFSObject(w, r, repo, object); !ok { return } } serveGenericArtifact(w, r, cached) } +// fetchLFSObject caches an object the cache doesn't have. When it can't, or +// the request is a HEAD, it answers the request and returns false. +func (h *GitHandler) fetchLFSObject(w http.ResponseWriter, r *http.Request, repo gitRepository, object lfsObject) (*CacheResult, bool) { + download, kept := h.lfsDownloads.get(repo.key, object, time.Now()) + if !kept { + var ok bool + if download, ok = h.lfsFreshDownload(w, r, repo, object); !ok { + return nil, false + } + } + if r.Method == http.MethodHead { + // The upstream has the object, and a HEAD needs no more than that: + // fetching it is left to a GET. + w.Header().Set("Content-Type", gitLFSObjectMediaType) + w.Header().Set("Content-Length", strconv.FormatInt(object.Size, 10)) + return nil, false + } + h.lfsDownloads.drop(repo.key, object.OID) + cached, err := h.cacheLFSObject(r.Context(), repo, object, download) + if err != nil && kept && retryLFSDownload(r.Context(), err) { + // The kept URL may have expired early or been revoked. + var ok bool + if download, ok = h.lfsFreshDownload(w, r, repo, object); !ok { + return nil, false + } + cached, err = h.cacheLFSObject(r.Context(), repo, object, download) + } + if err != nil { + h.proxy.serveArtifactError(w, err, "failed to fetch LFS object") + return nil, false + } + return cached, true +} + +// lfsFreshDownload asks the upstream batch API for object's download. When +// there is none it answers the request and returns false. +func (h *GitHandler) lfsFreshDownload(w http.ResponseWriter, r *http.Request, repo gitRepository, object lfsObject) (lfsAction, bool) { + upstream, err := h.lfsBatch(r.Context(), repo, nil, []lfsObject{object}) + if err != nil { + h.serveLFSUpstreamError(w, r, repo, err) + return lfsAction{}, false + } + download, status, msg := lfsDownload(upstream, object) + if status != 0 { + lfsError(w, status, msg) + return lfsAction{}, false + } + return download, true +} + +// cacheLFSObject caches object, which the upstream offers as download. A +// copy already stored under another name, such as by a fork of the +// repository, saves the download. +func (h *GitHandler) cacheLFSObject(ctx context.Context, repo gitRepository, object lfsObject, download lfsAction) (*CacheResult, error) { + header := make(http.Header, len(download.Header)) + for key, value := range download.Header { + header.Set(key, value) + } + // Download URLs are often signed, granting access to the object until + // they expire, so only the URL without its query is logged and + // recorded. The object must have the size the client asked for. + ctx, recorded := withSignedDownload(ctx, download.Href, object.Size) + if !h.proxy.StreamArtifacts { + stored, err := h.proxy.DB.FindStoredContent(object.OID) + if err != nil { + h.proxy.Logger.Warn("git: looking up a stored copy of an LFS object failed", "oid", object.OID, "error", err) + } + if stored != nil && stored.Size == object.Size { + cached, err := h.proxy.cacheStoredCopy(ctx, gitEcosystem, repo.lfsName(), object.OID, object.OID, + recorded, object.OID, stored) + if err == nil { + return cached, nil + } + h.proxy.Logger.Warn("git: copying a stored LFS object failed, downloading it", "oid", object.OID, "error", err) + } + } + return h.proxy.getOrFetchArtifactFromURL(ctx, gitEcosystem, repo.lfsName(), object.OID, object.OID, + recorded, header, object.OID, "") +} + +// retryLFSDownload reports whether a download from a kept URL failed in a way +// a fresh URL might not. +func retryLFSDownload(ctx context.Context, err error) bool { + return ctx.Err() == nil && + !errors.Is(err, ErrArtifactDigestMismatch) && + !errors.Is(err, errDownloadSize) && + !errors.Is(err, ErrVersionDenied) && + !errors.Is(err, ErrArtifactBlocked) +} + // lfsDownload returns the download action the upstream gave for object, or // the status and message to answer with when it gave none. func lfsDownload(resp *lfsBatchResponse, object lfsObject) (lfsAction, int, string) { @@ -286,8 +357,7 @@ func lfsDownload(resp *lfsBatchResponse, object lfsObject) (lfsAction, int, stri return lfsAction{}, http.StatusBadGateway, "upstream refused the object" } download, ok := got.Actions["download"] - u, err := url.Parse(download.Href) - if !ok || err != nil || (u.Scheme != "https" && u.Scheme != "http") || u.Host == "" { + if !ok || !validLFSHref(download.Href) { return lfsAction{}, http.StatusBadGateway, "upstream gave no download for the object" } return download, 0, "" @@ -295,6 +365,12 @@ func lfsDownload(resp *lfsBatchResponse, object lfsObject) (lfsAction, int, stri return lfsAction{}, http.StatusNotFound, "object not found" } +// validLFSHref reports whether href is an absolute HTTP(S) URL. +func validLFSHref(href string) bool { + u, err := url.Parse(href) + return err == nil && (u.Scheme == "https" || u.Scheme == "http") && u.Host != "" +} + // lfsBatch asks the upstream's LFS batch API for download actions for // objects. upstream.auth applies to it as to the repository. func (h *GitHandler) lfsBatch(ctx context.Context, repo gitRepository, ref json.RawMessage, objects []lfsObject) (*lfsBatchResponse, error) { @@ -383,3 +459,101 @@ func lfsError(w http.ResponseWriter, status int, msg string) { w.WriteHeader(status) _ = json.NewEncoder(w).Encode(map[string]string{"message": msg}) } + +const ( + // gitLFSKeptDownloads bounds how many download URLs from batch responses + // are kept for the GETs that follow them. + gitLFSKeptDownloads = 4096 + // gitLFSKeptTTL is how long a download URL is kept when the upstream + // gives no expiry, and gitLFSKeptMaxTTL how long at most. + gitLFSKeptTTL = 5 * time.Minute + gitLFSKeptMaxTTL = time.Hour + // gitLFSExpiryMargin is how long before its expiry a URL is no longer + // used, so a download started with it can finish. + gitLFSExpiryMargin = 30 * time.Second +) + +// lfsDownloads keeps the download actions of recent batch responses, so the +// GET for an object a client just got in a batch needs no second batch +// request. They stay in memory, never reach a client and expire with the +// URLs. +type lfsDownloads struct { + mu sync.Mutex + entries map[lfsDownloadKey]lfsKeptDownload +} + +type lfsDownloadKey struct { + repo, oid string +} + +type lfsKeptDownload struct { + size int64 + action lfsAction + expires time.Time +} + +// put keeps action for object of repo until shortly before it expires. +func (d *lfsDownloads) put(repo string, object lfsObject, action lfsAction, now time.Time) { + if !validLFSHref(action.Href) { + return + } + expires := now.Add(gitLFSKeptTTL) + if action.ExpiresIn > 0 || action.ExpiresAt != "" { + expires = now.Add(gitLFSKeptMaxTTL) + } + if action.ExpiresIn > 0 { + expires = minTime(expires, now.Add(time.Duration(action.ExpiresIn)*time.Second)) + } + if at, err := time.Parse(time.RFC3339, action.ExpiresAt); err == nil { + expires = minTime(expires, at) + } + expires = expires.Add(-gitLFSExpiryMargin) + if !expires.After(now) { + return + } + d.mu.Lock() + defer d.mu.Unlock() + if d.entries == nil { + d.entries = make(map[lfsDownloadKey]lfsKeptDownload) + } + if len(d.entries) >= gitLFSKeptDownloads { + for key, kept := range d.entries { + if !kept.expires.After(now) { + delete(d.entries, key) + } + } + } + if len(d.entries) >= gitLFSKeptDownloads { + // Still full: make room by dropping any one. + for key := range d.entries { + delete(d.entries, key) + break + } + } + d.entries[lfsDownloadKey{repo, object.OID}] = lfsKeptDownload{size: object.Size, action: action, expires: expires} +} + +// get returns the action kept for object of repo, if it hasn't expired. +func (d *lfsDownloads) get(repo string, object lfsObject, now time.Time) (lfsAction, bool) { + d.mu.Lock() + defer d.mu.Unlock() + kept, ok := d.entries[lfsDownloadKey{repo, object.OID}] + if !ok || kept.size != object.Size || !kept.expires.After(now) { + return lfsAction{}, false + } + return kept.action, true +} + +// drop forgets the action kept for oid of repo, once a GET has used it. +func (d *lfsDownloads) drop(repo, oid string) { + d.mu.Lock() + defer d.mu.Unlock() + delete(d.entries, lfsDownloadKey{repo, oid}) +} + +func minTime(a, b time.Time) time.Time { + if b.Before(a) { + return b + } + return a +} diff --git a/internal/handler/git_lfs_test.go b/internal/handler/git_lfs_test.go index 62cb3aa..37bc6aa 100644 --- a/internal/handler/git_lfs_test.go +++ b/internal/handler/git_lfs_test.go @@ -5,6 +5,7 @@ import ( "crypto/sha256" "encoding/hex" "encoding/json" + "errors" "fmt" "io" "log/slog" @@ -13,6 +14,7 @@ import ( "strings" "sync/atomic" "testing" + "time" "github.com/git-pkgs/proxy/internal/config" "github.com/git-pkgs/registries/fetch" @@ -35,6 +37,9 @@ type lfsUpstream struct { downloads atomic.Int32 // batchStatus, when set, is the status the batch API fails with. batchStatus atomic.Int32 + // signature is the signature the batch API hands out and the store + // accepts; changing it expires the URLs handed out before. + signature atomic.Value // lastBatch is the last batch request the upstream got. lastBatch atomic.Value // logs is what the proxy logged. @@ -46,9 +51,10 @@ func newLFSUpstream(t *testing.T) *lfsUpstream { sum := sha256.Sum256([]byte(lfsTestContent)) u := &lfsUpstream{oid: hex.EncodeToString(sum[:])} u.content.Store(lfsTestContent) + u.signature.Store(lfsSignature) u.store = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.URL.Path != "/objects/"+u.oid || r.Header.Get("X-Signed") != "token" || - r.URL.Query().Get("Signature") != lfsSignature { + r.URL.Query().Get("Signature") != u.signature.Load().(string) { http.Error(w, "forbidden", http.StatusForbidden) return } @@ -56,7 +62,11 @@ func newLFSUpstream(t *testing.T) *lfsUpstream { _, _ = io.WriteString(w, u.content.Load().(string)) })) u.git = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.Method != http.MethodPost || r.URL.Path != "/acme/widget.git/info/lfs/objects/batch" { + // acme/widget has the object, as does its fork; stranger/widget + // exists but doesn't have it. + repo, _, _ := strings.Cut(r.URL.Path, ".git/info/lfs/objects/batch") + hasObject, exists := map[string]bool{"/acme/widget": true, "/fork/widget": true, "/stranger/widget": false}[repo] + if r.Method != http.MethodPost || !exists { http.NotFound(w, r) return } @@ -73,14 +83,14 @@ func newLFSUpstream(t *testing.T) *lfsUpstream { u.lastBatch.Store(req) resp := lfsBatchResponse{Transfer: "basic"} for _, object := range req.Objects { - if object.OID != u.oid { + if object.OID != u.oid || !hasObject { resp.Objects = append(resp.Objects, lfsObjectResponse{OID: object.OID, Size: object.Size, Error: &lfsObjectError{Code: http.StatusNotFound, Message: "Object does not exist"}}) continue } resp.Objects = append(resp.Objects, lfsObjectResponse{OID: object.OID, Size: object.Size, Actions: map[string]lfsAction{"download": { - Href: u.store.URL + "/objects/" + u.oid + "?Expires=1&Signature=" + lfsSignature, + Href: u.store.URL + "/objects/" + u.oid + "?Expires=1&Signature=" + u.signature.Load().(string), Header: map[string]string{"X-Signed": "token"}, }}}) } @@ -328,3 +338,146 @@ func TestGitLFSHeadDoesNotDownload(t *testing.T) { t.Errorf("HEAD of a missing object: status %d, want 404", w.Code) } } + +func TestGitLFSObjectAfterBatchNeedsNoSecondBatch(t *testing.T) { + u := newLFSUpstream(t) + h := newLFSHandler(t, u) + size := int64(len(lfsTestContent)) + path := fmt.Sprintf("/up/acme/widget.git/info/lfs/objects/%s/%d", u.oid, size) + batch := lfsBatchBody("download", lfsObject{OID: u.oid, Size: size}) + + if w := lfsRequest(h, http.MethodPost, "/up/acme/widget.git/info/lfs/objects/batch", batch); w.Code != http.StatusOK { + t.Fatalf("batch: status %d, body %s", w.Code, w.Body) + } + if w := lfsRequest(h, http.MethodGet, path, ""); w.Code != http.StatusOK || w.Body.String() != lfsTestContent { + t.Fatalf("GET: status %d, body %q", w.Code, w.Body) + } + if u.batches.Load() != 1 || u.downloads.Load() != 1 { + t.Errorf("batches %d, downloads %d; want the client's batch and one download", u.batches.Load(), u.downloads.Load()) + } + + // A URL that stopped working before its time is replaced by a fresh one. + other := newLFSUpstream(t) + h = newLFSHandler(t, other) + if w := lfsRequest(h, http.MethodPost, "/up/acme/widget.git/info/lfs/objects/batch", batch); w.Code != http.StatusOK { + t.Fatalf("batch: status %d, body %s", w.Code, w.Body) + } + other.signature.Store("rotated") + if w := lfsRequest(h, http.MethodGet, path, ""); w.Code != http.StatusOK || w.Body.String() != lfsTestContent { + t.Fatalf("GET with a revoked URL: status %d, body %q", w.Code, w.Body) + } + if other.batches.Load() != 2 { + t.Errorf("batches = %d, want a second one for a fresh URL", other.batches.Load()) + } +} + +func TestGitLFSKeptDownloadsExpire(t *testing.T) { + now := time.Date(2026, 10, 25, 12, 0, 0, 0, time.UTC) + object := lfsObject{OID: strings.Repeat("a", 64), Size: 3} + href := "https://objects.example/a?sig=1" + tests := []struct { + name string + action lfsAction + at time.Duration + kept bool + }{ + {"no expiry", lfsAction{Href: href}, 4 * time.Minute, true}, + {"no expiry, later", lfsAction{Href: href}, 5 * time.Minute, false}, + {"expires_in", lfsAction{Href: href, ExpiresIn: 600}, 9 * time.Minute, true}, + {"expires_in, within the margin", lfsAction{Href: href, ExpiresIn: 600}, 10*time.Minute - 10*time.Second, false}, + {"expires_in too short to use", lfsAction{Href: href, ExpiresIn: 10}, 0, false}, + {"expires_at", lfsAction{Href: href, ExpiresAt: now.Add(20 * time.Minute).Format(time.RFC3339)}, 19 * time.Minute, true}, + {"expires_at, past", lfsAction{Href: href, ExpiresAt: now.Add(20 * time.Minute).Format(time.RFC3339)}, 20 * time.Minute, false}, + {"long expiry capped", lfsAction{Href: href, ExpiresIn: 86400}, 2 * time.Hour, false}, + {"relative href", lfsAction{Href: "/a"}, 0, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var d lfsDownloads + d.put("repo", object, tt.action, now) + if _, kept := d.get("repo", object, now.Add(tt.at)); kept != tt.kept { + t.Errorf("kept = %v, want %v", kept, tt.kept) + } + }) + } + + var d lfsDownloads + d.put("repo", object, lfsAction{Href: href}, now) + if _, kept := d.get("repo", lfsObject{OID: object.OID, Size: 4}, now); kept { + t.Error("kept for another size") + } + if _, kept := d.get("other", object, now); kept { + t.Error("kept for another repository") + } + for i := range gitLFSKeptDownloads + 10 { + d.put("repo", lfsObject{OID: fmt.Sprintf("%064x", i), Size: 1}, lfsAction{Href: href}, now) + } + if len(d.entries) > gitLFSKeptDownloads { + t.Errorf("%d entries kept, want at most %d", len(d.entries), gitLFSKeptDownloads) + } +} + +func TestGitLFSObjectOfAnotherSizeIsRefused(t *testing.T) { + u := newLFSUpstream(t) + h := newLFSHandler(t, u) + path := fmt.Sprintf("/up/acme/widget.git/info/lfs/objects/%s/%d", u.oid, len(lfsTestContent)-1) + + if w := lfsRequest(h, http.MethodGet, path, ""); w.Code == http.StatusOK { + t.Fatalf("GET at the wrong size: status %d, body %q", w.Code, w.Body) + } + if u.downloads.Load() != 1 || u.batches.Load() != 1 { + t.Errorf("downloads %d, batches %d; want one of each, without a retry", u.downloads.Load(), u.batches.Load()) + } +} + +func TestExactSizeBody(t *testing.T) { + for _, tt := range []struct { + body string + size int64 + ok bool + }{ + {"abcdef", 6, true}, + {"abcdef", 5, false}, + {"abcdef", 7, false}, + {"", 0, true}, + {"a", 0, false}, + } { + got, err := io.ReadAll(&exactSizeBody{ReadCloser: io.NopCloser(strings.NewReader(tt.body)), left: tt.size}) + if (err == nil) != tt.ok || (err != nil && !errors.Is(err, errDownloadSize)) { + t.Errorf("%q at size %d: err %v, want ok %v", tt.body, tt.size, err, tt.ok) + } + if int64(len(got)) > tt.size+1 { + t.Errorf("%q at size %d: read %d bytes", tt.body, tt.size, len(got)) + } + } +} + +func TestGitLFSObjectCopiedFromAnotherRepository(t *testing.T) { + u := newLFSUpstream(t) + h := newLFSHandler(t, u) + object := func(repo string) string { + return fmt.Sprintf("/up/%s.git/info/lfs/objects/%s/%d", repo, u.oid, len(lfsTestContent)) + } + + if w := lfsRequest(h, http.MethodGet, object("acme/widget"), ""); w.Code != http.StatusOK { + t.Fatalf("GET: status %d, body %s", w.Code, w.Body) + } + w := lfsRequest(h, http.MethodGet, object("fork/widget"), "") + if w.Code != http.StatusOK || w.Body.String() != lfsTestContent { + t.Fatalf("GET from the fork: status %d, body %q", w.Code, w.Body) + } + if u.downloads.Load() != 1 || u.batches.Load() != 2 { + t.Errorf("downloads %d, batches %d; want the fork's object copied after its own batch", u.downloads.Load(), u.batches.Load()) + } + cached, err := h.proxy.GetCachedArtifact(t.Context(), gitEcosystem, "up/fork/widget", u.oid, u.oid) + if err != nil || cached == nil { + t.Fatalf("fork's object not cached: %v, %v", cached, err) + } + _ = cached.Reader.Close() + + // A repository the upstream says lacks the object doesn't get it, though + // a copy is stored. + if w := lfsRequest(h, http.MethodGet, object("stranger/widget"), ""); w.Code != http.StatusNotFound { + t.Errorf("GET from a repository without the object: status %d, body %s; want 404", w.Code, w.Body) + } +} diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 909e76e..363314e 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -2252,6 +2252,33 @@ func (p *Proxy) coalescedFetchFromURL(ctx context.Context, ecosystem, name, vers }) } +// cacheStoredCopy caches a copy of the object stored at source under a new +// name, instead of downloading the same bytes again. It shares the coalescing +// key a download from upstreamURL would use, so the two never run at once. +func (p *Proxy) cacheStoredCopy(ctx context.Context, ecosystem, name, version, filename, upstreamURL, upstreamHash string, source *database.StoredContent) (*CacheResult, error) { + pkgPURL, versionPURL, err := packagePURLStrings(ecosystem, name, version) + if err != nil { + return nil, err + } + if p.versionDenied(ecosystem, name, version) || p.Denylist.Denied(versionPURL) { + return nil, fmt.Errorf("%w: %s", ErrVersionDenied, versionPURL) + } + key := artifactCoalesceKey(versionPURL, filename, upstreamURL, upstreamHash, "") + recheck := func() (artifacts.Artifact, string, bool) { + return p.cachedArtifactRecord(pkgPURL, versionPURL, filename, upstreamHash) + } + return p.coalesceFetch(ctx, key, recheck, func(fetchCtx context.Context) (artifacts.Artifact, string, error) { + body, err := p.Storage.Open(fetchCtx, source.StoragePath) + if err != nil { + return artifacts.Artifact{}, "", fmt.Errorf("opening stored copy: %w", err) + } + p.Logger.Info("caching a stored copy", "ecosystem", ecosystem, "name", name, "version", version, + "source", source.StoragePath) + return p.storeArtifact(fetchCtx, ecosystem, name, version, filename, pkgPURL, versionPURL, upstreamURL, upstreamHash, "", + &fetch.Artifact{Body: body, Size: source.Size, ContentType: source.ContentType.String}) + }) +} + // getCachedArtifactWithUpstreamHash returns a cached artifact whose recorded // content hash matches the checksum the upstream currently declares for it. // This detects an upstream re-publishing under the same version, which the @@ -2335,17 +2362,24 @@ type observedFetcher interface { type signedDownloadKey struct{} // signedDownload is a download URL carrying a signature, and the URL without -// its query that stands for it everywhere but the request itself. +// its query that stands for it everywhere but the request itself. size, when +// not negative, is the exact size the download must have. type signedDownload struct { signed, recorded string + size int64 } +// errDownloadSize is a download larger or smaller than the size it was +// expected to have. +var errDownloadSize = errors.New("download does not have the expected size") + // withSignedDownload returns a context under which a fetch of the returned // URL requests signedURL instead. The returned URL, signedURL without its // query and fragment, is what is logged, recorded with the artifact and used // to coalesce fetches, so a signature that grants access to the object -// stays out of logs and the database. -func withSignedDownload(ctx context.Context, signedURL string) (context.Context, string) { +// stays out of logs and the database. A download of other than size bytes +// is cut off with errDownloadSize, unless size is negative. +func withSignedDownload(ctx context.Context, signedURL string, size int64) (context.Context, string) { u, err := url.Parse(signedURL) if err != nil { return ctx, signedURL @@ -2354,7 +2388,7 @@ func withSignedDownload(ctx context.Context, signedURL string) (context.Context, u.RawQuery, u.ForceQuery = "", false u.Fragment, u.RawFragment = "", "" recorded := u.String() - return context.WithValue(ctx, signedDownloadKey{}, signedDownload{signed: signedURL, recorded: recorded}), recorded + return context.WithValue(ctx, signedDownloadKey{}, signedDownload{signed: signedURL, recorded: recorded, size: size}), recorded } // redactedFetchError is a fetch error whose text names the recorded URL @@ -2381,9 +2415,39 @@ func (p *Proxy) fetchUpstream(ctx context.Context, downloadURL string, headers h if err != nil { return nil, &redactedFetchError{err: err, download: download} } + if download.size < 0 { + return artifact, nil + } + if artifact.Size >= 0 && artifact.Size != download.size { + _ = artifact.Body.Close() + return nil, fmt.Errorf("%w: upstream announced %d bytes, want %d", errDownloadSize, artifact.Size, download.size) + } + artifact.Body = &exactSizeBody{ReadCloser: artifact.Body, left: download.size} return artifact, nil } +// exactSizeBody fails a body that turns out longer or shorter than left +// bytes, without reading more than one byte past it. +type exactSizeBody struct { + io.ReadCloser + left int64 +} + +func (b *exactSizeBody) Read(p []byte) (int, error) { + if int64(len(p)) > b.left+1 { + p = p[:b.left+1] + } + n, err := b.ReadCloser.Read(p) + b.left -= int64(n) + switch { + case b.left < 0: + return n, fmt.Errorf("%w: upstream sent more bytes", errDownloadSize) + case errors.Is(err, io.EOF) && b.left > 0: + return n, fmt.Errorf("%w: upstream sent %d bytes too few", errDownloadSize, b.left) + } + return n, err +} + func (p *Proxy) fetchUpstreamURL(ctx context.Context, downloadURL string, headers http.Header) (*fetch.Artifact, error) { report, _ := ctx.Value(upstreamLastModifiedKey{}).(func(time.Time)) observer, ok := p.Fetcher.(observedFetcher) From bcda24e15c8dd639745ebe57c0a8e8b9b93db5f5 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 25 Oct 2026 16:53:11 +0100 Subject: [PATCH 444/633] Document timeouts for large LFS objects --- docs/configuration.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/configuration.md b/docs/configuration.md index bd53a3e..269d5bd 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -853,6 +853,14 @@ is cut off. If the object's content is already stored under another repository, such as a fork, it is copied from there instead of downloaded, once the upstream has confirmed this repository has it; the copy takes its own space. + +Large LFS objects may need longer timeouts. The proxy downloads an object it +doesn't have before sending any of it, within `artifact_timeout`, and the +whole response, download included, must fit within `write_timeout`; both +default to 5 minutes. git-lfs gives up on a response after 30 seconds without +data (`lfs.activitytimeout`) and retries. A retry waits for the download still +running, so the object is served once it is cached, but a download that +outlasts git-lfs's retries fails that clone or pull until it finishes. Download URLs are often signed; only the URL without its query is logged and recorded with the object. An object is only cached once its content matches its sha256 object ID. Objects are cached per repository, so one From 17e4fc763f24d3f643b4f8b541c5eedba1ae933d Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 25 Oct 2026 19:37:22 +0100 Subject: [PATCH 445/633] Mirror Helm charts Mirroring pkg:helm PURLs failed with an unsupported ecosystem error. The namespace now names the upstream.helm repository, as the Helm handler caches charts per repository: pkg:helm/{repository}/{chart}@{version}. The chart is found in the repository's index through the handler's own index lookup, fetched from the first URL the release lists, as helm does, checked against the listed digest and cached under the repository, digest and filename the handler serves it from. The index goes through the metadata cache, so helm finds the chart there while the repository is down. Unversioned PURLs list every version in the index. Charts without a repository, from an unconfigured one, or listed in an OCI registry are refused; the last are pulled through /v2, and the error names the pkg:oci PURL that mirrors them. --- cmd/proxy/main.go | 22 ++- docs/configuration.md | 2 + internal/handler/helm.go | 36 ++++- internal/handler/helm_mirror.go | 67 ++++++++++ internal/handler/helm_mirror_test.go | 17 +++ internal/handler/helm_pypi_rewrite_test.go | 3 +- internal/mirror/helm.go | 40 ++++++ internal/mirror/mirror.go | 7 + internal/mirror/versions.go | 6 + internal/server/mirror_fetch_test.go | 4 +- internal/server/mirror_helm_test.go | 147 +++++++++++++++++++++ internal/server/mirror_versions.go | 12 ++ internal/server/mirror_versions_test.go | 5 +- 13 files changed, 353 insertions(+), 15 deletions(-) create mode 100644 internal/handler/helm_mirror.go create mode 100644 internal/handler/helm_mirror_test.go create mode 100644 internal/mirror/helm.go create mode 100644 internal/server/mirror_helm_test.go diff --git a/cmd/proxy/main.go b/cmd/proxy/main.go index a61534a..6832dd3 100644 --- a/cmd/proxy/main.go +++ b/cmd/proxy/main.go @@ -125,6 +125,7 @@ import ( "github.com/git-pkgs/proxy/internal/config" "github.com/git-pkgs/proxy/internal/database" "github.com/git-pkgs/proxy/internal/denylist" + "github.com/git-pkgs/proxy/internal/handler" "github.com/git-pkgs/proxy/internal/memlimit" "github.com/git-pkgs/proxy/internal/mirror" "github.com/git-pkgs/proxy/internal/server" @@ -558,18 +559,13 @@ func runMirror() { proxy.MetadataTTL = cfg.ParseMetadataTTL() proxy.MetadataMaxSize = cfg.ParseMetadataMaxSize() - m := mirror.New(proxy, db, store, logger, *concurrency) - m.Images = images - m.Versions = server.NewMirrorVersions(cfg.Upstream, proxy.HTTPClient) + m := newCommandMirror(cfg, proxy, images, db, store, logger, *concurrency) switch s := source.(type) { case *mirror.PURLSource: s.Versions = m.Versions case *mirror.SBOMSource: s.Versions = m.Versions } - if cfg.Scanning.Enabled { - refuseUnscannedMirroring(m) - } ctx, cancel := context.WithCancel(context.Background()) go func() { @@ -616,6 +612,20 @@ func runMirror() { } } +// newCommandMirror builds the mirror proxy mirror runs, which fetches and +// lists versions from the configured upstreams. +func newCommandMirror(cfg *config.Config, proxy *handler.Proxy, images *handler.ContainerHandler, + db *database.DB, store storage.Storage, logger *slog.Logger, workers int) *mirror.Mirror { + m := mirror.New(proxy, db, store, logger, workers) + m.Images = images + m.Charts = handler.NewHelmHandler(proxy, cfg.BaseURL, cfg.Upstream.Helm) + m.Versions = server.NewMirrorVersions(cfg.Upstream, proxy.HTTPClient) + if cfg.Scanning.Enabled { + refuseUnscannedMirroring(m) + } + return m +} + // mirrorExitCode returns the exit status for a finished mirror run: 1 when any // package version failed to mirror, 0 otherwise. func mirrorExitCode(progress *mirror.Progress) int { diff --git a/docs/configuration.md b/docs/configuration.md index 1049f2b..ba97446 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1530,6 +1530,8 @@ A conda version is mirrored with every build anaconda.org lists for it, across a A Composer release is mirrored with the dist archive `upstream.composer` lists for it, checked against its shasum when one is listed, and cached under the name `/composer/files/...` serves it as. Give the version as the repository lists it, for example `pkg:composer/monolog/monolog@3.8.1`. Branches such as `dev-main` aren't mirrored, since their archive changes with every push, and neither are packages in `upstream.composer_routes`. +A Helm chart is mirrored from a repository in `upstream.helm`, named as the namespace, for example `pkg:helm/prometheus-community/prometheus@27.1.0` with `prometheus-community` configured. The chart is fetched from the first URL the repository's `index.yaml` lists for that version, checked against the listed digest, and cached under the repository and digest `/helm/{name}/charts/...` serves it from. The index is cached as well, so with [`cache_metadata`](#metadata-caching) on `helm pull` works while the repository is down. Without a version every version in the index is mirrored. Charts named without a repository, or from one that isn't configured, are refused, and so are charts the index lists in an OCI registry: helm pulls those through `/v2`, so mirror them as `pkg:oci` images. + | Flag | Default | Description | |------|---------|-------------| | `--sbom` | | Path to CycloneDX or SPDX SBOM file | diff --git a/internal/handler/helm.go b/internal/handler/helm.go index b800069..07e2f74 100644 --- a/internal/handler/helm.go +++ b/internal/handler/helm.go @@ -1,6 +1,7 @@ package handler import ( + "context" "crypto/sha256" "encoding/hex" "errors" @@ -82,7 +83,7 @@ func (h *HelmHandler) handleIndex(w http.ResponseWriter, r *http.Request) { return } - index := h.fetchIndex(r, repository, upstreamURL) + index := h.fetchIndex(r.Context(), repository, upstreamURL) if index.err != nil { h.serveIndexError(w, index.err) return @@ -130,7 +131,7 @@ func (h *HelmHandler) handleChart(w http.ResponseWriter, r *http.Request) { return } - index := h.fetchIndex(r, repository, upstreamURL) + index := h.fetchIndex(r.Context(), repository, upstreamURL) if index.err != nil { h.serveIndexError(w, index.err) return @@ -182,9 +183,9 @@ func (h *HelmHandler) repositoryForRequest(r *http.Request) (name, upstreamURL s return name, upstreamURL, ok } -func (h *HelmHandler) fetchIndex(r *http.Request, repository, upstreamURL string) metadataResult { +func (h *HelmHandler) fetchIndex(ctx context.Context, repository, upstreamURL string) metadataResult { return h.proxy.fetchMetadataDocument( - r.Context(), + ctx, helmMetadataEcosystem, h.indexCacheKey(repository, upstreamURL), upstreamURL+"/"+helmIndexFilename, @@ -278,10 +279,11 @@ func (h *HelmHandler) findChartDownload(repository, upstreamURL string, index me } // helmChartLookup holds the HTTP chart downloads an index lists, by chart -// digest, in index order. +// digest, in index order, and each release by chart name and version. type helmChartLookup struct { indexDigest string byDigest map[string][]helmChartEntry + byRelease map[helmRelease]helmReleaseDownload } type helmChartEntry struct { @@ -290,6 +292,16 @@ type helmChartEntry struct { downloads []helmChartDownload } +type helmRelease struct{ chart, version string } + +// helmReleaseDownload is what helm downloads for a release: the first URL +// the index lists for it, checked against its digest. +type helmReleaseDownload struct { + digest string + created time.Time + first helmChartDownload +} + // chartLookup returns the charts the index lists, parsing it only when the // last lookup built for the repository came from other bytes. An index // without a digest, read with the metadata cache off, is parsed every time. @@ -326,7 +338,7 @@ func (h *HelmHandler) buildChartLookup(upstreamURL string, body []byte) (*helmCh return nil, err } - lookup := &helmChartLookup{byDigest: make(map[string][]helmChartEntry)} + lookup := &helmChartLookup{byDigest: make(map[string][]helmChartEntry), byRelease: make(map[helmRelease]helmReleaseDownload)} for i := 0; i < len(entries.Content); i += 2 { chartName := entries.Content[i].Value releases := entries.Content[i+1] @@ -339,6 +351,18 @@ func (h *HelmHandler) buildChartLookup(upstreamURL string, body []byte) (*helmCh continue } entry := helmChartEntry{name: chartName, created: chart.created} + if version := helmMappingValue(release, "version"); version != nil && version.Value != "" { + // helm takes the first release listed for a version. + key := helmRelease{chartName, version.Value} + if _, listed := lookup.byRelease[key]; !listed { + first := chart.downloads[0] + lookup.byRelease[key] = helmReleaseDownload{ + digest: chart.digest, + created: chart.created, + first: helmChartDownload{url: first.url, filename: first.filename, oci: first.oci}, + } + } + } for _, download := range chart.downloads { if !download.oci { // Drop the YAML node, so the lookup doesn't keep the diff --git a/internal/handler/helm_mirror.go b/internal/handler/helm_mirror.go new file mode 100644 index 0000000..fe2199d --- /dev/null +++ b/internal/handler/helm_mirror.go @@ -0,0 +1,67 @@ +package handler + +import ( + "context" + "fmt" + "path" + "slices" + "strings" +) + +// MirrorChart caches a chart release of a configured repository under the +// key handleChart serves it from, checked against the digest the index +// lists. The index is read through the metadata cache, as handleIndex reads +// it, so with metadata caching on helm finds the chart there as well while +// the repository is down. +func (h *HelmHandler) MirrorChart(ctx context.Context, repository, chart, version string) (*CacheResult, error) { + upstreamURL, ok := h.repositories[repository] + if !ok { + return nil, h.unknownRepositoryError(repository) + } + index := h.fetchIndex(ctx, repository, upstreamURL) + if index.err != nil { + return nil, fmt.Errorf("fetching the %s index: %w", repository, index.err) + } + charts, err := h.chartLookup(repository, upstreamURL, index) + if err != nil { + return nil, err + } + + release, ok := charts.byRelease[helmRelease{chart, version}] + switch { + case !ok: + return nil, fmt.Errorf("%w: %s %s is not in the %s index", ErrUpstreamNotFound, chart, version, repository) + case release.first.oci: + return nil, fmt.Errorf("%s %s is published as %s, which helm pulls through the container registry routes; mirror it as an image: %s", + chart, version, release.first.url, ociChartPURL(release.first.url, version)) + case h.chartOnCooldown(chart, release.created): + return nil, fmt.Errorf("%s %s is in cooldown, so the index doesn't list it", chart, version) + } + return h.proxy.GetOrFetchArtifactFromURLWithDigest(ctx, helmMetadataEcosystem, repository, release.digest, + release.first.filename, release.first.url, "sha256:"+release.digest) +} + +func (h *HelmHandler) unknownRepositoryError(repository string) error { + names := make([]string, 0, len(h.repositories)) + for name := range h.repositories { + names = append(names, name) + } + if len(names) == 0 { + return fmt.Errorf("unknown Helm repository %q: upstream.helm configures none", repository) + } + slices.Sort(names) + return fmt.Errorf("unknown Helm repository %q: upstream.helm configures %s", repository, strings.Join(names, ", ")) +} + +// ociChartPURL returns the image PURL that mirrors an OCI chart reference, +// tagged with the chart version when the reference has no tag or digest, as +// helm pulls it. +func ociChartPURL(reference, version string) string { + repository := strings.TrimPrefix(reference, "oci://") + if at := strings.LastIndex(repository, "@"); at >= 0 { + repository, version = repository[:at], repository[at+1:] + } else if colon := strings.LastIndex(repository, ":"); colon > strings.LastIndex(repository, "/") { + repository, version = repository[:colon], repository[colon+1:] + } + return "pkg:oci/" + path.Base(repository) + "@" + version + "?repository_url=" + repository +} diff --git a/internal/handler/helm_mirror_test.go b/internal/handler/helm_mirror_test.go new file mode 100644 index 0000000..617d35d --- /dev/null +++ b/internal/handler/helm_mirror_test.go @@ -0,0 +1,17 @@ +package handler + +import "testing" + +func TestOCIChartPURL(t *testing.T) { + tests := []struct{ reference, want string }{ + {"oci://ghcr.io/owner/demo:2.0.0", "pkg:oci/demo@2.0.0?repository_url=ghcr.io/owner/demo"}, + {"oci://ghcr.io/owner/demo", "pkg:oci/demo@1.0.0?repository_url=ghcr.io/owner/demo"}, + {"oci://localhost:5000/demo", "pkg:oci/demo@1.0.0?repository_url=localhost:5000/demo"}, + {"oci://ghcr.io/owner/demo@sha256:abc", "pkg:oci/demo@sha256:abc?repository_url=ghcr.io/owner/demo"}, + } + for _, tt := range tests { + if got := ociChartPURL(tt.reference, "1.0.0"); got != tt.want { + t.Errorf("ociChartPURL(%q) = %q, want %q", tt.reference, got, tt.want) + } + } +} diff --git a/internal/handler/helm_pypi_rewrite_test.go b/internal/handler/helm_pypi_rewrite_test.go index 8a8dd67..3500f8e 100644 --- a/internal/handler/helm_pypi_rewrite_test.go +++ b/internal/handler/helm_pypi_rewrite_test.go @@ -1,6 +1,7 @@ package handler import ( + "context" "errors" "fmt" "net/http" @@ -138,7 +139,7 @@ func TestHelmChartLookupFollowsIndex(t *testing.T) { proxy.MetadataTTL = time.Hour h := NewHelmHandler(proxy, "http://proxy.example", map[string]string{"stable": upstream.URL}) index := func() metadataResult { - return h.fetchIndex(httptest.NewRequest(http.MethodGet, "/", nil), "stable", upstream.URL) + return h.fetchIndex(context.Background(), "stable", upstream.URL) } first, err := h.findChartDownload("stable", upstream.URL, index(), digestA, "a-1.0.0.tgz") diff --git a/internal/mirror/helm.go b/internal/mirror/helm.go new file mode 100644 index 0000000..bda16dd --- /dev/null +++ b/internal/mirror/helm.go @@ -0,0 +1,40 @@ +package mirror + +import ( + "context" + "errors" + "strings" + + "github.com/git-pkgs/proxy/internal/handler" +) + +// ChartMirrorer caches Helm charts from the configured repositories the way +// the Helm handler serves them. +type ChartMirrorer interface { + MirrorChart(ctx context.Context, repository, chart, version string) (*handler.CacheResult, error) +} + +var ( + errHelmNoRepository = errors.New("helm charts need the upstream.helm repository as the namespace, such as pkg:helm/{repository}/{chart}@{version}") + errHelmUnavailable = errors.New("mirroring Helm charts isn't available here") +) + +// mirrorHelm mirrors a chart release. The PURL namespace names the +// upstream.helm repository, since the handler caches charts per repository. +func (m *Mirror) mirrorHelm(ctx context.Context, pv PackageVersion, tracker *progressTracker) { + repository, chart, ok := strings.Cut(pv.Name, "/") + if !ok || repository == "" || chart == "" { + m.recordFailure(tracker, pv, errHelmNoRepository) + return + } + if m.Charts == nil { + m.recordFailure(tracker, pv, errHelmUnavailable) + return + } + result, err := m.Charts.MirrorChart(ctx, repository, chart, pv.Version) + if err != nil { + m.recordFailure(tracker, pv, err) + return + } + m.recordResult(tracker, pv, result) +} diff --git a/internal/mirror/mirror.go b/internal/mirror/mirror.go index 6bd8d18..0a1142d 100644 --- a/internal/mirror/mirror.go +++ b/internal/mirror/mirror.go @@ -29,6 +29,9 @@ type Mirror struct { // other unsupported ecosystem. Images ImageMirrorer + // Charts mirrors helm charts. Without it they fail. + Charts ChartMirrorer + // Versions lists the versions of unversioned PURLs in mirror jobs and // schedules. Nil lists them from the public registries. Versions VersionLister @@ -236,6 +239,10 @@ func (m *Mirror) mirrorOne(ctx context.Context, pv PackageVersion, tracker *prog m.mirrorComposer(ctx, pv, tracker) return } + if pv.Ecosystem == "helm" { + m.mirrorHelm(ctx, pv, tracker) + return + } result, err := m.proxy.GetOrFetchArtifact(ctx, pv.Ecosystem, pv.Name, pv.Version, "") if err != nil { m.recordFailure(tracker, pv, err) diff --git a/internal/mirror/versions.go b/internal/mirror/versions.go index 1b68a6d..64fb36d 100644 --- a/internal/mirror/versions.go +++ b/internal/mirror/versions.go @@ -3,6 +3,7 @@ package mirror import ( "context" "fmt" + "strings" "github.com/git-pkgs/purl" "github.com/git-pkgs/registries" @@ -73,5 +74,10 @@ func (r RegistryVersions) fetchVersions(ctx context.Context, ecosystem, name str if err != nil { return nil, err } + if purlType == "helm" { + // The namespace names the repository, which URL resolves to the + // index the chart is listed in. + _, name, _ = strings.Cut(name, "/") + } return reg.FetchVersions(ctx, name) } diff --git a/internal/server/mirror_fetch_test.go b/internal/server/mirror_fetch_test.go index 142594c..86a5d8e 100644 --- a/internal/server/mirror_fetch_test.go +++ b/internal/server/mirror_fetch_test.go @@ -59,12 +59,14 @@ func mirrorWithVersions(t *testing.T, cfg *config.Config, versions mirror.Versio } logger := slog.New(slog.NewTextHandler(io.Discard, nil)) proxy, _ := NewMirrorProxy(cfg, db, store, logger) + proxy.CacheMetadata = true // as proxy mirror does m := mirror.New(proxy, db, store, logger, 1) + m.Charts = handler.NewHelmHandler(proxy, cfg.BaseURL, cfg.Upstream.Helm) m.Versions = versions if versions == nil { m.Versions = NewMirrorVersions(cfg.Upstream, proxy.HTTPClient) } - progress, err := m.Run(context.Background(), &mirror.PURLSource{PURLs: purls}) + progress, err := m.Run(context.Background(), &mirror.PURLSource{PURLs: purls, Versions: m.Versions}) if err != nil { t.Fatalf("Run() error = %v", err) } diff --git a/internal/server/mirror_helm_test.go b/internal/server/mirror_helm_test.go new file mode 100644 index 0000000..47833e6 --- /dev/null +++ b/internal/server/mirror_helm_test.go @@ -0,0 +1,147 @@ +package server + +import ( + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + + "github.com/git-pkgs/proxy/internal/config" + "github.com/git-pkgs/proxy/internal/handler" +) + +// helmUpstream serves an HTTP Helm repository listing demo 1.0.0 at a +// relative URL and demo 2.0.0 in an OCI registry, and the 1.0.0 chart. +type helmUpstream struct { + *httptest.Server + chart string + digest string + chartHits atomic.Int32 +} + +func newHelmUpstream(t *testing.T) *helmUpstream { + t.Helper() + u := &helmUpstream{chart: "demo chart archive"} + sum := sha256.Sum256([]byte(u.chart)) + u.digest = hex.EncodeToString(sum[:]) + u.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/index.yaml": + _, _ = fmt.Fprintf(w, `apiVersion: v1 +entries: + demo: + - name: demo + version: 2.0.0 + digest: %s + urls: [oci://ghcr.io/owner/demo:2.0.0] + - name: demo + version: 1.0.0 + digest: %s + urls: [charts/demo-1.0.0.tgz] +`, strings.Repeat("b", 64), u.digest) + case "/charts/demo-1.0.0.tgz": + u.chartHits.Add(1) + _, _ = io.WriteString(w, u.chart) + default: + http.NotFound(w, r) + } + })) + t.Cleanup(u.Close) + return u +} + +func helmMirrorConfig(upstream *helmUpstream) *config.Config { + cfg := config.Default() + cfg.Upstream.Helm = map[string]string{"stable": upstream.URL} + return cfg +} + +func TestMirrorHelmChartIsServedFromCache(t *testing.T) { + upstream := newHelmUpstream(t) + cfg := helmMirrorConfig(upstream) + + progress, proxy := mirrorWithConfig(t, cfg, "pkg:helm/stable/demo@1.0.0") + + if progress.Completed != 1 || upstream.chartHits.Load() != 1 { + t.Fatalf("completed = %d, chart hits = %d, errors = %v", progress.Completed, upstream.chartHits.Load(), progress.Errors) + } + + // helm reads the index and then the chart it points to, both from the + // cache once the repository is down. + upstream.Close() + helm := handler.NewHelmHandler(proxy, "http://proxy.test", cfg.Upstream.Helm).Routes() + chartPath := "/stable/charts/" + upstream.digest + "/demo-1.0.0.tgz" + rec := httptest.NewRecorder() + helm.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/stable/index.yaml", nil)) + if rec.Code != http.StatusOK || !strings.Contains(rec.Body.String(), "http://proxy.test/helm"+chartPath) { + t.Fatalf("index from cache: status = %d, body %q; want 200 listing %s", rec.Code, rec.Body.String(), chartPath) + } + rec = httptest.NewRecorder() + helm.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, chartPath, nil)) + if rec.Code != http.StatusOK || rec.Body.String() != upstream.chart { + t.Errorf("chart from cache: status = %d, body %q; want 200 and the mirrored chart", rec.Code, strings.TrimSpace(rec.Body.String())) + } +} + +func TestMirrorHelmListsVersions(t *testing.T) { + upstream := newHelmUpstream(t) + + progress, _ := mirrorWithConfig(t, helmMirrorConfig(upstream), "pkg:helm/stable/demo") + + // 2.0.0 is in an OCI registry and refused. + if progress.Total != 2 || progress.Completed != 1 || progress.Failed != 1 || upstream.chartHits.Load() != 1 { + t.Errorf("total = %d, completed = %d, failed = %d, chart hits = %d, errors = %v; want both versions listed and 1.0.0 mirrored", + progress.Total, progress.Completed, progress.Failed, upstream.chartHits.Load(), progress.Errors) + } +} + +func TestMirrorHelmRefusesDigestMismatch(t *testing.T) { + upstream := newHelmUpstream(t) + want := upstream.chart + upstream.chart = "tampered chart archive" + cfg := helmMirrorConfig(upstream) + + progress, proxy := mirrorWithConfig(t, cfg, "pkg:helm/stable/demo@1.0.0") + + if progress.Failed != 1 || progress.Completed != 0 { + t.Fatalf("completed = %d, failed = %d, errors = %v; want the chart refused", progress.Completed, progress.Failed, progress.Errors) + } + sum := sha256.Sum256([]byte(want)) + upstream.Close() + rec := httptest.NewRecorder() + handler.NewHelmHandler(proxy, "http://proxy.test", cfg.Upstream.Helm).Routes().ServeHTTP(rec, + httptest.NewRequest(http.MethodGet, "/stable/charts/"+hex.EncodeToString(sum[:])+"/demo-1.0.0.tgz", nil)) + if rec.Code == http.StatusOK { + t.Error("the tampered chart was cached") + } +} + +func TestMirrorHelmRefusals(t *testing.T) { + tests := []struct { + name, purl, want string + }{ + {"OCI chart", "pkg:helm/stable/demo@2.0.0", "pkg:oci/demo@2.0.0?repository_url=ghcr.io/owner/demo"}, + {"unknown repository", "pkg:helm/other/demo@1.0.0", `unknown Helm repository "other": upstream.helm configures stable`}, + {"no repository", "pkg:helm/demo@1.0.0", "pkg:helm/{repository}/{chart}@{version}"}, + {"unknown version", "pkg:helm/stable/demo@3.0.0", "not in the stable index"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + upstream := newHelmUpstream(t) + + progress, _ := mirrorWithConfig(t, helmMirrorConfig(upstream), tt.purl) + + if progress.Failed != 1 || len(progress.Errors) != 1 || !strings.Contains(progress.Errors[0].Error, tt.want) { + t.Errorf("failed = %d, errors = %v; want an error mentioning %q", progress.Failed, progress.Errors, tt.want) + } + if upstream.chartHits.Load() != 0 { + t.Error("a chart was fetched") + } + }) + } +} diff --git a/internal/server/mirror_versions.go b/internal/server/mirror_versions.go index 404388d..e093a98 100644 --- a/internal/server/mirror_versions.go +++ b/internal/server/mirror_versions.go @@ -30,6 +30,7 @@ func NewMirrorVersions(upstream config.UpstreamConfig, client *http.Client) mirr func (s *Server) newMirror(proxy *handler.Proxy, workers int) *mirror.Mirror { m := mirror.New(proxy, s.db, s.storage, s.logger, workers) m.Images = s.newContainerHandler(proxy) + m.Charts = handler.NewHelmHandler(proxy, s.cfg.BaseURL, s.cfg.Upstream.Helm) m.Versions = NewMirrorVersions(s.cfg.Upstream, proxy.HTTPClient) return m } @@ -80,6 +81,17 @@ func mirrorVersionsURL(u config.UpstreamConfig, purlType, name string) (string, return "", nil case "julia": return publicOnly("upstream.julia", u.Julia, d.Julia) + case "helm": + // The namespace names the repository; see RegistryVersions. + repository, _, ok := strings.Cut(name, "/") + if !ok { + return "", errors.New("helm charts need the upstream.helm repository as the namespace, such as pkg:helm/{repository}/{chart}") + } + repositoryURL, ok := u.Helm[repository] + if !ok { + return "", fmt.Errorf("unknown Helm repository %q: it isn't in upstream.helm", repository) + } + return strings.TrimSuffix(repositoryURL, "/"), nil default: return "", fmt.Errorf("listing %s versions isn't supported; give a version", purlType) } diff --git a/internal/server/mirror_versions_test.go b/internal/server/mirror_versions_test.go index 898c212..89fbea3 100644 --- a/internal/server/mirror_versions_test.go +++ b/internal/server/mirror_versions_test.go @@ -96,6 +96,7 @@ func TestMirrorVersionsURL(t *testing.T) { custom.ComposerRoutes = map[string]string{"acme/*": "https://composer.example.com"} custom.CRAN = "https://cran.example.com" custom.HomebrewAPI = "https://brew.example.com/api" + custom.Helm = map[string]string{"stable": "https://charts.example.com/stable/"} custom.Cargo = "https://index.example.com" custom.Conda = "https://conda.example.com" custom.Julia = "https://julia.example.com" @@ -128,7 +129,9 @@ func TestMirrorVersionsURL(t *testing.T) { {"conda channel", custom, "conda", "numpy", "", "upstream.conda"}, {"julia default", config.Default().Upstream, "julia", "JSON", "", ""}, {"julia server", custom, "julia", "JSON", "", "upstream.julia"}, - {"helm", custom, "helm", "nginx", "", "give a version"}, + {"helm", custom, "helm", "stable/nginx", "https://charts.example.com/stable", ""}, + {"helm without repository", custom, "helm", "nginx", "", "namespace"}, + {"helm unknown repository", custom, "helm", "other/nginx", "", "upstream.helm"}, {"swift", custom, "swift", "apple/swift-nio", "", "give a version"}, {"apk", custom, "apk", "curl", "", "give a version"}, } From 3f83d9c0cecfc733ad05479c8adb6bdec8ccf41a Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 25 Oct 2026 19:37:46 +0100 Subject: [PATCH 446/633] Mirror Julia packages proxy mirror and mirror jobs now take pkg:julia PURLs with the uuid qualifier. The tree hash of the version comes from the General registry, after checking the uuid is the one General lists for the name, and the tarball is fetched from upstream.julia at /package/{uuid}/{tree-hash} and cached under the name and tree hash the Julia handler uses. The handler names a package from the registry's Registry.toml, which it loads from the upstream. Without the upstream it fell back to caching and looking up by uuid, so a package cached under its name, mirrored or fetched before a restart, was not served offline. It now looks the tarball up by its tree hash in that case. Julia versions are listed from General whatever upstream.julia is, as packages are identified by uuid and tree hash on every Pkg server serving General. --- cmd/proxy/main.go | 2 +- docs/configuration.md | 4 +- internal/database/cached_names.go | 21 +++ internal/handler/julia.go | 63 ++++++++- internal/mirror/julia.go | 88 +++++++++++++ internal/mirror/mirror.go | 8 ++ internal/mirror/source.go | 6 +- internal/mirror/versions.go | 15 ++- internal/mirror/versions_test.go | 20 +++ internal/server/mirror_fetch_test.go | 1 + internal/server/mirror_julia_test.go | 162 ++++++++++++++++++++++++ internal/server/mirror_versions.go | 15 ++- internal/server/mirror_versions_test.go | 2 +- 13 files changed, 393 insertions(+), 14 deletions(-) create mode 100644 internal/database/cached_names.go create mode 100644 internal/mirror/julia.go create mode 100644 internal/server/mirror_julia_test.go diff --git a/cmd/proxy/main.go b/cmd/proxy/main.go index a61534a..7f212ea 100644 --- a/cmd/proxy/main.go +++ b/cmd/proxy/main.go @@ -560,7 +560,7 @@ func runMirror() { m := mirror.New(proxy, db, store, logger, *concurrency) m.Images = images - m.Versions = server.NewMirrorVersions(cfg.Upstream, proxy.HTTPClient) + server.UseMirrorUpstreams(m, cfg.Upstream, proxy.HTTPClient) switch s := source.(type) { case *mirror.PURLSource: s.Versions = m.Versions diff --git a/docs/configuration.md b/docs/configuration.md index 1049f2b..58dde96 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1530,6 +1530,8 @@ A conda version is mirrored with every build anaconda.org lists for it, across a A Composer release is mirrored with the dist archive `upstream.composer` lists for it, checked against its shasum when one is listed, and cached under the name `/composer/files/...` serves it as. Give the version as the repository lists it, for example `pkg:composer/monolog/monolog@3.8.1`. Branches such as `dev-main` aren't mirrored, since their archive changes with every push, and neither are packages in `upstream.composer_routes`. +A Julia package is mirrored with the source tarball `upstream.julia` serves at `/package/{uuid}/{tree-hash}`, cached under the package's name and tree hash as `/julia/package/...` serves it. The PURL needs the `uuid` qualifier, for example `pkg:julia/Example@0.5.5?uuid=7876af07-990d-54b4-ab0e-23690620f79a`. Only packages of the General registry can be mirrored: the uuid and tree hash are read from General on GitHub, also with another `upstream.julia`, since they are the same on every Pkg server serving General. The tarball isn't checked by the proxy, as when serving; Pkg checks the tree hash of what it unpacks, except on Windows. Dependencies, the artifacts in a package's `Artifacts.toml` and the registry aren't mirrored. An offline `Pkg.add` also needs the registry: one installed on the client, or the one the proxy cached when a client last fetched it, which it serves offline with `cache_metadata` on. + | Flag | Default | Description | |------|---------|-------------| | `--sbom` | | Path to CycloneDX or SPDX SBOM file | @@ -1550,7 +1552,7 @@ proxy mirror pkg:npm/lodash@4.17.21 pkg:cargo/serde@1.0.0 ### Unversioned PURLs -A PURL without a version, here or in a mirror API job, mirrors every version the configured upstream lists, through the same client, so `upstream.auth` and `upstream.allow_private_hosts` apply. npm packages in `upstream.npm_routes` are listed from their route. Listing works for npm, gem, golang, hex (from `upstream.hex_api`), pub, pypi, maven, nuget, composer, cran and brew. For cargo, conda and julia the upstream doesn't serve the API used for listing, so versions come from the public registry while the upstream is the default one; with another upstream, and for Composer packages in `upstream.composer_routes` or any other ecosystem, give a version. +A PURL without a version, here or in a mirror API job, mirrors every version the configured upstream lists, through the same client, so `upstream.auth` and `upstream.allow_private_hosts` apply. npm packages in `upstream.npm_routes` are listed from their route. Listing works for npm, gem, golang, hex (from `upstream.hex_api`), pub, pypi, maven, nuget, composer, cran and brew. For cargo and conda the upstream doesn't serve the API used for listing, so versions come from the public registry while the upstream is the default one; with another upstream, and for Composer packages in `upstream.composer_routes` or any other ecosystem, give a version. Julia versions are always listed from the General registry. ### Container images diff --git a/internal/database/cached_names.go b/internal/database/cached_names.go new file mode 100644 index 0000000..0b9563b --- /dev/null +++ b/internal/database/cached_names.go @@ -0,0 +1,21 @@ +package database + +// CachedArtifactNames returns the names of the packages of ecosystem that +// have a cached artifact called filename, for protocols whose file names +// identify an artifact without its package name. +func (db *DB) CachedArtifactNames(ecosystem, filename string) ([]string, error) { + var names []string + query := db.Rebind(` + SELECT DISTINCT packages.name + FROM artifacts + JOIN versions ON versions.purl = artifacts.version_purl + JOIN packages ON packages.purl = versions.package_purl + WHERE packages.ecosystem = ? AND artifacts.filename = ? + AND artifacts.storage_path IS NOT NULL AND artifacts.fetched_at IS NOT NULL + ORDER BY packages.name + `) + if err := db.Select(&names, query, ecosystem, filename); err != nil { + return nil, err + } + return names, nil +} diff --git a/internal/handler/julia.go b/internal/handler/julia.go index 67693b2..6ec0209 100644 --- a/internal/handler/julia.go +++ b/internal/handler/julia.go @@ -110,7 +110,7 @@ func (h *JuliaHandler) handleRegistry(w http.ResponseWriter, r *http.Request) { h.proxy.Logger.Debug("julia registry request", "uuid", uuid, "hash", hash) upstreamURL := h.upstreamURL + r.URL.Path - result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "julia", juliaRegistryName, hash, hash+".tar.gz", upstreamURL) + result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "julia", juliaRegistryName, hash, juliaTarball(hash), upstreamURL) if err != nil { h.proxy.serveArtifactError(w, err, "failed to fetch registry") return @@ -137,8 +137,15 @@ func (h *JuliaHandler) handlePackage(w http.ResponseWriter, r *http.Request) { h.proxy.Logger.Info("julia package request", "name", name, "uuid", uuid, "hash", hash) + if name == uuid { + if result := h.cachedPackage(r.Context(), hash); result != nil { + ServeArtifactRequest(w, r, result) + return + } + } + upstreamURL := h.upstreamURL + r.URL.Path - result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "julia", name, hash, hash+".tar.gz", upstreamURL) + result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "julia", name, hash, juliaTarball(hash), upstreamURL) if err != nil { h.proxy.serveArtifactError(w, err, "failed to fetch package") return @@ -147,6 +154,56 @@ func (h *JuliaHandler) handlePackage(w http.ResponseWriter, r *http.Request) { ServeArtifactRequest(w, r, result) } +// cachedPackage returns the package tarball with tree hash hash from the +// cache, under whatever name it was cached as, or nil. Without the name map, +// which needs the registry from upstream, a request can't be matched to the +// name its package was cached under, but the tarball is identified by its +// tree hash alone. The registry and artifact tarballs cached under the same +// file name are skipped. +func (h *JuliaHandler) cachedPackage(ctx context.Context, hash string) *CacheResult { + if h.proxy.DB == nil { + return nil + } + filename := juliaTarball(hash) + names, err := h.proxy.DB.CachedArtifactNames("julia", filename) + if err != nil { + h.proxy.Logger.Warn("looking up cached julia package", "hash", hash, "error", err) + return nil + } + for _, name := range names { + if name == juliaRegistryName || name == juliaArtifactName { + continue + } + result, err := h.proxy.GetCachedArtifact(ctx, "julia", name, hash, filename) + if err == nil && result != nil { + return result + } + } + return nil +} + +// JuliaPackageDownload returns the URL of a package's tarball on the Pkg +// server at upstreamURL, or the default one when empty, and the file name the +// Julia handler caches it under. The handler caches a package tarball as +// version treeHash, the git tree hash of the package's source, under the +// package's name in the General registry. +func JuliaPackageDownload(upstreamURL, uuid, treeHash string) (url, filename string, err error) { + if !validJuliaUUID(uuid) { + return "", "", fmt.Errorf("invalid julia package uuid %q", uuid) + } + if !juliaHexPattern.MatchString(treeHash) { + return "", "", fmt.Errorf("invalid julia tree hash %q", treeHash) + } + base := configuredUpstreamURL(upstreamURL, juliaUpstream) + return base + "/package/" + uuid + "/" + treeHash, juliaTarball(treeHash), nil +} + +// juliaTarball is the file name a registry, package or artifact tarball is +// cached under. +func juliaTarball(hash string) string { + return hash + ".tar.gz" +} + // handleArtifact serves an immutable binary artifact tarball. Artifacts are // anonymous content-addressed blobs with no associated package name. func (h *JuliaHandler) handleArtifact(w http.ResponseWriter, r *http.Request) { @@ -159,7 +216,7 @@ func (h *JuliaHandler) handleArtifact(w http.ResponseWriter, r *http.Request) { h.proxy.Logger.Debug("julia artifact request", "hash", hash) upstreamURL := h.upstreamURL + r.URL.Path - result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "julia", juliaArtifactName, hash, hash+".tar.gz", upstreamURL) + result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "julia", juliaArtifactName, hash, juliaTarball(hash), upstreamURL) if err != nil { h.proxy.serveArtifactError(w, err, "failed to fetch artifact") return diff --git a/internal/mirror/julia.go b/internal/mirror/julia.go new file mode 100644 index 0000000..30efb68 --- /dev/null +++ b/internal/mirror/julia.go @@ -0,0 +1,88 @@ +package mirror + +import ( + "context" + "errors" + "fmt" + "strings" + + "github.com/git-pkgs/proxy/internal/handler" + "github.com/git-pkgs/registries" +) + +// PackageFinder finds a package's registry entry, which for Julia gives the +// UUID the package is registered under. +type PackageFinder interface { + FindPackage(ctx context.Context, ecosystem, name string) (*registries.Package, error) +} + +// FindPackage implements PackageFinder. +func (r RegistryVersions) FindPackage(ctx context.Context, ecosystem, name string) (*registries.Package, error) { + reg, err := r.registry(ecosystem, name) + if err != nil { + return nil, err + } + return reg.FetchPackage(ctx, name) +} + +// mirrorJulia mirrors the source tarball of a version of a package in the +// General registry from the Pkg server, under the name and tree hash the +// Julia handler caches it as. The registry gives the tree hash, and the +// tarball is fetched unchecked, as the handler fetches it: Pkg checks the +// tree hash of what it extracts. The registry and the package's artifacts +// are not mirrored. +func (m *Mirror) mirrorJulia(ctx context.Context, pv PackageVersion, tracker *progressTracker) { + fail := func(err error) { m.recordFailure(tracker, pv, err) } + uuid := strings.ToLower(pv.Qualifiers["uuid"]) + if uuid == "" { + fail(fmt.Errorf("julia packages need a uuid qualifier, such as pkg:julia/%s@%s?uuid=", pv.Name, pv.Version)) + return + } + + pkg, err := m.packageFinder().FindPackage(ctx, pv.Ecosystem, pv.Name) + if err != nil { + fail(err) + return + } + registered, _ := pkg.Metadata["uuid"].(string) + if !strings.EqualFold(registered, uuid) { + fail(fmt.Errorf("%s in the General registry has uuid %s, not %s; only General packages can be mirrored", pv.Name, registered, uuid)) + return + } + // The handler names packages as the registry does. + name := pkg.Name + if name == "" { + name = pv.Name + } + + v, err := m.versionFinder().FindVersion(ctx, pv.Ecosystem, name, pv.Version) + if err != nil { + fail(err) + return + } + treeHash, _ := v.Metadata["git-tree-sha1"].(string) + if treeHash == "" { + fail(errors.New("the registry lists no tree hash for this version")) + return + } + url, filename, err := handler.JuliaPackageDownload(m.JuliaUpstream, uuid, treeHash) + if err != nil { + fail(err) + return + } + result, err := m.proxy.GetOrFetchArtifactFromURL(ctx, pv.Ecosystem, name, treeHash, filename, url) + if err != nil { + fail(err) + return + } + m.recordResult(tracker, pv, result) +} + +// packageFinder finds packages where Versions lists versions, so both come +// from the same registry. +func (m *Mirror) packageFinder() PackageFinder { //nolint:ireturn // any finder will do + if finder, ok := m.Versions.(PackageFinder); ok { + return finder + } + return RegistryVersions{} +} diff --git a/internal/mirror/mirror.go b/internal/mirror/mirror.go index 6bd8d18..2fa626a 100644 --- a/internal/mirror/mirror.go +++ b/internal/mirror/mirror.go @@ -33,6 +33,10 @@ type Mirror struct { // schedules. Nil lists them from the public registries. Versions VersionLister + // JuliaUpstream is the Pkg server Julia packages are fetched from. Empty + // uses https://pkg.julialang.org. + JuliaUpstream string + // RefusePackages, when set, fails every item Images does not take with // this reason instead of fetching it, so nothing is cached. RefusePackages string @@ -236,6 +240,10 @@ func (m *Mirror) mirrorOne(ctx context.Context, pv PackageVersion, tracker *prog m.mirrorComposer(ctx, pv, tracker) return } + if pv.Ecosystem == "julia" { + m.mirrorJulia(ctx, pv, tracker) + return + } result, err := m.proxy.GetOrFetchArtifact(ctx, pv.Ecosystem, pv.Name, pv.Version, "") if err != nil { m.recordFailure(tracker, pv, err) diff --git a/internal/mirror/source.go b/internal/mirror/source.go index d8778b5..b742e00 100644 --- a/internal/mirror/source.go +++ b/internal/mirror/source.go @@ -16,7 +16,8 @@ type PackageVersion struct { Version string // Qualifiers are the PURL qualifiers. Images use repository_url, tag and // arch; PyPI uses file_name, and conda subdir, build and type, to select - // which files of the version are mirrored. + // which files of the version are mirrored. Julia uses uuid to identify + // the package. Qualifiers map[string]string // Platforms limit which platforms of an image index are mirrored. Platforms []string @@ -75,6 +76,9 @@ func (s *PURLSource) Enumerate(ctx context.Context, fn func(PackageVersion) erro name = p.Namespace + "/" + p.Name } qualifiers := fileQualifiers(ecosystem, p.Qualifiers.Map()) + if ecosystem == "julia" { + qualifiers = map[string]string{"uuid": p.Qualifier("uuid")} + } // Images are never expanded to all tags, and conda packages without // a channel aren't listed; both fail when they are mirrored. diff --git a/internal/mirror/versions.go b/internal/mirror/versions.go index 1b68a6d..2c33e7e 100644 --- a/internal/mirror/versions.go +++ b/internal/mirror/versions.go @@ -57,6 +57,15 @@ func (r RegistryVersions) FindVersion(ctx context.Context, ecosystem, name, vers } func (r RegistryVersions) fetchVersions(ctx context.Context, ecosystem, name string) ([]registries.Version, error) { + reg, err := r.registry(ecosystem, name) + if err != nil { + return nil, err + } + return reg.FetchVersions(ctx, name) +} + +// registry returns the client for the registry name is listed in. +func (r RegistryVersions) registry(ecosystem, name string) (registries.Registry, error) { //nolint:ireturn // registries.New returns the interface purlType := purl.EcosystemToPURLType(ecosystem) var baseURL string if r.URL != nil { @@ -69,9 +78,5 @@ func (r RegistryVersions) fetchVersions(ctx context.Context, ecosystem, name str if client == nil { client = registries.DefaultClient() } - reg, err := registries.New(purlType, baseURL, client) - if err != nil { - return nil, err - } - return reg.FetchVersions(ctx, name) + return registries.New(purlType, baseURL, client) } diff --git a/internal/mirror/versions_test.go b/internal/mirror/versions_test.go index 9c04a21..29f9f41 100644 --- a/internal/mirror/versions_test.go +++ b/internal/mirror/versions_test.go @@ -96,3 +96,23 @@ func TestSourceFromRequestUsesMirrorVersions(t *testing.T) { t.Errorf("SBOM source lists versions through %v, want the mirror's lister", got) } } + +func TestPURLSourceKeepsJuliaUUID(t *testing.T) { + const uuid = "7876af07-990d-54b4-ab0e-23690620f79a" + versions := &fakeVersions{versions: []string{"0.5.5"}} + source := &PURLSource{ + PURLs: []string{"pkg:julia/Example@0.5.4?uuid=" + uuid, "pkg:julia/Example?uuid=" + uuid}, + Versions: versions, + } + + items := enumerate(t, source) + + qualifiers := map[string]string{"uuid": uuid} + want := []PackageVersion{ + {Ecosystem: "julia", Name: "Example", Version: "0.5.4", Qualifiers: qualifiers}, + {Ecosystem: "julia", Name: "Example", Version: "0.5.5", Qualifiers: qualifiers}, + } + if !reflect.DeepEqual(items, want) { + t.Errorf("items = %v, want %v", items, want) + } +} diff --git a/internal/server/mirror_fetch_test.go b/internal/server/mirror_fetch_test.go index 142594c..5ab3ca3 100644 --- a/internal/server/mirror_fetch_test.go +++ b/internal/server/mirror_fetch_test.go @@ -61,6 +61,7 @@ func mirrorWithVersions(t *testing.T, cfg *config.Config, versions mirror.Versio proxy, _ := NewMirrorProxy(cfg, db, store, logger) m := mirror.New(proxy, db, store, logger, 1) m.Versions = versions + m.JuliaUpstream = cfg.Upstream.Julia if versions == nil { m.Versions = NewMirrorVersions(cfg.Upstream, proxy.HTTPClient) } diff --git a/internal/server/mirror_julia_test.go b/internal/server/mirror_julia_test.go new file mode 100644 index 0000000..607a2dc --- /dev/null +++ b/internal/server/mirror_julia_test.go @@ -0,0 +1,162 @@ +package server + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + + "github.com/git-pkgs/proxy/internal/config" + "github.com/git-pkgs/proxy/internal/handler" + "github.com/git-pkgs/proxy/internal/mirror" + "github.com/git-pkgs/registries" +) + +const ( + juliaExampleUUID = "7876af07-990d-54b4-ab0e-23690620f79a" + juliaExampleTree = "e1f0e1a832ccd8e97d6d0348dec33ee139a5aeaf" + juliaGeneralUUID = "23338594-aafe-5451-b93e-139f81909106" + juliaRegistryTree = "465d1e55da75e226473a37c40bf2c20aee19234d" +) + +// juliaUpstream serves the General registry's files for Example, which has +// version 0.5.5, and as a Pkg server /registries, the General registry +// tarball and Example's 0.5.5 source tarball. It counts package downloads. +type juliaUpstream struct { + *httptest.Server + tarball string + packageHits atomic.Int32 + registryHits atomic.Int32 +} + +func newJuliaUpstream(t *testing.T) *juliaUpstream { + t.Helper() + u := &juliaUpstream{tarball: "Example 0.5.5 source"} + registry := juliaRegistryTarball(t, "[packages]\n"+juliaExampleUUID+` = { name = "Example", path = "E/Example" }`+"\n") + u.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/E/Example/Package.toml": + _, _ = io.WriteString(w, "name = \"Example\"\nuuid = \""+juliaExampleUUID+"\"\nrepo = \"https://github.com/JuliaLang/Example.jl.git\"\n") + case "/E/Example/Versions.toml": + _, _ = io.WriteString(w, "[\"0.5.4\"]\ngit-tree-sha1 = \"11820aa9c229fd3833d4bd69e5e75ef4e7273bf1\"\n\n[\"0.5.5\"]\ngit-tree-sha1 = \""+juliaExampleTree+"\"\n") + case "/registries": + _, _ = io.WriteString(w, "/registry/"+juliaGeneralUUID+"/"+juliaRegistryTree+"\n") + case "/registry/" + juliaGeneralUUID + "/" + juliaRegistryTree: + u.registryHits.Add(1) + _, _ = w.Write(registry) + case "/package/" + juliaExampleUUID + "/" + juliaExampleTree: + u.packageHits.Add(1) + _, _ = io.WriteString(w, u.tarball) + default: + http.NotFound(w, r) + } + })) + t.Cleanup(u.Close) + return u +} + +// juliaRegistryTarball returns a gzipped registry tarball holding registryToml. +func juliaRegistryTarball(t *testing.T, registryToml string) []byte { + t.Helper() + var buf bytes.Buffer + gz := gzip.NewWriter(&buf) + tw := tar.NewWriter(gz) + if err := tw.WriteHeader(&tar.Header{Name: "Registry.toml", Mode: 0o644, Size: int64(len(registryToml))}); err != nil { + t.Fatal(err) + } + if _, err := io.WriteString(tw, registryToml); err != nil { + t.Fatal(err) + } + if err := tw.Close(); err != nil { + t.Fatal(err) + } + if err := gz.Close(); err != nil { + t.Fatal(err) + } + return buf.Bytes() +} + +// mirrorJulia mirrors purls from upstream as the Pkg server, with the +// registry's files read from it too. +func mirrorJulia(t *testing.T, upstream *juliaUpstream, purls ...string) (*mirror.Progress, *handler.Proxy) { + t.Helper() + cfg := config.Default() + cfg.Upstream.Julia = upstream.URL + versions := mirror.RegistryVersions{ + Client: registries.NewClient(registries.WithHTTPClient(upstream.Client())), + URL: func(string, string) (string, error) { return upstream.URL, nil }, + } + return mirrorWithVersions(t, cfg, versions, purls...) +} + +func getJuliaPackage(julia *handler.JuliaHandler) *httptest.ResponseRecorder { + rec := httptest.NewRecorder() + julia.Routes().ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/package/"+juliaExampleUUID+"/"+juliaExampleTree, nil)) + return rec +} + +func TestMirrorJuliaPackage(t *testing.T) { + upstream := newJuliaUpstream(t) + + progress, proxy := mirrorJulia(t, upstream, "pkg:julia/Example@0.5.5?uuid="+juliaExampleUUID) + + if progress.Completed != 1 || progress.Failed != 0 || upstream.packageHits.Load() != 1 { + t.Fatalf("completed = %d, failed = %d, package hits = %d, errors = %v", progress.Completed, progress.Failed, upstream.packageHits.Load(), progress.Errors) + } + if cached, _ := proxy.GetCachedArtifact(t.Context(), "julia", "Example", juliaExampleTree, juliaExampleTree+".tar.gz"); cached == nil { + t.Fatal("the tarball isn't cached under Example and its tree hash") + } else { + _ = cached.Reader.Close() + } + + // With the registry's names loaded, the handler finds the tarball under + // the package's name without fetching it again. + julia := handler.NewJuliaHandlerWithUpstream(proxy, upstream.URL) + if rec := getJuliaPackage(julia); rec.Code != http.StatusOK || rec.Body.String() != upstream.tarball { + t.Errorf("status = %d, body = %q, want 200 and the tarball", rec.Code, rec.Body) + } + if upstream.registryHits.Load() == 0 { + t.Error("the handler didn't load the registry's names") + } + + // Without the upstream, a new handler can't load the names, and finds + // the tarball by its tree hash. + upstream.Close() + if rec := getJuliaPackage(handler.NewJuliaHandlerWithUpstream(proxy, upstream.URL)); rec.Code != http.StatusOK || rec.Body.String() != upstream.tarball { + t.Errorf("from cache: status = %d, body = %q, want 200 and the tarball", rec.Code, rec.Body) + } + if got := upstream.packageHits.Load(); got != 1 { + t.Errorf("package fetched %d times, want 1", got) + } +} + +func TestMirrorJuliaRefusesOtherUUID(t *testing.T) { + upstream := newJuliaUpstream(t) + + progress, _ := mirrorJulia(t, upstream, "pkg:julia/Example@0.5.5?uuid=00000000-0000-0000-0000-000000000000") + + if progress.Failed != 1 || len(progress.Errors) != 1 || !strings.Contains(progress.Errors[0].Error, juliaExampleUUID) { + t.Errorf("failed = %d, errors = %v; want a failure naming General's uuid", progress.Failed, progress.Errors) + } + if upstream.packageHits.Load() != 0 { + t.Error("a tarball was fetched for a package General doesn't have") + } +} + +func TestServerMirrorFetchesJuliaFromUpstream(t *testing.T) { + cfg := config.Default() + cfg.Upstream.Julia = "https://julia.example.com" + logger := slog.New(slog.NewTextHandler(io.Discard, nil)) + proxy, _ := NewMirrorProxy(cfg, nil, nil, logger) + s := &Server{cfg: cfg, logger: logger, ecr: newECRTokens(logger)} + + if got := s.newMirror(proxy, 1).JuliaUpstream; got != cfg.Upstream.Julia { + t.Errorf("JuliaUpstream = %q, want upstream.julia", got) + } +} diff --git a/internal/server/mirror_versions.go b/internal/server/mirror_versions.go index 404388d..25bee85 100644 --- a/internal/server/mirror_versions.go +++ b/internal/server/mirror_versions.go @@ -30,10 +30,18 @@ func NewMirrorVersions(upstream config.UpstreamConfig, client *http.Client) mirr func (s *Server) newMirror(proxy *handler.Proxy, workers int) *mirror.Mirror { m := mirror.New(proxy, s.db, s.storage, s.logger, workers) m.Images = s.newContainerHandler(proxy) - m.Versions = NewMirrorVersions(s.cfg.Upstream, proxy.HTTPClient) + UseMirrorUpstreams(m, s.cfg.Upstream, proxy.HTTPClient) return m } +// UseMirrorUpstreams points m at the configured upstreams: it lists +// versions from them, through client, and fetches Julia packages from +// upstream.julia. +func UseMirrorUpstreams(m *mirror.Mirror, upstream config.UpstreamConfig, client *http.Client) { + m.Versions = NewMirrorVersions(upstream, client) + m.JuliaUpstream = upstream.Julia +} + // mirrorVersionsURL returns the registry base URL to list a package's // versions from: the upstream the proxy serves it from, where that upstream // has the API the registries client reads. An empty URL means the public @@ -79,7 +87,10 @@ func mirrorVersionsURL(u config.UpstreamConfig, purlType, name string) (string, } return "", nil case "julia": - return publicOnly("upstream.julia", u.Julia, d.Julia) + // Packages are identified by uuid and tree hash, the same on every + // Pkg server serving General, so General's are used whatever the + // upstream. + return "", nil default: return "", fmt.Errorf("listing %s versions isn't supported; give a version", purlType) } diff --git a/internal/server/mirror_versions_test.go b/internal/server/mirror_versions_test.go index 898c212..7e59dd4 100644 --- a/internal/server/mirror_versions_test.go +++ b/internal/server/mirror_versions_test.go @@ -127,7 +127,7 @@ func TestMirrorVersionsURL(t *testing.T) { {"conda default", config.Default().Upstream, "conda", "numpy", "", ""}, {"conda channel", custom, "conda", "numpy", "", "upstream.conda"}, {"julia default", config.Default().Upstream, "julia", "JSON", "", ""}, - {"julia server", custom, "julia", "JSON", "", "upstream.julia"}, + {"julia server lists from General", custom, "julia", "JSON", "", ""}, {"helm", custom, "helm", "nginx", "", "give a version"}, {"swift", custom, "swift", "apple/swift-nio", "", "give a version"}, {"apk", custom, "apk", "curl", "", "give a version"}, From 2fed8901aeb07a033e5f2aa310dbbbb372cbe517 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 25 Oct 2026 20:29:03 +0100 Subject: [PATCH 447/633] Send artifacts to the client while a cache miss is stored On a cache miss the proxy stored the whole artifact before sending any of it, so clients with an idle timeout gave up on large artifacts: git-lfs after 30 seconds, pacman after 10. The caller that starts a shared fetch now gets the bytes as they arrive while they are stored; requests joining the fetch, HEAD and range requests still wait for the stored copy. The client's copy is checked against the upstream's digest as it is read, and a mismatch aborts the response before it completes. A client going away leaves the copy to storage running. A storage error leaves the client's copy running: the client gets the artifact, and the failure is logged at error level and counted in proxy_artifacts_served_uncached_total. A block-mode scanner has to see the whole artifact first, so with one configured for the ecosystem a miss is handled as before. --- docs/architecture.md | 9 +- docs/configuration.md | 21 +- internal/handler/handler.go | 12 +- internal/handler/stream_miss.go | 255 +++++++++++++++ internal/handler/stream_miss_test.go | 444 +++++++++++++++++++++++++++ internal/metrics/metrics.go | 17 + internal/scanner/group.go | 14 + internal/scanner/group_test.go | 24 ++ internal/server/middleware.go | 5 +- 9 files changed, 787 insertions(+), 14 deletions(-) create mode 100644 internal/handler/stream_miss.go create mode 100644 internal/handler/stream_miss_test.go diff --git a/docs/architecture.md b/docs/architecture.md index e7ce8c8..b2f31c6 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -58,11 +58,14 @@ Metadata is not cached - always fetched fresh. This ensures clients see new vers **Cache Miss:** - Resolve download URL using Resolver - Fetch artifact from upstream using Fetcher + - Copy the body to Storage and, for a client GET without a `Range` + header, to the client at once (`storeWhileServing`); the client's copy + is checked against the upstream's digest as it is read, and a mismatch + aborts it. A block-mode scanner, `HEAD` and range requests, and + requests joining a fetch already running wait for the stored file. - Store artifact in Storage (returns size, hash) - Create/update database records (package, version, artifact) - - Open stored file - - Return reader to handler - - Handler streams file to client + - Requests that waited open the stored file and stream it to their client Artifact handlers honor single byte ranges when the storage reader supports efficient seeking (currently the local filesystem reader). They advertise diff --git a/docs/configuration.md b/docs/configuration.md index 269d5bd..f52601b 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -72,6 +72,14 @@ storage: `storage.max_size` counts cached artifacts and cached metadata documents (`cache_metadata`). When the total goes over the limit, the least recently served entries of either kind are evicted first; an evicted metadata document is fetched from upstream again on the next request for it. Pushed artifacts are not counted or evicted. An artifact replaced by a refetch stays in storage for at least an hour, or `storage.direct_serve_ttl` if longer, so requests already reading it can finish, and storage use can exceed the limit by what was replaced in that time. +### Cache misses + +On a cache miss the proxy sends the artifact to the client while it stores it, so the client gets its first bytes as soon as the upstream sends them, rather than after the whole artifact is stored. Clients that give up after a while without data, such as git-lfs (30 seconds) or pacman (10 seconds), no longer time out on large artifacts. The response goes out chunked, without `Content-Length`, as with `cache_artifacts: false`; once stored, the artifact is served with its size. + +The bytes are checked against the digest the upstream declares, where it declares one, as they are sent. On a mismatch the connection is aborted before the response completes, so the client never receives a tampered artifact as a good one, and nothing is cached. If the client goes away, the artifact is still stored. If storing it fails, for example because the disk is full, the client still gets the whole artifact; the failure is logged at error level and counted in `proxy_artifacts_served_uncached_total`, so a cache that has stopped filling doesn't go unnoticed while every request succeeds. + +Requests for the same artifact that arrive while it is being stored wait for the stored copy, as do `HEAD` requests and requests with a `Range` header, which are answered from it with its size. With a `block`-mode scanner configured for the ecosystem, a miss is stored and scanned before anything is sent, as before (see [Artifact Scanning](#artifact-scanning)). + ### Serving artifacts without storing them With `storage.cache_artifacts: false` the proxy streams every artifact download from upstream to the client and stores no artifacts. Metadata is still filtered and cached, so cooldown and the denylist apply as usual, and the database still holds the publish times cooldown needs. Use it when another caching layer, such as an Artifactory remote repository, sits in front of the proxy and caching artifacts twice only costs storage. @@ -854,13 +862,10 @@ repository, such as a fork, it is copied from there instead of downloaded, once the upstream has confirmed this repository has it; the copy takes its own space. -Large LFS objects may need longer timeouts. The proxy downloads an object it -doesn't have before sending any of it, within `artifact_timeout`, and the -whole response, download included, must fit within `write_timeout`; both -default to 5 minutes. git-lfs gives up on a response after 30 seconds without -data (`lfs.activitytimeout`) and retries. A retry waits for the download still -running, so the object is served once it is cached, but a download that -outlasts git-lfs's retries fails that clone or pull until it finishes. +Large LFS objects may need longer timeouts. An object the proxy doesn't +have is sent to git-lfs as it is downloaded (see [Cache misses](#cache-misses)), +but the download must finish within `artifact_timeout` and the whole response +within `write_timeout`; both default to 5 minutes. Download URLs are often signed; only the URL without its query is logged and recorded with the object. An object is only cached once its content matches its sha256 object ID. Objects are cached per repository, so one @@ -1248,7 +1253,7 @@ scanning: ### How caching defers to a scan verdict -The proxy never uploads artifact bytes to a scanner. When an artifact is fetched from upstream, it's stored to the configured storage backend first, exactly as without scanning. If scanning is enabled for the artifact's ecosystem, the proxy then notifies each applicable scanner with package metadata and a short-lived, HMAC-signed URL pointing at the internal `/_internal/scan-fetch` route; each scanner GETs that URL itself to pull the exact bytes staged in storage and runs its own scan against them. +The proxy never uploads artifact bytes to a scanner. When an artifact is fetched from upstream, it's stored to the configured storage backend first, exactly as without scanning. If scanning is enabled for the artifact's ecosystem, the proxy then notifies each applicable scanner with package metadata and a short-lived, HMAC-signed URL pointing at the internal `/_internal/scan-fetch` route; each scanner GETs that URL itself to pull the exact bytes staged in storage and runs its own scan against them. With a `block`-mode scanner for the ecosystem, the client gets nothing before the verdict; with only `monitor`-mode scanners, a cache miss is sent to the client while it is stored, and the scan runs after (see [Cache misses](#cache-misses)). Scanners configured for the same ecosystem all run concurrently, never sequentially. The moment any `block`-mode scanner reports a not-allowed verdict (or errors, unless `fail_open` is set), the proxy cancels the in-flight calls to the other scanners and deletes the staged artifact — it's never committed to the cache database, so it was never visible to a client. If nothing blocks, the proxy waits for every `block`-mode scanner to finish before caching the artifact and serving it. A `monitor`-mode scanner's findings are logged and never gate the wait or the caching decision, even when it reports not-allowed. diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 363314e..c96358c 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -652,7 +652,7 @@ func (p *Proxy) fetchAndCache(ctx context.Context, ecosystem, name, version, fil } metrics.RecordUpstreamFetch(ecosystem, fetchDuration) - return p.storeArtifact(ctx, ecosystem, name, version, filename, pkgPURL, versionPURL, info.URL, "", "", artifact) + return p.storeFetched(ctx, ecosystem, name, version, filename, pkgPURL, versionPURL, info.URL, "", "", artifact) } // storeArtifact writes a fetched artifact to storage, verifies it against @@ -848,6 +848,9 @@ func (p *Proxy) coalesceFetch(ctx context.Context, key string, recheck func() (a p.fetchMu.Unlock() if !joined { + if streamsCacheMiss(ctx) { + return p.runSharedFetchStreaming(ctx, key, f, recheck, commit) + } return p.runSharedFetch(ctx, key, f, recheck, commit) } @@ -940,6 +943,11 @@ func (p *Proxy) runSharedFetch(ctx context.Context, key string, f *inflightFetch if err != nil { return nil, err } + if h := handoffFrom(ctx); h != nil && h.handedOver() { + // The caller already has the bytes; opening the stored copy would + // cost a read it never uses, a GET on S3 or GCS. + return nil, nil + } return p.openStoredArtifact(ctx, stored, path) } @@ -2339,7 +2347,7 @@ func (p *Proxy) fetchAndCacheFromURL(ctx context.Context, ecosystem, name, versi return artifacts.Artifact{}, "", fmt.Errorf("fetching from upstream: %w", err) } - return p.storeArtifact(ctx, ecosystem, name, version, filename, pkgPURL, versionPURL, downloadURL, upstreamHash, declared, artifact) + return p.storeFetched(ctx, ecosystem, name, version, filename, pkgPURL, versionPURL, downloadURL, upstreamHash, declared, artifact) } type upstreamLastModifiedKey struct{} diff --git a/internal/handler/stream_miss.go b/internal/handler/stream_miss.go new file mode 100644 index 0000000..9ddcc20 --- /dev/null +++ b/internal/handler/stream_miss.go @@ -0,0 +1,255 @@ +package handler + +import ( + "context" + "errors" + "io" + "strings" + "sync" + + "github.com/git-pkgs/artifacts" + "github.com/git-pkgs/proxy/internal/metrics" + "github.com/git-pkgs/registries/fetch" + "github.com/opencontainers/go-digest" +) + +// On a cache miss the proxy used to store the whole artifact before sending +// any of it, so a client with an idle timeout (git-lfs gives up after 30 +// seconds without data, pacman after 10) could give up on a large one. A +// marked request instead gets the bytes as they arrive from upstream while +// they are stored: the caller that starts the shared fetch is handed a +// stream, and callers joining it still wait for the stored artifact. + +type streamCacheMissKey struct{} + +// StreamCacheMiss marks ctx so that on an artifact cache miss the bytes are +// sent to the client while they are being stored. The server sets it on GET +// requests without a Range header; a range of a stream that is still +// arriving can't be served. +func StreamCacheMiss(ctx context.Context) context.Context { + return context.WithValue(ctx, streamCacheMissKey{}, true) +} + +func streamsCacheMiss(ctx context.Context) bool { + stream, _ := ctx.Value(streamCacheMissKey{}).(bool) + return stream && !servingHead(ctx) +} + +// missHandoff passes the client's stream from the shared fetch, which runs on +// its own goroutine, to the caller that started it. The caller can leave +// before a stream is offered, and the fetch must then not write to a stream +// nobody reads. +type missHandoff struct { + mu sync.Mutex + ready chan struct{} + stream *CacheResult + taken bool + abandoned bool +} + +type missHandoffKey struct{} + +func handoffFrom(ctx context.Context) *missHandoff { + h, _ := ctx.Value(missHandoffKey{}).(*missHandoff) + return h +} + +// offer hands stream to the caller and reports whether it took it. +func (h *missHandoff) offer(stream *CacheResult) bool { + h.mu.Lock() + defer h.mu.Unlock() + if h.abandoned { + return false + } + h.stream, h.taken = stream, true + close(h.ready) + return true +} + +// handedOver reports whether the caller took a stream. +func (h *missHandoff) handedOver() bool { + h.mu.Lock() + defer h.mu.Unlock() + return h.taken +} + +// runSharedFetchStreaming runs the shared fetch on its own goroutine and +// returns as soon as it offers a stream, or with its result if it finishes +// without one: on a recheck hit, an artifact another instance stored, an +// error, or an artifact a block-mode scanner has to see first. Waiters and +// the fetch lease still see the fetch end only once the artifact is stored. +func (p *Proxy) runSharedFetchStreaming(ctx context.Context, key string, f *inflightFetch, recheck func() (artifacts.Artifact, string, bool), commit func(context.Context) (artifacts.Artifact, string, error)) (*CacheResult, error) { + h := &missHandoff{ready: make(chan struct{})} + type outcome struct { + res *CacheResult + err error + } + done := make(chan outcome, 1) + go func() { + res, err := p.runSharedFetch(context.WithValue(ctx, missHandoffKey{}, h), key, f, recheck, commit) + h.mu.Lock() + defer h.mu.Unlock() + if h.taken || h.abandoned { + // The caller has the stream, or has gone; nobody reads this. + if res != nil && res.Reader != nil { + _ = res.Reader.Close() + } + return + } + done <- outcome{res, err} + }() + + select { + case <-h.ready: + return h.stream, nil + case o := <-done: + return o.res, o.err + case <-ctx.Done(): + h.mu.Lock() + defer h.mu.Unlock() + if h.taken { + return h.stream, nil + } + h.abandoned = true + select { + case o := <-done: + if o.res != nil && o.res.Reader != nil { + _ = o.res.Reader.Close() + } + default: + } + return nil, ctx.Err() + } +} + +// storeFetched stores an artifact fetched from upstream. When the caller +// that started the fetch is waiting for a stream, it is sent the bytes as +// they are stored; see storeWhileServing. +func (p *Proxy) storeFetched(ctx context.Context, ecosystem, name, version, filename, pkgPURL, versionPURL, upstreamURL, upstreamHash, declared string, artifact *fetch.Artifact) (artifacts.Artifact, string, error) { + h := handoffFrom(ctx) + if h == nil || p.Scanners.Blocks(ecosystem) { + return p.storeArtifact(ctx, ecosystem, name, version, filename, pkgPURL, versionPURL, upstreamURL, upstreamHash, declared, artifact) + } + return p.storeWhileServing(ctx, h, ecosystem, name, version, filename, pkgPURL, versionPURL, upstreamURL, upstreamHash, declared, artifact) +} + +// errNoReaderLeft stops reading upstream once neither storage nor the +// client takes the bytes any more. +var errNoReaderLeft = errors.New("neither storage nor the client is reading the artifact") + +// storeWhileServing copies the upstream body to storage and to the client at +// once. The client's copy is checked against upstreamHash and declared as it +// is read, and a mismatch fails it before its last bytes, so the response is +// aborted and the client discards it; storeArtifact checks the stored copy +// the same way and doesn't cache it. A client that goes away leaves the copy +// to storage running. A storage error leaves the client's copy running, so +// the client still gets the artifact; that is logged and counted, since every +// request would otherwise succeed while nothing is cached. The client's +// stream ends once the last byte is checked, before storeArtifact's scan and +// database update. +func (p *Proxy) storeWhileServing(ctx context.Context, h *missHandoff, ecosystem, name, version, filename, pkgPURL, versionPURL, upstreamURL, upstreamHash, declared string, artifact *fetch.Artifact) (artifacts.Artifact, string, error) { + hash := strings.ToLower(upstreamHash) + checks, err := newIntegrityChecks(hash, declared) + if err != nil { + _ = artifact.Body.Close() + return artifacts.Artifact{}, "", err + } + + clientR, clientW := io.Pipe() + var client io.ReadCloser = clientR + if hash != "" || declared != "" { + if client, err = checks.wrapFailOnMismatch(clientR, func(reason string) { + p.Logger.Warn("artifact sent on a cache miss failed its integrity check; aborting the response", + "purl", versionPURL, "filename", filename, "reason", reason) + }); err != nil { + _ = artifact.Body.Close() + return artifacts.Artifact{}, "", err + } + } + stream := &CacheResult{ + Reader: client, + Artifact: artifacts.Artifact{ + PURL: versionPURL, + Filename: filename, + MediaType: artifact.ContentType, + // Unknown until stored, so the response goes out chunked and + // can still be aborted. + Size: -1, + }, + } + if hash != "" { + stream.Artifact.Digest = digest.Digest("sha256:" + hash) + } + if !h.offer(stream) { + _ = clientR.Close() + return p.storeArtifact(ctx, ecosystem, name, version, filename, pkgPURL, versionPURL, upstreamURL, upstreamHash, declared, artifact) + } + + storeR, storeW := io.Pipe() + type stored struct { + artifact artifacts.Artifact + path string + err error + } + storeDone := make(chan stored, 1) + go func() { + // Not cancelled with ctx: the client's request ends once it has the + // last byte, while storage may still be finishing the upload and the + // database write. Reading the upstream still stops with ctx, which + // ends the copy to storage too. + a, path, err := p.storeArtifact(context.WithoutCancel(ctx), ecosystem, name, version, filename, pkgPURL, versionPURL, upstreamURL, upstreamHash, declared, + &fetch.Artifact{Body: storeR, Size: artifact.Size, ContentType: artifact.ContentType, ETag: artifact.ETag}) + // storeArtifact closes storeR; this covers a return before it read + // anything, so the copy below never blocks on it. + _ = storeR.Close() + storeDone <- stored{a, path, err} + }() + + readErr, clientOK := copyToBoth(artifact.Body, storeW, clientW) + _ = artifact.Body.Close() + _ = storeW.CloseWithError(readErr) + _ = clientW.CloseWithError(readErr) + + out := <-storeDone + if out.err != nil && readErr == nil && clientOK { + p.Logger.Error("artifact sent to the client but not cached", + "purl", versionPURL, "filename", filename, "url", upstreamURL, "error", out.err) + metrics.RecordArtifactServedUncached(ecosystem, metrics.FailureDetail{ + PURL: versionPURL, Filename: filename, URL: upstreamURL, Err: out.err}) + } + return out.artifact, out.path, out.err +} + +// copyToBoth copies src to store and client until src ends, dropping either +// destination once a write to it fails. It returns the error that ended the +// copy, nil at the end of src, and whether the client took every byte. +func copyToBoth(src io.Reader, store, client io.Writer) (error, bool) { + buffer := artifactCopyBufferPool.Get().(*[]byte) + defer artifactCopyBufferPool.Put(buffer) + buf := *buffer + storeOK, clientOK := true, true + for { + n, err := src.Read(buf) + if n > 0 { + if storeOK { + if _, werr := store.Write(buf[:n]); werr != nil { + storeOK = false + } + } + if clientOK { + if _, werr := client.Write(buf[:n]); werr != nil { + clientOK = false + } + } + if !storeOK && !clientOK { + return errNoReaderLeft, false + } + } + if errors.Is(err, io.EOF) { + return nil, clientOK + } + if err != nil { + return err, false + } + } +} diff --git a/internal/handler/stream_miss_test.go b/internal/handler/stream_miss_test.go new file mode 100644 index 0000000..e3b449b --- /dev/null +++ b/internal/handler/stream_miss_test.go @@ -0,0 +1,444 @@ +package handler + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/git-pkgs/proxy/internal/config" + "github.com/git-pkgs/proxy/internal/metrics" + "github.com/git-pkgs/proxy/internal/scanner" + "github.com/git-pkgs/registries/fetch" + "github.com/prometheus/client_golang/prometheus/testutil" +) + +// lockedBuffer is a log destination safe for the goroutine that stores a +// streamed artifact to write to while a test reads it. +type lockedBuffer struct { + mu sync.Mutex + buf bytes.Buffer +} + +func (b *lockedBuffer) Write(p []byte) (int, error) { + b.mu.Lock() + defer b.mu.Unlock() + return b.buf.Write(p) +} + +func (b *lockedBuffer) String() string { + b.mu.Lock() + defer b.mu.Unlock() + return b.buf.String() +} + +// streamMissSetup is a Git LFS upstream whose object store sends the first +// half of an object, then waits for release before sending the rest, and a +// proxy serving it to clients the way the server marks their requests. +type streamMissSetup struct { + proxy *Proxy + store *mockStorage + client *httptest.Server + git *httptest.Server + objects *httptest.Server + release chan struct{} + content string // what the object store sends + oid string // the object ID clients ask for + path string + downloads atomic.Int32 + logs lockedBuffer +} + +// streamMissContent spans many copy buffers: a stream checked against its +// digest holds back its latest chunk until the next one arrives. +var streamMissContent = strings.Repeat("0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ\n", 400<<10/63) + +// streamMissPrefix is how much a client must get while the upstream still +// holds back the second half. +const streamMissPrefix = 64 << 10 + +func newStreamMiss(t *testing.T, served string) *streamMissSetup { + t.Helper() + return newStreamMissMarked(t, served, true) +} + +// newStreamMissMarked is newStreamMiss, with requests marked to detach +// their upstream fetch as the server does, or not, as when the server has +// no detached fetch to spare. +func newStreamMissMarked(t *testing.T, served string, detached bool) *streamMissSetup { + t.Helper() + sum := sha256.Sum256([]byte(streamMissContent)) + s := &streamMissSetup{content: served, oid: hex.EncodeToString(sum[:]), release: make(chan struct{})} + s.objects = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + s.downloads.Add(1) + half := len(s.content) / 2 + _, _ = io.WriteString(w, s.content[:half]) + w.(http.Flusher).Flush() + select { + case <-s.release: + case <-r.Context().Done(): + return + } + _, _ = io.WriteString(w, s.content[half:]) + })) + s.git = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var req lfsBatchRequest + _ = json.NewDecoder(r.Body).Decode(&req) + resp := lfsBatchResponse{Transfer: "basic"} + for _, o := range req.Objects { + resp.Objects = append(resp.Objects, lfsObjectResponse{OID: o.OID, Size: o.Size, + Actions: map[string]lfsAction{"download": {Href: s.objects.URL + "/objects/" + o.OID}}}) + } + w.Header().Set("Content-Type", gitLFSMediaType) + _ = json.NewEncoder(w).Encode(resp) + })) + + proxy, _, store, _ := setupTestProxy(t) + store.seekable = true + proxy.Logger = slog.New(slog.NewTextHandler(&s.logs, nil)) + proxy.HTTPClient = s.git.Client() + fetcher := fetch.NewFetcher(fetch.WithHTTPClient(s.objects.Client()), fetch.WithMaxRetries(0)) + proxy.Fetcher = fetcher + s.proxy, s.store = proxy, store + h := NewGitHandler(proxy, nil, config.UpstreamConfig{Git: map[string]config.GitUpstream{ + "up": {URL: s.git.URL}, + }}).WithBaseURL("http://proxy.test/") + routes := h.Routes() + // As the server marks client requests. + s.client = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + ctx := StreamCacheMiss(r.Context()) + if detached { + ctx = DetachUpstreamFetch(ctx) + } + routes.ServeHTTP(w, r.WithContext(ctx)) + })) + s.path = fmt.Sprintf("/up/acme/widget.git/info/lfs/objects/%s/%d", s.oid, len(streamMissContent)) + t.Cleanup(func() { + s.releaseUpstream() + s.client.Close() + s.git.Close() + s.objects.Close() + _ = fetcher.Close() + }) + return s +} + +func (s *streamMissSetup) releaseUpstream() { + select { + case <-s.release: + default: + close(s.release) + } +} + +// get requests the object. The response headers must arrive in time even +// while the upstream holds back the second half. +func (s *streamMissSetup) get(t *testing.T) *http.Response { + t.Helper() + type result struct { + resp *http.Response + err error + } + got := make(chan result, 1) + go func() { + resp, err := s.client.Client().Get(s.client.URL + s.path) + got <- result{resp, err} + }() + select { + case r := <-got: + if r.err != nil { + t.Fatal(r.err) + } + return r.resp + case <-time.After(5 * time.Second): + // Once the upstream is released the response arrives; close it. + t.Cleanup(func() { + s.releaseUpstream() + if r := <-got; r.resp != nil { + _ = r.resp.Body.Close() + } + }) + t.Fatal("no response while the upstream was still sending") + return nil + } +} + +// cached waits for the stored copy, which is written after the client's +// stream ends. +func (s *streamMissSetup) cached(t *testing.T) bool { + t.Helper() + deadline := time.Now().Add(5 * time.Second) + for { + res, err := s.proxy.GetCachedArtifact(t.Context(), gitEcosystem, "up/acme/widget", s.oid, s.oid) + if err == nil && res != nil { + _ = res.Reader.Close() + return true + } + if time.Now().After(deadline) { + return false + } + time.Sleep(10 * time.Millisecond) + } +} + +// readFirst reads the first streamMissPrefix bytes of body, failing if they +// don't arrive in time. +func readFirst(t *testing.T, body io.Reader) string { + t.Helper() + got := make(chan string, 1) + go func() { + buf := make([]byte, streamMissPrefix) + _, _ = io.ReadFull(body, buf) + got <- string(buf) + }() + select { + case s := <-got: + return s + case <-time.After(5 * time.Second): + t.Fatal("no bytes reached the client while the upstream was still sending") + return "" + } +} + +func TestCacheMissStreamsBeforeUpstreamFinishes(t *testing.T) { + s := newStreamMiss(t, streamMissContent) + resp := s.get(t) + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode != http.StatusOK || resp.ContentLength != -1 { + t.Fatalf("status %d, Content-Length %d; want 200, chunked", resp.StatusCode, resp.ContentLength) + } + if first := readFirst(t, resp.Body); first != streamMissContent[:streamMissPrefix] { + t.Fatal("first bytes differ from the artifact's") + } + + s.releaseUpstream() + rest, err := io.ReadAll(resp.Body) + if err != nil || string(rest) != streamMissContent[streamMissPrefix:] { + t.Fatalf("rest: %d bytes, %v", len(rest), err) + } + if !s.cached(t) { + t.Fatal("streamed artifact was not cached") + } + + // Served from the cache now, with its size. + s.objects.Close() + again := s.get(t) + body, _ := io.ReadAll(again.Body) + _ = again.Body.Close() + if again.StatusCode != http.StatusOK || string(body) != streamMissContent || again.ContentLength != int64(len(streamMissContent)) { + t.Errorf("cached GET: status %d, length %d, %d bytes", again.StatusCode, again.ContentLength, len(body)) + } + if s.downloads.Load() != 1 { + t.Errorf("downloads = %d, want 1", s.downloads.Load()) + } +} + +func TestCacheMissStreamAbortedOnMismatch(t *testing.T) { + tampered := "X" + streamMissContent[1:] + s := newStreamMiss(t, tampered) + s.releaseUpstream() + resp := s.get(t) + body, err := io.ReadAll(resp.Body) + _ = resp.Body.Close() + if err == nil { + t.Fatalf("read %d bytes without error; want the response aborted", len(body)) + } + if string(body) == tampered { + t.Error("client got every byte of an artifact that doesn't match its object ID") + } + // Nothing to wait for: storeArtifact has refused it once the stream ends. + time.Sleep(100 * time.Millisecond) + if res, _ := s.proxy.GetCachedArtifact(t.Context(), gitEcosystem, "up/acme/widget", s.oid, s.oid); res != nil { + _ = res.Reader.Close() + t.Error("mismatching artifact was cached") + } +} + +func TestCacheMissStreamFinishesWhenStorageFails(t *testing.T) { + s := newStreamMiss(t, streamMissContent) + s.store.mu.Lock() + s.store.storeErr = errors.New("no space left on device") + s.store.mu.Unlock() + before := testutil.ToFloat64(metrics.ArtifactsServedUncached.WithLabelValues(gitEcosystem)) + s.releaseUpstream() + + resp := s.get(t) + body, err := io.ReadAll(resp.Body) + _ = resp.Body.Close() + if err != nil || resp.StatusCode != http.StatusOK || string(body) != streamMissContent { + t.Fatalf("status %d, %d bytes, %v; want the whole artifact", resp.StatusCode, len(body), err) + } + deadline := time.Now().Add(5 * time.Second) + for testutil.ToFloat64(metrics.ArtifactsServedUncached.WithLabelValues(gitEcosystem)) == before && time.Now().Before(deadline) { + time.Sleep(10 * time.Millisecond) + } + if got := testutil.ToFloat64(metrics.ArtifactsServedUncached.WithLabelValues(gitEcosystem)) - before; got != 1 { + t.Errorf("served-uncached count went up by %v, want 1", got) + } + if !strings.Contains(s.logs.String(), "level=ERROR") || !strings.Contains(s.logs.String(), "not cached") { + t.Errorf("storage failure not logged at error level:\n%s", s.logs.String()) + } +} + +func TestCacheMissStoredAfterClientLeaves(t *testing.T) { + s := newStreamMiss(t, streamMissContent) + resp := s.get(t) + readFirst(t, resp.Body) + _ = resp.Body.Close() + s.client.CloseClientConnections() + + s.releaseUpstream() + if !s.cached(t) { + t.Fatal("artifact not cached after the client that started the fetch left") + } +} + +func TestCacheMissWaiterGetsStoredArtifact(t *testing.T) { + s := newStreamMiss(t, streamMissContent) + first := s.get(t) + defer func() { _ = first.Body.Close() }() + readFirst(t, first.Body) + + second := make(chan string, 1) + go func() { + resp, err := s.client.Client().Get(s.client.URL + s.path) + if err != nil { + second <- err.Error() + return + } + body, _ := io.ReadAll(resp.Body) + _ = resp.Body.Close() + second <- fmt.Sprintf("%d %d %s", resp.StatusCode, resp.ContentLength, body) + }() + select { + case got := <-second: + t.Fatalf("second request answered before the artifact was stored: %s", got) + case <-time.After(200 * time.Millisecond): + } + + s.releaseUpstream() + if _, err := io.ReadAll(first.Body); err != nil { + t.Fatal(err) + } + want := fmt.Sprintf("200 %d %s", len(streamMissContent), streamMissContent) + if got := <-second; got != want { + t.Errorf("second request = %.60q..., want the stored artifact with its length", got) + } + if s.downloads.Load() != 1 { + t.Errorf("downloads = %d, want 1", s.downloads.Load()) + } +} + +func newStreamMissScanGroup(t *testing.T, mode string) *scanner.Group { + t.Helper() + g, err := scanner.NewGroup(config.ScanningConfig{ + Enabled: true, + Timeout: "15s", + SigningKey: "test-signing-key", + Scanners: []config.ScannerConfig{ + {Name: "test-scanner", URL: newTestScanServer(t, true, "").URL, Mode: mode}, + }, + }, slog.New(slog.NewTextHandler(io.Discard, nil))) + if err != nil { + t.Fatal(err) + } + return g +} + +// A block-mode scanner must see the whole artifact before any of it is +// served, so a miss isn't streamed. +func TestCacheMissNotStreamedWithBlockingScanner(t *testing.T) { + s := newStreamMiss(t, streamMissContent) + s.proxy.ScanSigningKey = []byte("test-signing-key") + s.proxy.Scanners = newStreamMissScanGroup(t, "block") + + got := make(chan *http.Response, 1) + go func() { + resp, err := s.client.Client().Get(s.client.URL + s.path) + if err != nil { + t.Error(err) + } + got <- resp + }() + select { + case resp := <-got: + _ = resp.Body.Close() + t.Fatal("response started before the upstream finished, with a block-mode scanner") + case <-time.After(300 * time.Millisecond): + } + s.releaseUpstream() + resp := <-got + body, _ := io.ReadAll(resp.Body) + _ = resp.Body.Close() + if resp.StatusCode != http.StatusOK || string(body) != streamMissContent || resp.ContentLength != int64(len(streamMissContent)) { + t.Errorf("status %d, length %d, %d bytes; want the scanned, stored artifact", resp.StatusCode, resp.ContentLength, len(body)) + } +} + +// A monitor-mode scanner doesn't gate the artifact, so the miss is streamed +// and the scan runs after the client's stream ends. +func TestCacheMissStreamedWithMonitoringScanner(t *testing.T) { + s := newStreamMiss(t, streamMissContent) + s.proxy.ScanSigningKey = []byte("test-signing-key") + s.proxy.Scanners = newStreamMissScanGroup(t, "monitor") + + resp := s.get(t) + defer func() { _ = resp.Body.Close() }() + readFirst(t, resp.Body) + s.releaseUpstream() + if _, err := io.ReadAll(resp.Body); err != nil { + t.Fatal(err) + } + if !s.cached(t) { + t.Error("artifact not cached") + } +} + +// slowStorage stores like mockStorage, but takes a while to finish once it +// has read the last byte, as an upload to S3 does. +type slowStorage struct { + *mockStorage + delay time.Duration +} + +func (s *slowStorage) Store(ctx context.Context, path string, r io.Reader) (int64, string, error) { + data, err := io.ReadAll(r) + if err != nil { + return 0, "", err + } + select { + case <-time.After(s.delay): + case <-ctx.Done(): + return 0, "", ctx.Err() + } + return s.mockStorage.Store(ctx, path, bytes.NewReader(data)) +} + +// Without a detached fetch, storing still finishes after the client's +// request has ended with the last byte. +func TestCacheMissStoredAfterResponseWithoutDetach(t *testing.T) { + s := newStreamMissMarked(t, streamMissContent, false) + s.proxy.Storage = &slowStorage{mockStorage: s.store, delay: 300 * time.Millisecond} + s.releaseUpstream() + resp := s.get(t) + body, err := io.ReadAll(resp.Body) + _ = resp.Body.Close() + if err != nil || string(body) != streamMissContent { + t.Fatalf("%d bytes, %v", len(body), err) + } + if !s.cached(t) { + t.Fatal("artifact not cached once the response ended") + } +} diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go index 708b5b3..2acde79 100644 --- a/internal/metrics/metrics.go +++ b/internal/metrics/metrics.go @@ -163,6 +163,14 @@ var ( }, ) + ArtifactsServedUncached = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "proxy_artifacts_served_uncached_total", + Help: "Artifacts sent to the client on a cache miss that could not be cached, such as when storage is full", + }, + []string{"ecosystem"}, + ) + IntegrityFailures = prometheus.NewCounterVec( prometheus.CounterOpts{ Name: "proxy_integrity_failures_total", @@ -319,6 +327,7 @@ func init() { UpstreamRateLimitHolds, StorageOperationDuration, StorageErrors, + ArtifactsServedUncached, ActiveRequests, IntegrityFailures, HealthProbeFailures, @@ -465,6 +474,14 @@ func RecordStorageError(operation string, detail FailureDetail) { recordFailure("proxy_storage_errors_total", map[string]string{"operation": operation}, detail) } +// RecordArtifactServedUncached counts an artifact a client got on a cache +// miss although storing it failed, and keeps the detail for /ui/analytics. +func RecordArtifactServedUncached(ecosystem string, detail FailureDetail) { + ecosystem = purl.NormalizeEcosystem(ecosystem) + ArtifactsServedUncached.WithLabelValues(ecosystem).Inc() + recordFailure("proxy_artifacts_served_uncached_total", map[string]string{"ecosystem": ecosystem}, detail) +} + // RecordScanResult tracks a completed pre-cache scan call. func RecordScanResult(ecosystem, scannerName string, allowed bool, duration time.Duration) { ecosystem = purl.NormalizeEcosystem(ecosystem) diff --git a/internal/scanner/group.go b/internal/scanner/group.go index 227a037..cf5930f 100644 --- a/internal/scanner/group.go +++ b/internal/scanner/group.go @@ -91,6 +91,20 @@ func (g *Group) Covers(ecosystem string) bool { return g.Enabled() && len(g.applicable(ecosystem)) > 0 } +// Blocks reports whether a block-mode scanner applies to ecosystem, so an +// artifact must be scanned whole before any of it is served. +func (g *Group) Blocks(ecosystem string) bool { + if !g.Enabled() { + return false + } + for _, e := range g.applicable(ecosystem) { + if e.mode == modeBlock { + return true + } + } + return false +} + // Timeout returns the per-scan-call timeout used to bound the signed fetch // URL's validity. func (g *Group) Timeout() time.Duration { diff --git a/internal/scanner/group_test.go b/internal/scanner/group_test.go index 48cbebc..cbbaa01 100644 --- a/internal/scanner/group_test.go +++ b/internal/scanner/group_test.go @@ -270,3 +270,27 @@ func TestGroup_Scan_ErrorFailOpen(t *testing.T) { t.Error("Allowed = false, want true — FailOpen must treat scanner errors as allowed") } } + +func TestGroupBlocks(t *testing.T) { + g, err := NewGroup(config.ScanningConfig{ + Enabled: true, + SigningKey: "key", + Scanners: []config.ScannerConfig{ + {Name: "watch", URL: "http://watch.invalid", Mode: "monitor"}, + {Name: "gate", URL: "http://gate.invalid", Mode: "block", Ecosystems: []string{"npm"}}, + }, + }, slog.New(slog.NewTextHandler(io.Discard, nil))) + if err != nil { + t.Fatal(err) + } + if !g.Blocks("npm") { + t.Error("Blocks(npm) = false with a block-mode scanner for npm") + } + if g.Blocks("pypi") { + t.Error("Blocks(pypi) = true with only a monitor-mode scanner for it") + } + var none *Group + if none.Blocks("npm") { + t.Error("nil group blocks") + } +} diff --git a/internal/server/middleware.go b/internal/server/middleware.go index f0dd831..7d2bd98 100644 --- a/internal/server/middleware.go +++ b/internal/server/middleware.go @@ -44,8 +44,11 @@ func GetRequestID(ctx context.Context) string { func markRequestContext(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { ctx := handler.DetachUpstreamFetch(r.Context()) - if r.Method == http.MethodHead { + switch { + case r.Method == http.MethodHead: ctx = handler.ServingHead(ctx) + case r.Method == http.MethodGet && r.Header.Get("Range") == "": + ctx = handler.StreamCacheMiss(ctx) } next.ServeHTTP(w, r.WithContext(ctx)) }) From 673a581f9eac1eae7f2d017dfeddcf105a557c23 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 25 Oct 2026 21:48:59 +0100 Subject: [PATCH 448/633] Mirror Swift registry releases proxy mirror and mirror jobs now take pkg:swift PURLs whose namespace is the registry scope, such as pkg:swift/apple/swift-nio@2.62.0 for apple.swift-nio. The source archive is fetched from upstream.swift, checked against the SHA-256 checksum in the release metadata and cached under the key the archive route serves. PURLs whose namespace is a source repository URL don't name a registry identity and are refused. Unversioned PURLs list releases from upstream.swift. SwiftPM reads the release list and Package.swift before it downloads an archive, and the handler passed both through uncached, so a mirrored release still couldn't be resolved with the upstream down. With cache_metadata the handler now keeps them in the metadata cache along with their Link header, which carries release list pages and the version-specific manifests, and serves them when the upstream fails. Manifests are keyed by tools version, so the 5.8 of an alternate link and the 5.8.0 SwiftPM requests share a key. The mirror stores every release list page and each manifest a release links. --- docs/architecture.md | 1 + docs/configuration.md | 6 +- internal/handler/relay_test.go | 2 +- internal/handler/swift.go | 118 ++++++-------- internal/handler/swift_cache.go | 131 +++++++++++++++ internal/handler/swift_cache_test.go | 76 +++++++++ internal/handler/swift_mirror.go | 125 +++++++++++++++ internal/handler/swift_mirror_test.go | 22 +++ internal/mirror/mirror.go | 4 + internal/mirror/swift.go | 64 ++++++++ internal/mirror/swift_test.go | 16 ++ internal/mirror/versions.go | 6 + internal/server/mirror_swift_test.go | 205 ++++++++++++++++++++++++ internal/server/mirror_versions.go | 2 + internal/server/mirror_versions_test.go | 3 +- 15 files changed, 710 insertions(+), 71 deletions(-) create mode 100644 internal/handler/swift_cache.go create mode 100644 internal/handler/swift_cache_test.go create mode 100644 internal/handler/swift_mirror.go create mode 100644 internal/handler/swift_mirror_test.go create mode 100644 internal/mirror/swift.go create mode 100644 internal/mirror/swift_test.go create mode 100644 internal/server/mirror_swift_test.go diff --git a/docs/architecture.md b/docs/architecture.md index e7ce8c8..5bb1b12 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -304,6 +304,7 @@ HTTP protocol handlers for each registry type. **SwiftHandler:** - Proxies the Swift Package Registry v1 read endpoints - Rewrites release URLs and caches source archives +- With `cache_metadata`, caches release metadata, release lists and manifests (with their `Link` header) and serves them when the upstream fails **GitHandler:** - Serves `/git/{name}/{repo}` from `internal/gitcache`, read-only over smart HTTP diff --git a/docs/configuration.md b/docs/configuration.md index 269d5bd..0966090 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1328,6 +1328,8 @@ Some metadata never changes once published, and is cached like an artifact, with Composer archive downloads look up their upstream URL in the same cached metadata, so an archive is resolved without asking Packagist again and stays installable while it is down. Only a version or file the cached metadata does not list yet is looked up upstream. With or without `cache_metadata`, a cached archive of a tagged release from the default repository is served without any metadata lookup; with cooldown enabled it is checked against the publish time recorded when the archive was first resolved. Branch (dev) versions, routed packages and archives picked by a query string such as `?sha=` are always resolved from the metadata. +Swift release lists and manifests are cached with the upstream's `Link` header, which carries the release list's pages and the version-specific manifests a client chooses from. + The `proxy mirror` command always enables metadata caching regardless of this setting. ### Metadata TTL @@ -1565,6 +1567,8 @@ A conda version is mirrored with every build anaconda.org lists for it, across a A Composer release is mirrored with the dist archive `upstream.composer` lists for it, checked against its shasum when one is listed, and cached under the name `/composer/files/...` serves it as. Give the version as the repository lists it, for example `pkg:composer/monolog/monolog@3.8.1`. Branches such as `dev-main` aren't mirrored, since their archive changes with every push, and neither are packages in `upstream.composer_routes`. +A Swift release is mirrored from `upstream.swift` with its source archive, checked against the SHA-256 checksum in the release metadata and cached under the key `/swift/{scope}/{name}/{version}.zip` serves. Give the registry scope as the namespace, for example `pkg:swift/apple/swift-argument-parser@1.5.0` for `apple.swift-argument-parser`. A PURL whose namespace is a source repository, such as `pkg:swift/github.com/apple/swift-argument-parser`, doesn't name a registry identity and is refused. The release metadata, the package's release list and the release's `Package.swift` with its version-specific manifests are stored in the metadata cache too, so `swift package resolve` works with the upstream unreachable. Serving them, and the archive, offline needs `cache_metadata` on the server, since the archive is served with the checksum from the release metadata. + | Flag | Default | Description | |------|---------|-------------| | `--sbom` | | Path to CycloneDX or SPDX SBOM file | @@ -1585,7 +1589,7 @@ proxy mirror pkg:npm/lodash@4.17.21 pkg:cargo/serde@1.0.0 ### Unversioned PURLs -A PURL without a version, here or in a mirror API job, mirrors every version the configured upstream lists, through the same client, so `upstream.auth` and `upstream.allow_private_hosts` apply. npm packages in `upstream.npm_routes` are listed from their route. Listing works for npm, gem, golang, hex (from `upstream.hex_api`), pub, pypi, maven, nuget, composer, cran and brew. For cargo, conda and julia the upstream doesn't serve the API used for listing, so versions come from the public registry while the upstream is the default one; with another upstream, and for Composer packages in `upstream.composer_routes` or any other ecosystem, give a version. +A PURL without a version, here or in a mirror API job, mirrors every version the configured upstream lists, through the same client, so `upstream.auth` and `upstream.allow_private_hosts` apply. npm packages in `upstream.npm_routes` are listed from their route. Listing works for npm, gem, golang, hex (from `upstream.hex_api`), pub, pypi, maven, nuget, composer, cran, brew and swift. For cargo, conda and julia the upstream doesn't serve the API used for listing, so versions come from the public registry while the upstream is the default one; with another upstream, and for Composer packages in `upstream.composer_routes` or any other ecosystem, give a version. ### Container images diff --git a/internal/handler/relay_test.go b/internal/handler/relay_test.go index a7a8fb7..ec60460 100644 --- a/internal/handler/relay_test.go +++ b/internal/handler/relay_test.go @@ -53,7 +53,7 @@ func TestRelayRoutes(t *testing.T) { "/conan/v2/conans/demo/1.0/user/stable/revisions/rev/files/other.txt", "/composer/search.json", "/pypi/simple/", "/gem/api/v1/dependencies", "/gem/info/demo", "/conda/conda-forge/noarch/repodata.json", - "/hex/packages/demo", "/swift/scope/demo/1.0.0/Package.swift", + "/hex/packages/demo", "/v2/library/demo/manifests/latest", "/v2/library/demo/tags/list", "/npm/-/npm/v1/keys", } { diff --git a/internal/handler/swift.go b/internal/handler/swift.go index d7d403f..7de0cd1 100644 --- a/internal/handler/swift.go +++ b/internal/handler/swift.go @@ -104,23 +104,23 @@ func (h *SwiftHandler) handlePackageReleases(w http.ResponseWriter, r *http.Requ scope, name = canonicalSwiftPackage(scope, name) upstreamURL := h.buildUpstreamURL(scope, name, "", "", r.URL.RawQuery) - body, contentType, responseHeaders, err := h.fetchMetadataWithHeaders( - r.Context(), upstreamURL, requestAccept(r, swiftAcceptJSON), + doc, err := h.fetchDocument( + r.Context(), swiftReleasesCacheKey(scope, name, r.URL.RawQuery), upstreamURL, requestAccept(r, swiftAcceptJSON), ) if err != nil { h.writeMetadataError(w, err) return } - rewritten, err := h.rewriteReleaseURLs(scope, name, body) + rewritten, err := h.rewriteReleaseURLs(scope, name, doc.body) if err != nil { h.proxy.Logger.Warn("failed to rewrite Swift release URLs", "error", err) - rewritten = body + rewritten = doc.body } - for _, link := range responseHeaders.Values("Link") { - w.Header().Add("Link", h.rewriteLinkHeader(link, upstreamURL)) + if doc.link != "" { + w.Header().Set("Link", h.rewriteLinkHeader(doc.link, upstreamURL)) } - writeSwiftMetadata(w, r, rewritten, contentType) + writeSwiftMetadata(w, r, rewritten, doc.contentType) } func (h *SwiftHandler) handleRelease(w http.ResponseWriter, r *http.Request) { @@ -161,7 +161,17 @@ func (h *SwiftHandler) handleManifest(w http.ResponseWriter, r *http.Request) { scope, name = canonicalSwiftPackage(scope, name) upstreamURL := h.buildUpstreamURL(scope, name, version, "Package.swift", r.URL.RawQuery) - h.proxySwiftResource(w, r, upstreamURL, swiftAcceptManifest) + doc, err := h.fetchDocument( + r.Context(), swiftManifestCacheKey(scope, name, version, r.URL.RawQuery), upstreamURL, requestAccept(r, swiftAcceptManifest), + ) + if err != nil { + h.writeMetadataError(w, err) + return + } + if doc.link != "" { + w.Header().Set("Link", h.rewriteLinkHeader(doc.link, upstreamURL)) + } + writeSwiftMetadata(w, r, doc.body, doc.contentType) } func (h *SwiftHandler) handleIdentifiers(w http.ResponseWriter, r *http.Request) { @@ -194,14 +204,13 @@ func (h *SwiftHandler) handleSourceArchive(w http.ResponseWriter, r *http.Reques } scope, name = canonicalSwiftPackage(scope, name) - packageName := scope + "/" + name - filename := fmt.Sprintf("%s-%s.zip", name, version) - upstreamURL := h.buildUpstreamURL(scope, name, version+".zip", "", r.URL.RawQuery) - packagePURL, versionPURL := packageurl.MakeCacheStrings("swift", packageName, version) - if packagePURL == "" || versionPURL == "" { - h.writeArtifactError(w, fmt.Errorf("%w: swift %q", errUnsupportedPackageIdentity, packageName)) + key, err := swiftSourceArchiveKey(scope, name, version) + if err != nil { + h.writeArtifactError(w, err) return } + filename, packagePURL, versionPURL := key.filename, key.packagePURL, key.versionPURL + upstreamURL := h.buildUpstreamURL(scope, name, version+".zip", "", r.URL.RawQuery) archiveInfo, infoErr := h.fetchArchiveInfo(r.Context(), scope, name, version) if infoErr != nil { h.writeArtifactError(w, fmt.Errorf("fetching release metadata: %w", infoErr)) @@ -215,10 +224,7 @@ func (h *SwiftHandler) handleSourceArchive(w http.ResponseWriter, r *http.Reques headers := make(http.Header) headers.Set("Accept", requestAccept(r, swiftAcceptArchive)) - result, err := h.proxy.getOrFetchArtifactFromURLWithCachePURLs( - r.Context(), "swift", packageName, version, filename, packagePURL, versionPURL, - upstreamURL, headers, archiveInfo.checksum, "", - ) + result, err := h.fetchSourceArchive(r.Context(), key, version, upstreamURL, headers, archiveInfo) if err != nil { h.writeArtifactError(w, err) return @@ -229,6 +235,32 @@ func (h *SwiftHandler) handleSourceArchive(w http.ResponseWriter, r *http.Reques ServeArtifactRequest(w, r, result) } +// swiftArchiveKey is the cache key a release's source archive is stored +// under. +type swiftArchiveKey struct { + packageName, filename, packagePURL, versionPURL string +} + +// swiftSourceArchiveKey returns the cache key of the source archive of the +// canonical (lowercased) scope and name at version. +func swiftSourceArchiveKey(scope, name, version string) (swiftArchiveKey, error) { + key := swiftArchiveKey{packageName: scope + "/" + name, filename: fmt.Sprintf("%s-%s.zip", name, version)} + key.packagePURL, key.versionPURL = packageurl.MakeCacheStrings("swift", key.packageName, version) + if key.packagePURL == "" || key.versionPURL == "" { + return swiftArchiveKey{}, fmt.Errorf("%w: swift %q", errUnsupportedPackageIdentity, key.packageName) + } + return key, nil +} + +// fetchSourceArchive returns the source archive under key from the cache, or +// fetches it from upstreamURL, verified against the checksum in info. +func (h *SwiftHandler) fetchSourceArchive(ctx context.Context, key swiftArchiveKey, version, upstreamURL string, headers http.Header, info swiftArchiveInfo) (*CacheResult, error) { + return h.proxy.getOrFetchArtifactFromURLWithCachePURLs( + ctx, "swift", key.packageName, version, key.filename, key.packagePURL, key.versionPURL, + upstreamURL, headers, info.checksum, "", + ) +} + func (h *SwiftHandler) handleSourceArchiveHead( w http.ResponseWriter, r *http.Request, @@ -424,56 +456,6 @@ func swiftDigestHeader(checksum string) string { return "sha-256=" + base64.StdEncoding.EncodeToString(digest) } -func (h *SwiftHandler) proxySwiftResource(w http.ResponseWriter, r *http.Request, upstreamURL, defaultAccept string) { - req, err := http.NewRequestWithContext(r.Context(), r.Method, upstreamURL, nil) - if err != nil { - writeSwiftProblem(w, http.StatusInternalServerError, "failed to create upstream request") - return - } - req.Header.Set("Accept", requestAccept(r, defaultAccept)) - for _, name := range []string{"If-Modified-Since", "If-None-Match"} { - if value := r.Header.Get(name); value != "" { - req.Header.Set(name, value) - } - } - - resp, err := h.proxy.HTTPClient.Do(req) - if err != nil { - if writeUpstreamHeld(w, err) { - return - } - writeSwiftProblem(w, http.StatusBadGateway, "upstream request failed") - return - } - defer func() { _ = resp.Body.Close() }() - - h.proxy.relayResponse(w, r, resp, func(dst, src http.Header) { - copySwiftResponseHeaders(dst, src) - if location := src.Get("Location"); location != "" { - dst.Set("Location", h.rewriteRegistryURL(location, upstreamURL)) - } - for _, link := range src.Values("Link") { - dst.Add("Link", h.rewriteLinkHeader(link, upstreamURL)) - } - if dst.Get("Content-Version") == "" { - dst.Set("Content-Version", swiftContentVersion) - } - }) -} - -func copySwiftResponseHeaders(dst, src http.Header) { - for _, name := range []string{ - "Cache-Control", "Content-Disposition", "Content-Language", headerContentLength, - headerContentType, "Content-Version", "Digest", headerETag, headerLastModified, - "Retry-After", "Vary", "Warning", "X-Swift-Package-Signature", - "X-Swift-Package-Signature-Format", - } { - for _, value := range src.Values(name) { - dst.Add(name, value) - } - } -} - func (h *SwiftHandler) rewriteLinkHeader(value, upstreamRequestURL string) string { var result strings.Builder for len(value) > 0 { diff --git a/internal/handler/swift_cache.go b/internal/handler/swift_cache.go new file mode 100644 index 0000000..e149331 --- /dev/null +++ b/internal/handler/swift_cache.go @@ -0,0 +1,131 @@ +package handler + +import ( + "bytes" + "context" + "database/sql" + "errors" + "net/url" + "strings" + "time" + + "github.com/git-pkgs/proxy/internal/database" +) + +// With cache_metadata, release lists and manifests are kept in the metadata +// cache along with the upstream Link header, which carries the pagination of +// release lists and the version-specific manifests a client picks from. They +// are served from the cache within metadata_ttl, and after it when the +// upstream fails, as release metadata is. + +// swiftDocument is a release list or manifest as the upstream sent it. +type swiftDocument struct { + body []byte + contentType string + // link is the upstream Link header, its values joined, with the + // upstream's URLs. + link string + fetchedAt time.Time +} + +// fetchDocument returns the document at upstreamURL, from the metadata cache +// under cacheKey while it is fresh or when the upstream fails, and otherwise +// from upstream, storing it. +func (h *SwiftHandler) fetchDocument(ctx context.Context, cacheKey, upstreamURL, accept string) (*swiftDocument, error) { + cached := h.loadDocument(ctx, cacheKey) + if cached != nil && h.proxy.MetadataTTL > 0 && time.Since(cached.fetchedAt) < h.proxy.MetadataTTL { + return cached, nil + } + body, contentType, headers, err := h.fetchMetadataWithHeaders(ctx, upstreamURL, accept) + if err != nil { + if cached == nil || errors.Is(err, ErrUpstreamNotFound) { + return nil, err + } + h.proxy.Logger.Warn("upstream Swift metadata fetch failed, serving cached copy", + "url", upstreamURL, "error", err) + return cached, nil + } + doc := &swiftDocument{ + body: body, + contentType: contentType, + link: strings.Join(headers.Values("Link"), ", "), + fetchedAt: time.Now(), + } + h.storeDocument(ctx, cacheKey, doc) + return doc, nil +} + +func (h *SwiftHandler) documentsCached() bool { + return h.proxy.CacheMetadata && h.proxy.DB != nil && h.proxy.Storage != nil +} + +// loadDocument returns the cached document under cacheKey, or nil. +func (h *SwiftHandler) loadDocument(ctx context.Context, cacheKey string) *swiftDocument { + if !h.documentsCached() { + return nil + } + entry, err := h.proxy.DB.GetMetadataCache("swift", cacheKey) + if err != nil || entry == nil || !entry.FetchedAt.Valid { + return nil + } + reader, err := h.proxy.Storage.Open(ctx, entry.StoragePath) + if err != nil { + return nil + } + defer func() { _ = reader.Close() }() + body, err := h.proxy.ReadMetadata(reader) + if err != nil { + return nil + } + h.proxy.recordMetadataRead(entry) + return &swiftDocument{ + body: body, + contentType: entry.ContentType.String, + link: entry.Link.String, + fetchedAt: entry.FetchedAt.Time, + } +} + +// storeDocument caches doc under cacheKey. A failure is logged; the +// document is still served. +func (h *SwiftHandler) storeDocument(ctx context.Context, cacheKey string, doc *swiftDocument) { + if !h.documentsCached() { + return + } + storagePath := metadataStoragePath("swift", cacheKey) + size, _, err := h.proxy.Storage.Store(ctx, storagePath, bytes.NewReader(doc.body)) + if err == nil { + err = h.proxy.DB.UpsertMetadataCache(&database.MetadataCacheEntry{ + Ecosystem: "swift", + Name: cacheKey, + StoragePath: storagePath, + Link: sql.NullString{String: doc.link, Valid: doc.link != ""}, + ContentType: sql.NullString{String: doc.contentType, Valid: doc.contentType != ""}, + Size: sql.NullInt64{Int64: size, Valid: true}, + FetchedAt: sql.NullTime{Time: doc.fetchedAt, Valid: true}, + }) + } + if err != nil { + h.proxy.Logger.Warn("failed to cache Swift metadata", "key", cacheKey, "error", err) + } +} + +func swiftReleasesCacheKey(scope, name, rawQuery string) string { + return swiftMetadataCacheKey("releases", scope, name, rawQuery) +} + +// swiftManifestCacheKey keys a manifest by its swift-version as a tools +// version, so the 5.8 of a manifest's alternate link and the 5.8.0 a client +// requests for it share a key. +func swiftManifestCacheKey(scope, name, version, rawQuery string) string { + if query, err := url.ParseQuery(rawQuery); err == nil { + if tools := query.Get("swift-version"); tools != "" { + for strings.Count(tools, ".") > 0 && strings.HasSuffix(tools, ".0") { + tools = strings.TrimSuffix(tools, ".0") + } + query.Set("swift-version", tools) + } + rawQuery = query.Encode() + } + return swiftMetadataCacheKey("manifest", scope, name, version, rawQuery) +} diff --git a/internal/handler/swift_cache_test.go b/internal/handler/swift_cache_test.go new file mode 100644 index 0000000..06692c0 --- /dev/null +++ b/internal/handler/swift_cache_test.go @@ -0,0 +1,76 @@ +package handler + +import ( + "io" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" +) + +func TestSwiftReleaseListAndManifestServedFromCacheWhenUpstreamFails(t *testing.T) { + var down atomic.Bool + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if down.Load() { + http.Error(w, "down", http.StatusServiceUnavailable) + return + } + w.Header().Set("Link", `; rel="next"`) + if strings.HasSuffix(r.URL.Path, "/Package.swift") { + w.Header().Set("Content-Type", "text/x-swift") + _, _ = io.WriteString(w, "// swift-tools-version:5.9") + return + } + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, `{"releases":{"1.0.0":{}}}`) + })) + defer upstream.Close() + + for _, cacheMetadata := range []bool{true, false} { + proxy, _, _, _ := setupTestProxy(t) + proxy.CacheMetadata = cacheMetadata + routes := NewSwiftHandler(proxy, "https://proxy.example", upstream.URL+"/registry").Routes() + get := func(path string) *httptest.ResponseRecorder { + w := httptest.NewRecorder() + routes.ServeHTTP(w, httptest.NewRequest(http.MethodGet, path, nil)) + return w + } + down.Store(false) + for _, path := range []string{"/apple/example", "/apple/example/1.0.0/Package.swift"} { + if w := get(path); w.Code != http.StatusOK { + t.Fatalf("%s: status = %d with the upstream up", path, w.Code) + } + } + + down.Store(true) + for path, body := range map[string]string{"/apple/example": `"1.0.0"`, "/apple/example/1.0.0/Package.swift": "swift-tools-version:5.9"} { + w := get(path) + if !cacheMetadata { + if w.Code == http.StatusOK { + t.Errorf("%s: served without cache_metadata while the upstream is down", path) + } + continue + } + if w.Code != http.StatusOK || !strings.Contains(w.Body.String(), body) { + t.Errorf("%s: status = %d, body %q; want the cached copy", path, w.Code, w.Body.String()) + } + if link := w.Header().Get("Link"); link != `; rel="next"` { + t.Errorf("%s: Link = %q, want the cached Link rewritten", path, link) + } + } + } +} + +func TestSwiftManifestCacheKeyIgnoresToolsVersionZeros(t *testing.T) { + key := swiftManifestCacheKey("apple", "example", "1.0.0", "swift-version=5.8") + if got := swiftManifestCacheKey("apple", "example", "1.0.0", "swift-version=5.8.0"); got != key { + t.Error("5.8.0 and 5.8 have different keys") + } + if got := swiftManifestCacheKey("apple", "example", "1.0.0", "swift-version=5.9"); got == key { + t.Error("5.9 and 5.8 share a key") + } + if got := swiftManifestCacheKey("apple", "example", "1.0.0", ""); got == key { + t.Error("Package.swift and the 5.8 manifest share a key") + } +} diff --git a/internal/handler/swift_mirror.go b/internal/handler/swift_mirror.go new file mode 100644 index 0000000..3374cc9 --- /dev/null +++ b/internal/handler/swift_mirror.go @@ -0,0 +1,125 @@ +package handler + +import ( + "context" + "fmt" + "net/http" + "net/url" + "strings" +) + +// MirrorRelease caches the source archive of release version of the registry +// package scope.name, under the key the source archive route serves it from, +// verified against the SHA-256 checksum its release metadata lists. The +// release metadata is cached as that route reads it, and with cache_metadata +// so are the package's release list and the release's manifests, which a +// client reads before it downloads the archive. A client can then resolve and +// fetch the release without the upstream. +func (h *SwiftHandler) MirrorRelease(ctx context.Context, scope, name, version string) (*CacheResult, error) { + if !validSwiftPackageReference(scope, name, version) { + return nil, fmt.Errorf("invalid Swift registry release %s.%s@%s", scope, name, version) + } + scope, name = canonicalSwiftPackage(scope, name) + key, err := swiftSourceArchiveKey(scope, name, version) + if err != nil { + return nil, err + } + if h.documentsCached() { + if err := h.mirrorReleaseList(ctx, scope, name); err != nil { + return nil, fmt.Errorf("fetching release list: %w", err) + } + if err := h.mirrorManifests(ctx, scope, name, version); err != nil { + return nil, fmt.Errorf("fetching manifest: %w", err) + } + } + info, err := h.fetchArchiveInfo(ctx, scope, name, version) + if err != nil { + return nil, fmt.Errorf("fetching release metadata: %w", err) + } + headers := http.Header{"Accept": {swiftAcceptArchive}} + return h.fetchSourceArchive(ctx, key, version, h.buildUpstreamURL(scope, name, version+".zip", "", ""), headers, info) +} + +// maxSwiftReleasePages bounds the release list pages a mirror follows. +const maxSwiftReleasePages = 100 + +// mirrorReleaseList caches every page of the release list, under the keys +// the release list route reads for the page links it rewrites. +func (h *SwiftHandler) mirrorReleaseList(ctx context.Context, scope, name string) error { + query := "" + seen := map[string]bool{} + for page := 0; page < maxSwiftReleasePages && !seen[query]; page++ { + seen[query] = true + upstreamURL := h.buildUpstreamURL(scope, name, "", "", query) + doc, err := h.fetchDocument(ctx, swiftReleasesCacheKey(scope, name, query), upstreamURL, swiftAcceptJSON) + if err != nil { + return err + } + next := swiftLinkQueries(doc.link, "next", upstreamURL) + if len(next) == 0 { + return nil + } + query = next[0] + } + return nil +} + +// mirrorManifests caches Package.swift of a release and the version-specific +// manifests its Link header lists, under the keys the manifest route reads +// for the alternate links it rewrites. +func (h *SwiftHandler) mirrorManifests(ctx context.Context, scope, name, version string) error { + upstreamURL := h.buildUpstreamURL(scope, name, version, "Package.swift", "") + doc, err := h.fetchDocument(ctx, swiftManifestCacheKey(scope, name, version, ""), upstreamURL, swiftAcceptManifest) + if err != nil { + return err + } + for _, query := range swiftLinkQueries(doc.link, "alternate", upstreamURL) { + alternateURL := h.buildUpstreamURL(scope, name, version, "Package.swift", query) + if _, err := h.fetchDocument(ctx, swiftManifestCacheKey(scope, name, version, query), alternateURL, swiftAcceptManifest); err != nil { + return fmt.Errorf("%s: %w", query, err) + } + } + return nil +} + +// swiftLinkQueries returns the queries of the links with relation rel in +// link, a Link header, that point at requestURL's own path with a query, +// such as the next page of a release list or a version-specific manifest. +func swiftLinkQueries(link, rel, requestURL string) []string { + base, err := url.Parse(requestURL) + if err != nil { + return nil + } + var queries []string + for _, value := range strings.Split(link, ",") { + target, params, ok := strings.Cut(strings.TrimSpace(value), ";") + if !ok || !strings.HasPrefix(target, "<") || !strings.HasSuffix(target, ">") || !swiftLinkHasRel(params, rel) { + continue + } + ref, err := url.Parse(strings.TrimSpace(target[1 : len(target)-1])) + if err != nil { + continue + } + resolved := base.ResolveReference(ref) + if resolved.Host != base.Host || resolved.EscapedPath() != base.EscapedPath() || resolved.RawQuery == "" { + continue + } + queries = append(queries, resolved.RawQuery) + } + return queries +} + +func swiftLinkHasRel(params, rel string) bool { + for _, param := range strings.Split(params, ";") { + key, value, ok := strings.Cut(strings.TrimSpace(param), "=") + if !ok || !strings.EqualFold(strings.TrimSpace(key), "rel") { + continue + } + for _, r := range strings.Fields(strings.Trim(strings.TrimSpace(value), `"`)) { + if strings.EqualFold(r, rel) { + return true + } + } + } + return false +} diff --git a/internal/handler/swift_mirror_test.go b/internal/handler/swift_mirror_test.go new file mode 100644 index 0000000..8359a5e --- /dev/null +++ b/internal/handler/swift_mirror_test.go @@ -0,0 +1,22 @@ +package handler + +import ( + "reflect" + "testing" +) + +func TestSwiftLinkQueries(t *testing.T) { + const manifest = "https://registry.example/swift/apple/example/1.0.0/Package.swift" + link := `; rel="alternate"; filename="Package@swift-5.8.swift", ` + + `; rel=alternate, ` + + `; rel="alternate", ` + + `; rel="alternate", ` + + `; rel="canonical"` + + got := swiftLinkQueries(link, "alternate", manifest) + + want := []string{"swift-version=5.8", "swift-version=5.7"} + if !reflect.DeepEqual(got, want) { + t.Errorf("swiftLinkQueries() = %q, want %q", got, want) + } +} diff --git a/internal/mirror/mirror.go b/internal/mirror/mirror.go index 6bd8d18..cca5d23 100644 --- a/internal/mirror/mirror.go +++ b/internal/mirror/mirror.go @@ -236,6 +236,10 @@ func (m *Mirror) mirrorOne(ctx context.Context, pv PackageVersion, tracker *prog m.mirrorComposer(ctx, pv, tracker) return } + if pv.Ecosystem == "swift" { + m.mirrorSwift(ctx, pv, tracker) + return + } result, err := m.proxy.GetOrFetchArtifact(ctx, pv.Ecosystem, pv.Name, pv.Version, "") if err != nil { m.recordFailure(tracker, pv, err) diff --git a/internal/mirror/swift.go b/internal/mirror/swift.go new file mode 100644 index 0000000..516276d --- /dev/null +++ b/internal/mirror/swift.go @@ -0,0 +1,64 @@ +package mirror + +import ( + "context" + "fmt" + "strings" + + "github.com/git-pkgs/proxy/internal/handler" +) + +// mirrorSwift mirrors the source archive of a Swift registry release, with +// its release metadata, from the registry the proxy serves Swift packages +// from. +func (m *Mirror) mirrorSwift(ctx context.Context, pv PackageVersion, tracker *progressTracker) { + scope, name, err := swiftScopeAndName(pv.Name) + if err != nil { + m.recordFailure(tracker, pv, err) + return + } + upstream, err := m.registryURL("swift", pv.Name) + if err != nil { + m.recordFailure(tracker, pv, err) + return + } + result, err := handler.NewSwiftHandler(m.proxy, "", upstream).MirrorRelease(ctx, scope, name, pv.Version) + if err != nil { + m.recordFailure(tracker, pv, err) + return + } + m.recordResult(tracker, pv, result) +} + +// registryURL returns the registry base URL Versions lists purlType packages +// from, which is the configured upstream, or empty for the default one. +func (m *Mirror) registryURL(purlType, name string) (string, error) { + if r, ok := m.Versions.(RegistryVersions); ok && r.URL != nil { + return r.URL(purlType, name) + } + return "", nil +} + +// swiftScopeAndName splits the name of a pkg:swift PURL, its namespace and +// name, into a registry scope and package name. The namespace of a registry +// package is its scope, as in pkg:swift/apple/swift-nio for the identity +// apple.swift-nio. A namespace naming a source repository, such as +// pkg:swift/github.com/apple/swift-nio, isn't a registry identity and is +// refused. +func swiftScopeAndName(name string) (scope, pkg string, _ error) { + scope, pkg, ok := strings.Cut(name, "/") + if !ok || strings.ContainsAny(scope, ".:") || strings.Contains(pkg, "/") { + return "", "", fmt.Errorf("pkg:swift/%s names a source repository; mirroring needs a registry scope as the namespace, such as pkg:swift/apple/swift-nio for apple.swift-nio", name) + } + return scope, pkg, nil +} + +// swiftIdentity returns the registry identity, scope.name, of the name of a +// pkg:swift PURL. +func swiftIdentity(name string) (string, error) { + scope, pkg, err := swiftScopeAndName(name) + if err != nil { + return "", err + } + return scope + "." + pkg, nil +} diff --git a/internal/mirror/swift_test.go b/internal/mirror/swift_test.go new file mode 100644 index 0000000..2d05615 --- /dev/null +++ b/internal/mirror/swift_test.go @@ -0,0 +1,16 @@ +package mirror + +import "testing" + +func TestSwiftIdentity(t *testing.T) { + for name, want := range map[string]string{ + "apple/swift-nio": "apple.swift-nio", + "github.com/apple/swift-nio": "", + "swift-nio": "", + } { + got, err := swiftIdentity(name) + if got != want || (want == "") != (err != nil) { + t.Errorf("swiftIdentity(%q) = %q, %v; want %q", name, got, err, want) + } + } +} diff --git a/internal/mirror/versions.go b/internal/mirror/versions.go index 1b68a6d..973ff69 100644 --- a/internal/mirror/versions.go +++ b/internal/mirror/versions.go @@ -57,6 +57,12 @@ func (r RegistryVersions) FindVersion(ctx context.Context, ecosystem, name, vers } func (r RegistryVersions) fetchVersions(ctx context.Context, ecosystem, name string) ([]registries.Version, error) { + if ecosystem == "swift" { + var err error + if name, err = swiftIdentity(name); err != nil { + return nil, err + } + } purlType := purl.EcosystemToPURLType(ecosystem) var baseURL string if r.URL != nil { diff --git a/internal/server/mirror_swift_test.go b/internal/server/mirror_swift_test.go new file mode 100644 index 0000000..d10b682 --- /dev/null +++ b/internal/server/mirror_swift_test.go @@ -0,0 +1,205 @@ +package server + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + + "github.com/git-pkgs/proxy/internal/config" + "github.com/git-pkgs/proxy/internal/database" + "github.com/git-pkgs/proxy/internal/handler" + "github.com/git-pkgs/proxy/internal/mirror" + "github.com/git-pkgs/proxy/internal/storage" +) + +// swiftUpstream serves the Swift registry API for acme.widget with releases +// 1.0.0, on the first page of the release list, and 1.1.0, on the second. +// Each release has a Package.swift that links a manifest for Swift 5.8. +type swiftUpstream struct { + *httptest.Server + // checksum overrides the archive checksum the release metadata lists. + checksum string + hits atomic.Int32 + archiveHits atomic.Int32 +} + +func swiftArchive(version string) string { return "zip of acme.widget " + version } + +func newSwiftUpstream(t *testing.T) *swiftUpstream { + t.Helper() + u := &swiftUpstream{} + u.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + u.hits.Add(1) + w.Header().Set("Content-Version", "1") + rest, ok := strings.CutPrefix(r.URL.Path, "/acme/widget") + switch { + case !ok: + http.NotFound(w, r) + case rest == "": + page := map[string]string{"": "1.0.0", "page=2": "1.1.0"}[r.URL.RawQuery] + if page == "" { + http.NotFound(w, r) + return + } + if r.URL.RawQuery == "" { + w.Header().Set("Link", fmt.Sprintf(`<%s/acme/widget?page=2>; rel="next"`, u.URL)) + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{"releases": map[string]any{ + page: map[string]string{"url": u.URL + "/acme/widget/" + page}, + }}) + case strings.HasSuffix(rest, ".zip"): + u.archiveHits.Add(1) + _, _ = io.WriteString(w, swiftArchive(strings.TrimSuffix(rest[1:], ".zip"))) + case strings.HasSuffix(rest, "/Package.swift"): + version := strings.TrimSuffix(rest[1:], "/Package.swift") + w.Header().Set("Content-Type", "text/x-swift") + if r.URL.Query().Get("swift-version") == "5.8" { + _, _ = io.WriteString(w, "// swift-tools-version:5.8\n// "+version) + return + } + w.Header().Set("Link", fmt.Sprintf(`<%s/acme/widget/%s/Package.swift?swift-version=5.8>; rel="alternate"; filename="Package@swift-5.8.swift"; swift-tools-version="5.8"`, u.URL, version)) + _, _ = io.WriteString(w, "// swift-tools-version:5.9\n// "+version) + default: + version := rest[1:] + sum := sha256.Sum256([]byte(swiftArchive(version))) + checksum := hex.EncodeToString(sum[:]) + if u.checksum != "" { + checksum = u.checksum + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{ + "id": "acme.widget", "version": version, + "resources": []map[string]string{{"name": "source-archive", "type": "application/zip", "checksum": checksum}}, + }) + } + })) + t.Cleanup(u.Close) + return u +} + +// mirrorSwift mirrors purls from upstream the way proxy mirror does, with +// metadata caching on. +func mirrorSwift(t *testing.T, upstream *swiftUpstream, purls ...string) (*mirror.Progress, *handler.Proxy) { + t.Helper() + cfg := config.Default() + cfg.Upstream.Swift = upstream.URL + cfg.Upstream.AllowLoopback = true + db, err := database.Create(t.TempDir() + "/test.db") + if err != nil { + t.Fatal(err) + } + if err := db.MigrateSchema(); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = db.Close() }) + store, err := storage.OpenBucket(context.Background(), "file://"+t.TempDir()) + if err != nil { + t.Fatal(err) + } + logger := slog.New(slog.NewTextHandler(io.Discard, nil)) + proxy, _ := NewMirrorProxy(cfg, db, store, logger) + proxy.CacheMetadata = true + m := mirror.New(proxy, db, store, logger, 1) + m.Versions = NewMirrorVersions(cfg.Upstream, proxy.HTTPClient) + progress, err := m.Run(context.Background(), &mirror.PURLSource{PURLs: purls, Versions: m.Versions}) + if err != nil { + t.Fatalf("Run() error = %v", err) + } + return progress, proxy +} + +func swiftRegistryGet(t *testing.T, routes http.Handler, path, accept string) *httptest.ResponseRecorder { + t.Helper() + req := httptest.NewRequest(http.MethodGet, path, nil) + req.Header.Set("Accept", accept) + rec := httptest.NewRecorder() + routes.ServeHTTP(rec, req) + return rec +} + +func TestMirrorSwiftReleaseIsServedFromCache(t *testing.T) { + upstream := newSwiftUpstream(t) + + progress, proxy := mirrorSwift(t, upstream, "pkg:swift/Acme/Widget@1.0.0") + + if progress.Completed != 1 || upstream.archiveHits.Load() != 1 { + t.Fatalf("completed = %d, archive hits = %d, errors = %v", progress.Completed, upstream.archiveHits.Load(), progress.Errors) + } + + // SwiftPM resolves and downloads the release with the upstream down: + // the release list and its pages, the release metadata, both manifests + // and the archive come from the cache. + upstream.Close() + routes := handler.NewSwiftHandler(proxy, "http://proxy.test", upstream.URL).Routes() + const jsonType = "application/vnd.swift.registry.v1+json" + const swiftType = "application/vnd.swift.registry.v1+swift" + checks := []struct { + path, accept, body, link string + }{ + {"/acme/widget", jsonType, "http://proxy.test/swift/acme/widget/1.0.0", ""}, + {"/acme/widget?page=2", jsonType, "http://proxy.test/swift/acme/widget/1.1.0", ""}, + {"/acme/widget/1.0.0", jsonType, `"source-archive"`, ""}, + {"/acme/widget/1.0.0/Package.swift", swiftType, "swift-tools-version:5.9", ""}, + {"/acme/widget/1.0.0/Package.swift?swift-version=5.8.0", swiftType, "swift-tools-version:5.8", ""}, + {"/acme/widget/1.0.0.zip", "application/vnd.swift.registry.v1+zip", swiftArchive("1.0.0"), ""}, + } + for _, check := range checks { + rec := swiftRegistryGet(t, routes, check.path, check.accept) + if rec.Code != http.StatusOK || !strings.Contains(rec.Body.String(), check.body) { + t.Errorf("%s: status = %d, body %q; want 200 with %q", check.path, rec.Code, strings.TrimSpace(rec.Body.String()), check.body) + } + if link := rec.Header().Get("Link"); !strings.Contains(link, check.link) { + t.Errorf("%s: Link = %q, want %q", check.path, link, check.link) + } + } +} + +func TestMirrorSwiftRefusesChecksumMismatch(t *testing.T) { + upstream := newSwiftUpstream(t) + upstream.checksum = strings.Repeat("0", 64) + + progress, proxy := mirrorSwift(t, upstream, "pkg:swift/acme/widget@1.0.0") + + if progress.Failed != 1 || progress.Completed != 0 { + t.Fatalf("completed = %d, failed = %d, errors = %v; want the archive refused", progress.Completed, progress.Failed, progress.Errors) + } + upstream.Close() + routes := handler.NewSwiftHandler(proxy, "http://proxy.test", upstream.URL).Routes() + if rec := swiftRegistryGet(t, routes, "/acme/widget/1.0.0.zip", "application/vnd.swift.registry.v1+zip"); rec.Code == http.StatusOK { + t.Error("the refused archive was cached") + } +} + +func TestMirrorSwiftListsReleases(t *testing.T) { + upstream := newSwiftUpstream(t) + + progress, _ := mirrorSwift(t, upstream, "pkg:swift/acme/widget") + + if progress.Total != 2 || progress.Completed != 2 || upstream.archiveHits.Load() != 2 { + t.Errorf("total = %d, completed = %d, archive hits = %d, errors = %v; want both releases mirrored", + progress.Total, progress.Completed, upstream.archiveHits.Load(), progress.Errors) + } +} + +func TestMirrorSwiftRefusesRepositoryNamespaces(t *testing.T) { + upstream := newSwiftUpstream(t) + + progress, _ := mirrorSwift(t, upstream, "pkg:swift/github.com/acme/widget@1.0.0") + + if progress.Failed != 1 || len(progress.Errors) != 1 || !strings.Contains(progress.Errors[0].Error, "registry scope") { + t.Errorf("failed = %d, errors = %v; want the repository PURL refused", progress.Failed, progress.Errors) + } + if upstream.hits.Load() != 0 { + t.Error("the upstream was asked for a repository PURL") + } +} diff --git a/internal/server/mirror_versions.go b/internal/server/mirror_versions.go index 404388d..09ab3b7 100644 --- a/internal/server/mirror_versions.go +++ b/internal/server/mirror_versions.go @@ -80,6 +80,8 @@ func mirrorVersionsURL(u config.UpstreamConfig, purlType, name string) (string, return "", nil case "julia": return publicOnly("upstream.julia", u.Julia, d.Julia) + case "swift": + return upstreamOr(u.Swift, d.Swift), nil default: return "", fmt.Errorf("listing %s versions isn't supported; give a version", purlType) } diff --git a/internal/server/mirror_versions_test.go b/internal/server/mirror_versions_test.go index 898c212..143fac8 100644 --- a/internal/server/mirror_versions_test.go +++ b/internal/server/mirror_versions_test.go @@ -99,6 +99,7 @@ func TestMirrorVersionsURL(t *testing.T) { custom.Cargo = "https://index.example.com" custom.Conda = "https://conda.example.com" custom.Julia = "https://julia.example.com" + custom.Swift = "https://swift.example.com/registry/" tests := []struct { name string @@ -129,7 +130,7 @@ func TestMirrorVersionsURL(t *testing.T) { {"julia default", config.Default().Upstream, "julia", "JSON", "", ""}, {"julia server", custom, "julia", "JSON", "", "upstream.julia"}, {"helm", custom, "helm", "nginx", "", "give a version"}, - {"swift", custom, "swift", "apple/swift-nio", "", "give a version"}, + {"swift", custom, "swift", "apple.swift-nio", "https://swift.example.com/registry", ""}, {"apk", custom, "apk", "curl", "", "give a version"}, } for _, tt := range tests { From 8ed0edb90a2b9194d0a0614ffdfb0b8eabff0076 Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Sun, 25 Oct 2026 22:19:24 +0100 Subject: [PATCH 449/633] Extract portable npm metadata transformations npm metadata rewriting depended on handler.Proxy, which pulls SQL drivers, cloud SDKs and metrics into its import graph. Move the transformation into internal/metadata/npm with explicit policy, evaluation time, public URL and hosted-version inputs, and share the JSON scanner between npm and Composer through internal/jsonscan. Rebased onto main: rewritten tarball URLs are still logged once per document, now through an OnTarballsRewritten callback that reports the count. --- internal/handler/composer.go | 61 ++- internal/handler/denylist_test.go | 6 +- internal/handler/npm.go | 511 ++---------------- internal/handler/npm_hosted.go | 27 +- internal/handler/npm_portable_test.go | 107 ++++ internal/handler/npm_rewrite_bench_test.go | 2 +- internal/handler/npm_rewrite_test.go | 155 ------ internal/handler/npm_test.go | 10 +- internal/{handler => jsonscan}/jsonscan.go | 149 +++-- .../{handler => jsonscan}/jsonscan_test.go | 38 +- .../npm/denylist.go} | 9 +- internal/metadata/npm/metadata.go | 471 ++++++++++++++++ internal/metadata/npm/metadata_test.go | 153 ++++++ internal/metadata/npm/policy_test.go | 161 ++++++ internal/metadata/npm/testdata/packument.json | 12 + 15 files changed, 1120 insertions(+), 752 deletions(-) create mode 100644 internal/handler/npm_portable_test.go rename internal/{handler => jsonscan}/jsonscan.go (55%) rename internal/{handler => jsonscan}/jsonscan_test.go (71%) rename internal/{handler/npm_denylist.go => metadata/npm/denylist.go} (82%) create mode 100644 internal/metadata/npm/metadata.go create mode 100644 internal/metadata/npm/metadata_test.go create mode 100644 internal/metadata/npm/policy_test.go create mode 100644 internal/metadata/npm/testdata/packument.json diff --git a/internal/handler/composer.go b/internal/handler/composer.go index 7ced6d6..116d106 100644 --- a/internal/handler/composer.go +++ b/internal/handler/composer.go @@ -13,6 +13,7 @@ import ( "strings" "time" + "github.com/git-pkgs/proxy/internal/jsonscan" "github.com/git-pkgs/proxy/internal/retention" "github.com/git-pkgs/purl" ) @@ -219,7 +220,7 @@ func (h *ComposerHandler) rewriteMetadataWithLocal(body []byte, localPackage str if !json.Valid(body) { return nil, nil, errors.New("composer metadata is not valid JSON") } - format, _, err := lookupJSONString(body, "minified") + format, _, err := jsonscan.LookupString(body, "minified") if err != nil { return nil, nil, err } @@ -229,17 +230,17 @@ func (h *ComposerHandler) rewriteMetadataWithLocal(body []byte, localPackage str var kept []string listed := false out.Grow(len(body) + len(body)/8) - err = rewriteJSONMembers(&out, body, func(out *bytes.Buffer, m jsonMember) (bool, error) { - packages := m.value(body) - if !jsonKeyIs(m.key(body), "packages") || packages[0] != '{' { + err = jsonscan.RewriteMembers(&out, body, func(out *bytes.Buffer, m jsonscan.Member) (bool, error) { + packages := m.Value(body) + if !jsonscan.KeyIs(m.Key(body), "packages") || packages[0] != '{' { return false, nil } - return true, rewriteJSONMembers(out, packages, func(out *bytes.Buffer, p jsonMember) (bool, error) { - versions := p.value(packages) + return true, jsonscan.RewriteMembers(out, packages, func(out *bytes.Buffer, p jsonscan.Member) (bool, error) { + versions := p.Value(packages) if versions[0] != '[' { return false, nil } - packageName, err := jsonKey(p.key(packages)) + packageName, err := jsonscan.Key(p.Key(packages)) if err != nil { return false, err } @@ -279,8 +280,8 @@ func (h *ComposerHandler) writeVersions(out *bytes.Buffer, packageName string, v _, routed := h.routes.match(packageName) out.WriteByte('[') - err := forEachJSONElement(versions, func(entry []byte) error { - if minified && jsonStringIs(entry, composerDevReset) { + err := jsonscan.EachElement(versions, func(entry []byte) error { + if minified && jsonscan.StringIs(entry, composerDevReset) { upstream.reset() written.reset() devReset = true @@ -427,15 +428,15 @@ func disableDefaultNotification(fields *composerFields) { // rewriteDist returns a version's dist object with its url pointing at this // proxy. A dist without a string url is returned unchanged. func (h *ComposerHandler) rewriteDist(packageName, version string, dist []byte) []byte { - urlMember, ok, err := findJSONMember(dist, "url") - if err != nil || !ok || dist[urlMember.valStart] != '"' { + urlMember, ok, err := jsonscan.FindMember(dist, "url") + if err != nil || !ok || dist[urlMember.ValStart] != '"' { return dist } var upstreamURL string - if err := json.Unmarshal(urlMember.value(dist), &upstreamURL); err != nil { + if err := json.Unmarshal(urlMember.Value(dist), &upstreamURL); err != nil { return dist } - distType, _, _ := lookupJSONString(dist, "type") + distType, _, _ := jsonscan.LookupString(dist, "type") proxyURL, ok := h.proxyDistURL(packageName, version, upstreamURL, distType) if !ok { @@ -445,10 +446,10 @@ func (h *ComposerHandler) rewriteDist(packageName, version string, dist []byte) if err != nil { return dist } - rewritten := make([]byte, 0, len(dist)-len(urlMember.value(dist))+len(encoded)) - rewritten = append(rewritten, dist[:urlMember.valStart]...) + rewritten := make([]byte, 0, len(dist)-len(urlMember.Value(dist))+len(encoded)) + rewritten = append(rewritten, dist[:urlMember.ValStart]...) rewritten = append(rewritten, encoded...) - return append(rewritten, dist[urlMember.valEnd:]...) + return append(rewritten, dist[urlMember.ValEnd:]...) } // proxyDistURL returns the URL this proxy serves a version's archive at, or @@ -592,9 +593,9 @@ func (f *composerFields) set(name string, key, value []byte) { // apply overlays the members of a version entry. In minified metadata the // value "__unset" removes the field instead. func (f *composerFields) apply(entry []byte, minified bool) error { - return forEachJSONMember(entry, func(m jsonMember) error { - key, value := m.key(entry), m.value(entry) - if minified && jsonStringIs(value, composerUnset) { + return jsonscan.EachMember(entry, func(m jsonscan.Member) error { + key, value := m.Key(entry), m.Value(entry) + if minified && jsonscan.StringIs(value, composerUnset) { value = nil } if bytes.IndexByte(key, '\\') < 0 { @@ -603,7 +604,7 @@ func (f *composerFields) apply(entry []byte, minified bool) error { return nil } } - name, err := jsonKey(key) + name, err := jsonscan.Key(key) if err != nil { return err } @@ -950,11 +951,11 @@ func (h *ComposerHandler) findDownloadURLFromMetadata(ctx context.Context, metaU // the minified format as it needs, since it runs on each archive the proxy // has not cached yet. func composerDistURL(body []byte, packageName, version string) (composerDist, error) { - format, _, err := lookupJSONString(body, "minified") + format, _, err := jsonscan.LookupString(body, "minified") if err != nil { return composerDist{}, err } - versions, err := lookupJSON(body, "packages", packageName) + versions, err := jsonscan.Lookup(body, "packages", packageName) if err != nil || len(versions) == 0 || versions[0] != '[' { return composerDist{}, err } @@ -962,8 +963,8 @@ func composerDistURL(body []byte, packageName, version string) (composerDist, er fields := newComposerFields() var dist composerDist - err = forEachJSONElement(versions, func(entry []byte) error { - if minified && jsonStringIs(entry, composerDevReset) { + err = jsonscan.EachElement(versions, func(entry []byte) error { + if minified && jsonscan.StringIs(entry, composerDevReset) { fields.reset() return nil } @@ -976,20 +977,20 @@ func composerDistURL(body []byte, packageName, version string) (composerDist, er if err := fields.apply(entry, minified); err != nil { return err } - if !jsonStringIs(fields.get("version"), version) { + if !jsonscan.StringIs(fields.get("version"), version) { return nil } distRaw := fields.get("dist") - dist.url, _, _ = lookupJSONString(distRaw, "url") + dist.url, _, _ = jsonscan.LookupString(distRaw, "url") if dist.url != "" { - dist.distType, _, _ = lookupJSONString(distRaw, "type") - dist.shasum, _, _ = lookupJSONString(distRaw, "shasum") + dist.distType, _, _ = jsonscan.LookupString(distRaw, "type") + dist.shasum, _, _ = jsonscan.LookupString(distRaw, "shasum") dist.published = fields.get("time") - return errStopScan + return jsonscan.ErrStop } return nil }) - if errors.Is(err, errStopScan) { + if errors.Is(err, jsonscan.ErrStop) { err = nil } return dist, err diff --git a/internal/handler/denylist_test.go b/internal/handler/denylist_test.go index e89f186..bb41e38 100644 --- a/internal/handler/denylist_test.go +++ b/internal/handler/denylist_test.go @@ -80,7 +80,7 @@ func TestNPMDenylistMetadata(t *testing.T) { h := NewNPMHandler(p, "https://proxy.example", "") for _, times := range []string{"", `,"time":{"1.0.0":"2020-01-01T00:00:00Z","2.0.0":"2021-01-01T00:00:00Z"}`} { body := `{"versions":{"1.0.0":{},"2.0.0":{},"3.0.0-beta.1":{}},"dist-tags":{"latest":"2.0.0","beta":"2.0.0","next":"3.0.0-beta.1"}` + times + `}` - out, err := h.rewriteMetadata("@scope/demo", []byte(body)) + out, _, err := h.rewriteMetadataKeeping("@scope/demo", []byte(body)) if err != nil { t.Fatal(err) } @@ -88,7 +88,7 @@ func TestNPMDenylistMetadata(t *testing.T) { t.Fatalf("incorrect metadata: %s", out) } } - out, err := h.rewriteMetadata("@scope/demo", []byte(`{"versions":{"2.0.0":{}},"dist-tags":{"latest":"2.0.0"}}`)) + out, _, err := h.rewriteMetadataKeeping("@scope/demo", []byte(`{"versions":{"2.0.0":{}},"dist-tags":{"latest":"2.0.0"}}`)) if err != nil || strings.Contains(string(out), "2.0.0") { t.Fatalf("all denied: %s, %v", out, err) } @@ -208,7 +208,7 @@ func TestDenylistAndCooldownCompose(t *testing.T) { setTestDenylist(t, p, "pkg:npm/demo@1.0.0") h := NewNPMHandler(p, "https://proxy.test", "") body := fmt.Sprintf(`{"versions":{"1.0.0":{},"2.0.0":{},"3.0.0":{}},"time":{"1.0.0":"2020-01-01T00:00:00Z","2.0.0":"2021-01-01T00:00:00Z","3.0.0":%q},"dist-tags":{"latest":"3.0.0"}}`, time.Now().UTC().Format(time.RFC3339)) - out, err := h.rewriteMetadata("demo", []byte(body)) + out, _, err := h.rewriteMetadataKeeping("demo", []byte(body)) if err != nil || strings.Contains(string(out), "1.0.0") || strings.Contains(string(out), "3.0.0") || !strings.Contains(string(out), `"latest":"2.0.0"`) { t.Fatalf("combined policy: %s, %v", out, err) } diff --git a/internal/handler/npm.go b/internal/handler/npm.go index b749a40..4ced2f2 100644 --- a/internal/handler/npm.go +++ b/internal/handler/npm.go @@ -2,21 +2,18 @@ package handler import ( "bytes" - "encoding/json" "errors" "fmt" "io" - "maps" "net/http" "net/url" - "reflect" - "slices" - "sort" "strings" "sync" "time" "github.com/git-pkgs/integrity" + "github.com/git-pkgs/proxy/internal/jsonscan" + "github.com/git-pkgs/proxy/internal/metadata/npm" "github.com/git-pkgs/proxy/internal/retention" ) @@ -339,11 +336,11 @@ func (h *NPMHandler) packumentValidator() func([]byte) error { // leaving it to the usual handling. func requireFullPackument(body []byte) error { var hasTime, hasModified bool - err := forEachJSONMember(body, func(m jsonMember) error { - switch key := m.key(body); { - case jsonKeyIs(key, "time"): + err := jsonscan.EachMember(body, func(m jsonscan.Member) error { + switch key := m.Key(body); { + case jsonscan.KeyIs(key, "time"): hasTime = true - case jsonKeyIs(key, "modified"): + case jsonscan.KeyIs(key, "modified"): hasModified = true } return nil @@ -436,52 +433,13 @@ func (h *NPMHandler) npmVersionMetadataUsesFilteredPackument(packageName string) // rewriteNPMVersionFromPackument picks one version entry out of an already // filtered packument and rewrites its tarball URL. func (h *NPMHandler) rewriteNPMVersionFromPackument(packageName, specifier string, packument []byte) ([]byte, error) { - filtered, err := h.rewriteMetadata(packageName, packument) - if err != nil { - return nil, err - } - - target := specifier - if resolved, ok, err := lookupJSONString(filtered, "dist-tags", specifier); err != nil { - return nil, err - } else if ok { - target = resolved - } - - if h.proxy.versionDenied("npm", packageName, target) { - return nil, ErrVersionDenied - } - - versionRaw, err := lookupJSON(filtered, "versions", target) - if err != nil { - return nil, err - } - if len(versionRaw) == 0 { - return nil, ErrVersionNotFound - } - - var out bytes.Buffer - h.writeNPMVersion(&out, packageName, target, versionRaw) - return out.Bytes(), nil + out, err := npm.Version(packageName, specifier, packument, h.npmRewriteOptions(packageName, nil)) + return out, npmRewriteError(err) } -// rewriteMetadata rewrites tarball URLs in npm package metadata to point at this proxy. -// If cooldown is enabled, versions published too recently are filtered out. -// -// The document is never decoded as a whole. A full packument can run to tens -// of megabytes, and decoding it into generic maps took many times that in -// memory for every rewrite. Instead the version names, time map and dist-tags -// are decoded to decide what to keep, and the response is assembled from -// slices of the original bytes: only each version's tarball URL, and the time -// map and dist-tags when filtering changed them, are written fresh. Everything -// else reaches the client byte for byte as upstream sent it. -func (h *NPMHandler) rewriteMetadata(packageName string, body []byte) ([]byte, error) { - out, _, err := h.rewriteMetadataKeeping(packageName, body) - return out, err -} - -// rewriteMetadataKeeping is rewriteMetadata that also reports the versions it -// kept. +// rewriteMetadataKeeping rewrites tarball URLs in npm package metadata to +// point at this proxy, filters the versions cooldown and the denylist +// withhold, and reports the versions it kept. See npm.Rewrite. func (h *NPMHandler) rewriteMetadataKeeping(packageName string, body []byte) ([]byte, []string, error) { return h.rewriteMetadataWithLocal(packageName, body, nil) } @@ -491,394 +449,49 @@ func (h *NPMHandler) rewriteMetadataKeeping(packageName string, body []byte) ([] // version. Cooldown and the denylist only filter upstream versions, and // dist-tags stay as upstream has them. func (h *NPMHandler) rewriteMetadataWithLocal(packageName string, body []byte, local []npmLocalEntry) ([]byte, []string, error) { - doc, err := indexNPMPackument(body) - if err != nil { - return nil, nil, err - } + out, kept, err := npm.Rewrite(packageName, body, h.npmRewriteOptions(packageName, local)) + return out, kept, npmRewriteError(err) +} - versionsRaw := doc.value(doc.versionsAt) - if len(versionsRaw) == 0 || versionsRaw[0] != '{' { - version, ok, err := lookupJSONString(body, "version") - if err != nil { - return nil, nil, err - } - if ok { - if h.proxy.versionDenied("npm", packageName, version) { - return nil, nil, ErrVersionDenied - } - var out bytes.Buffer - if h.writeNPMVersion(&out, packageName, version, body) { - h.logTarballRewrites(packageName, 1) - } - return out.Bytes(), []string{version}, nil - } - if len(h.proxy.Denylist.Versions(canonicalPackagePURL("npm", packageName))) != 0 { - return nil, nil, errors.New("npm metadata has no versions object") - } - return body, nil, nil // No versions to rewrite +// npmRewriteOptions gathers what a rewrite of packageName's metadata depends +// on from the proxy: its URL, the cooldown and denylist policy, the time +// cooldown is evaluated at, and the pushed versions to list. The callbacks +// log what the rewrite did. +func (h *NPMHandler) npmRewriteOptions(packageName string, local []npmLocalEntry) npm.Options { + return npm.Options{ + ProxyURL: h.proxyURL, + Cooldown: h.proxy.Cooldown, + DeniedVersions: h.proxy.Denylist.Versions(canonicalPackagePURL("npm", packageName)), + EvaluatedAt: time.Now(), + LocalVersions: local, + // Logged once per document: a line per version flooded debug logs, + // since packages such as eslint list hundreds of versions and with + // cooldown on every request rewrites the document again. + OnTarballsRewritten: func(count int) { + h.proxy.Logger.Debug("rewrote tarball URLs", "package", packageName, "count", count) + }, + OnCooldownFilter: func(version, publishedAt string) { + h.proxy.Logger.Info("cooldown: filtering npm version", "package", packageName, "version", version, + "published", publishedAt) + }, } +} - // The filters only look at which versions exist, so they run on a map of - // version names alongside the decoded time map and dist-tags. - versions := map[string]any{} - if err := forEachJSONMember(versionsRaw, func(m jsonMember) error { - version, err := jsonKey(m.key(versionsRaw)) - versions[version] = nil +// npmRewriteError maps the rewrite's errors to the handler's, which the +// routes answer with 403 and 404. +func npmRewriteError(err error) error { + switch { + case errors.Is(err, npm.ErrVersionDenied): + return ErrVersionDenied + case errors.Is(err, npm.ErrVersionNotFound): + return ErrVersionNotFound + default: return err - }); err != nil { - return nil, nil, err - } - metadata := map[string]any{} - timeMap := doc.decodeObject(doc.timeAt, metadata, "time") - distTags := doc.decodeObject(doc.tagsAt, metadata, "dist-tags") - timeEntries := len(timeMap) - originalTags := maps.Clone(distTags) - - h.applyCooldownFiltering(metadata, versions, packageName) - h.applyDenylistFiltering(metadata, versions, packageName) - for _, e := range local { - delete(versions, e.version) - } - - // Untouched members are copied as they are; the three the filters - // handled are written from their filtered state, and only when it changed. - out, err := doc.write(func(out *bytes.Buffer, i int) error { - switch { - case i == doc.versionsAt: - return h.writeNPMVersions(out, packageName, versionsRaw, versions, local) - case i == doc.timeAt && timeMap != nil && (len(timeMap) != timeEntries || len(local) > 0): - pushedAt := make([][2]string, 0, len(local)) - for _, e := range local { - pushedAt = append(pushedAt, [2]string{e.version, e.time}) - } - return writeFilteredJSONObjectWith(out, doc.value(i), func(key string) bool { - _, ok := timeMap[key] - _, pushed := findNPMLocalEntry(local, key) - return ok && !pushed - }, pushedAt) - case i == doc.tagsAt && distTags != nil && !reflect.DeepEqual(distTags, originalTags): - encoded, err := json.Marshal(distTags) - out.Write(encoded) - return err - default: - out.Write(doc.value(i)) - return nil - } - }) - kept := slices.Collect(maps.Keys(versions)) - for _, e := range local { - kept = append(kept, e.version) - } - return out, kept, err -} - -// npmPackument is a packument's top-level members as offsets into its bytes, -// with the positions of the members rewriteMetadata filters, or -1 for those -// it lacks. A key that repeats is recorded at its last occurrence, the one -// JSON.parse keeps. -type npmPackument struct { - body []byte - members []jsonMember - versionsAt, timeAt, tagsAt int -} - -func indexNPMPackument(body []byte) (*npmPackument, error) { - if !json.Valid(body) { - return nil, errors.New("npm metadata is not valid JSON") - } - doc := &npmPackument{body: body, versionsAt: -1, timeAt: -1, tagsAt: -1} - err := forEachJSONMember(body, func(m jsonMember) error { - switch key := m.key(body); { - case jsonKeyIs(key, "versions"): - doc.versionsAt = len(doc.members) - case jsonKeyIs(key, "time"): - doc.timeAt = len(doc.members) - case jsonKeyIs(key, "dist-tags"): - doc.tagsAt = len(doc.members) - } - doc.members = append(doc.members, m) - return nil - }) - return doc, err -} - -// value returns the raw value of member i, or nil when i is -1. -func (d *npmPackument) value(i int) []byte { - if i < 0 { - return nil - } - return d.members[i].value(d.body) -} - -// decodeObject decodes member i into metadata[name] and returns it when it is -// an object. The filters edit the returned map in place. -func (d *npmPackument) decodeObject(i int, metadata map[string]any, name string) map[string]any { - raw := d.value(i) - if raw == nil { - return nil - } - var value any - if err := json.Unmarshal(raw, &value); err != nil { - return nil - } - metadata[name] = value - object, _ := value.(map[string]any) - return object -} - -// write assembles the document in member order, with writeValue writing each -// member's value. Earlier duplicates of the filtered keys are dropped: they -// would carry unfiltered data, and JSON.parse ignores them anyway. -func (d *npmPackument) write(writeValue func(out *bytes.Buffer, i int) error) ([]byte, error) { - var out bytes.Buffer - out.Grow(len(d.body) + len(d.body)/8) - out.WriteByte('{') - first := true - for i, m := range d.members { - if i != d.versionsAt && i != d.timeAt && i != d.tagsAt { - key := m.key(d.body) - if jsonKeyIs(key, "versions") || jsonKeyIs(key, "time") || jsonKeyIs(key, "dist-tags") { - continue - } - } - if !first { - out.WriteByte(',') - } - first = false - out.Write(m.key(d.body)) - out.WriteByte(':') - if err := writeValue(&out, i); err != nil { - return nil, err - } - } - out.WriteByte('}') - return out.Bytes(), nil -} - -// writeNPMVersions writes the versions object, keeping the versions still in -// keep and pointing each kept version's tarball at this proxy, followed by the -// pushed versions in local. -func (h *NPMHandler) writeNPMVersions(out *bytes.Buffer, packageName string, versionsRaw []byte, keep map[string]any, local []npmLocalEntry) error { - out.WriteByte('{') - first := true - rewritten := 0 - err := forEachJSONMember(versionsRaw, func(m jsonMember) error { - version, err := jsonKey(m.key(versionsRaw)) - if err != nil { - return err - } - if _, ok := keep[version]; !ok { - return nil - } - if !first { - out.WriteByte(',') - } - first = false - out.Write(m.key(versionsRaw)) - out.WriteByte(':') - if h.writeNPMVersion(out, packageName, version, m.value(versionsRaw)) { - rewritten++ - } - return nil - }) - h.logTarballRewrites(packageName, rewritten) - for _, e := range local { - if !first { - out.WriteByte(',') - } - first = false - key, _ := json.Marshal(e.version) - out.Write(key) - out.WriteByte(':') - out.Write(e.entry) - } - out.WriteByte('}') - return err -} - -// writeNPMVersion writes one version entry with its dist.tarball pointing at -// this proxy, and reports whether it rewrote the tarball. An entry without a -// string tarball is written unchanged. -func (h *NPMHandler) writeNPMVersion(out *bytes.Buffer, packageName, version string, raw []byte) bool { - dist, ok, err := findJSONMember(raw, "dist") - if err != nil || !ok { - out.Write(raw) - return false - } - distRaw := dist.value(raw) - tarball, ok, err := findJSONMember(distRaw, "tarball") - if err != nil || !ok || distRaw[tarball.valStart] != '"' { - out.Write(raw) - return false - } - var oldTarball string - if err := json.Unmarshal(tarball.value(distRaw), &oldTarball); err != nil { - out.Write(raw) - return false - } - - newTarball := h.proxyTarballURL(packageName, version, oldTarball) - encoded, err := json.Marshal(newTarball) - if err != nil { - out.Write(raw) - return false - } - out.Write(raw[:dist.valStart+tarball.valStart]) - out.Write(encoded) - out.Write(raw[dist.valStart+tarball.valEnd:]) - return true -} - -// logTarballRewrites logs once per document how many tarball URLs it -// rewrote. A line per version flooded debug logs: packages such as eslint -// list hundreds of versions, and with cooldown on every request rewrites -// the document again. -func (h *NPMHandler) logTarballRewrites(packageName string, count int) { - if count > 0 { - h.proxy.Logger.Debug("rewrote tarball URLs", "package", packageName, "count", count) } } -// writeFilteredJSONObjectWith keeps members whose decoded keys match keep, -// then appends extra string members. -func writeFilteredJSONObjectWith(out *bytes.Buffer, obj []byte, keep func(string) bool, extra [][2]string) error { - out.WriteByte('{') - first := true - err := forEachJSONMember(obj, func(m jsonMember) error { - key, err := jsonKey(m.key(obj)) - if err != nil { - return err - } - if !keep(key) { - return nil - } - if !first { - out.WriteByte(',') - } - first = false - out.Write(obj[m.keyStart:m.valEnd]) - return nil - }) - for _, member := range extra { - if !first { - out.WriteByte(',') - } - first = false - key, _ := json.Marshal(member[0]) - value, _ := json.Marshal(member[1]) - out.Write(key) - out.WriteByte(':') - out.Write(value) - } - out.WriteByte('}') - return err -} - -// applyCooldownFiltering removes versions that are too recently published, -// and updates dist-tags.latest if the current latest was filtered out. -func (h *NPMHandler) applyCooldownFiltering(metadata map[string]any, versions map[string]any, packageName string) { - if h.proxy.Cooldown == nil || !h.proxy.Cooldown.Enabled() { - return - } - - timeMap, _ := metadata["time"].(map[string]any) - if timeMap == nil { - return - } - - packagePURL := canonicalPackagePURL("npm", packageName) - - for version := range versions { - publishedStr, ok := timeMap[version].(string) - if !ok { - continue - } - publishedAt, err := time.Parse(time.RFC3339, publishedStr) - if err != nil { - continue - } - if !h.proxy.Cooldown.IsAllowed("npm", packagePURL, publishedAt) { - h.proxy.Logger.Info("cooldown: filtering npm version", - "package", packageName, "version", version, - "published", publishedStr) - delete(versions, version) - delete(timeMap, version) - } - } - - h.updateDistTagsLatest(metadata, versions, timeMap) -} - -// updateDistTagsLatest updates the dist-tags.latest field if the current latest -// version was removed by cooldown filtering. -func (h *NPMHandler) updateDistTagsLatest(metadata, versions, timeMap map[string]any) { - distTags, ok := metadata["dist-tags"].(map[string]any) - if !ok { - return - } - - latest, ok := distTags["latest"].(string) - if !ok { - return - } - - if _, exists := versions[latest]; exists { - return - } - - if newLatest := h.findNewestVersion(versions, timeMap); newLatest != "" { - distTags["latest"] = newLatest - } -} - -// proxyTarballURL returns the proxy URL that serves the tarball upstream -// lists at tarball for this version. func (h *NPMHandler) proxyTarballURL(packageName, version, tarball string) string { - filename := tarball - if idx := strings.LastIndex(tarball, "/"); idx >= 0 { - filename = tarball[idx+1:] - } - if h.extractVersionFromFilename(packageName, filename) != version { - _, shortName, scoped := strings.Cut(packageName, "/") - if !scoped { - shortName = packageName - } - filename = shortName + "-" + version + ".tgz" - } - - return fmt.Sprintf("%s/npm/%s/-/%s", h.proxyURL, url.PathEscape(packageName), filename) -} - -// findNewestVersion returns the version string with the most recent timestamp -// from the remaining versions, using the time map. -func (h *NPMHandler) findNewestVersion(versions map[string]any, timeMap map[string]any) string { - if timeMap == nil { - return "" - } - - type versionTime struct { - version string - t time.Time - } - - var vts []versionTime - for v := range versions { - if ts, ok := timeMap[v].(string); ok { - if t, err := time.Parse(time.RFC3339, ts); err == nil { - vts = append(vts, versionTime{v, t}) - } - } - } - - if len(vts) == 0 { - return "" - } - - sort.Slice(vts, func(i, j int) bool { - return vts[i].t.After(vts[j].t) - }) - - return vts[0].version + return npm.TarballURL(h.proxyURL, packageName, version, tarball) } // handleDownload serves a package tarball, fetching and caching from upstream if needed. @@ -995,13 +608,13 @@ func (h *NPMHandler) getTarball(r *http.Request, registryURL, packageName, versi // returns "" when neither is usable, and the tarball is fetched unverified as // before. func npmVersionIntegrity(body []byte, version string) string { - sri, _, _ := lookupJSONString(body, "versions", version, "dist", "integrity") + sri, _, _ := jsonscan.LookupString(body, "versions", version, "dist", "integrity") if sri != "" { if _, err := integrity.ParseSRI(sri); err == nil { return sri } } - shasum, _, _ := lookupJSONString(body, "versions", version, "dist", "shasum") + shasum, _, _ := jsonscan.LookupString(body, "versions", version, "dist", "shasum") return sha1Checksum(shasum) } @@ -1009,7 +622,7 @@ func npmVersionIntegrity(body []byte, version string) string { // or "" if it has none. It reads that one value in place rather than decoding // every version, since it runs on each tarball the proxy has not cached yet. func npmVersionTarball(body []byte, version string) string { - tarball, _, _ := lookupJSONString(body, "versions", version, "dist", "tarball") + tarball, _, _ := jsonscan.LookupString(body, "versions", version, "dist", "tarball") return tarball } @@ -1077,7 +690,7 @@ func (h *NPMHandler) versionInCooldown(packageName, version string, metadata fun return false, err } - published, ok, err := lookupJSONString(body, "time", version) + published, ok, err := jsonscan.LookupString(body, "time", version) if err != nil { return false, fmt.Errorf("parsing npm metadata: %w", err) } @@ -1173,28 +786,6 @@ func (h *NPMHandler) parseDownloadPath(path string) (packageName, filename strin return packageName, filename } -// extractVersionFromFilename extracts version from npm tarball filename. -// e.g., "lodash-4.17.21.tgz" -> "4.17.21" -// e.g., "core-7.23.0.tgz" for @babel/core -> "7.23.0" func (h *NPMHandler) extractVersionFromFilename(packageName, filename string) string { - // Remove .tgz extension - if !strings.HasSuffix(filename, ".tgz") { - return "" - } - base := strings.TrimSuffix(filename, ".tgz") - - // For scoped packages, the filename uses the short name - shortName := packageName - if strings.Contains(packageName, "/") { - parts := strings.SplitN(packageName, "/", scopedParts) - shortName = parts[1] - } - - // Expected format: {shortName}-{version} - prefix := shortName + "-" - if !strings.HasPrefix(base, prefix) { - return "" - } - - return strings.TrimPrefix(base, prefix) + return npm.VersionFromFilename(packageName, filename) } diff --git a/internal/handler/npm_hosted.go b/internal/handler/npm_hosted.go index 4c786e8..77f9227 100644 --- a/internal/handler/npm_hosted.go +++ b/internal/handler/npm_hosted.go @@ -8,15 +8,10 @@ import ( "time" "github.com/Masterminds/semver/v3" + "github.com/git-pkgs/proxy/internal/metadata/npm" ) -// npmLocalEntry is a pushed version as a packument lists it: the version -// entry built from its package.json, and the time it was pushed. -type npmLocalEntry struct { - version string - entry json.RawMessage - time string -} +type npmLocalEntry = npm.LocalVersion // npmLocalEntries returns the pushed versions of a package as packument // entries, or nil when there are none or hosted packages are disabled. @@ -42,14 +37,14 @@ func (h *NPMHandler) npmLocalEntries(packageName string) ([]npmLocalEntry, error if err != nil { return nil, err } - entries = append(entries, npmLocalEntry{version: lv.Version, entry: entry, time: lv.PushedAt.Format(time.RFC3339)}) + entries = append(entries, npmLocalEntry{Version: lv.Version, Entry: entry, PublishedAt: lv.PushedAt.Format(time.RFC3339)}) } return entries, nil } func findNPMLocalEntry(local []npmLocalEntry, version string) (npmLocalEntry, bool) { for _, e := range local { - if e.version == version { + if e.Version == version { return e, true } } @@ -81,7 +76,7 @@ func (h *NPMHandler) serveLocalMetadata(w http.ResponseWriter, r *http.Request, case req.versionLookup && pushed: w.Header().Set(headerContentType, contentTypeJSON) w.WriteHeader(http.StatusOK) - _, _ = w.Write(e.entry) + _, _ = w.Write(e.Entry) case h.proxy.hostedOnly("npm", req.packageName): h.serveHostedOnlyMetadata(w, r, req.packageName, req.specifier, req.versionLookup, req.metadataCacheKey, local) case len(local) > 0 && !req.versionLookup: @@ -145,7 +140,7 @@ func (h *NPMHandler) serveHostedOnlyMetadata(w http.ResponseWriter, r *http.Requ } w.Header().Set(headerContentType, contentTypeJSON) w.WriteHeader(http.StatusOK) - _, _ = w.Write(e.entry) + _, _ = w.Write(e.Entry) return } out, err := localNPMPackument(packageName, local) @@ -162,8 +157,8 @@ func localNPMPackument(packageName string, local []npmLocalEntry) ([]byte, error versions := make(map[string]json.RawMessage, len(local)) times := make(map[string]string, len(local)) for _, e := range local { - versions[e.version] = e.entry - times[e.version] = e.time + versions[e.Version] = e.Entry + times[e.Version] = e.PublishedAt } return json.Marshal(map[string]any{ "name": packageName, @@ -180,10 +175,10 @@ func latestNPMVersion(local []npmLocalEntry) string { var latestSemver *semver.Version semverOnly := true for _, e := range local { - if e.time > latestTime { - latest, latestTime = e.version, e.time + if e.PublishedAt > latestTime { + latest, latestTime = e.Version, e.PublishedAt } - v, err := semver.NewVersion(e.version) + v, err := semver.NewVersion(e.Version) if err != nil { semverOnly = false continue diff --git a/internal/handler/npm_portable_test.go b/internal/handler/npm_portable_test.go new file mode 100644 index 0000000..d0bb193 --- /dev/null +++ b/internal/handler/npm_portable_test.go @@ -0,0 +1,107 @@ +package handler + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "log/slog" + "net/http" + "os" + "strings" + "testing" + + "github.com/git-pkgs/cooldown" +) + +func TestNPMMetadataLogsPolicyAndURLChanges(t *testing.T) { + body, err := os.ReadFile("../metadata/npm/testdata/packument.json") + if err != nil { + t.Fatal(err) + } + f := newNPMHostedFixture(t, map[string]string{"/@example%2Fwidgets": string(body)}) + f.proxy.Cooldown = &cooldown.Config{Default: "7d"} + setTestDenylist(t, f.proxy, "pkg:npm/@example/widgets@2.0.0") + var logs bytes.Buffer + f.proxy.Logger = slog.New(slog.NewJSONHandler(&logs, &slog.HandlerOptions{Level: slog.LevelDebug})) + if rec := f.get(t, "/@example/widgets"); rec.Code != http.StatusOK { + t.Fatalf("HTTP %d: %s", rec.Code, rec.Body.String()) + } + want := map[string]map[string]string{ + "cooldown: filtering npm version": {"level": "INFO", "package": "@example/widgets", "version": "3.0.0", "published": "2099-01-01T00:00:00Z"}, + "rewrote tarball URLs": {"level": "DEBUG", "package": "@example/widgets", "count": "1"}, + } + decoder := json.NewDecoder(&logs) + for { + var record map[string]any + if err := decoder.Decode(&record); err == io.EOF { + break + } else if err != nil { + t.Fatal(err) + } + message, _ := record["msg"].(string) + fields, ok := want[message] + if !ok { + continue + } + for key, value := range fields { + if fmt.Sprint(record[key]) != value { + t.Errorf("%s: %s = %v, want %q", message, key, record[key], value) + } + } + delete(want, message) + } + if len(want) != 0 { + t.Errorf("missing log records: %v", want) + } +} + +func TestNPMRoutesWithPortableMetadata(t *testing.T) { + body, err := os.ReadFile("../metadata/npm/testdata/packument.json") + if err != nil { + t.Fatal(err) + } + for _, hosted := range []bool{false, true} { + name := "upstream" + if hosted { + name = "hosted override" + } + t.Run(name, func(t *testing.T) { + f := newNPMHostedFixture(t, map[string]string{"/@example%2Fwidgets": string(body)}) + f.proxy.Cooldown = &cooldown.Config{Default: "7d"} + setTestDenylist(t, f.proxy, "pkg:npm/@example/widgets@2.0.0") + if hosted { + f.push(t, "@example/widgets", "3.0.0", `{"name":"@example/widgets","version":"3.0.0","dependencies":{"local":"1.0.0"}}`) + } + rec := f.get(t, "/@example/widgets") + doc := decodePackument(t, rec) + count := 1 + if hosted { + count = 2 + } + if len(doc.Versions) != count || doc.DistTags["latest"] != "1.0.0" || len(doc.DistTags) != 1 { + t.Fatalf("filtered metadata: %s", rec.Body.String()) + } + if doc.Versions["1.0.0"].Dist.Tarball != "http://proxy.local/npm/@example%2Fwidgets/-/widgets-1.0.0.tgz" || !strings.Contains(rec.Body.String(), "12345678901234567890") { + t.Fatalf("rewritten metadata: %s", rec.Body.String()) + } + if hosted && doc.Versions["3.0.0"].Dependencies["local"] != "1.0.0" { + t.Fatalf("hosted override: %s", rec.Body.String()) + } + checkNPMVersionResponses(t, f, hosted) + }) + } +} + +func checkNPMVersionResponses(t *testing.T, f npmHostedFixture, hosted bool) { + t.Helper() + for specifier, want := range map[string]int{"latest": http.StatusOK, "1.0.0": http.StatusOK, "2.0.0": http.StatusForbidden, "3.0.0": http.StatusNotFound} { + if hosted && specifier == "3.0.0" { + want = http.StatusOK + } + got := f.get(t, "/@example/widgets/"+specifier) + if got.Code != want { + t.Errorf("%s: HTTP %d, want %d: %s", specifier, got.Code, want, got.Body.String()) + } + } +} diff --git a/internal/handler/npm_rewrite_bench_test.go b/internal/handler/npm_rewrite_bench_test.go index b200c5f..86552db 100644 --- a/internal/handler/npm_rewrite_bench_test.go +++ b/internal/handler/npm_rewrite_bench_test.go @@ -43,7 +43,7 @@ func BenchmarkNPMRewriteMetadata(b *testing.B) { b.SetBytes(int64(len(body))) b.ReportAllocs() for b.Loop() { - if _, err := h.rewriteMetadata("big", body); err != nil { + if _, _, err := h.rewriteMetadataKeeping("big", body); err != nil { b.Fatal(err) } } diff --git a/internal/handler/npm_rewrite_test.go b/internal/handler/npm_rewrite_test.go index 8521082..1386503 100644 --- a/internal/handler/npm_rewrite_test.go +++ b/internal/handler/npm_rewrite_test.go @@ -1,9 +1,7 @@ package handler import ( - "encoding/json" "errors" - "fmt" "net/http" "net/http/httptest" "strings" @@ -16,159 +14,6 @@ import ( "github.com/prometheus/client_golang/prometheus/testutil" ) -// TestNPMRewriteMetadataKeepsUpstreamBytes checks that only tarball URLs -// change: everything else, including numbers encoding/json would round -// through float64 and characters it would HTML-escape, is copied verbatim. -func TestNPMRewriteMetadataKeepsUpstreamBytes(t *testing.T) { - h := &NPMHandler{proxy: testProxy(), proxyURL: "http://proxy"} - input := `{"name":"demo","_big":12345678901234567890,"readme":"&",` + - `"versions":{"1.0.0":{"version":"1.0.0","dist":{"shasum":"s","tarball":"https://registry.npmjs.org/demo/-/demo-1.0.0.tgz","integrity":"i"}}}}` - - out, err := h.rewriteMetadata("demo", []byte(input)) - if err != nil { - t.Fatal(err) - } - want := strings.Replace(input, "https://registry.npmjs.org/demo/-/demo-1.0.0.tgz", "http://proxy/npm/demo/-/demo-1.0.0.tgz", 1) - if string(out) != want { - t.Errorf("rewrite =\n%s\nwant\n%s", out, want) - } -} - -func TestNPMRewriteMetadataUnusualEntries(t *testing.T) { - h := &NPMHandler{proxy: testProxy(), proxyURL: "http://proxy"} - input := `{ - "versions": { - "1.0.0": "not an object", - "2.0.0": {"dist": "no object either"}, - "3.0.0": {"dist": {"tarball": 7}}, - "4.0.0": {"dist": {"tarball": "https://registry.npmjs.org/demo/-/renamed.tgz"}}, - "5.0.0": {"dist": {}} - } - }` - out, err := h.rewriteMetadata("demo", []byte(input)) - if err != nil { - t.Fatal(err) - } - var doc struct { - Versions map[string]json.RawMessage `json:"versions"` - } - if err := json.Unmarshal(out, &doc); err != nil { - t.Fatalf("output is not JSON: %v\n%s", err, out) - } - for v, want := range map[string]string{ - "1.0.0": `"not an object"`, - "2.0.0": `{"dist": "no object either"}`, - "3.0.0": `{"dist": {"tarball": 7}}`, - "4.0.0": `{"dist": {"tarball": "http://proxy/npm/demo/-/demo-4.0.0.tgz"}}`, - "5.0.0": `{"dist": {}}`, - } { - if got := string(doc.Versions[v]); got != want { - t.Errorf("%s = %s, want %s", v, got, want) - } - } -} - -func TestNPMRewriteMetadataInvalidJSON(t *testing.T) { - h := &NPMHandler{proxy: testProxy(), proxyURL: "http://proxy"} - for _, input := range []string{`{"versions": {}`, `[1]`, `{"versions": {}} trailing`} { - if _, err := h.rewriteMetadata("demo", []byte(input)); err == nil { - t.Errorf("%s: expected an error", input) - } - } -} - -func TestNPMRewriteMetadataNoVersions(t *testing.T) { - h := &NPMHandler{proxy: testProxy(), proxyURL: "http://proxy"} - input := `{"name":"demo","versions":[]}` - out, err := h.rewriteMetadata("demo", []byte(input)) - if err != nil || string(out) != input { - t.Errorf("rewrite = %s, %v; want input unchanged", out, err) - } - - setTestDenylist(t, h.proxy, "pkg:npm/demo@1.0.0") - if _, err := h.rewriteMetadata("demo", []byte(input)); err == nil { - t.Error("expected an error when a denylisted package has no versions object") - } -} - -// TestNPMRewriteMetadataCooldownAndDenylist runs both filters together and -// checks that versions, their time entries and the tags pointing at them are -// all removed, with latest moved to the newest version left. -func TestNPMRewriteMetadataCooldownAndDenylist(t *testing.T) { - now := time.Now() - ts := func(age time.Duration) string { return now.Add(-age).UTC().Format(time.RFC3339) } - - proxy := testProxy() - proxy.Cooldown = &cooldown.Config{Default: "3d"} - setTestDenylist(t, proxy, "pkg:npm/demo@2.0.0") - h := &NPMHandler{proxy: proxy, proxyURL: "http://proxy"} - - version := func(v string) string { - return fmt.Sprintf(`%q:{"version":%q,"dist":{"tarball":"https://registry.npmjs.org/demo/-/demo-%s.tgz"}}`, v, v, v) - } - input := `{"name":"demo",` + - `"dist-tags":{"latest":"3.0.0","beta":"2.0.0","next":"1.0.0"},` + - `"versions":{` + version("1.0.0") + `,` + version("2.0.0") + `,` + version("3.0.0") + `},` + - `"time":{"created":"` + ts(1000*time.Hour) + `","1.0.0":"` + ts(900*time.Hour) + `","2.0.0":"` + ts(800*time.Hour) + `","3.0.0":"` + ts(time.Hour) + `"}}` - - out, err := h.rewriteMetadata("demo", []byte(input)) - if err != nil { - t.Fatal(err) - } - var doc struct { - DistTags map[string]string `json:"dist-tags"` - Versions map[string]json.RawMessage `json:"versions"` - Time map[string]string `json:"time"` - } - if err := json.Unmarshal(out, &doc); err != nil { - t.Fatalf("output is not JSON: %v\n%s", err, out) - } - if len(doc.Versions) != 1 || doc.Versions["1.0.0"] == nil { - t.Errorf("versions = %v, want only 1.0.0", keysOf(doc.Versions)) - } - if _, ok := doc.Time["2.0.0"]; ok { - t.Error("time still lists denylisted 2.0.0") - } - if _, ok := doc.Time["3.0.0"]; ok { - t.Error("time still lists 3.0.0, which is inside cooldown") - } - if _, ok := doc.Time["created"]; !ok { - t.Error("time lost its created entry") - } - want := map[string]string{"latest": "1.0.0", "next": "1.0.0"} - if fmt.Sprint(doc.DistTags) != fmt.Sprint(want) { - t.Errorf("dist-tags = %v, want %v", doc.DistTags, want) - } - if !strings.Contains(string(doc.Versions["1.0.0"]), "http://proxy/npm/demo/-/demo-1.0.0.tgz") { - t.Errorf("1.0.0 tarball not rewritten: %s", doc.Versions["1.0.0"]) - } -} - -// TestNPMRewriteMetadataDuplicateKeys checks that a repeated top-level key -// cannot smuggle an unfiltered copy of the versions past the filters. -func TestNPMRewriteMetadataDuplicateKeys(t *testing.T) { - proxy := testProxy() - setTestDenylist(t, proxy, "pkg:npm/demo@2.0.0") - h := &NPMHandler{proxy: proxy, proxyURL: "http://proxy"} - input := `{"versions":{"2.0.0":{"dist":{"tarball":"https://registry.npmjs.org/demo/-/demo-2.0.0.tgz"}}},` + - `"versions":{"1.0.0":{"dist":{"tarball":"https://registry.npmjs.org/demo/-/demo-1.0.0.tgz"}},"2.0.0":{"dist":{"tarball":"https://registry.npmjs.org/demo/-/demo-2.0.0.tgz"}}}}` - out, err := h.rewriteMetadata("demo", []byte(input)) - if err != nil { - t.Fatal(err) - } - if strings.Contains(string(out), "2.0.0") || strings.Count(string(out), `"versions"`) != 1 { - t.Errorf("rewrite = %s", out) - } -} - -func keysOf(m map[string]json.RawMessage) []string { - keys := make([]string, 0, len(m)) - for k := range m { - keys = append(keys, k) - } - return keys -} - // TestMetadataServesRewriteWithoutReadingStorage checks that a repeat // request inside the metadata TTL is answered from the rewrite cache by the // stored digest alone, without reading the cached document back. diff --git a/internal/handler/npm_test.go b/internal/handler/npm_test.go index 0867776..0e083a3 100644 --- a/internal/handler/npm_test.go +++ b/internal/handler/npm_test.go @@ -155,7 +155,7 @@ func TestNPMRewriteMetadata(t *testing.T) { } }` - output, err := h.rewriteMetadata("lodash", []byte(input)) + output, _, err := h.rewriteMetadataKeeping("lodash", []byte(input)) if err != nil { t.Fatalf("rewriteMetadata failed: %v", err) } @@ -186,7 +186,7 @@ func TestNPMRewriteMetadataLogsOncePerDocument(t *testing.T) { "4.17.19":{"dist":{"tarball":"https://registry.npmjs.org/lodash/-/lodash-4.17.19.tgz"}}, "4.17.20":{"dist":{"tarball":"https://registry.npmjs.org/lodash/-/lodash-4.17.20.tgz"}}, "4.17.21":{"dist":{"tarball":"https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz"}}}}` - if _, err := h.rewriteMetadata("lodash", []byte(input)); err != nil { + if _, _, err := h.rewriteMetadataKeeping("lodash", []byte(input)); err != nil { t.Fatalf("rewriteMetadata failed: %v", err) } @@ -215,7 +215,7 @@ func TestNPMRewriteMetadataScopedPackage(t *testing.T) { } }` - output, err := h.rewriteMetadata("@babel/core", []byte(input)) + output, _, err := h.rewriteMetadataKeeping("@babel/core", []byte(input)) if err != nil { t.Fatalf("rewriteMetadata failed: %v", err) } @@ -330,7 +330,7 @@ func TestNPMRewriteMetadataCooldown(t *testing.T) { } }` - output, err := h.rewriteMetadata("testpkg", []byte(input)) + output, _, err := h.rewriteMetadataKeeping("testpkg", []byte(input)) if err != nil { t.Fatalf("rewriteMetadata failed: %v", err) } @@ -386,7 +386,7 @@ func TestNPMRewriteMetadataCooldownExemptPackage(t *testing.T) { } }` - output, err := h.rewriteMetadata("testpkg", []byte(input)) + output, _, err := h.rewriteMetadataKeeping("testpkg", []byte(input)) if err != nil { t.Fatalf("rewriteMetadata failed: %v", err) } diff --git a/internal/handler/jsonscan.go b/internal/jsonscan/jsonscan.go similarity index 55% rename from internal/handler/jsonscan.go rename to internal/jsonscan/jsonscan.go index 2c95aa4..3478e69 100644 --- a/internal/handler/jsonscan.go +++ b/internal/jsonscan/jsonscan.go @@ -1,4 +1,4 @@ -package handler +package jsonscan import ( "bytes" @@ -16,24 +16,24 @@ import ( // the document. Callers that copy unvisited bytes into a response validate the // whole document first with json.Valid. -var errMalformedJSON = errors.New("malformed JSON") +var ErrMalformed = errors.New("malformed JSON") -// errNotJSONObject is returned when a value expected to be an object is not. -var errNotJSONObject = errors.New("JSON value is not an object") +// ErrNotObject is returned when a value expected to be an object is not. +var ErrNotObject = errors.New("JSON value is not an object") -// errStopScan is returned by a callback to end a walk early once it has +// ErrStop is returned by a callback to end a walk early once it has // found what it was looking for. -var errStopScan = errors.New("stop scanning") +var ErrStop = errors.New("stop scanning") -// jsonMember is one member of a JSON object as offsets into the bytes that +// Member is one member of a JSON object as offsets into the bytes that // were scanned. The key span includes its quotes. -type jsonMember struct { - keyStart, keyEnd int - valStart, valEnd int +type Member struct { + KeyStart, KeyEnd int + ValStart, ValEnd int } -func (m jsonMember) key(data []byte) []byte { return data[m.keyStart:m.keyEnd] } -func (m jsonMember) value(data []byte) []byte { return data[m.valStart:m.valEnd] } +func (m Member) Key(data []byte) []byte { return data[m.KeyStart:m.KeyEnd] } +func (m Member) Value(data []byte) []byte { return data[m.ValStart:m.ValEnd] } func skipJSONSpace(data []byte, i int) int { for i < len(data) { @@ -57,13 +57,13 @@ func skipJSONString(data []byte, i int) (int, error) { return j + 1, nil } } - return 0, errMalformedJSON + return 0, ErrMalformed } // skipJSONValue returns the index just past the value starting at data[i]. func skipJSONValue(data []byte, i int) (int, error) { if i >= len(data) { - return 0, errMalformedJSON + return 0, ErrMalformed } switch data[i] { case '"': @@ -87,9 +87,9 @@ func skipJSONValue(data []byte, i int) (int, error) { } } } - return 0, errMalformedJSON + return 0, ErrMalformed case '}', ']', ',', ':': - return 0, errMalformedJSON + return 0, ErrMalformed default: // A number, true, false or null runs to the next delimiter. j := i @@ -104,13 +104,13 @@ func skipJSONValue(data []byte, i int) (int, error) { } } -// forEachJSONMember calls fn for each member of the object obj holds, in -// document order. It returns errNotJSONObject if obj is not an object, and +// EachMember calls fn for each member of the object obj holds, in +// document order. It returns ErrNotObject if obj is not an object, and // stops early with fn's error if fn returns one. -func forEachJSONMember(obj []byte, fn func(jsonMember) error) error { +func EachMember(obj []byte, fn func(Member) error) error { i := skipJSONSpace(obj, 0) if i >= len(obj) || obj[i] != '{' { - return errNotJSONObject + return ErrNotObject } i = skipJSONSpace(obj, i+1) if i < len(obj) && obj[i] == '}' { @@ -118,7 +118,7 @@ func forEachJSONMember(obj []byte, fn func(jsonMember) error) error { } for { if i >= len(obj) || obj[i] != '"' { - return errMalformedJSON + return ErrMalformed } keyEnd, err := skipJSONString(obj, i) if err != nil { @@ -126,19 +126,19 @@ func forEachJSONMember(obj []byte, fn func(jsonMember) error) error { } colon := skipJSONSpace(obj, keyEnd) if colon >= len(obj) || obj[colon] != ':' { - return errMalformedJSON + return ErrMalformed } valStart := skipJSONSpace(obj, colon+1) valEnd, err := skipJSONValue(obj, valStart) if err != nil { return err } - if err := fn(jsonMember{keyStart: i, keyEnd: keyEnd, valStart: valStart, valEnd: valEnd}); err != nil { + if err := fn(Member{KeyStart: i, KeyEnd: keyEnd, ValStart: valStart, ValEnd: valEnd}); err != nil { return err } next := skipJSONSpace(obj, valEnd) if next >= len(obj) { - return errMalformedJSON + return ErrMalformed } switch obj[next] { case ',': @@ -146,17 +146,17 @@ func forEachJSONMember(obj []byte, fn func(jsonMember) error) error { case '}': return nil default: - return errMalformedJSON + return ErrMalformed } } } -// forEachJSONElement calls fn with each element of the array arr holds, in +// EachElement calls fn with each element of the array arr holds, in // order, and stops early with fn's error if fn returns one. -func forEachJSONElement(arr []byte, fn func(element []byte) error) error { +func EachElement(arr []byte, fn func(element []byte) error) error { i := skipJSONSpace(arr, 0) if i >= len(arr) || arr[i] != '[' { - return errMalformedJSON + return ErrMalformed } i = skipJSONSpace(arr, i+1) if i < len(arr) && arr[i] == ']' { @@ -172,7 +172,7 @@ func forEachJSONElement(arr []byte, fn func(element []byte) error) error { } next := skipJSONSpace(arr, end) if next >= len(arr) { - return errMalformedJSON + return ErrMalformed } switch arr[next] { case ',': @@ -180,22 +180,22 @@ func forEachJSONElement(arr []byte, fn func(element []byte) error) error { case ']': return nil default: - return errMalformedJSON + return ErrMalformed } } } -// rewriteJSONMembers copies the object obj holds to out byte for byte, +// RewriteMembers copies the object obj holds to out byte for byte, // except that rewrite may write a member's value itself. It returns true // when it did; otherwise the original value is copied. -func rewriteJSONMembers(out *bytes.Buffer, obj []byte, rewrite func(out *bytes.Buffer, m jsonMember) (bool, error)) error { +func RewriteMembers(out *bytes.Buffer, obj []byte, rewrite func(out *bytes.Buffer, m Member) (bool, error)) error { copied := 0 - err := forEachJSONMember(obj, func(m jsonMember) error { - out.Write(obj[copied:m.valStart]) - copied = m.valStart + err := EachMember(obj, func(m Member) error { + out.Write(obj[copied:m.ValStart]) + copied = m.ValStart rewritten, err := rewrite(out, m) if rewritten { - copied = m.valEnd + copied = m.ValEnd } return err }) @@ -203,8 +203,8 @@ func rewriteJSONMembers(out *bytes.Buffer, obj []byte, rewrite func(out *bytes.B return err } -// jsonKey decodes a quoted key as forEachJSONMember reports it. -func jsonKey(raw []byte) (string, error) { +// Key decodes a quoted key as EachMember reports it. +func Key(raw []byte) (string, error) { if bytes.IndexByte(raw, '\\') < 0 { return string(raw[1 : len(raw)-1]), nil } @@ -213,27 +213,27 @@ func jsonKey(raw []byte) (string, error) { return key, err } -// jsonKeyIs reports whether a quoted key decodes to name. -func jsonKeyIs(raw []byte, name string) bool { +// KeyIs reports whether a quoted key decodes to name. +func KeyIs(raw []byte, name string) bool { if bytes.IndexByte(raw, '\\') < 0 { return string(raw[1:len(raw)-1]) == name } - key, err := jsonKey(raw) + key, err := Key(raw) return err == nil && key == name } -// jsonStringIs reports whether raw is a JSON string that decodes to s. -func jsonStringIs(raw []byte, s string) bool { - return len(raw) >= 2 && raw[0] == '"' && jsonKeyIs(raw, s) +// StringIs reports whether raw is a JSON string that decodes to s. +func StringIs(raw []byte, s string) bool { + return len(raw) >= 2 && raw[0] == '"' && KeyIs(raw, s) } -// findJSONMember returns the member of obj named name. When the key repeats, +// FindMember returns the member of obj named name. When the key repeats, // the last one wins, as it does for JSON.parse and encoding/json. -func findJSONMember(obj []byte, name string) (jsonMember, bool, error) { - var found jsonMember +func FindMember(obj []byte, name string) (Member, bool, error) { + var found Member ok := false - err := forEachJSONMember(obj, func(m jsonMember) error { - if jsonKeyIs(m.key(obj), name) { + err := EachMember(obj, func(m Member) error { + if KeyIs(m.Key(obj), name) { found, ok = m, true } return nil @@ -241,28 +241,28 @@ func findJSONMember(obj []byte, name string) (jsonMember, bool, error) { return found, ok, err } -// lookupJSON follows path through nested objects in doc and returns the +// Lookup follows path through nested objects in doc and returns the // value at its end. It returns nil and no error when a key along the path is // missing or names something other than an object. -func lookupJSON(doc []byte, path ...string) ([]byte, error) { +func Lookup(doc []byte, path ...string) ([]byte, error) { value := doc for _, name := range path { - m, ok, err := findJSONMember(value, name) - if errors.Is(err, errNotJSONObject) { + m, ok, err := FindMember(value, name) + if errors.Is(err, ErrNotObject) { return nil, nil } if err != nil || !ok { return nil, err } - value = m.value(value) + value = m.Value(value) } return value, nil } -// lookupJSONString is lookupJSON for a string value. ok is false when the +// LookupString is Lookup for a string value. ok is false when the // value is missing or is not a string. -func lookupJSONString(doc []byte, path ...string) (string, bool, error) { - raw, err := lookupJSON(doc, path...) +func LookupString(doc []byte, path ...string) (string, bool, error) { + raw, err := Lookup(doc, path...) if err != nil || len(raw) == 0 || raw[0] != '"' { return "", false, err } @@ -272,3 +272,38 @@ func lookupJSONString(doc []byte, path ...string) (string, bool, error) { } return s, true, nil } + +// WriteFilteredObject writes the object obj holds, keeping the members +// whose decoded key keep accepts, followed by extra string members. +func WriteFilteredObject(out *bytes.Buffer, obj []byte, keep func(string) bool, extra [][2]string) error { + out.WriteByte('{') + first := true + err := EachMember(obj, func(m Member) error { + key, err := Key(m.Key(obj)) + if err != nil { + return err + } + if !keep(key) { + return nil + } + if !first { + out.WriteByte(',') + } + first = false + out.Write(obj[m.KeyStart:m.ValEnd]) + return nil + }) + for _, member := range extra { + if !first { + out.WriteByte(',') + } + first = false + key, _ := json.Marshal(member[0]) + value, _ := json.Marshal(member[1]) + out.Write(key) + out.WriteByte(':') + out.Write(value) + } + out.WriteByte('}') + return err +} diff --git a/internal/handler/jsonscan_test.go b/internal/jsonscan/jsonscan_test.go similarity index 71% rename from internal/handler/jsonscan_test.go rename to internal/jsonscan/jsonscan_test.go index 0825000..a14ac85 100644 --- a/internal/handler/jsonscan_test.go +++ b/internal/jsonscan/jsonscan_test.go @@ -1,4 +1,4 @@ -package handler +package jsonscan import ( "bytes" @@ -34,7 +34,7 @@ func TestLookupJSONString(t *testing.T) { {[]string{"name", "x"}, "", false}, } for _, c := range cases { - got, ok, err := lookupJSONString(doc, c.path...) + got, ok, err := LookupString(doc, c.path...) if err != nil || ok != c.wantOK || got != c.want { t.Errorf("lookup %v = %q, %v, %v; want %q, %v", c.path, got, ok, err, c.want, c.wantOK) } @@ -43,7 +43,7 @@ func TestLookupJSONString(t *testing.T) { func TestLookupJSONEscapedKey(t *testing.T) { doc := []byte(`{"versions": {"1.0.0": {"dist": {"tarball": "u"}}}}`) - got, ok, err := lookupJSONString(doc, "versions", "1.0.0", "dist", "tarball") + got, ok, err := LookupString(doc, "versions", "1.0.0", "dist", "tarball") if err != nil || !ok || got != "u" { t.Errorf("lookup = %q, %v, %v", got, ok, err) } @@ -57,9 +57,9 @@ func TestLookupJSONMalformed(t *testing.T) { `{"a": "unterminated}`, `{"a": [1, 2}`, } { - _, _, err := lookupJSONString([]byte(doc), "versions", "1.0.0", "dist", "tarball") - if !errors.Is(err, errMalformedJSON) { - t.Errorf("%s: err = %v, want errMalformedJSON", doc, err) + _, _, err := LookupString([]byte(doc), "versions", "1.0.0", "dist", "tarball") + if !errors.Is(err, ErrMalformed) { + t.Errorf("%s: err = %v, want ErrMalformed", doc, err) } } } @@ -67,8 +67,8 @@ func TestLookupJSONMalformed(t *testing.T) { func TestForEachJSONMemberOffsets(t *testing.T) { doc := []byte(` { "a" : 1 , "b":{"c":[true,null]} ,"d":"x\"y" } `) var got [][2]string - err := forEachJSONMember(doc, func(m jsonMember) error { - got = append(got, [2]string{string(m.key(doc)), string(m.value(doc))}) + err := EachMember(doc, func(m Member) error { + got = append(got, [2]string{string(m.Key(doc)), string(m.Value(doc))}) return nil }) want := [][2]string{{`"a"`, `1`}, {`"b"`, `{"c":[true,null]}`}, {`"d"`, `"x\"y"`}} @@ -81,17 +81,17 @@ func TestForEachJSONMemberOffsets(t *testing.T) { } } - if err := forEachJSONMember([]byte(`[1]`), func(jsonMember) error { return nil }); !errors.Is(err, errNotJSONObject) { - t.Errorf("array: err = %v, want errNotJSONObject", err) + if err := EachMember([]byte(`[1]`), func(Member) error { return nil }); !errors.Is(err, ErrNotObject) { + t.Errorf("array: err = %v, want ErrNotObject", err) } - if err := forEachJSONMember([]byte(`{}`), func(jsonMember) error { t.Error("called for empty object"); return nil }); err != nil { + if err := EachMember([]byte(`{}`), func(Member) error { t.Error("called for empty object"); return nil }); err != nil { t.Errorf("empty object: err = %v", err) } } func TestWriteFilteredJSONObject(t *testing.T) { var out bytes.Buffer - err := writeFilteredJSONObjectWith(&out, []byte(`{"a": 1, "b": {"x": 2}, "c": 3}`), func(k string) bool { return k != "b" }, nil) + err := WriteFilteredObject(&out, []byte(`{"a": 1, "b": {"x": 2}, "c": 3}`), func(k string) bool { return k != "b" }, nil) if err != nil || out.String() != `{"a": 1,"c": 3}` { t.Errorf("filtered = %s, %v", out.String(), err) } @@ -99,7 +99,7 @@ func TestWriteFilteredJSONObject(t *testing.T) { func TestForEachJSONElement(t *testing.T) { var got []string - err := forEachJSONElement([]byte(` [ 1, "a,]", {"b": [2]} ,[] ] `), func(e []byte) error { + err := EachElement([]byte(` [ 1, "a,]", {"b": [2]} ,[] ] `), func(e []byte) error { got = append(got, string(e)) return nil }) @@ -113,11 +113,11 @@ func TestForEachJSONElement(t *testing.T) { } } - if err := forEachJSONElement([]byte(`[]`), func([]byte) error { t.Error("called for empty array"); return nil }); err != nil { + if err := EachElement([]byte(`[]`), func([]byte) error { t.Error("called for empty array"); return nil }); err != nil { t.Errorf("empty array: err = %v", err) } for _, bad := range []string{`{}`, `[1,`, `[1 2]`, `[1`} { - if err := forEachJSONElement([]byte(bad), func([]byte) error { return nil }); err == nil { + if err := EachElement([]byte(bad), func([]byte) error { return nil }); err == nil { t.Errorf("%s: expected an error", bad) } } @@ -126,8 +126,8 @@ func TestForEachJSONElement(t *testing.T) { func TestRewriteJSONMembers(t *testing.T) { obj := []byte(`{ "a": 1, "b": [2], "c" : 3 }`) var out bytes.Buffer - err := rewriteJSONMembers(&out, obj, func(out *bytes.Buffer, m jsonMember) (bool, error) { - if !jsonKeyIs(m.key(obj), "b") { + err := RewriteMembers(&out, obj, func(out *bytes.Buffer, m Member) (bool, error) { + if !KeyIs(m.Key(obj), "b") { return false, nil } out.WriteString(`"B"`) @@ -148,8 +148,8 @@ func TestJSONStringIs(t *testing.T) { ``: false, `["~dev"]`: false, } { - if got := jsonStringIs([]byte(raw), "~dev"); got != want { - t.Errorf("jsonStringIs(%s) = %v, want %v", raw, got, want) + if got := StringIs([]byte(raw), "~dev"); got != want { + t.Errorf("StringIs(%s) = %v, want %v", raw, got, want) } } } diff --git a/internal/handler/npm_denylist.go b/internal/metadata/npm/denylist.go similarity index 82% rename from internal/handler/npm_denylist.go rename to internal/metadata/npm/denylist.go index 7f8b87d..d34d4d3 100644 --- a/internal/handler/npm_denylist.go +++ b/internal/metadata/npm/denylist.go @@ -1,12 +1,9 @@ -package handler +package npm import "github.com/Masterminds/semver/v3" -func (h *NPMHandler) applyDenylistFiltering(metadata, versions map[string]any, packageName string) { - if h.proxy.Denylist == nil { - return - } - denied := h.proxy.Denylist.Versions(canonicalPackagePURL("npm", packageName)) +func (h *transformer) applyDenylistFiltering(metadata, versions map[string]any) { + denied := h.DeniedVersions if len(denied) == 0 { return } diff --git a/internal/metadata/npm/metadata.go b/internal/metadata/npm/metadata.go new file mode 100644 index 0000000..dcf0347 --- /dev/null +++ b/internal/metadata/npm/metadata.go @@ -0,0 +1,471 @@ +// Package npm transforms registry metadata without storage or network access. +package npm + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "maps" + "net/url" + "reflect" + "slices" + "sort" + "strings" + "time" + + "github.com/git-pkgs/cooldown" + "github.com/git-pkgs/proxy/internal/jsonscan" + "github.com/git-pkgs/proxy/internal/packageurl" +) + +var ErrVersionDenied = errors.New("package version blocked by denylist") +var ErrVersionNotFound = errors.New("version not found") + +// CooldownPolicy evaluates version age at the caller's supplied time. +type CooldownPolicy interface { + Enabled() bool + Evaluate(ecosystem, packagePURL string, publishedAt, evaluatedAt time.Time) cooldown.Decision +} + +// LocalVersion is an already approved hosted version and its publication time. +type LocalVersion struct { + Version string + Entry json.RawMessage + PublishedAt string +} + +// Options supplies the public URL, policy and hosted versions for a rewrite. +// EvaluatedAt is used as given; the transformation never reads the clock. +type Options struct { + ProxyURL string + Cooldown CooldownPolicy + DeniedVersions map[string]bool + EvaluatedAt time.Time + LocalVersions []LocalVersion + // Optional callbacks report changes synchronously without retaining events. + // OnTarballsRewritten is called at most once per document, with how many + // tarball URLs it pointed at the proxy, and not at all when none. + OnTarballsRewritten func(count int) + OnCooldownFilter func(version, publishedAt string) +} + +type transformer struct{ Options } + +// Rewrite filters upstream versions and rewrites their tarball URLs, preserving +// untouched JSON values verbatim. Hosted entries override upstream versions. +// The returned version set is unordered. The document, denylist and hosted +// entries are not modified. +func Rewrite(packageName string, body []byte, options Options) ([]byte, []string, error) { + return (&transformer{options}).rewriteMetadataWithLocal(packageName, body, options.LocalVersions) +} + +// Version resolves a version or dist-tag through the filtered packument. +func Version(packageName, specifier string, packument []byte, options Options) ([]byte, error) { + filtered, _, err := Rewrite(packageName, packument, options) + if err != nil { + return nil, err + } + + target := specifier + if resolved, ok, err := jsonscan.LookupString(filtered, "dist-tags", specifier); err != nil { + return nil, err + } else if ok { + target = resolved + } + + if options.DeniedVersions[target] { + return nil, ErrVersionDenied + } + + versionRaw, err := jsonscan.Lookup(filtered, "versions", target) + if err != nil { + return nil, err + } + if len(versionRaw) == 0 { + return nil, ErrVersionNotFound + } + + var out bytes.Buffer + (&transformer{options}).writeNPMVersion(&out, packageName, target, versionRaw) + return out.Bytes(), nil +} + +func (h *transformer) rewriteMetadataWithLocal(packageName string, body []byte, local []LocalVersion) ([]byte, []string, error) { + doc, err := indexNPMPackument(body) + if err != nil { + return nil, nil, err + } + + versionsRaw := doc.value(doc.versionsAt) + if len(versionsRaw) == 0 || versionsRaw[0] != '{' { + version, ok, err := jsonscan.LookupString(body, "version") + if err != nil { + return nil, nil, err + } + if ok { + if h.DeniedVersions[version] { + return nil, nil, ErrVersionDenied + } + var out bytes.Buffer + if h.writeNPMVersion(&out, packageName, version, body) { + h.tarballsRewritten(1) + } + return out.Bytes(), []string{version}, nil + } + if len(h.DeniedVersions) != 0 { + return nil, nil, errors.New("npm metadata has no versions object") + } + return body, nil, nil // No versions to rewrite + } + + // The filters only look at which versions exist, so they run on a map of + // version names alongside the decoded time map and dist-tags. + versions := map[string]any{} + if err := jsonscan.EachMember(versionsRaw, func(m jsonscan.Member) error { + version, err := jsonscan.Key(m.Key(versionsRaw)) + versions[version] = nil + return err + }); err != nil { + return nil, nil, err + } + metadata := map[string]any{} + timeMap := doc.decodeObject(doc.timeAt, metadata, "time") + distTags := doc.decodeObject(doc.tagsAt, metadata, "dist-tags") + timeEntries := len(timeMap) + originalTags := maps.Clone(distTags) + + h.applyCooldownFiltering(metadata, versions, packageName) + h.applyDenylistFiltering(metadata, versions) + for _, e := range local { + delete(versions, e.Version) + } + + // Untouched members are copied as they are; the three the filters + // handled are written from their filtered state, and only when it changed. + out, err := doc.write(func(out *bytes.Buffer, i int) error { + switch { + case i == doc.versionsAt: + return h.writeNPMVersions(out, packageName, versionsRaw, versions, local) + case i == doc.timeAt && timeMap != nil && (len(timeMap) != timeEntries || len(local) > 0): + pushedAt := make([][2]string, 0, len(local)) + for _, e := range local { + pushedAt = append(pushedAt, [2]string{e.Version, e.PublishedAt}) + } + return jsonscan.WriteFilteredObject(out, doc.value(i), func(key string) bool { + _, ok := timeMap[key] + pushed := slices.ContainsFunc(local, func(e LocalVersion) bool { return e.Version == key }) + return ok && !pushed + }, pushedAt) + case i == doc.tagsAt && distTags != nil && !reflect.DeepEqual(distTags, originalTags): + encoded, err := json.Marshal(distTags) + out.Write(encoded) + return err + default: + out.Write(doc.value(i)) + return nil + } + }) + kept := slices.Collect(maps.Keys(versions)) + for _, e := range local { + kept = append(kept, e.Version) + } + return out, kept, err +} + +// npmPackument is a packument's top-level members as offsets into its bytes, +// with the positions of the members rewriteMetadata filters, or -1 for those +// it lacks. A key that repeats is recorded at its last occurrence, the one +// JSON.parse keeps. +type npmPackument struct { + body []byte + members []jsonscan.Member + versionsAt, timeAt, tagsAt int +} + +func indexNPMPackument(body []byte) (*npmPackument, error) { + if !json.Valid(body) { + return nil, errors.New("npm metadata is not valid JSON") + } + doc := &npmPackument{body: body, versionsAt: -1, timeAt: -1, tagsAt: -1} + err := jsonscan.EachMember(body, func(m jsonscan.Member) error { + switch key := m.Key(body); { + case jsonscan.KeyIs(key, "versions"): + doc.versionsAt = len(doc.members) + case jsonscan.KeyIs(key, "time"): + doc.timeAt = len(doc.members) + case jsonscan.KeyIs(key, "dist-tags"): + doc.tagsAt = len(doc.members) + } + doc.members = append(doc.members, m) + return nil + }) + return doc, err +} + +// value returns the raw value of member i, or nil when i is -1. +func (d *npmPackument) value(i int) []byte { + if i < 0 { + return nil + } + return d.members[i].Value(d.body) +} + +// decodeObject decodes member i into metadata[name] and returns it when it is +// an object. The filters edit the returned map in place. +func (d *npmPackument) decodeObject(i int, metadata map[string]any, name string) map[string]any { + raw := d.value(i) + if raw == nil { + return nil + } + var value any + if err := json.Unmarshal(raw, &value); err != nil { + return nil + } + metadata[name] = value + object, _ := value.(map[string]any) + return object +} + +// write assembles the document in member order, with writeValue writing each +// member's value. Earlier duplicates of the filtered keys are dropped: they +// would carry unfiltered data, and JSON.parse ignores them anyway. +func (d *npmPackument) write(writeValue func(out *bytes.Buffer, i int) error) ([]byte, error) { + var out bytes.Buffer + out.Grow(len(d.body) + len(d.body)/8) + out.WriteByte('{') + first := true + for i, m := range d.members { + if i != d.versionsAt && i != d.timeAt && i != d.tagsAt { + key := m.Key(d.body) + if jsonscan.KeyIs(key, "versions") || jsonscan.KeyIs(key, "time") || jsonscan.KeyIs(key, "dist-tags") { + continue + } + } + if !first { + out.WriteByte(',') + } + first = false + out.Write(m.Key(d.body)) + out.WriteByte(':') + if err := writeValue(&out, i); err != nil { + return nil, err + } + } + out.WriteByte('}') + return out.Bytes(), nil +} + +// writeNPMVersions writes the versions object, keeping the versions still in +// keep and pointing each kept version's tarball at this proxy, followed by the +// pushed versions in local. +func (h *transformer) writeNPMVersions(out *bytes.Buffer, packageName string, versionsRaw []byte, keep map[string]any, local []LocalVersion) error { + out.WriteByte('{') + first := true + rewritten := 0 + err := jsonscan.EachMember(versionsRaw, func(m jsonscan.Member) error { + version, err := jsonscan.Key(m.Key(versionsRaw)) + if err != nil { + return err + } + if _, ok := keep[version]; !ok { + return nil + } + if !first { + out.WriteByte(',') + } + first = false + out.Write(m.Key(versionsRaw)) + out.WriteByte(':') + if h.writeNPMVersion(out, packageName, version, m.Value(versionsRaw)) { + rewritten++ + } + return nil + }) + h.tarballsRewritten(rewritten) + for _, e := range local { + if !first { + out.WriteByte(',') + } + first = false + key, _ := json.Marshal(e.Version) + out.Write(key) + out.WriteByte(':') + out.Write(e.Entry) + } + out.WriteByte('}') + return err +} + +// writeNPMVersion writes one version entry with its dist.tarball pointing at +// this proxy, and reports whether it rewrote the tarball. An entry without a +// string tarball is written unchanged. +func (h *transformer) writeNPMVersion(out *bytes.Buffer, packageName, version string, raw []byte) bool { + dist, ok, err := jsonscan.FindMember(raw, "dist") + if err != nil || !ok { + out.Write(raw) + return false + } + distRaw := dist.Value(raw) + tarball, ok, err := jsonscan.FindMember(distRaw, "tarball") + if err != nil || !ok || distRaw[tarball.ValStart] != '"' { + out.Write(raw) + return false + } + var oldTarball string + if err := json.Unmarshal(tarball.Value(distRaw), &oldTarball); err != nil { + out.Write(raw) + return false + } + + newTarball := TarballURL(h.ProxyURL, packageName, version, oldTarball) + encoded, err := json.Marshal(newTarball) + if err != nil { + out.Write(raw) + return false + } + out.Write(raw[:dist.ValStart+tarball.ValStart]) + out.Write(encoded) + out.Write(raw[dist.ValStart+tarball.ValEnd:]) + return true +} + +// tarballsRewritten reports once per document how many tarball URLs it +// rewrote. A report per version flooded debug logs: packages such as eslint +// list hundreds of versions, and with cooldown on every request rewrites the +// document again. +func (h *transformer) tarballsRewritten(count int) { + if count > 0 && h.OnTarballsRewritten != nil { + h.OnTarballsRewritten(count) + } +} + +// applyCooldownFiltering removes versions that are too recently published, +// and updates dist-tags.latest if the current latest was filtered out. +func (h *transformer) applyCooldownFiltering(metadata map[string]any, versions map[string]any, packageName string) { + if h.Cooldown == nil || !h.Cooldown.Enabled() { + return + } + + timeMap, _ := metadata["time"].(map[string]any) + if timeMap == nil { + return + } + + packagePURL := packageurl.MakeString("npm", packageName, "") + + for version := range versions { + publishedStr, ok := timeMap[version].(string) + if !ok { + continue + } + publishedAt, err := time.Parse(time.RFC3339, publishedStr) + if err != nil { + continue + } + if !h.Cooldown.Evaluate("npm", packagePURL, publishedAt, h.EvaluatedAt).Allowed { + if h.OnCooldownFilter != nil { + h.OnCooldownFilter(version, publishedStr) + } + delete(versions, version) + delete(timeMap, version) + } + } + + h.updateDistTagsLatest(metadata, versions, timeMap) +} + +// updateDistTagsLatest updates the dist-tags.latest field if the current latest +// version was removed by cooldown filtering. +func (h *transformer) updateDistTagsLatest(metadata, versions, timeMap map[string]any) { + distTags, ok := metadata["dist-tags"].(map[string]any) + if !ok { + return + } + + latest, ok := distTags["latest"].(string) + if !ok { + return + } + + if _, exists := versions[latest]; exists { + return + } + + if newLatest := h.findNewestVersion(versions, timeMap); newLatest != "" { + distTags["latest"] = newLatest + } +} + +// TarballURL returns the proxy URL that serves the tarball upstream +// lists at tarball for this version. +func TarballURL(proxyURL, packageName, version, tarball string) string { + filename := tarball + if idx := strings.LastIndex(tarball, "/"); idx >= 0 { + filename = tarball[idx+1:] + } + if VersionFromFilename(packageName, filename) != version { + _, shortName, scoped := strings.Cut(packageName, "/") + if !scoped { + shortName = packageName + } + filename = shortName + "-" + version + ".tgz" + } + + return fmt.Sprintf("%s/npm/%s/-/%s", proxyURL, url.PathEscape(packageName), filename) +} + +// findNewestVersion returns the version string with the most recent timestamp +// from the remaining versions, using the time map. +func (h *transformer) findNewestVersion(versions map[string]any, timeMap map[string]any) string { + if timeMap == nil { + return "" + } + + type versionTime struct { + version string + t time.Time + } + + var vts []versionTime + for v := range versions { + if ts, ok := timeMap[v].(string); ok { + if t, err := time.Parse(time.RFC3339, ts); err == nil { + vts = append(vts, versionTime{v, t}) + } + } + } + + if len(vts) == 0 { + return "" + } + + sort.Slice(vts, func(i, j int) bool { + return vts[i].t.After(vts[j].t) + }) + + return vts[0].version +} + +// VersionFromFilename extracts a version from an npm tarball filename. +func VersionFromFilename(packageName, filename string) string { + // Remove .tgz extension + if !strings.HasSuffix(filename, ".tgz") { + return "" + } + base := strings.TrimSuffix(filename, ".tgz") + + // For scoped packages, the filename uses the short name + shortName := packageName + if _, name, scoped := strings.Cut(packageName, "/"); scoped { + shortName = name + } + + // Expected format: {shortName}-{version} + prefix := shortName + "-" + if !strings.HasPrefix(base, prefix) { + return "" + } + + return strings.TrimPrefix(base, prefix) +} diff --git a/internal/metadata/npm/metadata_test.go b/internal/metadata/npm/metadata_test.go new file mode 100644 index 0000000..a1e5e4b --- /dev/null +++ b/internal/metadata/npm/metadata_test.go @@ -0,0 +1,153 @@ +package npm + +import ( + "encoding/json" + "fmt" + "github.com/git-pkgs/cooldown" + "strings" + "testing" + "time" +) + +// TestNPMRewriteMetadataKeepsUpstreamBytes checks that only tarball URLs +// change: everything else, including numbers encoding/json would round +// through float64 and characters it would HTML-escape, is copied verbatim. +func TestNPMRewriteMetadataKeepsUpstreamBytes(t *testing.T) { + options := Options{ProxyURL: "http://proxy"} + input := `{"name":"demo","_big":12345678901234567890,"readme":"&",` + + `"versions":{"1.0.0":{"version":"1.0.0","dist":{"shasum":"s","tarball":"https://registry.npmjs.org/demo/-/demo-1.0.0.tgz","integrity":"i"}}}}` + + out, _, err := Rewrite("demo", []byte(input), options) + if err != nil { + t.Fatal(err) + } + want := strings.Replace(input, "https://registry.npmjs.org/demo/-/demo-1.0.0.tgz", "http://proxy/npm/demo/-/demo-1.0.0.tgz", 1) + if string(out) != want { + t.Errorf("rewrite =\n%s\nwant\n%s", out, want) + } +} + +func TestNPMRewriteMetadataUnusualEntries(t *testing.T) { + options := Options{ProxyURL: "http://proxy"} + input := `{ + "versions": { + "1.0.0": "not an object", + "2.0.0": {"dist": "no object either"}, + "3.0.0": {"dist": {"tarball": 7}}, + "4.0.0": {"dist": {"tarball": "https://registry.npmjs.org/demo/-/renamed.tgz"}}, + "5.0.0": {"dist": {}} + } + }` + out, _, err := Rewrite("demo", []byte(input), options) + if err != nil { + t.Fatal(err) + } + var doc struct { + Versions map[string]json.RawMessage `json:"versions"` + } + if err := json.Unmarshal(out, &doc); err != nil { + t.Fatalf("output is not JSON: %v\n%s", err, out) + } + for v, want := range map[string]string{ + "1.0.0": `"not an object"`, + "2.0.0": `{"dist": "no object either"}`, + "3.0.0": `{"dist": {"tarball": 7}}`, + "4.0.0": `{"dist": {"tarball": "http://proxy/npm/demo/-/demo-4.0.0.tgz"}}`, + "5.0.0": `{"dist": {}}`, + } { + if got := string(doc.Versions[v]); got != want { + t.Errorf("%s = %s, want %s", v, got, want) + } + } +} + +func TestNPMRewriteMetadataInvalidJSON(t *testing.T) { + options := Options{ProxyURL: "http://proxy"} + for _, input := range []string{`{"versions": {}`, `[1]`, `{"versions": {}} trailing`} { + if _, _, err := Rewrite("demo", []byte(input), options); err == nil { + t.Errorf("%s: expected an error", input) + } + } +} + +func TestNPMRewriteMetadataNoVersions(t *testing.T) { + options := Options{ProxyURL: "http://proxy"} + input := `{"name":"demo","versions":[]}` + out, _, err := Rewrite("demo", []byte(input), options) + if err != nil || string(out) != input { + t.Errorf("rewrite = %s, %v; want input unchanged", out, err) + } + + options.DeniedVersions = map[string]bool{"1.0.0": true} + if _, _, err := Rewrite("demo", []byte(input), options); err == nil { + t.Error("expected an error when a denylisted package has no versions object") + } +} + +// TestNPMRewriteMetadataCooldownAndDenylist runs both filters together and +// checks that versions, their time entries and the tags pointing at them are +// all removed, with latest moved to the newest version left. +func TestNPMRewriteMetadataCooldownAndDenylist(t *testing.T) { + now := time.Date(2026, 10, 18, 12, 0, 0, 0, time.UTC) + ts := func(age time.Duration) string { return now.Add(-age).UTC().Format(time.RFC3339) } + + options := Options{ProxyURL: "http://proxy", EvaluatedAt: time.Date(2026, 10, 18, 12, 0, 0, 0, time.UTC)} + options.Cooldown = &cooldown.Config{Default: "3d"} + options.DeniedVersions = map[string]bool{"2.0.0": true} + + version := func(v string) string { + return fmt.Sprintf(`%q:{"version":%q,"dist":{"tarball":"https://registry.npmjs.org/demo/-/demo-%s.tgz"}}`, v, v, v) + } + input := `{"name":"demo",` + + `"dist-tags":{"latest":"3.0.0","beta":"2.0.0","next":"1.0.0"},` + + `"versions":{` + version("1.0.0") + `,` + version("2.0.0") + `,` + version("3.0.0") + `},` + + `"time":{"created":"` + ts(1000*time.Hour) + `","1.0.0":"` + ts(900*time.Hour) + `","2.0.0":"` + ts(800*time.Hour) + `","3.0.0":"` + ts(time.Hour) + `"}}` + + out, _, err := Rewrite("demo", []byte(input), options) + if err != nil { + t.Fatal(err) + } + var doc struct { + DistTags map[string]string `json:"dist-tags"` + Versions map[string]json.RawMessage `json:"versions"` + Time map[string]string `json:"time"` + } + if err := json.Unmarshal(out, &doc); err != nil { + t.Fatalf("output is not JSON: %v\n%s", err, out) + } + if len(doc.Versions) != 1 || doc.Versions["1.0.0"] == nil { + t.Errorf("versions = %v, want only 1.0.0", doc.Versions) + } + if _, ok := doc.Time["2.0.0"]; ok { + t.Error("time still lists denylisted 2.0.0") + } + if _, ok := doc.Time["3.0.0"]; ok { + t.Error("time still lists 3.0.0, which is inside cooldown") + } + if _, ok := doc.Time["created"]; !ok { + t.Error("time lost its created entry") + } + want := map[string]string{"latest": "1.0.0", "next": "1.0.0"} + if fmt.Sprint(doc.DistTags) != fmt.Sprint(want) { + t.Errorf("dist-tags = %v, want %v", doc.DistTags, want) + } + if !strings.Contains(string(doc.Versions["1.0.0"]), "http://proxy/npm/demo/-/demo-1.0.0.tgz") { + t.Errorf("1.0.0 tarball not rewritten: %s", doc.Versions["1.0.0"]) + } +} + +// TestNPMRewriteMetadataDuplicateKeys checks that a repeated top-level key +// cannot smuggle an unfiltered copy of the versions past the filters. +func TestNPMRewriteMetadataDuplicateKeys(t *testing.T) { + options := Options{ProxyURL: "http://proxy", EvaluatedAt: time.Date(2026, 10, 18, 12, 0, 0, 0, time.UTC)} + options.DeniedVersions = map[string]bool{"2.0.0": true} + input := `{"versions":{"2.0.0":{"dist":{"tarball":"https://registry.npmjs.org/demo/-/demo-2.0.0.tgz"}}},` + + `"versions":{"1.0.0":{"dist":{"tarball":"https://registry.npmjs.org/demo/-/demo-1.0.0.tgz"}},"2.0.0":{"dist":{"tarball":"https://registry.npmjs.org/demo/-/demo-2.0.0.tgz"}}}}` + out, _, err := Rewrite("demo", []byte(input), options) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(out), "2.0.0") || strings.Count(string(out), `"versions"`) != 1 { + t.Errorf("rewrite = %s", out) + } +} diff --git a/internal/metadata/npm/policy_test.go b/internal/metadata/npm/policy_test.go new file mode 100644 index 0000000..7726d29 --- /dev/null +++ b/internal/metadata/npm/policy_test.go @@ -0,0 +1,161 @@ +package npm + +import ( + "bytes" + _ "embed" + "encoding/json" + "errors" + "slices" + "strings" + "testing" + "time" + + "github.com/git-pkgs/cooldown" +) + +//go:embed testdata/packument.json +var packument []byte + +func TestRewriteCallbacks(t *testing.T) { + options := Options{ProxyURL: "https://proxy.example", Cooldown: &cooldown.Config{Default: "7d"}, + DeniedVersions: map[string]bool{"2.0.0": true}, EvaluatedAt: time.Date(2030, 1, 1, 0, 0, 0, 0, time.UTC)} + want, kept, err := Rewrite("@example/widgets", packument, options) + if err != nil { + t.Fatal(err) + } + var rewrites, filtered int + options.OnTarballsRewritten = func(count int) { + rewrites++ + if count != 1 { + t.Errorf("rewrite count = %d, want 1", count) + } + } + options.OnCooldownFilter = func(version, publishedAt string) { + filtered++ + if version != "3.0.0" || publishedAt != "2099-01-01T00:00:00Z" { + t.Errorf("filter event: %s, %s", version, publishedAt) + } + } + out, gotKept, err := Rewrite("@example/widgets", packument, options) + if err != nil || !bytes.Equal(out, want) || !slices.Equal(gotKept, kept) || rewrites != 1 || filtered != 1 { + t.Fatalf("callbacks: rewrites=%d, filtered=%d, kept=%v, error=%v", rewrites, filtered, gotKept, err) + } +} + +func TestRewriteHostedOverride(t *testing.T) { + options := Options{ + ProxyURL: "https://proxy.example", + Cooldown: &cooldown.Config{Default: "7d"}, + DeniedVersions: map[string]bool{"2.0.0": true}, + EvaluatedAt: time.Date(2030, 1, 1, 0, 0, 0, 0, time.UTC), + LocalVersions: []LocalVersion{{Version: "3.0.0", PublishedAt: "2030-01-01T00:00:00Z", + Entry: json.RawMessage(`{"version":"3.0.0","dist":{"tarball":"https://proxy.example/npm/@example%2Fwidgets/-/widgets-3.0.0.tgz","integrity":"sha512-local"}}`)}}, + } + before := bytes.Clone(packument) + localBefore := bytes.Clone(options.LocalVersions[0].Entry) + out, kept, err := Rewrite("@example/widgets", packument, options) + if err != nil { + t.Fatal(err) + } + slices.Sort(kept) + if !slices.Equal(kept, []string{"1.0.0", "3.0.0"}) { + t.Fatalf("kept = %v", kept) + } + var doc struct { + Versions map[string]json.RawMessage `json:"versions"` + Tags map[string]string `json:"dist-tags"` + Time map[string]string `json:"time"` + } + if err := json.Unmarshal(out, &doc); err != nil { + t.Fatal(err) + } + if len(doc.Versions) != 2 || !bytes.Equal(doc.Versions["3.0.0"], localBefore) || doc.Time["3.0.0"] != options.LocalVersions[0].PublishedAt { + t.Fatalf("hosted override lost: %s", out) + } + if len(doc.Tags) != 1 || doc.Tags["latest"] != "1.0.0" || doc.Time["2.0.0"] != "" || doc.Time["created"] == "" { + t.Fatalf("policy output: %s", out) + } + for _, want := range []string{"12345678901234567890", "Widgets & tools", "https://proxy.example/npm/@example%2Fwidgets/-/widgets-1.0.0.tgz"} { + if !strings.Contains(string(out), want) { + t.Errorf("output lost %q", want) + } + } + if !bytes.Equal(packument, before) || !bytes.Equal(options.LocalVersions[0].Entry, localBefore) || len(options.DeniedVersions) != 1 || !options.DeniedVersions["2.0.0"] { + t.Fatal("rewrite mutated its input") + } + again, _, err := Rewrite("@example/widgets", packument, options) + if err != nil || !bytes.Equal(out, again) { + t.Fatalf("repeat changed output: %v", err) + } +} + +func TestVersionPolicyErrors(t *testing.T) { + options := Options{ProxyURL: "https://proxy.example", Cooldown: &cooldown.Config{Default: "7d"}, + DeniedVersions: map[string]bool{"2.0.0": true}, EvaluatedAt: time.Date(2030, 1, 1, 0, 0, 0, 0, time.UTC)} + for _, tc := range []struct { + specifier string + want error + }{ + {"1.0.0", nil}, {"latest", nil}, {"2.0.0", ErrVersionDenied}, {"3.0.0", ErrVersionNotFound}, {"beta", ErrVersionNotFound}, {"missing", ErrVersionNotFound}, + } { + t.Run(tc.specifier, func(t *testing.T) { + var counts []int + options := options + options.OnTarballsRewritten = func(count int) { counts = append(counts, count) } + out, err := Version("@example/widgets", tc.specifier, packument, options) + if !errors.Is(err, tc.want) { + t.Fatalf("error = %v, want %v", err, tc.want) + } + // The packument is reported once; picking one version out of it + // is not a second report. + if !slices.Equal(counts, []int{1}) { + t.Errorf("rewrite reports = %v, want [1]", counts) + } + if tc.want == nil && (!bytes.Contains(out, []byte(`"version": "1.0.0"`)) || !bytes.Contains(out, []byte("https://proxy.example/npm/@example%2Fwidgets/-/widgets-1.0.0.tgz"))) { + t.Fatalf("resolved version: %s", out) + } + }) + } +} + +func TestRewriteCooldownEvaluationTime(t *testing.T) { + body := []byte(`{"versions":{"1.0.0":{}},"time":{"1.0.0":"2030-01-01T00:00:00Z"}}`) + boundary := time.Date(2030, 1, 8, 0, 0, 0, 0, time.UTC) + options := Options{Cooldown: &cooldown.Config{Default: "7d"}} + for _, tc := range []struct { + at time.Time + count int + }{ + {boundary.Add(-time.Nanosecond), 0}, {boundary, 1}, {boundary.Add(time.Second), 1}, + } { + options.EvaluatedAt = tc.at + _, kept, err := Rewrite("@example/widgets", body, options) + if err != nil || len(kept) != tc.count { + t.Errorf("at %s: kept %v, error %v", tc.at, kept, err) + } + } + options.Cooldown = &cooldown.Config{Default: "7d", Packages: map[string]string{"pkg:npm/%40example/widgets": "0"}} + options.EvaluatedAt = boundary.Add(-time.Hour) + _, kept, err := Rewrite("@example/widgets", body, options) + if err != nil || len(kept) != 1 { + t.Fatalf("scoped exemption: kept %v, error %v", kept, err) + } +} + +func TestRewriteVersionDocument(t *testing.T) { + body := []byte(`{"name":"@example/widgets","version":"1.0.0","dist":{"tarball":"https://cdn.example/abcdef.tgz"}}`) + options := Options{ProxyURL: "https://proxy.example"} + out, kept, err := Rewrite("@example/widgets", body, options) + if err != nil || !slices.Equal(kept, []string{"1.0.0"}) || !bytes.Contains(out, []byte("https://proxy.example/npm/@example%2Fwidgets/-/widgets-1.0.0.tgz")) { + t.Fatalf("rewrite: %s, %v, %v", out, kept, err) + } + var counts []int + options.OnTarballsRewritten = func(count int) { counts = append(counts, count) } + if _, _, err := Rewrite("@example/widgets", body, options); err != nil || !slices.Equal(counts, []int{1}) { + t.Fatalf("version document reported %v, %v", counts, err) + } + options.DeniedVersions = map[string]bool{"1.0.0": true} + if _, _, err := Rewrite("@example/widgets", body, options); !errors.Is(err, ErrVersionDenied) { + t.Fatalf("denied version: %v", err) + } +} diff --git a/internal/metadata/npm/testdata/packument.json b/internal/metadata/npm/testdata/packument.json new file mode 100644 index 0000000..cdb36b5 --- /dev/null +++ b/internal/metadata/npm/testdata/packument.json @@ -0,0 +1,12 @@ +{ + "name": "@example/widgets", + "dist-tags": {"latest": "3.0.0", "beta": "2.0.0"}, + "versions": { + "1.0.0": {"name": "@example/widgets", "version": "1.0.0", "dependencies": {"left-pad": "^1.3.0"}, "dist": {"integrity": "sha512-example", "tarball": "https://registry.npmjs.org/@example/widgets/-/widgets-1.0.0.tgz"}}, + "2.0.0": {"name": "@example/widgets", "version": "2.0.0", "dist": {"tarball": "https://registry.npmjs.org/@example/widgets/-/widgets-2.0.0.tgz"}}, + "3.0.0": {"name": "@example/widgets", "version": "3.0.0", "dist": {"tarball": "https://cdn.example/abcdef.tgz"}} + }, + "time": {"created": "2020-01-01T00:00:00Z", "1.0.0": "2020-01-01T00:00:00Z", "2.0.0": "2021-01-01T00:00:00Z", "3.0.0": "2099-01-01T00:00:00Z"}, + "readme": "Widgets & tools", + "downloads": 12345678901234567890 +} From e070618021499c29ca4b725251448ba7066d56f6 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 25 Oct 2026 22:19:37 +0100 Subject: [PATCH 450/633] Move abbreviated packument detection into internal/metadata/npm Whether a packument is the abbreviated format is a property of the document, so it sits with the rest of the npm metadata code; the handler keeps only the error it turns that into. Add a test that keeps the package's imports to the portable set, give jsonscan a package comment, and describe the new packages in the architecture docs. --- docs/architecture.md | 12 +++++-- internal/handler/npm.go | 21 +++--------- internal/jsonscan/jsonscan.go | 20 +++++------ internal/metadata/npm/imports_test.go | 47 ++++++++++++++++++++++++++ internal/metadata/npm/metadata.go | 19 +++++++++++ internal/metadata/npm/metadata_test.go | 14 ++++++++ 6 files changed, 105 insertions(+), 28 deletions(-) create mode 100644 internal/metadata/npm/imports_test.go diff --git a/docs/architecture.md b/docs/architecture.md index e7ce8c8..660d810 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -34,7 +34,7 @@ The proxy is a caching HTTP server that sits between package manager clients and 1. Client requests `GET /npm/lodash` 2. NPMHandler receives request 3. Handler fetches metadata from upstream `registry.npmjs.org/lodash` -4. Handler rewrites tarball URLs in metadata to point at proxy +4. Handler rewrites tarball URLs in metadata to point at proxy, and filters versions withheld by cooldown or the denylist (`internal/metadata/npm`) 5. Handler returns modified metadata to client Metadata is not cached - always fetched fresh. This ensures clients see new versions immediately. @@ -294,7 +294,7 @@ HTTP protocol handlers for each registry type. **NPMHandler:** - `handlePackageMetadata()` - Proxy + rewrite metadata - `handleDownload()` - Serve cached artifact -- Rewrites tarball URLs to point at proxy +- Fetches, caches and serves packuments, and caches their rewrites; the rewrite itself is `internal/metadata/npm` **CargoHandler:** - `handleConfig()` - Return registry config @@ -310,6 +310,14 @@ HTTP protocol handlers for each registry type. - Validates repository paths and the `repos` allowlist, and maps cache errors to statuses - Serves the Git LFS API under `{repo}.git/info/lfs/`: forwards download batches upstream and points each object at `objects/{oid}/{size}`, which is served from the artifact cache (ecosystem `git`, name `{name}/{repo}`, version the object ID); a miss uses the download URL kept from the batch or asks the upstream batch API for a fresh one, recorded without its signed query, copies the object from another repository when its content is already stored, and checks the object against its sha256 ID; a batch the upstream can't answer is answered from the cache +### `internal/metadata/npm` + +The npm metadata rewrite, separated from the handler so it can run where the proxy's infrastructure can't, such as WASI. It takes a packument's bytes and everything else as explicit inputs (proxy URL, cooldown policy, the time cooldown is evaluated at, denied versions, pushed versions) and returns the rewritten document with the versions it kept. It filters versions by cooldown and the denylist, moves `latest` off a withheld version, lists pushed versions in place of upstream ones, and points each tarball URL at the proxy, copying everything else byte for byte. It does no I/O and never reads the clock; callbacks report what it did, which the handler logs. It imports no database, storage, metrics, HTTP or logging packages, which a test checks. + +### `internal/jsonscan` + +Reads and rewrites members of a JSON document in place, by byte offsets, without decoding the whole document. Shared by the npm and Composer metadata rewrites. + ### `internal/gitcache` Local bare clones of upstream Git repositories, outside the artifact cache and the database, at `{git.cache_dir}/{name}-{URL hash}/{repo}.git`. The hash of the upstream's URL means repointing an upstream starts a fresh cache; at startup only the configured upstreams' directories are loaded, and those of their old URLs removed. diff --git a/internal/handler/npm.go b/internal/handler/npm.go index 4ced2f2..c86dfd0 100644 --- a/internal/handler/npm.go +++ b/internal/handler/npm.go @@ -329,23 +329,12 @@ func (h *NPMHandler) packumentValidator() func([]byte) error { return requireFullPackument } -// requireFullPackument rejects an abbreviated packument: one with the -// top-level "modified" field the abbreviated format carries and no "time" -// map. A full document without a time map, as some registries serve, is -// let through as before, and so is anything that is not a JSON object, -// leaving it to the usual handling. +// requireFullPackument rejects an abbreviated packument (see +// npm.Abbreviated). A full document without a time map, as some registries +// serve, is let through as before, and so is anything that is not a JSON +// object, leaving it to the usual handling. func requireFullPackument(body []byte) error { - var hasTime, hasModified bool - err := jsonscan.EachMember(body, func(m jsonscan.Member) error { - switch key := m.Key(body); { - case jsonscan.KeyIs(key, "time"): - hasTime = true - case jsonscan.KeyIs(key, "modified"): - hasModified = true - } - return nil - }) - if err == nil && hasModified && !hasTime { + if npm.Abbreviated(body) { return errAbbreviatedPackument } return nil diff --git a/internal/jsonscan/jsonscan.go b/internal/jsonscan/jsonscan.go index 3478e69..7fcade7 100644 --- a/internal/jsonscan/jsonscan.go +++ b/internal/jsonscan/jsonscan.go @@ -1,3 +1,12 @@ +// Package jsonscan walks a JSON document in place, reporting where +// object members sit as offsets into the original bytes. It lets a caller +// read or replace one value in a large metadata document without decoding the +// rest of it into Go values, which for an npm packument costs many times the +// document's size. +// +// It checks structure only as far as it needs to find its way through +// the document. Callers that copy unvisited bytes into a response validate the +// whole document first with json.Valid. package jsonscan import ( @@ -6,16 +15,7 @@ import ( "errors" ) -// The helpers in this file walk a JSON document in place, reporting where -// object members sit as offsets into the original bytes. They let a handler -// read or replace one value in a large metadata document without decoding the -// rest of it into Go values, which for an npm packument costs many times the -// document's size. -// -// They check structure only as far as they need to find their way through -// the document. Callers that copy unvisited bytes into a response validate the -// whole document first with json.Valid. - +// ErrMalformed is returned when the document is not well-formed JSON. var ErrMalformed = errors.New("malformed JSON") // ErrNotObject is returned when a value expected to be an object is not. diff --git a/internal/metadata/npm/imports_test.go b/internal/metadata/npm/imports_test.go new file mode 100644 index 0000000..0f04bc1 --- /dev/null +++ b/internal/metadata/npm/imports_test.go @@ -0,0 +1,47 @@ +package npm + +import ( + "os/exec" + "slices" + "strings" + "testing" +) + +// TestPortableImports keeps the package free of the proxy's infrastructure: +// no handler, database, storage, metrics, HTTP or logging, so it builds for +// targets such as WASI where those are not available. +func TestPortableImports(t *testing.T) { + goTool, err := exec.LookPath("go") + if err != nil { + t.Skip("go tool not found") + } + out, err := exec.Command(goTool, "list", "-deps", "-f", "{{.Standard}} {{.ImportPath}}", ".").Output() + if err != nil { + t.Fatalf("go list: %v", err) + } + allowed := []string{ + "github.com/Masterminds/semver/v3", + "github.com/git-pkgs/cooldown", + "github.com/git-pkgs/proxy/internal/jsonscan", + "github.com/git-pkgs/proxy/internal/metadata/npm", + "github.com/git-pkgs/proxy/internal/packageurl", + "github.com/git-pkgs/purl", + "github.com/git-pkgs/vers", + "github.com/package-url/packageurl-go", + } + // database/sql/driver comes in through semver, which implements its + // Valuer interface; it is only interfaces, no driver. + forbiddenStd := []string{"database/sql", "log/slog", "net", "net/http"} + for line := range strings.Lines(string(out)) { + standard, path, _ := strings.Cut(strings.TrimSpace(line), " ") + if standard == "true" { + if slices.Contains(forbiddenStd, path) { + t.Errorf("imports %s", path) + } + continue + } + if !slices.Contains(allowed, path) { + t.Errorf("imports %s, which is not on the portable list", path) + } + } +} diff --git a/internal/metadata/npm/metadata.go b/internal/metadata/npm/metadata.go index dcf0347..62e531f 100644 --- a/internal/metadata/npm/metadata.go +++ b/internal/metadata/npm/metadata.go @@ -91,6 +91,25 @@ func Version(packageName, specifier string, packument []byte, options Options) ( return out.Bytes(), nil } +// Abbreviated reports whether body is an abbreviated packument: one with the +// top-level "modified" field the abbreviated format carries and no "time" +// map, which cooldown needs. A full document without a time map, as some +// registries serve, is not abbreviated, and neither is anything that is not +// a JSON object. +func Abbreviated(body []byte) bool { + var hasTime, hasModified bool + err := jsonscan.EachMember(body, func(m jsonscan.Member) error { + switch key := m.Key(body); { + case jsonscan.KeyIs(key, "time"): + hasTime = true + case jsonscan.KeyIs(key, "modified"): + hasModified = true + } + return nil + }) + return err == nil && hasModified && !hasTime +} + func (h *transformer) rewriteMetadataWithLocal(packageName string, body []byte, local []LocalVersion) ([]byte, []string, error) { doc, err := indexNPMPackument(body) if err != nil { diff --git a/internal/metadata/npm/metadata_test.go b/internal/metadata/npm/metadata_test.go index a1e5e4b..5d68dd6 100644 --- a/internal/metadata/npm/metadata_test.go +++ b/internal/metadata/npm/metadata_test.go @@ -151,3 +151,17 @@ func TestNPMRewriteMetadataDuplicateKeys(t *testing.T) { t.Errorf("rewrite = %s", out) } } + +func TestAbbreviated(t *testing.T) { + for body, want := range map[string]bool{ + `{"name":"demo","modified":"2024-01-01T00:00:00Z","versions":{}}`: true, + `{"name":"demo","modified":"2024-01-01T00:00:00Z","time":{},"versions":{}}`: false, + `{"name":"demo","versions":{}}`: false, + `["modified"]`: false, + `{"modified":`: false, + } { + if got := Abbreviated([]byte(body)); got != want { + t.Errorf("Abbreviated(%s) = %v, want %v", body, got, want) + } + } +} From 24acee273118d841911d9b04998dd6d18cc88e75 Mon Sep 17 00:00:00 2001 From: Christian Heim Date: Mon, 26 Oct 2026 08:49:50 +0100 Subject: [PATCH 451/633] When a refetched file can be neither recorded nor queued for deletion, it is still served to everyone sharing the fetch and deleted once the last of them has closed it, so it no longer stays in storage with nothing pointing at it. A record that committed despite the reported error keeps its file. --- internal/handler/handler.go | 134 +++++++++++- internal/handler/unrecorded_test.go | 304 ++++++++++++++++++++++++++++ 2 files changed, 428 insertions(+), 10 deletions(-) create mode 100644 internal/handler/unrecorded_test.go diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 62d63ae..dd880db 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -14,6 +14,7 @@ import ( "strconv" "strings" "sync" + "sync/atomic" "time" "github.com/git-pkgs/artifacts" @@ -617,14 +618,79 @@ func (p *Proxy) storeArtifact(ctx context.Context, ecosystem, name, version, fil p.Logger.Warn("failed to update cache database", "error", err) // Continue anyway - we have the file. Queue it for deletion in case // no record points at it; the queue skips it while one does. - if qErr := p.DB.QueuePendingDelete(storagePath); qErr != nil { - p.Logger.Warn("failed to queue unrecorded artifact for deletion", "path", storagePath, "error", qErr) + if qErr := p.DB.QueuePendingDelete(storagePath); qErr != nil && !p.recordedAt(versionPURL, filename, storagePath) { + p.Logger.Warn("failed to queue unrecorded artifact for deletion, deleting it once served", + "path", storagePath, "error", qErr) + return sharedArtifact, storagePath, errUnrecorded } } return sharedArtifact, storagePath, nil } +// recordedAt reports whether the artifact's record points at storagePath. A +// commit can apply and still report an error, so storeArtifact checks before +// treating an object as unrecorded. A failed lookup counts as not recorded. +func (p *Proxy) recordedAt(versionPURL, filename, storagePath string) bool { + a, err := p.DB.GetArtifact(versionPURL, filename) + return err == nil && a != nil && a.StoragePath.Valid && a.StoragePath.String == storagePath +} + +// errUnrecorded comes back from storeArtifact together with a usable artifact +// and path when neither a record nor a queue entry could be written for it. +// Nothing else would ever delete the object, so the fetch serves it to +// everyone sharing it and deletes it when the last of them is done. +var errUnrecorded = errors.New("stored artifact has no record") + +// unrecordedObject deletes an unrecorded object once every caller sharing +// its fetch has released it. +type unrecordedObject struct { + refs atomic.Int32 + delete func() +} + +func (u *unrecordedObject) release() { + if u.refs.Add(-1) == 0 { + u.delete() + } +} + +// releaseOnClose releases an unrecorded object once its reader is closed. +type releaseOnClose struct { + io.ReadCloser + release func() +} + +func (r *releaseOnClose) Close() error { + err := r.ReadCloser.Close() + r.release() + return err +} + +// seekableReleaseOnClose keeps range requests working on a seekable reader. +type seekableReleaseOnClose struct { + releaseOnClose + io.Seeker +} + +// serveUnrecorded gives one caller its own reader over an unrecorded object +// and releases the caller's hold on it when that reader closes, or right away +// if it can't be opened. +func (p *Proxy) serveUnrecorded(ctx context.Context, u *unrecordedObject, artifact artifacts.Artifact, path string) (*CacheResult, error) { + res, err := p.openStoredArtifact(ctx, artifact, path) + if err != nil { + u.release() + return nil, err + } + r := releaseOnClose{ReadCloser: res.Reader, release: u.release} + if s, ok := res.Reader.(io.Seeker); ok { + res.Reader = &seekableReleaseOnClose{releaseOnClose: r, Seeker: s} + } else { + res.Reader = &r + } + return res, nil +} + // openStoredArtifact gives one caller its own reader over just-committed // bytes. Callers sharing a fetch cannot share a handle: it has one read // position, so they would consume each other's bytes and the first Close would @@ -676,13 +742,17 @@ func (p *Proxy) cachedArtifactRecord(pkgPURL, versionPURL, filename, upstreamHas // fetch panicked out of it. var errSharedFetchAbandoned = errors.New("shared upstream fetch did not complete") -// inflightFetch is one upstream fetch that concurrent callers share. val and -// err are written before done closes and read only after, so the close is the -// handoff. +// inflightFetch is one upstream fetch that concurrent callers share. val, err +// and unrecorded are written before done closes and read only after, so the +// close is the handoff. waiters counts callers waiting on it, guarded by +// fetchMu; done closes under fetchMu too, so a waiter leaving early knows +// whether it was counted. type inflightFetch struct { - done chan struct{} - val fetchedArtifact - err error + done chan struct{} + val fetchedArtifact + err error + unrecorded *unrecordedObject + waiters int } // coalesceFetch runs commit at most once for concurrent callers sharing key, @@ -715,7 +785,9 @@ func (p *Proxy) coalesceFetch(ctx context.Context, key string, recheck func() (a p.inFlight = make(map[string]*inflightFetch) } f, joined := p.inFlight[key] - if !joined { + if joined { + f.waiters++ + } else { f = &inflightFetch{done: make(chan struct{})} p.inFlight[key] = f } @@ -728,15 +800,35 @@ func (p *Proxy) coalesceFetch(ctx context.Context, key string, recheck func() (a select { case <-ctx.Done(): // This caller gave up; the fetch continues for everyone else. + p.leaveFetch(f) return nil, ctx.Err() case <-f.done: } if f.err != nil { return nil, f.err } + if f.unrecorded != nil { + return p.serveUnrecorded(ctx, f.unrecorded, f.val.artifact, f.val.storagePath) + } return p.openStoredArtifact(ctx, f.val.artifact, f.val.storagePath) } +// leaveFetch withdraws a waiter that gave up. Once the fetch has finished the +// waiter was counted as holding any unrecorded object, so it releases it. +func (p *Proxy) leaveFetch(f *inflightFetch) { + p.fetchMu.Lock() + select { + case <-f.done: + p.fetchMu.Unlock() + if f.unrecorded != nil { + f.unrecorded.release() + } + default: + f.waiters-- + p.fetchMu.Unlock() + } +} + // runSharedFetch performs the fetch that joined callers are waiting on. It is // never abandoned early, and always releases the key and wakes the waiters. func (p *Proxy) runSharedFetch(ctx context.Context, key string, f *inflightFetch, recheck func() (artifacts.Artifact, string, bool), commit func(context.Context) (artifacts.Artifact, string, error)) (*CacheResult, error) { @@ -746,8 +838,14 @@ func (p *Proxy) runSharedFetch(ctx context.Context, key string, f *inflightFetch defer func() { p.fetchMu.Lock() delete(p.inFlight, key) - p.fetchMu.Unlock() + if f.unrecorded != nil { + // Hand each waiter a hold, then drop the one kept while the + // waiters were still unknown. + f.unrecorded.refs.Add(int32(f.waiters)) + defer f.unrecorded.release() + } close(f.done) + p.fetchMu.Unlock() }() // A caller checks the cache before reaching here, so a fetch that finished @@ -767,13 +865,29 @@ func (p *Proxy) runSharedFetch(ctx context.Context, key string, f *inflightFetch commitCtx = context.WithoutCancel(ctx) } stored, path, err := commit(commitCtx) + unrecorded := errors.Is(err, errUnrecorded) + if unrecorded { + err = nil + } f.val, f.err = fetchedArtifact{artifact: stored, storagePath: path}, err if err != nil { return nil, err } + if unrecorded { + f.unrecorded = &unrecordedObject{delete: func() { p.deleteUnrecorded(path) }} + f.unrecorded.refs.Add(1) // this caller + f.unrecorded.refs.Add(1) // until the waiters are counted + return p.serveUnrecorded(ctx, f.unrecorded, stored, path) + } return p.openStoredArtifact(ctx, stored, path) } +func (p *Proxy) deleteUnrecorded(path string) { + if err := p.Storage.Delete(context.Background(), path); err != nil { + p.Logger.Warn("failed to delete unrecorded artifact", "path", path, "error", err) + } +} + type detachUpstreamFetchKey struct{} // DetachUpstreamFetch marks ctx so that an artifact fetch started under it diff --git a/internal/handler/unrecorded_test.go b/internal/handler/unrecorded_test.go new file mode 100644 index 0000000..87b6a26 --- /dev/null +++ b/internal/handler/unrecorded_test.go @@ -0,0 +1,304 @@ +package handler + +import ( + "context" + "errors" + "io" + "maps" + "net/http" + "net/http/httptest" + "slices" + "sync" + "testing" + "time" +) + +// These tests refetch an artifact whose stored file is gone while +// pending_deletes refuses writes. The record can't move to the new file and +// the file can't be queued, so the fetch must serve it to every caller sharing +// it and delete it once the last of them is done. + +const ( + unrecordedContent = "refetched content" + unrecordedURL = "https://example.com/gone.tgz" +) + +func setupUnrecorded(t *testing.T) (*Proxy, *mockStorage) { + t.Helper() + proxy, db, store, _ := setupTestProxy(t) + seedPackage(t, db, store, "npm", "gone", "1.0.0", "gone-1.0.0.tgz", "old content") + oldPath := recordedStoragePath(t, db, "pkg:npm/gone@1.0.0", "gone-1.0.0.tgz") + delete(store.files, oldPath) + if _, err := db.Exec(`CREATE TRIGGER refuse_queue BEFORE INSERT ON pending_deletes + BEGIN SELECT RAISE(ABORT, 'queue refused'); END`); err != nil { + t.Fatalf("creating trigger: %v", err) + } + t.Cleanup(func() { + if got := recordedStoragePath(t, db, "pkg:npm/gone@1.0.0", "gone-1.0.0.tgz"); got != oldPath { + t.Errorf("record points at %q, want it kept at %q", got, oldPath) + } + }) + return proxy, store +} + +func fetchGone(ctx context.Context, proxy *Proxy) (*CacheResult, error) { + return proxy.GetOrFetchArtifactFromURL(ctx, "npm", "gone", "1.0.0", "gone-1.0.0.tgz", unrecordedURL) +} + +func storedFiles(store *mockStorage) []string { + store.mu.Lock() + defer store.mu.Unlock() + return slices.Collect(maps.Keys(store.files)) +} + +func readAndClose(t *testing.T, res *CacheResult) { + t.Helper() + body, err := io.ReadAll(res.Reader) + _ = res.Reader.Close() + if err != nil || string(body) != unrecordedContent { + t.Errorf("body = %q, %v, want %q", body, err, unrecordedContent) + } +} + +// waitForWaiters blocks until n callers wait on the one fetch in flight. +func waitForWaiters(t *testing.T, proxy *Proxy, n int) { + t.Helper() + deadline := time.Now().Add(5 * time.Second) + for time.Now().Before(deadline) { + proxy.fetchMu.Lock() + got := -1 + for _, f := range proxy.inFlight { + got = f.waiters + } + proxy.fetchMu.Unlock() + if got == n { + return + } + time.Sleep(time.Millisecond) + } + t.Fatalf("never saw %d waiters on the fetch", n) +} + +func TestUnrecordedRefetchIsDeletedOnceServed(t *testing.T) { + proxy, store := setupUnrecorded(t) + fetcher := &countingFetcher{content: unrecordedContent} + proxy.Fetcher = fetcher + + for range 2 { + res, err := fetchGone(context.Background(), proxy) + if err != nil { + t.Fatalf("fetch: %v", err) + } + if got := storedFiles(store); len(got) != 1 { + t.Errorf("stored files while serving = %v, want the refetched one", got) + } + readAndClose(t, res) + if got := storedFiles(store); len(got) != 0 { + t.Errorf("stored files after serving = %v, want none left untracked", got) + } + } +} + +func TestUnrecordedRefetchServesRangeRequests(t *testing.T) { + proxy, store := setupUnrecorded(t) + store.seekable = true + proxy.Fetcher = &countingFetcher{content: unrecordedContent} + + res, err := fetchGone(context.Background(), proxy) + if err != nil { + t.Fatalf("fetch: %v", err) + } + req := httptest.NewRequest(http.MethodGet, "/gone-1.0.0.tgz", nil) + req.Header.Set("Range", "bytes=2-5") + rec := httptest.NewRecorder() + ServeArtifactRequest(rec, req, res) + + if rec.Code != http.StatusPartialContent || rec.Body.String() != unrecordedContent[2:6] { + t.Errorf("range response = %d %q, want 206 %q", rec.Code, rec.Body.String(), unrecordedContent[2:6]) + } + if got := storedFiles(store); len(got) != 0 { + t.Errorf("stored files after serving = %v, want none left untracked", got) + } +} + +// TestUnrecordedRefetchWaitsForEveryCaller keeps the object until the caller +// that ran the fetch and every caller that waited on it have closed. +func TestUnrecordedRefetchWaitsForEveryCaller(t *testing.T) { + const waiters = 4 + proxy, store := setupUnrecorded(t) + fetcher := newGatedFetcher(unrecordedContent) + proxy.Fetcher = fetcher + + leader := make(chan *CacheResult, 1) + go func() { + res, err := fetchGone(context.Background(), proxy) + if err != nil { + t.Errorf("leader: %v", err) + } + leader <- res + }() + <-fetcher.entered + + results := make(chan *CacheResult, waiters) + var wg sync.WaitGroup + for range waiters { + wg.Go(func() { + res, err := fetchGone(context.Background(), proxy) + if err != nil { + t.Errorf("waiter: %v", err) + return + } + results <- res + }) + } + waitForWaiters(t, proxy, waiters) + close(fetcher.release) + + // The leader holds its reader open while every waiter reads and closes. + lead := <-leader + wg.Wait() + close(results) + for res := range results { + readAndClose(t, res) + } + if got := storedFiles(store); len(got) != 1 { + t.Fatalf("stored files with the leader still reading = %v, want the refetched one", got) + } + readAndClose(t, lead) + if got := storedFiles(store); len(got) != 0 { + t.Errorf("stored files after every caller closed = %v, want none", got) + } +} + +// ctxStorage fails to open under a cancelled context, as cloud backends do. +type ctxStorage struct{ *mockStorage } + +func (s ctxStorage) Open(ctx context.Context, path string) (io.ReadCloser, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + return s.mockStorage.Open(ctx, path) +} + +// TestUnrecordedRefetchOutlivesDetachedCaller is the retry a detached fetch +// exists for: the client that started it gave up, and its retry waits on the +// same fetch. The retry must get the bytes rather than a deleted object. +func TestUnrecordedRefetchOutlivesDetachedCaller(t *testing.T) { + proxy, store := setupUnrecorded(t) + proxy.Storage = ctxStorage{store} + fetcher := newGatedFetcher(unrecordedContent) + proxy.Fetcher = fetcher + + ctx, cancel := context.WithCancel(DetachUpstreamFetch(context.Background())) + leaderDone := make(chan struct{}) + go func() { + defer close(leaderDone) + res, _ := fetchGone(ctx, proxy) + drain(res) + }() + <-fetcher.entered + cancel() + + retry := make(chan *CacheResult, 1) + go func() { + res, err := fetchGone(context.Background(), proxy) + if err != nil { + t.Errorf("retry: %v", err) + } + retry <- res + }() + waitForWaiters(t, proxy, 1) + close(fetcher.release) + <-leaderDone + + if res := <-retry; res != nil { + readAndClose(t, res) + } + if got := storedFiles(store); len(got) != 0 { + t.Errorf("stored files after the retry closed = %v, want none", got) + } +} + +// TestUnrecordedRefetchIgnoresWaiterThatLeft checks that a waiter giving up +// before the fetch ends does not keep the object forever. +func TestUnrecordedRefetchIgnoresWaiterThatLeft(t *testing.T) { + proxy, store := setupUnrecorded(t) + fetcher := newGatedFetcher(unrecordedContent) + proxy.Fetcher = fetcher + + leader := make(chan *CacheResult, 1) + go func() { + res, err := fetchGone(context.Background(), proxy) + if err != nil { + t.Errorf("leader: %v", err) + } + leader <- res + }() + <-fetcher.entered + + ctx, cancel := context.WithCancel(context.Background()) + left := make(chan error, 1) + go func() { + _, err := fetchGone(ctx, proxy) + left <- err + }() + waitForWaiters(t, proxy, 1) + cancel() + if err := <-left; !errors.Is(err, context.Canceled) { + t.Fatalf("waiter that left: err = %v, want context.Canceled", err) + } + close(fetcher.release) + + readAndClose(t, <-leader) + if got := storedFiles(store); len(got) != 0 { + t.Errorf("stored files after the leader closed = %v, want none", got) + } +} + +// TestLeaveFetch covers both ways a waiter can give up: before the fetch +// finished, when it was not handed a hold yet, and after, when it was. +func TestLeaveFetch(t *testing.T) { + proxy, _, _, _ := setupTestProxy(t) + deleted := false + newFetch := func() *inflightFetch { + f := &inflightFetch{done: make(chan struct{}), waiters: 1} + f.unrecorded = &unrecordedObject{delete: func() { deleted = true }} + f.unrecorded.refs.Add(1) + return f + } + + f := newFetch() + proxy.leaveFetch(f) + if f.waiters != 0 || deleted { + t.Errorf("before done: waiters = %d, deleted = %v, want 0 and false", f.waiters, deleted) + } + + f = newFetch() + close(f.done) + proxy.leaveFetch(f) + if f.waiters != 1 || !deleted { + t.Errorf("after done: waiters = %d, deleted = %v, want 1 and true", f.waiters, deleted) + } +} + +// TestRecordedAt is the check that keeps an object whose record update +// committed despite reporting an error. +func TestRecordedAt(t *testing.T) { + proxy, db, store, _ := setupTestProxy(t) + seedPackage(t, db, store, "npm", "kept", "1.0.0", "kept-1.0.0.tgz", "content") + path := recordedStoragePath(t, db, "pkg:npm/kept@1.0.0", "kept-1.0.0.tgz") + + if !proxy.recordedAt("pkg:npm/kept@1.0.0", "kept-1.0.0.tgz", path) { + t.Error("record pointing at the path not seen") + } + if proxy.recordedAt("pkg:npm/kept@1.0.0", "kept-1.0.0.tgz", "other/path.tgz") { + t.Error("record pointing elsewhere taken as recorded") + } + if proxy.recordedAt("pkg:npm/none@1.0.0", "none-1.0.0.tgz", path) { + t.Error("missing record taken as recorded") + } + _ = db.Close() + if proxy.recordedAt("pkg:npm/kept@1.0.0", "kept-1.0.0.tgz", path) { + t.Error("failed lookup taken as recorded") + } +} From 257d159ecf22633da5ffd2e121907d663b0cba84 Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Mon, 26 Oct 2026 10:19:24 +0000 Subject: [PATCH 452/633] Release an unrecorded object's hold only once per reader --- internal/handler/handler.go | 3 ++- internal/handler/unrecorded_test.go | 38 +++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 1 deletion(-) diff --git a/internal/handler/handler.go b/internal/handler/handler.go index a4007fc..fc5b912 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -802,7 +802,8 @@ func (p *Proxy) serveUnrecorded(ctx context.Context, u *unrecordedObject, artifa u.release() return nil, err } - r := releaseOnClose{ReadCloser: res.Reader, release: u.release} + // Once, so a second Close can't drop another caller's hold. + r := releaseOnClose{ReadCloser: res.Reader, release: sync.OnceFunc(u.release)} if s, ok := res.Reader.(io.Seeker); ok { res.Reader = &seekableReleaseOnClose{releaseOnClose: r, Seeker: s} } else { diff --git a/internal/handler/unrecorded_test.go b/internal/handler/unrecorded_test.go index 87b6a26..3533a10 100644 --- a/internal/handler/unrecorded_test.go +++ b/internal/handler/unrecorded_test.go @@ -170,6 +170,44 @@ func TestUnrecordedRefetchWaitsForEveryCaller(t *testing.T) { } } +func TestUnrecordedRefetchSurvivesDoubleClose(t *testing.T) { + proxy, store := setupUnrecorded(t) + fetcher := newGatedFetcher(unrecordedContent) + proxy.Fetcher = fetcher + + leader := make(chan *CacheResult, 1) + go func() { + res, err := fetchGone(context.Background(), proxy) + if err != nil { + t.Errorf("leader: %v", err) + } + leader <- res + }() + <-fetcher.entered + + waiter := make(chan *CacheResult, 1) + go func() { + res, err := fetchGone(context.Background(), proxy) + if err != nil { + t.Errorf("waiter: %v", err) + } + waiter <- res + }() + waitForWaiters(t, proxy, 1) + close(fetcher.release) + + lead, wait := <-leader, <-waiter + readAndClose(t, lead) + _ = lead.Reader.Close() + if got := storedFiles(store); len(got) != 1 { + t.Fatalf("stored files with the waiter still reading = %v, want the refetched one", got) + } + readAndClose(t, wait) + if got := storedFiles(store); len(got) != 0 { + t.Errorf("stored files after every caller closed = %v, want none", got) + } +} + // ctxStorage fails to open under a cancelled context, as cloud backends do. type ctxStorage struct{ *mockStorage } From 391c5b709fc6c3a43b72c67af58364313903f34c Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Mon, 26 Oct 2026 16:40:15 +0100 Subject: [PATCH 453/633] Index artifacts by filename and shorten Julia comments CachedArtifactNames filters on filename alone, which the unique (version_purl, filename) index can't serve. --- internal/database/retention_test.go | 33 +++++++++++++++++++++++++++++ internal/database/schema.go | 15 +++++++++++++ internal/handler/julia.go | 4 +--- internal/mirror/julia.go | 8 ++----- 4 files changed, 51 insertions(+), 9 deletions(-) diff --git a/internal/database/retention_test.go b/internal/database/retention_test.go index 213ae22..bd9ad28 100644 --- a/internal/database/retention_test.go +++ b/internal/database/retention_test.go @@ -394,3 +394,36 @@ func TestGetRetentionCandidatesTimezone(t *testing.T) { } }) } + +func TestArtifactFilenameIndex(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + indexQuery := `SELECT COUNT(*) FROM sqlite_master WHERE type = 'index' AND name = 'idx_artifacts_filename'` + if db.Dialect() == DialectPostgres { + indexQuery = `SELECT COUNT(*) FROM pg_indexes WHERE indexname = 'idx_artifacts_filename'` + } + hasIndex := func() bool { + t.Helper() + var count int + if err := db.Get(&count, indexQuery); err != nil { + t.Fatalf("querying indexes: %v", err) + } + return count == 1 + } + + if !hasIndex() { + t.Error("fresh schema has no idx_artifacts_filename") + } + if _, err := db.Exec(`DROP INDEX idx_artifacts_filename`); err != nil { + t.Fatalf("dropping index: %v", err) + } + if err := migrateAddArtifactFilenameIndex(db); err != nil { + t.Fatalf("migration: %v", err) + } + if err := migrateAddArtifactFilenameIndex(db); err != nil { + t.Fatalf("migration is not idempotent: %v", err) + } + if !hasIndex() { + t.Error("migration did not create idx_artifacts_filename") + } + }) +} diff --git a/internal/database/schema.go b/internal/database/schema.go index 46a21ec..f0ec651 100644 --- a/internal/database/schema.go +++ b/internal/database/schema.go @@ -80,6 +80,7 @@ CREATE TABLE IF NOT EXISTS artifacts ( CREATE UNIQUE INDEX IF NOT EXISTS idx_artifacts_version_filename ON artifacts(version_purl, filename); CREATE INDEX IF NOT EXISTS idx_artifacts_storage_path ON artifacts(storage_path); CREATE INDEX IF NOT EXISTS idx_artifacts_content_hash ON artifacts(content_hash); +CREATE INDEX IF NOT EXISTS idx_artifacts_filename ON artifacts(filename); CREATE INDEX IF NOT EXISTS idx_artifacts_last_accessed ON artifacts(last_accessed_at); CREATE TABLE IF NOT EXISTS pending_deletes ( @@ -222,6 +223,7 @@ CREATE TABLE IF NOT EXISTS artifacts ( CREATE UNIQUE INDEX IF NOT EXISTS idx_artifacts_version_filename ON artifacts(version_purl, filename); CREATE INDEX IF NOT EXISTS idx_artifacts_storage_path ON artifacts(storage_path); CREATE INDEX IF NOT EXISTS idx_artifacts_content_hash ON artifacts(content_hash); +CREATE INDEX IF NOT EXISTS idx_artifacts_filename ON artifacts(filename); CREATE INDEX IF NOT EXISTS idx_artifacts_last_accessed ON artifacts(last_accessed_at); CREATE TABLE IF NOT EXISTS pending_deletes ( @@ -323,6 +325,7 @@ CREATE TABLE IF NOT EXISTS artifacts ( CREATE UNIQUE INDEX IF NOT EXISTS idx_artifacts_version_filename ON artifacts(version_purl, filename); CREATE INDEX IF NOT EXISTS idx_artifacts_storage_path ON artifacts(storage_path); CREATE INDEX IF NOT EXISTS idx_artifacts_content_hash ON artifacts(content_hash); +CREATE INDEX IF NOT EXISTS idx_artifacts_filename ON artifacts(filename); CREATE INDEX IF NOT EXISTS idx_artifacts_last_accessed ON artifacts(last_accessed_at); ` @@ -347,6 +350,7 @@ CREATE TABLE IF NOT EXISTS artifacts ( CREATE UNIQUE INDEX IF NOT EXISTS idx_artifacts_version_filename ON artifacts(version_purl, filename); CREATE INDEX IF NOT EXISTS idx_artifacts_storage_path ON artifacts(storage_path); CREATE INDEX IF NOT EXISTS idx_artifacts_content_hash ON artifacts(content_hash); +CREATE INDEX IF NOT EXISTS idx_artifacts_filename ON artifacts(filename); CREATE INDEX IF NOT EXISTS idx_artifacts_last_accessed ON artifacts(last_accessed_at); ` @@ -463,6 +467,7 @@ var migrations = []migration{ {"016_add_fetch_leases", migrateAddFetchLeases}, {"017_add_artifact_content_hash_index", migrateAddArtifactContentHashIndex}, {"018_add_upstream_holds", migrateAddUpstreamHolds}, + {"019_add_artifact_filename_index", migrateAddArtifactFilenameIndex}, } // isTableNotFound returns true if the error indicates a missing table. @@ -835,6 +840,16 @@ func migrateAddArtifactContentHashIndex(db *DB) error { return nil } +// migrateAddArtifactFilenameIndex indexes artifacts by filename, for +// lookups that don't know the version, such as CachedArtifactNames. +func migrateAddArtifactFilenameIndex(db *DB) error { + _, err := db.Exec(`CREATE INDEX IF NOT EXISTS idx_artifacts_filename ON artifacts(filename)`) + if err != nil { + return fmt.Errorf("creating artifacts filename index: %w", err) + } + return nil +} + // migrateAddMetadataDates creates the table of Last-Modified dates for // rewritten metadata, which replicas sharing the database agree on. func migrateAddMetadataDates(db *DB) error { diff --git a/internal/handler/julia.go b/internal/handler/julia.go index 6ec0209..dbac313 100644 --- a/internal/handler/julia.go +++ b/internal/handler/julia.go @@ -184,9 +184,7 @@ func (h *JuliaHandler) cachedPackage(ctx context.Context, hash string) *CacheRes // JuliaPackageDownload returns the URL of a package's tarball on the Pkg // server at upstreamURL, or the default one when empty, and the file name the -// Julia handler caches it under. The handler caches a package tarball as -// version treeHash, the git tree hash of the package's source, under the -// package's name in the General registry. +// Julia handler caches it under. func JuliaPackageDownload(upstreamURL, uuid, treeHash string) (url, filename string, err error) { if !validJuliaUUID(uuid) { return "", "", fmt.Errorf("invalid julia package uuid %q", uuid) diff --git a/internal/mirror/julia.go b/internal/mirror/julia.go index 30efb68..613ba76 100644 --- a/internal/mirror/julia.go +++ b/internal/mirror/julia.go @@ -25,12 +25,8 @@ func (r RegistryVersions) FindPackage(ctx context.Context, ecosystem, name strin return reg.FetchPackage(ctx, name) } -// mirrorJulia mirrors the source tarball of a version of a package in the -// General registry from the Pkg server, under the name and tree hash the -// Julia handler caches it as. The registry gives the tree hash, and the -// tarball is fetched unchecked, as the handler fetches it: Pkg checks the -// tree hash of what it extracts. The registry and the package's artifacts -// are not mirrored. +// mirrorJulia fetches the tarball unchecked because Pkg checks the tree hash +// of what it extracts. func (m *Mirror) mirrorJulia(ctx context.Context, pv PackageVersion, tracker *progressTracker) { fail := func(err error) { m.recordFailure(tracker, pv, err) } uuid := strings.ToLower(pv.Qualifiers["uuid"]) From 8ea75a30a313e8ae8cc252a95d0a4d2d5f3ec29e Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Mon, 26 Oct 2026 16:43:42 +0100 Subject: [PATCH 454/633] Check cached Swift documents against their digest and resolve source PURLs Release lists and manifests record their sha256 digest and are read back through storedMetadataEntry and openMetadataEntry, so a corrupted copy is fetched again. pkg:swift PURLs naming a source repository are looked up through the registry's /identifiers. --- docs/configuration.md | 2 +- internal/handler/swift_cache.go | 33 ++++++++---------- internal/handler/swift_cache_test.go | 48 ++++++++++++++++++++++++++ internal/handler/swift_mirror.go | 46 +++++++++++++++++++++---- internal/mirror/swift.go | 51 +++++++++++++++------------- internal/mirror/swift_test.go | 33 ++++++++++++++++-- internal/mirror/versions.go | 49 ++++++++++++++++++++++---- internal/server/mirror_swift_test.go | 30 ++++++++++++---- 8 files changed, 227 insertions(+), 65 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index afd08d2..6939865 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1571,7 +1571,7 @@ A Composer release is mirrored with the dist archive `upstream.composer` lists f A Helm chart is mirrored from a repository in `upstream.helm`, named as the namespace, for example `pkg:helm/prometheus-community/prometheus@27.1.0` with `prometheus-community` configured. The chart is fetched from the first URL the repository's `index.yaml` lists for that version, checked against the listed digest, and cached under the repository and digest `/helm/{name}/charts/...` serves it from. The index is cached as well, so with [`cache_metadata`](#metadata-caching) on `helm pull` works while the repository is down. Without a version every version in the index is mirrored. Charts named without a repository, or from one that isn't configured, are refused, and so are charts the index lists in an OCI registry: helm pulls those through `/v2`, so mirror them as `pkg:oci` images. -A Swift release is mirrored from `upstream.swift` with its source archive, checked against the SHA-256 checksum in the release metadata and cached under the key `/swift/{scope}/{name}/{version}.zip` serves. Give the registry scope as the namespace, for example `pkg:swift/apple/swift-argument-parser@1.5.0` for `apple.swift-argument-parser`. A PURL whose namespace is a source repository, such as `pkg:swift/github.com/apple/swift-argument-parser`, doesn't name a registry identity and is refused. The release metadata, the package's release list and the release's `Package.swift` with its version-specific manifests are stored in the metadata cache too, so `swift package resolve` works with the upstream unreachable. Serving them, and the archive, offline needs `cache_metadata` on the server, since the archive is served with the checksum from the release metadata. +A Swift release is mirrored from `upstream.swift` with its source archive, checked against the SHA-256 checksum in the release metadata and cached under the key `/swift/{scope}/{name}/{version}.zip` serves. Give the registry scope as the namespace, for example `pkg:swift/apple/swift-argument-parser@1.5.0` for `apple.swift-argument-parser`, or the source repository, as SBOM tools write it: `pkg:swift/github.com/apple/swift-argument-parser@1.5.0` is looked up through the upstream's `/identifiers`, and the first identity it lists is mirrored. The release metadata, the package's release list and the release's `Package.swift` with its version-specific manifests are stored in the metadata cache too, so `swift package resolve` works with the upstream unreachable. Serving them, and the archive, offline needs `cache_metadata` on the server, since the archive is served with the checksum from the release metadata. | Flag | Default | Description | |------|---------|-------------| diff --git a/internal/handler/swift_cache.go b/internal/handler/swift_cache.go index e149331..d778179 100644 --- a/internal/handler/swift_cache.go +++ b/internal/handler/swift_cache.go @@ -12,12 +12,6 @@ import ( "github.com/git-pkgs/proxy/internal/database" ) -// With cache_metadata, release lists and manifests are kept in the metadata -// cache along with the upstream Link header, which carries the pagination of -// release lists and the version-specific manifests a client picks from. They -// are served from the cache within metadata_ttl, and after it when the -// upstream fails, as release metadata is. - // swiftDocument is a release list or manifest as the upstream sent it. type swiftDocument struct { body []byte @@ -30,7 +24,9 @@ type swiftDocument struct { // fetchDocument returns the document at upstreamURL, from the metadata cache // under cacheKey while it is fresh or when the upstream fails, and otherwise -// from upstream, storing it. +// from upstream, storing it. The upstream Link header is cached with it, +// since it carries the pagination of release lists and the version-specific +// manifests a client picks from. func (h *SwiftHandler) fetchDocument(ctx context.Context, cacheKey, upstreamURL, accept string) (*swiftDocument, error) { cached := h.loadDocument(ctx, cacheKey) if cached != nil && h.proxy.MetadataTTL > 0 && time.Since(cached.fetchedAt) < h.proxy.MetadataTTL { @@ -64,11 +60,11 @@ func (h *SwiftHandler) loadDocument(ctx context.Context, cacheKey string) *swift if !h.documentsCached() { return nil } - entry, err := h.proxy.DB.GetMetadataCache("swift", cacheKey) - if err != nil || entry == nil || !entry.FetchedAt.Valid { + entry := h.proxy.storedMetadataEntry("swift", cacheKey, true) + if entry == nil { return nil } - reader, err := h.proxy.Storage.Open(ctx, entry.StoragePath) + reader, err := h.proxy.openMetadataEntry(ctx, "swift", cacheKey, entry) if err != nil { return nil } @@ -93,16 +89,17 @@ func (h *SwiftHandler) storeDocument(ctx context.Context, cacheKey string, doc * return } storagePath := metadataStoragePath("swift", cacheKey) - size, _, err := h.proxy.Storage.Store(ctx, storagePath, bytes.NewReader(doc.body)) + size, hash, err := h.proxy.Storage.Store(ctx, storagePath, bytes.NewReader(doc.body)) if err == nil { err = h.proxy.DB.UpsertMetadataCache(&database.MetadataCacheEntry{ - Ecosystem: "swift", - Name: cacheKey, - StoragePath: storagePath, - Link: sql.NullString{String: doc.link, Valid: doc.link != ""}, - ContentType: sql.NullString{String: doc.contentType, Valid: doc.contentType != ""}, - Size: sql.NullInt64{Int64: size, Valid: true}, - FetchedAt: sql.NullTime{Time: doc.fetchedAt, Valid: true}, + Ecosystem: "swift", + Name: cacheKey, + StoragePath: storagePath, + Link: sql.NullString{String: doc.link, Valid: doc.link != ""}, + ContentType: sql.NullString{String: doc.contentType, Valid: doc.contentType != ""}, + ContentDigest: sql.NullString{String: "sha256:" + hash, Valid: hash != ""}, + Size: sql.NullInt64{Int64: size, Valid: true}, + FetchedAt: sql.NullTime{Time: doc.fetchedAt, Valid: true}, }) } if err != nil { diff --git a/internal/handler/swift_cache_test.go b/internal/handler/swift_cache_test.go index 06692c0..2b194a5 100644 --- a/internal/handler/swift_cache_test.go +++ b/internal/handler/swift_cache_test.go @@ -1,12 +1,14 @@ package handler import ( + "context" "io" "net/http" "net/http/httptest" "strings" "sync/atomic" "testing" + "time" ) func TestSwiftReleaseListAndManifestServedFromCacheWhenUpstreamFails(t *testing.T) { @@ -62,6 +64,52 @@ func TestSwiftReleaseListAndManifestServedFromCacheWhenUpstreamFails(t *testing. } } +// TestSwiftCorruptManifestRefetched checks that a cached manifest whose +// stored bytes no longer match the recorded digest is fetched again, even +// within metadata_ttl. +func TestSwiftCorruptManifestRefetched(t *testing.T) { + var hits atomic.Int32 + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + hits.Add(1) + w.Header().Set("Content-Type", "text/x-swift") + _, _ = io.WriteString(w, "// swift-tools-version:5.9") + })) + defer upstream.Close() + + proxy, db, store, _ := setupTestProxy(t) + proxy.CacheMetadata = true + proxy.MetadataTTL = time.Hour + routes := NewSwiftHandler(proxy, "https://proxy.example", upstream.URL+"/registry").Routes() + get := func() *httptest.ResponseRecorder { + w := httptest.NewRecorder() + routes.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/apple/example/1.0.0/Package.swift", nil)) + return w + } + + get() + entry, err := db.GetMetadataCache("swift", swiftManifestCacheKey("apple", "example", "1.0.0", "")) + if err != nil || entry == nil { + t.Fatalf("manifest not cached: %v", err) + } + if !strings.HasPrefix(entry.ContentDigest.String, "sha256:") { + t.Fatalf("ContentDigest = %q, want a sha256 digest", entry.ContentDigest.String) + } + if w := get(); hits.Load() != 1 || w.Code != http.StatusOK { + t.Fatalf("fresh cached manifest: status %d, %d upstream fetches; want it served from the cache", w.Code, hits.Load()) + } + + if _, _, err := store.Store(context.Background(), entry.StoragePath, strings.NewReader("// tampered")); err != nil { + t.Fatal(err) + } + w := get() + if hits.Load() != 2 { + t.Errorf("upstream fetches = %d, want the corrupt manifest fetched again", hits.Load()) + } + if w.Code != http.StatusOK || !strings.Contains(w.Body.String(), "swift-tools-version:5.9") { + t.Errorf("status = %d, body %q; want the upstream manifest", w.Code, w.Body.String()) + } +} + func TestSwiftManifestCacheKeyIgnoresToolsVersionZeros(t *testing.T) { key := swiftManifestCacheKey("apple", "example", "1.0.0", "swift-version=5.8") if got := swiftManifestCacheKey("apple", "example", "1.0.0", "swift-version=5.8.0"); got != key { diff --git a/internal/handler/swift_mirror.go b/internal/handler/swift_mirror.go index 3374cc9..d3944e3 100644 --- a/internal/handler/swift_mirror.go +++ b/internal/handler/swift_mirror.go @@ -2,19 +2,16 @@ package handler import ( "context" + "encoding/json" + "errors" "fmt" "net/http" "net/url" "strings" ) -// MirrorRelease caches the source archive of release version of the registry -// package scope.name, under the key the source archive route serves it from, -// verified against the SHA-256 checksum its release metadata lists. The -// release metadata is cached as that route reads it, and with cache_metadata -// so are the package's release list and the release's manifests, which a -// client reads before it downloads the archive. A client can then resolve and -// fetch the release without the upstream. +// MirrorRelease caches release version of scope.name so that a client can +// resolve and fetch the release without the upstream. func (h *SwiftHandler) MirrorRelease(ctx context.Context, scope, name, version string) (*CacheResult, error) { if !validSwiftPackageReference(scope, name, version) { return nil, fmt.Errorf("invalid Swift registry release %s.%s@%s", scope, name, version) @@ -123,3 +120,38 @@ func swiftLinkHasRel(params, rel string) bool { } return false } + +// ResolveIdentity returns the registry scope and name the upstream lists for +// the source repository at sourceURL. The answer is cached as the identifiers +// route caches it. +func (h *SwiftHandler) ResolveIdentity(ctx context.Context, sourceURL string) (scope, name string, _ error) { + query := url.Values{"url": {sourceURL}}.Encode() + body, _, err := h.proxy.FetchOrCacheMetadata(ctx, "swift", swiftMetadataCacheKey("identifiers", query), + h.upstreamURL+"/identifiers?"+query, swiftAcceptJSON) + if errors.Is(err, ErrUpstreamNotFound) { + return "", "", fmt.Errorf("the Swift registry has no package for %s", sourceURL) + } + if err != nil { + return "", "", err + } + return SwiftIdentityOf(body, sourceURL) +} + +// SwiftIdentityOf returns the scope and name of the first identity in body, +// a registry's answer to an identifiers lookup for sourceURL. +func SwiftIdentityOf(body []byte, sourceURL string) (scope, name string, _ error) { + var answer struct { + Identifiers []string `json:"identifiers"` + } + if err := json.Unmarshal(body, &answer); err != nil { + return "", "", fmt.Errorf("parsing the Swift registry's identifiers for %s: %w", sourceURL, err) + } + if len(answer.Identifiers) == 0 { + return "", "", fmt.Errorf("the Swift registry has no package for %s", sourceURL) + } + scope, name, ok := strings.Cut(answer.Identifiers[0], ".") + if !ok || scope == "" || name == "" { + return "", "", fmt.Errorf("the Swift registry answered %s with the invalid identity %q", sourceURL, answer.Identifiers[0]) + } + return scope, name, nil +} diff --git a/internal/mirror/swift.go b/internal/mirror/swift.go index 516276d..4616c8f 100644 --- a/internal/mirror/swift.go +++ b/internal/mirror/swift.go @@ -12,17 +12,23 @@ import ( // its release metadata, from the registry the proxy serves Swift packages // from. func (m *Mirror) mirrorSwift(ctx context.Context, pv PackageVersion, tracker *progressTracker) { - scope, name, err := swiftScopeAndName(pv.Name) - if err != nil { - m.recordFailure(tracker, pv, err) - return - } upstream, err := m.registryURL("swift", pv.Name) if err != nil { m.recordFailure(tracker, pv, err) return } - result, err := handler.NewSwiftHandler(m.proxy, "", upstream).MirrorRelease(ctx, scope, name, pv.Version) + h := handler.NewSwiftHandler(m.proxy, "", upstream) + var scope, name string + if source := swiftSourceURL(pv.Name); source != "" { + scope, name, err = h.ResolveIdentity(ctx, source) + } else { + scope, name, err = swiftScopeAndName(pv.Name) + } + if err != nil { + m.recordFailure(tracker, pv, err) + return + } + result, err := h.MirrorRelease(ctx, scope, name, pv.Version) if err != nil { m.recordFailure(tracker, pv, err) return @@ -39,26 +45,23 @@ func (m *Mirror) registryURL(purlType, name string) (string, error) { return "", nil } -// swiftScopeAndName splits the name of a pkg:swift PURL, its namespace and -// name, into a registry scope and package name. The namespace of a registry -// package is its scope, as in pkg:swift/apple/swift-nio for the identity -// apple.swift-nio. A namespace naming a source repository, such as -// pkg:swift/github.com/apple/swift-nio, isn't a registry identity and is -// refused. +// swiftSourceURL returns the repository URL of a pkg:swift name whose +// namespace starts with a host, as in pkg:swift/github.com/apple/swift-nio, +// or "" for a name whose namespace is a registry scope. +func swiftSourceURL(name string) string { + if host, _, ok := strings.Cut(name, "/"); ok && strings.Contains(host, ".") { + return "https://" + name + } + return "" +} + +// swiftScopeAndName splits the name of a pkg:swift PURL whose namespace is +// a registry scope, as in pkg:swift/apple/swift-nio for the identity +// apple.swift-nio. func swiftScopeAndName(name string) (scope, pkg string, _ error) { scope, pkg, ok := strings.Cut(name, "/") - if !ok || strings.ContainsAny(scope, ".:") || strings.Contains(pkg, "/") { - return "", "", fmt.Errorf("pkg:swift/%s names a source repository; mirroring needs a registry scope as the namespace, such as pkg:swift/apple/swift-nio for apple.swift-nio", name) + if !ok || scope == "" || pkg == "" || strings.ContainsAny(scope, ".:") || strings.Contains(pkg, "/") { + return "", "", fmt.Errorf("pkg:swift/%s needs a registry scope or a source repository as the namespace, such as pkg:swift/apple/swift-nio or pkg:swift/github.com/apple/swift-nio", name) } return scope, pkg, nil } - -// swiftIdentity returns the registry identity, scope.name, of the name of a -// pkg:swift PURL. -func swiftIdentity(name string) (string, error) { - scope, pkg, err := swiftScopeAndName(name) - if err != nil { - return "", err - } - return scope + "." + pkg, nil -} diff --git a/internal/mirror/swift_test.go b/internal/mirror/swift_test.go index 2d05615..4ae0764 100644 --- a/internal/mirror/swift_test.go +++ b/internal/mirror/swift_test.go @@ -1,14 +1,41 @@ package mirror -import "testing" +import ( + "context" + "net/http" + "net/http/httptest" + "testing" + + "github.com/git-pkgs/registries" +) func TestSwiftIdentity(t *testing.T) { + registry := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/identifiers" || r.Header.Get("Accept") != "application/vnd.swift.registry.v1+json" { + http.Error(w, "unexpected request", http.StatusBadRequest) + return + } + switch r.URL.Query().Get("url") { + case "https://github.com/apple/swift-nio": + _, _ = w.Write([]byte(`{"identifiers":["apple.swift-nio","mirror.swift-nio"]}`)) + case "https://github.com/acme/empty": + _, _ = w.Write([]byte(`{"identifiers":[]}`)) + default: + http.NotFound(w, r) + } + })) + defer registry.Close() + for name, want := range map[string]string{ "apple/swift-nio": "apple.swift-nio", - "github.com/apple/swift-nio": "", + "github.com/apple/swift-nio": "apple.swift-nio", + "github.com/acme/empty": "", + "github.com/acme/unknown": "", "swift-nio": "", + "apple/": "", + "apple/swift-nio/extra": "", } { - got, err := swiftIdentity(name) + got, err := swiftIdentity(context.Background(), registries.DefaultClient(), registry.URL, name) if got != want || (want == "") != (err != nil) { t.Errorf("swiftIdentity(%q) = %q, %v; want %q", name, got, err, want) } diff --git a/internal/mirror/versions.go b/internal/mirror/versions.go index fce9c2a..ad22039 100644 --- a/internal/mirror/versions.go +++ b/internal/mirror/versions.go @@ -2,9 +2,15 @@ package mirror import ( "context" + "encoding/json" + "errors" "fmt" + "net/http" + "net/url" "strings" + "github.com/git-pkgs/proxy/internal/config" + "github.com/git-pkgs/proxy/internal/handler" "github.com/git-pkgs/purl" "github.com/git-pkgs/registries" _ "github.com/git-pkgs/registries/all" @@ -58,12 +64,6 @@ func (r RegistryVersions) FindVersion(ctx context.Context, ecosystem, name, vers } func (r RegistryVersions) fetchVersions(ctx context.Context, ecosystem, name string) ([]registries.Version, error) { - if ecosystem == "swift" { - var err error - if name, err = swiftIdentity(name); err != nil { - return nil, err - } - } purlType := purl.EcosystemToPURLType(ecosystem) var baseURL string if r.URL != nil { @@ -76,6 +76,12 @@ func (r RegistryVersions) fetchVersions(ctx context.Context, ecosystem, name str if client == nil { client = registries.DefaultClient() } + if ecosystem == "swift" { + var err error + if name, err = swiftIdentity(ctx, client, baseURL, name); err != nil { + return nil, err + } + } reg, err := registries.New(purlType, baseURL, client) if err != nil { return nil, err @@ -87,3 +93,34 @@ func (r RegistryVersions) fetchVersions(ctx context.Context, ecosystem, name str } return reg.FetchVersions(ctx, name) } + +// swiftIdentity returns the registry identity, scope.name, of the name of a +// pkg:swift PURL, looking a source repository up on the registry at baseURL. +func swiftIdentity(ctx context.Context, client *registries.Client, baseURL, name string) (string, error) { + source := swiftSourceURL(name) + if source == "" { + scope, pkg, err := swiftScopeAndName(name) + if err != nil { + return "", err + } + return scope + "." + pkg, nil + } + if baseURL == "" { + baseURL = config.DefaultSwiftUpstream + } + var body json.RawMessage + _, err := client.GetJSONWithHeaders(ctx, + strings.TrimSuffix(baseURL, "/")+"/identifiers?"+url.Values{"url": {source}}.Encode(), + http.Header{"Accept": {"application/vnd.swift.registry.v1+json"}}, &body) + if httpErr := (*registries.HTTPError)(nil); errors.As(err, &httpErr) && httpErr.IsNotFound() { + return "", fmt.Errorf("the Swift registry has no package for %s", source) + } + if err != nil { + return "", err + } + scope, pkg, err := handler.SwiftIdentityOf(body, source) + if err != nil { + return "", err + } + return scope + "." + pkg, nil +} diff --git a/internal/server/mirror_swift_test.go b/internal/server/mirror_swift_test.go index d10b682..a7050c3 100644 --- a/internal/server/mirror_swift_test.go +++ b/internal/server/mirror_swift_test.go @@ -42,6 +42,13 @@ func newSwiftUpstream(t *testing.T) *swiftUpstream { w.Header().Set("Content-Version", "1") rest, ok := strings.CutPrefix(r.URL.Path, "/acme/widget") switch { + case r.URL.Path == "/identifiers": + if r.URL.Query().Get("url") != "https://github.com/acme/widget" { + http.NotFound(w, r) + return + } + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, `{"identifiers":["acme.widget"]}`) case !ok: http.NotFound(w, r) case rest == "": @@ -191,15 +198,26 @@ func TestMirrorSwiftListsReleases(t *testing.T) { } } -func TestMirrorSwiftRefusesRepositoryNamespaces(t *testing.T) { +func TestMirrorSwiftResolvesRepositoryPURLs(t *testing.T) { upstream := newSwiftUpstream(t) - progress, _ := mirrorSwift(t, upstream, "pkg:swift/github.com/acme/widget@1.0.0") + progress, proxy := mirrorSwift(t, upstream, "pkg:swift/github.com/acme/widget@1.0.0", "pkg:swift/github.com/acme/widget", "pkg:swift/github.com/acme/other@1.0.0") - if progress.Failed != 1 || len(progress.Errors) != 1 || !strings.Contains(progress.Errors[0].Error, "registry scope") { - t.Errorf("failed = %d, errors = %v; want the repository PURL refused", progress.Failed, progress.Errors) + // The listed PURL's 1.0.0 is the versioned one, so it's mirrored once. + if progress.Completed != 2 || progress.Failed != 1 || upstream.archiveHits.Load() != 2 { + t.Fatalf("completed = %d, failed = %d, archive hits = %d, errors = %v; want both acme.widget releases mirrored and the unknown repository refused", + progress.Completed, progress.Failed, upstream.archiveHits.Load(), progress.Errors) } - if upstream.hits.Load() != 0 { - t.Error("the upstream was asked for a repository PURL") + if len(progress.Errors) != 1 || !strings.Contains(progress.Errors[0].Error, "no package for https://github.com/acme/other") { + t.Errorf("errors = %v; want the unknown repository named", progress.Errors) + } + + // The lookup is cached too, for clients that resolve a repository URL + // through the registry. + upstream.Close() + routes := handler.NewSwiftHandler(proxy, "http://proxy.test", upstream.URL).Routes() + rec := swiftRegistryGet(t, routes, "/identifiers?url=https%3A%2F%2Fgithub.com%2Facme%2Fwidget", "application/vnd.swift.registry.v1+json") + if rec.Code != http.StatusOK || !strings.Contains(rec.Body.String(), "acme.widget") { + t.Errorf("identifiers: status = %d, body %q; want the cached lookup", rec.Code, rec.Body.String()) } } From 9a5e5a2c9a85db5cb9de165726af4384c678c87b Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Mon, 26 Oct 2026 16:44:11 +0100 Subject: [PATCH 455/633] Keep a slow client from holding back a streamed miss The upstream read fed the client through a pipe, so it ran at the client's speed. The client is now fed from a bounded buffer and, once it falls behind, continues from the stored copy where it left off. A caller handed the stream takes no hold on an unrecorded object. --- docs/configuration.md | 2 +- internal/handler/stream_miss.go | 240 +++++++++++++++++++++++---- internal/handler/stream_miss_test.go | 123 ++++++++++++++ internal/handler/unrecorded_test.go | 56 +++++++ 4 files changed, 384 insertions(+), 37 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 936be4d..e92a0ac 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -76,7 +76,7 @@ storage: On a cache miss the proxy sends the artifact to the client while it stores it, so the client gets its first bytes as soon as the upstream sends them, rather than after the whole artifact is stored. Clients that give up after a while without data, such as git-lfs (30 seconds) or pacman (10 seconds), no longer time out on large artifacts. The response goes out chunked, without `Content-Length`, as with `cache_artifacts: false`; once stored, the artifact is served with its size. -The bytes are checked against the digest the upstream declares, where it declares one, as they are sent. On a mismatch the connection is aborted before the response completes, so the client never receives a tampered artifact as a good one, and nothing is cached. If the client goes away, the artifact is still stored. If storing it fails, for example because the disk is full, the client still gets the whole artifact; the failure is logged at error level and counted in `proxy_artifacts_served_uncached_total`, so a cache that has stopped filling doesn't go unnoticed while every request succeeds. +The bytes are checked against the digest the upstream declares, where it declares one, as they are sent. On a mismatch the connection is aborted before the response completes, so the client never receives a tampered artifact as a good one, and nothing is cached. The upstream is read at its own speed whatever the client's: a client more than 8 MB behind continues from the stored copy once it is stored, so a slow client doesn't hold back the download, other requests waiting for it, or `artifact_timeout`. If the client goes away, the artifact is still stored. If storing it fails, for example because the disk is full, the client still gets the whole artifact; the failure is logged at error level and counted in `proxy_artifacts_served_uncached_total`, so a cache that has stopped filling doesn't go unnoticed while every request succeeds. Requests for the same artifact that arrive while it is being stored wait for the stored copy, as do `HEAD` requests and requests with a `Range` header, which are answered from it with its size. With a `block`-mode scanner configured for the ecosystem, a miss is stored and scanned before anything is sent, as before (see [Artifact Scanning](#artifact-scanning)). diff --git a/internal/handler/stream_miss.go b/internal/handler/stream_miss.go index 9ddcc20..bc401b2 100644 --- a/internal/handler/stream_miss.go +++ b/internal/handler/stream_miss.go @@ -3,6 +3,7 @@ package handler import ( "context" "errors" + "fmt" "io" "strings" "sync" @@ -13,12 +14,12 @@ import ( "github.com/opencontainers/go-digest" ) -// On a cache miss the proxy used to store the whole artifact before sending -// any of it, so a client with an idle timeout (git-lfs gives up after 30 -// seconds without data, pacman after 10) could give up on a large one. A -// marked request instead gets the bytes as they arrive from upstream while -// they are stored: the caller that starts the shared fetch is handed a -// stream, and callers joining it still wait for the stored artifact. +// On a cache miss a marked request is sent the bytes as they arrive from +// upstream while they are stored, rather than after the whole artifact is +// stored. Clients with idle timeouts need this for large artifacts: git-lfs +// gives up after 30 seconds without data, pacman after 10. The caller that +// starts the shared fetch is handed the stream; callers joining it still wait +// for the stored artifact. type streamCacheMissKey struct{} @@ -138,15 +139,9 @@ func (p *Proxy) storeFetched(ctx context.Context, ecosystem, name, version, file var errNoReaderLeft = errors.New("neither storage nor the client is reading the artifact") // storeWhileServing copies the upstream body to storage and to the client at -// once. The client's copy is checked against upstreamHash and declared as it -// is read, and a mismatch fails it before its last bytes, so the response is -// aborted and the client discards it; storeArtifact checks the stored copy -// the same way and doesn't cache it. A client that goes away leaves the copy -// to storage running. A storage error leaves the client's copy running, so -// the client still gets the artifact; that is logged and counted, since every -// request would otherwise succeed while nothing is cached. The client's -// stream ends once the last byte is checked, before storeArtifact's scan and -// database update. +// once, at the upstream's speed; see missFeed for a client that can't keep up. +// The client's copy is checked before its last bytes, so a mismatch aborts +// the response. A storage failure leaves the client's copy running. func (p *Proxy) storeWhileServing(ctx context.Context, h *missHandoff, ecosystem, name, version, filename, pkgPURL, versionPURL, upstreamURL, upstreamHash, declared string, artifact *fetch.Artifact) (artifacts.Artifact, string, error) { hash := strings.ToLower(upstreamHash) checks, err := newIntegrityChecks(hash, declared) @@ -155,10 +150,10 @@ func (p *Proxy) storeWhileServing(ctx context.Context, h *missHandoff, ecosystem return artifacts.Artifact{}, "", err } - clientR, clientW := io.Pipe() - var client io.ReadCloser = clientR + feed := newMissFeed(clientFeedLimit) + var client io.ReadCloser = feed if hash != "" || declared != "" { - if client, err = checks.wrapFailOnMismatch(clientR, func(reason string) { + if client, err = checks.wrapFailOnMismatch(feed, func(reason string) { p.Logger.Warn("artifact sent on a cache miss failed its integrity check; aborting the response", "purl", versionPURL, "filename", filename, "reason", reason) }); err != nil { @@ -181,7 +176,7 @@ func (p *Proxy) storeWhileServing(ctx context.Context, h *missHandoff, ecosystem stream.Artifact.Digest = digest.Digest("sha256:" + hash) } if !h.offer(stream) { - _ = clientR.Close() + _ = feed.Close() return p.storeArtifact(ctx, ecosystem, name, version, filename, pkgPURL, versionPURL, upstreamURL, upstreamHash, declared, artifact) } @@ -205,13 +200,17 @@ func (p *Proxy) storeWhileServing(ctx context.Context, h *missHandoff, ecosystem storeDone <- stored{a, path, err} }() - readErr, clientOK := copyToBoth(artifact.Body, storeW, clientW) + readErr := copyToBoth(artifact.Body, storeW, feed) _ = artifact.Body.Close() _ = storeW.CloseWithError(readErr) - _ = clientW.CloseWithError(readErr) + feed.end(readErr) out := <-storeDone - if out.err != nil && readErr == nil && clientOK { + if feed.lagging() { + // Opened before returning, so an unrecorded object is still there. + feed.resolve(p.openStoredForFeed(ctx, out.artifact, out.path, out.err)) + } + if out.err != nil && readErr == nil && feed.fedFully() { p.Logger.Error("artifact sent to the client but not cached", "purl", versionPURL, "filename", filename, "url", upstreamURL, "error", out.err) metrics.RecordArtifactServedUncached(ecosystem, metrics.FailureDetail{ @@ -220,14 +219,27 @@ func (p *Proxy) storeWhileServing(ctx context.Context, h *missHandoff, ecosystem return out.artifact, out.path, out.err } -// copyToBoth copies src to store and client until src ends, dropping either -// destination once a write to it fails. It returns the error that ended the -// copy, nil at the end of src, and whether the client took every byte. -func copyToBoth(src io.Reader, store, client io.Writer) (error, bool) { +// openStoredForFeed opens the stored copy for a client that fell behind. An +// unrecorded object is opened too: it is deleted only after this returns. +func (p *Proxy) openStoredForFeed(ctx context.Context, a artifacts.Artifact, path string, storeErr error) (io.ReadCloser, error) { + if storeErr != nil && !errors.Is(storeErr, errUnrecorded) { + return nil, storeErr + } + res, err := p.openStoredArtifact(ctx, a, path) + if err != nil { + return nil, err + } + return res.Reader, nil +} + +// copyToBoth copies src to store and to the client's feed until src ends, +// dropping the store once a write to it fails. It returns the error that +// ended the copy, nil at the end of src. Feeding the client never blocks. +func copyToBoth(src io.Reader, store io.Writer, client *missFeed) error { buffer := artifactCopyBufferPool.Get().(*[]byte) defer artifactCopyBufferPool.Put(buffer) buf := *buffer - storeOK, clientOK := true, true + storeOK := true for { n, err := src.Read(buf) if n > 0 { @@ -236,20 +248,176 @@ func copyToBoth(src io.Reader, store, client io.Writer) (error, bool) { storeOK = false } } - if clientOK { - if _, werr := client.Write(buf[:n]); werr != nil { - clientOK = false - } - } - if !storeOK && !clientOK { - return errNoReaderLeft, false + // A client that has gone, or that waits for the stored copy, + // has no use for the rest once storing has failed. + if !client.push(buf[:n]) && !storeOK { + return errNoReaderLeft } } if errors.Is(err, io.EOF) { - return nil, clientOK + return nil } if err != nil { - return err, false + return err } } } + +// clientFeedLimit bounds the bytes held for a client that reads a streamed +// miss slower than they arrive. A variable so tests can lower it. +var clientFeedLimit = 8 << 20 + +// missFeed is the client's side of a streamed miss. The upstream read pushes +// into it without blocking, so a slow client can't hold back the copy to +// storage, the fetch lease or the callers waiting on the fetch. Once the +// client is more than the limit behind, it stops being fed: it reads what +// is held, then continues from the stored copy where it left off. +type missFeed struct { + mu sync.Mutex + cond *sync.Cond + chunks [][]byte + held int + limit int + // fed counts the bytes pushed before the client fell behind, where it + // continues in the stored copy. + fed int64 + ended bool + endErr error + behind bool + closed bool + early bool // closed before reading everything it was fed + + stored io.ReadCloser + storedErr error + resolved bool + skipped bool +} + +func newMissFeed(limit int) *missFeed { + m := &missFeed{limit: limit} + m.cond = sync.NewCond(&m.mu) + return m +} + +// push queues a copy of b and reports whether the client is still fed. +func (m *missFeed) push(b []byte) bool { + m.mu.Lock() + defer m.mu.Unlock() + if m.closed || m.behind { + return false + } + if m.held+len(b) > m.limit { + m.behind = true + return false + } + m.chunks = append(m.chunks, append([]byte(nil), b...)) + m.held += len(b) + m.fed += int64(len(b)) + m.cond.Broadcast() + return true +} + +// end records how the upstream read ended. +func (m *missFeed) end(err error) { + m.mu.Lock() + defer m.mu.Unlock() + m.ended, m.endErr = true, err + m.cond.Broadcast() +} + +func (m *missFeed) lagging() bool { + m.mu.Lock() + defer m.mu.Unlock() + return m.behind && !m.closed +} + +// fedFully reports whether the client was sent every byte from upstream. +func (m *missFeed) fedFully() bool { + m.mu.Lock() + defer m.mu.Unlock() + return !m.behind && !m.early +} + +// resolve gives a client that fell behind the stored copy, or the error +// that left none. +func (m *missFeed) resolve(r io.ReadCloser, err error) { + m.mu.Lock() + defer m.mu.Unlock() + if m.closed { + if r != nil { + _ = r.Close() + } + return + } + m.stored, m.storedErr, m.resolved = r, err, true + m.cond.Broadcast() +} + +func (m *missFeed) Read(p []byte) (int, error) { + m.mu.Lock() + for { + switch { + case m.closed: + m.mu.Unlock() + return 0, io.ErrClosedPipe + case len(m.chunks) > 0: + n := copy(p, m.chunks[0]) + if m.chunks[0] = m.chunks[0][n:]; len(m.chunks[0]) == 0 { + m.chunks[0] = nil + m.chunks = m.chunks[1:] + } + m.held -= n + m.mu.Unlock() + return n, nil + case m.behind && m.resolved: + r, err, skipped := m.stored, m.storedErr, m.skipped + m.skipped = true + m.mu.Unlock() + if err != nil { + return 0, err + } + if !skipped { + if err := skipTo(r, m.fed); err != nil { + return 0, err + } + } + return r.Read(p) + case !m.behind && m.ended: + err := m.endErr + m.mu.Unlock() + if err == nil { + err = io.EOF + } + return 0, err + } + m.cond.Wait() + } +} + +func (m *missFeed) Close() error { + m.mu.Lock() + defer m.mu.Unlock() + if m.closed { + return nil + } + m.closed = true + m.early = !m.ended || len(m.chunks) > 0 + m.chunks, m.held = nil, 0 + if m.stored != nil { + _ = m.stored.Close() + } + m.cond.Broadcast() + return nil +} + +// skipTo moves r past the first n bytes. +func skipTo(r io.Reader, n int64) error { + if s, ok := r.(io.Seeker); ok { + _, err := s.Seek(n, io.SeekStart) + return err + } + if _, err := io.CopyN(io.Discard, r, n); err != nil { + return fmt.Errorf("skipping to byte %d of the stored artifact: %w", n, err) + } + return nil +} diff --git a/internal/handler/stream_miss_test.go b/internal/handler/stream_miss_test.go index e3b449b..172be1b 100644 --- a/internal/handler/stream_miss_test.go +++ b/internal/handler/stream_miss_test.go @@ -442,3 +442,126 @@ func TestCacheMissStoredAfterResponseWithoutDetach(t *testing.T) { t.Fatal("artifact not cached once the response ended") } } + +// lowerFeedLimit makes a client fall behind after n bytes. +func lowerFeedLimit(t *testing.T, n int) { + t.Helper() + old := clientFeedLimit + clientFeedLimit = n + t.Cleanup(func() { clientFeedLimit = old }) +} + +// TestCacheMissSlowClientDoesNotHoldBackFetch has the caller that starts a +// streamed miss read nothing until a second caller, waiting on the same +// fetch, has the stored artifact. The slow client then gets every byte, the +// ones past the limit from the stored copy. +func TestCacheMissSlowClientDoesNotHoldBackFetch(t *testing.T) { + for _, seekable := range []bool{false, true} { + t.Run(fmt.Sprintf("seekable=%v", seekable), func(t *testing.T) { testSlowClient(t, seekable) }) + } +} + +func testSlowClient(t *testing.T, seekable bool) { + t.Helper() + lowerFeedLimit(t, 64<<10) + proxy, _, store, _ := setupTestProxy(t) + store.seekable = seekable + fetcher := newGatedFetcher(streamMissContent) + proxy.Fetcher = fetcher + get := func(ctx context.Context) (*CacheResult, error) { + return proxy.GetOrFetchArtifactFromURL(ctx, "npm", "big", "1.0.0", "big-1.0.0.tgz", "https://example.com/big.tgz") + } + + slow := make(chan *CacheResult, 1) + go func() { + res, err := get(StreamCacheMiss(context.Background())) + if err != nil { + t.Errorf("slow client: %v", err) + } + slow <- res + }() + <-fetcher.entered + waiter := make(chan *CacheResult, 1) + go func() { + res, err := get(context.Background()) + if err != nil { + t.Errorf("waiter: %v", err) + } + waiter <- res + }() + waitForWaiters(t, proxy, 1) + close(fetcher.release) + + lead := <-slow + if lead == nil { + t.FailNow() + } + defer func() { _ = lead.Reader.Close() }() + if lead.Artifact.Size != -1 { + t.Fatalf("slow client got size %d, want a stream", lead.Artifact.Size) + } + select { + case res := <-waiter: + if res == nil { + t.FailNow() + } + body, err := io.ReadAll(res.Reader) + _ = res.Reader.Close() + if err != nil || string(body) != streamMissContent { + t.Fatalf("waiter: %d bytes, %v", len(body), err) + } + case <-time.After(5 * time.Second): + t.Fatal("waiter held back by a client that reads nothing") + } + + body, err := io.ReadAll(lead.Reader) + if err != nil || string(body) != streamMissContent { + t.Fatalf("slow client: %d bytes, %v; want the whole artifact", len(body), err) + } + if fetcher.calls.Load() != 1 { + t.Errorf("upstream fetches = %d, want 1", fetcher.calls.Load()) + } +} + +// TestCacheMissSlowClientFailsWithoutStoredCopy: a client that fell behind +// has nothing to continue from when storing fails, so its response fails +// rather than ending short. +func TestCacheMissSlowClientFailsWithoutStoredCopy(t *testing.T) { + lowerFeedLimit(t, 64<<10) + proxy, _, store, _ := setupTestProxy(t) + store.storeErr = errors.New("no space left on device") + proxy.Fetcher = &countingFetcher{content: streamMissContent} + + res, err := proxy.GetOrFetchArtifactFromURL(StreamCacheMiss(context.Background()), "npm", "big", "1.0.0", "big-1.0.0.tgz", "https://example.com/big.tgz") + if err != nil { + t.Fatal(err) + } + body, err := io.ReadAll(res.Reader) + _ = res.Reader.Close() + if err == nil { + t.Fatalf("read %d bytes without error; want the response to fail", len(body)) + } +} + +func TestMissFeedCloseWhileWaiting(t *testing.T) { + m := newMissFeed(4) + done := make(chan error, 1) + go func() { + _, err := m.Read(make([]byte, 8)) + done <- err + }() + time.Sleep(10 * time.Millisecond) + _ = m.Close() + select { + case err := <-done: + if !errors.Is(err, io.ErrClosedPipe) { + t.Errorf("Read after Close = %v", err) + } + case <-time.After(time.Second): + t.Fatal("Read still blocked after Close") + } + if m.push([]byte("ab")) { + t.Error("push to a closed feed reported the client fed") + } + m.resolve(io.NopCloser(strings.NewReader("x")), nil) +} diff --git a/internal/handler/unrecorded_test.go b/internal/handler/unrecorded_test.go index 3533a10..75325f2 100644 --- a/internal/handler/unrecorded_test.go +++ b/internal/handler/unrecorded_test.go @@ -340,3 +340,59 @@ func TestRecordedAt(t *testing.T) { t.Error("failed lookup taken as recorded") } } + +// TestUnrecordedStreamedMiss streams an unrecorded refetch to the caller +// that started it: it holds no reader on the stored object, so the object is +// deleted once the waiters are done, and the waiters still get it. +func TestUnrecordedStreamedMiss(t *testing.T) { + for _, behind := range []bool{false, true} { + t.Run(map[bool]string{false: "fed", true: "behind"}[behind], func(t *testing.T) { + if behind { + old := clientFeedLimit + clientFeedLimit = 4 + t.Cleanup(func() { clientFeedLimit = old }) + } + proxy, store := setupUnrecorded(t) + fetcher := newGatedFetcher(unrecordedContent) + proxy.Fetcher = fetcher + + leader := make(chan *CacheResult, 1) + go func() { + res, err := fetchGone(StreamCacheMiss(context.Background()), proxy) + if err != nil { + t.Errorf("leader: %v", err) + } + leader <- res + }() + <-fetcher.entered + waiter := make(chan *CacheResult, 1) + go func() { + res, err := fetchGone(context.Background(), proxy) + if err != nil { + t.Errorf("waiter: %v", err) + } + waiter <- res + }() + waitForWaiters(t, proxy, 1) + close(fetcher.release) + + lead := <-leader + if lead == nil || lead.Artifact.Size != -1 { + t.Fatalf("leader = %+v, want a stream", lead) + } + w := <-waiter + if w == nil { + t.FailNow() + } + readAndClose(t, w) + deadline := time.Now().Add(5 * time.Second) + for len(storedFiles(store)) != 0 && time.Now().Before(deadline) { + time.Sleep(time.Millisecond) + } + if got := storedFiles(store); len(got) != 0 { + t.Errorf("stored files after the waiter closed = %v, want none", got) + } + readAndClose(t, lead) + }) + } +} From 7e2bf15e0036e939c40e91134ab5c9f1494bedc2 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Mon, 26 Oct 2026 18:35:17 +0100 Subject: [PATCH 456/633] Release a new minor version when breaking changes were merged The monthly release skipped the month when a pull request labelled breaking had merged, leaving the release to be cut by hand. It now cuts a new minor version instead (a new major version from 1.0 on). --- .github/workflows/monthly-release.yml | 27 +++++++++++++++++---------- CONTRIBUTING.md | 2 +- 2 files changed, 18 insertions(+), 11 deletions(-) diff --git a/.github/workflows/monthly-release.yml b/.github/workflows/monthly-release.yml index 6afee9d..22bec6e 100644 --- a/.github/workflows/monthly-release.yml +++ b/.github/workflows/monthly-release.yml @@ -1,8 +1,10 @@ -name: Monthly patch release +name: Monthly release -# Cuts a patch release on the 1st of each month when main is green and -# something has merged since the last release. Scheduled runs only report -# what they would do until the repository variable MONTHLY_RELEASE is "true". +# Cuts a release on the 1st of each month when main is green and something +# has merged since the last release: a patch release, or a new minor version +# if a pull request labelled "breaking" was merged (a new major version from +# 1.0 on). Scheduled runs only report what they would do until the +# repository variable MONTHLY_RELEASE is "true". on: schedule: @@ -76,15 +78,20 @@ jobs: fi since=$(git log -1 --format=%cI "$last") - breaking=$(gh pr list -R "$REPO" --state merged --base main \ + breaking=$(gh pr list -R "$REPO" --state merged --base main --limit 1000 \ --search "merged:>$since label:breaking" --json number --jq '.[].number') - if [ -n "$breaking" ]; then - echo "::warning::breaking changes merged since $last (#$(echo "$breaking" | paste -sd, - | sed 's/,/, #/g')), release a new minor version by hand" - exit 0 - fi IFS=. read -r major minor patch <<<"${last#v}" - tag="v$major.$minor.$((patch + 1))" + if [ -z "$breaking" ]; then + tag="v$major.$minor.$((patch + 1))" + else + echo "Breaking changes merged since $last: #$(echo "$breaking" | sort -n | paste -sd, - | sed 's/,/, #/g')" + if [ "$major" -eq 0 ]; then + tag="v0.$((minor + 1)).0" + else + tag="v$((major + 1)).0.0" + fi + fi echo "$merged commits since $last, next release $tag at $sha" echo "tag=$tag" >> "$GITHUB_OUTPUT" echo "sha=$sha" >> "$GITHUB_OUTPUT" diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 9bb2fa1..6359283 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -222,7 +222,7 @@ Open an issue before a bigger change: anything that changes behaviour users rely ## Releases -A patch release is cut on the 1st of each month by `.github/workflows/monthly-release.yml` when `main` is green and something has merged since the last tag. It is skipped if a pull request labelled `breaking` was merged since then: label pull requests that change behaviour users rely on, and release those as a new minor version by hand. +A release is cut on the 1st of each month by `.github/workflows/monthly-release.yml` when `main` is green and something has merged since the last tag. It is a patch release, or a new minor version if a pull request labelled `breaking` was merged since then (a new major version from 1.0 on), so label pull requests that change behaviour users rely on. Scheduled runs only report what they would release until the repository variable `MONTHLY_RELEASE` is set to `true`. The workflow can also be started by hand from the Actions tab, as a dry run by default. From 1887b124cf406e1b6e175d42e9bf57e4a02cb725 Mon Sep 17 00:00:00 2001 From: Ondrej Kokes Date: Tue, 27 Oct 2026 08:50:11 +0100 Subject: [PATCH 457/633] Add extraVolumes, extraVolumeMounts and extraInitContainers to Helm chart The pod only ever got the config, data and tmp volumes, so there was no way to mount a private CA certificate, file-based credentials or a projected token, or to run init containers. All three values default to empty, leaving the rendered Deployment unchanged. Render the new blocks in CI and document the private CA setup in the README. Closes #614 Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 10 ++++++++ README.md | 21 +++++++++++++++++ deploy/charts/proxy/templates/deployment.yaml | 10 ++++++++ deploy/charts/proxy/values.yaml | 23 +++++++++++++++++++ 4 files changed, 64 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c8d0bc7..4d2e324 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -110,3 +110,13 @@ jobs: --set 'ingress.hosts[0].paths[0].path=/' \ --set 'ingress.hosts[0].paths[0].pathType=Prefix' \ >/dev/null + helm template proxy deploy/charts/proxy \ + --set 'extraVolumes[0].name=corp-ca' \ + --set 'extraVolumes[0].configMap.name=corp-ca' \ + --set 'extraVolumeMounts[0].name=corp-ca' \ + --set 'extraVolumeMounts[0].mountPath=/etc/proxy/ca' \ + --set 'extraVolumeMounts[0].readOnly=true' \ + --set 'extraInitContainers[0].name=wait-for-db' \ + --set 'extraInitContainers[0].image=busybox:1.37' \ + --set 'extraInitContainers[0].command={sh,-c,exit 0}' \ + >/dev/null diff --git a/README.md b/README.md index bcfd327..20c6d5d 100644 --- a/README.md +++ b/README.md @@ -104,6 +104,27 @@ using SQLite and filesystem artifact storage under `/data`. See [`deploy/charts/proxy/values.yaml`](deploy/charts/proxy/values.yaml) for ingress, external database and object-storage configuration options. +To trust a private CA, for example when upstreams sit behind a TLS-intercepting +corporate proxy or are internal registries signed by an in-house CA, mount the +certificate with `extraVolumes` and `extraVolumeMounts` and add its directory +to `SSL_CERT_DIR`. The proxy process makes all upstream TLS connections itself, +Git fetches included, so no other configuration is needed. Public CAs stay +trusted alongside the private one: + +```yaml +extraVolumes: + - name: corp-ca + configMap: + name: corp-ca # e.g. key corp-ca.crt, PEM-encoded +extraVolumeMounts: + - name: corp-ca + mountPath: /etc/proxy/ca + readOnly: true +extraEnv: + - name: SSL_CERT_DIR + value: /etc/ssl/certs:/etc/proxy/ca +``` + ## Quick Start ```bash diff --git a/deploy/charts/proxy/templates/deployment.yaml b/deploy/charts/proxy/templates/deployment.yaml index 7a1027b..f04728a 100644 --- a/deploy/charts/proxy/templates/deployment.yaml +++ b/deploy/charts/proxy/templates/deployment.yaml @@ -33,6 +33,10 @@ spec: imagePullSecrets: {{- toYaml . | nindent 8 }} {{- end }} + {{- with .Values.extraInitContainers }} + initContainers: + {{- toYaml . | nindent 8 }} + {{- end }} containers: - name: {{ .Chart.Name }} image: {{ include "proxy.image" . | quote }} @@ -72,6 +76,9 @@ spec: mountPath: {{ .Values.persistence.mountPath }} - name: tmp mountPath: /tmp + {{- with .Values.extraVolumeMounts }} + {{- toYaml . | nindent 12 }} + {{- end }} volumes: - name: config configMap: @@ -93,6 +100,9 @@ spec: {{- else }} {} {{- end }} + {{- with .Values.extraVolumes }} + {{- toYaml . | nindent 8 }} + {{- end }} {{- with .Values.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} diff --git a/deploy/charts/proxy/values.yaml b/deploy/charts/proxy/values.yaml index bdff879..5936ef3 100644 --- a/deploy/charts/proxy/values.yaml +++ b/deploy/charts/proxy/values.yaml @@ -56,6 +56,29 @@ extraEnvFrom: [] # - secretRef: # name: proxy-object-storage +# Additional volumes for the pod and mounts for the proxy container, for +# example a ConfigMap holding a private CA certificate. Go adds certificates +# from the directories in SSL_CERT_DIR to its trusted roots (set it with +# extraEnv); SSL_CERT_FILE replaces the system bundle instead. All upstream +# TLS, including Git fetches, goes through the proxy process. +extraVolumes: [] +# - name: corp-ca +# configMap: +# name: corp-ca + +extraVolumeMounts: [] +# - name: corp-ca +# mountPath: /etc/proxy/ca +# readOnly: true + +# Init containers run before the proxy starts. They can use extraVolumes, but +# extraVolumeMounts applies only to the proxy container, so each init container +# needs its own volumeMounts. +extraInitContainers: [] +# - name: wait-for-db +# image: busybox:1.37 +# command: ["sh", "-c", "until nc -z postgres 5432; do sleep 2; done"] + persistence: enabled: true # Keep this aligned with config.data.storage.url and config.data.database.path From 4094b0fca91231c6ffe883dd8ba8ec7c85ecc22a Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Tue, 27 Oct 2026 12:16:50 +0100 Subject: [PATCH 458/633] Create breaking monthly releases as drafts A breaking release gets upgrade notes before it goes out. release.yml creates it as a draft and publish.yml isn't started; publishing the draft by hand starts publish.yml for the images and the chart. --- .github/workflows/monthly-release.yml | 21 +++++++++++++++++---- .github/workflows/publish.yml | 4 ++++ .github/workflows/release.yml | 7 ++++++- CONTRIBUTING.md | 2 ++ 4 files changed, 29 insertions(+), 5 deletions(-) diff --git a/.github/workflows/monthly-release.yml b/.github/workflows/monthly-release.yml index 22bec6e..6e1c28f 100644 --- a/.github/workflows/monthly-release.yml +++ b/.github/workflows/monthly-release.yml @@ -3,7 +3,9 @@ name: Monthly release # Cuts a release on the 1st of each month when main is green and something # has merged since the last release: a patch release, or a new minor version # if a pull request labelled "breaking" was merged (a new major version from -# 1.0 on). Scheduled runs only report what they would do until the +# 1.0 on). A breaking release is created as a draft, and its images and chart +# are published only when someone publishes the draft, after writing the +# upgrade notes. Scheduled runs only report what they would do until the # repository variable MONTHLY_RELEASE is "true". on: @@ -95,6 +97,7 @@ jobs: echo "$merged commits since $last, next release $tag at $sha" echo "tag=$tag" >> "$GITHUB_OUTPUT" echo "sha=$sha" >> "$GITHUB_OUTPUT" + echo "draft=$([ -n "$breaking" ] && echo true || echo false)" >> "$GITHUB_OUTPUT" # A tag pushed with GITHUB_TOKEN doesn't start other workflows, so the # release workflows are dispatched on the new tag instead. @@ -104,13 +107,23 @@ jobs: TAG: ${{ steps.plan.outputs.tag }} SHA: ${{ steps.plan.outputs.sha }} DRY_RUN: ${{ steps.plan.outputs.dry_run }} + DRAFT: ${{ steps.plan.outputs.draft }} run: | set -euo pipefail + if [ "$DRAFT" = true ]; then + what="start release.yml for a draft release" + else + what="start release.yml and publish.yml" + fi if [ "$DRY_RUN" = true ]; then - echo "Dry run: would tag $TAG at $SHA and start release.yml and publish.yml" + echo "Dry run: would tag $TAG at $SHA and $what" exit 0 fi gh api "repos/$REPO/git/refs" -f ref="refs/tags/$TAG" -f sha="$SHA" >/dev/null - gh workflow run release.yml -R "$REPO" --ref "$TAG" - gh workflow run publish.yml -R "$REPO" --ref "$TAG" + gh workflow run release.yml -R "$REPO" --ref "$TAG" -f draft="$DRAFT" + if [ "$DRAFT" = true ]; then + echo "::notice::$TAG is a draft release: add the upgrade notes and publish it to push the images and chart" + else + gh workflow run publish.yml -R "$REPO" --ref "$TAG" + fi echo "Tagged $TAG at $SHA" diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index b7ac470..0fadcbf 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -5,6 +5,10 @@ on: push: tags: - "v*" + # A draft release made by monthly-release.yml is published by hand. Releases + # published with GITHUB_TOKEN don't start this. + release: + types: [published] permissions: {} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 64c1e47..491d6ec 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -6,6 +6,11 @@ on: - "v*" # Started by monthly-release.yml on the tag it creates. workflow_dispatch: + inputs: + draft: + description: Create the release as a draft + type: boolean + default: false permissions: contents: write @@ -33,6 +38,6 @@ jobs: - uses: goreleaser/goreleaser-action@a4f02ab5355a589de60d917a659ce617ed19b3de # v7.3.0 with: version: "~> v2" - args: release --clean + args: release --clean${{ inputs.draft && ' --draft' || '' }} env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6359283..8cee063 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -224,6 +224,8 @@ Open an issue before a bigger change: anything that changes behaviour users rely A release is cut on the 1st of each month by `.github/workflows/monthly-release.yml` when `main` is green and something has merged since the last tag. It is a patch release, or a new minor version if a pull request labelled `breaking` was merged since then (a new major version from 1.0 on), so label pull requests that change behaviour users rely on. +A patch release is published on its own. A breaking release is created as a draft, without images or a chart: write the upgrade notes in it and publish it, which starts `publish.yml` for the images and the chart. + Scheduled runs only report what they would release until the repository variable `MONTHLY_RELEASE` is set to `true`. The workflow can also be started by hand from the Actions tab, as a dry run by default. To release by hand, push a `v*` tag. From fb346ae3559130fbdc48f52d2f7c9b3911fbbd1a Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Tue, 27 Oct 2026 12:30:12 +0100 Subject: [PATCH 459/633] Hold an unrecorded object for a client reading its stored copy A client that fell behind is switched to the stored copy, which for an unrecorded refetch is deleted once the waiters close. It now takes its own hold, released when it closes. --- internal/handler/handler.go | 1 + internal/handler/stream_miss.go | 41 +++++++++- internal/handler/unrecorded_test.go | 113 ++++++++++++++++------------ 3 files changed, 104 insertions(+), 51 deletions(-) diff --git a/internal/handler/handler.go b/internal/handler/handler.go index a2cd811..b3ca1c4 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -1052,6 +1052,7 @@ func (p *Proxy) runSharedFetch(ctx context.Context, key string, f *inflightFetch f.unrecorded = &unrecordedObject{delete: func() { p.deleteUnrecorded(path) }} f.unrecorded.refs.Add(1) // until the waiters are counted if handedOver { + h.holdUnrecorded(f.unrecorded) return nil, nil } f.unrecorded.refs.Add(1) // this caller diff --git a/internal/handler/stream_miss.go b/internal/handler/stream_miss.go index bc401b2..fb8ae62 100644 --- a/internal/handler/stream_miss.go +++ b/internal/handler/stream_miss.go @@ -46,6 +46,9 @@ type missHandoff struct { stream *CacheResult taken bool abandoned bool + // feed is the stream's source. Set and read only on the fetch's + // goroutine. + feed *missFeed } type missHandoffKey struct{} @@ -67,6 +70,14 @@ func (h *missHandoff) offer(stream *CacheResult) bool { return true } +// holdUnrecorded gives a client that fell behind, and reads the stored copy +// of u, its own hold on u until it closes, so u isn't deleted under it. +func (h *missHandoff) holdUnrecorded(u *unrecordedObject) { + if h.feed != nil { + h.feed.holdUntilClose(u) + } +} + // handedOver reports whether the caller took a stream. func (h *missHandoff) handedOver() bool { h.mu.Lock() @@ -179,6 +190,7 @@ func (p *Proxy) storeWhileServing(ctx context.Context, h *missHandoff, ecosystem _ = feed.Close() return p.storeArtifact(ctx, ecosystem, name, version, filename, pkgPURL, versionPURL, upstreamURL, upstreamHash, declared, artifact) } + h.feed = feed storeR, storeW := io.Pipe() type stored struct { @@ -207,7 +219,8 @@ func (p *Proxy) storeWhileServing(ctx context.Context, h *missHandoff, ecosystem out := <-storeDone if feed.lagging() { - // Opened before returning, so an unrecorded object is still there. + // Opened before returning, so an unrecorded object is still there; + // runSharedFetch then gives the client a hold on it. feed.resolve(p.openStoredForFeed(ctx, out.artifact, out.path, out.err)) } if out.err != nil && readErr == nil && feed.fedFully() { @@ -220,7 +233,8 @@ func (p *Proxy) storeWhileServing(ctx context.Context, h *missHandoff, ecosystem } // openStoredForFeed opens the stored copy for a client that fell behind. An -// unrecorded object is opened too: it is deleted only after this returns. +// unrecorded object is opened too: it is deleted only after this returns, and +// the client holds it from then on (missHandoff.holdUnrecorded). func (p *Proxy) openStoredForFeed(ctx context.Context, a artifacts.Artifact, path string, storeErr error) (io.ReadCloser, error) { if storeErr != nil && !errors.Is(storeErr, errUnrecorded) { return nil, storeErr @@ -291,6 +305,8 @@ type missFeed struct { storedErr error resolved bool skipped bool + // release drops the client's hold on an unrecorded stored copy. + release func() } func newMissFeed(limit int) *missFeed { @@ -353,6 +369,18 @@ func (m *missFeed) resolve(r io.ReadCloser, err error) { m.cond.Broadcast() } +// holdUntilClose takes a hold on u for a client reading its stored copy and +// releases it when the client closes. +func (m *missFeed) holdUntilClose(u *unrecordedObject) { + m.mu.Lock() + defer m.mu.Unlock() + if m.closed || m.stored == nil { + return + } + u.refs.Add(1) + m.release = u.release +} + func (m *missFeed) Read(p []byte) (int, error) { m.mu.Lock() for { @@ -396,8 +424,8 @@ func (m *missFeed) Read(p []byte) (int, error) { func (m *missFeed) Close() error { m.mu.Lock() - defer m.mu.Unlock() if m.closed { + m.mu.Unlock() return nil } m.closed = true @@ -406,7 +434,14 @@ func (m *missFeed) Close() error { if m.stored != nil { _ = m.stored.Close() } + release := m.release + m.release = nil m.cond.Broadcast() + m.mu.Unlock() + if release != nil { + // Outside the lock: the last release deletes the object. + release() + } return nil } diff --git a/internal/handler/unrecorded_test.go b/internal/handler/unrecorded_test.go index 75325f2..d87ddc9 100644 --- a/internal/handler/unrecorded_test.go +++ b/internal/handler/unrecorded_test.go @@ -342,57 +342,74 @@ func TestRecordedAt(t *testing.T) { } // TestUnrecordedStreamedMiss streams an unrecorded refetch to the caller -// that started it: it holds no reader on the stored object, so the object is -// deleted once the waiters are done, and the waiters still get it. +// that started it, and the waiters still get the stored object. A caller fed +// from upstream holds no reader on it, so it is deleted once the waiters are +// done; one that fell behind reads it, so it is deleted once that caller is +// done too. func TestUnrecordedStreamedMiss(t *testing.T) { for _, behind := range []bool{false, true} { t.Run(map[bool]string{false: "fed", true: "behind"}[behind], func(t *testing.T) { - if behind { - old := clientFeedLimit - clientFeedLimit = 4 - t.Cleanup(func() { clientFeedLimit = old }) - } - proxy, store := setupUnrecorded(t) - fetcher := newGatedFetcher(unrecordedContent) - proxy.Fetcher = fetcher - - leader := make(chan *CacheResult, 1) - go func() { - res, err := fetchGone(StreamCacheMiss(context.Background()), proxy) - if err != nil { - t.Errorf("leader: %v", err) - } - leader <- res - }() - <-fetcher.entered - waiter := make(chan *CacheResult, 1) - go func() { - res, err := fetchGone(context.Background(), proxy) - if err != nil { - t.Errorf("waiter: %v", err) - } - waiter <- res - }() - waitForWaiters(t, proxy, 1) - close(fetcher.release) - - lead := <-leader - if lead == nil || lead.Artifact.Size != -1 { - t.Fatalf("leader = %+v, want a stream", lead) - } - w := <-waiter - if w == nil { - t.FailNow() - } - readAndClose(t, w) - deadline := time.Now().Add(5 * time.Second) - for len(storedFiles(store)) != 0 && time.Now().Before(deadline) { - time.Sleep(time.Millisecond) - } - if got := storedFiles(store); len(got) != 0 { - t.Errorf("stored files after the waiter closed = %v, want none", got) - } - readAndClose(t, lead) + testUnrecordedStreamedMiss(t, behind) }) } } + +func testUnrecordedStreamedMiss(t *testing.T, behind bool) { + t.Helper() + if behind { + old := clientFeedLimit + clientFeedLimit = 4 + t.Cleanup(func() { clientFeedLimit = old }) + } + proxy, store := setupUnrecorded(t) + fetcher := newGatedFetcher(unrecordedContent) + proxy.Fetcher = fetcher + + leader := make(chan *CacheResult, 1) + go func() { + res, err := fetchGone(StreamCacheMiss(context.Background()), proxy) + if err != nil { + t.Errorf("leader: %v", err) + } + leader <- res + }() + <-fetcher.entered + waiter := make(chan *CacheResult, 1) + go func() { + res, err := fetchGone(context.Background(), proxy) + if err != nil { + t.Errorf("waiter: %v", err) + } + waiter <- res + }() + waitForWaiters(t, proxy, 1) + close(fetcher.release) + + lead := <-leader + if lead == nil || lead.Artifact.Size != -1 { + t.Fatalf("leader = %+v, want a stream", lead) + } + w := <-waiter + if w == nil { + t.FailNow() + } + readAndClose(t, w) + if behind { + // The leader reads the stored copy, so it must outlive the + // waiter. + if len(storedFiles(store)) == 0 { + t.Fatal("stored copy deleted while the leader still reads it") + } + readAndClose(t, lead) + } + deadline := time.Now().Add(5 * time.Second) + for len(storedFiles(store)) != 0 && time.Now().Before(deadline) { + time.Sleep(time.Millisecond) + } + if got := storedFiles(store); len(got) != 0 { + t.Errorf("stored files after every reader closed = %v, want none", got) + } + if !behind { + readAndClose(t, lead) + } +} From b79cd4d14ad2566ed9d7590ad3c57d4d7a901749 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Tue, 27 Oct 2026 13:03:42 +0100 Subject: [PATCH 460/633] Show the cooldown configuration in the web UI The dashboard shows the default and the ecosystem overrides, and counts package overrides and patterns without naming them, since the UI has no auth. A package page shows the cooldown that applies to the package and the kind of rule it comes from. Also add FillVersionPublishedAt, which stores a publish time only where none is stored, for recording the versions cooldown holds back. --- docs/configuration.md | 2 + internal/database/database_test.go | 42 ++++++ internal/database/queries.go | 30 ++++ internal/server/cooldown_view.go | 122 ++++++++++++++++ internal/server/cooldown_view_test.go | 137 ++++++++++++++++++ internal/server/dashboard.go | 3 + internal/server/server.go | 6 + .../server/templates/pages/dashboard.html | 15 ++ .../server/templates/pages/package_show.html | 3 + 9 files changed, 360 insertions(+) create mode 100644 internal/server/cooldown_view.go create mode 100644 internal/server/cooldown_view_test.go diff --git a/docs/configuration.md b/docs/configuration.md index f10e4b2..b0dee95 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1076,6 +1076,8 @@ Exact `packages` entries take precedence over patterns. When several patterns ma Resolution order: exact package override, then package pattern, then ecosystem override, then global default. This lets you set a conservative default while exempting trusted package families. +The web UI's dashboard shows the default and the ecosystem overrides, and how many package overrides and patterns there are; their names aren't shown, since the UI has no authentication. A package's page shows the cooldown that applies to it and which kind of rule it comes from. + Currently supported for npm, PyPI, pub.dev, Composer, Cargo, NuGet, Conda, RubyGems, Hex, Go and Maven. Most of these ecosystems include publish timestamps in their metadata; Go and Maven are described below. Go has no publish time. The module proxy's `.info` gives the commit time of the revision a version points at, so a version tagged on an old commit counts as old. The proxy reads each listed version's `.info` once, stores the time, and leaves versions in cooldown out of `@v/list` and `@latest`. When upstream's `@latest` is in cooldown, it serves the newest release out of cooldown instead. Requests for a version in cooldown, including its `.info`, `.mod` and `.zip`, and a branch or commit query that resolves to one, get 403 rather than 404, because the go command moves on to the next `GOPROXY` entry, such as `direct`, after a 404. If a version's `.info` can't be fetched, the list or download fails with 502 instead of letting the version through. diff --git a/internal/database/database_test.go b/internal/database/database_test.go index f8956a6..09f5a93 100644 --- a/internal/database/database_test.go +++ b/internal/database/database_test.go @@ -1112,6 +1112,48 @@ func BenchmarkMigrateSchemaFullyMigrated(b *testing.B) { } } +func TestFillVersionPublishedAt(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + const purl, pkg = "pkg:npm/leftpad@1.0.0", "pkg:npm/leftpad" + first := time.Date(2020, 1, 2, 3, 4, 5, 0, time.UTC) + publishedAt := func() sql.NullTime { + t.Helper() + got, err := db.GetVersionByPURL(purl) + if err != nil || got == nil { + t.Fatalf("GetVersionByPURL: %v, %v", got, err) + } + return got.PublishedAt + } + + // A row without a time, as the artifact cache writes it, is filled. + if err := db.UpsertVersion(&Version{PURL: purl, PackagePURL: pkg}); err != nil { + t.Fatalf("UpsertVersion: %v", err) + } + if err := db.FillVersionPublishedAt(purl, pkg, first.In(time.FixedZone("CEST", 2*60*60))); err != nil { + t.Fatalf("FillVersionPublishedAt: %v", err) + } + if got := publishedAt(); !got.Valid || !got.Time.Equal(first) { + t.Fatalf("PublishedAt = %v, want %v", got, first) + } + + // A stored time is kept. + if err := db.FillVersionPublishedAt(purl, pkg, first.Add(time.Hour)); err != nil { + t.Fatalf("FillVersionPublishedAt again: %v", err) + } + if got := publishedAt(); !got.Time.Equal(first) { + t.Fatalf("PublishedAt after a second fill = %v, want %v kept", got.Time, first) + } + + // A version the proxy hasn't seen gets a row. + if err := db.FillVersionPublishedAt("pkg:npm/leftpad@2.0.0", pkg, first); err != nil { + t.Fatalf("FillVersionPublishedAt new: %v", err) + } + if got, err := db.GetVersionByPURL("pkg:npm/leftpad@2.0.0"); err != nil || got == nil || !got.PublishedAt.Time.Equal(first) { + t.Fatalf("new version = %+v, %v, want published %v", got, err, first) + } + }) +} + func TestVersionPublishedAtPreserved(t *testing.T) { runWithBothDatabases(t, func(t *testing.T, db *DB) { publishedAt := time.Date(2020, 1, 2, 3, 4, 5, 0, time.UTC) diff --git a/internal/database/queries.go b/internal/database/queries.go index d7150e8..0bd5e82 100644 --- a/internal/database/queries.go +++ b/internal/database/queries.go @@ -223,6 +223,36 @@ func (db *DB) SetVersionPublishedAt(versionPURL, packagePURL string, publishedAt return nil } +// FillVersionPublishedAt records a version's publish time, in UTC, only where +// none is stored yet, creating the versions row if needed. Callers that see +// the same version on every metadata request write it once without reading +// first. +func (db *DB) FillVersionPublishedAt(versionPURL, packagePURL string, publishedAt time.Time) error { + now := time.Now() + query := ` + INSERT INTO versions (purl, package_purl, published_at, created_at, updated_at) + VALUES (?, ?, ?, ?, ?) + ON CONFLICT(purl) DO UPDATE SET + published_at = excluded.published_at, + updated_at = excluded.updated_at + WHERE versions.published_at IS NULL + ` + if db.dialect == DialectPostgres { + query = ` + INSERT INTO versions (purl, package_purl, published_at, created_at, updated_at) + VALUES ($1, $2, $3, $4, $5) + ON CONFLICT(purl) DO UPDATE SET + published_at = EXCLUDED.published_at, + updated_at = EXCLUDED.updated_at + WHERE versions.published_at IS NULL + ` + } + if _, err := db.Exec(query, versionPURL, packagePURL, publishedAt.UTC(), now, now); err != nil { + return fmt.Errorf("filling version publish time: %w", err) + } + return nil +} + // Artifact queries func (db *DB) GetArtifact(versionPURL, filename string) (*Artifact, error) { diff --git a/internal/server/cooldown_view.go b/internal/server/cooldown_view.go new file mode 100644 index 0000000..57a64ec --- /dev/null +++ b/internal/server/cooldown_view.go @@ -0,0 +1,122 @@ +package server + +import ( + "fmt" + "sort" + "time" + + "github.com/git-pkgs/cooldown" + "github.com/git-pkgs/proxy/internal/config" + "github.com/git-pkgs/proxy/internal/packageurl" +) + +// CooldownSummary describes the cooldown configuration on the dashboard. The +// UI has no auth, so package overrides and patterns are only counted: their +// names often list internal packages. +type CooldownSummary struct { + Enabled bool + Default string + Ecosystems []CooldownOverride + PackageOverrides int + PackagePatterns int +} + +// CooldownOverride is one ecosystem's cooldown. +type CooldownOverride struct { + Ecosystem string + Duration string +} + +// PackageCooldown is the cooldown that applies to one package, and the kind +// of rule it comes from. +type PackageCooldown struct { + Duration string + Rule string +} + +// cooldownView answers the UI's cooldown questions from the configuration. +type cooldownView struct { + cfg config.CooldownConfig + policy *cooldown.Policy + packages map[string]string + // patterns sets every package pattern to the same non-zero duration over + // a zero default, so a non-zero result means a pattern matched, with the + // policy's own matching. + patterns *cooldown.Policy +} + +func newCooldownView(cfg config.CooldownConfig, policy *cooldown.Policy) (*cooldownView, error) { + probe := make(map[string]string, len(cfg.PackagePatterns)) + for glob := range cfg.PackagePatterns { + probe[glob] = "1h" + } + patterns, err := cooldown.NewPolicy(nil, probe) + if err != nil { + return nil, fmt.Errorf("configuring cooldown package patterns: %w", err) + } + return &cooldownView{cfg: cfg, policy: policy, packages: cfg.NormalizedPackages(), patterns: patterns}, nil +} + +func (v *cooldownView) enabled() bool { + return v != nil && v.policy != nil && v.policy.Enabled() +} + +// summary returns the dashboard's view of the configuration. +func (v *cooldownView) summary() CooldownSummary { + if !v.enabled() { + return CooldownSummary{} + } + s := CooldownSummary{ + Enabled: true, + Default: formatConfigDuration(v.cfg.Default), + PackageOverrides: len(v.cfg.Packages), + PackagePatterns: len(v.cfg.PackagePatterns), + } + for eco, d := range v.cfg.Ecosystems { + s.Ecosystems = append(s.Ecosystems, CooldownOverride{Ecosystem: eco, Duration: formatConfigDuration(d)}) + } + sort.Slice(s.Ecosystems, func(i, j int) bool { return s.Ecosystems[i].Ecosystem < s.Ecosystems[j].Ecosystem }) + return s +} + +// forPackage returns the cooldown for a package, or nil with cooldown off. +func (v *cooldownView) forPackage(ecosystem, name string) *PackageCooldown { + if !v.enabled() { + return nil + } + purl := packageurl.MakeString(ecosystem, name, "") + rule := "default" + if _, ok := v.packages[purl]; ok { + rule = "package override" + } else if v.patterns.For(ecosystem, purl) != 0 { + rule = "package pattern" + } else if _, ok := v.cfg.Ecosystems[ecosystem]; ok { + rule = "ecosystem override" + } + return &PackageCooldown{Duration: formatCooldown(v.policy.For(ecosystem, purl)), Rule: rule} +} + +func formatConfigDuration(s string) string { + d, err := cooldown.ParseDuration(s) + if err != nil { + return s + } + return formatCooldown(d) +} + +// formatCooldown writes d as the configuration does: "3d", "12h", "30m", +// and "off" for zero. +func formatCooldown(d time.Duration) string { + const day = 24 * time.Hour + switch { + case d <= 0: + return "off" + case d%day == 0: + return fmt.Sprintf("%dd", d/day) + case d%time.Hour == 0: + return fmt.Sprintf("%dh", d/time.Hour) + case d%time.Minute == 0: + return fmt.Sprintf("%dm", d/time.Minute) + } + return d.String() +} diff --git a/internal/server/cooldown_view_test.go b/internal/server/cooldown_view_test.go new file mode 100644 index 0000000..41ff789 --- /dev/null +++ b/internal/server/cooldown_view_test.go @@ -0,0 +1,137 @@ +package server + +import ( + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/git-pkgs/cooldown" + "github.com/git-pkgs/proxy/internal/config" + "github.com/git-pkgs/proxy/internal/database" +) + +func testCooldownView(t *testing.T, cfg config.CooldownConfig) *cooldownView { + t.Helper() + policy, err := cooldown.NewPolicy(&cooldown.Config{ + Default: cfg.Default, + Ecosystems: cfg.Ecosystems, + Packages: cfg.NormalizedPackages(), + }, cfg.PackagePatterns) + if err != nil { + t.Fatalf("NewPolicy: %v", err) + } + v, err := newCooldownView(cfg, policy) + if err != nil { + t.Fatalf("newCooldownView: %v", err) + } + return v +} + +func TestCooldownViewForPackage(t *testing.T) { + v := testCooldownView(t, config.CooldownConfig{ + Default: "3d", + Ecosystems: map[string]string{"npm": "7d", "composer": "48h"}, + Packages: map[string]string{"pkg:composer/phpunit/phpunit": "0"}, + PackagePatterns: map[string]string{"pkg:npm/@corp/*": "0", "pkg:composer/acme/*": "12h"}, + }) + tests := []struct { + ecosystem, name string + want PackageCooldown + }{ + {"cargo", "serde", PackageCooldown{"3d", "default"}}, + {"npm", "lodash", PackageCooldown{"7d", "ecosystem override"}}, + {"composer", "monolog/monolog", PackageCooldown{"2d", "ecosystem override"}}, + {"composer", "phpunit/phpunit", PackageCooldown{"off", "package override"}}, + {"composer", "acme/tool", PackageCooldown{"12h", "package pattern"}}, + {"npm", "@corp/ui", PackageCooldown{"off", "package pattern"}}, + } + for _, tt := range tests { + got := v.forPackage(tt.ecosystem, tt.name) + if got == nil || *got != tt.want { + t.Errorf("forPackage(%s, %s) = %+v, want %+v", tt.ecosystem, tt.name, got, tt.want) + } + } +} + +func TestCooldownViewOff(t *testing.T) { + var unset *cooldownView + for _, v := range []*cooldownView{unset, testCooldownView(t, config.CooldownConfig{})} { + if got := v.forPackage("npm", "lodash"); got != nil { + t.Errorf("forPackage with cooldown off = %+v, want nil", got) + } + if got := v.summary(); got.Enabled { + t.Errorf("summary with cooldown off = %+v, want disabled", got) + } + } +} + +// TestCooldownViewOnlyPattern checks that a pattern alone turns cooldown on. +func TestCooldownViewOnlyPattern(t *testing.T) { + v := testCooldownView(t, config.CooldownConfig{PackagePatterns: map[string]string{"pkg:npm/*": "1d"}}) + if got := v.forPackage("npm", "lodash"); got == nil || *got != (PackageCooldown{"1d", "package pattern"}) { + t.Errorf("forPackage = %+v, want 1d from the pattern", got) + } + if got := v.forPackage("cargo", "serde"); got == nil || *got != (PackageCooldown{"off", "default"}) { + t.Errorf("forPackage outside the pattern = %+v, want off by default", got) + } +} + +func TestFormatCooldown(t *testing.T) { + for d, want := range map[time.Duration]string{ + 0: "off", + 72 * time.Hour: "3d", + 36 * time.Hour: "36h", + 90 * time.Minute: "90m", + 90*time.Minute + 30*time.Second: "1h30m30s", + } { + if got := formatCooldown(d); got != want { + t.Errorf("formatCooldown(%v) = %q, want %q", d, got, want) + } + } +} + +// TestCooldownPages checks that the dashboard shows the configuration +// without the names of package overrides and patterns, and the package page +// the cooldown that applies to the package. +func TestCooldownPages(t *testing.T) { + ts := newTestServer(t) + defer ts.close() + ts.server.cooldown = testCooldownView(t, config.CooldownConfig{ + Default: "3d", + Ecosystems: map[string]string{"npm": "7d"}, + Packages: map[string]string{"pkg:npm/internal-secret": "0"}, + PackagePatterns: map[string]string{"pkg:npm/@corp-secret/*": "0"}, + }) + if err := ts.db.UpsertPackage(&database.Package{PURL: "pkg:npm/lodash", Ecosystem: "npm", Name: "lodash"}); err != nil { + t.Fatalf("UpsertPackage: %v", err) + } + + get := func(path string) string { + t.Helper() + w := httptest.NewRecorder() + ts.handler.ServeHTTP(w, httptest.NewRequest(http.MethodGet, path, nil)) + if w.Code != http.StatusOK { + t.Fatalf("GET %s: status %d", path, w.Code) + } + return w.Body.String() + } + + dashboard := get("/ui/") + for _, want := range []string{"Cooldown", "3d", "npm:", "7d", "Package overrides:", "Package patterns:"} { + if !strings.Contains(dashboard, want) { + t.Errorf("dashboard is missing %q", want) + } + } + for _, secret := range []string{"internal-secret", "corp-secret"} { + if strings.Contains(dashboard, secret) { + t.Errorf("dashboard shows %q", secret) + } + } + + page := get("/ui/package/npm/lodash") + if !strings.Contains(page, "7d") || !strings.Contains(page, "(ecosystem override)") { + t.Error("package page doesn't show the 7d ecosystem override") + } +} diff --git a/internal/server/dashboard.go b/internal/server/dashboard.go index da866a6..7e14af6 100644 --- a/internal/server/dashboard.go +++ b/internal/server/dashboard.go @@ -13,6 +13,7 @@ type DashboardData struct { Layout Stats DashboardStats EnrichmentStats EnrichmentStatsView + Cooldown CooldownSummary RecentPackages []PackageInfo PopularPackages []PackageInfo } @@ -68,6 +69,8 @@ type PackageShowData struct { Versions []database.Version Vulnerabilities []database.Vulnerability LicenseCategory string + // Cooldown is nil with cooldown off. + Cooldown *PackageCooldown } // VersionShowData contains data for rendering the version show page. diff --git a/internal/server/server.go b/internal/server/server.go index 12619a6..ea87e08 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -126,6 +126,7 @@ type Server struct { sources sourceTracker scheduler *mirror.Scheduler git *gitcache.Cache + cooldown *cooldownView // browseLimits overrides defaultBrowseLimits when set; tests use it. browseLimits archives.StreamOptions @@ -254,6 +255,9 @@ func (s *Server) serve(listener net.Listener) error { return fmt.Errorf("configuring cooldown policy: %w", err) } proxy.Cooldown = cooldownPolicy + if s.cooldown, err = newCooldownView(s.cfg.Cooldown, cooldownPolicy); err != nil { + return err + } policy, err := denylist.New(s.cfg.Denylist.Packages) if err != nil { return fmt.Errorf("configuring denylist: %w", err) @@ -832,6 +836,7 @@ func (s *Server) handleRoot(w http.ResponseWriter, r *http.Request) { TotalVersions: stats.TotalVersions, }, EnrichmentStats: enrichmentStatsView(enrichStats), + Cooldown: s.cooldown.summary(), } for _, p := range popular { @@ -1169,6 +1174,7 @@ func (s *Server) showPackage(w http.ResponseWriter, r *http.Request, ecosystem, Versions: versions, Vulnerabilities: vulns, LicenseCategory: categorizeLicense(pkg.License), + Cooldown: s.cooldown.forPackage(pkg.Ecosystem, pkg.Name), } if err := s.templates.Render(w, "package_show", data); err != nil { diff --git a/internal/server/templates/pages/dashboard.html b/internal/server/templates/pages/dashboard.html index e48e205..eb4c786 100644 --- a/internal/server/templates/pages/dashboard.html +++ b/internal/server/templates/pages/dashboard.html @@ -25,6 +25,21 @@ {{template "security_overview" .EnrichmentStats}} {{end}} +{{with .Cooldown}}{{if .Enabled}} +
+

Cooldown

+

Versions published more recently than this are left out of package metadata.

+
+
Default:
{{.Default}}
+ {{range .Ecosystems}} +
{{.Ecosystem}}:
{{.Duration}}
+ {{end}} + {{if .PackageOverrides}}
Package overrides:
{{.PackageOverrides}}
{{end}} + {{if .PackagePatterns}}
Package patterns:
{{.PackagePatterns}}
{{end}} +
+
+{{end}}{{end}} +
diff --git a/internal/server/templates/pages/package_show.html b/internal/server/templates/pages/package_show.html index a2724a8..b0dd29a 100644 --- a/internal/server/templates/pages/package_show.html +++ b/internal/server/templates/pages/package_show.html @@ -18,6 +18,9 @@ {{if .Package.LatestVersion.Valid}}
Latest Version:
{{.Package.LatestVersion.String}}
{{end}} + {{with .Cooldown}} +
Cooldown:
{{.Duration}} ({{.Rule}})
+ {{end}} {{if .Package.License.Valid}}
License:
{{.Package.License.String}}
{{end}} From b2553ddd49187fa9f3f6c243ced07b86121bd105 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Tue, 27 Oct 2026 16:15:31 +0100 Subject: [PATCH 461/633] Pass Composer security advisories and filter lists through The proxy's packages.json listed only metadata, search and notify URLs, so a client using the proxy as its only Composer repository lost `composer audit`, `audit.block-insecure` and Packagist's malware filter list without any warning. The service index now carries the security-advisories and filter entries of the upstream repository's own packages.json, cached for the metadata TTL, with URLs pointing at the proxy. An upstream that does not offer them gets nothing advertised, so a custom upstream.composer_repository without the API keeps working as before. Advisories and filter entries embedded in p2 metadata were already passed on unchanged. Advisories queries and the filter summary are relayed uncached. Names of routed packages and hosted-only vendors are removed from advisories queries first, so private package names are not sent to the public API. --- README.md | 11 + docs/configuration.md | 3 + internal/handler/composer.go | 14 +- internal/handler/composer_advisories.go | 268 ++++++++++++++++ internal/handler/composer_advisories_test.go | 308 +++++++++++++++++++ 5 files changed, 602 insertions(+), 2 deletions(-) create mode 100644 internal/handler/composer_advisories.go create mode 100644 internal/handler/composer_advisories_test.go diff --git a/README.md b/README.md index 20c6d5d..eb0db0e 100644 --- a/README.md +++ b/README.md @@ -387,6 +387,15 @@ Or set globally: composer config -g repositories.proxy composer http://localhost:8080/composer ``` +`composer audit`, `audit.block-insecure` and Packagist's malware filter list +work through the proxy when `upstream.composer_repository` offers them in its +`packages.json`, as repo.packagist.org does. The proxy advertises them in its +own `packages.json` and passes the advisories and filter list requests through +to the upstream uncached. Names of packages in `upstream.composer_routes` or +`hosted.composer_vendors` are removed from advisories requests before they are +sent. Versions withheld by [cooldown](#version-cooldown) are not excluded from +the report. + To serve private packages through the same URL, see [Private package routes](docs/configuration.md#private-package-routes). ### Conan (C/C++) @@ -1263,6 +1272,8 @@ Recently cached: | `GET /maven/*` | Maven repository protocol | | `GET /nuget/*` | NuGet V3 API | | `GET /composer/*` | Composer/Packagist protocol | +| `GET`, `POST /composer/api/security-advisories/` | Composer security advisories, passed through to upstream without private package names | +| `GET /composer/lists/summary.json` | Composer filter list summary, passed through to upstream uncached | | `GET /conan/*` | Conan C/C++ protocol | | `GET /chef/*` | Chef Supermarket protocol (universe, cookbook API and downloads) | | `GET /conda/*` | Conda/Anaconda protocol | diff --git a/docs/configuration.md b/docs/configuration.md index f10e4b2..5497b91 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -339,6 +339,8 @@ upstream: composer_repository: "https://upstream-proxy.example.com/composer" ``` +The proxy reads `packages.json` from `upstream.composer_repository`, cached for the metadata TTL, and passes on its `security-advisories` and `filter` entries (Packagist's malware list) with URLs pointing at the proxy. An upstream that offers neither is still usable; `composer audit` then reports nothing from it. A filter `api-url` is not passed on; Composer reads filter entries from package metadata instead. + `upstream.hex_api` is used for cooldown timestamps and must expose Hex's `/api/packages/{name}` JSON endpoint. `upstream.chef` is the Supermarket served at `/chef/`, for Berkshelf (`source "https://proxy.example.com/chef"`) and Policyfiles (`default_source :supermarket, "https://proxy.example.com/chef"`). The upstream has to serve the Supermarket API: `/universe`, `/api/v1/cookbooks/{name}`, its versions and their downloads. A private Supermarket that needs credentials takes them from `upstream.auth`, scoped to its URL. A download redirect to another host, such as the storage bucket behind Supermarket, does not receive them: @@ -379,6 +381,7 @@ upstream: - Composer metadata of routed packages carries an empty `notification-url`, so Composer does not report their installs to the default upstream's `notify-batch` endpoint. - Use `upstream.auth` for the registry's credentials. The proxy has no authentication of its own, so every client that can reach it can download routed packages. - Search (`/composer/search.json`) and the package list (`/composer/packages/list.json`) still come from the default upstream only. +- Composer security advisories and filter lists come from the default upstream. Routed package names, and those in `hosted.composer_vendors`, are removed from advisories requests before they are sent, so they only get advisories that their route's own metadata carries. - npm audit, signing-key, search and ping requests still go to `upstream.npm`. An audit request lists every package in the dependency tree, routed ones included. - Routes can only be set in the configuration file, not with environment variables. diff --git a/internal/handler/composer.go b/internal/handler/composer.go index 116d106..bf04da0 100644 --- a/internal/handler/composer.go +++ b/internal/handler/composer.go @@ -100,12 +100,19 @@ func (h *ComposerHandler) Routes() http.Handler { mux.HandleFunc("GET /search.json", h.proxyUpstream) mux.HandleFunc("GET /packages/list.json", h.proxyUpstream) + // Security advisories and filter lists, as the default repository offers them + mux.HandleFunc("GET "+composerAdvisoriesPath, h.handleSecurityAdvisories) + mux.HandleFunc("POST "+composerAdvisoriesPath, h.handleSecurityAdvisories) + mux.HandleFunc("GET "+composerFilterSummaryPath, h.handleFilterSummary) + return mux } -// handleServiceIndex returns the Composer repository service index. +// handleServiceIndex returns the Composer repository service index. It +// carries the security advisories and filter list entries of the default +// repository's own index when that has them; if the repository cannot be +// reached and nothing is cached, the index goes out without them. func (h *ComposerHandler) handleServiceIndex(w http.ResponseWriter, r *http.Request) { - // Return a minimal service index pointing to our proxy index := map[string]any{ "packages": map[string]any{}, "metadata-url": h.proxyURL + "/composer/p2/%package%.json", @@ -113,6 +120,9 @@ func (h *ComposerHandler) handleServiceIndex(w http.ResponseWriter, r *http.Requ "search": h.proxyURL + "/composer/search.json?q=%query%&type=%type%", "providers-lazy-url": h.proxyURL + "/composer/p2/%package%.json", } + if upstream, ok := h.upstreamIndex(r.Context()); ok { + upstream.advertise(index, h.proxyURL) + } w.Header().Set(headerContentType, "application/json") _ = json.NewEncoder(w).Encode(index) diff --git a/internal/handler/composer_advisories.go b/internal/handler/composer_advisories.go new file mode 100644 index 0000000..e77da37 --- /dev/null +++ b/internal/handler/composer_advisories.go @@ -0,0 +1,268 @@ +package handler + +import ( + "context" + "encoding/json" + "errors" + "io" + "net/http" + "net/url" + "strings" +) + +const ( + composerAdvisoriesPath = "/api/security-advisories/" + composerFilterSummaryPath = "/lists/summary.json" + composerPackagesParam = "packages" + + // composerAdvisoriesMaxBody caps an advisories query. Composer sends at + // most 500 package names per request. + composerAdvisoriesMaxBody = 1 << 20 +) + +// composerUpstreamIndex holds the parts of the default repository's +// packages.json that the proxy advertises in its own: the security +// advisories API and the filter lists (Packagist's malware list). Composer +// reads advisories and filter entries for each package from its p2 metadata, +// which the proxy passes on unchanged, and asks these URLs for the rest. +type composerUpstreamIndex struct { + advisoriesURL string + advisoriesMetadata bool + filterLists json.RawMessage + filterSummaryURL string +} + +// upstreamIndex fetches the default repository's packages.json, cached for +// the metadata TTL. ok is false when the repository could not be reached and +// nothing is cached; a document without the fields yields an empty index. +func (h *ComposerHandler) upstreamIndex(ctx context.Context) (composerUpstreamIndex, bool) { + indexURL := h.repoURL + "/packages.json" + cacheKey := hashedCacheKey(h.repoURL, "/packages.json") + res := h.proxy.fetchMetadataDocument(ctx, "composer", cacheKey, indexURL, contentTypeJSON, nil) + if res.err != nil { + h.proxy.Logger.Warn("failed to fetch upstream composer packages.json", "url", indexURL, "error", res.err) + return composerUpstreamIndex{}, false + } + return parseComposerUpstreamIndex(indexURL, res.body), true +} + +func parseComposerUpstreamIndex(indexURL string, body []byte) composerUpstreamIndex { + var doc struct { + SecurityAdvisories struct { + Metadata bool `json:"metadata"` + APIURL string `json:"api-url"` + } `json:"security-advisories"` + Filter struct { + Metadata bool `json:"metadata"` + Lists json.RawMessage `json:"lists"` + SummaryURL string `json:"summary-url"` + } `json:"filter"` + } + var index composerUpstreamIndex + if err := json.Unmarshal(body, &doc); err != nil { + return index + } + // Composer refuses advisories configured without an api-url unless the + // repository lists its packages, which the proxy does not. + if apiURL, ok := resolveComposerURL(indexURL, doc.SecurityAdvisories.APIURL); ok { + index.advisoriesURL = apiURL + index.advisoriesMetadata = doc.SecurityAdvisories.Metadata + } + // Composer only applies filter lists a repository marks with metadata. + // A filter api-url is not passed on: without it Composer reads the + // entries from package metadata, narrowed by the summary. + if doc.Filter.Metadata && len(doc.Filter.Lists) > 0 && doc.Filter.Lists[0] == '{' { + index.filterLists = doc.Filter.Lists + index.filterSummaryURL, _ = resolveComposerURL(indexURL, doc.Filter.SummaryURL) + } + return index +} + +// resolveComposerURL resolves a URL from packages.json against the document's +// own, as Composer does, and accepts only http and https. +func resolveComposerURL(base, ref string) (string, bool) { + if ref == "" { + return "", false + } + b, err := url.Parse(base) + if err != nil { + return "", false + } + u, err := b.Parse(ref) + if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" { + return "", false + } + return u.String(), true +} + +// advertise adds the advisories and filter entries to the proxy's +// packages.json, pointing at the proxy. +func (index composerUpstreamIndex) advertise(packages map[string]any, proxyURL string) { + if index.advisoriesURL != "" { + packages["security-advisories"] = map[string]any{ + "metadata": index.advisoriesMetadata, + "api-url": proxyURL + "/composer" + composerAdvisoriesPath, + } + } + if index.filterLists != nil { + filter := map[string]any{"metadata": true, "lists": index.filterLists} + if index.filterSummaryURL != "" { + filter["summary-url"] = proxyURL + "/composer" + composerFilterSummaryPath + } + packages["filter"] = filter + } +} + +// handleSecurityAdvisories relays a security advisories query to the URL the +// default repository advertises. Composer sends the package names as +// packages[] in a POST form; a GET query is accepted too. Names served by a +// package route or a hosted-only vendor are removed first, so private package +// names are never sent to the public API. +// +// Responses are not cached: each query names a different set of packages. +func (h *ComposerHandler) handleSecurityAdvisories(w http.ResponseWriter, r *http.Request) { + query, err := h.advisoriesQuery(w, r) + if err != nil { + var maxBytesErr *http.MaxBytesError + if errors.As(err, &maxBytesErr) { + JSONError(w, http.StatusRequestEntityTooLarge, "advisories request too large") + return + } + JSONError(w, http.StatusBadRequest, "could not read advisories request") + return + } + + asked := len(query[composerPackagesParam+"[]"]) + public := h.publicPackageNames(query[composerPackagesParam+"[]"]) + if asked > 0 && len(public) == 0 { + w.Header().Set(headerContentType, contentTypeJSON) + _, _ = io.WriteString(w, `{"advisories":{}}`) + return + } + if asked > 0 { + query[composerPackagesParam+"[]"] = public + } + + index, ok := h.upstreamIndex(r.Context()) + if !ok { + JSONError(w, http.StatusBadGateway, "upstream repository unavailable") + return + } + if index.advisoriesURL == "" { + JSONError(w, http.StatusNotFound, "upstream repository has no security advisories API") + return + } + + req, err := h.advisoriesRequest(r, index.advisoriesURL, query) + if err != nil { + JSONError(w, http.StatusInternalServerError, "failed to create request") + return + } + h.proxy.Logger.Debug("composer advisories request", "packages", len(public), "withheld", asked-len(public)) + + resp, err := h.proxy.HTTPClient.Do(req) + if err != nil { + if writeUpstreamHeld(w, err) { + return + } + h.proxy.Logger.Error("composer advisories request failed", "error", err) + JSONError(w, http.StatusBadGateway, "failed to reach upstream repository") + return + } + defer func() { _ = resp.Body.Close() }() + + h.proxy.relayResponse(w, r, resp, nil) +} + +// advisoriesQuery reads the query of an advisories request, from the form +// body of a POST or the URL of a GET. Package names come back under +// "packages[]" whichever spelling the client used: PHP's http_build_query, +// which Composer uses, numbers them as packages[0], packages[1], ... +func (h *ComposerHandler) advisoriesQuery(w http.ResponseWriter, r *http.Request) (url.Values, error) { + raw := r.URL.RawQuery + if r.Method == http.MethodPost { + body, err := io.ReadAll(http.MaxBytesReader(w, r.Body, composerAdvisoriesMaxBody)) + if err != nil { + return nil, err + } + raw = string(body) + } + values, err := url.ParseQuery(raw) + if err != nil { + return nil, err + } + query := url.Values{} + for key, vals := range values { + if key == composerPackagesParam || strings.HasPrefix(key, composerPackagesParam+"[") { + key = composerPackagesParam + "[]" + } + query[key] = append(query[key], vals...) + } + return query, nil +} + +// publicPackageNames drops the names the default repository does not serve. +func (h *ComposerHandler) publicPackageNames(names []string) []string { + public := make([]string, 0, len(names)) + for _, name := range names { + if _, routed := h.routes.match(name); routed || h.proxy.hostedOnly("composer", name) { + continue + } + public = append(public, name) + } + return public +} + +// advisoriesRequest builds the upstream request, sending the query the way +// the client did. +func (h *ComposerHandler) advisoriesRequest(r *http.Request, apiURL string, query url.Values) (*http.Request, error) { + u, err := url.Parse(apiURL) + if err != nil { + return nil, err + } + var body io.Reader + if r.Method == http.MethodPost { + body = strings.NewReader(query.Encode()) + } else { + u.RawQuery = joinQuery(u.RawQuery, query.Encode()) + } + req, err := http.NewRequestWithContext(r.Context(), r.Method, u.String(), body) + if err != nil { + return nil, err + } + if r.Method == http.MethodPost { + req.Header.Set(headerContentType, "application/x-www-form-urlencoded") + } + for _, header := range []string{headerAccept, headerAcceptEncoding} { + if v := r.Header.Get(header); v != "" { + req.Header.Set(header, v) + } + } + return req, nil +} + +func joinQuery(a, b string) string { + if a == "" || b == "" { + return a + b + } + return a + "&" + b +} + +// handleFilterSummary relays the filter list summary the default repository +// advertises. It lists every package on a filter list, so the request names +// no packages. It is not cached, so a newly listed package is blocked as soon +// as the upstream lists it; Composer revalidates its own copy with +// If-Modified-Since, which is passed on. +func (h *ComposerHandler) handleFilterSummary(w http.ResponseWriter, r *http.Request) { + index, ok := h.upstreamIndex(r.Context()) + if !ok { + JSONError(w, http.StatusBadGateway, "upstream repository unavailable") + return + } + if index.filterSummaryURL == "" { + JSONError(w, http.StatusNotFound, "upstream repository has no filter list summary") + return + } + h.proxy.ProxyUpstream(w, r, index.filterSummaryURL, + []string{headerAccept, headerAcceptEncoding, "If-Modified-Since", "If-None-Match"}) +} diff --git a/internal/handler/composer_advisories_test.go b/internal/handler/composer_advisories_test.go new file mode 100644 index 0000000..6adc7d2 --- /dev/null +++ b/internal/handler/composer_advisories_test.go @@ -0,0 +1,308 @@ +package handler + +import ( + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "net/url" + "slices" + "strings" + "sync" + "testing" +) + +// composerAdvisoryUpstream serves a packages.json advertising security +// advisories and a malware filter list, and records the advisories and +// summary requests it gets. +type composerAdvisoryUpstream struct { + *httptest.Server + index string + + mu sync.Mutex + requests []*recordedRequest +} + +type recordedRequest struct { + method string + path string + query url.Values + body string + header http.Header +} + +const composerAdvisoriesResponse = `{"advisories":{"public/library":[{"advisoryId":"PKSA-1","affectedVersions":"<1.0.1"}]}}` + +const composerFilterSummary = `{"filter":{"malware":{"evil/package":"1.0.0"}}}` + +func newComposerAdvisoryUpstream(t *testing.T) *composerAdvisoryUpstream { + t.Helper() + u := &composerAdvisoryUpstream{} + u.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, _ := io.ReadAll(r.Body) + u.mu.Lock() + u.requests = append(u.requests, &recordedRequest{r.Method, r.URL.Path, r.URL.Query(), string(body), r.Header.Clone()}) + u.mu.Unlock() + w.Header().Set("Content-Type", "application/json") + switch r.URL.Path { + case "/packages.json": + _, _ = io.WriteString(w, u.index) + case "/api/security-advisories/": + _, _ = io.WriteString(w, composerAdvisoriesResponse) + case "/lists/all/summary.json": + if r.Header.Get("If-Modified-Since") != "" { + w.WriteHeader(http.StatusNotModified) + return + } + _, _ = io.WriteString(w, composerFilterSummary) + default: + http.NotFound(w, r) + } + })) + // Like Packagist's: the API on one host, the summary as a relative URL. + u.index = `{ + "packages": [], + "metadata-url": "/p2/%package%.json", + "security-advisories": {"metadata": true, "api-url": "` + u.URL + `/api/security-advisories/"}, + "filter": {"metadata": true, "lists": {"malware": {"enabled": true}}, "summary-url": "/lists/all/summary.json"} + }` + t.Cleanup(u.Close) + return u +} + +// requestsTo returns the recorded requests for path. +func (u *composerAdvisoryUpstream) requestsTo(path string) []*recordedRequest { + u.mu.Lock() + defer u.mu.Unlock() + var out []*recordedRequest + for _, r := range u.requests { + if r.path == path { + out = append(out, r) + } + } + return out +} + +func newAdvisoryComposerHandler(t *testing.T, upstream *composerAdvisoryUpstream) *ComposerHandler { + t.Helper() + proxy, _, _, _ := setupTestProxy(t) + proxy.HostedEnabled = true + proxy.HostedComposerVendors = []string{"acme"} + return NewComposerHandlerWithUpstreams(proxy, "http://proxy.test", upstream.URL, upstream.URL). + WithPackageRoutes(map[string]string{"example/*": "https://packages.example.com/composer"}) +} + +func serveComposerIndex(t *testing.T, h *ComposerHandler) map[string]any { + t.Helper() + w := serveComposer(h, "/packages.json") + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want 200; body: %s", w.Code, w.Body) + } + var index map[string]any + if err := json.Unmarshal(w.Body.Bytes(), &index); err != nil { + t.Fatalf("parsing packages.json: %v", err) + } + return index +} + +func TestComposerServiceIndexAdvertisesAdvisoriesAndFilter(t *testing.T) { + upstream := newComposerAdvisoryUpstream(t) + index := serveComposerIndex(t, newAdvisoryComposerHandler(t, upstream)) + + advisories, _ := index["security-advisories"].(map[string]any) + if advisories["api-url"] != "http://proxy.test/composer/api/security-advisories/" || advisories["metadata"] != true { + t.Errorf("security-advisories = %v, want the proxy's api-url with metadata", index["security-advisories"]) + } + filter, _ := index["filter"].(map[string]any) + if filter["summary-url"] != "http://proxy.test/composer/lists/summary.json" || filter["metadata"] != true { + t.Errorf("filter = %v, want the proxy's summary-url with metadata", index["filter"]) + } + lists, _ := filter["lists"].(map[string]any) + if _, ok := lists["malware"]; !ok { + t.Errorf("filter lists = %v, want upstream's malware list", filter["lists"]) + } + if index["metadata-url"] != "http://proxy.test/composer/p2/%package%.json" { + t.Errorf("metadata-url = %v, want the proxy's", index["metadata-url"]) + } +} + +func TestComposerServiceIndexWithoutUpstreamSupport(t *testing.T) { + tests := []struct { + name string + index string + }{ + {"not advertised", `{"packages": [], "metadata-url": "/p2/%package%.json"}`}, + {"advisories without api-url", `{"security-advisories": {"metadata": true}}`}, + {"filter without metadata", `{"filter": {"lists": {"malware": {"enabled": true}}, "api-url": "/filter"}}`}, + {"not json", ``}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + upstream := newComposerAdvisoryUpstream(t) + upstream.index = tt.index + index := serveComposerIndex(t, newAdvisoryComposerHandler(t, upstream)) + for _, key := range []string{"security-advisories", "filter"} { + if _, ok := index[key]; ok { + t.Errorf("%s advertised although upstream does not offer it: %v", key, index[key]) + } + } + }) + } +} + +func TestComposerServiceIndexUpstreamDown(t *testing.T) { + upstream := newComposerAdvisoryUpstream(t) + h := newAdvisoryComposerHandler(t, upstream) + upstream.Close() + + index := serveComposerIndex(t, h) + if _, ok := index["security-advisories"]; ok { + t.Errorf("security-advisories advertised without an upstream index: %v", index["security-advisories"]) + } + if index["metadata-url"] == nil { + t.Error("packages.json lacks metadata-url while the upstream is down") + } +} + +func serveAdvisories(h *ComposerHandler, method, target, body string) *httptest.ResponseRecorder { + var reader io.Reader + if body != "" { + reader = strings.NewReader(body) + } + req := httptest.NewRequest(method, target, reader) + if method == http.MethodPost { + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + } + w := httptest.NewRecorder() + h.Routes().ServeHTTP(w, req) + return w +} + +func TestComposerAdvisoriesWithholdPrivateNames(t *testing.T) { + // The form Composer sends: PHP's http_build_query numbers the entries. + form := url.Values{ + "packages[0]": {"public/library"}, + "packages[1]": {"example/private"}, + "packages[2]": {"Acme/hosted"}, + "packages[3]": {"other/library"}, + } + tests := []struct { + name, method, target, body string + }{ + {"POST form", http.MethodPost, "/api/security-advisories/", form.Encode()}, + {"GET query", http.MethodGet, "/api/security-advisories/?" + form.Encode(), ""}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + upstream := newComposerAdvisoryUpstream(t) + h := newAdvisoryComposerHandler(t, upstream) + + w := serveAdvisories(h, tt.method, tt.target, tt.body) + + if w.Code != http.StatusOK || w.Body.String() != composerAdvisoriesResponse { + t.Fatalf("got %d %q, want upstream's advisories", w.Code, w.Body) + } + sent := upstream.requestsTo("/api/security-advisories/") + if len(sent) != 1 { + t.Fatalf("upstream got %d advisories requests, want 1", len(sent)) + } + if sent[0].method != tt.method { + t.Errorf("upstream method = %s, want %s", sent[0].method, tt.method) + } + query := sent[0].query + if tt.method == http.MethodPost { + query, _ = url.ParseQuery(sent[0].body) + } + got := query["packages[]"] + slices.Sort(got) + if want := []string{"other/library", "public/library"}; !slices.Equal(got, want) { + t.Errorf("upstream was asked about %v, want only the public packages %v", got, want) + } + }) + } +} + +func TestComposerAdvisoriesOnlyPrivateNames(t *testing.T) { + upstream := newComposerAdvisoryUpstream(t) + h := newAdvisoryComposerHandler(t, upstream) + + w := serveAdvisories(h, http.MethodPost, "/api/security-advisories/", "packages%5B0%5D=example%2Fprivate") + + if w.Code != http.StatusOK || w.Body.String() != `{"advisories":{}}` { + t.Errorf("got %d %q, want an empty advisories answer", w.Code, w.Body) + } + if sent := upstream.requestsTo("/api/security-advisories/"); len(sent) != 0 { + t.Errorf("upstream was asked although every package is private: %+v", sent[0]) + } +} + +func TestComposerAdvisoriesUpdatedSince(t *testing.T) { + upstream := newComposerAdvisoryUpstream(t) + h := newAdvisoryComposerHandler(t, upstream) + + w := serveAdvisories(h, http.MethodGet, "/api/security-advisories/?updatedSince=1700000000", "") + + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want 200", w.Code) + } + sent := upstream.requestsTo("/api/security-advisories/") + if len(sent) != 1 || sent[0].query.Get("updatedSince") != "1700000000" { + t.Errorf("updatedSince was not passed on: %+v", sent) + } +} + +func TestComposerAdvisoriesNotOffered(t *testing.T) { + upstream := newComposerAdvisoryUpstream(t) + upstream.index = `{"packages": []}` + h := newAdvisoryComposerHandler(t, upstream) + + if w := serveAdvisories(h, http.MethodPost, "/api/security-advisories/", "packages%5B%5D=public%2Flibrary"); w.Code != http.StatusNotFound { + t.Errorf("advisories status = %d, want 404 when upstream has no API", w.Code) + } + if w := serveComposer(h, "/lists/summary.json"); w.Code != http.StatusNotFound { + t.Errorf("summary status = %d, want 404 when upstream has no filter", w.Code) + } +} + +func TestComposerFilterSummaryRelayed(t *testing.T) { + upstream := newComposerAdvisoryUpstream(t) + h := newAdvisoryComposerHandler(t, upstream) + + w := serveComposer(h, "/lists/summary.json") + if w.Code != http.StatusOK || w.Body.String() != composerFilterSummary { + t.Fatalf("got %d %q, want upstream's summary", w.Code, w.Body) + } + + req := httptest.NewRequest(http.MethodGet, "/lists/summary.json", nil) + req.Header.Set("If-Modified-Since", "Wed, 21 Oct 2026 15:45:01 GMT") + w = httptest.NewRecorder() + h.Routes().ServeHTTP(w, req) + if w.Code != http.StatusNotModified { + t.Errorf("revalidation status = %d, want 304 passed on from upstream", w.Code) + } +} + +func TestComposerMetadataKeepsAdvisoriesAndFilter(t *testing.T) { + h := NewComposerHandler(testProxy(), "http://proxy.test") + body := []byte(`{ + "minified": "composer/2.0", + "packages": {"evil/package": [{"name": "evil/package", "version": "1.0.0", "dist": {"url": "https://example.com/evil.zip", "type": "zip"}}]}, + "security-advisories": [{"advisoryId": "PKSA-1", "affectedVersions": "<2.0.0"}], + "filter": {"malware": [{"constraint": "1.0.0", "reason": "malware", "id": "PKFE-1"}]} + }`) + + out, _, err := h.rewriteMetadataKeeping(body) + if err != nil { + t.Fatalf("rewrite: %v", err) + } + var doc map[string]json.RawMessage + if err := json.Unmarshal(out, &doc); err != nil { + t.Fatalf("parsing rewritten metadata: %v", err) + } + if !strings.Contains(string(doc["security-advisories"]), "PKSA-1") { + t.Errorf("security-advisories lost in rewrite: %s", out) + } + if !strings.Contains(string(doc["filter"]), "PKFE-1") { + t.Errorf("filter lost in rewrite: %s", out) + } +} From 29ecb20733366efaa6cbe89269f2fa4e9d75cf59 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Tue, 27 Oct 2026 16:16:04 +0100 Subject: [PATCH 462/633] Serve the Go vulnerability database at /go/vulndb govulncheck reads vuln.go.dev directly, so on networks that only reach the proxy it cannot run. upstream.go_vulndb (default https://vuln.go.dev) is now served at /go/vulndb for `govulncheck -db`. Only the files the database API defines are passed on (index/db, index/modules, index/vulns and ID/{id}, as .json and .json.gz), with IDs limited to letters, digits, '-' and '_'. They are stored and served byte-for-byte; .json.gz files are gzip data without Content-Encoding. All files, entries included, follow the metadata TTL since entries change when their modified date does, and fall back to the cached copy when the database is unreachable. Non-JSON responses are refused so a block page does not replace the copy kept for an outage. HEAD is allowed under /vulndb/, because govulncheck checks a non-default database with HEAD on index/modules.json.gz. --- README.md | 8 + cmd/proxy/main.go | 1 + docs/configuration.md | 43 ++++++ internal/config/config.go | 7 + internal/config/config_test.go | 4 + internal/handler/go.go | 14 +- internal/handler/go_vulndb.go | 129 +++++++++++++++++ internal/handler/go_vulndb_test.go | 225 +++++++++++++++++++++++++++++ internal/server/go_vulndb_test.go | 39 +++++ internal/server/server.go | 1 + 10 files changed, 470 insertions(+), 1 deletion(-) create mode 100644 internal/handler/go_vulndb.go create mode 100644 internal/handler/go_vulndb_test.go create mode 100644 internal/server/go_vulndb_test.go diff --git a/README.md b/README.md index 20c6d5d..5b6a11b 100644 --- a/README.md +++ b/README.md @@ -239,6 +239,14 @@ the `go` command contacts the database directly as before. Set checksum databases are listed under `upstream.go_sumdbs`, see [Go checksum database](docs/configuration.md#go-checksum-database). +govulncheck can use the proxy for the Go vulnerability database: + +```bash +govulncheck -db http://localhost:8080/go/vulndb ./... +``` + +See [Go vulnerability database](docs/configuration.md#go-vulnerability-database). + ### Homebrew Point Homebrew's JSON API and artifact domain at the proxy: diff --git a/cmd/proxy/main.go b/cmd/proxy/main.go index b19f689..525a4f6 100644 --- a/cmd/proxy/main.go +++ b/cmd/proxy/main.go @@ -240,6 +240,7 @@ func runServe() { fmt.Fprintf(os.Stderr, " PROXY_UPSTREAM_CARGO_DOWNLOAD Cargo download upstream URL\n") fmt.Fprintf(os.Stderr, " PROXY_UPSTREAM_GEM RubyGems upstream URL\n") fmt.Fprintf(os.Stderr, " PROXY_UPSTREAM_GO Go module proxy upstream URL\n") + fmt.Fprintf(os.Stderr, " PROXY_UPSTREAM_GO_VULNDB Go vulnerability database URL\n") fmt.Fprintf(os.Stderr, " PROXY_UPSTREAM_HEX Hex repository upstream URL\n") fmt.Fprintf(os.Stderr, " PROXY_UPSTREAM_HEX_API Hex API upstream URL\n") fmt.Fprintf(os.Stderr, " PROXY_UPSTREAM_PUB pub registry upstream URL\n") diff --git a/docs/configuration.md b/docs/configuration.md index f10e4b2..5a484dd 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -292,6 +292,7 @@ Each upstream used by a built-in package route can be set in YAML or JSON under | `upstream.gem` | `PROXY_UPSTREAM_GEM` | `https://rubygems.org` | | `upstream.go` | `PROXY_UPSTREAM_GO` | `https://proxy.golang.org` | | `upstream.go_sumdbs_offline` | `PROXY_UPSTREAM_GO_SUMDBS_OFFLINE` | `false` | +| `upstream.go_vulndb` | `PROXY_UPSTREAM_GO_VULNDB` | `https://vuln.go.dev` | | `upstream.hex` | `PROXY_UPSTREAM_HEX` | `https://repo.hex.pm` | | `upstream.hex_api` | `PROXY_UPSTREAM_HEX_API` | `https://hex.pm` | | `upstream.pub` | `PROXY_UPSTREAM_PUB` | `https://pub.dev` | @@ -670,6 +671,48 @@ If `upstream.go` points at a module proxy that serves the checksum database itself, such as Athens, point the entry there instead, for example `sum.golang.org: "https://athens.example/sumdb/sum.golang.org"`. +### Go vulnerability database + +```yaml +upstream: + + # The Go vulnerability database, served at /go/vulndb. + go_vulndb: "https://vuln.go.dev" +``` + +Point govulncheck at the proxy with its `-db` flag: + +```bash +govulncheck -db https://proxy.example.com/go/vulndb ./... +``` + +The proxy passes on the files the +[database API](https://go.dev/security/vuln/database#api) defines: +`index/db.json`, `index/modules.json`, `index/vulns.json` and `ID/{id}.json`, +each also as `.json.gz`, which is what govulncheck fetches. Other paths get +404 without reaching the database. The files are stored and served +byte-for-byte: a `.json.gz` file is gzip data, not a response with +`Content-Encoding: gzip`. + +With `cache_metadata` enabled, every file follows `metadata_ttl`, entries +included, since the database updates an entry when its `modified` date +changes. A response that is not JSON (after decompressing a `.json.gz` file) +is refused and not stored. When the database cannot be reached, cached files +are served however old, with a `Warning` header. Without `cache_metadata` +the requests are forwarded and nothing is stored. + +On a network where clients cannot reach `vuln.go.dev`, only files someone +fetched through the proxy before an outage are available during it. To have +every entry cached, fetch the indexes and all entries through the proxy while +it is online, for example: + +```bash +base=https://proxy.example.com/go/vulndb +for f in db modules vulns; do curl -sfo /dev/null "$base/index/$f.json.gz"; done +curl -sf "$base/index/vulns.json" | jq -r '.[].id' | + xargs -I{} curl -sfo /dev/null "$base/ID/{}.json.gz" +``` + ### Debian archives ```yaml diff --git a/internal/config/config.go b/internal/config/config.go index a660699..0a49349 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -621,6 +621,11 @@ type UpstreamConfig struct { // Requires cache_metadata. Default: false. GoSumDBsOffline bool `json:"go_sumdbs_offline" yaml:"go_sumdbs_offline"` + // GoVulnDB is the Go vulnerability database served at /go/vulndb, for + // govulncheck with GOVULNDB pointing at the proxy. + // Default: https://vuln.go.dev + GoVulnDB string `json:"go_vulndb" yaml:"go_vulndb"` + // Hex is the upstream Hex repository URL. // Default: https://repo.hex.pm Hex string `json:"hex" yaml:"hex"` @@ -1246,6 +1251,7 @@ func Default() *Config { CargoDownload: "https://static.crates.io/crates", Gem: "https://rubygems.org", Go: "https://proxy.golang.org", + GoVulnDB: "https://vuln.go.dev", Hex: "https://repo.hex.pm", HexAPI: "https://hex.pm", Pub: "https://pub.dev", @@ -1412,6 +1418,7 @@ func (c *Config) LoadFromEnv() { setEnvString(&c.Upstream.CargoDownload, "PROXY_UPSTREAM_CARGO_DOWNLOAD") setEnvString(&c.Upstream.Gem, "PROXY_UPSTREAM_GEM") setEnvString(&c.Upstream.Go, "PROXY_UPSTREAM_GO") + setEnvString(&c.Upstream.GoVulnDB, "PROXY_UPSTREAM_GO_VULNDB") setEnvString(&c.Upstream.Hex, "PROXY_UPSTREAM_HEX") setEnvString(&c.Upstream.HexAPI, "PROXY_UPSTREAM_HEX_API") setEnvString(&c.Upstream.Pub, "PROXY_UPSTREAM_PUB") diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 7590a64..2f89de2 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -21,6 +21,7 @@ func upstreamConfigValues(upstream UpstreamConfig) map[string]string { "cargo_download": upstream.CargoDownload, "gem": upstream.Gem, "go": upstream.Go, + "go_vulndb": upstream.GoVulnDB, "hex": upstream.Hex, "hex_api": upstream.HexAPI, "pub": upstream.Pub, @@ -53,6 +54,7 @@ func defaultUpstreamValues() map[string]string { "cargo_download": "https://static.crates.io/crates", "gem": "https://rubygems.org", "go": "https://proxy.golang.org", + "go_vulndb": "https://vuln.go.dev", "hex": "https://repo.hex.pm", "hex_api": "https://hex.pm", "pub": "https://pub.dev", @@ -85,6 +87,7 @@ func upstreamEnvironmentVariables() map[string]string { "cargo_download": "PROXY_UPSTREAM_CARGO_DOWNLOAD", "gem": "PROXY_UPSTREAM_GEM", "go": "PROXY_UPSTREAM_GO", + "go_vulndb": "PROXY_UPSTREAM_GO_VULNDB", "hex": "PROXY_UPSTREAM_HEX", "hex_api": "PROXY_UPSTREAM_HEX_API", "pub": "PROXY_UPSTREAM_PUB", @@ -379,6 +382,7 @@ upstream: cargo_download: "https://upstream.example.com/cargo_download" gem: "https://upstream.example.com/gem" go: "https://upstream.example.com/go" + go_vulndb: "https://upstream.example.com/go_vulndb" hex: "https://upstream.example.com/hex" hex_api: "https://upstream.example.com/hex_api" pub: "https://upstream.example.com/pub" diff --git a/internal/handler/go.go b/internal/handler/go.go index add1348..59c0937 100644 --- a/internal/handler/go.go +++ b/internal/handler/go.go @@ -37,6 +37,7 @@ type GoHandler struct { sumDBs map[string]string sumDBOffline bool sumDBProbes goSumDBProbes + vulnDBURL string } // NewGoHandler creates a new Go module proxy handler. @@ -46,6 +47,7 @@ func NewGoHandler(proxy *Proxy, proxyURL string) *GoHandler { upstreamURL: goUpstream, proxyURL: strings.TrimSuffix(proxyURL, "/"), sumDBs: map[string]string{defaultGoSumDBName: defaultGoSumDBURL}, + vulnDBURL: defaultGoVulnDBURL, } } @@ -60,7 +62,9 @@ func NewGoHandlerWithUpstream(proxy *Proxy, proxyURL, upstreamURL string) *GoHan func (h *GoHandler) Routes() http.Handler { // Go module paths can contain slashes, so just use the handler directly return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.Method != http.MethodGet { + // govulncheck sends HEAD for index/modules.json.gz to check that a + // GOVULNDB other than vuln.go.dev is a vulnerability database. + if r.Method != http.MethodGet && (r.Method != http.MethodHead || !strings.HasPrefix(r.URL.Path, "/vulndb/")) { http.Error(w, "method not allowed", http.StatusMethodNotAllowed) return } @@ -78,6 +82,14 @@ func (h *GoHandler) handleRequest(w http.ResponseWriter, r *http.Request) { return } + // So do vulnerability database requests, which govulncheck sends when + // GOVULNDB points at /go/vulndb. A module path's first element holds a + // dot, so neither prefix can be a module. + if rest, ok := strings.CutPrefix(path, "vulndb/"); ok { + h.handleVulnDB(w, r, rest) + return + } + // Check for @v/ pattern to identify module requests if idx := strings.Index(path, "/@v/"); idx >= 0 { module := path[:idx] diff --git a/internal/handler/go_vulndb.go b/internal/handler/go_vulndb.go new file mode 100644 index 0000000..8531171 --- /dev/null +++ b/internal/handler/go_vulndb.go @@ -0,0 +1,129 @@ +package handler + +import ( + "bytes" + "compress/gzip" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "strings" +) + +const ( + // defaultGoVulnDBURL is the database govulncheck uses when GOVULNDB is + // not set. + defaultGoVulnDBURL = "https://vuln.go.dev" + + // goVulnDBMaxIDLength bounds an entry ID in a request path. Go IDs look + // like GO-2024-1234; other OSV databases use longer ones such as + // GHSA-xxxx-xxxx-xxxx. + goVulnDBMaxIDLength = 128 + + // goVulnDBMaxJSONSize bounds a decompressed .json.gz response while it is + // checked. index/modules.json, the largest file, is a few MB. + goVulnDBMaxJSONSize = 256 << 20 +) + +// SetVulnDB sets the Go vulnerability database served at /vulndb/. An empty +// URL keeps the default, vuln.go.dev. +func (h *GoHandler) SetVulnDB(vulnDBURL string) { + h.vulnDBURL = configuredUpstreamURL(vulnDBURL, defaultGoVulnDBURL) +} + +// handleVulnDB serves the Go vulnerability database (GOVULNDB) at +// /vulndb/{path}. Only the files the database API defines are passed on: +// index/db.json, index/modules.json, index/vulns.json and ID/{id}.json, each +// also as .json.gz, which is what govulncheck fetches. Every file can change, +// so all of them follow the metadata TTL and fall back to the cached copy +// when the database cannot be reached. The bytes are stored and served +// unchanged: a .json.gz file is gzip data, not a gzip-encoded response. +func (h *GoHandler) handleVulnDB(w http.ResponseWriter, r *http.Request, path string) { + if !validGoVulnDBPath(path) { + http.NotFound(w, r) + return + } + upstreamURL := h.vulnDBURL + "/" + path + p := h.proxy + if !p.CacheMetadata { + p.proxyMetadataStream(w, r, upstreamURL, "identity", "*/*") + return + } + cacheKey := goVulnDBCacheKey(h.vulnDBURL, path) + res := p.lookupMetadata(r.Context(), goEcosystem, cacheKey, upstreamURL, "*/*", "identity", goVulnDBResponseCheck(path)) + if res.err != nil { + p.writeMetadataFetchError(w, res.err) + return + } + p.writeMetadataCachedResponseWithEncoding(w, r, goEcosystem, cacheKey, res.body, res.contentType, res.contentEncoding) +} + +// validGoVulnDBPath reports whether path is a file the vulnerability +// database API defines. Entry IDs are limited to letters, digits, '-' and +// '_', which keeps '.', '/', '%' and the like out of the upstream URL. +func validGoVulnDBPath(path string) bool { + name, ok := strings.CutSuffix(path, ".json.gz") + if !ok { + name, ok = strings.CutSuffix(path, ".json") + } + if !ok { + return false + } + switch name { + case "index/db", "index/modules", "index/vulns": + return true + } + id, ok := strings.CutPrefix(name, "ID/") + if !ok || id == "" || len(id) > goVulnDBMaxIDLength { + return false + } + for _, c := range id { + if !isASCIILetterOrDigit(c) && c != '-' && c != '_' { + return false + } + } + return true +} + +func isASCIILetterOrDigit(c rune) bool { + return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') +} + +// goVulnDBResponseCheck returns a check that a response to path is JSON, +// after decompressing it for a .json.gz path. A 200 that is something else +// (an egress proxy's block page, a cut-off download) is refused rather than +// cached, where it would also replace the copy kept for an outage. +func goVulnDBResponseCheck(path string) func([]byte) error { + compressed := strings.HasSuffix(path, ".gz") + return func(body []byte) error { + data := body + if compressed { + zr, err := gzip.NewReader(bytes.NewReader(body)) + if err != nil { + return fmt.Errorf("vulnerability database file %s is not gzip data: %w", path, err) + } + data, err = io.ReadAll(io.LimitReader(zr, goVulnDBMaxJSONSize+1)) + if err == nil && len(data) > goVulnDBMaxJSONSize { + err = errors.New("too large") + } + if err != nil { + return fmt.Errorf("vulnerability database file %s does not decompress: %w", path, err) + } + } + if !json.Valid(data) { + return fmt.Errorf("vulnerability database file %s is not JSON", path) + } + return nil + } +} + +// goVulnDBCacheKey hashes the database URL and the request path, as +// goSumDBCacheKey does, so repointing upstream.go_vulndb does not serve the +// old database's entries. +func goVulnDBCacheKey(vulnDBURL, path string) string { + digest := sha256.Sum256([]byte(vulnDBURL + "\x00" + path)) + return "vulndb/" + hex.EncodeToString(digest[:]) +} diff --git a/internal/handler/go_vulndb_test.go b/internal/handler/go_vulndb_test.go new file mode 100644 index 0000000..162ff9a --- /dev/null +++ b/internal/handler/go_vulndb_test.go @@ -0,0 +1,225 @@ +package handler + +import ( + "bytes" + "compress/gzip" + "io" + "net/http" + "net/http/httptest" + "testing" + "time" +) + +const ( + testVulnDBDB = `{"modified":"2026-10-21T00:27:29Z"}` + testVulnDBModules = `[{"path":"golang.org/x/text","vulns":[{"id":"GO-2021-0113","modified":"2026-10-21T00:27:29Z","fixed":"0.3.7"}]}]` + testVulnDBEntry = `{"schema_version":"1.3.1","id":"GO-2021-0113","modified":"2026-10-21T00:27:29Z"}` +) + +func gzipString(t *testing.T, s string) string { + t.Helper() + var buf bytes.Buffer + zw := gzip.NewWriter(&buf) + if _, err := io.WriteString(zw, s); err != nil { + t.Fatal(err) + } + if err := zw.Close(); err != nil { + t.Fatal(err) + } + return buf.String() +} + +// testVulnDBFiles is a small vulnerability database laid out as vuln.go.dev +// serves it: every file as .json and .json.gz, the latter as gzip data with a +// JSON content type and no Content-Encoding. +func testVulnDBFiles(t *testing.T) map[string]string { + t.Helper() + files := map[string]string{ + "/index/db.json": testVulnDBDB, + "/index/modules.json": testVulnDBModules, + "/index/vulns.json": `[{"id":"GO-2021-0113","modified":"2026-10-21T00:27:29Z"}]`, + "/ID/GO-2021-0113.json": testVulnDBEntry, + "/ID/GHSA-ppp9-7jff-5vj2.json": testVulnDBEntry, + } + all := make(map[string]string, 2*len(files)) + for path, body := range files { + all[path] = body + all[path+".gz"] = gzipString(t, body) + } + return all +} + +func newVulnDBTestHandler(t *testing.T, vulnDBURL string, cacheMetadata bool, ttl time.Duration) *GoHandler { + t.Helper() + proxy, _, _, _ := setupTestProxy(t) + proxy.HTTPClient = http.DefaultClient + proxy.CacheMetadata = cacheMetadata + proxy.MetadataTTL = ttl + h := NewGoHandler(proxy, "http://proxy.example") + h.SetVulnDB(vulnDBURL) + return h +} + +func TestGoVulnDB_DefaultsToVulnGoDev(t *testing.T) { + h := NewGoHandler(nil, "http://proxy.example") + if h.vulnDBURL != defaultGoVulnDBURL { + t.Errorf("vulnDBURL = %q, want %q", h.vulnDBURL, defaultGoVulnDBURL) + } + h.SetVulnDB("") + if h.vulnDBURL != defaultGoVulnDBURL { + t.Errorf("after SetVulnDB(\"\"): vulnDBURL = %q, want %q", h.vulnDBURL, defaultGoVulnDBURL) + } + h.SetVulnDB("https://vulndb.example/") + if h.vulnDBURL != "https://vulndb.example" { + t.Errorf("vulnDBURL = %q, want the trailing slash trimmed", h.vulnDBURL) + } +} + +func TestGoVulnDB_ServesFilesUnchanged(t *testing.T) { + files := testVulnDBFiles(t) + for _, cacheMetadata := range []bool{true, false} { + db := newFakeSumDB(t, files) + h := newVulnDBTestHandler(t, db.server.URL, cacheMetadata, time.Hour) + for path, want := range files { + w := serveGoRequest(h, "/vulndb"+path) + if w.Code != http.StatusOK || w.Body.String() != want { + t.Errorf("cache_metadata=%v %s: status = %d, body = %q, want 200 with the upstream bytes", cacheMetadata, path, w.Code, w.Body.String()) + } + if enc := w.Header().Get(headerContentEncoding); enc != "" { + t.Errorf("cache_metadata=%v %s: Content-Encoding = %q, want none", cacheMetadata, path, enc) + } + } + } +} + +func TestGoVulnDB_HeadForSchemaCheck(t *testing.T) { + db := newFakeSumDB(t, testVulnDBFiles(t)) + h := newVulnDBTestHandler(t, db.server.URL, true, time.Hour) + + w := httptest.NewRecorder() + h.Routes().ServeHTTP(w, httptest.NewRequest(http.MethodHead, "/vulndb/index/modules.json.gz", nil)) + if w.Code != http.StatusOK { + t.Errorf("HEAD index/modules.json.gz: status = %d, want 200", w.Code) + } + + w = httptest.NewRecorder() + h.Routes().ServeHTTP(w, httptest.NewRequest(http.MethodHead, "/golang.org/x/text/@v/list", nil)) + if w.Code != http.StatusMethodNotAllowed { + t.Errorf("HEAD on a module path: status = %d, want 405", w.Code) + } +} + +func TestGoVulnDB_CachedWithinTTL(t *testing.T) { + db := newFakeSumDB(t, testVulnDBFiles(t)) + h := newVulnDBTestHandler(t, db.server.URL, true, time.Hour) + + for range 3 { + if w := serveGoRequest(h, "/vulndb/index/modules.json.gz"); w.Code != http.StatusOK { + t.Fatalf("status = %d, want 200", w.Code) + } + } + if n := db.requests("/index/modules.json.gz"); n != 1 { + t.Errorf("upstream requests = %d, want 1", n) + } +} + +func TestGoVulnDB_RefreshedAfterTTL(t *testing.T) { + db := newFakeSumDB(t, testVulnDBFiles(t)) + h := newVulnDBTestHandler(t, db.server.URL, true, time.Nanosecond) + + serveGoRequest(h, "/vulndb/ID/GO-2021-0113.json") + updated := `{"id":"GO-2021-0113","modified":"2026-10-27T00:00:00Z"}` + db.setFile("/ID/GO-2021-0113.json", updated) + if w := serveGoRequest(h, "/vulndb/ID/GO-2021-0113.json"); w.Body.String() != updated { + t.Errorf("body = %q, want the modified entry %q", w.Body.String(), updated) + } +} + +func TestGoVulnDB_StaleFallbackWhenUnreachable(t *testing.T) { + files := testVulnDBFiles(t) + db := newFakeSumDB(t, files) + h := newVulnDBTestHandler(t, db.server.URL, true, time.Nanosecond) + + for path := range files { + if w := serveGoRequest(h, "/vulndb"+path); w.Code != http.StatusOK { + t.Fatalf("%s online: status = %d, want 200", path, w.Code) + } + } + + db.down.Store(true) + for path, want := range files { + w := serveGoRequest(h, "/vulndb"+path) + if w.Code != http.StatusOK || w.Body.String() != want { + t.Errorf("%s offline: status = %d, body = %q, want 200 with the cached copy", path, w.Code, w.Body.String()) + } + if w.Header().Get("Warning") == "" { + t.Errorf("%s offline: no Warning header, want the response marked stale", path) + } + } + if w := serveGoRequest(h, "/vulndb/ID/GO-2099-0001.json.gz"); w.Code != http.StatusBadGateway { + t.Errorf("uncached entry offline: status = %d, want 502", w.Code) + } +} + +func TestGoVulnDB_RefusesNonJSON(t *testing.T) { + const blockPage = "blocked by policy" + db := newFakeSumDB(t, map[string]string{ + "/index/db.json": blockPage, + "/index/db.json.gz": blockPage, + "/index/modules.json.gz": gzipString(t, blockPage), + "/ID/GO-2021-0113.json.gz": gzipString(t, testVulnDBEntry)[:20], + }) + h := newVulnDBTestHandler(t, db.server.URL, true, time.Hour) + + for _, path := range []string{"/index/db.json", "/index/db.json.gz", "/index/modules.json.gz", "/ID/GO-2021-0113.json.gz"} { + if w := serveGoRequest(h, "/vulndb"+path); w.Code != http.StatusBadGateway { + t.Errorf("%s: status = %d, want 502", path, w.Code) + } + } + + // Nothing bad was cached: once upstream serves the real file, it is used. + db.setFile("/index/db.json", testVulnDBDB) + if w := serveGoRequest(h, "/vulndb/index/db.json"); w.Code != http.StatusOK || w.Body.String() != testVulnDBDB { + t.Errorf("after fix: status = %d, body = %q, want 200 with %q", w.Code, w.Body.String(), testVulnDBDB) + } +} + +func TestGoVulnDB_NotFound(t *testing.T) { + db := newFakeSumDB(t, nil) + h := newVulnDBTestHandler(t, db.server.URL, true, time.Hour) + if w := serveGoRequest(h, "/vulndb/ID/GO-2099-0001.json.gz"); w.Code != http.StatusNotFound { + t.Errorf("status = %d, want 404", w.Code) + } +} + +func TestGoVulnDB_InvalidPathsNeverReachUpstream(t *testing.T) { + db := newFakeSumDB(t, nil) + h := newVulnDBTestHandler(t, db.server.URL, true, time.Hour) + + for _, target := range []string{ + "/vulndb", + "/vulndb/", + "/vulndb/index", + "/vulndb/index/db", + "/vulndb/index/db.json.zip", + "/vulndb/index/other.json", + "/vulndb/vulndb.zip", + "/vulndb/ID/", + "/vulndb/ID/.json", + "/vulndb/ID/GO-2021-0113", + "/vulndb/ID/GO-2021-0113.txt", + "/vulndb/ID/../index/db.json", + "/vulndb/ID/..%2Findex%2Fdb.json", + "/vulndb/ID/GO.2021.json", + "/vulndb/ID/a/b.json", + "/vulndb/ID/GO%3F.json", + "/vulndb/ID/" + string(bytes.Repeat([]byte("A"), goVulnDBMaxIDLength+1)) + ".json", + } { + if w := serveGoRequest(h, target); w.Code != http.StatusNotFound { + t.Errorf("%s: status = %d, want 404", target, w.Code) + } + } + if n := db.totalRequests(); n != 0 { + t.Errorf("upstream requests = %d, want 0", n) + } +} diff --git a/internal/server/go_vulndb_test.go b/internal/server/go_vulndb_test.go new file mode 100644 index 0000000..bdde56d --- /dev/null +++ b/internal/server/go_vulndb_test.go @@ -0,0 +1,39 @@ +package server + +import ( + "io" + "log/slog" + "net/http" + "net/http/httptest" + "testing" + + "github.com/git-pkgs/proxy/internal/config" + "github.com/git-pkgs/proxy/internal/handler" + "github.com/go-chi/chi/v5" +) + +func TestGoVulnDBReachesGoHandler(t *testing.T) { + const dbJSON = `{"modified":"2026-10-21T00:27:29Z"}` + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/index/db.json" { + http.NotFound(w, r) + return + } + _, _ = io.WriteString(w, dbJSON) + })) + defer upstream.Close() + + cfg := config.Default() + cfg.BaseURL = "https://proxy.example" + cfg.Upstream.GoVulnDB = upstream.URL + p := &handler.Proxy{HTTPClient: upstream.Client(), Logger: slog.New(slog.NewTextHandler(io.Discard, nil))} + s := &Server{cfg: cfg} + r := chi.NewRouter() + s.mountProtocolHandlers(r, p) + + w := httptest.NewRecorder() + r.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/go/vulndb/index/db.json", nil)) + if w.Code != http.StatusOK || w.Body.String() != dbJSON { + t.Errorf("status = %d, body = %q; want index/db.json from upstream.go_vulndb", w.Code, w.Body.String()) + } +} diff --git a/internal/server/server.go b/internal/server/server.go index 12619a6..48ee8b3 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -447,6 +447,7 @@ func (s *Server) mountProtocolHandlers(r chi.Router, proxy *handler.Proxy) { gemHandler := handler.NewGemHandlerWithUpstream(proxy, s.cfg.BaseURL, s.cfg.Upstream.Gem) goHandler := handler.NewGoHandlerWithUpstream(proxy, s.cfg.BaseURL, s.cfg.Upstream.Go) goHandler.SetSumDBs(s.cfg.Upstream.GoSumDBs, s.cfg.Upstream.GoSumDBsOffline) + goHandler.SetVulnDB(s.cfg.Upstream.GoVulnDB) hexHandler := handler.NewHexHandlerWithUpstreams( proxy, s.cfg.BaseURL, From 1abb8c029a3783197f5856831dc0a5f5b921f658 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Tue, 27 Oct 2026 16:18:26 +0100 Subject: [PATCH 463/633] Serve dated snapshots of Alpine APK indexes Alpine mirrors keep only the current build of each package, so once a package is upgraded `apk add curl=8.5.0-r0` fails: the current index no longer lists the old build, even when the proxy still has it cached. With upstream.apk_snapshots set (e.g. "90d"), each version of an index or detached signature the metadata cache stores is copied to _snapshots/alpine/ and recorded in a new metadata_snapshots table with the time it was first served. /apk/{name}@{YYYY-MM-DD}/ serves the version that was current at the end of that day; packages below it are the plain path's requests. The kept bytes are the signed upstream bytes, so apk verifies them unchanged. An hourly job deletes versions replaced longer ago than the configured duration and queues their blobs on the pending-delete queue; the newest version of each index stays. Packages listed by kept snapshots are not protected from eviction or retention; the docs say so. --- README.md | 5 + docs/configuration.md | 46 +++++ internal/config/config.go | 26 +++ internal/config/config_test.go | 35 ++++ internal/database/database_test.go | 2 +- internal/database/metadata_snapshots.go | 136 ++++++++++++++ internal/database/metadata_snapshots_test.go | 108 +++++++++++ internal/database/queries.go | 1 + internal/database/schema.go | 59 ++++++ internal/handler/apk.go | 20 +- internal/handler/apk_snapshots.go | 187 +++++++++++++++++++ internal/handler/apk_snapshots_test.go | 168 +++++++++++++++++ internal/server/apk_snapshots.go | 48 +++++ internal/server/apk_snapshots_test.go | 63 +++++++ internal/server/server.go | 4 +- 15 files changed, 902 insertions(+), 6 deletions(-) create mode 100644 internal/database/metadata_snapshots.go create mode 100644 internal/database/metadata_snapshots_test.go create mode 100644 internal/handler/apk_snapshots.go create mode 100644 internal/handler/apk_snapshots_test.go create mode 100644 internal/server/apk_snapshots.go create mode 100644 internal/server/apk_snapshots_test.go diff --git a/README.md b/README.md index 20c6d5d..c360866 100644 --- a/README.md +++ b/README.md @@ -722,6 +722,11 @@ signature verification keeps working. Indexes use the metadata cache (`metadata_ttl`, stale fallback); `.apk` packages are stored in the shared artifact cache and remain available when the upstream is unreachable. +Set `upstream.apk_snapshots` (e.g. `"90d"`) to keep replaced indexes and serve +them by date at `/apk/alpine@2026-10-01/v3.22/main`, so pinned package versions +stay installable after the mirror moves on. See +[Alpine APK snapshots](docs/configuration.md#alpine-apk-snapshots). + To proxy other mirrors or private repositories, configure named upstreams under `upstream.apk` (this replaces the built-in default; re-add `alpine` if you still want it): diff --git a/docs/configuration.md b/docs/configuration.md index f10e4b2..f2faf46 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -314,6 +314,7 @@ Each upstream used by a built-in package route can be set in YAML or JSON under | `upstream.rpm` | `PROXY_UPSTREAM_RPM` | `https://dl.fedoraproject.org/pub/fedora/linux` | | `upstream.homebrew_api` | `PROXY_UPSTREAM_HOMEBREW_API` | `https://formulae.brew.sh/api` | | `upstream.homebrew_artifact` | `PROXY_UPSTREAM_HOMEBREW_ARTIFACT` | `https://ghcr.io` | +| `upstream.apk_snapshots` | `PROXY_UPSTREAM_APK_SNAPSHOTS` | disabled; see [Alpine APK snapshots](#alpine-apk-snapshots) | Private, ULA, CGNAT, and loopback addresses are rejected by default. Add each private upstream hostname or IP address to `upstream.allow_private_hosts`. The matching environment variable accepts a comma-separated list. Loopback upstreams also require `upstream.allow_loopback: true`. That setting permits upstream requests and redirects to reach any loopback address. @@ -566,6 +567,8 @@ with the metadata TTL and served byte-for-byte unchanged so apk signature verification keeps working; `.apk` packages use the shared artifact cache. When `upstream.apk` is empty, a single repository named `alpine` pointing at the official mirror is available; configuring any entry replaces that default. +Earlier index versions can be kept and served by date; see +[Alpine APK snapshots](#alpine-apk-snapshots). ```yaml upstream: @@ -613,6 +616,49 @@ as signatures and keys, is passed through uncached. When `upstream.freebsd` is e repository named `freebsd` pointing at `https://pkg.freebsd.org` is available; configuring any entry replaces that default. +### Alpine APK snapshots + +```yaml +upstream: + # Keep replaced APK indexes for 90 days. + apk_snapshots: "90d" +``` + +Alpine mirrors only keep the current build of each package, so after an +upgrade `apk add curl=8.5.0-r0` fails because the index no longer lists that +build. With `upstream.apk_snapshots` set, the proxy keeps every version of an +index (and of a detached signature) it stores, and serves them under a dated +repository name: + +``` +http://localhost:8080/apk/alpine@2026-10-01/v3.22/main +``` + +`/apk/{name}@{YYYY-MM-DD}/…` returns the index that was current at the end of +that day (UTC). The kept files are the signed upstream bytes, so apk verifies +them as usual. A day before the first stored version of an index, or a day +that has not started yet, is `404`. Today's snapshot can still change until the +day ends. Packages under a snapshot path are the same as under the plain path: +they come from the artifact cache, or from upstream if not cached; a build the +upstream has dropped and the proxy never cached is `404`. A configured +repository whose name contains `@` takes precedence over a snapshot path. + +Snapshots are recorded when an index is requested through the plain path, so +they only exist for repositories clients use. A version counts from the first +time the proxy served it, which can be up to `metadata_ttl` after upstream +published it. A version is deleted once it was replaced more than the +configured duration ago; the newest version of each index is always kept. +Durations take a `d` suffix; empty or `"0"` disables snapshots. Snapshots need +`cache_metadata`. Their files live under `_snapshots/` in storage and do not +count towards `storage.max_size`. + +Snapshots keep indexes, not packages. A package listed by an old snapshot can +still be evicted by `storage.max_size` or [retention](#retention), and is then +only available while upstream still has it. Retention counts from a package's +last download, so a pinned build nobody installed for longer than the `alpine` +retention is evicted even while a snapshot lists it. Use package rules with +`"0"` for builds that must stay installable. + ### Go checksum database ```yaml diff --git a/internal/config/config.go b/internal/config/config.go index a660699..6b90541 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -744,6 +744,12 @@ type UpstreamConfig struct { // Default when empty: {"alpine": "https://dl-cdn.alpinelinux.org/alpine"}. APK map[string]string `json:"apk" yaml:"apk"` + // APKSnapshots keeps earlier versions of Alpine APK indexes for this + // long after upstream replaced them, and serves them at + // /apk/{name}@{YYYY-MM-DD}/. Accepts a "d" suffix, e.g. "90d". + // Empty or "0" disables snapshots. + APKSnapshots string `json:"apk_snapshots" yaml:"apk_snapshots"` + // Arch maps repository names to Arch Linux pacman mirror base URLs, // served at /arch/{name}/. The remaining request path mirrors the // upstream layout, e.g. /arch/archlinux/core/os/x86_64/core.db. @@ -852,6 +858,11 @@ func (u *UpstreamConfig) Validate() error { if err := validateNamedUpstreams("upstream.apk", u.APK); err != nil { return err } + if u.APKSnapshots != "" { + if _, err := parseRetentionDuration("upstream.apk_snapshots", u.APKSnapshots); err != nil { + return err + } + } if err := validateNamedUpstreams("upstream.arch", u.Arch); err != nil { return err } @@ -1431,6 +1442,7 @@ func (c *Config) LoadFromEnv() { setEnvString(&c.Upstream.Swift, "PROXY_UPSTREAM_SWIFT") setEnvString(&c.Upstream.OCIDefault, "PROXY_UPSTREAM_OCI_DEFAULT") setEnvString(&c.Upstream.Debian, "PROXY_UPSTREAM_DEBIAN") + setEnvString(&c.Upstream.APKSnapshots, "PROXY_UPSTREAM_APK_SNAPSHOTS") setEnvString(&c.Upstream.RPM, "PROXY_UPSTREAM_RPM") setEnvString(&c.Upstream.HomebrewAPI, "PROXY_UPSTREAM_HOMEBREW_API") setEnvString(&c.Upstream.HomebrewArtifact, "PROXY_UPSTREAM_HOMEBREW_ARTIFACT") @@ -1928,6 +1940,20 @@ func (c *Config) ParseGradleBuildCacheMaxUploadSize() int64 { return size } +// ParseAPKSnapshots returns how long earlier Alpine APK indexes are kept +// after upstream replaced them. Returns 0 when snapshots are disabled or the +// value is invalid. +func (c *Config) ParseAPKSnapshots() time.Duration { + if c.Upstream.APKSnapshots == "" { + return 0 + } + d, err := parseDayDuration(c.Upstream.APKSnapshots) + if err != nil || d < 0 { + return 0 + } + return d +} + // ParseGradleBuildCacheMaxAge returns age-based eviction threshold. // Returns 0 when disabled or invalid. func (c *Config) ParseGradleBuildCacheMaxAge() time.Duration { diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 7590a64..e3f2dc2 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -2052,3 +2052,38 @@ upstream: t.Errorf("generic_paths = %v", cfg.Upstream.GenericPaths) } } + +func TestAPKSnapshots(t *testing.T) { + cfg := Default() + if got := cfg.ParseAPKSnapshots(); got != 0 { + t.Errorf("default ParseAPKSnapshots = %v, want 0", got) + } + + for value, want := range map[string]time.Duration{ + "90d": 90 * 24 * time.Hour, + "12h": 12 * time.Hour, + "0": 0, + } { + cfg.Upstream.APKSnapshots = value + if err := cfg.Validate(); err != nil { + t.Errorf("Validate(%q): %v", value, err) + } + if got := cfg.ParseAPKSnapshots(); got != want { + t.Errorf("ParseAPKSnapshots(%q) = %v, want %v", value, got, want) + } + } + + for _, value := range []string{"forever", "-1d"} { + cfg.Upstream.APKSnapshots = value + if err := cfg.Validate(); err == nil { + t.Errorf("Validate(%q) succeeded, want an error", value) + } + } + + cfg = Default() + t.Setenv("PROXY_UPSTREAM_APK_SNAPSHOTS", "30d") + cfg.LoadFromEnv() + if cfg.Upstream.APKSnapshots != "30d" { + t.Errorf("APKSnapshots from env = %q, want 30d", cfg.Upstream.APKSnapshots) + } +} diff --git a/internal/database/database_test.go b/internal/database/database_test.go index f8956a6..a61f91d 100644 --- a/internal/database/database_test.go +++ b/internal/database/database_test.go @@ -613,7 +613,7 @@ func createTestPostgresDB(t *testing.T) *DB { // Drop and recreate every table CreateSchema creates for clean test state; // leftover migration records make the next CreateSchema fail on the // migrations primary key. - tables := []string{"artifacts", "pending_deletes", "versions", "packages", "vulnerabilities", "metadata_cache", "metadata_dates", "job_runs", "fetch_leases", "upstream_holds", "migrations", "schema_info"} + tables := []string{"artifacts", "pending_deletes", "versions", "packages", "vulnerabilities", "metadata_cache", "metadata_dates", "job_runs", "fetch_leases", "upstream_holds", "metadata_snapshots", "migrations", "schema_info"} for _, table := range tables { _, _ = db.Exec("DROP TABLE IF EXISTS " + table + " CASCADE") } diff --git a/internal/database/metadata_snapshots.go b/internal/database/metadata_snapshots.go new file mode 100644 index 0000000..b174aab --- /dev/null +++ b/internal/database/metadata_snapshots.go @@ -0,0 +1,136 @@ +package database + +import ( + "database/sql" + "fmt" + "math" + "time" +) + +// Metadata snapshots keep earlier versions of a metadata document after +// upstream replaced it, so a request for a past date can be answered with the +// document that was current then. Each row is one version, valid from +// valid_from_ms (Unix milliseconds, the time the proxy first stored it) until +// the next row for the same ecosystem and name. + +// MetadataSnapshot is one kept version of a metadata document. +type MetadataSnapshot struct { + ID int64 `db:"id"` + Ecosystem string `db:"ecosystem"` + Name string `db:"name"` + StoragePath string `db:"storage_path"` + ContentType sql.NullString `db:"content_type"` + ContentEncoding sql.NullString `db:"content_encoding"` + ContentDigest string `db:"content_digest"` + Size int64 `db:"size"` + ValidFromMS int64 `db:"valid_from_ms"` +} + +// ValidFrom returns the time the snapshot became current. +func (s *MetadataSnapshot) ValidFrom() time.Time { + return time.UnixMilli(s.ValidFromMS) +} + +const metadataSnapshotColumns = `id, ecosystem, name, storage_path, content_type, content_encoding, + content_digest, size, valid_from_ms` + +// MetadataSnapshotAt returns the snapshot of ecosystem and name that was +// current at at: the newest one that became current before it. It returns nil +// when there is none. +func (db *DB) MetadataSnapshotAt(ecosystem, name string, at time.Time) (*MetadataSnapshot, error) { + return db.metadataSnapshotBefore(ecosystem, name, at.UnixMilli()) +} + +// LatestMetadataSnapshot returns the newest snapshot of ecosystem and name, +// or nil when there is none. +func (db *DB) LatestMetadataSnapshot(ecosystem, name string) (*MetadataSnapshot, error) { + return db.metadataSnapshotBefore(ecosystem, name, math.MaxInt64) +} + +func (db *DB) metadataSnapshotBefore(ecosystem, name string, beforeMS int64) (*MetadataSnapshot, error) { + var snap MetadataSnapshot + err := db.Get(&snap, db.Rebind(`SELECT `+metadataSnapshotColumns+` + FROM metadata_snapshots + WHERE ecosystem = ? AND name = ? AND valid_from_ms < ? + ORDER BY valid_from_ms DESC, id DESC + LIMIT 1`), ecosystem, name, beforeMS) + if err == sql.ErrNoRows { + return nil, nil + } + if err != nil { + return nil, fmt.Errorf("reading metadata snapshot: %w", err) + } + return &snap, nil +} + +// AddMetadataSnapshot records snap as the current version of its document, +// unless the newest snapshot already has its digest, and reports whether it +// did. The check and the insert are one statement, so concurrent callers +// recording the same document add it once. The casts let Postgres type the +// parameters of the INSERT ... SELECT. +func (db *DB) AddMetadataSnapshot(snap *MetadataSnapshot) (bool, error) { + res, err := db.Exec(db.Rebind(` + INSERT INTO metadata_snapshots (ecosystem, name, storage_path, content_type, content_encoding, + content_digest, size, valid_from_ms) + SELECT CAST(? AS TEXT), CAST(? AS TEXT), CAST(? AS TEXT), CAST(? AS TEXT), CAST(? AS TEXT), + CAST(? AS TEXT), CAST(? AS BIGINT), CAST(? AS BIGINT) + WHERE COALESCE(( + SELECT content_digest FROM metadata_snapshots + WHERE ecosystem = ? AND name = ? + ORDER BY valid_from_ms DESC, id DESC + LIMIT 1 + ), '') <> CAST(? AS TEXT) + `), snap.Ecosystem, snap.Name, snap.StoragePath, snap.ContentType, snap.ContentEncoding, + snap.ContentDigest, snap.Size, snap.ValidFromMS, + snap.Ecosystem, snap.Name, snap.ContentDigest) + if err != nil { + return false, fmt.Errorf("recording metadata snapshot: %w", err) + } + n, err := res.RowsAffected() + return n > 0, err +} + +// PruneMetadataSnapshots deletes the snapshots of ecosystem that stopped +// being current before cutoff, that is those replaced by a newer snapshot +// that became current before cutoff, and queues their blobs for deletion. The +// newest snapshot of each document is always kept. It returns the number of +// snapshots deleted. +func (db *DB) PruneMetadataSnapshots(ecosystem string, cutoff time.Time) (int, error) { + tx, err := db.Beginx() + if err != nil { + return 0, err + } + defer func() { _ = tx.Rollback() }() + + var expired []struct { + ID int64 `db:"id"` + StoragePath string `db:"storage_path"` + } + err = tx.Select(&expired, db.Rebind(` + SELECT s.id, s.storage_path FROM metadata_snapshots s + WHERE s.ecosystem = ? + AND EXISTS ( + SELECT 1 FROM metadata_snapshots n + WHERE n.ecosystem = s.ecosystem AND n.name = s.name + AND (n.valid_from_ms > s.valid_from_ms OR (n.valid_from_ms = s.valid_from_ms AND n.id > s.id)) + AND n.valid_from_ms < ? + ) + `), ecosystem, cutoff.UnixMilli()) + if err != nil { + return 0, fmt.Errorf("listing expired metadata snapshots: %w", err) + } + + queuedAt := time.Now().UTC() + for _, row := range expired { + if _, err := tx.Exec(db.Rebind(`DELETE FROM metadata_snapshots WHERE id = ?`), row.ID); err != nil { + return 0, fmt.Errorf("deleting metadata snapshot: %w", err) + } + if _, err := tx.Exec(db.Rebind(queuePendingDeleteQuery), row.StoragePath, queuedAt); err != nil { + return 0, fmt.Errorf("queueing metadata snapshot blob: %w", err) + } + } + if err := tx.Commit(); err != nil { + return 0, err + } + return len(expired), nil +} diff --git a/internal/database/metadata_snapshots_test.go b/internal/database/metadata_snapshots_test.go new file mode 100644 index 0000000..ae859fd --- /dev/null +++ b/internal/database/metadata_snapshots_test.go @@ -0,0 +1,108 @@ +package database + +import ( + "testing" + "time" +) + +func addSnapshot(t *testing.T, db *DB, name, digest string, validFrom time.Time) bool { + t.Helper() + added, err := db.AddMetadataSnapshot(&MetadataSnapshot{ + Ecosystem: "alpine", + Name: name, + StoragePath: "_snapshots/alpine/" + name + "/" + digest, + ContentDigest: digest, + Size: 1, + ValidFromMS: validFrom.UnixMilli(), + }) + if err != nil { + t.Fatalf("AddMetadataSnapshot: %v", err) + } + return added +} + +func snapshotDigestAt(t *testing.T, db *DB, name string, at time.Time) string { + t.Helper() + snap, err := db.MetadataSnapshotAt("alpine", name, at) + if err != nil { + t.Fatalf("MetadataSnapshotAt: %v", err) + } + if snap == nil { + return "" + } + return snap.ContentDigest +} + +func TestMetadataSnapshots_AddAndLookup(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + day := time.Date(2026, 3, 1, 0, 0, 0, 0, time.UTC) + if !addSnapshot(t, db, "idx", "sha256:a", day) { + t.Fatal("first snapshot not added") + } + if addSnapshot(t, db, "idx", "sha256:a", day.Add(time.Hour)) { + t.Error("unchanged digest added again") + } + if !addSnapshot(t, db, "idx", "sha256:b", day.Add(48*time.Hour)) { + t.Fatal("changed digest not added") + } + // A rollback to an earlier version is a new snapshot. + if !addSnapshot(t, db, "idx", "sha256:a", day.Add(96*time.Hour)) { + t.Fatal("rolled back digest not added") + } + + for _, tc := range []struct { + at time.Time + want string + }{ + {day, ""}, + {day.Add(time.Minute), "sha256:a"}, + {day.Add(72 * time.Hour), "sha256:b"}, + {day.Add(100 * time.Hour), "sha256:a"}, + } { + if got := snapshotDigestAt(t, db, "idx", tc.at); got != tc.want { + t.Errorf("snapshot at %s = %q, want %q", tc.at, got, tc.want) + } + } + latest, err := db.LatestMetadataSnapshot("alpine", "idx") + if err != nil || latest == nil || !latest.ValidFrom().Equal(day.Add(96*time.Hour)) { + t.Errorf("LatestMetadataSnapshot = %+v, %v", latest, err) + } + }) +} + +func TestMetadataSnapshots_Prune(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + now := time.Date(2026, 6, 1, 0, 0, 0, 0, time.UTC) + addSnapshot(t, db, "idx", "sha256:a", now.AddDate(0, 0, -100)) + addSnapshot(t, db, "idx", "sha256:b", now.AddDate(0, 0, -60)) + addSnapshot(t, db, "idx", "sha256:c", now.AddDate(0, 0, -10)) + // The only version of another index stays however old it is. + addSnapshot(t, db, "other", "sha256:x", now.AddDate(0, 0, -200)) + + pruned, err := db.PruneMetadataSnapshots("alpine", now.AddDate(0, 0, -30)) + if err != nil { + t.Fatalf("PruneMetadataSnapshots: %v", err) + } + // a was replaced 60 days ago, b only 10 days ago. + if pruned != 1 { + t.Errorf("pruned = %d, want 1", pruned) + } + if got := snapshotDigestAt(t, db, "idx", now.AddDate(0, 0, -90)); got != "" { + t.Errorf("snapshot of a pruned day = %q, want none", got) + } + if got := snapshotDigestAt(t, db, "idx", now.AddDate(0, 0, -50)); got != "sha256:b" { + t.Errorf("snapshot 50 days ago = %q, want sha256:b", got) + } + if got := snapshotDigestAt(t, db, "other", now); got != "sha256:x" { + t.Errorf("only snapshot of other = %q, want sha256:x", got) + } + + due, err := db.GetDuePendingDeletes(time.Now().Add(time.Hour), 10) + if err != nil { + t.Fatal(err) + } + if len(due) != 1 || due[0] != "_snapshots/alpine/idx/sha256:a" { + t.Errorf("queued deletes = %v, want the pruned blob", due) + } + }) +} diff --git a/internal/database/queries.go b/internal/database/queries.go index d7150e8..973bd08 100644 --- a/internal/database/queries.go +++ b/internal/database/queries.go @@ -645,6 +645,7 @@ func (db *DB) GetDuePendingDeletes(cutoff time.Time, limit int) ([]string, error WHERE queued_at < ? AND NOT EXISTS (SELECT 1 FROM artifacts WHERE artifacts.storage_path = pending_deletes.path) AND NOT EXISTS (SELECT 1 FROM metadata_cache WHERE metadata_cache.storage_path = pending_deletes.path) + AND NOT EXISTS (SELECT 1 FROM metadata_snapshots WHERE metadata_snapshots.storage_path = pending_deletes.path) ORDER BY queued_at LIMIT ? `) diff --git a/internal/database/schema.go b/internal/database/schema.go index f0ec651..6d31ff4 100644 --- a/internal/database/schema.go +++ b/internal/database/schema.go @@ -154,6 +154,20 @@ CREATE TABLE IF NOT EXISTS upstream_holds ( until_ms INTEGER NOT NULL ); +CREATE TABLE IF NOT EXISTS metadata_snapshots ( + id INTEGER PRIMARY KEY, + ecosystem TEXT NOT NULL, + name TEXT NOT NULL, + storage_path TEXT NOT NULL, + content_type TEXT, + content_encoding TEXT, + content_digest TEXT NOT NULL, + size INTEGER NOT NULL, + valid_from_ms INTEGER NOT NULL +); +CREATE INDEX IF NOT EXISTS idx_metadata_snapshots_key ON metadata_snapshots(ecosystem, name, valid_from_ms); +CREATE INDEX IF NOT EXISTS idx_metadata_snapshots_path ON metadata_snapshots(storage_path); + CREATE TABLE IF NOT EXISTS migrations ( name TEXT NOT NULL PRIMARY KEY, applied_at DATETIME NOT NULL @@ -297,6 +311,20 @@ CREATE TABLE IF NOT EXISTS upstream_holds ( until_ms BIGINT NOT NULL ); +CREATE TABLE IF NOT EXISTS metadata_snapshots ( + id SERIAL PRIMARY KEY, + ecosystem TEXT NOT NULL, + name TEXT NOT NULL, + storage_path TEXT NOT NULL, + content_type TEXT, + content_encoding TEXT, + content_digest TEXT NOT NULL, + size BIGINT NOT NULL, + valid_from_ms BIGINT NOT NULL +); +CREATE INDEX IF NOT EXISTS idx_metadata_snapshots_key ON metadata_snapshots(ecosystem, name, valid_from_ms); +CREATE INDEX IF NOT EXISTS idx_metadata_snapshots_path ON metadata_snapshots(storage_path); + CREATE TABLE IF NOT EXISTS migrations ( name TEXT NOT NULL PRIMARY KEY, applied_at TIMESTAMP NOT NULL @@ -468,6 +496,7 @@ var migrations = []migration{ {"017_add_artifact_content_hash_index", migrateAddArtifactContentHashIndex}, {"018_add_upstream_holds", migrateAddUpstreamHolds}, {"019_add_artifact_filename_index", migrateAddArtifactFilenameIndex}, + {"020_add_metadata_snapshots", migrateAddMetadataSnapshots}, } // isTableNotFound returns true if the error indicates a missing table. @@ -850,6 +879,36 @@ func migrateAddArtifactFilenameIndex(db *DB) error { return nil } +// migrateAddMetadataSnapshots creates the table of kept earlier versions of +// metadata documents, which dated snapshot requests are served from. +func migrateAddMetadataSnapshots(db *DB) error { + pk, integer := "id INTEGER PRIMARY KEY", "INTEGER" + if db.dialect == DialectPostgres { + pk, integer = "id SERIAL PRIMARY KEY", "BIGINT" + } + statements := []string{ + fmt.Sprintf(`CREATE TABLE IF NOT EXISTS metadata_snapshots ( + %s, + ecosystem TEXT NOT NULL, + name TEXT NOT NULL, + storage_path TEXT NOT NULL, + content_type TEXT, + content_encoding TEXT, + content_digest TEXT NOT NULL, + size %s NOT NULL, + valid_from_ms %s NOT NULL + )`, pk, integer, integer), + `CREATE INDEX IF NOT EXISTS idx_metadata_snapshots_key ON metadata_snapshots(ecosystem, name, valid_from_ms)`, + `CREATE INDEX IF NOT EXISTS idx_metadata_snapshots_path ON metadata_snapshots(storage_path)`, + } + for _, query := range statements { + if _, err := db.Exec(query); err != nil { + return fmt.Errorf("creating metadata_snapshots: %w", err) + } + } + return nil +} + // migrateAddMetadataDates creates the table of Last-Modified dates for // rewritten metadata, which replicas sharing the database agree on. func migrateAddMetadataDates(db *DB) error { diff --git a/internal/handler/apk.go b/internal/handler/apk.go index 9024fe0..071ae3a 100644 --- a/internal/handler/apk.go +++ b/internal/handler/apk.go @@ -1,11 +1,13 @@ package handler import ( + "context" "crypto/sha256" "encoding/hex" "net/http" "regexp" "strings" + "time" "github.com/git-pkgs/proxy/internal/retention" "github.com/git-pkgs/purl" @@ -51,6 +53,9 @@ type APKHandler struct { proxy *Proxy proxyURL string repositories map[string]string + // snapshots is how long replaced index versions are kept; 0 disables + // snapshots. See WithSnapshots. + snapshots time.Duration } // NewAPKHandler creates an Alpine APK repository protocol handler. @@ -87,14 +92,18 @@ func (h *APKHandler) Routes() http.Handler { return } - repository, rest, ok := strings.Cut(path, "/") - upstreamURL, found := h.repositories[repository] + segment, rest, ok := strings.Cut(path, "/") + repository, upstreamURL, day, snapshot, found := h.resolveRepository(segment) if !ok || rest == "" || !found { http.NotFound(w, r) return } switch { + case snapshot && (isAPKIndex(rest) || isAPKSignature(rest)): + // A dated snapshot serves the kept version of the index; packages + // and other files below are the same as under the plain path. + h.serveSnapshot(w, r, repository, upstreamURL, rest, day) case isAPKIndex(rest) || isAPKSignature(rest): // Indexes and detached signatures are signed upstream metadata. // Cache them with the metadata TTL and serve the stored bytes @@ -160,8 +169,11 @@ func (h *APKHandler) handlePackageDownload(w http.ResponseWriter, r *http.Reques // metadata cache. Stored bytes are re-served verbatim, which keeps embedded // and detached signatures valid. func (h *APKHandler) handleMetadata(w http.ResponseWriter, r *http.Request, repository, upstreamURL, path string) { - h.proxy.ProxyCached(w, r, upstreamURL+"/"+path, apkEcosystem, - h.metadataCacheKey(repository, upstreamURL, path), "*/*") + cacheKey := h.metadataCacheKey(repository, upstreamURL, path) + h.proxy.ProxyCached(w, r, upstreamURL+"/"+path, apkEcosystem, cacheKey, "*/*") + if h.snapshotsEnabled() { + h.recordSnapshot(context.WithoutCancel(r.Context()), cacheKey) + } } // metadataCacheKey derives the metadata cache key from the repository name, diff --git a/internal/handler/apk_snapshots.go b/internal/handler/apk_snapshots.go new file mode 100644 index 0000000..cca04d6 --- /dev/null +++ b/internal/handler/apk_snapshots.go @@ -0,0 +1,187 @@ +package handler + +import ( + "context" + "io" + "net/http" + "strconv" + "strings" + "time" + + "github.com/git-pkgs/proxy/internal/database" +) + +// Dated snapshots of APK indexes. +// +// Alpine mirrors keep only the current build of each package, so once a +// package is upgraded the index no longer lists the old build and +// `apk add name=old-version` fails. With snapshots enabled, every version of an +// index (and of a detached signature) the proxy stores is kept, and +// /apk/{name}@{YYYY-MM-DD}/... serves the version that was current at the end +// of that day (UTC). The kept bytes are the signed upstream bytes, so apk +// verifies them as usual. Packages under a snapshot path are the same +// requests as under the plain path. + +// apkSnapshotDateLayout is the date format of a snapshot path segment. +const apkSnapshotDateLayout = "2006-01-02" + +// WithSnapshots keeps earlier versions of the indexes this handler serves and +// makes them available under /apk/{name}@{YYYY-MM-DD}/. keep is how long a +// version is kept after it was replaced; 0 disables snapshots. +func (h *APKHandler) WithSnapshots(keep time.Duration) *APKHandler { + h.snapshots = keep + return h +} + +// snapshotsEnabled reports whether snapshots are recorded and served. They +// need the metadata cache, which stores the documents they are copied from. +func (h *APKHandler) snapshotsEnabled() bool { + return h.snapshots > 0 && h.proxy.CacheMetadata && h.proxy.DB != nil && h.proxy.Storage != nil +} + +// resolveRepository maps the first path segment to a configured repository. +// A segment of the form {name}@{YYYY-MM-DD} names a snapshot of repository +// name for that day, when snapshots are enabled and no repository is +// configured under the full segment. ok is false for anything else. +func (h *APKHandler) resolveRepository(segment string) (repository, upstreamURL string, day time.Time, snapshot, ok bool) { + if upstreamURL, found := h.repositories[segment]; found { + return segment, upstreamURL, time.Time{}, false, true + } + if !h.snapshotsEnabled() { + return "", "", time.Time{}, false, false + } + at := strings.LastIndexByte(segment, '@') + if at < 0 { + return "", "", time.Time{}, false, false + } + upstreamURL, found := h.repositories[segment[:at]] + if !found { + return "", "", time.Time{}, false, false + } + day, err := time.Parse(apkSnapshotDateLayout, segment[at+1:]) + if err != nil { + return "", "", time.Time{}, false, false + } + return segment[:at], upstreamURL, day, true, true +} + +// serveSnapshot answers a request for an index or signature under a snapshot +// path with the version that was current at the end of day. A day for which +// no version was stored, or one that has not begun yet, is 404. +func (h *APKHandler) serveSnapshot(w http.ResponseWriter, r *http.Request, repository, upstreamURL, path string, day time.Time) { + end := day.AddDate(0, 0, 1) + if day.After(time.Now()) { + http.NotFound(w, r) + return + } + snap, err := h.proxy.DB.MetadataSnapshotAt(apkEcosystem, h.metadataCacheKey(repository, upstreamURL, path), end) + if err != nil { + h.proxy.Logger.Error("apk snapshot lookup failed", "repository", repository, "path", path, "error", err) + http.Error(w, "internal error", http.StatusInternalServerError) + return + } + if snap == nil { + http.NotFound(w, r) + return + } + + if writeMetadataValidators(w, r, cachedMeta{etag: `"` + snap.ContentDigest + `"`, lastModified: snap.ValidFrom()}) { + return + } + body, err := h.proxy.Storage.Open(r.Context(), snap.StoragePath) + if err != nil { + h.proxy.Logger.Error("apk snapshot unreadable", "repository", repository, "path", path, + "storage_path", snap.StoragePath, "error", err) + http.Error(w, "snapshot unavailable", http.StatusInternalServerError) + return + } + defer func() { _ = body.Close() }() + + if snap.ContentType.String != "" { + w.Header().Set(headerContentType, snap.ContentType.String) + } + w.Header().Set(headerContentLength, strconv.FormatInt(snap.Size, 10)) + if snap.ContentEncoding.String != "" { + w.Header().Set(headerContentEncoding, snap.ContentEncoding.String) + } + w.WriteHeader(http.StatusOK) + if r.Method != http.MethodHead { + _, _ = io.Copy(w, body) + } +} + +// recordSnapshot keeps the document now cached under cacheKey as a snapshot +// when it differs from the newest one kept. It runs after the document was +// served, so a failure only means this version is recorded on a later +// request instead. +func (h *APKHandler) recordSnapshot(ctx context.Context, cacheKey string) { + entry, err := h.proxy.DB.GetMetadataCache(apkEcosystem, cacheKey) + if err != nil || entry == nil || !entry.ContentDigest.Valid { + return + } + digest := entry.ContentDigest.String + latest, err := h.proxy.DB.LatestMetadataSnapshot(apkEcosystem, cacheKey) + if err != nil || (latest != nil && latest.ContentDigest == digest) { + return + } + + validFrom := time.Now() + storagePath := apkSnapshotStoragePath(cacheKey, digest, validFrom) + size, ok := h.copySnapshotBlob(ctx, entry.StoragePath, storagePath, digest) + if !ok { + return + } + added, err := h.proxy.DB.AddMetadataSnapshot(&database.MetadataSnapshot{ + Ecosystem: apkEcosystem, + Name: cacheKey, + StoragePath: storagePath, + ContentType: entry.ContentType, + ContentEncoding: entry.ContentEncoding, + ContentDigest: digest, + Size: size, + ValidFromMS: validFrom.UnixMilli(), + }) + if err != nil { + h.proxy.Logger.Warn("failed to record apk snapshot", "key", cacheKey, "error", err) + } + if err != nil || !added { + // Another request recorded this version first. + h.discardSnapshotBlob(ctx, storagePath) + } +} + +// copySnapshotBlob copies the cached document at from to to and reports its +// size, provided the copied bytes still have digest. A concurrent fetch can +// replace the cached document after its row was read; that version is +// recorded by the request that stored it. +func (h *APKHandler) copySnapshotBlob(ctx context.Context, from, to, digest string) (int64, bool) { + src, err := h.proxy.Storage.Open(ctx, from) + if err != nil { + return 0, false + } + defer func() { _ = src.Close() }() + size, hash, err := h.proxy.Storage.Store(ctx, to, src) + if err != nil { + h.proxy.Logger.Warn("failed to store apk snapshot", "path", to, "error", err) + return 0, false + } + if "sha256:"+hash != digest { + h.discardSnapshotBlob(ctx, to) + return 0, false + } + return size, true +} + +func (h *APKHandler) discardSnapshotBlob(ctx context.Context, path string) { + if err := h.proxy.Storage.Delete(ctx, path); err != nil { + h.proxy.Logger.Warn("failed to discard apk snapshot blob", "path", path, "error", err) + } +} + +// apkSnapshotStoragePath is where one kept version of the document cached +// under cacheKey is stored. The time it became current keeps a version that +// returns after an upgrade was rolled back apart from its earlier copy. +func apkSnapshotStoragePath(cacheKey, digest string, validFrom time.Time) string { + return "_snapshots/" + apkEcosystem + "/" + cacheKey + "/" + + strconv.FormatInt(validFrom.UnixMilli(), 10) + "-" + strings.TrimPrefix(digest, "sha256:") +} diff --git a/internal/handler/apk_snapshots_test.go b/internal/handler/apk_snapshots_test.go new file mode 100644 index 0000000..50291c1 --- /dev/null +++ b/internal/handler/apk_snapshots_test.go @@ -0,0 +1,168 @@ +package handler + +import ( + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + "time" + + "github.com/git-pkgs/proxy/internal/database" + "github.com/git-pkgs/registries/fetch" +) + +const apkSnapshotIndexPath = "/v3.22/main/x86_64/APKINDEX.tar.gz" + +// apkSnapshotUpstream serves the index held in index, initially "index v1", +// and counts package downloads. +type apkSnapshotUpstream struct { + index atomic.Value + downloads atomic.Int32 + server *httptest.Server +} + +func newAPKSnapshotUpstream(t *testing.T) *apkSnapshotUpstream { + t.Helper() + u := &apkSnapshotUpstream{} + u.index.Store("index v1") + u.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case apkSnapshotIndexPath: + _, _ = w.Write([]byte(u.index.Load().(string))) + case "/v3.22/main/x86_64/curl-8.5.0-r0.apk": + u.downloads.Add(1) + _, _ = w.Write([]byte("curl 8.5.0")) + default: + http.NotFound(w, r) + } + })) + t.Cleanup(u.server.Close) + return u +} + +// newSnapshotAPKHandler returns an APK handler with snapshots enabled whose +// metadata TTL is 0, so every index request asks upstream. +func newSnapshotAPKHandler(t *testing.T, upstreamURL string) (*APKHandler, *database.DB) { + t.Helper() + proxy, db, _, _ := setupTestProxy(t) + proxy.CacheMetadata = true + proxy.MetadataTTL = 0 + proxy.HTTPClient = http.DefaultClient + fetcher := fetch.NewFetcher(fetch.WithHTTPClient(http.DefaultClient), fetch.WithMaxRetries(0)) + proxy.Fetcher = fetcher + t.Cleanup(func() { _ = fetcher.Close() }) + h := NewAPKHandler(proxy, "http://proxy.example", map[string]string{"alpine": upstreamURL}). + WithSnapshots(30 * 24 * time.Hour) + return h, db +} + +func snapshotCount(t *testing.T, db *database.DB) int { + t.Helper() + var n int + if err := db.Get(&n, `SELECT COUNT(*) FROM metadata_snapshots`); err != nil { + t.Fatal(err) + } + return n +} + +func expectAPKResponse(t *testing.T, h *APKHandler, target string, status int, body string) { + t.Helper() + w := serveAPKRequest(h, target) + if w.Code != status { + t.Fatalf("%s: status = %d, want %d: %s", target, w.Code, status, w.Body.String()) + } + if status == http.StatusOK && w.Body.String() != body { + t.Errorf("%s: body = %q, want %q", target, w.Body.String(), body) + } +} + +// TestAPKSnapshots_ServeIndexCurrentAtDate records two versions of an index +// through the plain path and checks that each dated path serves the version +// that was current at the end of that day. +func TestAPKSnapshots_ServeIndexCurrentAtDate(t *testing.T) { + upstream := newAPKSnapshotUpstream(t) + h, db := newSnapshotAPKHandler(t, upstream.server.URL) + + expectAPKResponse(t, h, "/alpine"+apkSnapshotIndexPath, http.StatusOK, "index v1") + // Pretend v1 was first stored on 2026-01-10. + recorded := time.Date(2026, 1, 10, 12, 0, 0, 0, time.UTC) + if _, err := db.Exec(`UPDATE metadata_snapshots SET valid_from_ms = ?`, recorded.UnixMilli()); err != nil { + t.Fatal(err) + } + + upstream.index.Store("index v2") + expectAPKResponse(t, h, "/alpine"+apkSnapshotIndexPath, http.StatusOK, "index v2") + // An unchanged index adds no snapshot. + expectAPKResponse(t, h, "/alpine"+apkSnapshotIndexPath, http.StatusOK, "index v2") + if got := snapshotCount(t, db); got != 2 { + t.Fatalf("snapshots = %d, want 2", got) + } + + today := time.Now().UTC().Format(apkSnapshotDateLayout) + future := time.Now().UTC().AddDate(0, 0, 2).Format(apkSnapshotDateLayout) + expectAPKResponse(t, h, "/alpine@2026-01-09"+apkSnapshotIndexPath, http.StatusNotFound, "") + expectAPKResponse(t, h, "/alpine@2026-01-10"+apkSnapshotIndexPath, http.StatusOK, "index v1") + expectAPKResponse(t, h, "/alpine@"+today+apkSnapshotIndexPath, http.StatusOK, "index v2") + expectAPKResponse(t, h, "/alpine@"+future+apkSnapshotIndexPath, http.StatusNotFound, "") + expectAPKResponse(t, h, "/alpine@2026-13-01"+apkSnapshotIndexPath, http.StatusNotFound, "") + expectAPKResponse(t, h, "/other@2026-01-10"+apkSnapshotIndexPath, http.StatusNotFound, "") + expectAPKResponse(t, h, "/alpine@2026-01-10/v3.22/community/x86_64/APKINDEX.tar.gz", http.StatusNotFound, "") +} + +func TestAPKSnapshots_ConditionalRequest(t *testing.T) { + upstream := newAPKSnapshotUpstream(t) + h, _ := newSnapshotAPKHandler(t, upstream.server.URL) + expectAPKResponse(t, h, "/alpine"+apkSnapshotIndexPath, http.StatusOK, "index v1") + + target := "/alpine@" + time.Now().UTC().Format(apkSnapshotDateLayout) + apkSnapshotIndexPath + first := serveAPKRequest(h, target) + etag := first.Header().Get("ETag") + if first.Code != http.StatusOK || etag == "" { + t.Fatalf("status = %d, ETag = %q, want 200 with an ETag", first.Code, etag) + } + req := httptest.NewRequest(http.MethodGet, target, nil) + req.Header.Set("If-None-Match", etag) + w := httptest.NewRecorder() + h.Routes().ServeHTTP(w, req) + if w.Code != http.StatusNotModified { + t.Errorf("conditional request: status = %d, want 304", w.Code) + } +} + +// TestAPKSnapshots_PackagesShareTheCache checks that a package requested +// under a snapshot path is the cached artifact of the plain path. +func TestAPKSnapshots_PackagesShareTheCache(t *testing.T) { + upstream := newAPKSnapshotUpstream(t) + h, _ := newSnapshotAPKHandler(t, upstream.server.URL) + + pkg := "/v3.22/main/x86_64/curl-8.5.0-r0.apk" + expectAPKResponse(t, h, "/alpine"+pkg, http.StatusOK, "curl 8.5.0") + expectAPKResponse(t, h, "/alpine@2026-01-10"+pkg, http.StatusOK, "curl 8.5.0") + if got := upstream.downloads.Load(); got != 1 { + t.Errorf("upstream downloads = %d, want 1", got) + } + expectAPKResponse(t, h, "/alpine@2026-01-10/v3.22/main/x86_64/gone-1.0-r0.apk", http.StatusNotFound, "") +} + +func TestAPKSnapshots_DisabledByDefault(t *testing.T) { + upstream := newAPKSnapshotUpstream(t) + h, db := newSnapshotAPKHandler(t, upstream.server.URL) + h.WithSnapshots(0) + + expectAPKResponse(t, h, "/alpine"+apkSnapshotIndexPath, http.StatusOK, "index v1") + if got := snapshotCount(t, db); got != 0 { + t.Errorf("snapshots = %d, want 0", got) + } + today := time.Now().UTC().Format(apkSnapshotDateLayout) + expectAPKResponse(t, h, "/alpine@"+today+apkSnapshotIndexPath, http.StatusNotFound, "") +} + +// TestAPKSnapshots_ConfiguredNameWins checks that a repository whose +// configured name contains '@' is served as configured, not as a snapshot. +func TestAPKSnapshots_ConfiguredNameWins(t *testing.T) { + upstream := newAPKSnapshotUpstream(t) + h, _ := newSnapshotAPKHandler(t, upstream.server.URL) + h.repositories["alpine@2026-01-10"] = upstream.server.URL + + expectAPKResponse(t, h, "/alpine@2026-01-10"+apkSnapshotIndexPath, http.StatusOK, "index v1") +} diff --git a/internal/server/apk_snapshots.go b/internal/server/apk_snapshots.go new file mode 100644 index 0000000..ee8e3c9 --- /dev/null +++ b/internal/server/apk_snapshots.go @@ -0,0 +1,48 @@ +package server + +import ( + "context" + "time" +) + +// apkSnapshotPruneInterval is how often snapshots kept past +// upstream.apk_snapshots are deleted. +const apkSnapshotPruneInterval = time.Hour + +// startAPKSnapshotPruning deletes APK index snapshots that were replaced more +// than upstream.apk_snapshots ago, once at startup and then every hour. Their +// blobs go through the pending-delete queue like evicted cache entries. +func (s *Server) startAPKSnapshotPruning(ctx context.Context) { + keep := s.cfg.ParseAPKSnapshots() + if keep <= 0 { + return + } + if !s.cfg.CacheMetadata { + s.logger.Warn("upstream.apk_snapshots needs cache_metadata, snapshots disabled") + return + } + + ticker := time.NewTicker(apkSnapshotPruneInterval) + defer ticker.Stop() + for { + s.pruneAPKSnapshots(ctx, keep, time.Now()) + select { + case <-ctx.Done(): + return + case <-ticker.C: + } + } +} + +func (s *Server) pruneAPKSnapshots(ctx context.Context, keep time.Duration, now time.Time) { + s.runExclusive(ctx, "apk-snapshot-prune", func() { + n, err := s.db.PruneMetadataSnapshots("alpine", now.Add(-keep)) + if err != nil { + s.logger.Warn("apk snapshots: prune failed", "error", err) + return + } + if n > 0 { + s.logger.Info("apk snapshots: pruned", "deleted", n) + } + }) +} diff --git a/internal/server/apk_snapshots_test.go b/internal/server/apk_snapshots_test.go new file mode 100644 index 0000000..191e073 --- /dev/null +++ b/internal/server/apk_snapshots_test.go @@ -0,0 +1,63 @@ +package server + +import ( + "context" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/git-pkgs/proxy/internal/config" + "github.com/git-pkgs/proxy/internal/database" + "github.com/git-pkgs/proxy/internal/handler" + "github.com/git-pkgs/proxy/internal/storage" + "github.com/git-pkgs/registries/fetch" + "github.com/go-chi/chi/v5" +) + +// TestAPKSnapshotsConfigured checks that upstream.apk_snapshots enables the +// dated APK paths, with the digest the file storage records for the cached +// index matching the copy kept as a snapshot. +func TestAPKSnapshotsConfigured(t *testing.T) { + const index = "signed index" + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = io.WriteString(w, index) + })) + defer upstream.Close() + + db, err := database.Create(t.TempDir() + "/test.db") + if err != nil { + t.Fatalf("creating database: %v", err) + } + t.Cleanup(func() { _ = db.Close() }) + store, err := storage.OpenBucket(context.Background(), "file://"+t.TempDir()) + if err != nil { + t.Fatalf("opening storage: %v", err) + } + logger := slog.New(slog.NewTextHandler(io.Discard, nil)) + proxy := handler.NewProxy(db, store, fetch.NewFetcher(), fetch.NewResolver(), logger) + proxy.HTTPClient = upstream.Client() + proxy.CacheMetadata = true + + cfg := config.Default() + cfg.BaseURL = "http://proxy.test" + cfg.Upstream.APK = map[string]string{"alpine": upstream.URL} + cfg.Upstream.APKSnapshots = "30d" + s := &Server{cfg: cfg, logger: logger} + r := chi.NewRouter() + s.mountProtocolHandlers(r, proxy) + + today := time.Now().UTC().Format("2006-01-02") + for _, target := range []string{ + "/apk/alpine/v3.22/main/x86_64/APKINDEX.tar.gz", + "/apk/alpine@" + today + "/v3.22/main/x86_64/APKINDEX.tar.gz", + } { + w := httptest.NewRecorder() + r.ServeHTTP(w, httptest.NewRequest(http.MethodGet, target, nil)) + if w.Code != http.StatusOK || w.Body.String() != index { + t.Fatalf("%s: status = %d, body = %q, want 200 %q", target, w.Code, w.Body.String(), index) + } + } +} diff --git a/internal/server/server.go b/internal/server/server.go index 12619a6..fb378ac 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -425,6 +425,7 @@ func (s *Server) serve(listener net.Listener) error { s.background.Go(func() { backfillMetadataAccess(bgCtx, s.db, s.logger, metadataBackfillBatch) }) s.background.Go(func() { s.startEvictionLoop(bgCtx) }) s.background.Go(func() { s.startRetentionLoop(bgCtx) }) + s.background.Go(func() { s.startAPKSnapshotPruning(bgCtx) }) s.background.Go(func() { s.startReclaimLoop(bgCtx) }) if listener != nil { @@ -489,7 +490,8 @@ func (s *Server) mountProtocolHandlers(r chi.Router, proxy *handler.Proxy) { containerHandler := s.newContainerHandler(proxy) helmHandler := handler.NewHelmHandlerWithOCIRegistries( proxy, s.cfg.BaseURL, s.cfg.Upstream.Helm, s.cfg.Upstream.OCIDefault, s.cfg.Upstream.OCI) - apkHandler := handler.NewAPKHandler(proxy, s.cfg.BaseURL, s.cfg.Upstream.APK) + apkHandler := handler.NewAPKHandler(proxy, s.cfg.BaseURL, s.cfg.Upstream.APK). + WithSnapshots(s.cfg.ParseAPKSnapshots()) archHandler := handler.NewArchHandler(proxy, s.cfg.BaseURL, s.cfg.Upstream.Arch) freebsdHandler := handler.NewFreeBSDHandler(proxy, s.cfg.BaseURL, s.cfg.Upstream.FreeBSD) debianHandler := handler.NewDebianHandler( From e97d4aa306b69da92817e7099a0af719201f2263 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Tue, 27 Oct 2026 16:19:14 +0100 Subject: [PATCH 464/633] Answer npm dist-tag ls from the served packument `npm dist-tag ls` reads /-/package/{name}/dist-tags. The proxy had no route for it, so the request ended up in the metadata path and failed. The tags are taken from the packument the proxy serves rather than relayed from upstream, so latest never names a version cooldown holds back, routed packages use their route, and pushed versions and hosted-only scopes count as they do for the packument. Changing tags stays unsupported and still gets 405. --- README.md | 1 + docs/configuration.md | 4 +- internal/handler/npm.go | 5 + internal/handler/npm_dist_tags.go | 115 +++++++++++++++++++++ internal/handler/npm_dist_tags_test.go | 136 +++++++++++++++++++++++++ 5 files changed, 259 insertions(+), 2 deletions(-) create mode 100644 internal/handler/npm_dist_tags.go create mode 100644 internal/handler/npm_dist_tags_test.go diff --git a/README.md b/README.md index 20c6d5d..cfee785 100644 --- a/README.md +++ b/README.md @@ -1253,6 +1253,7 @@ Recently cached: | `GET /npm/*` | npm registry protocol | | `POST /npm/-/npm/v1/security/*` | npm/pnpm/Yarn audit endpoints, passed through to upstream | | `GET /npm/-/npm/v1/keys` | npm registry signing keys, passed through to upstream | +| `GET /npm/-/package/{name}/dist-tags` | `npm dist-tag ls`, answered from the packument the proxy serves (cooldown and pushed versions applied); changing tags is not supported | | `GET /npm/-/v1/search`, `GET /npm/-/ping` | `npm search` and `npm ping`, passed through to upstream uncached | | `GET /cargo/*` | Cargo sparse index protocol | | `GET /gem/*` | RubyGems protocol | diff --git a/docs/configuration.md b/docs/configuration.md index f10e4b2..ea498ac 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1093,7 +1093,7 @@ RubyGems, Hex and PyPI keep their publish times outside the metadata clients rea For RubyGems and Hex, both documents go through the metadata cache. With [`cache_metadata`](#metadata-caching) they are served from it within `metadata_ttl` without asking the upstream, revalidated with their ETags after it, and filtered from the last cached copies, marked stale, while the upstream is down; without it they are still fetched once for concurrent requests. Each document has its own TTL, so the compact index or Hex package can be newer than the cached times and list a version they don't have. The times are then fetched again before filtering, and if that fails, such versions are left out until the times can be read. The filtered index or package carries an ETag of its own, so Bundler and Mix revalidate it with `304 Not Modified`. Publish times are compared with the current time on each request, but the filtered result is kept for up to a minute (see [rewritten metadata cache](#rewritten-metadata-cache)), so a version leaving cooldown can show up a minute late; a newly published version shows up within `metadata_ttl`, as with other cached metadata. Upstream errors are answered as for other cached metadata (404, or 502) rather than relayed. -With cooldown on, the proxy asks npm for full packuments, since the abbreviated ones it uses otherwise carry no publish times. Abbreviated packuments cached before cooldown was turned on are fetched again in full rather than served, including as a stale copy when the upstream is down. +With cooldown on, the proxy asks npm for full packuments, since the abbreviated ones it uses otherwise carry no publish times. Abbreviated packuments cached before cooldown was turned on are fetched again in full rather than served, including as a stale copy when the upstream is down. When `latest` is in cooldown, the packument tags the newest version out of cooldown instead, and `npm dist-tag ls` shows the same tags. Conda cooldown covers `repodata.json`, `current_repodata.json`, `repodata_from_packages.json` and sharded repodata (CEP 16). Other forms of the repodata, such as `repodata.json.zst`, `repodata.json.bz2` and `repodata.jlap` patches, would list versions still in cooldown, so with cooldown on they are answered 404 and conda and mamba fall back to `repodata.json`. With `cache_metadata`, `repodata.json` is read through the metadata cache as it is without cooldown: within `metadata_ttl` the stored copy is filtered as it is read from storage, after it the upstream is asked with its ETag, and while the upstream is down the stale copy is filtered. Once a document has been filtered, the proxy remembers the ETag of the result until the first entry it left out leaves cooldown, so a client revalidating with `If-None-Match` gets `304 Not Modified` without the document being filtered or sent again. The first response for each document has no ETag, as it is sent while it is filtered. Without `cache_metadata`, `repodata.json` is fetched and filtered on every request. Clients cache shards by the hash in the shard index, so with cooldown on the proxy serves a rewritten index whose shard hashes change every hour and filters each shard as it is fetched. Versions leaving cooldown show up within the hour, and clients fetch the shards they need again once an hour. If a channel's shard index points packages or shards somewhere the proxy doesn't route, the proxy answers 404 for it and clients fall back to `repodata.json`. @@ -1462,7 +1462,7 @@ npm and Composer archives must contain the package manifest: `package.json` in t Pushed versions are listed in the package metadata clients resolve against, with the dependencies from their manifest: - A pushed version takes the place of an upstream version with the same number. -- Pushed versions are added next to the upstream ones and are never held back by cooldown. npm `dist-tags` stay as the upstream has them. +- Pushed versions are added next to the upstream ones and are never held back by cooldown. npm `dist-tags` stay as the upstream has them, also in `npm dist-tag ls`. - A package the upstream doesn't know is served from its pushed versions alone. Its npm `latest` tag is the highest pushed version. - Pushed Composer dev versions (`dev-*`, `*-dev`) are listed in the `~dev` metadata file. - npm entries carry the archive's sha512 as `dist.integrity`. Composer entries carry its sha256 as `dist.reference` and leave `shasum` empty. diff --git a/internal/handler/npm.go b/internal/handler/npm.go index c86dfd0..59ea27e 100644 --- a/internal/handler/npm.go +++ b/internal/handler/npm.go @@ -107,6 +107,11 @@ func (h *NPMHandler) Routes() http.Handler { return } + if strings.HasPrefix(r.URL.Path, npmPackagePrefix) && strings.HasSuffix(r.URL.Path, npmDistTagsSuffix) { + h.handleDistTags(w, r) + return + } + path := strings.TrimPrefix(r.URL.Path, "/") // Check if this is a tarball download (contains /-/) diff --git a/internal/handler/npm_dist_tags.go b/internal/handler/npm_dist_tags.go new file mode 100644 index 0000000..eb3741c --- /dev/null +++ b/internal/handler/npm_dist_tags.go @@ -0,0 +1,115 @@ +package handler + +import ( + "context" + "errors" + "net/http" + "strings" + + "github.com/git-pkgs/proxy/internal/jsonscan" +) + +// npmPackagePrefix and npmDistTagsSuffix frame the path `npm dist-tag ls` +// reads: /-/package/{name}/dist-tags, with a scoped name's slash encoded. +const ( + npmPackagePrefix = "/-/package/" + npmDistTagsSuffix = "/dist-tags" +) + +// handleDistTags answers `npm dist-tag ls` from the dist-tags of the +// packument the proxy serves, not from upstream's, so a tag never names a +// version cooldown or the denylist holds back, and pushed versions and +// package routes count as they do for the packument. Changing tags is not +// supported; Routes answers anything but GET with 405. +func (h *NPMHandler) handleDistTags(w http.ResponseWriter, r *http.Request) { + packageName := strings.TrimSuffix(strings.TrimPrefix(r.URL.Path, npmPackagePrefix), npmDistTagsSuffix) + if !validNPMPackageName(packageName) { + JSONError(w, http.StatusBadRequest, "invalid package name") + return + } + + packument, stale, err := h.servedPackument(r.Context(), packageName) + if err != nil { + if r.Context().Err() != nil { + return + } + if errors.Is(err, ErrUpstreamNotFound) || errors.Is(err, ErrPackageNotFound) { + JSONError(w, http.StatusNotFound, ErrPackageNotFound.Error()) + return + } + if writeUpstreamHeld(w, err) { + return + } + h.proxy.Logger.Error("failed to read npm dist-tags", "package", packageName, "error", err) + JSONError(w, http.StatusBadGateway, "failed to fetch from upstream") + return + } + + tags, err := jsonscan.Lookup(packument, "dist-tags") + if err != nil || len(tags) == 0 || tags[0] != '{' { + tags = []byte("{}") + } + markStale(w, stale) + w.Header().Set(headerContentType, contentTypeJSON) + w.WriteHeader(http.StatusOK) + _, _ = w.Write(tags) +} + +// servedPackument returns the packument of packageName as the proxy serves +// it: from the package's registry or route, filtered by cooldown and the +// denylist, with pushed versions added, or built from the pushed versions +// alone in a hosted-only scope. stale reports a cached copy served because +// upstream failed. +func (h *NPMHandler) servedPackument(ctx context.Context, packageName string) (body []byte, stale bool, err error) { + local, err := h.npmLocalEntries(packageName) + if err != nil { + return nil, false, err + } + if h.proxy.hostedOnly("npm", packageName) { + if len(local) == 0 { + return nil, false, ErrPackageNotFound + } + body, err := localNPMPackument(packageName, local) + return body, false, err + } + + registryURL, cacheKey := h.sourceFor(packageName) + _, registryPath := npmMetadataPath(packageName) + upstream := h.proxy.fetchMetadataDocument(ctx, "npm", cacheKey, registryURL+"/"+registryPath, h.packumentAccept(), h.packumentValidator()) + switch { + case errors.Is(upstream.err, ErrUpstreamNotFound) && len(local) > 0: + body, err := localNPMPackument(packageName, local) + return body, false, err + case upstream.err != nil: + return nil, false, upstream.err + case len(local) > 0: + body, _, err := h.rewriteMetadataWithLocal(packageName, upstream.body, local) + return body, upstream.stale, err + } + + rewritten, err := h.proxy.cachedRewrite(ctx, "npm", h.proxyURL, cacheKey, upstream.body, upstream.digest, func(b []byte) ([]byte, []string, error) { + return h.rewriteMetadataKeeping(packageName, b) + }) + if err != nil { + if h.proxy.mustFilterMetadata("npm", packageName) { + return nil, false, err + } + // The packument route serves upstream's document when the rewrite + // fails and nothing has to be filtered out. + return upstream.body, upstream.stale, nil + } + return rewritten.body, upstream.stale, nil +} + +// validNPMPackageName reports whether name is a plain or scoped package +// name, without path traversal. +func validNPMPackageName(name string) bool { + if name == "" || containsPathTraversal(name) || strings.ContainsAny(name, "\\") { + return false + } + scope, rest, scoped := strings.Cut(name, "/") + if !scoped { + return !strings.HasPrefix(name, "@") + } + return len(scope) > 1 && scope[0] == '@' && rest != "" && !strings.Contains(rest, "/") +} diff --git a/internal/handler/npm_dist_tags_test.go b/internal/handler/npm_dist_tags_test.go new file mode 100644 index 0000000..a23a1c9 --- /dev/null +++ b/internal/handler/npm_dist_tags_test.go @@ -0,0 +1,136 @@ +package handler + +import ( + "encoding/json" + "net/http" + "testing" + "time" + + "github.com/git-pkgs/cooldown" +) + +// taggedPackument has 1.0.0 published a month ago and 2.0.0 an hour ago, +// tagged latest. +func taggedPackument(name string) func(string) string { + month := time.Now().Add(-30 * 24 * time.Hour).UTC().Format(time.RFC3339) + hour := time.Now().Add(-time.Hour).UTC().Format(time.RFC3339) + return func(baseURL string) string { + return `{"name":"` + name + `","dist-tags":{"latest":"2.0.0","stable":"1.0.0"},"versions":{` + + `"1.0.0":{"version":"1.0.0","dist":{"tarball":"` + baseURL + `/x/-/x-1.0.0.tgz"}},` + + `"2.0.0":{"version":"2.0.0","dist":{"tarball":"` + baseURL + `/x/-/x-2.0.0.tgz"}}},` + + `"time":{"1.0.0":"` + month + `","2.0.0":"` + hour + `"}}` + } +} + +func decodeDistTags(t *testing.T, code int, body []byte) map[string]string { + t.Helper() + if code != http.StatusOK { + t.Fatalf("status = %d, body: %s", code, body) + } + var tags map[string]string + if err := json.Unmarshal(body, &tags); err != nil { + t.Fatalf("dist-tags are not a JSON object: %v: %s", err, body) + } + return tags +} + +func TestNPMDistTags(t *testing.T) { + upstream := newNPMUpstream(t, map[string]func(string) string{"/demo": taggedPackument("demo")}, http.StatusOK) + proxy, _, _, _ := setupTestProxy(t) + proxy.HTTPClient = http.DefaultClient + h := NewNPMHandler(proxy, "http://proxy.test", upstream.URL) + + w := serveNPM(t, h, http.MethodGet, "/-/package/demo/dist-tags", nil, nil) + tags := decodeDistTags(t, w.Code, w.Body.Bytes()) + if tags["latest"] != "2.0.0" || tags["stable"] != "1.0.0" || len(tags) != 2 { + t.Errorf("dist-tags = %v", tags) + } +} + +// TestNPMDistTagsFollowCooldown checks that latest never names a version +// cooldown holds back, as in the packument. +func TestNPMDistTagsFollowCooldown(t *testing.T) { + upstream := newNPMUpstream(t, map[string]func(string) string{"/demo": taggedPackument("demo")}, http.StatusOK) + proxy, _, _, _ := setupTestProxy(t) + proxy.HTTPClient = http.DefaultClient + proxy.Cooldown = &cooldown.Config{Default: "7d"} + h := NewNPMHandler(proxy, "http://proxy.test", upstream.URL) + + w := serveNPM(t, h, http.MethodGet, "/-/package/demo/dist-tags", nil, nil) + if tags := decodeDistTags(t, w.Code, w.Body.Bytes()); tags["latest"] != "1.0.0" { + t.Errorf("dist-tags = %v, want latest 1.0.0", tags) + } +} + +func TestNPMDistTagsRoutedPackage(t *testing.T) { + public := newNPMUpstream(t, map[string]func(string) string{"/@example/widgets": routedTestPackument("9.9.9", "")}, http.StatusOK) + private := newNPMUpstream(t, map[string]func(string) string{routedNPMPath: routedTestPackument("1.0.0", "/registry")}, http.StatusOK) + proxy, _, _, _ := setupTestProxy(t) + proxy.HTTPClient = http.DefaultClient + h := newRoutedNPMHandler(proxy, public, private) + + w := serveNPM(t, h, http.MethodGet, "/-/package/@example%2fwidgets/dist-tags", nil, nil) + if tags := decodeDistTags(t, w.Code, w.Body.Bytes()); tags["latest"] != "1.0.0" { + t.Errorf("dist-tags = %v, want the route's latest 1.0.0", tags) + } + if got := public.requested(); len(got) != 0 { + t.Errorf("default registry was queried for a routed package: %v", got) + } +} + +func TestNPMDistTagsHostedOnlyScope(t *testing.T) { + f := newNPMHostedFixture(t, nil) + f.proxy.HostedNPMScopes = []string{"@acme"} + f.push(t, "@acme/private", "1.10.0", `{"name":"@acme/private","version":"1.10.0"}`) + f.push(t, "@acme/private", "1.9.0", `{"name":"@acme/private","version":"1.9.0"}`) + + rec := f.get(t, "/-/package/@acme%2fprivate/dist-tags") + if tags := decodeDistTags(t, rec.Code, rec.Body.Bytes()); tags["latest"] != "1.10.0" { + t.Errorf("dist-tags = %v, want latest 1.10.0", tags) + } + if rec := f.get(t, "/-/package/@acme%2fother/dist-tags"); rec.Code != http.StatusNotFound { + t.Errorf("unpushed package status = %d, want 404", rec.Code) + } + if *f.upstreamRequests != 0 { + t.Errorf("hosted-only scope sent %d requests upstream", *f.upstreamRequests) + } +} + +func TestNPMDistTagsErrors(t *testing.T) { + upstream := newNPMUpstream(t, map[string]func(string) string{}, http.StatusOK) + proxy, _, _, _ := setupTestProxy(t) + proxy.HTTPClient = http.DefaultClient + h := NewNPMHandler(proxy, "http://proxy.test", upstream.URL) + + tests := []struct { + method, target string + want int + }{ + {http.MethodGet, "/-/package/missing/dist-tags", http.StatusNotFound}, + {http.MethodGet, "/-/package/@scope/dist-tags", http.StatusBadRequest}, + {http.MethodGet, "/-/package/a/b/c/dist-tags", http.StatusBadRequest}, + {http.MethodGet, "/-/package/..%2f..%2fetc/dist-tags", http.StatusBadRequest}, + {http.MethodPut, "/-/package/demo/dist-tags/beta", http.StatusMethodNotAllowed}, + {http.MethodDelete, "/-/package/demo/dist-tags/beta", http.StatusMethodNotAllowed}, + } + for _, tt := range tests { + if w := serveNPM(t, h, tt.method, tt.target, nil, nil); w.Code != tt.want { + t.Errorf("%s %s: status = %d, want %d", tt.method, tt.target, w.Code, tt.want) + } + } + if got := upstream.requested(); len(got) != 1 { + t.Errorf("upstream requests = %v, want only the missing package", got) + } +} + +// TestNPMDistTagsWithPushedVersion checks that a push next to upstream +// versions leaves the tags as upstream has them, as in the packument. +func TestNPMDistTagsWithPushedVersion(t *testing.T) { + f := newNPMHostedFixture(t, map[string]string{"/request": requestPackument}) + f.push(t, "request", "2.88.4", `{"name":"request","version":"2.88.4"}`) + + rec := f.get(t, "/-/package/request/dist-tags") + if tags := decodeDistTags(t, rec.Code, rec.Body.Bytes()); tags["latest"] != "2.88.2" { + t.Errorf("dist-tags = %v, want upstream's latest 2.88.2", tags) + } +} From 92b1fb20a6d451693d4d5347092f505a9a398d10 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Tue, 27 Oct 2026 16:31:47 +0100 Subject: [PATCH 465/633] Follow npm and Packagist change feeds to revalidate changed metadata Within metadata_ttl cached npm and Composer metadata is served without asking upstream, so a new release can take that long to show up. Both registries publish a feed of the packages that changed. upstream.npm_change_feed and upstream.composer_change_feed (off by default, require cache_metadata) start a reader per feed that runs every minute and clears fetched_at on the cached rows of listed packages that were fetched before the read. The rows keep their document and ETag, so the next request revalidates them upstream. Routed packages are cached under other keys and never match. A Packagist resync expires all cached Composer metadata. Failed reads back off up to 15 minutes. The feed position is kept in memory, so after a restart the reader starts from the feed's current position. --- README.md | 7 +- config.example.yaml | 9 + docs/configuration.md | 22 ++ internal/changefeed/changefeed.go | 261 +++++++++++++++++ internal/changefeed/changefeed_test.go | 198 +++++++++++++ internal/config/config.go | 20 ++ internal/config/config_test.go | 35 +++ internal/database/metadata_cache_test.go | 90 ++++++ internal/database/queries.go | 46 +++ internal/metrics/metrics.go | 16 ++ internal/server/analytics_coverage_test.go | 40 +-- internal/server/analytics_runtime.go | 2 + internal/server/analytics_runtime_test.go | 18 ++ internal/server/change_feeds.go | 94 ++++++ internal/server/change_feeds_test.go | 272 ++++++++++++++++++ internal/server/server.go | 1 + internal/server/server_test.go | 7 - .../templates/components/runtime_metrics.html | 1 + 18 files changed, 1110 insertions(+), 29 deletions(-) create mode 100644 internal/changefeed/changefeed.go create mode 100644 internal/changefeed/changefeed_test.go create mode 100644 internal/server/change_feeds.go create mode 100644 internal/server/change_feeds_test.go diff --git a/README.md b/README.md index 20c6d5d..4f0c033 100644 --- a/README.md +++ b/README.md @@ -188,7 +188,7 @@ therefore come from upstream's database, not from the proxy's own vulnerability data, and versions withheld by [cooldown](#version-cooldown) are not excluded from the report. -To serve private packages through the same registry URL, see [Private package routes](docs/configuration.md#private-package-routes). +To serve private packages through the same registry URL, see [Private package routes](docs/configuration.md#private-package-routes). To see new releases before `metadata_ttl` runs out, see [Change feeds](docs/configuration.md#change-feeds). ### Cargo @@ -387,7 +387,7 @@ Or set globally: composer config -g repositories.proxy composer http://localhost:8080/composer ``` -To serve private packages through the same URL, see [Private package routes](docs/configuration.md#private-package-routes). +To serve private packages through the same URL, see [Private package routes](docs/configuration.md#private-package-routes). To see new releases before `metadata_ttl` runs out, see [Change feeds](docs/configuration.md#change-feeds). ### Conan (C/C++) @@ -1521,6 +1521,7 @@ The proxy exposes Prometheus metrics at `GET /metrics`. All metric names are pre | `proxy_metadata_cache_entries` | gauge | `ecosystem` | Number of cached metadata documents per ecosystem. | | `proxy_artifacts_evicted_total` | counter | `reason`, `ecosystem` | Artifacts evicted by the size limit (`lru`) or by age (`retention`) | | `proxy_metadata_evicted_total` | counter | `ecosystem` | Cached metadata documents evicted by the size limit | +| `proxy_metadata_expired_by_change_feed_total` | counter | `ecosystem` | Cached metadata documents set to revalidate because an upstream [change feed](docs/configuration.md#change-feeds) listed them | | `proxy_fetch_lease_waits_total` | counter | `kind` | Cache misses (`artifact` or `metadata`) that waited for another instance sharing the database to fetch them | | `proxy_upstream_fetch_duration_seconds` | histogram | `ecosystem` | Time spent fetching from upstream | | `proxy_upstream_errors_total` | counter | `ecosystem`, `error_type` | Upstream fetch failures | @@ -1591,7 +1592,7 @@ The `kind` label on `proxy_cache_hits_total` and `proxy_cache_misses_total` keep `ecosystem` means three slightly different things across `/metrics`, and queries that join across them need to know which. -**From the package record, normalized.** The six `proxy_ecosystem_*` gauges, `proxy_cache_hits_total`, `proxy_cache_misses_total`, the three `proxy_metadata_cache_*` metrics, `proxy_artifacts_evicted_total`, `proxy_metadata_evicted_total`, `proxy_integrity_failures_total` and the scan metrics. Aliases collapse here: `gem` reads as `rubygems`, `composer` as `packagist`, `go` as `golang`. +**From the package record, normalized.** The six `proxy_ecosystem_*` gauges, `proxy_cache_hits_total`, `proxy_cache_misses_total`, the three `proxy_metadata_cache_*` metrics, `proxy_artifacts_evicted_total`, `proxy_metadata_evicted_total`, `proxy_metadata_expired_by_change_feed_total`, `proxy_integrity_failures_total` and the scan metrics. Aliases collapse here: `gem` reads as `rubygems`, `composer` as `packagist`, `go` as `golang`. **From the request path.** `proxy_requests_total`, `proxy_request_duration_seconds` and `proxy_response_bytes_total`. The names mostly coincide with the normalized ones -- these also report `rubygems`, `packagist` and `golang` -- but the Debian route reports `debian` where the package record says `deb`, and any path that is not a package endpoint reports `other`, which corresponds to no ecosystem at all. diff --git a/config.example.yaml b/config.example.yaml index d3b6da4..8ad3759 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -189,6 +189,10 @@ upstream: # npm_routes: # "@example/*": "https://npm.example.com/registry" + # Follow the npm registry's change feed, so a new release is fetched on the + # next request instead of after metadata_ttl. Requires cache_metadata. + # npm_change_feed: true + # Cargo sparse index URL cargo: "https://index.crates.io" @@ -245,6 +249,11 @@ upstream: # composer_routes: # "example/*": "https://composer.example.com/packages" + # Follow the change feed of the Packagist at composer, so a new release is + # fetched on the next request instead of after metadata_ttl. Requires + # cache_metadata. + # composer_change_feed: true + # Conan registry URL conan: "https://center.conan.io" diff --git a/docs/configuration.md b/docs/configuration.md index f10e4b2..e18263f 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -287,6 +287,7 @@ Each upstream used by a built-in package route can be set in YAML or JSON under | `upstream.allow_loopback` | `PROXY_UPSTREAM_ALLOW_LOOPBACK` | `false` | | `upstream.npm` | `PROXY_UPSTREAM_NPM` | `https://registry.npmjs.org` | | `upstream.npm_full_metadata` | `PROXY_UPSTREAM_NPM_FULL_METADATA` | `false` | +| `upstream.npm_change_feed` | `PROXY_UPSTREAM_NPM_CHANGE_FEED` | `false` (see [change feeds](#change-feeds)) | | `upstream.cargo` | `PROXY_UPSTREAM_CARGO` | `https://index.crates.io` | | `upstream.cargo_download` | `PROXY_UPSTREAM_CARGO_DOWNLOAD` | `https://static.crates.io/crates` | | `upstream.gem` | `PROXY_UPSTREAM_GEM` | `https://rubygems.org` | @@ -303,6 +304,7 @@ Each upstream used by a built-in package route can be set in YAML or JSON under | `upstream.nuget_search` | `PROXY_UPSTREAM_NUGET_SEARCH` | `https://azuresearch-usnc.nuget.org` | | `upstream.composer` | `PROXY_UPSTREAM_COMPOSER` | `https://packagist.org` | | `upstream.composer_repository` | `PROXY_UPSTREAM_COMPOSER_REPOSITORY` | `https://repo.packagist.org` | +| `upstream.composer_change_feed` | `PROXY_UPSTREAM_COMPOSER_CHANGE_FEED` | `false` (see [change feeds](#change-feeds)) | | `upstream.conan` | `PROXY_UPSTREAM_CONAN` | `https://center.conan.io` | | `upstream.chef` | `PROXY_UPSTREAM_CHEF` | `https://supermarket.chef.io` | | `upstream.conda` | `PROXY_UPSTREAM_CONDA` | `https://conda.anaconda.org` | @@ -1346,6 +1348,26 @@ Set to `"0"` to always revalidate with upstream (ETag-based conditional requests When upstream is unreachable and the cached entry is past its TTL, the proxy serves the stale cached copy with a `Warning: 110 - "Response is Stale"` header so clients can tell the data may be outdated. +### Change feeds + +Within `metadata_ttl` a new release isn't seen. For npm and Composer the proxy can instead follow the registry's change feed, which lists the packages that changed, and revalidate just those: + +```yaml +cache_metadata: true +upstream: + npm_change_feed: true + composer_change_feed: true +``` + +Or via environment variables: `PROXY_UPSTREAM_NPM_CHANGE_FEED=true`, `PROXY_UPSTREAM_COMPOSER_CHANGE_FEED=true`. Both require `cache_metadata`, and do nothing with a `metadata_ttl` of `"0"`. + +Every minute the proxy reads what changed since its last read. Cached metadata of a listed package that was fetched before the read is marked as needing revalidation, and the next request for it asks the upstream with its `ETag` (a `304` when nothing changed). Artifacts are not touched. Listed packages the proxy hasn't cached cost nothing beyond the read. The feed is read in full, so the upstream doesn't learn which packages the proxy has cached. Packages in `upstream.npm_routes` or `upstream.composer_routes` are cached under other keys and never matched. + +- Composer reads `{upstream.composer}/metadata/changes.json`, which packagist.org advertises as `metadata-changes-url`. Both `vendor/package` and its `vendor/package~dev` file are covered. Packagist keeps 24 hours of changes; if the proxy's position is older than that, Packagist asks for a resync and all cached Composer metadata is revalidated. repo.packagist.org sends `Cache-Control: max-age=900`, but Packagist purges its CDN when a package changes, so revalidation gets the new file within seconds; the proxy doesn't apply that header either way. +- npm reads `https://replicate.npmjs.com/registry/_changes`, the public registry's feed, whatever `upstream.npm` is. It is only useful when `upstream.npm` is registry.npmjs.org or serves the same packages. Packuments (`/npm/{name}`) are covered; version and dist-tag documents (`/npm/{name}/latest`) still wait out `metadata_ttl`. A busy minute on npm lists a few thousand packages; the proxy reads up to 100,000 per pass and continues where it stopped on the next. + +A new release therefore shows up about a minute after the registry lists it rather than up to `metadata_ttl` later. The feed position is kept in memory: after a restart the proxy starts from the feed's current position, and changes made while it was down are covered by `metadata_ttl` as before. While a feed can't be read the proxy logs a warning and retries with a growing delay, up to 15 minutes, and `metadata_ttl` applies meanwhile. A request whose upstream fetch overlaps the change and the feed read can still store the old document for one TTL. Each instance sharing a database reads the feeds itself. `proxy_metadata_expired_by_change_feed_total` counts the cached documents marked, by ecosystem, and debug logging names each one. + ### Metadata size limit Upstream metadata responses are buffered in memory before being rewritten and served. `metadata_max_size` caps that buffer to protect against OOM from a misbehaving upstream. Some npm packages with thousands of versions (for example `renovate`) exceed the 100 MB default, so raise this if you see `metadata response exceeds size limit` in the logs. Conda `repodata.json` with cooldown enabled is filtered as it streams and isn't held in memory decompressed. With `cache_metadata` the limit applies to the compressed document as it does without cooldown; one over it is filtered as it streams from the upstream and not cached. diff --git a/internal/changefeed/changefeed.go b/internal/changefeed/changefeed.go new file mode 100644 index 0000000..2e2d48f --- /dev/null +++ b/internal/changefeed/changefeed.go @@ -0,0 +1,261 @@ +// Package changefeed reads the feeds in which public registries list the +// packages that changed since a given point, so the proxy can revalidate its +// cached metadata for them before metadata_ttl runs out. +package changefeed + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "slices" + "strconv" + "strings" +) + +// Feed is one registry's change feed. It remembers how far it has read, in +// memory only: after a restart it starts again from the registry's current +// position. +type Feed interface { + // Ecosystem is the metadata cache ecosystem the changed names belong to. + Ecosystem() string + // Changes returns what changed since the previous call. The first call + // only finds the feed's current position and reports no changes. On error + // the position stays where it was, so the next call asks again. + Changes(ctx context.Context) (Changes, error) +} + +// Changes is one read of a feed. +type Changes struct { + // Names are the cache keys of the documents that changed. + Names []string + // All is set when the feed can't say what changed, so every document of + // the ecosystem has to be treated as changed. + All bool +} + +// maxFeedResponse bounds a feed response read into memory. A full page of +// the npm feed is about 1.5 MB. +const maxFeedResponse = 64 << 20 + +// getJSON fetches u and decodes its JSON body into v. A status other than 200 +// is an error unless allowStatus also accepts it. +func getJSON(ctx context.Context, client *http.Client, u string, v any, allowStatus ...int) error { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil) + if err != nil { + return fmt.Errorf("creating request: %w", err) + } + req.Header.Set("Accept", "application/json") + resp, err := client.Do(req) + if err != nil { + return err + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode != http.StatusOK && !slices.Contains(allowStatus, resp.StatusCode) { + _, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, maxFeedResponse)) + return fmt.Errorf("%s: unexpected status %d", redact(u), resp.StatusCode) + } + body, err := io.ReadAll(io.LimitReader(resp.Body, maxFeedResponse+1)) + if err != nil { + return fmt.Errorf("%s: reading response: %w", redact(u), err) + } + if len(body) > maxFeedResponse { + return fmt.Errorf("%s: response larger than %d bytes", redact(u), maxFeedResponse) + } + if err := json.Unmarshal(body, v); err != nil { + return fmt.Errorf("%s: decoding response: %w", redact(u), err) + } + return nil +} + +// redact drops credentials and the query from u for error messages. +func redact(u string) string { + parsed, err := url.Parse(u) + if err != nil { + return "feed" + } + parsed.User = nil + parsed.RawQuery = "" + return parsed.String() +} + +// withQuery returns base with the query parameters in q added. +func withQuery(base string, q url.Values) string { + sep := "?" + if strings.Contains(base, "?") { + sep = "&" + } + return base + sep + q.Encode() +} + +// ComposerFeedURL is where a Packagist instance at apiURL, such as +// https://packagist.org, publishes its metadata change feed. Packagist +// advertises it as "metadata-changes-url" in its packages.json. +func ComposerFeedURL(apiURL string) string { + return strings.TrimSuffix(apiURL, "/") + "/metadata/changes.json" +} + +// ComposerFeed reads Packagist's metadata change feed +// (https://packagist.org/apidoc#track-package-updates). Its position is a +// timestamp in units of 1/10000 s. Asked without one it answers 400 with the +// current timestamp; asked with one older than its 24 hours of history it +// answers with a single "resync" action. Each update names the file that +// changed, "vendor/package" or "vendor/package~dev", which is also the key +// the Composer handler caches it under; a delete names the package, and +// covers both files. +type ComposerFeed struct { + client *http.Client + url string + since int64 +} + +// NewComposerFeed returns the feed at feedURL, see ComposerFeedURL. +func NewComposerFeed(client *http.Client, feedURL string) *ComposerFeed { + return &ComposerFeed{client: client, url: feedURL} +} + +func (f *ComposerFeed) Ecosystem() string { return "composer" } + +type composerChanges struct { + Actions []struct { + Type string `json:"type"` + Package string `json:"package"` + } `json:"actions"` + Timestamp int64 `json:"timestamp"` + Error string `json:"error"` +} + +func (f *ComposerFeed) Changes(ctx context.Context) (Changes, error) { + if f.since == 0 { + var start composerChanges + if err := getJSON(ctx, f.client, f.url, &start, http.StatusBadRequest); err != nil { + return Changes{}, err + } + if start.Timestamp <= 0 { + return Changes{}, errors.New("composer change feed: no timestamp to start from") + } + f.since = start.Timestamp + return Changes{}, nil + } + + var page composerChanges + u := withQuery(f.url, url.Values{"since": {strconv.FormatInt(f.since, 10)}}) + if err := getJSON(ctx, f.client, u, &page); err != nil { + return Changes{}, err + } + if page.Error != "" || page.Timestamp <= 0 { + return Changes{}, fmt.Errorf("composer change feed: %q", page.Error) + } + var changes Changes + for _, a := range page.Actions { + switch a.Type { + case "update": + changes.Names = append(changes.Names, a.Package) + case "delete": + // A deleted package takes its dev versions with it. + changes.Names = append(changes.Names, a.Package, a.Package+"~dev") + case "resync": + // The position is too old for the feed to list what changed. + changes.All = true + } + } + f.since = page.Timestamp + return changes, nil +} + +// NPMFeedURL is the change feed of the public npm registry. +const NPMFeedURL = "https://replicate.npmjs.com/registry/_changes" + +// npmPageLimit is the most changes the npm feed returns in one response. +const npmPageLimit = 10000 + +// npmMaxPages bounds how many pages one Changes call reads. The rest is read +// on the next call. +const npmMaxPages = 10 + +// NPMFeed reads the npm registry's CouchDB-style changes feed at +// replicate.npmjs.com. Its position is a sequence, which npm asks followers +// to treat as opaque; the database document at the feed's parent URL, with +// its trailing slash (without it the URL is 404), gives the current one as +// update_seq. A page holds at most 10000 changes (1000 when no limit is +// given). Each result's id is a package name, which is also the key the npm +// handler caches its packument under. +type NPMFeed struct { + client *http.Client + url string + // since is the sequence exactly as the feed returned it. + since json.RawMessage +} + +// NewNPMFeed returns the feed at feedURL, see NPMFeedURL. +func NewNPMFeed(client *http.Client, feedURL string) *NPMFeed { + return &NPMFeed{client: client, url: feedURL} +} + +func (f *NPMFeed) Ecosystem() string { return "npm" } + +type npmChanges struct { + Results []struct { + ID string `json:"id"` + } `json:"results"` + LastSeq json.RawMessage `json:"last_seq"` +} + +func (f *NPMFeed) Changes(ctx context.Context) (Changes, error) { + if f.since == nil { + var info struct { + UpdateSeq json.RawMessage `json:"update_seq"` + } + if err := getJSON(ctx, f.client, strings.TrimSuffix(f.url, "_changes"), &info); err != nil { + return Changes{}, err + } + if len(info.UpdateSeq) == 0 { + return Changes{}, errors.New("npm change feed: no update_seq to start from") + } + f.since = info.UpdateSeq + return Changes{}, nil + } + + var changes Changes + for range npmMaxPages { + var page npmChanges + u := withQuery(f.url, url.Values{ + "since": {seqParam(f.since)}, + "limit": {strconv.Itoa(npmPageLimit)}, + }) + if err := getJSON(ctx, f.client, u, &page); err != nil { + // Changes already read are reported, and the position kept + // after them, so they aren't lost to a later page's failure. + if len(changes.Names) > 0 { + return changes, nil + } + return Changes{}, err + } + if len(page.LastSeq) == 0 { + return changes, errors.New("npm change feed: response without last_seq") + } + for _, r := range page.Results { + if r.ID != "" && !strings.HasPrefix(r.ID, "_design/") { + changes.Names = append(changes.Names, r.ID) + } + } + f.since = page.LastSeq + if len(page.Results) < npmPageLimit { + break + } + } + return changes, nil +} + +// seqParam returns a sequence as the since parameter takes it: a number as +// is, a string without its quotes. +func seqParam(seq json.RawMessage) string { + var s string + if json.Unmarshal(seq, &s) == nil { + return s + } + return string(seq) +} diff --git a/internal/changefeed/changefeed_test.go b/internal/changefeed/changefeed_test.go new file mode 100644 index 0000000..21a4765 --- /dev/null +++ b/internal/changefeed/changefeed_test.go @@ -0,0 +1,198 @@ +package changefeed + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "slices" + "strconv" + "strings" + "sync" + "sync/atomic" + "testing" +) + +// fakePackagist serves a change feed the way packagist.org does: without +// since, 400 with the current timestamp; with an old since, a resync; with a +// known one, the actions recorded after it. +type fakePackagist struct { + mu sync.Mutex + now int64 + actions map[int64]string // since -> JSON actions + asked []string +} + +func (p *fakePackagist) ServeHTTP(w http.ResponseWriter, r *http.Request) { + p.mu.Lock() + defer p.mu.Unlock() + since := r.URL.Query().Get("since") + p.asked = append(p.asked, since) + w.Header().Set("Content-Type", "application/json") + if since == "" { + w.WriteHeader(http.StatusBadRequest) + _, _ = fmt.Fprintf(w, `{"error":"Invalid or missing since query parameter","timestamp":%d}`, p.now) + return + } + n, _ := strconv.ParseInt(since, 10, 64) + actions, ok := p.actions[n] + if !ok { + actions = `[{"type":"resync","package":"*","time":1}]` + } + _, _ = fmt.Fprintf(w, `{"actions":%s,"timestamp":%d}`, actions, p.now) +} + +func TestComposerFeed(t *testing.T) { + p := &fakePackagist{now: 100, actions: map[int64]string{ + 100: `[{"type":"update","package":"acme/lib","time":1},{"type":"update","package":"acme/lib~dev","time":1},{"type":"delete","package":"acme/gone","time":1}]`, + 200: `[]`, + }} + srv := httptest.NewServer(p) + defer srv.Close() + feed := NewComposerFeed(srv.Client(), ComposerFeedURL(srv.URL+"/")) + ctx := context.Background() + + if c, err := feed.Changes(ctx); err != nil || c.Names != nil || c.All { + t.Fatalf("first read = %+v, %v; want the start position only", c, err) + } + p.mu.Lock() + p.now = 200 + p.mu.Unlock() + c, err := feed.Changes(ctx) + if err != nil { + t.Fatal(err) + } + if want := []string{"acme/lib", "acme/lib~dev", "acme/gone", "acme/gone~dev"}; !slices.Equal(c.Names, want) || c.All { + t.Errorf("changes = %+v, want %v", c, want) + } + if c, err := feed.Changes(ctx); err != nil || len(c.Names) != 0 || c.All { + t.Errorf("changes without news = %+v, %v", c, err) + } + + // A position the feed no longer has history for gets a resync. + feed.since = 50 + if c, err := feed.Changes(ctx); err != nil || !c.All { + t.Errorf("changes after a resync = %+v, %v, want All", c, err) + } + if want := []string{"", "100", "200", "50"}; !slices.Equal(p.asked, want) { + t.Errorf("asked since %q, want %q", p.asked, want) + } +} + +func TestComposerFeedErrorKeepsPosition(t *testing.T) { + var fail atomic.Bool + fail.Store(true) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if fail.Load() { + w.WriteHeader(http.StatusServiceUnavailable) + return + } + if r.URL.Query().Get("since") != "7" { + t.Errorf("since = %q, want 7", r.URL.Query().Get("since")) + } + _, _ = w.Write([]byte(`{"actions":[{"type":"update","package":"acme/lib","time":1}],"timestamp":8}`)) + })) + defer srv.Close() + feed := &ComposerFeed{client: srv.Client(), url: srv.URL, since: 7} + if _, err := feed.Changes(context.Background()); err == nil { + t.Fatal("want an error from a failing feed") + } + if feed.since != 7 { + t.Fatalf("since = %d after an error, want 7", feed.since) + } + fail.Store(false) + if c, err := feed.Changes(context.Background()); err != nil || !slices.Equal(c.Names, []string{"acme/lib"}) { + t.Errorf("changes after recovery = %+v, %v", c, err) + } +} + +// fakeReplicate serves a CouchDB-style changes feed over seqs 1..len(ids). +type fakeReplicate struct { + mu sync.Mutex + ids []string + limit int + asked []string +} + +func (f *fakeReplicate) ServeHTTP(w http.ResponseWriter, r *http.Request) { + f.mu.Lock() + defer f.mu.Unlock() + w.Header().Set("Content-Type", "application/json") + if r.URL.Path == "/registry/" { + _, _ = fmt.Fprintf(w, `{"db_name":"registry","update_seq":%d}`, len(f.ids)) + return + } + if r.URL.Path != "/registry/_changes" { + w.WriteHeader(http.StatusNotFound) + return + } + since, _ := strconv.Atoi(r.URL.Query().Get("since")) + limit, _ := strconv.Atoi(r.URL.Query().Get("limit")) + f.asked = append(f.asked, r.URL.Query().Get("since")) + if f.limit > 0 { + limit = min(limit, f.limit) + } + var results []string + last := since + for seq := since + 1; seq <= len(f.ids) && len(results) < limit; seq++ { + results = append(results, fmt.Sprintf(`{"seq":%d,"id":%q,"changes":[{"rev":"1-a"}]}`, seq, f.ids[seq-1])) + last = seq + } + _, _ = fmt.Fprintf(w, `{"results":[%s],"last_seq":%d}`, strings.Join(results, ","), last) +} + +func TestNPMFeed(t *testing.T) { + f := &fakeReplicate{ids: []string{"old"}} + srv := httptest.NewServer(f) + defer srv.Close() + feed := NewNPMFeed(srv.Client(), srv.URL+"/registry/_changes") + ctx := context.Background() + + if c, err := feed.Changes(ctx); err != nil || c.Names != nil { + t.Fatalf("first read = %+v, %v; want the start position only", c, err) + } + f.mu.Lock() + f.ids = append(f.ids, "lodash", "@scope/pkg", "_design/app") + f.mu.Unlock() + c, err := feed.Changes(ctx) + if err != nil { + t.Fatal(err) + } + if want := []string{"lodash", "@scope/pkg"}; !slices.Equal(c.Names, want) { + t.Errorf("changes = %v, want %v", c.Names, want) + } + if c, err := feed.Changes(ctx); err != nil || len(c.Names) != 0 { + t.Errorf("changes without news = %+v, %v", c, err) + } + if want := []string{"1", "4"}; !slices.Equal(f.asked, want) { + t.Errorf("asked since %q, want %q", f.asked, want) + } +} + +// TestNPMFeedPages checks that a backlog longer than a page is read page by +// page, and that one read stops after npmMaxPages. +func TestNPMFeedPages(t *testing.T) { + f := &fakeReplicate{limit: npmPageLimit} + srv := httptest.NewServer(f) + defer srv.Close() + feed := &NPMFeed{client: srv.Client(), url: srv.URL + "/registry/_changes", since: []byte("0")} + f.ids = make([]string, npmPageLimit*(npmMaxPages+1)+5) + for i := range f.ids { + f.ids[i] = "pkg-" + strconv.Itoa(i) + } + + c, err := feed.Changes(context.Background()) + if err != nil { + t.Fatal(err) + } + if len(c.Names) != npmPageLimit*npmMaxPages { + t.Errorf("first read returned %d names, want %d", len(c.Names), npmPageLimit*npmMaxPages) + } + c, err = feed.Changes(context.Background()) + if err != nil { + t.Fatal(err) + } + if len(c.Names) != npmPageLimit+5 { + t.Errorf("second read returned %d names, want %d", len(c.Names), npmPageLimit+5) + } +} diff --git a/internal/config/config.go b/internal/config/config.go index a660699..1787dc2 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -590,6 +590,12 @@ type UpstreamConfig struct { // wins. NPMRoutes map[string]string `json:"npm_routes" yaml:"npm_routes"` + // NPMChangeFeed follows the public npm registry's change feed and + // revalidates cached packuments of packages listed in it on their next + // request, rather than serving them until metadata_ttl runs out. + // Requires cache_metadata. Default: false. + NPMChangeFeed bool `json:"npm_change_feed" yaml:"npm_change_feed"` + // Cargo is the upstream cargo index URL. // Default: https://index.crates.io Cargo string `json:"cargo" yaml:"cargo"` @@ -682,6 +688,12 @@ type UpstreamConfig struct { // matching pattern wins. ComposerRoutes map[string]string `json:"composer_routes" yaml:"composer_routes"` + // ComposerChangeFeed follows the metadata change feed of + // upstream.composer and revalidates cached metadata of packages listed + // in it on their next request, rather than serving it until + // metadata_ttl runs out. Requires cache_metadata. Default: false. + ComposerChangeFeed bool `json:"composer_change_feed" yaml:"composer_change_feed"` + // Conan is the upstream Conan registry URL. // Default: https://center.conan.io Conan string `json:"conan" yaml:"conan"` @@ -969,6 +981,12 @@ func (c *Config) validateUpstream() error { if c.Upstream.GoSumDBsOffline && !c.CacheMetadata { return fmt.Errorf("upstream.go_sumdbs_offline requires cache_metadata") } + if c.Upstream.NPMChangeFeed && !c.CacheMetadata { + return fmt.Errorf("upstream.npm_change_feed requires cache_metadata") + } + if c.Upstream.ComposerChangeFeed && !c.CacheMetadata { + return fmt.Errorf("upstream.composer_change_feed requires cache_metadata") + } return nil } @@ -1407,6 +1425,7 @@ func (c *Config) LoadFromEnv() { setEnvBool(&c.Upstream.AllowLoopback, "PROXY_UPSTREAM_ALLOW_LOOPBACK") setEnvString(&c.Upstream.NPM, "PROXY_UPSTREAM_NPM") setEnvBool(&c.Upstream.NPMFullMetadata, "PROXY_UPSTREAM_NPM_FULL_METADATA") + setEnvBool(&c.Upstream.NPMChangeFeed, "PROXY_UPSTREAM_NPM_CHANGE_FEED") setEnvBool(&c.Upstream.GoSumDBsOffline, "PROXY_UPSTREAM_GO_SUMDBS_OFFLINE") setEnvString(&c.Upstream.Cargo, "PROXY_UPSTREAM_CARGO") setEnvString(&c.Upstream.CargoDownload, "PROXY_UPSTREAM_CARGO_DOWNLOAD") @@ -1423,6 +1442,7 @@ func (c *Config) LoadFromEnv() { setEnvString(&c.Upstream.NuGetSearch, "PROXY_UPSTREAM_NUGET_SEARCH") setEnvString(&c.Upstream.Composer, "PROXY_UPSTREAM_COMPOSER") setEnvString(&c.Upstream.ComposerRepository, "PROXY_UPSTREAM_COMPOSER_REPOSITORY") + setEnvBool(&c.Upstream.ComposerChangeFeed, "PROXY_UPSTREAM_COMPOSER_CHANGE_FEED") setEnvString(&c.Upstream.Conan, "PROXY_UPSTREAM_CONAN") setEnvString(&c.Upstream.Chef, "PROXY_UPSTREAM_CHEF") setEnvString(&c.Upstream.Conda, "PROXY_UPSTREAM_CONDA") diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 7590a64..0730e01 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -1137,6 +1137,17 @@ func TestLoadMetadataTTLFromEnv(t *testing.T) { } } +func TestLoadChangeFeedsFromEnv(t *testing.T) { + cfg := Default() + t.Setenv("PROXY_UPSTREAM_NPM_CHANGE_FEED", "true") + t.Setenv("PROXY_UPSTREAM_COMPOSER_CHANGE_FEED", "true") + cfg.LoadFromEnv() + + if !cfg.Upstream.NPMChangeFeed || !cfg.Upstream.ComposerChangeFeed { + t.Errorf("change feeds = npm %t, composer %t, want both on", cfg.Upstream.NPMChangeFeed, cfg.Upstream.ComposerChangeFeed) + } +} + func TestParseGradleBuildCacheConfig(t *testing.T) { cfg := Default() @@ -1725,6 +1736,30 @@ func TestValidateGoSumDBs(t *testing.T) { cfg.Upstream.GoSumDBsOffline = true }, }, + { + name: "npm change feed requires metadata caching", + modify: func(cfg *Config) { + cfg.CacheMetadata = false + cfg.Upstream.NPMChangeFeed = true + }, + wantErr: true, + }, + { + name: "Composer change feed requires metadata caching", + modify: func(cfg *Config) { + cfg.CacheMetadata = false + cfg.Upstream.ComposerChangeFeed = true + }, + wantErr: true, + }, + { + name: "change feeds with metadata caching", + modify: func(cfg *Config) { + cfg.CacheMetadata = true + cfg.Upstream.NPMChangeFeed = true + cfg.Upstream.ComposerChangeFeed = true + }, + }, } for _, tt := range tests { diff --git a/internal/database/metadata_cache_test.go b/internal/database/metadata_cache_test.go index 0e19f6c..ee16c25 100644 --- a/internal/database/metadata_cache_test.go +++ b/internal/database/metadata_cache_test.go @@ -2,7 +2,9 @@ package database import ( "database/sql" + "fmt" "path/filepath" + "slices" "testing" "time" ) @@ -448,3 +450,91 @@ func TestRefreshMetadataCache(t *testing.T) { t.Errorf("refresh of a missing row = %v, %v, want false", refreshed, err) } } + +// TestExpireMetadataCache checks that expiring clears fetched_at only on the +// named rows of the ecosystem fetched before the cutoff, and keeps their +// documents and validators. +func TestExpireMetadataCache(t *testing.T) { + db := setupMetadataCacheDB(t) + cutoff := time.Now().UTC().Truncate(time.Second) + rows := []struct { + ecosystem, name string + fetched time.Time + }{ + {"composer", "acme/lib", cutoff.Add(-time.Minute)}, + {"composer", "acme/lib~dev", cutoff.Add(-time.Minute)}, + {"composer", "acme/new", cutoff.Add(time.Second)}, + {"composer", "acme/other", cutoff.Add(-time.Minute)}, + {"composer", "routes/abc/acme/lib", cutoff.Add(-time.Minute)}, + {testEcosystemNPM, "acme/lib", cutoff.Add(-time.Minute)}, + } + for _, r := range rows { + err := db.UpsertMetadataCache(&MetadataCacheEntry{ + Ecosystem: r.ecosystem, + Name: r.name, + StoragePath: "_metadata/" + r.ecosystem + "/" + r.name, + ETag: sql.NullString{String: `"abc"`, Valid: true}, + ContentDigest: sql.NullString{String: "sha256:aaaa", Valid: true}, + FetchedAt: sql.NullTime{Time: r.fetched, Valid: true}, + }) + if err != nil { + t.Fatal(err) + } + } + + expired, err := db.ExpireMetadataCache("composer", []string{"acme/lib", "acme/lib~dev", "acme/new", "acme/missing"}, cutoff) + if err != nil { + t.Fatal(err) + } + slices.Sort(expired) + if want := []string{"acme/lib", "acme/lib~dev"}; !slices.Equal(expired, want) { + t.Errorf("expired = %v, want %v", expired, want) + } + got, err := db.GetMetadataCache("composer", "acme/lib") + if err != nil { + t.Fatal(err) + } + if got.FetchedAt.Valid || got.ETag.String != `"abc"` || got.ContentDigest.String != "sha256:aaaa" { + t.Errorf("expired row = %+v, want no fetched_at and the same validators", got) + } + for _, r := range []struct{ ecosystem, name string }{{"composer", "acme/new"}, {"composer", "acme/other"}, {testEcosystemNPM, "acme/lib"}} { + if got, _ := db.GetMetadataCache(r.ecosystem, r.name); !got.FetchedAt.Valid { + t.Errorf("%s %s expired, want it left alone", r.ecosystem, r.name) + } + } + + expired, err = db.ExpireAllMetadataCache("composer", cutoff) + if err != nil { + t.Fatal(err) + } + if want := []string{"acme/other"}; !slices.Equal(expired, want) { + t.Errorf("expired all = %v, want %v (routed and newer rows kept)", expired, want) + } +} + +// TestExpireMetadataCacheBatches checks that more names than one statement +// takes are all expired. +func TestExpireMetadataCacheBatches(t *testing.T) { + db := setupMetadataCacheDB(t) + cutoff := time.Now().UTC() + names := make([]string, expireMetadataBatch+2) + for i := range names { + names[i] = fmt.Sprintf("pkg-%d", i) + } + for _, name := range []string{names[0], names[len(names)-1]} { + err := db.UpsertMetadataCache(&MetadataCacheEntry{ + Ecosystem: testEcosystemNPM, Name: name, StoragePath: "_metadata/npm/" + name, + FetchedAt: sql.NullTime{Time: cutoff.Add(-time.Minute), Valid: true}, + }) + if err != nil { + t.Fatal(err) + } + } + expired, err := db.ExpireMetadataCache(testEcosystemNPM, names, cutoff) + if err != nil { + t.Fatal(err) + } + if len(expired) != 2 { + t.Errorf("expired = %v, want both stored names", expired) + } +} diff --git a/internal/database/queries.go b/internal/database/queries.go index d7150e8..d97ef25 100644 --- a/internal/database/queries.go +++ b/internal/database/queries.go @@ -4,7 +4,9 @@ import ( "database/sql" "errors" "fmt" + "slices" "sort" + "strings" "time" "github.com/git-pkgs/artifacts" @@ -1357,6 +1359,50 @@ func (db *DB) ForgetMetadataDocument(ecosystem, name, from, to string) error { return err } +// expireMetadataBatch is how many names one ExpireMetadataCache statement +// covers, well below the bind parameter limits of SQLite and Postgres. +const expireMetadataBatch = 500 + +// ExpireMetadataCache clears fetched_at on the metadata cache rows for +// ecosystem and names that were fetched before cutoff, and returns the names +// of the rows it changed. The rows keep their documents and validators, so +// the next lookup of each revalidates it upstream instead of serving it as +// fresh. It is for an upstream that reports which documents changed. +func (db *DB) ExpireMetadataCache(ecosystem string, names []string, cutoff time.Time) ([]string, error) { + var expired []string + for batch := range slices.Chunk(names, expireMetadataBatch) { + args := []any{time.Now(), ecosystem, cutoff.UTC()} + for _, name := range batch { + args = append(args, name) + } + var got []string + err := db.DB.Select(&got, db.Rebind(` + UPDATE metadata_cache SET fetched_at = NULL, updated_at = ? + WHERE ecosystem = ? AND fetched_at < ? + AND name IN (`+strings.TrimSuffix(strings.Repeat("?, ", len(batch)), ", ")+`) + RETURNING name + `), args...) + if err != nil { + return expired, err + } + expired = append(expired, got...) + } + return expired, nil +} + +// ExpireAllMetadataCache is ExpireMetadataCache for every row of ecosystem, +// except those of packages routed to another registry (named "routes/..."), +// for an upstream that can't say which of its documents changed. +func (db *DB) ExpireAllMetadataCache(ecosystem string, cutoff time.Time) ([]string, error) { + var expired []string + err := db.DB.Select(&expired, db.Rebind(` + UPDATE metadata_cache SET fetched_at = NULL, updated_at = ? + WHERE ecosystem = ? AND fetched_at < ? AND name NOT LIKE 'routes/%' + RETURNING name + `), time.Now(), ecosystem, cutoff.UTC()) + return expired, err +} + // UpsertMetadataCache writes entry and marks it accessed now, since a // document is stored because a request asked for it. func (db *DB) UpsertMetadataCache(entry *MetadataCacheEntry) error { diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go index 708b5b3..984a607 100644 --- a/internal/metrics/metrics.go +++ b/internal/metrics/metrics.go @@ -86,6 +86,14 @@ var ( []string{"ecosystem"}, ) + MetadataExpiredByChangeFeed = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "proxy_metadata_expired_by_change_feed_total", + Help: "Cached metadata documents set to revalidate because an upstream change feed listed them, by ecosystem", + }, + []string{"ecosystem"}, + ) + FetchLeaseWaits = prometheus.NewCounterVec( prometheus.CounterOpts{ Name: "proxy_fetch_lease_waits_total", @@ -311,6 +319,7 @@ func init() { CachedArtifacts, ArtifactsEvicted, MetadataEvicted, + MetadataExpiredByChangeFeed, FetchLeaseWaits, UpstreamFetchDuration, UpstreamErrors, @@ -409,6 +418,13 @@ func RecordMetadataEvicted(ecosystem string) { MetadataEvicted.WithLabelValues(purl.NormalizeEcosystem(ecosystem)).Inc() } +// RecordMetadataExpiredByChangeFeed counts n metadata documents an upstream +// change feed listed as changed. ecosystem is the metadata cache's, +// normalized like the cache hit counter. +func RecordMetadataExpiredByChangeFeed(ecosystem string, n int) { + MetadataExpiredByChangeFeed.WithLabelValues(purl.NormalizeEcosystem(ecosystem)).Add(float64(n)) +} + // RecordFetchLeaseWait counts a cache miss that found another instance // sharing the database already fetching it. func RecordFetchLeaseWait(kind string) { diff --git a/internal/server/analytics_coverage_test.go b/internal/server/analytics_coverage_test.go index 82e5ca2..02e67e2 100644 --- a/internal/server/analytics_coverage_test.go +++ b/internal/server/analytics_coverage_test.go @@ -34,25 +34,26 @@ var metricSurface = map[string]string{ "proxy_client_response_bytes_total": "Runtime: Request sources -- By client", // Registry-derived, shown in the Runtime card. - "proxy_requests_total": "Runtime: Requests + Responses by status", - "proxy_request_duration_seconds": "Runtime: Mean latency", - "proxy_active_requests": "Runtime: In flight", - "proxy_cache_hits_total": "Runtime: Artifact and metadata cache lookups + artifact hit rate", - "proxy_cache_misses_total": "Runtime: Artifact and metadata cache lookups + artifact hit rate", - "proxy_metadata_cache_served_bytes_total": "Runtime: Metadata cache lookups -- served from cache", - "proxy_upstream_fetch_duration_seconds": "Runtime: Upstream fetches + mean fetch", - "proxy_upstream_errors_total": "Runtime: Upstream errors", - "proxy_storage_operation_duration_seconds": "Runtime: Storage operations", - "proxy_storage_errors_total": "Runtime: Storage errors", - "proxy_integrity_failures_total": "Runtime: Integrity failures", - "proxy_health_probe_failures_total": "Runtime: Health probe failures", - "proxy_circuit_breaker_state": "Runtime: Circuit breakers", - "proxy_circuit_breaker_trips_total": "Runtime: Circuit breaker trips", - "proxy_artifacts_evicted_total": "Runtime: Evicted artifacts", - "proxy_metadata_evicted_total": "Runtime: Evicted metadata", - "proxy_scan_duration_seconds": "Runtime: Pre-cache scanning -- Scans", - "proxy_scan_blocked_total": "Runtime: Pre-cache scanning -- Artifacts blocked", - "proxy_scan_errors_total": "Runtime: Pre-cache scanning -- Scan errors", + "proxy_requests_total": "Runtime: Requests + Responses by status", + "proxy_request_duration_seconds": "Runtime: Mean latency", + "proxy_active_requests": "Runtime: In flight", + "proxy_cache_hits_total": "Runtime: Artifact and metadata cache lookups + artifact hit rate", + "proxy_cache_misses_total": "Runtime: Artifact and metadata cache lookups + artifact hit rate", + "proxy_metadata_cache_served_bytes_total": "Runtime: Metadata cache lookups -- served from cache", + "proxy_upstream_fetch_duration_seconds": "Runtime: Upstream fetches + mean fetch", + "proxy_upstream_errors_total": "Runtime: Upstream errors", + "proxy_storage_operation_duration_seconds": "Runtime: Storage operations", + "proxy_storage_errors_total": "Runtime: Storage errors", + "proxy_integrity_failures_total": "Runtime: Integrity failures", + "proxy_health_probe_failures_total": "Runtime: Health probe failures", + "proxy_circuit_breaker_state": "Runtime: Circuit breakers", + "proxy_circuit_breaker_trips_total": "Runtime: Circuit breaker trips", + "proxy_artifacts_evicted_total": "Runtime: Evicted artifacts", + "proxy_metadata_evicted_total": "Runtime: Evicted metadata", + "proxy_metadata_expired_by_change_feed_total": "Runtime: Metadata changed upstream", + "proxy_scan_duration_seconds": "Runtime: Pre-cache scanning -- Scans", + "proxy_scan_blocked_total": "Runtime: Pre-cache scanning -- Artifacts blocked", + "proxy_scan_errors_total": "Runtime: Pre-cache scanning -- Scan errors", } // TestEveryMetricIsSurfaced fails when a metric is registered but has no home @@ -80,6 +81,7 @@ func TestEveryMetricIsSurfaced(t *testing.T) { metrics.UpdateCacheStats(1, 1) metrics.RecordArtifactEvicted("retention", "npm") metrics.RecordMetadataEvicted("npm") + metrics.RecordMetadataExpiredByChangeFeed("npm", 1) metrics.UpdateEcosystemStats([]metrics.EcosystemStats{{Ecosystem: "npm", DownloadedBytes: 1}}) metrics.UpdateMetadataCacheStats([]metrics.MetadataCacheStats{{Ecosystem: "npm", Entries: 1, Size: 1}}) diff --git a/internal/server/analytics_runtime.go b/internal/server/analytics_runtime.go index 5773879..103e33e 100644 --- a/internal/server/analytics_runtime.go +++ b/internal/server/analytics_runtime.go @@ -35,6 +35,7 @@ type RuntimeView struct { ProbeFailures []LabelledCount Evictions []LabelledCount MetadataEvictions []LabelledCount + ChangeFeedExpired []LabelledCount Breakers []BreakerRow BreakerTrips []LabelledCount @@ -161,6 +162,7 @@ func runtimeView(snap *metrics.Snapshot, failureDetails bool) RuntimeView { v.ProbeFailures = failures("proxy_health_probe_failures_total", "step") v.Evictions = countRows(snap, "proxy_artifacts_evicted_total", "reason", "ecosystem") v.MetadataEvictions = countRows(snap, "proxy_metadata_evicted_total", "ecosystem") + v.ChangeFeedExpired = countRows(snap, "proxy_metadata_expired_by_change_feed_total", "ecosystem") v.Breakers = breakerRows(snap) v.BreakerTrips = failures("proxy_circuit_breaker_trips_total", "registry") diff --git a/internal/server/analytics_runtime_test.go b/internal/server/analytics_runtime_test.go index 66c154e..03bc236 100644 --- a/internal/server/analytics_runtime_test.go +++ b/internal/server/analytics_runtime_test.go @@ -170,6 +170,24 @@ func TestRuntimeViewShowsMetadataEvictions(t *testing.T) { } } +func TestRuntimeViewShowsChangeFeedExpiries(t *testing.T) { + reg := prometheus.NewRegistry() + expired := prometheus.NewCounterVec( + prometheus.CounterOpts{Name: "proxy_metadata_expired_by_change_feed_total", Help: "t"}, []string{"ecosystem"}) + reg.MustRegister(expired) + expired.WithLabelValues("packagist").Add(2) + + snap, err := metrics.GatherFrom(reg) + if err != nil { + t.Fatalf("GatherFrom: %v", err) + } + v := runtimeView(snap, false) + + if len(v.ChangeFeedExpired) != 1 || v.ChangeFeedExpired[0].Label != "packagist" || v.ChangeFeedExpired[0].Count != "2" { + t.Errorf("ChangeFeedExpired = %+v, want packagist = 2", v.ChangeFeedExpired) + } +} + // A counter sitting at zero carries no information and would bury the rows // that do, so it is left out. func TestRuntimeViewDropsZeroCounters(t *testing.T) { diff --git a/internal/server/change_feeds.go b/internal/server/change_feeds.go new file mode 100644 index 0000000..059bd2f --- /dev/null +++ b/internal/server/change_feeds.go @@ -0,0 +1,94 @@ +package server + +import ( + "context" + "log/slog" + "net/http" + "time" + + "github.com/git-pkgs/proxy/internal/changefeed" + "github.com/git-pkgs/proxy/internal/database" + "github.com/git-pkgs/proxy/internal/metrics" +) + +// changeFeedInterval is how often an upstream change feed is read. +const changeFeedInterval = time.Minute + +// changeFeedMaxBackoff caps the wait between reads of a failing feed. +const changeFeedMaxBackoff = 15 * time.Minute + +// startChangeFeeds follows the change feeds the configuration turns on. Each +// instance sharing a database follows them itself: the reads are small, and +// expiring a row twice does no harm. +func (s *Server) startChangeFeeds(ctx context.Context, client *http.Client, metadataTTL time.Duration) { + var feeds []changefeed.Feed + if s.cfg.Upstream.NPMChangeFeed { + feeds = append(feeds, changefeed.NewNPMFeed(client, changefeed.NPMFeedURL)) + } + if s.cfg.Upstream.ComposerChangeFeed { + feeds = append(feeds, changefeed.NewComposerFeed(client, changefeed.ComposerFeedURL(s.cfg.Upstream.Composer))) + } + if len(feeds) == 0 { + return + } + if !s.cfg.CacheMetadata || metadataTTL <= 0 { + // Every request already revalidates, so there is nothing to expire. + s.logger.Info("change feeds not followed: metadata is not served from cache without revalidating") + return + } + for _, feed := range feeds { + s.background.Go(func() { followChangeFeed(ctx, s.db, s.logger, feed, changeFeedInterval) }) + } +} + +// followChangeFeed reads feed every interval until ctx ends and expires the +// cached metadata it lists. After a failed read it waits twice as long as +// before, up to changeFeedMaxBackoff. +func followChangeFeed(ctx context.Context, db *database.DB, logger *slog.Logger, feed changefeed.Feed, interval time.Duration) { + wait := interval + for { + if err := readChangeFeed(ctx, db, logger, feed); err != nil { + if ctx.Err() != nil { + return + } + wait = min(wait+wait, changeFeedMaxBackoff) + logger.Warn("change feed: read failed", "ecosystem", feed.Ecosystem(), "error", err, "retry_in", wait) + } else { + wait = interval + } + timer := time.NewTimer(wait) + select { + case <-ctx.Done(): + timer.Stop() + return + case <-timer.C: + } + } +} + +// readChangeFeed reads feed once and expires the cached metadata of what it +// lists, so the next request for it revalidates upstream. Only rows fetched +// before the read finished are expired: a row fetched after it was fetched +// after every change the read returned. If expiring fails, the feed has +// already moved past the changes, and those rows wait out metadata_ttl. +func readChangeFeed(ctx context.Context, db *database.DB, logger *slog.Logger, feed changefeed.Feed) error { + changes, err := feed.Changes(ctx) + if err != nil { + return err + } + cutoff := time.Now() + ecosystem := feed.Ecosystem() + var expired []string + switch { + case changes.All: + logger.Info("change feed: upstream asked for a full resync, revalidating all cached metadata", "ecosystem", ecosystem) + expired, err = db.ExpireAllMetadataCache(ecosystem, cutoff) + case len(changes.Names) > 0: + expired, err = db.ExpireMetadataCache(ecosystem, changes.Names, cutoff) + } + for _, name := range expired { + logger.Debug("change feed: cached metadata changed upstream", "ecosystem", ecosystem, "key", name) + } + metrics.RecordMetadataExpiredByChangeFeed(ecosystem, len(expired)) + return err +} diff --git a/internal/server/change_feeds_test.go b/internal/server/change_feeds_test.go new file mode 100644 index 0000000..90441ac --- /dev/null +++ b/internal/server/change_feeds_test.go @@ -0,0 +1,272 @@ +package server + +import ( + "context" + "crypto/sha256" + "errors" + "fmt" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/git-pkgs/proxy/internal/changefeed" + "github.com/git-pkgs/proxy/internal/config" + "github.com/git-pkgs/proxy/internal/database" + "github.com/git-pkgs/proxy/internal/handler" + "github.com/git-pkgs/proxy/internal/storage" + "github.com/git-pkgs/registries/fetch" + "github.com/go-chi/chi/v5" +) + +// changingRegistry serves npm packuments, Composer metadata, and both +// registries' change feeds, all from one set of documents that a test can +// change. The feeds list every document changed since the previous read. +type changingRegistry struct { + *httptest.Server + mu sync.Mutex + docs map[string]string // path -> body + changed []string // paths changed since the feed was last read + feedSeq int + fetches map[string]int +} + +func newChangingRegistry(t *testing.T) *changingRegistry { + t.Helper() + reg := &changingRegistry{docs: map[string]string{}, fetches: map[string]int{}} + reg.Server = httptest.NewServer(http.HandlerFunc(reg.serve)) + t.Cleanup(reg.Close) + return reg +} + +func (reg *changingRegistry) set(path, body string) { + reg.mu.Lock() + defer reg.mu.Unlock() + if _, ok := reg.docs[path]; ok { + reg.changed = append(reg.changed, path) + } + reg.docs[path] = body +} + +func (reg *changingRegistry) fetched(path string) int { + reg.mu.Lock() + defer reg.mu.Unlock() + return reg.fetches[path] +} + +// takeChanged returns the changed paths with prefix and forgets them. +func (reg *changingRegistry) takeChanged(prefix string) []string { + var taken, kept []string + for _, p := range reg.changed { + if name, ok := strings.CutPrefix(p, prefix); ok { + taken = append(taken, name) + } else { + kept = append(kept, p) + } + } + reg.changed = kept + return taken +} + +func (reg *changingRegistry) serve(w http.ResponseWriter, r *http.Request) { + reg.mu.Lock() + defer reg.mu.Unlock() + w.Header().Set("Content-Type", "application/json") + switch r.URL.Path { + case "/metadata/changes.json": + if r.URL.Query().Get("since") == "" { + w.WriteHeader(http.StatusBadRequest) + _, _ = io.WriteString(w, `{"error":"missing since","timestamp":1}`) + return + } + var actions []string + for _, name := range reg.takeChanged("/p2/") { + actions = append(actions, fmt.Sprintf(`{"type":"update","package":%q,"time":1}`, strings.TrimSuffix(name, ".json"))) + } + _, _ = fmt.Fprintf(w, `{"actions":[%s],"timestamp":2}`, strings.Join(actions, ",")) + return + case "/replicate/registry/": + _, _ = fmt.Fprintf(w, `{"update_seq":%d}`, reg.feedSeq) + return + case "/replicate/registry/_changes": + var results []string + for _, name := range reg.takeChanged("/npm/") { + reg.feedSeq++ + results = append(results, fmt.Sprintf(`{"seq":%d,"id":%q}`, reg.feedSeq, name)) + } + _, _ = fmt.Fprintf(w, `{"results":[%s],"last_seq":%d}`, strings.Join(results, ","), reg.feedSeq) + return + } + reg.fetches[r.URL.Path]++ + body, ok := reg.docs[r.URL.Path] + if !ok { + http.NotFound(w, r) + return + } + etag := fmt.Sprintf(`"%x"`, sha256.Sum256([]byte(body))) + if r.Header.Get("If-None-Match") == etag { + w.WriteHeader(http.StatusNotModified) + return + } + w.Header().Set("ETag", etag) + _, _ = io.WriteString(w, body) +} + +func composerDoc(versions ...string) string { + var entries []string + for _, v := range versions { + entries = append(entries, fmt.Sprintf(`{"name":"acme/lib","version":%q,"version_normalized":"%s.0","dist":{"type":"zip","url":"https://example.test/acme/lib/%s.zip","reference":"r%s"}}`, v, v, v, v)) + } + return `{"packages":{"acme/lib":[` + strings.Join(entries, ",") + `]},"minified":"composer/2.0"}` +} + +func npmDoc(versions ...string) string { + var entries []string + for _, v := range versions { + entries = append(entries, fmt.Sprintf(`%q:{"name":"left-pad","version":%q,"dist":{"tarball":"https://registry.example.test/left-pad/-/left-pad-%s.tgz"}}`, v, v, v)) + } + return `{"name":"left-pad","dist-tags":{"latest":"` + versions[len(versions)-1] + `"},"versions":{` + strings.Join(entries, ",") + `}}` +} + +// changeFeedRouter mounts the npm and Composer handlers on reg, with +// metadata cached for an hour. +func changeFeedRouter(t *testing.T, reg *changingRegistry) (http.Handler, *database.DB) { + t.Helper() + db, err := database.Create(t.TempDir() + "/test.db") + if err != nil { + t.Fatalf("creating database: %v", err) + } + t.Cleanup(func() { _ = db.Close() }) + store, err := storage.OpenBucket(context.Background(), "file://"+t.TempDir()) + if err != nil { + t.Fatalf("opening storage: %v", err) + } + fetcher := fetch.NewFetcher(fetch.WithHTTPClient(reg.Client()), fetch.WithMaxRetries(0)) + t.Cleanup(func() { _ = fetcher.Close() }) + proxy := handler.NewProxy(db, store, fetcher, fetch.NewResolver(), slog.New(slog.NewTextHandler(io.Discard, nil))) + proxy.HTTPClient = reg.Client() + proxy.CacheMetadata = true + proxy.MetadataTTL = time.Hour + proxy.SetMetadataRewriteCacheSize(1 << 20) + + cfg := config.Default() + cfg.BaseURL = "http://proxy.example" + cfg.Upstream.NPM = reg.URL + "/npm" + cfg.Upstream.Composer = reg.URL + cfg.Upstream.ComposerRepository = reg.URL + s := &Server{cfg: cfg} + r := chi.NewRouter() + s.mountProtocolHandlers(r, proxy) + return r, db +} + +func getBody(t *testing.T, h http.Handler, path string) string { + t.Helper() + w := httptest.NewRecorder() + h.ServeHTTP(w, httptest.NewRequest(http.MethodGet, path, nil)) + if w.Code != http.StatusOK { + t.Fatalf("GET %s = %d: %s", path, w.Code, w.Body.String()) + } + return w.Body.String() +} + +// TestChangeFeedRevalidatesChangedMetadata checks the whole path: cached +// metadata is served within its TTL even after upstream changed, until the +// change feed lists the package, after which the next request revalidates +// and serves the new release. Packages the feed doesn't list stay cached. +func TestChangeFeedRevalidatesChangedMetadata(t *testing.T) { + reg := newChangingRegistry(t) + reg.set("/p2/acme/lib.json", composerDoc("1.0.0")) + reg.set("/p2/acme/other.json", `{"packages":{"acme/other":[]},"minified":"composer/2.0"}`) + reg.set("/npm/left-pad", npmDoc("1.0.0")) + h, db := changeFeedRouter(t, reg) + logger := slog.New(slog.NewTextHandler(io.Discard, nil)) + feeds := []changefeed.Feed{ + changefeed.NewComposerFeed(reg.Client(), changefeed.ComposerFeedURL(reg.URL)), + changefeed.NewNPMFeed(reg.Client(), reg.URL+"/replicate/registry/_changes"), + } + for _, feed := range feeds { + if err := readChangeFeed(context.Background(), db, logger, feed); err != nil { + t.Fatalf("starting %s feed: %v", feed.Ecosystem(), err) + } + } + + getBody(t, h, "/composer/p2/acme/lib.json") + getBody(t, h, "/composer/p2/acme/other.json") + getBody(t, h, "/npm/left-pad") + + reg.set("/p2/acme/lib.json", composerDoc("1.0.0", "1.1.0")) + reg.set("/npm/left-pad", npmDoc("1.0.0", "1.1.0")) + if body := getBody(t, h, "/composer/p2/acme/lib.json"); strings.Contains(body, "1.1.0") { + t.Fatal("composer metadata revalidated before the feed listed it; the test proves nothing") + } + if body := getBody(t, h, "/npm/left-pad"); strings.Contains(body, "1.1.0") { + t.Fatal("npm packument revalidated before the feed listed it; the test proves nothing") + } + + for _, feed := range feeds { + if err := readChangeFeed(context.Background(), db, logger, feed); err != nil { + t.Fatalf("reading %s feed: %v", feed.Ecosystem(), err) + } + } + + if body := getBody(t, h, "/composer/p2/acme/lib.json"); !strings.Contains(body, "1.1.0") { + t.Errorf("composer metadata after the feed listed it = %s, want 1.1.0", body) + } + if body := getBody(t, h, "/npm/left-pad"); !strings.Contains(body, "1.1.0") { + t.Errorf("npm packument after the feed listed it = %s, want 1.1.0", body) + } + getBody(t, h, "/composer/p2/acme/other.json") + if n := reg.fetched("/p2/acme/other.json"); n != 1 { + t.Errorf("unlisted package fetched %d times, want once", n) + } + if n := reg.fetched("/p2/acme/lib.json"); n != 2 { + t.Errorf("listed package fetched %d times, want twice", n) + } +} + +// countingFeed fails its first reads and then reports no changes. +type countingFeed struct { + reads atomic.Int32 + failures int32 +} + +func (f *countingFeed) Ecosystem() string { return "npm" } + +func (f *countingFeed) Changes(context.Context) (changefeed.Changes, error) { + if f.reads.Add(1) <= f.failures { + return changefeed.Changes{}, errors.New("feed down") + } + return changefeed.Changes{}, nil +} + +// TestFollowChangeFeedStopsWithContext checks that a failing feed is retried +// rather than given up on, and that following stops when ctx ends. +func TestFollowChangeFeedStopsWithContext(t *testing.T) { + feed := &countingFeed{failures: 2} + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + go func() { + defer close(done) + followChangeFeed(ctx, nil, slog.New(slog.NewTextHandler(io.Discard, nil)), feed, time.Millisecond) + }() + deadline := time.After(5 * time.Second) + for feed.reads.Load() < 4 { + select { + case <-deadline: + t.Fatalf("feed read %d times, want it read again after failing", feed.reads.Load()) + case <-time.After(time.Millisecond): + } + } + cancel() + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatal("followChangeFeed did not return after ctx ended") + } +} diff --git a/internal/server/server.go b/internal/server/server.go index 12619a6..8d3a746 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -397,6 +397,7 @@ func (s *Server) serve(listener net.Listener) error { }) } s.startMirrorSchedules(bgCtx, proxy) + s.startChangeFeeds(bgCtx, proxy.HTTPClient, proxy.MetadataTTL) // Hosted package endpoints (opt-in via hosted.enabled or PROXY_HOSTED_ENABLED) if s.cfg.Hosted.Enabled { diff --git a/internal/server/server_test.go b/internal/server/server_test.go index 292643c..06f6b23 100644 --- a/internal/server/server_test.go +++ b/internal/server/server_test.go @@ -633,13 +633,6 @@ func TestSwiftCachedVersionPURLUsesStoredPackagePURL(t *testing.T) { } } -func min(a, b int) int { - if a < b { - return a - } - return b -} - func TestNPMPackageMetadata(t *testing.T) { ts := newTestServer(t) defer ts.close() diff --git a/internal/server/templates/components/runtime_metrics.html b/internal/server/templates/components/runtime_metrics.html index 18a3140..b5427b7 100644 --- a/internal/server/templates/components/runtime_metrics.html +++ b/internal/server/templates/components/runtime_metrics.html @@ -60,6 +60,7 @@ {{template "runtime_counts" dict "Title" "Health probe failures" "Rows" .ProbeFailures "Empty" "None"}} {{template "runtime_counts" dict "Title" "Evicted artifacts" "Rows" .Evictions "Empty" "None"}} {{template "runtime_counts" dict "Title" "Evicted metadata" "Rows" .MetadataEvictions "Empty" "None"}} + {{template "runtime_counts" dict "Title" "Metadata changed upstream" "Rows" .ChangeFeedExpired "Empty" "None"}} {{template "runtime_counts" dict "Title" "Circuit breaker trips" "Rows" .BreakerTrips "Empty" "None"}}
From 0bc0e903f872f5b49328811e6d8f30c20820dcd2 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Tue, 27 Oct 2026 19:30:34 +0100 Subject: [PATCH 466/633] Add a project inventory built from uploaded SBOMs Repositories upload an SBOM of their default branch, for example from syft in CI, to PUT /api/projects/{project}/sbom. The proxy records the package versions it lists, keyed by the same unversioned PURL the handlers record packages under, so the inventory joins with what the proxy has cached and the vulnerabilities it holds. That answers which repository uses which version of a package: the new /ui/projects pages list each project's package versions with outdated and vulnerability markers, package pages show the projects per version, the dashboard lists vulnerable versions in use, and GET /api/projects?purl= does the same for scripts. The feature is off by default. Uploads and deletes need projects.token; an upload replaces the project's components in one transaction. projects.max_age removes projects that stopped uploading. --- README.md | 12 + cmd/proxy/main.go | 5 +- config.example.yaml | 9 + docs/architecture.md | 4 + docs/configuration.md | 82 ++++ docs/swagger/docs.go | 288 +++++++++++++- docs/swagger/swagger.json | 288 +++++++++++++- go.mod | 2 +- internal/config/config.go | 10 + internal/config/projects.go | 53 +++ internal/config/projects_test.go | 69 ++++ internal/database/database_test.go | 2 +- internal/database/projects.go | 257 +++++++++++++ internal/database/projects_test.go | 228 +++++++++++ internal/database/schema.go | 76 ++++ internal/inventory/inventory.go | 185 +++++++++ internal/inventory/inventory_test.go | 133 +++++++ .../inventory/testdata/syft-cyclonedx.json | 1 + internal/server/dashboard.go | 6 + internal/server/layout.go | 3 + internal/server/projects_api.go | 335 +++++++++++++++++ internal/server/projects_api_test.go | 354 ++++++++++++++++++ internal/server/projects_prune.go | 43 +++ internal/server/projects_prune_test.go | 48 +++ internal/server/projects_ui.go | 287 ++++++++++++++ internal/server/projects_ui_test.go | 137 +++++++ internal/server/server.go | 14 + .../templates/components/severity_badge.html | 3 + internal/server/templates/layout/header.html | 1 + .../server/templates/pages/dashboard.html | 27 ++ .../server/templates/pages/package_show.html | 28 ++ .../server/templates/pages/project_show.html | 53 +++ .../server/templates/pages/projects_list.html | 33 ++ 33 files changed, 3071 insertions(+), 5 deletions(-) create mode 100644 internal/config/projects.go create mode 100644 internal/config/projects_test.go create mode 100644 internal/database/projects.go create mode 100644 internal/database/projects_test.go create mode 100644 internal/inventory/inventory.go create mode 100644 internal/inventory/inventory_test.go create mode 100644 internal/inventory/testdata/syft-cyclonedx.json create mode 100644 internal/server/projects_api.go create mode 100644 internal/server/projects_api_test.go create mode 100644 internal/server/projects_prune.go create mode 100644 internal/server/projects_prune_test.go create mode 100644 internal/server/projects_ui.go create mode 100644 internal/server/projects_ui_test.go create mode 100644 internal/server/templates/components/severity_badge.html create mode 100644 internal/server/templates/pages/project_show.html create mode 100644 internal/server/templates/pages/projects_list.html diff --git a/README.md b/README.md index 20c6d5d..61edf9d 100644 --- a/README.md +++ b/README.md @@ -1300,6 +1300,17 @@ Registered when `hosted.enabled` is set. Pushing and deleting require `Authoriza | `GET /api/hosted/{ecosystem}/{name}` | List the archives pushed for a package | | `DELETE /api/hosted/{ecosystem}/{name}/{version}` | Delete a pushed version | +### Projects API + +Registered when `projects.enabled` is set. Uploading and deleting require `Authorization: Bearer `. See [Project inventory](docs/configuration.md#project-inventory). + +| Endpoint | Description | +|----------|-------------| +| `PUT /api/projects/{project}/sbom` | Replace a project's recorded package versions with the ones the SBOM in the body lists | +| `GET /api/projects` | List projects, or with `?purl=pkg:npm/lodash@4.17.20` the projects using a package or version | +| `GET /api/projects/{project}` | A project's recorded package versions | +| `DELETE /api/projects/{project}` | Delete a project | + ### Cache API Registered when `cache_delete` is set. It has no authentication of its own. @@ -1503,6 +1514,7 @@ The proxy serves a web UI under `/ui`. No separate frontend build is needed -- t - **Version detail** (`/ui/package/{ecosystem}/{name}/{version}`) -- per-version metadata, integrity hash, artifact cache status, and hit counts. - **Source browser** (`/ui/package/{ecosystem}/{name}/{version}/browse`) -- browse files inside cached archives with syntax highlighting for text files and image previews. - **Version diff** (`/ui/package/{ecosystem}/{name}/compare/{v1}...{v2}`) -- side-by-side diff of two cached versions showing added, removed, and changed files. +- **Projects** (`/ui/projects`) -- with the [project inventory](docs/configuration.md#project-inventory) enabled, the package versions each repository's uploaded SBOM lists, marked when outdated or vulnerable. Package pages then show which projects use each version, and the dashboard lists vulnerable versions in use. ## Monitoring diff --git a/cmd/proxy/main.go b/cmd/proxy/main.go index b19f689..728a579 100644 --- a/cmd/proxy/main.go +++ b/cmd/proxy/main.go @@ -8,7 +8,7 @@ // @securityDefinitions.apikey BearerAuth // @in header // @name Authorization -// @description "Bearer " followed by hosted.token +// @description "Bearer " followed by hosted.token, or projects.token for /api/projects // // The proxy caches package artifacts from upstream registries (npm, cargo, etc.) // providing faster, more reliable access for development teams. @@ -93,6 +93,9 @@ // PROXY_HOSTED_MAX_UPLOAD_SIZE - Max size of one pushed archive (default "100MB") // PROXY_HOSTED_NPM_SCOPES - npm scopes served from pushed packages only // PROXY_HOSTED_COMPOSER_VENDORS - Composer vendors served from pushed packages only +// PROXY_PROJECTS_ENABLED - Enable the project inventory (/api/projects, /ui/projects) +// PROXY_PROJECTS_TOKEN - Bearer token for uploading and deleting project SBOMs +// PROXY_PROJECTS_MAX_AGE - Remove projects not uploaded for this long (e.g. "90d") // PROXY_URL - Proxy URL for proxy hosted // // Example: diff --git a/config.example.yaml b/config.example.yaml index d3b6da4..a875322 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -442,6 +442,15 @@ gradle: # npm_scopes: ["@acme"] # composer_vendors: ["acme"] +# Project inventory (disabled by default). Repositories upload an SBOM to +# PUT /api/projects/{project}/sbom, and the UI shows which project uses +# which package versions. Project names are visible on the unauthenticated UI. +# projects: +# enabled: true +# token: "${PROXY_PROJECTS_TOKEN}" +# # Drop projects that have not uploaded for this long ("" keeps them) +# max_age: "90d" + # Re-mirror container image tags on a schedule, so a moved tag is picked up # even when nobody pulls it. An unchanged tag costs one HEAD request. # mirror: diff --git a/docs/architecture.md b/docs/architecture.md index 660d810..0105d3f 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -363,6 +363,10 @@ Selective package mirroring for pre-populating the proxy cache. Supports multipl The package also provides a `MetadataCache` for storing raw upstream metadata blobs so the proxy can serve metadata responses offline. The `JobStore` manages async mirror jobs exposed via the `/api/mirror` endpoints. +### `internal/inventory` + +Turns an SBOM uploaded to `/api/projects` into the package versions recorded for the project. Package URLs are mapped to the ecosystem names and unversioned PURLs the handlers record packages under, so a project's components join with `packages` by `purl`. Also compares versions for the outdated and vulnerability markers on the project pages. + ### `internal/config` Configuration loading. diff --git a/docs/configuration.md b/docs/configuration.md index f10e4b2..c5c6f7a 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1504,6 +1504,88 @@ Generic files never come from an upstream: anything not pushed answers 404. Whil Pushed archives are stored like cached ones but marked as local. The cache size limit and storage retention never remove them; only a delete does. Scanning, when enabled, checks pushed archives like fetched ones. +## Project inventory + +The project inventory answers "which repository uses which version of a package?". Each repository uploads an SBOM of its default branch, and the proxy records the package versions it lists. The UI then shows: + +- `/ui/projects`: every project, and per project its package versions, marked when the proxy knows a newer version or vulnerabilities for that version. +- On a package page: which projects use each version, including versions the proxy hasn't cached. +- On the dashboard: vulnerable versions in use and the projects using them. + +It is disabled by default. Uploads and deletes require a token: + +```yaml +projects: + enabled: true + token: "${PROXY_PROJECTS_TOKEN}" + max_age: "90d" +``` + +| Config | Environment | Description | +|--------|-------------|-------------| +| `projects.enabled` | `PROXY_PROJECTS_ENABLED` | Register the `/api/projects` endpoints and the UI pages (default `false`) | +| `projects.token` | `PROXY_PROJECTS_TOKEN` | Bearer token required to upload and delete; required when enabled, supports `${VAR}` | +| `projects.max_age` | `PROXY_PROJECTS_MAX_AGE` | Remove projects that haven't uploaded for this long, e.g. `90d` or `720h`; empty or `0` keeps them (default) | + +Removing old projects runs hourly. With several instances on one database, one of them does it. + +### Uploading an SBOM + +`PUT /api/projects/{project}/sbom` takes a CycloneDX (JSON or XML) or SPDX (JSON or tag-value) document of up to 32 MiB and replaces everything recorded for the project with what it lists. The optional `ref` and `branch` query parameters are stored and shown with the project. + +The project name is a path such as `group/sub/repo`: segments of letters, digits, `.`, `_` and `-`, at most 255 characters. It can be sent as path segments (`/api/projects/group/sub/repo/sbom`) or URL-encoded as one segment (`/api/projects/group%2Fsub%2Frepo/sbom`). + +```bash +syft dir:. -o cyclonedx-json > sbom.json +curl -X PUT --data-binary @sbom.json \ + -H "Authorization: Bearer $PROXY_PROJECTS_TOKEN" \ + "https://proxy.example.com/api/projects/acme/shop/web/sbom?ref=$(git rev-parse HEAD)&branch=main" +``` + +The response counts the recorded package versions and the skipped entries: + +```json +{"project":"acme/shop/web","components":10,"skipped":5} +``` + +Entries are recorded by their package URL. Entries without one (files, the operating system) and package types the proxy doesn't serve (such as `pkg:github`) are skipped. Duplicates are recorded once. + +A GitLab CI job on the default branch could look like this: + +```yaml +sbom: + image: alpine:3.20 + rules: + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + script: + - apk add --no-cache curl + - curl -sSfL https://raw.githubusercontent.com/anchore/syft/main/install.sh | sh -s -- -b /usr/local/bin + - syft dir:. -o cyclonedx-json > sbom.json + - > + curl -sSf -X PUT --data-binary @sbom.json + -H "Authorization: Bearer $PROXY_PROJECTS_TOKEN" + "https://proxy.example.com/api/projects/$CI_PROJECT_PATH/sbom?ref=$CI_COMMIT_SHA&branch=$CI_COMMIT_BRANCH" +``` + +### Querying + +```bash +curl https://proxy.example.com/api/projects # all projects +curl https://proxy.example.com/api/projects/acme/shop/web # one project's package versions +curl 'https://proxy.example.com/api/projects?purl=pkg:npm/lodash' # projects using lodash +curl 'https://proxy.example.com/api/projects?purl=pkg:npm/lodash@4.17.20' # projects using lodash 4.17.20 +curl -X DELETE -H "Authorization: Bearer $PROXY_PROJECTS_TOKEN" \ + https://proxy.example.com/api/projects/acme/shop/web +``` + +Reading needs no token, like the rest of `/api`. + +### What it shows + +The inventory reflects what the projects' SBOMs list, not what was downloaded through the proxy: a project that installs from another registry still shows up, and the proxy may not have cached any of its packages. Outdated and vulnerability markers use what the proxy knows about a package: its recorded latest version and the vulnerabilities recorded for it, matched against the version a project uses by the vulnerability's fixed version. A package the proxy has no record of has neither. + +Project names and the packages they use are shown on `/ui` and `/api/projects` without authentication. If that is sensitive, restrict both at your ingress or reverse proxy. + ## Mirror API The `/api/mirror` endpoints are disabled by default. Enable them to allow starting mirror jobs via HTTP: diff --git a/docs/swagger/docs.go b/docs/swagger/docs.go index e35a86f..7ddf630 100644 --- a/docs/swagger/docs.go +++ b/docs/swagger/docs.go @@ -460,6 +460,192 @@ const docTemplate = `{ } } }, + "/api/projects": { + "get": { + "description": "Lists the projects that uploaded an SBOM. With purl, lists the projects using that package, or only that version of it when the purl has one; each entry then carries the version the project uses.", + "produces": [ + "application/json" + ], + "tags": [ + "projects" + ], + "summary": "List projects", + "parameters": [ + { + "type": "string", + "description": "Package URL, e.g. pkg:npm/lodash or pkg:npm/lodash@4.17.20", + "name": "purl", + "in": "query" + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/server.ProjectListResponse" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/server.ErrorResponse" + } + } + } + } + }, + "/api/projects/{project}": { + "get": { + "description": "Returns the project with the package versions recorded for it.", + "produces": [ + "application/json" + ], + "tags": [ + "projects" + ], + "summary": "Get a project", + "parameters": [ + { + "type": "string", + "description": "Project path, e.g. group/sub/repo", + "name": "project", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/server.ProjectResponse" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/server.ErrorResponse" + } + }, + "404": { + "description": "Not Found", + "schema": { + "$ref": "#/definitions/server.ErrorResponse" + } + } + } + }, + "delete": { + "security": [ + { + "BearerAuth": [] + } + ], + "description": "Removes the project and the package versions recorded for it.", + "tags": [ + "projects" + ], + "summary": "Delete a project", + "parameters": [ + { + "type": "string", + "description": "Project path, e.g. group/sub/repo", + "name": "project", + "in": "path", + "required": true + } + ], + "responses": { + "204": { + "description": "No Content" + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/server.ErrorResponse" + } + }, + "401": { + "description": "Unauthorized", + "schema": { + "$ref": "#/definitions/server.ErrorResponse" + } + }, + "404": { + "description": "Not Found", + "schema": { + "$ref": "#/definitions/server.ErrorResponse" + } + } + } + } + }, + "/api/projects/{project}/sbom": { + "put": { + "security": [ + { + "BearerAuth": [] + } + ], + "description": "Replaces the package versions recorded for the project with the ones the SBOM lists. CycloneDX (JSON or XML) and SPDX (JSON or tag-value) are accepted, e.g. the output of ` + "`" + `syft dir:. -o cyclonedx-json` + "`" + `. Entries without a package URL, or of a package type the proxy doesn't serve, are skipped. The project may be given as path segments or URL-encoded as one segment.", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "projects" + ], + "summary": "Upload a project's SBOM", + "parameters": [ + { + "type": "string", + "description": "Project path, e.g. group/sub/repo", + "name": "project", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "Commit the SBOM was generated from", + "name": "ref", + "in": "query" + }, + { + "type": "string", + "description": "Branch the SBOM was generated from", + "name": "branch", + "in": "query" + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/server.ProjectUploadResponse" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/server.ErrorResponse" + } + }, + "401": { + "description": "Unauthorized", + "schema": { + "$ref": "#/definitions/server.ErrorResponse" + } + }, + "413": { + "description": "Request Entity Too Large", + "schema": { + "$ref": "#/definitions/server.ErrorResponse" + } + } + } + } + }, "/api/search": { "get": { "produces": [ @@ -1120,6 +1306,106 @@ const docTemplate = `{ } } }, + "server.ProjectComponent": { + "type": "object", + "properties": { + "ecosystem": { + "type": "string" + }, + "name": { + "type": "string" + }, + "purl": { + "type": "string" + }, + "version": { + "type": "string" + } + } + }, + "server.ProjectListResponse": { + "type": "object", + "properties": { + "projects": { + "type": "array", + "items": { + "$ref": "#/definitions/server.ProjectSummary" + } + } + } + }, + "server.ProjectResponse": { + "type": "object", + "properties": { + "branch": { + "type": "string" + }, + "component_count": { + "description": "ComponentCount is the number of package versions recorded.", + "type": "integer" + }, + "components": { + "type": "array", + "items": { + "$ref": "#/definitions/server.ProjectComponent" + } + }, + "name": { + "type": "string" + }, + "ref": { + "type": "string" + }, + "uploaded_at": { + "type": "string" + }, + "version": { + "description": "Version is set when listing the projects using a package: the version\nof it this project uses.", + "type": "string" + } + } + }, + "server.ProjectSummary": { + "type": "object", + "properties": { + "branch": { + "type": "string" + }, + "component_count": { + "description": "ComponentCount is the number of package versions recorded.", + "type": "integer" + }, + "name": { + "type": "string" + }, + "ref": { + "type": "string" + }, + "uploaded_at": { + "type": "string" + }, + "version": { + "description": "Version is set when listing the projects using a package: the version\nof it this project uses.", + "type": "string" + } + } + }, + "server.ProjectUploadResponse": { + "type": "object", + "properties": { + "components": { + "description": "Components is the number of package versions recorded.", + "type": "integer" + }, + "project": { + "type": "string" + }, + "skipped": { + "description": "Skipped is the number of SBOM entries without a package URL or of a\npackage type the proxy doesn't serve.", + "type": "integer" + } + } + }, "server.SearchPackageResult": { "type": "object", "properties": { @@ -1206,7 +1492,7 @@ const docTemplate = `{ }, "securityDefinitions": { "BearerAuth": { - "description": "\"Bearer \" followed by hosted.token", + "description": "\"Bearer \" followed by hosted.token, or projects.token for /api/projects", "type": "apiKey", "name": "Authorization", "in": "header" diff --git a/docs/swagger/swagger.json b/docs/swagger/swagger.json index 09e4f11..eb6f3b9 100644 --- a/docs/swagger/swagger.json +++ b/docs/swagger/swagger.json @@ -453,6 +453,192 @@ } } }, + "/api/projects": { + "get": { + "description": "Lists the projects that uploaded an SBOM. With purl, lists the projects using that package, or only that version of it when the purl has one; each entry then carries the version the project uses.", + "produces": [ + "application/json" + ], + "tags": [ + "projects" + ], + "summary": "List projects", + "parameters": [ + { + "type": "string", + "description": "Package URL, e.g. pkg:npm/lodash or pkg:npm/lodash@4.17.20", + "name": "purl", + "in": "query" + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/server.ProjectListResponse" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/server.ErrorResponse" + } + } + } + } + }, + "/api/projects/{project}": { + "get": { + "description": "Returns the project with the package versions recorded for it.", + "produces": [ + "application/json" + ], + "tags": [ + "projects" + ], + "summary": "Get a project", + "parameters": [ + { + "type": "string", + "description": "Project path, e.g. group/sub/repo", + "name": "project", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/server.ProjectResponse" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/server.ErrorResponse" + } + }, + "404": { + "description": "Not Found", + "schema": { + "$ref": "#/definitions/server.ErrorResponse" + } + } + } + }, + "delete": { + "security": [ + { + "BearerAuth": [] + } + ], + "description": "Removes the project and the package versions recorded for it.", + "tags": [ + "projects" + ], + "summary": "Delete a project", + "parameters": [ + { + "type": "string", + "description": "Project path, e.g. group/sub/repo", + "name": "project", + "in": "path", + "required": true + } + ], + "responses": { + "204": { + "description": "No Content" + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/server.ErrorResponse" + } + }, + "401": { + "description": "Unauthorized", + "schema": { + "$ref": "#/definitions/server.ErrorResponse" + } + }, + "404": { + "description": "Not Found", + "schema": { + "$ref": "#/definitions/server.ErrorResponse" + } + } + } + } + }, + "/api/projects/{project}/sbom": { + "put": { + "security": [ + { + "BearerAuth": [] + } + ], + "description": "Replaces the package versions recorded for the project with the ones the SBOM lists. CycloneDX (JSON or XML) and SPDX (JSON or tag-value) are accepted, e.g. the output of `syft dir:. -o cyclonedx-json`. Entries without a package URL, or of a package type the proxy doesn't serve, are skipped. The project may be given as path segments or URL-encoded as one segment.", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "projects" + ], + "summary": "Upload a project's SBOM", + "parameters": [ + { + "type": "string", + "description": "Project path, e.g. group/sub/repo", + "name": "project", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "Commit the SBOM was generated from", + "name": "ref", + "in": "query" + }, + { + "type": "string", + "description": "Branch the SBOM was generated from", + "name": "branch", + "in": "query" + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/server.ProjectUploadResponse" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/server.ErrorResponse" + } + }, + "401": { + "description": "Unauthorized", + "schema": { + "$ref": "#/definitions/server.ErrorResponse" + } + }, + "413": { + "description": "Request Entity Too Large", + "schema": { + "$ref": "#/definitions/server.ErrorResponse" + } + } + } + } + }, "/api/search": { "get": { "produces": [ @@ -1113,6 +1299,106 @@ } } }, + "server.ProjectComponent": { + "type": "object", + "properties": { + "ecosystem": { + "type": "string" + }, + "name": { + "type": "string" + }, + "purl": { + "type": "string" + }, + "version": { + "type": "string" + } + } + }, + "server.ProjectListResponse": { + "type": "object", + "properties": { + "projects": { + "type": "array", + "items": { + "$ref": "#/definitions/server.ProjectSummary" + } + } + } + }, + "server.ProjectResponse": { + "type": "object", + "properties": { + "branch": { + "type": "string" + }, + "component_count": { + "description": "ComponentCount is the number of package versions recorded.", + "type": "integer" + }, + "components": { + "type": "array", + "items": { + "$ref": "#/definitions/server.ProjectComponent" + } + }, + "name": { + "type": "string" + }, + "ref": { + "type": "string" + }, + "uploaded_at": { + "type": "string" + }, + "version": { + "description": "Version is set when listing the projects using a package: the version\nof it this project uses.", + "type": "string" + } + } + }, + "server.ProjectSummary": { + "type": "object", + "properties": { + "branch": { + "type": "string" + }, + "component_count": { + "description": "ComponentCount is the number of package versions recorded.", + "type": "integer" + }, + "name": { + "type": "string" + }, + "ref": { + "type": "string" + }, + "uploaded_at": { + "type": "string" + }, + "version": { + "description": "Version is set when listing the projects using a package: the version\nof it this project uses.", + "type": "string" + } + } + }, + "server.ProjectUploadResponse": { + "type": "object", + "properties": { + "components": { + "description": "Components is the number of package versions recorded.", + "type": "integer" + }, + "project": { + "type": "string" + }, + "skipped": { + "description": "Skipped is the number of SBOM entries without a package URL or of a\npackage type the proxy doesn't serve.", + "type": "integer" + } + } + }, "server.SearchPackageResult": { "type": "object", "properties": { @@ -1199,7 +1485,7 @@ }, "securityDefinitions": { "BearerAuth": { - "description": "\"Bearer \" followed by hosted.token", + "description": "\"Bearer \" followed by hosted.token, or projects.token for /api/projects", "type": "apiKey", "name": "Authorization", "in": "header" diff --git a/go.mod b/go.mod index 94fbe6c..2213d68 100644 --- a/go.mod +++ b/go.mod @@ -20,6 +20,7 @@ require ( github.com/git-pkgs/registries v0.11.1 github.com/git-pkgs/sbom v0.1.7 github.com/git-pkgs/spdx v0.3.3 + github.com/git-pkgs/vers v0.7.2 github.com/go-chi/chi/v5 v5.5.0 github.com/jmoiron/sqlx v1.4.0 github.com/klauspost/compress v1.20.1 @@ -76,7 +77,6 @@ require ( github.com/ecosyste-ms/ecosystems-go v0.5.0 // indirect github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a // indirect github.com/git-pkgs/pom v0.1.8 // indirect - github.com/git-pkgs/vers v0.7.2 // indirect github.com/git-pkgs/vulns v0.2.4 // indirect github.com/github/go-spdx/v2 v2.7.0 // indirect github.com/go-logr/logr v1.4.4 // indirect diff --git a/internal/config/config.go b/internal/config/config.go index a660699..42a4b9e 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -211,6 +211,9 @@ type Config struct { // Hosted configures pushing packages to the proxy (/api/hosted). Hosted HostedConfig `json:"hosted" yaml:"hosted"` + // Projects configures the project inventory (/api/projects). + Projects ProjectsConfig `json:"projects" yaml:"projects"` + // Gradle configures Gradle HttpBuildCache behavior. Gradle GradleConfig `json:"gradle" yaml:"gradle"` @@ -1448,6 +1451,9 @@ func (c *Config) LoadFromEnv() { setEnvString(&c.Hosted.MaxUploadSize, "PROXY_HOSTED_MAX_UPLOAD_SIZE") setEnvStringSlice(&c.Hosted.NPMScopes, "PROXY_HOSTED_NPM_SCOPES") setEnvStringSlice(&c.Hosted.ComposerVendors, "PROXY_HOSTED_COMPOSER_VENDORS") + setEnvBool(&c.Projects.Enabled, "PROXY_PROJECTS_ENABLED") + setEnvString(&c.Projects.Token, "PROXY_PROJECTS_TOKEN") + setEnvString(&c.Projects.MaxAge, "PROXY_PROJECTS_MAX_AGE") setEnvString(&c.MetadataTTL, "PROXY_METADATA_TTL") setEnvString(&c.MetadataMaxSize, "PROXY_METADATA_MAX_SIZE") setEnvString(&c.ArtifactMaxSize, "PROXY_ARTIFACT_MAX_SIZE") @@ -1622,6 +1628,10 @@ func (c *Config) validateComponents() error { HostedGenericRepository, HostedGenericRepository) } + if err := c.Projects.Validate(); err != nil { + return err + } + if err := c.Mirror.Validate(); err != nil { return err } diff --git a/internal/config/projects.go b/internal/config/projects.go new file mode 100644 index 0000000..f416007 --- /dev/null +++ b/internal/config/projects.go @@ -0,0 +1,53 @@ +package config + +import ( + "fmt" + "time" +) + +// ProjectsConfig configures the project inventory: repositories upload an +// SBOM to /api/projects, and the proxy shows which of them use which package +// versions. +type ProjectsConfig struct { + // Enabled registers the /api/projects endpoints and the UI pages. Off by + // default. + Enabled bool `json:"enabled" yaml:"enabled"` + + // Token is the bearer token required to upload and delete projects. + // Required when Enabled. Supports ${VAR_NAME} expansion like AuthConfig + // fields. + Token string `json:"token" yaml:"token"` + + // MaxAge removes projects that have not uploaded an SBOM for this long + // (e.g., "90d"). Empty or "0" keeps them until they are deleted. + MaxAge string `json:"max_age" yaml:"max_age"` + + maxAge time.Duration +} + +// TokenExpanded returns Token with ${VAR_NAME} references expanded. +func (p *ProjectsConfig) TokenExpanded() string { + return expandEnv(p.Token) +} + +// MaxAgeDuration returns the parsed MaxAge, 0 to keep projects. Validate +// must run first. +func (p *ProjectsConfig) MaxAgeDuration() time.Duration { + return p.maxAge +} + +// Validate checks the project inventory settings. +func (p *ProjectsConfig) Validate() error { + p.maxAge = 0 + if p.MaxAge != "" { + d, err := parseRetentionDuration("projects.max_age", p.MaxAge) + if err != nil { + return err + } + p.maxAge = d + } + if p.Enabled && p.TokenExpanded() == "" { + return fmt.Errorf("projects.token is required when projects.enabled is true") + } + return nil +} diff --git a/internal/config/projects_test.go b/internal/config/projects_test.go new file mode 100644 index 0000000..ce3c27c --- /dev/null +++ b/internal/config/projects_test.go @@ -0,0 +1,69 @@ +package config + +import ( + "testing" + "time" +) + +func TestProjectsConfigValidate(t *testing.T) { + t.Setenv("PROJECTS_TEST_TOKEN", "from-env") + + tests := []struct { + name string + projects ProjectsConfig + wantErr bool + maxAge time.Duration + }{ + {name: "disabled without token", projects: ProjectsConfig{}}, + {name: "enabled without token", projects: ProjectsConfig{Enabled: true}, wantErr: true}, + {name: "enabled with token", projects: ProjectsConfig{Enabled: true, Token: "s3cret"}}, + {name: "token from env reference", projects: ProjectsConfig{Enabled: true, Token: "${PROJECTS_TEST_TOKEN}"}}, + {name: "token env reference unset", projects: ProjectsConfig{Enabled: true, Token: "${PROJECTS_TEST_UNSET}"}, wantErr: true}, + {name: "max age in days", projects: ProjectsConfig{Enabled: true, Token: "s3cret", MaxAge: "90d"}, maxAge: 90 * 24 * time.Hour}, + {name: "max age in hours", projects: ProjectsConfig{Enabled: true, Token: "s3cret", MaxAge: "36h"}, maxAge: 36 * time.Hour}, + {name: "max age zero keeps projects", projects: ProjectsConfig{Enabled: true, Token: "s3cret", MaxAge: "0"}}, + {name: "invalid max age", projects: ProjectsConfig{Enabled: true, Token: "s3cret", MaxAge: "soon"}, wantErr: true}, + {name: "negative max age", projects: ProjectsConfig{Enabled: true, Token: "s3cret", MaxAge: "-1d"}, wantErr: true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := tt.projects.Validate() + if (err != nil) != tt.wantErr { + t.Fatalf("Validate() error = %v, wantErr %v", err, tt.wantErr) + } + if got := tt.projects.MaxAgeDuration(); err == nil && got != tt.maxAge { + t.Errorf("MaxAgeDuration() = %v, want %v", got, tt.maxAge) + } + }) + } +} + +func TestLoadFromEnvProjects(t *testing.T) { + cfg := Default() + t.Setenv("PROXY_PROJECTS_ENABLED", "true") + t.Setenv("PROXY_PROJECTS_TOKEN", "env-token") + t.Setenv("PROXY_PROJECTS_MAX_AGE", "30d") + + cfg.LoadFromEnv() + + if !cfg.Projects.Enabled { + t.Error("Projects.Enabled not loaded from PROXY_PROJECTS_ENABLED") + } + if cfg.Projects.TokenExpanded() != "env-token" { + t.Errorf("Projects.Token = %q, want env-token", cfg.Projects.Token) + } + if err := cfg.Validate(); err != nil { + t.Fatalf("Validate: %v", err) + } + if got := cfg.Projects.MaxAgeDuration(); got != 30*24*time.Hour { + t.Errorf("MaxAgeDuration() = %v, want 30 days", got) + } +} + +func TestValidateRejectsProjectsWithoutToken(t *testing.T) { + cfg := Default() + cfg.Projects = ProjectsConfig{Enabled: true} + if err := cfg.Validate(); err == nil { + t.Error("Validate() accepted projects.enabled without a token") + } +} diff --git a/internal/database/database_test.go b/internal/database/database_test.go index f8956a6..ea3b349 100644 --- a/internal/database/database_test.go +++ b/internal/database/database_test.go @@ -613,7 +613,7 @@ func createTestPostgresDB(t *testing.T) *DB { // Drop and recreate every table CreateSchema creates for clean test state; // leftover migration records make the next CreateSchema fail on the // migrations primary key. - tables := []string{"artifacts", "pending_deletes", "versions", "packages", "vulnerabilities", "metadata_cache", "metadata_dates", "job_runs", "fetch_leases", "upstream_holds", "migrations", "schema_info"} + tables := []string{"artifacts", "pending_deletes", "versions", "packages", "vulnerabilities", "metadata_cache", "metadata_dates", "job_runs", "fetch_leases", "upstream_holds", "projects", "project_components", "migrations", "schema_info"} for _, table := range tables { _, _ = db.Exec("DROP TABLE IF EXISTS " + table + " CASCADE") } diff --git a/internal/database/projects.go b/internal/database/projects.go new file mode 100644 index 0000000..b00596b --- /dev/null +++ b/internal/database/projects.go @@ -0,0 +1,257 @@ +package database + +import ( + "database/sql" + "errors" + "fmt" + "time" +) + +// Project is a repository that uploaded an SBOM to the project inventory. +type Project struct { + ID int64 `db:"id"` + Name string `db:"name"` + Ref sql.NullString `db:"ref"` + Branch sql.NullString `db:"branch"` + UploadedAt time.Time `db:"uploaded_at"` + // Components is the number of package versions recorded for it. + Components int64 `db:"components"` +} + +// ProjectComponent is a package version a project uses. PackagePURL is the +// unversioned package URL, as packages.purl records it. +type ProjectComponent struct { + PackagePURL string `db:"package_purl"` + Ecosystem string `db:"ecosystem"` + Name string `db:"name"` + Version string `db:"version"` +} + +// ProjectComponentInfo is a project's component together with the package +// record the proxy holds for it, if any. +type ProjectComponentInfo struct { + ProjectComponent + CachedEcosystem sql.NullString `db:"cached_ecosystem"` + CachedName sql.NullString `db:"cached_name"` + LatestVersion sql.NullString `db:"latest_version"` +} + +// ProjectUsage is a project using a version of a package. +type ProjectUsage struct { + Project + Version string `db:"version"` +} + +// ComponentVulnerability is a vulnerability recorded for the package of a +// component a project uses. It is not yet filtered by version: whether it +// affects Version depends on FixedVersion. +type ComponentVulnerability struct { + Project string `db:"project"` + ProjectComponent + CachedEcosystem string `db:"cached_ecosystem"` + CachedName string `db:"cached_name"` + VulnID string `db:"vuln_id"` + Severity sql.NullString `db:"severity"` + FixedVersion sql.NullString `db:"fixed_version"` + CVSSScore sql.NullFloat64 `db:"cvss_score"` +} + +const projectColumns = ` + SELECT pr.id, pr.name, pr.ref, pr.branch, pr.uploaded_at, + (SELECT COUNT(*) FROM project_components pc WHERE pc.project_id = pr.id) AS components` + +// SaveProject records the components of a project, replacing everything +// recorded for it before, in one transaction. Readers see either the old or +// the new set, never a mix. +func (db *DB) SaveProject(name, ref, branch string, uploadedAt time.Time, components []ProjectComponent) error { + tx, err := db.Beginx() + if err != nil { + return err + } + defer func() { _ = tx.Rollback() }() + + // The upsert locks the project row, so concurrent uploads of the same + // project and pruning take turns. + if _, err := tx.Exec(db.Rebind(` + INSERT INTO projects (name, ref, branch, uploaded_at) VALUES (?, ?, ?, ?) + ON CONFLICT(name) DO UPDATE SET ref = excluded.ref, branch = excluded.branch, uploaded_at = excluded.uploaded_at + `), name, nullString(ref), nullString(branch), uploadedAt.UTC()); err != nil { + return fmt.Errorf("recording project: %w", err) + } + var id int64 + if err := tx.Get(&id, db.Rebind(`SELECT id FROM projects WHERE name = ?`), name); err != nil { + return fmt.Errorf("reading project id: %w", err) + } + if _, err := tx.Exec(db.Rebind(`DELETE FROM project_components WHERE project_id = ?`), id); err != nil { + return fmt.Errorf("removing old components: %w", err) + } + + insert, err := tx.Preparex(db.Rebind(` + INSERT INTO project_components (project_id, package_purl, ecosystem, name, version) + VALUES (?, ?, ?, ?, ?) + ON CONFLICT(project_id, package_purl, version) DO NOTHING + `)) + if err != nil { + return err + } + defer func() { _ = insert.Close() }() + for _, c := range components { + if _, err := insert.Exec(id, c.PackagePURL, c.Ecosystem, c.Name, c.Version); err != nil { + return fmt.Errorf("recording component %s@%s: %w", c.PackagePURL, c.Version, err) + } + } + return tx.Commit() +} + +// DeleteProject removes a project and its components. It reports whether the +// project existed. +func (db *DB) DeleteProject(name string) (bool, error) { + tx, err := db.Beginx() + if err != nil { + return false, err + } + defer func() { _ = tx.Rollback() }() + + var id int64 + err = tx.Get(&id, db.Rebind(`DELETE FROM projects WHERE name = ? RETURNING id`), name) + if errors.Is(err, sql.ErrNoRows) { + return false, nil + } + if err != nil { + return false, fmt.Errorf("deleting project: %w", err) + } + if _, err := tx.Exec(db.Rebind(`DELETE FROM project_components WHERE project_id = ?`), id); err != nil { + return false, fmt.Errorf("deleting project components: %w", err) + } + return true, tx.Commit() +} + +// PruneProjects removes the projects last uploaded before cutoff, with their +// components, and returns how many it removed. A project uploaded again while +// the prune runs is kept. +func (db *DB) PruneProjects(cutoff time.Time) (int, error) { + var ids []int64 + if err := db.Select(&ids, db.Rebind(`SELECT id FROM projects WHERE uploaded_at < ?`), cutoff.UTC()); err != nil { + return 0, fmt.Errorf("listing expired projects: %w", err) + } + pruned := 0 + for _, id := range ids { + ok, err := db.pruneProject(id, cutoff) + if err != nil { + return pruned, err + } + if ok { + pruned++ + } + } + return pruned, nil +} + +func (db *DB) pruneProject(id int64, cutoff time.Time) (bool, error) { + tx, err := db.Beginx() + if err != nil { + return false, err + } + defer func() { _ = tx.Rollback() }() + + res, err := tx.Exec(db.Rebind(`DELETE FROM projects WHERE id = ? AND uploaded_at < ?`), id, cutoff.UTC()) + if err != nil { + return false, fmt.Errorf("pruning project: %w", err) + } + if n, err := res.RowsAffected(); err != nil || n == 0 { + return false, err + } + if _, err := tx.Exec(db.Rebind(`DELETE FROM project_components WHERE project_id = ?`), id); err != nil { + return false, fmt.Errorf("pruning project components: %w", err) + } + return true, tx.Commit() +} + +// ListProjects returns all projects ordered by name. +func (db *DB) ListProjects() ([]Project, error) { + projects := []Project{} + if err := db.Select(&projects, projectColumns+` FROM projects pr ORDER BY pr.name`); err != nil { + return nil, err + } + return projects, nil +} + +// GetProject returns the named project, or nil if there is none. +func (db *DB) GetProject(name string) (*Project, error) { + var p Project + err := db.Get(&p, db.Rebind(projectColumns+` FROM projects pr WHERE pr.name = ?`), name) + if errors.Is(err, sql.ErrNoRows) { + return nil, nil + } + if err != nil { + return nil, err + } + return &p, nil +} + +// GetProjectComponents returns the components of a project, ordered by +// ecosystem, name and version, with the package record the proxy holds for +// each. +func (db *DB) GetProjectComponents(projectID int64) ([]ProjectComponentInfo, error) { + components := []ProjectComponentInfo{} + err := db.Select(&components, db.Rebind(` + SELECT c.package_purl, c.ecosystem, c.name, c.version, + p.ecosystem AS cached_ecosystem, p.name AS cached_name, p.latest_version + FROM project_components c + LEFT JOIN packages p ON p.purl = c.package_purl + WHERE c.project_id = ? + ORDER BY c.ecosystem, c.name, c.version + `), projectID) + if err != nil { + return nil, err + } + return components, nil +} + +// GetProjectsUsingPackage returns the projects using a package, or only +// those using version when it is not empty, ordered by project and version. +func (db *DB) GetProjectsUsingPackage(packagePURL, version string) ([]ProjectUsage, error) { + query := projectColumns + `, c.version + FROM project_components c + JOIN projects pr ON pr.id = c.project_id + WHERE c.package_purl = ?` + args := []any{packagePURL} + if version != "" { + query += ` AND c.version = ?` + args = append(args, version) + } + usages := []ProjectUsage{} + if err := db.Select(&usages, db.Rebind(query+` ORDER BY pr.name, c.version`), args...); err != nil { + return nil, err + } + return usages, nil +} + +// GetComponentVulnerabilities returns the vulnerabilities the proxy holds for +// packages that projects use, for one project or, with an empty name, for all +// of them. Vulnerabilities are recorded per package, so only packages the +// proxy has a record of are matched. +func (db *DB) GetComponentVulnerabilities(project string) ([]ComponentVulnerability, error) { + query := ` + SELECT pr.name AS project, c.package_purl, c.ecosystem, c.name, c.version, + p.ecosystem AS cached_ecosystem, p.name AS cached_name, + v.vuln_id, v.severity, v.fixed_version, v.cvss_score + FROM project_components c + JOIN projects pr ON pr.id = c.project_id + JOIN packages p ON p.purl = c.package_purl + JOIN vulnerabilities v ON v.ecosystem = p.ecosystem AND v.package_name = p.name` + var args []any + if project != "" { + query += ` WHERE pr.name = ?` + args = append(args, project) + } + vulns := []ComponentVulnerability{} + if err := db.Select(&vulns, db.Rebind(query+` ORDER BY c.package_purl, c.version, pr.name, v.vuln_id`), args...); err != nil { + return nil, err + } + return vulns, nil +} + +func nullString(s string) sql.NullString { + return sql.NullString{String: s, Valid: s != ""} +} diff --git a/internal/database/projects_test.go b/internal/database/projects_test.go new file mode 100644 index 0000000..4eaaf50 --- /dev/null +++ b/internal/database/projects_test.go @@ -0,0 +1,228 @@ +package database + +import ( + "database/sql" + "slices" + "testing" + "time" +) + +var ( + lodash20 = ProjectComponent{PackagePURL: "pkg:npm/lodash", Ecosystem: "npm", Name: "lodash", Version: "4.17.20"} + lodash21 = ProjectComponent{PackagePURL: "pkg:npm/lodash", Ecosystem: "npm", Name: "lodash", Version: "4.17.21"} + console = ProjectComponent{PackagePURL: "pkg:composer/symfony/console", Ecosystem: "composer", Name: "symfony/console", Version: "v6.4.1"} +) + +func saveProject(t *testing.T, db *DB, name string, at time.Time, components ...ProjectComponent) { + t.Helper() + if err := db.SaveProject(name, "", "", at, components); err != nil { + t.Fatalf("SaveProject(%s): %v", name, err) + } +} + +func usageNames(usages []ProjectUsage) []string { + var names []string + for _, u := range usages { + names = append(names, u.Name+"@"+u.Version) + } + return names +} + +func TestSaveProjectReplacesComponents(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + now := time.Now().UTC().Truncate(time.Second) + if err := db.SaveProject("acme/web", "abc123", "main", now, []ProjectComponent{lodash20, console, lodash20}); err != nil { + t.Fatal(err) + } + saveProject(t, db, "acme/web", now.Add(time.Minute), lodash21) + + project, err := db.GetProject("acme/web") + if err != nil || project == nil { + t.Fatalf("GetProject = %v, %v", project, err) + } + if project.Components != 1 || project.Ref.Valid || project.Branch.Valid { + t.Errorf("project = %+v, want one component and ref and branch cleared", project) + } + if !project.UploadedAt.Equal(now.Add(time.Minute)) { + t.Errorf("uploaded_at = %v, want %v", project.UploadedAt, now.Add(time.Minute)) + } + components, err := db.GetProjectComponents(project.ID) + if err != nil { + t.Fatal(err) + } + if len(components) != 1 || components[0].ProjectComponent != lodash21 { + t.Errorf("components = %+v, want only lodash 4.17.21", components) + } + + missing, err := db.GetProject("acme/other") + if err != nil || missing != nil { + t.Errorf("GetProject(missing) = %v, %v", missing, err) + } + }) +} + +func TestGetProjectsUsingPackage(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + now := time.Now() + saveProject(t, db, "acme/web", now, lodash20, console) + saveProject(t, db, "acme/api", now, lodash21) + saveProject(t, db, "acme/cli", now, console) + + all, err := db.GetProjectsUsingPackage("pkg:npm/lodash", "") + if err != nil { + t.Fatal(err) + } + if got, want := usageNames(all), []string{"acme/api@4.17.21", "acme/web@4.17.20"}; !slices.Equal(got, want) { + t.Errorf("any version = %v, want %v", got, want) + } + one, err := db.GetProjectsUsingPackage("pkg:npm/lodash", "4.17.20") + if err != nil { + t.Fatal(err) + } + if got, want := usageNames(one), []string{"acme/web@4.17.20"}; !slices.Equal(got, want) { + t.Errorf("4.17.20 = %v, want %v", got, want) + } + none, err := db.GetProjectsUsingPackage("pkg:npm/lodash", "1.0.0") + if err != nil || len(none) != 0 { + t.Errorf("unused version = %v, %v", none, err) + } + }) +} + +func TestListAndDeleteProjects(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + now := time.Now() + saveProject(t, db, "acme/web", now, lodash20, console) + saveProject(t, db, "acme/api", now) + + projects, err := db.ListProjects() + if err != nil { + t.Fatal(err) + } + if len(projects) != 2 || projects[0].Name != "acme/api" || projects[0].Components != 0 || projects[1].Components != 2 { + t.Fatalf("projects = %+v", projects) + } + + deleted, err := db.DeleteProject("acme/web") + if err != nil || !deleted { + t.Fatalf("DeleteProject = %v, %v", deleted, err) + } + deleted, err = db.DeleteProject("acme/web") + if err != nil || deleted { + t.Errorf("second DeleteProject = %v, %v, want false", deleted, err) + } + var rows int + if err := db.Get(&rows, `SELECT COUNT(*) FROM project_components`); err != nil { + t.Fatal(err) + } + if rows != 0 { + t.Errorf("%d component rows left after the delete", rows) + } + }) +} + +func TestPruneProjects(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + now := time.Now() + saveProject(t, db, "acme/old", now.Add(-100*24*time.Hour), lodash20) + saveProject(t, db, "acme/new", now.Add(-time.Hour), lodash21) + + pruned, err := db.PruneProjects(now.Add(-90 * 24 * time.Hour)) + if err != nil { + t.Fatal(err) + } + if pruned != 1 { + t.Errorf("pruned = %d, want 1", pruned) + } + projects, err := db.ListProjects() + if err != nil { + t.Fatal(err) + } + if len(projects) != 1 || projects[0].Name != "acme/new" { + t.Errorf("projects after prune = %+v", projects) + } + usages, err := db.GetProjectsUsingPackage("pkg:npm/lodash", "") + if err != nil { + t.Fatal(err) + } + if got, want := usageNames(usages), []string{"acme/new@4.17.21"}; !slices.Equal(got, want) { + t.Errorf("usages after prune = %v, want %v", got, want) + } + }) +} + +func TestProjectComponentsJoinPackages(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + if err := db.UpsertPackage(&Package{ + PURL: "pkg:npm/lodash", Ecosystem: "npm", Name: "lodash", + LatestVersion: sql.NullString{String: "4.17.21", Valid: true}, + }); err != nil { + t.Fatal(err) + } + if err := db.UpsertVulnerability(&Vulnerability{ + VulnID: "GHSA-35jh-r3h4-6jhm", Ecosystem: "npm", PackageName: "lodash", + Severity: sql.NullString{String: "high", Valid: true}, + FixedVersion: sql.NullString{String: "4.17.21", Valid: true}, + }); err != nil { + t.Fatal(err) + } + saveProject(t, db, "acme/web", time.Now(), lodash20, console) + saveProject(t, db, "acme/api", time.Now(), lodash21) + + project, err := db.GetProject("acme/web") + if err != nil || project == nil { + t.Fatalf("GetProject = %v, %v", project, err) + } + components, err := db.GetProjectComponents(project.ID) + if err != nil { + t.Fatal(err) + } + if len(components) != 2 { + t.Fatalf("components = %+v", components) + } + // Ordered by ecosystem: composer first, which the proxy has no record of. + if components[0].CachedName.Valid { + t.Errorf("uncached component has a package record: %+v", components[0]) + } + if components[1].CachedName.String != "lodash" || components[1].LatestVersion.String != "4.17.21" { + t.Errorf("cached component = %+v", components[1]) + } + + vulns, err := db.GetComponentVulnerabilities("acme/web") + if err != nil { + t.Fatal(err) + } + if len(vulns) != 1 || vulns[0].Version != "4.17.20" || vulns[0].VulnID != "GHSA-35jh-r3h4-6jhm" { + t.Errorf("acme/web vulnerabilities = %+v", vulns) + } + all, err := db.GetComponentVulnerabilities("") + if err != nil { + t.Fatal(err) + } + if len(all) != 2 { + t.Errorf("all vulnerabilities = %+v, want one row per project", all) + } + }) +} + +func TestMigrateAddsProjects(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + // A database from before the project inventory. + for _, table := range []string{"project_components", "projects"} { + if _, err := db.Exec("DROP TABLE " + table); err != nil { + t.Fatalf("dropping %s: %v", table, err) + } + } + if _, err := db.Exec(db.Rebind("DELETE FROM migrations WHERE name = ?"), "020_add_projects"); err != nil { + t.Fatalf("forgetting the migration: %v", err) + } + if err := db.MigrateSchema(); err != nil { + t.Fatalf("MigrateSchema: %v", err) + } + saveProject(t, db, "acme/web", time.Now(), lodash20) + usages, err := db.GetProjectsUsingPackage("pkg:npm/lodash", "4.17.20") + if err != nil || len(usages) != 1 { + t.Fatalf("GetProjectsUsingPackage after migration = %v, %v", usages, err) + } + }) +} diff --git a/internal/database/schema.go b/internal/database/schema.go index f0ec651..861429e 100644 --- a/internal/database/schema.go +++ b/internal/database/schema.go @@ -154,6 +154,26 @@ CREATE TABLE IF NOT EXISTS upstream_holds ( until_ms INTEGER NOT NULL ); +CREATE TABLE IF NOT EXISTS projects ( + id INTEGER PRIMARY KEY, + name TEXT NOT NULL, + ref TEXT, + branch TEXT, + uploaded_at DATETIME NOT NULL +); +CREATE UNIQUE INDEX IF NOT EXISTS idx_projects_name ON projects(name); +CREATE INDEX IF NOT EXISTS idx_projects_uploaded_at ON projects(uploaded_at); + +CREATE TABLE IF NOT EXISTS project_components ( + project_id INTEGER NOT NULL, + package_purl TEXT NOT NULL, + ecosystem TEXT NOT NULL, + name TEXT NOT NULL, + version TEXT NOT NULL, + PRIMARY KEY (project_id, package_purl, version) +); +CREATE INDEX IF NOT EXISTS idx_project_components_package ON project_components(package_purl, version); + CREATE TABLE IF NOT EXISTS migrations ( name TEXT NOT NULL PRIMARY KEY, applied_at DATETIME NOT NULL @@ -297,6 +317,26 @@ CREATE TABLE IF NOT EXISTS upstream_holds ( until_ms BIGINT NOT NULL ); +CREATE TABLE IF NOT EXISTS projects ( + id SERIAL PRIMARY KEY, + name TEXT NOT NULL, + ref TEXT, + branch TEXT, + uploaded_at TIMESTAMP NOT NULL +); +CREATE UNIQUE INDEX IF NOT EXISTS idx_projects_name ON projects(name); +CREATE INDEX IF NOT EXISTS idx_projects_uploaded_at ON projects(uploaded_at); + +CREATE TABLE IF NOT EXISTS project_components ( + project_id INTEGER NOT NULL, + package_purl TEXT NOT NULL, + ecosystem TEXT NOT NULL, + name TEXT NOT NULL, + version TEXT NOT NULL, + PRIMARY KEY (project_id, package_purl, version) +); +CREATE INDEX IF NOT EXISTS idx_project_components_package ON project_components(package_purl, version); + CREATE TABLE IF NOT EXISTS migrations ( name TEXT NOT NULL PRIMARY KEY, applied_at TIMESTAMP NOT NULL @@ -468,6 +508,7 @@ var migrations = []migration{ {"017_add_artifact_content_hash_index", migrateAddArtifactContentHashIndex}, {"018_add_upstream_holds", migrateAddUpstreamHolds}, {"019_add_artifact_filename_index", migrateAddArtifactFilenameIndex}, + {"020_add_projects", migrateAddProjects}, } // isTableNotFound returns true if the error indicates a missing table. @@ -820,6 +861,41 @@ func migrateAddUpstreamHolds(db *DB) error { return nil } +// migrateAddProjects creates the tables of the project inventory: the +// projects that uploaded an SBOM and the package versions each one uses. +func migrateAddProjects(db *DB) error { + id, ts := "INTEGER PRIMARY KEY", sqliteDatetime + if db.dialect == DialectPostgres { + id, ts = "SERIAL PRIMARY KEY", postgresTimestamp + } + statements := []string{ + fmt.Sprintf(`CREATE TABLE IF NOT EXISTS projects ( + id %s, + name TEXT NOT NULL, + ref TEXT, + branch TEXT, + uploaded_at %s NOT NULL + )`, id, ts), + `CREATE UNIQUE INDEX IF NOT EXISTS idx_projects_name ON projects(name)`, + `CREATE INDEX IF NOT EXISTS idx_projects_uploaded_at ON projects(uploaded_at)`, + `CREATE TABLE IF NOT EXISTS project_components ( + project_id INTEGER NOT NULL, + package_purl TEXT NOT NULL, + ecosystem TEXT NOT NULL, + name TEXT NOT NULL, + version TEXT NOT NULL, + PRIMARY KEY (project_id, package_purl, version) + )`, + `CREATE INDEX IF NOT EXISTS idx_project_components_package ON project_components(package_purl, version)`, + } + for _, stmt := range statements { + if _, err := db.Exec(stmt); err != nil { + return fmt.Errorf("creating project tables: %w", err) + } + } + return nil +} + // migrateAddPendingDeletesQueuedAtIndex indexes the column reclaim orders // the queue by, which grows large after a retention sweep. func migrateAddPendingDeletesQueuedAtIndex(db *DB) error { diff --git a/internal/inventory/inventory.go b/internal/inventory/inventory.go new file mode 100644 index 0000000..954d46e --- /dev/null +++ b/internal/inventory/inventory.go @@ -0,0 +1,185 @@ +// Package inventory turns SBOMs uploaded for a project into the package +// versions the proxy records for it, named the way the proxy's handlers name +// the packages they cache, so the two can be joined. +package inventory + +import ( + "errors" + "fmt" + "strings" + + "github.com/git-pkgs/proxy/internal/packageurl" + "github.com/git-pkgs/purl" + "github.com/git-pkgs/sbom" + "github.com/git-pkgs/vers" +) + +// MaxProjectNameLength bounds a project name, such as a GitLab project path. +const MaxProjectNameLength = 255 + +// Component is one package version a project uses. +type Component struct { + // Ecosystem is the name the proxy's handlers store the package under, + // e.g. "npm", "composer", "golang" or "alpine". + Ecosystem string + // Name is the ecosystem-native package name, e.g. "@babel/core", + // "symfony/console" or "org.slf4j:slf4j-api". + Name string + // Version is the version as the SBOM gives it; it may be empty. + Version string + // PackagePURL is the unversioned package URL the proxy records the + // package under, so it matches packages.purl. + PackagePURL string +} + +// ecosystems maps the PURL types the proxy serves to the ecosystem name its +// handlers store packages under. +var ecosystems = map[string]string{ + "alpm": "arch", + "apk": "alpine", + "bsd": "freebsd", + "cargo": "cargo", + "chef": "chef", + "composer": "composer", + "conan": "conan", + "conda": "conda", + "cran": "cran", + "deb": "deb", + "gem": "gem", + "golang": "golang", + "hex": "hex", + "julia": "julia", + "maven": "maven", + "npm": "npm", + "nuget": "nuget", + "pub": "pub", + "pypi": "pypi", + "rpm": "rpm", + "swift": "swift", +} + +// distroTypes are PURL types whose namespace names the distribution +// ("debian", "alpine"), not part of the package name the proxy stores. +var distroTypes = map[string]bool{"alpm": true, "apk": true, "bsd": true, "deb": true, "rpm": true} + +// ErrUnsupportedPURL is returned for a package URL whose type the proxy +// doesn't serve. +var ErrUnsupportedPURL = errors.New("package type is not served by the proxy") + +// FromPURL maps a package URL to a component. Qualifiers and subpath are +// dropped; the version is kept as given. +func FromPURL(s string) (Component, error) { + p, err := purl.Parse(s) + if err != nil { + return Component{}, err + } + ecosystem, ok := ecosystems[p.Type] + if !ok { + return Component{}, fmt.Errorf("%w: %s", ErrUnsupportedPURL, p.Type) + } + name := p.FullName() + switch { + case distroTypes[p.Type]: + name = p.Name + case p.Type == "conda" && p.Namespace == "" && p.Qualifier("channel") != "": + name = p.Qualifier("channel") + "/" + p.Name + } + packagePURL, _ := packageurl.MakeCacheStrings(ecosystem, name, "") + if packagePURL == "" { + return Component{}, fmt.Errorf("package URL %q has no valid package name", s) + } + return Component{Ecosystem: ecosystem, Name: name, Version: p.Version, PackagePURL: packagePURL}, nil +} + +// Parse reads a CycloneDX (JSON or XML) or SPDX SBOM and returns the package +// versions it lists, without duplicates. Packages without a package URL, or +// of a type the proxy doesn't serve, are counted in skipped. +func Parse(data []byte) (components []Component, skipped int, err error) { + doc, err := sbom.Parse(data) + if err != nil { + return nil, 0, err + } + seen := make(map[[2]string]bool, len(doc.Packages)) + for _, pkg := range doc.Packages { + c, ok := componentOf(pkg) + if !ok { + skipped++ + continue + } + key := [2]string{c.PackagePURL, c.Version} + if seen[key] { + continue + } + seen[key] = true + components = append(components, c) + } + return components, skipped, nil +} + +func componentOf(pkg sbom.Package) (Component, bool) { + locator := pkg.PURL() + if locator == "" { + return Component{}, false + } + c, err := FromPURL(locator) + if err != nil { + return Component{}, false + } + if c.Version == "" { + c.Version = pkg.Version + } + return c, true +} + +// ValidateProjectName checks a project name such as "group/sub/repo": one or +// more segments of letters, digits, ".", "_" and "-", none of them "." or +// "..". +func ValidateProjectName(name string) error { + if name == "" { + return errors.New("project name is empty") + } + if len(name) > MaxProjectNameLength { + return fmt.Errorf("project name is longer than %d characters", MaxProjectNameLength) + } + for segment := range strings.SplitSeq(name, "/") { + if segment == "" || segment == "." || segment == ".." { + return fmt.Errorf("project name %q has an empty, \".\" or \"..\" segment", name) + } + for _, r := range segment { + if !validNameRune(r) { + return fmt.Errorf("project name %q may only contain letters, digits, '.', '_', '-' and '/'", name) + } + } + } + return nil +} + +func validNameRune(r rune) bool { + return r >= 'a' && r <= 'z' || r >= 'A' && r <= 'Z' || r >= '0' && r <= '9' || r == '.' || r == '_' || r == '-' +} + +// CompareVersions compares two versions of a package in ecosystem with the +// ecosystem's version rules, returning -1, 0 or 1. +func CompareVersions(ecosystem, a, b string) int { + return vers.CompareWithScheme(a, b, purl.EcosystemToPURLType(ecosystem)) +} + +// Outdated reports whether version is lower than latest. +func Outdated(ecosystem, version, latest string) bool { + if version == "" || latest == "" || version == latest { + return false + } + return CompareVersions(ecosystem, version, latest) < 0 +} + +// Affected reports whether a vulnerability fixed in fixedVersion affects +// version. A vulnerability without a fixed version affects every version. +func Affected(ecosystem, version, fixedVersion string) bool { + if fixedVersion == "" { + return true + } + if version == "" { + return false + } + return CompareVersions(ecosystem, version, fixedVersion) < 0 +} diff --git a/internal/inventory/inventory_test.go b/internal/inventory/inventory_test.go new file mode 100644 index 0000000..3d55d50 --- /dev/null +++ b/internal/inventory/inventory_test.go @@ -0,0 +1,133 @@ +package inventory + +import ( + "errors" + "os" + "slices" + "strings" + "testing" +) + +func TestParseSyftCycloneDX(t *testing.T) { + data, err := os.ReadFile("testdata/syft-cyclonedx.json") + if err != nil { + t.Fatal(err) + } + components, skipped, err := Parse(data) + if err != nil { + t.Fatalf("Parse: %v", err) + } + want := []Component{ + {Ecosystem: "npm", Name: "@babel/core", Version: "7.23.0", PackagePURL: "pkg:npm/%40babel/core"}, + {Ecosystem: "alpine", Name: "busybox", Version: "1.36.1-r15", PackagePURL: "pkg:apk/alpine/busybox"}, + {Ecosystem: "golang", Name: "github.com/inconshreveable/mousetrap", Version: "v1.1.0", PackagePURL: "pkg:golang/github.com/inconshreveable/mousetrap"}, + {Ecosystem: "golang", Name: "github.com/spf13/cobra", Version: "v1.8.0", PackagePURL: "pkg:golang/github.com/spf13/cobra"}, + {Ecosystem: "golang", Name: "github.com/spf13/pflag", Version: "v1.0.5", PackagePURL: "pkg:golang/github.com/spf13/pflag"}, + {Ecosystem: "npm", Name: "lodash", Version: "4.17.20", PackagePURL: "pkg:npm/lodash"}, + {Ecosystem: "composer", Name: "monolog/monolog", Version: "3.5.0", PackagePURL: "pkg:composer/monolog/monolog"}, + {Ecosystem: "alpine", Name: "musl", Version: "1.2.4-r2", PackagePURL: "pkg:apk/alpine/musl"}, + {Ecosystem: "composer", Name: "symfony/console", Version: "v6.4.1", PackagePURL: "pkg:composer/symfony/console"}, + {Ecosystem: "npm", Name: "webapp", Version: "1.0.0", PackagePURL: "pkg:npm/webapp"}, + } + if !slices.Equal(components, want) { + t.Errorf("components =\n%v\nwant\n%v", components, want) + } + // The operating system and the four lock files syft lists carry no PURL. + if skipped != 5 { + t.Errorf("skipped = %d, want 5", skipped) + } +} + +func TestParseDeduplicatesAndSkipsUnknownTypes(t *testing.T) { + doc := `{"bomFormat":"CycloneDX","specVersion":"1.5","components":[ + {"type":"library","name":"lodash","version":"4.17.21","purl":"pkg:npm/lodash@4.17.21"}, + {"type":"library","name":"lodash","version":"4.17.21","purl":"pkg:npm/lodash@4.17.21"}, + {"type":"library","name":"lodash","version":"4.17.20","purl":"pkg:npm/lodash@4.17.20"}, + {"type":"library","name":"checkout","version":"v4","purl":"pkg:github/actions/checkout@v4"}, + {"type":"library","name":"Django","version":"4.2.0","purl":"pkg:pypi/Django"}, + {"type":"library","name":"guava","version":"32.1.3-jre","purl":"pkg:maven/com.google.guava/guava@32.1.3-jre?type=jar"}, + {"type":"library","name":"libc6","version":"2.36-9","purl":"pkg:deb/debian/libc6@2.36-9?arch=amd64&distro=debian-12"}, + {"type":"library","name":"broken","purl":"not a purl"} + ]}` + components, skipped, err := Parse([]byte(doc)) + if err != nil { + t.Fatal(err) + } + want := []Component{ + {Ecosystem: "npm", Name: "lodash", Version: "4.17.21", PackagePURL: "pkg:npm/lodash"}, + {Ecosystem: "npm", Name: "lodash", Version: "4.17.20", PackagePURL: "pkg:npm/lodash"}, + {Ecosystem: "pypi", Name: "django", Version: "4.2.0", PackagePURL: "pkg:pypi/django"}, + {Ecosystem: "maven", Name: "com.google.guava:guava", Version: "32.1.3-jre", PackagePURL: "pkg:maven/com.google.guava/guava"}, + {Ecosystem: "deb", Name: "libc6", Version: "2.36-9", PackagePURL: "pkg:deb/libc6"}, + } + if !slices.Equal(components, want) { + t.Errorf("components =\n%v\nwant\n%v", components, want) + } + if skipped != 2 { + t.Errorf("skipped = %d, want 2", skipped) + } +} + +func TestParseRejectsNonSBOM(t *testing.T) { + if _, _, err := Parse([]byte(`{"hello":"world"}`)); err == nil { + t.Error("Parse accepted a document that is not an SBOM") + } +} + +func TestFromPURLUnsupported(t *testing.T) { + _, err := FromPURL("pkg:github/actions/checkout") + if !errors.Is(err, ErrUnsupportedPURL) { + t.Errorf("err = %v, want ErrUnsupportedPURL", err) + } +} + +func TestValidateProjectName(t *testing.T) { + valid := []string{"repo", "group/repo", "group/sub/my_repo-2.0", strings.Repeat("a", MaxProjectNameLength)} + for _, name := range valid { + if err := ValidateProjectName(name); err != nil { + t.Errorf("ValidateProjectName(%q) = %v, want nil", name, err) + } + } + invalid := []string{ + "", "/repo", "repo/", "group//repo", ".", "..", "group/../repo", "group/./repo", + "group/re po", "group/repo?x", "group\\repo", "grüppe/repo", "group/repo%2F", + strings.Repeat("a", MaxProjectNameLength+1), + } + for _, name := range invalid { + if err := ValidateProjectName(name); err == nil { + t.Errorf("ValidateProjectName(%q) = nil, want an error", name) + } + } +} + +func TestOutdatedAndAffected(t *testing.T) { + tests := []struct { + name string + ecosystem string + version string + other string + outdated bool + affected bool + }{ + {"npm older", "npm", "4.17.20", "4.17.21", true, true}, + {"npm same", "npm", "4.17.21", "4.17.21", false, false}, + {"npm newer", "npm", "4.18.0", "4.17.21", false, false}, + {"composer v prefix", "composer", "v6.4.1", "v6.4.2", true, true}, + {"golang", "golang", "v1.8.0", "v1.10.0", true, true}, + {"alpine revisions", "alpine", "1.36.1-r15", "1.36.1-r2", false, false}, + {"empty version", "npm", "", "1.0.0", false, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := Outdated(tt.ecosystem, tt.version, tt.other); got != tt.outdated { + t.Errorf("Outdated = %v, want %v", got, tt.outdated) + } + if got := Affected(tt.ecosystem, tt.version, tt.other); got != tt.affected { + t.Errorf("Affected = %v, want %v", got, tt.affected) + } + }) + } + if !Affected("npm", "1.0.0", "") { + t.Error("a vulnerability without a fix should affect every version") + } +} diff --git a/internal/inventory/testdata/syft-cyclonedx.json b/internal/inventory/testdata/syft-cyclonedx.json new file mode 100644 index 0000000..7647d68 --- /dev/null +++ b/internal/inventory/testdata/syft-cyclonedx.json @@ -0,0 +1 @@ +{"$schema":"http://cyclonedx.org/schema/bom-1.7.schema.json","bomFormat":"CycloneDX","specVersion":"1.7","serialNumber":"urn:uuid:d69cf664-a43e-4d65-93e3-e43e24d33c0e","version":1,"metadata":{"timestamp":"2026-10-27T19:17:53+01:00","tools":{"components":[{"type":"application","author":"anchore","name":"syft","version":"1.55.0"}]},"component":{"bom-ref":"af63bd4c8601b7f1","type":"file","name":"."}},"components":[{"bom-ref":"pkg:npm/%40babel/core@7.23.0?package-id=7b01fd7a9ce63128","type":"library","name":"@babel/core","version":"7.23.0","licenses":[{"license":{"id":"MIT"}}],"cpe":"cpe:2.3:a:\\@babel\\/core:\\@babel\\/core:7.23.0:*:*:*:*:*:*:*","purl":"pkg:npm/%40babel/core@7.23.0","properties":[{"name":"syft:package:foundBy","value":"javascript-lock-cataloger"},{"name":"syft:package:language","value":"javascript"},{"name":"syft:package:type","value":"npm"},{"name":"syft:package:metadataType","value":"javascript-npm-package-lock-entry"},{"name":"syft:location:0:path","value":"/package-lock.json"}]},{"bom-ref":"pkg:apk/alpine/busybox@1.36.1-r15?arch=x86_64&distro=alpine-3.19.1&package-id=464f552f6c89c192","type":"library","publisher":"Sören Tempel ","name":"busybox","version":"1.36.1-r15","description":"Size optimized toolbox of many common UNIX utilities","licenses":[{"license":{"id":"GPL-2.0-only"}}],"cpe":"cpe:2.3:a:busybox:busybox:1.36.1-r15:*:*:*:*:*:*:*","purl":"pkg:apk/alpine/busybox@1.36.1-r15?arch=x86_64&distro=alpine-3.19.1","externalReferences":[{"url":"https://busybox.net/","type":"distribution"}],"properties":[{"name":"syft:package:foundBy","value":"apk-db-cataloger"},{"name":"syft:package:type","value":"apk"},{"name":"syft:package:metadataType","value":"apk-db-entry"},{"name":"syft:location:0:path","value":"/lib/apk/db/installed"},{"name":"syft:metadata:gitCommitOfApkPort","value":"1b0b2f6e7d0e6a5b4c3d2e1f0a9b8c7d6e5f4a3b"},{"name":"syft:metadata:installedSize","value":"950272"},{"name":"syft:metadata:originPackage","value":"busybox"},{"name":"syft:metadata:pullChecksum","value":"Q1Xy0qB0j7Y7uQ2kGmWbN8vT6Ff0A4E="},{"name":"syft:metadata:pullDependencies:0","value":"so:libc.musl-x86_64.so.1"},{"name":"syft:metadata:size","value":"509268"}]},{"bom-ref":"pkg:golang/github.com/inconshreveable/mousetrap@v1.1.0?package-id=f6300927a9f6b106","type":"library","name":"github.com/inconshreveable/mousetrap","version":"v1.1.0","cpe":"cpe:2.3:a:inconshreveable:mousetrap:v1.1.0:*:*:*:*:*:*:*","purl":"pkg:golang/github.com/inconshreveable/mousetrap@v1.1.0","properties":[{"name":"syft:package:foundBy","value":"go-module-file-cataloger"},{"name":"syft:package:language","value":"go"},{"name":"syft:package:type","value":"go-module"},{"name":"syft:package:metadataType","value":"go-module-entry"},{"name":"syft:location:0:path","value":"/go.mod"},{"name":"syft:metadata:h1Digest","value":"h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8="}]},{"bom-ref":"pkg:golang/github.com/spf13/cobra@v1.8.0?package-id=81b06fbcbe87f414","type":"library","name":"github.com/spf13/cobra","version":"v1.8.0","cpe":"cpe:2.3:a:spf13:cobra:v1.8.0:*:*:*:*:*:*:*","purl":"pkg:golang/github.com/spf13/cobra@v1.8.0","properties":[{"name":"syft:package:foundBy","value":"go-module-file-cataloger"},{"name":"syft:package:language","value":"go"},{"name":"syft:package:type","value":"go-module"},{"name":"syft:package:metadataType","value":"go-module-entry"},{"name":"syft:location:0:path","value":"/go.mod"},{"name":"syft:metadata:h1Digest","value":"h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0="}]},{"bom-ref":"pkg:golang/github.com/spf13/pflag@v1.0.5?package-id=163f67459b76e534","type":"library","name":"github.com/spf13/pflag","version":"v1.0.5","cpe":"cpe:2.3:a:spf13:pflag:v1.0.5:*:*:*:*:*:*:*","purl":"pkg:golang/github.com/spf13/pflag@v1.0.5","properties":[{"name":"syft:package:foundBy","value":"go-module-file-cataloger"},{"name":"syft:package:language","value":"go"},{"name":"syft:package:type","value":"go-module"},{"name":"syft:package:metadataType","value":"go-module-entry"},{"name":"syft:location:0:path","value":"/go.mod"},{"name":"syft:metadata:h1Digest","value":"h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRPuqXzS0rkr6qZw="}]},{"bom-ref":"pkg:npm/lodash@4.17.20?package-id=ece3658907628a6b","type":"library","name":"lodash","version":"4.17.20","licenses":[{"license":{"id":"MIT"}}],"cpe":"cpe:2.3:a:lodash:lodash:4.17.20:*:*:*:*:node.js:*:*","purl":"pkg:npm/lodash@4.17.20","properties":[{"name":"syft:package:foundBy","value":"javascript-lock-cataloger"},{"name":"syft:package:language","value":"javascript"},{"name":"syft:package:type","value":"npm"},{"name":"syft:package:metadataType","value":"javascript-npm-package-lock-entry"},{"name":"syft:location:0:path","value":"/package-lock.json"}]},{"bom-ref":"pkg:composer/monolog/monolog@3.5.0?package-id=3abfdcdd00fd8d2f","type":"library","name":"monolog/monolog","version":"3.5.0","licenses":[{"license":{"id":"MIT"}}],"cpe":"cpe:2.3:a:monolog\\/monolog:monolog\\/monolog:3.5.0:*:*:*:*:*:*:*","purl":"pkg:composer/monolog/monolog@3.5.0","properties":[{"name":"syft:package:foundBy","value":"php-composer-lock-cataloger"},{"name":"syft:package:language","value":"php"},{"name":"syft:package:type","value":"php-composer"},{"name":"syft:package:metadataType","value":"php-composer-lock-entry"},{"name":"syft:location:0:path","value":"/composer.lock"}]},{"bom-ref":"pkg:apk/alpine/musl@1.2.4-r2?arch=x86_64&distro=alpine-3.19.1&package-id=9f0e3d80c35c10bc","type":"library","publisher":"Timo Teräs ","name":"musl","version":"1.2.4-r2","description":"the musl c library (libc) implementation","licenses":[{"license":{"id":"MIT"}}],"cpe":"cpe:2.3:a:musl-libc:musl:1.2.4-r2:*:*:*:*:*:*:*","purl":"pkg:apk/alpine/musl@1.2.4-r2?arch=x86_64&distro=alpine-3.19.1","externalReferences":[{"url":"https://musl.libc.org/","type":"distribution"}],"properties":[{"name":"syft:package:foundBy","value":"apk-db-cataloger"},{"name":"syft:package:type","value":"apk"},{"name":"syft:package:metadataType","value":"apk-db-entry"},{"name":"syft:cpe23","value":"cpe:2.3:a:musl_libc:musl:1.2.4-r2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:musl:musl:1.2.4-r2:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/lib/apk/db/installed"},{"name":"syft:metadata:gitCommitOfApkPort","value":"4b2f0d7b4e0e8a4b0e1b1b4d2b8a7a3d1a2f8e0c"},{"name":"syft:metadata:installedSize","value":"647168"},{"name":"syft:metadata:originPackage","value":"musl"},{"name":"syft:metadata:pullChecksum","value":"Q1N9CgYbNRm8SgkqbXqJhCAc6iqXo="},{"name":"syft:metadata:size","value":"407383"}]},{"bom-ref":"pkg:composer/symfony/console@v6.4.1?package-id=767524bf09da7da7","type":"library","name":"symfony/console","version":"v6.4.1","licenses":[{"license":{"id":"MIT"}}],"cpe":"cpe:2.3:a:symfony\\/console:symfony\\/console:v6.4.1:*:*:*:*:*:*:*","purl":"pkg:composer/symfony/console@v6.4.1","properties":[{"name":"syft:package:foundBy","value":"php-composer-lock-cataloger"},{"name":"syft:package:language","value":"php"},{"name":"syft:package:type","value":"php-composer"},{"name":"syft:package:metadataType","value":"php-composer-lock-entry"},{"name":"syft:location:0:path","value":"/composer.lock"}]},{"bom-ref":"pkg:npm/webapp@1.0.0?package-id=982a05090388c92d","type":"library","name":"webapp","version":"1.0.0","cpe":"cpe:2.3:a:webapp:webapp:1.0.0:*:*:*:*:*:*:*","purl":"pkg:npm/webapp@1.0.0","properties":[{"name":"syft:package:foundBy","value":"javascript-lock-cataloger"},{"name":"syft:package:language","value":"javascript"},{"name":"syft:package:type","value":"npm"},{"name":"syft:package:metadataType","value":"javascript-npm-package-lock-entry"},{"name":"syft:location:0:path","value":"/package-lock.json"}]},{"bom-ref":"os:alpine@3.19.1","type":"operating-system","name":"alpine","version":"3.19.1","description":"Alpine Linux v3.19","swid":{"tagId":"alpine","name":"alpine","version":"3.19.1"},"properties":[{"name":"syft:distro:extendedSupport","value":"false"},{"name":"syft:distro:id","value":"alpine"},{"name":"syft:distro:prettyName","value":"Alpine Linux v3.19"},{"name":"syft:distro:versionID","value":"3.19.1"}]},{"bom-ref":"c8bb62e6bded633e","type":"file","name":"/builds/acme/webapp/composer.lock","hashes":[{"alg":"SHA-1","content":"53c59ca459a8ff42918b5ed4fdcea1d8ba98432f"},{"alg":"SHA-256","content":"63738301cc7186a6645f00ae0a63ae6347e04552489f3cc70e92200463f5b4ac"}]},{"bom-ref":"3fc5a8d3d86e9790","type":"file","name":"/builds/acme/webapp/go.mod","hashes":[{"alg":"SHA-1","content":"c67c41265c415c41049b31f850b4497898eff1fd"},{"alg":"SHA-256","content":"f2303d4139e491238d8cb9a6c448e4c366b82bae3a646164465387df76e44f32"}]},{"bom-ref":"9f5aca292136191a","type":"file","name":"/builds/acme/webapp/lib/apk/db/installed","hashes":[{"alg":"SHA-1","content":"b2ebf78ea604d3a76359a262afd401ab966a36a1"},{"alg":"SHA-256","content":"d363cb96812dc6eb1d6f4c00879c2564cdbff238b6c1d0a37701d2adedd326e0"}]},{"bom-ref":"fd71c2238fc07657","type":"file","name":"/builds/acme/webapp/package-lock.json","hashes":[{"alg":"SHA-1","content":"fc26eadd11405354163eb0f53ffd2bb8cb56c5e7"},{"alg":"SHA-256","content":"69094786dd997053d1c7a81d4349bb5b608c08b576e67360f07cdc2726a041ea"}]}],"dependencies":[{"ref":"pkg:npm/webapp@1.0.0?package-id=982a05090388c92d","dependsOn":["pkg:npm/%40babel/core@7.23.0?package-id=7b01fd7a9ce63128","pkg:npm/lodash@4.17.20?package-id=ece3658907628a6b"]}]} diff --git a/internal/server/dashboard.go b/internal/server/dashboard.go index da866a6..ebcc333 100644 --- a/internal/server/dashboard.go +++ b/internal/server/dashboard.go @@ -15,6 +15,9 @@ type DashboardData struct { EnrichmentStats EnrichmentStatsView RecentPackages []PackageInfo PopularPackages []PackageInfo + // VulnerableInUse lists vulnerable package versions projects use, when + // the project inventory is enabled. + VulnerableInUse []VulnerableVersion } // DashboardStats contains cache statistics for the dashboard. @@ -68,6 +71,9 @@ type PackageShowData struct { Versions []database.Version Vulnerabilities []database.Vulnerability LicenseCategory string + // UsedBy lists the projects using each version, when the project + // inventory is enabled. + UsedBy []VersionUsage } // VersionShowData contains data for rendering the version show page. diff --git a/internal/server/layout.go b/internal/server/layout.go index 3a51876..1a63d2f 100644 --- a/internal/server/layout.go +++ b/internal/server/layout.go @@ -17,6 +17,8 @@ type Layout struct { CanonicalPath string // CacheDelete shows the controls for deleting cached artifacts. CacheDelete bool + // Projects shows the project inventory pages. + Projects bool } func (s *Server) layoutFor(r *http.Request) Layout { @@ -25,5 +27,6 @@ func (s *Server) layoutFor(r *http.Request) Layout { UIBaseURL: s.cfg.UIBaseURL, CanonicalPath: r.URL.Path, CacheDelete: s.cfg.CacheDelete, + Projects: s.cfg.Projects.Enabled, } } diff --git a/internal/server/projects_api.go b/internal/server/projects_api.go new file mode 100644 index 0000000..a007319 --- /dev/null +++ b/internal/server/projects_api.go @@ -0,0 +1,335 @@ +package server + +import ( + "errors" + "io" + "log/slog" + "net/http" + "net/url" + "strings" + "time" + + "github.com/git-pkgs/proxy/internal/database" + "github.com/git-pkgs/proxy/internal/inventory" + "github.com/git-pkgs/proxy/internal/packageurl" + "github.com/go-chi/chi/v5" +) + +// maxSBOMBytes caps an uploaded SBOM. syft's CycloneDX output for a large +// monorepo is a few MiB. +const maxSBOMBytes = 32 << 20 + +// maxRefLength caps the ref and branch query parameters of an upload. +const maxRefLength = 255 + +// sbomPathSuffix ends the upload path, /api/projects/{project}/sbom. +const sbomPathSuffix = "/sbom" + +// ProjectsAPIHandler serves /api/projects, where repositories upload the SBOM +// of their default branch so the proxy can show which of them use which +// package versions. +type ProjectsAPIHandler struct { + db *database.DB + logger *slog.Logger +} + +// NewProjectsAPIHandler creates the /api/projects handler. +func NewProjectsAPIHandler(db *database.DB, logger *slog.Logger) *ProjectsAPIHandler { + return &ProjectsAPIHandler{db: db, logger: logger} +} + +// ProjectUploadResponse summarizes an uploaded SBOM. +type ProjectUploadResponse struct { + Project string `json:"project"` + // Components is the number of package versions recorded. + Components int `json:"components"` + // Skipped is the number of SBOM entries without a package URL or of a + // package type the proxy doesn't serve. + Skipped int `json:"skipped"` +} + +// ProjectSummary describes a project in the inventory. +type ProjectSummary struct { + Name string `json:"name"` + Ref string `json:"ref,omitempty"` + Branch string `json:"branch,omitempty"` + UploadedAt time.Time `json:"uploaded_at"` + // ComponentCount is the number of package versions recorded. + ComponentCount int64 `json:"component_count"` + // Version is set when listing the projects using a package: the version + // of it this project uses. + Version string `json:"version,omitempty"` +} + +// ProjectListResponse lists projects. +type ProjectListResponse struct { + Projects []ProjectSummary `json:"projects"` +} + +// ProjectComponent is a package version a project uses. +type ProjectComponent struct { + Ecosystem string `json:"ecosystem"` + Name string `json:"name"` + Version string `json:"version"` + PURL string `json:"purl"` +} + +// ProjectResponse is a project with its components. +type ProjectResponse struct { + ProjectSummary + Components []ProjectComponent `json:"components"` +} + +// mountProjectAPI registers /api/projects when projects.enabled is set. +func (s *Server) mountProjectAPI(r chi.Router) { + if s.cfg.Projects.Enabled { + registerProjectRoutes(r, NewProjectsAPIHandler(s.db, s.logger), s.cfg.Projects.TokenExpanded()) + } +} + +// registerProjectRoutes mounts /api/projects. Reading is open like the rest +// of the API; uploading and deleting require the bearer token. +// +// A project name such as "group/sub/repo" is accepted both as path segments, +// /api/projects/group/sub/repo, and URL-encoded as one segment, +// /api/projects/group%2Fsub%2Frepo, which is how GitLab's CI_PROJECT_PATH +// usually ends up when a client escapes it. +func registerProjectRoutes(r chi.Router, h *ProjectsAPIHandler, token string) { + r.Get("/api/projects", h.HandleList) + r.Get("/api/projects/*", h.HandleGet) + r.Group(func(w chi.Router) { + w.Use(requireBearerToken(token)) + w.Put("/api/projects/*", h.HandleUpload) + w.Delete("/api/projects/*", h.HandleDelete) + }) +} + +// HandleUpload handles PUT /api/projects/{project}/sbom. +// @Summary Upload a project's SBOM +// @Description Replaces the package versions recorded for the project with the ones the SBOM lists. CycloneDX (JSON or XML) and SPDX (JSON or tag-value) are accepted, e.g. the output of `syft dir:. -o cyclonedx-json`. Entries without a package URL, or of a package type the proxy doesn't serve, are skipped. The project may be given as path segments or URL-encoded as one segment. +// @Tags projects +// @Accept json +// @Produce json +// @Security BearerAuth +// @Param project path string true "Project path, e.g. group/sub/repo" +// @Param ref query string false "Commit the SBOM was generated from" +// @Param branch query string false "Branch the SBOM was generated from" +// @Success 200 {object} ProjectUploadResponse +// @Failure 400 {object} ErrorResponse +// @Failure 401 {object} ErrorResponse +// @Failure 413 {object} ErrorResponse +// @Router /api/projects/{project}/sbom [put] +func (h *ProjectsAPIHandler) HandleUpload(w http.ResponseWriter, r *http.Request) { + path, err := projectPath(r) + if err != nil { + badRequest(w, err.Error()) + return + } + name, ok := strings.CutSuffix(path, sbomPathSuffix) + if !ok { + badRequest(w, "path must be /api/projects/{project}/sbom") + return + } + if err := inventory.ValidateProjectName(name); err != nil { + badRequest(w, err.Error()) + return + } + ref, branch := r.URL.Query().Get("ref"), r.URL.Query().Get("branch") + if len(ref) > maxRefLength || len(branch) > maxRefLength { + badRequest(w, "ref and branch must not be longer than 255 characters") + return + } + + data, err := io.ReadAll(http.MaxBytesReader(w, r.Body, maxSBOMBytes)) + var tooLarge *http.MaxBytesError + switch { + case errors.As(err, &tooLarge): + writeError(w, http.StatusRequestEntityTooLarge, ErrCodeBadRequest, "SBOM is larger than 32 MiB") + return + case err != nil: + badRequest(w, "failed to read request body") + return + case len(data) == 0: + badRequest(w, "request body is empty") + return + } + + components, skipped, err := inventory.Parse(data) + if err != nil { + badRequest(w, "request body is not a CycloneDX or SPDX SBOM: "+err.Error()) + return + } + records := make([]database.ProjectComponent, len(components)) + for i, c := range components { + records[i] = database.ProjectComponent{ + PackagePURL: c.PackagePURL, Ecosystem: c.Ecosystem, Name: c.Name, Version: c.Version, + } + } + if err := h.db.SaveProject(name, ref, branch, time.Now(), records); err != nil { + h.logger.Error("failed to save project", "project", name, "error", err) + internalError(w, "failed to save project") + return + } + h.logger.Info("recorded project SBOM", "project", name, "ref", ref, + "components", len(records), "skipped", skipped) + writeJSON(w, ProjectUploadResponse{Project: name, Components: len(records), Skipped: skipped}) +} + +// HandleDelete handles DELETE /api/projects/{project}. +// @Summary Delete a project +// @Description Removes the project and the package versions recorded for it. +// @Tags projects +// @Security BearerAuth +// @Param project path string true "Project path, e.g. group/sub/repo" +// @Success 204 +// @Failure 400 {object} ErrorResponse +// @Failure 401 {object} ErrorResponse +// @Failure 404 {object} ErrorResponse +// @Router /api/projects/{project} [delete] +func (h *ProjectsAPIHandler) HandleDelete(w http.ResponseWriter, r *http.Request) { + name, ok := validProjectPath(w, r) + if !ok { + return + } + deleted, err := h.db.DeleteProject(name) + switch { + case err != nil: + h.logger.Error("failed to delete project", "project", name, "error", err) + internalError(w, "failed to delete project") + case !deleted: + notFound(w, "project not found") + default: + h.logger.Info("deleted project", "project", name) + w.WriteHeader(http.StatusNoContent) + } +} + +// HandleList handles GET /api/projects. +// @Summary List projects +// @Description Lists the projects that uploaded an SBOM. With purl, lists the projects using that package, or only that version of it when the purl has one; each entry then carries the version the project uses. +// @Tags projects +// @Produce json +// @Param purl query string false "Package URL, e.g. pkg:npm/lodash or pkg:npm/lodash@4.17.20" +// @Success 200 {object} ProjectListResponse +// @Failure 400 {object} ErrorResponse +// @Router /api/projects [get] +func (h *ProjectsAPIHandler) HandleList(w http.ResponseWriter, r *http.Request) { + if query := r.URL.Query().Get("purl"); query != "" { + h.listUsages(w, query) + return + } + projects, err := h.db.ListProjects() + if err != nil { + h.logger.Error("failed to list projects", "error", err) + internalError(w, "failed to list projects") + return + } + resp := ProjectListResponse{Projects: make([]ProjectSummary, len(projects))} + for i, p := range projects { + resp.Projects[i] = projectSummary(p) + } + writeJSON(w, resp) +} + +func (h *ProjectsAPIHandler) listUsages(w http.ResponseWriter, query string) { + c, err := inventory.FromPURL(query) + if err != nil { + badRequest(w, "invalid purl: "+err.Error()) + return + } + usages, err := h.db.GetProjectsUsingPackage(c.PackagePURL, c.Version) + if err != nil { + h.logger.Error("failed to look up package usage", "purl", query, "error", err) + internalError(w, "failed to look up package usage") + return + } + resp := ProjectListResponse{Projects: make([]ProjectSummary, len(usages))} + for i, u := range usages { + resp.Projects[i] = projectSummary(u.Project) + resp.Projects[i].Version = u.Version + } + writeJSON(w, resp) +} + +// HandleGet handles GET /api/projects/{project}. +// @Summary Get a project +// @Description Returns the project with the package versions recorded for it. +// @Tags projects +// @Produce json +// @Param project path string true "Project path, e.g. group/sub/repo" +// @Success 200 {object} ProjectResponse +// @Failure 400 {object} ErrorResponse +// @Failure 404 {object} ErrorResponse +// @Router /api/projects/{project} [get] +func (h *ProjectsAPIHandler) HandleGet(w http.ResponseWriter, r *http.Request) { + name, ok := validProjectPath(w, r) + if !ok { + return + } + project, err := h.db.GetProject(name) + if err != nil { + h.logger.Error("failed to get project", "project", name, "error", err) + internalError(w, "failed to get project") + return + } + if project == nil { + notFound(w, "project not found") + return + } + components, err := h.db.GetProjectComponents(project.ID) + if err != nil { + h.logger.Error("failed to get project components", "project", name, "error", err) + internalError(w, "failed to get project components") + return + } + resp := ProjectResponse{ProjectSummary: projectSummary(*project), Components: make([]ProjectComponent, len(components))} + for i, c := range components { + resp.Components[i] = ProjectComponent{ + Ecosystem: c.Ecosystem, + Name: c.Name, + Version: c.Version, + PURL: packageurl.WithVersionString(c.PackagePURL, c.Version), + } + } + writeJSON(w, resp) +} + +func projectSummary(p database.Project) ProjectSummary { + return ProjectSummary{ + Name: p.Name, + Ref: p.Ref.String, + Branch: p.Branch.String, + UploadedAt: p.UploadedAt, + ComponentCount: p.Components, + } +} + +// projectPath returns the decoded wildcard of a /api/projects or +// /ui/projects route. chi hands it over still encoded when the request path +// had escapes its canonical form doesn't (wildcardIsEncoded), e.g. a project +// name sent as one segment with %2F. +func projectPath(r *http.Request) (string, error) { + wildcard := chi.URLParam(r, "*") + if !wildcardIsEncoded(r) { + return wildcard, nil + } + decoded, err := url.PathUnescape(wildcard) + if err != nil { + return "", errors.New("invalid escape in project path") + } + return decoded, nil +} + +// validProjectPath returns the project named by the route, answering 400 if +// it is not a valid project name. +func validProjectPath(w http.ResponseWriter, r *http.Request) (string, bool) { + name, err := projectPath(r) + if err == nil { + err = inventory.ValidateProjectName(name) + } + if err != nil { + badRequest(w, err.Error()) + return "", false + } + return name, true +} diff --git a/internal/server/projects_api_test.go b/internal/server/projects_api_test.go new file mode 100644 index 0000000..880a4cf --- /dev/null +++ b/internal/server/projects_api_test.go @@ -0,0 +1,354 @@ +package server + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "io" + "log/slog" + "net" + "net/http" + "net/http/httptest" + "net/url" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/git-pkgs/proxy/internal/config" + "github.com/git-pkgs/proxy/internal/database" + "github.com/go-chi/chi/v5" +) + +const testProjectsToken = "sbom-token" + +// syftSBOM is a CycloneDX document syft produced for a directory with an +// npm, Composer and Go lock file and an Alpine package database. It lists +// ten packages; five entries (the OS and the lock files) have no PURL. +func syftSBOM(t *testing.T) []byte { + t.Helper() + data, err := os.ReadFile("../inventory/testdata/syft-cyclonedx.json") + if err != nil { + t.Fatal(err) + } + return data +} + +type projectsTestServer struct { + server *Server + router http.Handler + db *database.DB +} + +// newProjectsTestServer routes the project API and the UI pages that show +// the inventory, with projects.enabled set. +func newProjectsTestServer(t *testing.T) projectsTestServer { + t.Helper() + db, err := database.Create(filepath.Join(t.TempDir(), "test.db")) + if err != nil { + t.Fatalf("database.Create: %v", err) + } + t.Cleanup(func() { _ = db.Close() }) + + logger := slog.New(slog.NewTextHandler(io.Discard, nil)) + cfg := &config.Config{Projects: config.ProjectsConfig{Enabled: true, Token: testProjectsToken}} + s := &Server{cfg: cfg, db: db, logger: logger, templates: &Templates{}} + + r := chi.NewRouter() + registerProjectRoutes(r, NewProjectsAPIHandler(db, logger), testProjectsToken) + r.Route("/ui", func(ui chi.Router) { + ui.Get("/", s.handleRoot) + ui.Get("/package/{ecosystem}/*", s.handlePackagePath) + s.mountProjectUI(ui) + }) + return projectsTestServer{server: s, router: r, db: db} +} + +func (ts projectsTestServer) do(t *testing.T, method, target, token string, body []byte) *httptest.ResponseRecorder { + t.Helper() + req := httptest.NewRequest(method, target, bytes.NewReader(body)) + if token != "" { + req.Header.Set("Authorization", "Bearer "+token) + } + rec := httptest.NewRecorder() + ts.router.ServeHTTP(rec, req) + return rec +} + +func (ts projectsTestServer) upload(t *testing.T, target string, body []byte) ProjectUploadResponse { + t.Helper() + rec := ts.do(t, http.MethodPut, target, testProjectsToken, body) + if rec.Code != http.StatusOK { + t.Fatalf("PUT %s = %d: %s", target, rec.Code, rec.Body) + } + var resp ProjectUploadResponse + if err := json.Unmarshal(rec.Body.Bytes(), &resp); err != nil { + t.Fatalf("decoding upload response: %v", err) + } + return resp +} + +func decodeJSON(t *testing.T, rec *httptest.ResponseRecorder, v any) { + t.Helper() + if rec.Code != http.StatusOK { + t.Fatalf("status = %d: %s", rec.Code, rec.Body) + } + if err := json.Unmarshal(rec.Body.Bytes(), v); err != nil { + t.Fatalf("decoding response: %v", err) + } +} + +func TestProjectUploadAndGet(t *testing.T) { + ts := newProjectsTestServer(t) + + resp := ts.upload(t, "/api/projects/acme/shop/web/sbom?ref=0123abcd&branch=main", syftSBOM(t)) + if resp != (ProjectUploadResponse{Project: "acme/shop/web", Components: 10, Skipped: 5}) { + t.Errorf("upload response = %+v", resp) + } + + // The same project, URL-encoded as one segment. + var project ProjectResponse + decodeJSON(t, ts.do(t, http.MethodGet, "/api/projects/acme%2Fshop%2Fweb", "", nil), &project) + if project.Name != "acme/shop/web" || project.Ref != "0123abcd" || project.Branch != "main" || project.ComponentCount != 10 { + t.Errorf("project = %+v", project.ProjectSummary) + } + var purls []string + for _, c := range project.Components { + purls = append(purls, c.PURL) + } + for _, want := range []string{"pkg:npm/%40babel/core@7.23.0", "pkg:composer/symfony/console@v6.4.1", "pkg:golang/github.com/spf13/cobra@v1.8.0", "pkg:apk/alpine/busybox@1.36.1-r15"} { + if !strings.Contains(strings.Join(purls, " "), want) { + t.Errorf("components %v do not include %s", purls, want) + } + } + + var list ProjectListResponse + decodeJSON(t, ts.do(t, http.MethodGet, "/api/projects", "", nil), &list) + if len(list.Projects) != 1 || list.Projects[0].Name != "acme/shop/web" || list.Projects[0].ComponentCount != 10 { + t.Errorf("list = %+v", list) + } +} + +func TestProjectUploadReplacesComponents(t *testing.T) { + ts := newProjectsTestServer(t) + ts.upload(t, "/api/projects/acme%2Fweb/sbom?ref=one", syftSBOM(t)) + resp := ts.upload(t, "/api/projects/acme%2Fweb/sbom", []byte(`{"bomFormat":"CycloneDX","specVersion":"1.5","components":[ + {"type":"library","name":"lodash","version":"4.17.21","purl":"pkg:npm/lodash@4.17.21"}]}`)) + if resp.Components != 1 || resp.Skipped != 0 { + t.Errorf("upload response = %+v", resp) + } + var project ProjectResponse + decodeJSON(t, ts.do(t, http.MethodGet, "/api/projects/acme/web", "", nil), &project) + if len(project.Components) != 1 || project.Components[0].PURL != "pkg:npm/lodash@4.17.21" || project.Ref != "" { + t.Errorf("project after second upload = %+v", project) + } +} + +func TestProjectLookupByPURL(t *testing.T) { + ts := newProjectsTestServer(t) + ts.upload(t, "/api/projects/acme/web/sbom", syftSBOM(t)) + ts.upload(t, "/api/projects/acme/api/sbom", []byte(`{"bomFormat":"CycloneDX","specVersion":"1.5","components":[ + {"type":"library","name":"lodash","version":"4.17.21","purl":"pkg:npm/lodash@4.17.21"}]}`)) + + tests := []struct { + purl string + want []string + }{ + {"pkg:npm/lodash", []string{"acme/api@4.17.21", "acme/web@4.17.20"}}, + {"pkg:npm/lodash@4.17.20", []string{"acme/web@4.17.20"}}, + {"pkg:npm/lodash@1.0.0", nil}, + {"pkg:npm/%40babel/core", []string{"acme/web@7.23.0"}}, + {"pkg:apk/alpine/musl@1.2.4-r2?arch=x86_64", []string{"acme/web@1.2.4-r2"}}, + } + for _, tt := range tests { + t.Run(tt.purl, func(t *testing.T) { + var list ProjectListResponse + decodeJSON(t, ts.do(t, http.MethodGet, "/api/projects?purl="+url.QueryEscape(tt.purl), "", nil), &list) + var got []string + for _, p := range list.Projects { + got = append(got, p.Name+"@"+p.Version) + } + if strings.Join(got, ",") != strings.Join(tt.want, ",") { + t.Errorf("projects = %v, want %v", got, tt.want) + } + }) + } + + for _, bad := range []string{"lodash", "pkg:github/actions/checkout@v4"} { + if rec := ts.do(t, http.MethodGet, "/api/projects?purl="+url.QueryEscape(bad), "", nil); rec.Code != http.StatusBadRequest { + t.Errorf("purl %q status = %d, want 400", bad, rec.Code) + } + } +} + +func TestProjectWritesRequireToken(t *testing.T) { + ts := newProjectsTestServer(t) + for _, token := range []string{"", "wrong-token"} { + if rec := ts.do(t, http.MethodPut, "/api/projects/acme/web/sbom", token, syftSBOM(t)); rec.Code != http.StatusUnauthorized { + t.Errorf("upload with token %q status = %d, want 401", token, rec.Code) + } + if rec := ts.do(t, http.MethodDelete, "/api/projects/acme/web", token, nil); rec.Code != http.StatusUnauthorized { + t.Errorf("delete with token %q status = %d, want 401", token, rec.Code) + } + } + if rec := ts.do(t, http.MethodGet, "/api/projects/acme/web", "", nil); rec.Code != http.StatusNotFound { + t.Errorf("project exists after unauthorized upload: status %d", rec.Code) + } +} + +func TestProjectDelete(t *testing.T) { + ts := newProjectsTestServer(t) + ts.upload(t, "/api/projects/acme/web/sbom", syftSBOM(t)) + if rec := ts.do(t, http.MethodDelete, "/api/projects/acme%2Fweb", testProjectsToken, nil); rec.Code != http.StatusNoContent { + t.Fatalf("delete status = %d: %s", rec.Code, rec.Body) + } + if rec := ts.do(t, http.MethodDelete, "/api/projects/acme/web", testProjectsToken, nil); rec.Code != http.StatusNotFound { + t.Errorf("second delete status = %d, want 404", rec.Code) + } + if rec := ts.do(t, http.MethodGet, "/api/projects/acme/web", "", nil); rec.Code != http.StatusNotFound { + t.Errorf("get after delete status = %d, want 404", rec.Code) + } +} + +func TestProjectPathsAreValidated(t *testing.T) { + ts := newProjectsTestServer(t) + uploads := []string{ + "/api/projects/sbom", + "/api/projects/acme/web", + "/api/projects/acme/../web/sbom", + "/api/projects/acme%2F..%2Fweb/sbom", + "/api/projects/..%2Fweb/sbom", + "/api/projects/acme//web/sbom", + "/api/projects/acme/./web/sbom", + "/api/projects/acme%20web/sbom", + "/api/projects/acme%2Fweb%3F/sbom", + "/api/projects/" + strings.Repeat("a", 256) + "/sbom", + } + for _, target := range uploads { + if rec := ts.do(t, http.MethodPut, target, testProjectsToken, syftSBOM(t)); rec.Code != http.StatusBadRequest { + t.Errorf("PUT %s status = %d, want 400", target, rec.Code) + } + } + for _, target := range []string{"/api/projects/", "/api/projects/acme/../web", "/api/projects/acme%2F..%2Fweb"} { + if rec := ts.do(t, http.MethodGet, target, "", nil); rec.Code != http.StatusBadRequest { + t.Errorf("GET %s status = %d, want 400", target, rec.Code) + } + if rec := ts.do(t, http.MethodDelete, target, testProjectsToken, nil); rec.Code != http.StatusBadRequest { + t.Errorf("DELETE %s status = %d, want 400", target, rec.Code) + } + } + if projects, err := ts.db.ListProjects(); err != nil || len(projects) != 0 { + t.Errorf("projects after invalid uploads = %v, %v", projects, err) + } +} + +func TestProjectUploadRejectsBadBodies(t *testing.T) { + ts := newProjectsTestServer(t) + tests := []struct { + name string + target string + body []byte + want int + }{ + {"empty", "/api/projects/acme/web/sbom", nil, http.StatusBadRequest}, + {"not an SBOM", "/api/projects/acme/web/sbom", []byte(`{"hello":"world"}`), http.StatusBadRequest}, + {"too large", "/api/projects/acme/web/sbom", make([]byte, maxSBOMBytes+1), http.StatusRequestEntityTooLarge}, + {"long ref", "/api/projects/acme/web/sbom?ref=" + strings.Repeat("a", maxRefLength+1), syftSBOM(t), http.StatusBadRequest}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if rec := ts.do(t, http.MethodPut, tt.target, testProjectsToken, tt.body); rec.Code != tt.want { + t.Errorf("status = %d, want %d: %s", rec.Code, tt.want, rec.Body) + } + }) + } +} + +// startProjectsServer runs the real server with the given projects config. +func startProjectsServer(t *testing.T, projects config.ProjectsConfig) string { + t.Helper() + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("reserving proxy address: %v", err) + } + t.Cleanup(func() { _ = listener.Close() }) + + tempDir := t.TempDir() + cfg := config.Default() + cfg.Listen = listener.Addr().String() + cfg.BaseURL = "http://" + cfg.Listen + cfg.Database.Path = filepath.Join(tempDir, "proxy.db") + cfg.Storage.URL = "file://" + filepath.Join(tempDir, "artifacts") + cfg.Projects = projects + if err := cfg.Validate(); err != nil { + t.Fatalf("validating config: %v", err) + } + + proxyServer, err := New(cfg, slog.New(slog.NewTextHandler(io.Discard, nil)), BuildInfo{Version: "test", Commit: "test"}) + if err != nil { + t.Fatalf("creating server: %v", err) + } + startErr := make(chan error, 1) + go func() { startErr <- proxyServer.Start(listener) }() + t.Cleanup(func() { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + _ = proxyServer.Shutdown(ctx) + if err := <-startErr; !errors.Is(err, http.ErrServerClosed) { + t.Errorf("server stopped with %v", err) + } + }) + return cfg.BaseURL +} + +func request(t *testing.T, method, url, token string, body []byte) (int, string) { + t.Helper() + req, err := http.NewRequest(method, url, bytes.NewReader(body)) + if err != nil { + t.Fatalf("building request: %v", err) + } + if token != "" { + req.Header.Set("Authorization", "Bearer "+token) + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatalf("%s %s: %v", method, url, err) + } + defer func() { _ = resp.Body.Close() }() + data, _ := io.ReadAll(resp.Body) + return resp.StatusCode, string(data) +} + +func TestProjectRoutesFollowConfig(t *testing.T) { + disabled := startProjectsServer(t, config.ProjectsConfig{}) + for _, path := range []string{"/api/projects", "/api/projects/acme/web", "/ui/projects"} { + if status, _ := request(t, http.MethodGet, disabled+path, "", nil); status != http.StatusNotFound { + t.Errorf("GET %s with projects disabled = %d, want 404", path, status) + } + } + if status, _ := request(t, http.MethodPut, disabled+"/api/projects/acme/web/sbom", testProjectsToken, syftSBOM(t)); status != http.StatusNotFound { + t.Errorf("upload with projects disabled = %d, want 404", status) + } + if _, body := request(t, http.MethodGet, disabled+"/ui/", "", nil); strings.Contains(body, `href="/ui/projects"`) { + t.Error("navigation links to projects while they are disabled") + } + + enabled := startProjectsServer(t, config.ProjectsConfig{Enabled: true, Token: testProjectsToken, MaxAge: "90d"}) + if status, _ := request(t, http.MethodPut, enabled+"/api/projects/acme%2Fweb/sbom", "", syftSBOM(t)); status != http.StatusUnauthorized { + t.Errorf("upload without token = %d, want 401", status) + } + if status, body := request(t, http.MethodPut, enabled+"/api/projects/acme%2Fweb/sbom", testProjectsToken, syftSBOM(t)); status != http.StatusOK { + t.Fatalf("upload = %d: %s", status, body) + } + if status, body := request(t, http.MethodGet, enabled+"/api/projects/acme/web", "", nil); status != http.StatusOK || !strings.Contains(body, `"name":"acme/web"`) { + t.Errorf("GET project = %d: %s", status, body) + } + if status, body := request(t, http.MethodGet, enabled+"/ui/projects", "", nil); status != http.StatusOK || !strings.Contains(body, `href="/ui/projects/acme/web"`) { + t.Errorf("projects page = %d, missing the project link", status) + } + if _, body := request(t, http.MethodGet, enabled+"/ui/", "", nil); !strings.Contains(body, `href="/ui/projects"`) { + t.Error("navigation does not link to projects") + } +} diff --git a/internal/server/projects_prune.go b/internal/server/projects_prune.go new file mode 100644 index 0000000..b9ba936 --- /dev/null +++ b/internal/server/projects_prune.go @@ -0,0 +1,43 @@ +package server + +import ( + "context" + "time" +) + +// projectPruneInterval is how often projects older than projects.max_age +// are removed. +const projectPruneInterval = time.Hour + +// startProjectPruning removes projects that have not uploaded an SBOM for +// projects.max_age, once at startup and then every projectPruneInterval. +func (s *Server) startProjectPruning(ctx context.Context) { + maxAge := s.cfg.Projects.MaxAgeDuration() + if !s.cfg.Projects.Enabled || maxAge <= 0 { + return + } + prune := func() { + s.runExclusive(ctx, "project-pruning", func() { + pruned, err := s.db.PruneProjects(time.Now().Add(-maxAge)) + if err != nil { + s.logger.Warn("pruning projects failed", "error", err) + } + if pruned > 0 { + s.logger.Info("pruned projects", "count", pruned, "max_age", maxAge) + } + }) + } + s.background.Go(func() { + prune() + ticker := time.NewTicker(projectPruneInterval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + prune() + } + } + }) +} diff --git a/internal/server/projects_prune_test.go b/internal/server/projects_prune_test.go new file mode 100644 index 0000000..0f74d55 --- /dev/null +++ b/internal/server/projects_prune_test.go @@ -0,0 +1,48 @@ +package server + +import ( + "context" + "testing" + "time" + + "github.com/git-pkgs/proxy/internal/config" +) + +func TestProjectPruningRemovesStaleProjects(t *testing.T) { + ts := newProjectsTestServer(t) + ts.server.cfg.Projects = config.ProjectsConfig{Enabled: true, Token: testProjectsToken, MaxAge: "30d"} + if err := ts.server.cfg.Projects.Validate(); err != nil { + t.Fatal(err) + } + if err := ts.db.SaveProject("acme/old", "", "", time.Now().Add(-31*24*time.Hour), nil); err != nil { + t.Fatal(err) + } + if err := ts.db.SaveProject("acme/new", "", "", time.Now(), nil); err != nil { + t.Fatal(err) + } + + ctx, cancel := context.WithCancel(context.Background()) + ts.server.startProjectPruning(ctx) + defer func() { + cancel() + ts.server.background.Wait() + }() + + deadline := time.Now().Add(5 * time.Second) + for { + old, err := ts.db.GetProject("acme/old") + if err != nil { + t.Fatal(err) + } + if old == nil { + break + } + if time.Now().After(deadline) { + t.Fatal("acme/old was not pruned") + } + time.Sleep(10 * time.Millisecond) + } + if current, err := ts.db.GetProject("acme/new"); err != nil || current == nil { + t.Errorf("acme/new was pruned: %v, %v", current, err) + } +} diff --git a/internal/server/projects_ui.go b/internal/server/projects_ui.go new file mode 100644 index 0000000..76ae8bd --- /dev/null +++ b/internal/server/projects_ui.go @@ -0,0 +1,287 @@ +package server + +import ( + "cmp" + "net/http" + "net/url" + "slices" + + "github.com/git-pkgs/proxy/internal/database" + "github.com/git-pkgs/proxy/internal/inventory" + "github.com/go-chi/chi/v5" +) + +// ProjectsListData contains data for rendering the projects list page. +type ProjectsListData struct { + Layout + Projects []ProjectRow +} + +// ProjectRow is a project in the UI. +type ProjectRow struct { + Name string + Ref string + Branch string + UploadedAt string + Components int64 +} + +// ProjectShowData contains data for rendering a project's page. +type ProjectShowData struct { + Layout + Project ProjectRow + Components []ProjectComponentRow + Vulnerable int + Outdated int +} + +// ProjectComponentRow is a package version a project uses, with what the +// proxy knows about it. +type ProjectComponentRow struct { + Ecosystem string + Name string + Version string + // Cached is set when the proxy has a package record, so the package + // page exists; CachedEcosystem and CachedName address it. + Cached bool + CachedEcosystem string + CachedName string + LatestVersion string + Outdated bool + VulnCount int + Severity string +} + +// VersionUsage lists the projects using one version of a package. +type VersionUsage struct { + Version string + EscapedVersion string + // Cached is set when the proxy has a record of the version. + Cached bool + Projects []string +} + +// VulnerableVersion is a package version with known vulnerabilities that +// projects use. +type VulnerableVersion struct { + Ecosystem string + Name string + Version string + Severity string + VulnCount int + Projects []string +} + +// mountProjectUI registers the project inventory pages when +// projects.enabled is set. +func (s *Server) mountProjectUI(ui chi.Router) { + if !s.cfg.Projects.Enabled { + return + } + ui.Get("/projects", s.handleProjectsList) + ui.Get("/projects/*", s.handleProjectShow) +} + +func (s *Server) handleProjectsList(w http.ResponseWriter, r *http.Request) { + projects, err := s.db.ListProjects() + if err != nil { + s.logger.Error("failed to list projects", "error", err) + http.Error(w, "failed to list projects", http.StatusInternalServerError) + return + } + data := ProjectsListData{Layout: s.layoutFor(r), Projects: make([]ProjectRow, len(projects))} + for i, p := range projects { + data.Projects[i] = projectRow(p) + } + if err := s.templates.Render(w, "projects_list", data); err != nil { + s.logger.Error("failed to render projects list", "error", err) + } +} + +func (s *Server) handleProjectShow(w http.ResponseWriter, r *http.Request) { + name, err := projectPath(r) + if err == nil { + err = inventory.ValidateProjectName(name) + } + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + project, err := s.db.GetProject(name) + if err != nil { + s.logger.Error("failed to get project", "project", name, "error", err) + http.Error(w, "failed to get project", http.StatusInternalServerError) + return + } + if project == nil { + http.Error(w, "project not found", http.StatusNotFound) + return + } + components, err := s.db.GetProjectComponents(project.ID) + if err != nil { + s.logger.Error("failed to get project components", "project", name, "error", err) + components = nil + } + vulns, err := s.db.GetComponentVulnerabilities(name) + if err != nil { + s.logger.Error("failed to get project vulnerabilities", "project", name, "error", err) + } + affecting := affectingVulnerabilities(vulns) + + data := ProjectShowData{Layout: s.layoutFor(r), Project: projectRow(*project)} + for _, c := range components { + row := ProjectComponentRow{ + Ecosystem: c.Ecosystem, + Name: c.Name, + Version: c.Version, + Cached: c.CachedName.Valid, + CachedEcosystem: c.CachedEcosystem.String, + CachedName: c.CachedName.String, + LatestVersion: c.LatestVersion.String, + Outdated: inventory.Outdated(c.Ecosystem, c.Version, c.LatestVersion.String), + } + if v, ok := affecting[componentKey{c.PackagePURL, c.Version}]; ok { + row.VulnCount, row.Severity = len(v.ids), v.severity + data.Vulnerable++ + } + if row.Outdated { + data.Outdated++ + } + data.Components = append(data.Components, row) + } + if err := s.templates.Render(w, "project_show", data); err != nil { + s.logger.Error("failed to render project page", "error", err) + } +} + +func projectRow(p database.Project) ProjectRow { + return ProjectRow{ + Name: p.Name, + Ref: p.Ref.String, + Branch: p.Branch.String, + UploadedAt: formatTimeAgo(p.UploadedAt), + Components: p.Components, + } +} + +// packageUsedBy lists the projects using each version of pkg, highest +// version first, or nothing when the project inventory is disabled. versions +// are the versions the proxy has records of; a version only projects know of +// is listed too. +func (s *Server) packageUsedBy(pkg *database.Package, versions []database.Version) []VersionUsage { + if !s.cfg.Projects.Enabled { + return nil + } + usages, err := s.db.GetProjectsUsingPackage(pkg.PURL, "") + if err != nil { + s.logger.Error("failed to get projects using package", "purl", pkg.PURL, "error", err) + return nil + } + known := make(map[string]bool, len(versions)) + for i := range versions { + known[versions[i].Version()] = true + } + byVersion := map[string]*VersionUsage{} + var result []*VersionUsage + for _, u := range usages { + vu, ok := byVersion[u.Version] + if !ok { + vu = &VersionUsage{Version: u.Version, EscapedVersion: url.PathEscape(u.Version), Cached: known[u.Version]} + byVersion[u.Version] = vu + result = append(result, vu) + } + vu.Projects = append(vu.Projects, u.Name) + } + slices.SortFunc(result, func(a, b *VersionUsage) int { + return inventory.CompareVersions(pkg.Ecosystem, b.Version, a.Version) + }) + out := make([]VersionUsage, len(result)) + for i, vu := range result { + out[i] = *vu + } + return out +} + +// vulnerableVersionsInUse returns the package versions projects use that +// have vulnerabilities, most severe and most used first, at most limit. It +// returns nothing when the project inventory is disabled. +func (s *Server) vulnerableVersionsInUse(limit int) []VulnerableVersion { + if !s.cfg.Projects.Enabled { + return nil + } + vulns, err := s.db.GetComponentVulnerabilities("") + if err != nil { + s.logger.Error("failed to get vulnerable versions in use", "error", err) + return nil + } + affecting := affectingVulnerabilities(vulns) + result := make([]VulnerableVersion, 0, len(affecting)) + for _, v := range affecting { + projects := make([]string, 0, len(v.projects)) + for p := range v.projects { + projects = append(projects, p) + } + slices.Sort(projects) + result = append(result, VulnerableVersion{ + Ecosystem: v.ecosystem, Name: v.name, Version: v.version, + Severity: v.severity, VulnCount: len(v.ids), Projects: projects, + }) + } + slices.SortFunc(result, func(a, b VulnerableVersion) int { + return cmp.Or( + cmp.Compare(severityRank(b.Severity), severityRank(a.Severity)), + cmp.Compare(len(b.Projects), len(a.Projects)), + cmp.Compare(a.Name, b.Name), + cmp.Compare(a.Version, b.Version), + ) + }) + if len(result) > limit { + result = result[:limit] + } + return result +} + +type componentKey struct{ packagePURL, version string } + +// affectingVersion collects the vulnerabilities affecting one package +// version, and the projects using it. +type affectingVersion struct { + ecosystem, name, version string + severity string + ids map[string]bool + projects map[string]bool +} + +// affectingVulnerabilities keeps the vulnerabilities that affect the version +// each component uses, grouped by package version. +func affectingVulnerabilities(vulns []database.ComponentVulnerability) map[componentKey]*affectingVersion { + out := map[componentKey]*affectingVersion{} + for _, v := range vulns { + if !inventory.Affected(v.Ecosystem, v.Version, v.FixedVersion.String) { + continue + } + key := componentKey{v.PackagePURL, v.Version} + a, ok := out[key] + if !ok { + a = &affectingVersion{ + ecosystem: v.CachedEcosystem, name: v.CachedName, version: v.Version, + ids: map[string]bool{}, projects: map[string]bool{}, + } + out[key] = a + } + a.ids[v.VulnID] = true + a.projects[v.Project] = true + if severityRank(v.Severity.String) > severityRank(a.severity) { + a.severity = v.Severity.String + } + } + return out +} + +// severityOrder lists severities from least to most severe. +var severityOrder = []string{"low", "medium", "high", "critical"} + +// severityRank orders severities; unknown ones rank below "low". +func severityRank(severity string) int { + return slices.Index(severityOrder, severity) + 1 +} diff --git a/internal/server/projects_ui_test.go b/internal/server/projects_ui_test.go new file mode 100644 index 0000000..18fc5c1 --- /dev/null +++ b/internal/server/projects_ui_test.go @@ -0,0 +1,137 @@ +package server + +import ( + "database/sql" + "net/http" + "strings" + "testing" + + "github.com/git-pkgs/proxy/internal/database" +) + +// seedLodash records lodash with 4.17.21 as its latest and only cached +// version, and a vulnerability fixed in 4.17.21. The syft fixture uses +// 4.17.20. +func seedLodash(t *testing.T, db *database.DB, withVuln bool) { + t.Helper() + if err := db.UpsertPackage(&database.Package{ + PURL: "pkg:npm/lodash", Ecosystem: "npm", Name: "lodash", + LatestVersion: sql.NullString{String: "4.17.21", Valid: true}, + }); err != nil { + t.Fatal(err) + } + if err := db.UpsertVersion(&database.Version{PURL: "pkg:npm/lodash@4.17.21", PackagePURL: "pkg:npm/lodash"}); err != nil { + t.Fatal(err) + } + if !withVuln { + return + } + if err := db.UpsertVulnerability(&database.Vulnerability{ + VulnID: "GHSA-35jh-r3h4-6jhm", Ecosystem: "npm", PackageName: "lodash", + Severity: sql.NullString{String: "high", Valid: true}, + FixedVersion: sql.NullString{String: "4.17.21", Valid: true}, + }); err != nil { + t.Fatal(err) + } +} + +func (ts projectsTestServer) page(t *testing.T, target string) string { + t.Helper() + rec := ts.do(t, http.MethodGet, target, "", nil) + if rec.Code != http.StatusOK { + t.Fatalf("GET %s = %d: %s", target, rec.Code, rec.Body) + } + return rec.Body.String() +} + +func assertContains(t *testing.T, page string, want ...string) { + t.Helper() + for _, w := range want { + if !strings.Contains(page, w) { + t.Errorf("page does not contain %q", w) + } + } +} + +func TestProjectPageShowsComponents(t *testing.T) { + ts := newProjectsTestServer(t) + seedLodash(t, ts.db, true) + ts.upload(t, "/api/projects/acme/web/sbom?branch=main", syftSBOM(t)) + + page := ts.page(t, "/ui/projects/acme/web") + assertContains(t, page, + "acme/web", + `href="/ui/package/npm/lodash"`, // cached, so linked + "outdated, latest 4.17.21", + "1 vulns", + "1 vulnerable", + "symfony/console", // not cached: listed without a link + "not cached", + ) + if strings.Contains(page, `href="/ui/package/composer/symfony/console"`) { + t.Error("links to the package page of a package the proxy has no record of") + } + + // Encoded as one segment, too. + assertContains(t, ts.page(t, "/ui/projects/acme%2Fweb"), "symfony/console") + + if rec := ts.do(t, http.MethodGet, "/ui/projects/acme/other", "", nil); rec.Code != http.StatusNotFound { + t.Errorf("unknown project status = %d, want 404", rec.Code) + } + if rec := ts.do(t, http.MethodGet, "/ui/projects/acme/../web", "", nil); rec.Code != http.StatusBadRequest { + t.Errorf("invalid project status = %d, want 400", rec.Code) + } + + list := ts.page(t, "/ui/projects") + assertContains(t, list, `href="/ui/projects/acme/web"`, "10 packages") +} + +func TestPackagePageShowsUsedBy(t *testing.T) { + ts := newProjectsTestServer(t) + seedLodash(t, ts.db, false) + ts.upload(t, "/api/projects/acme/web/sbom", syftSBOM(t)) + ts.upload(t, "/api/projects/acme/api/sbom", []byte(`{"bomFormat":"CycloneDX","specVersion":"1.5","components":[ + {"type":"library","name":"lodash","version":"4.17.21","purl":"pkg:npm/lodash@4.17.21"}]}`)) + + page := ts.page(t, "/ui/package/npm/lodash") + assertContains(t, page, + "Used by", + `href="/ui/package/npm/lodash/4.17.21"`, + `href="/ui/projects/acme/api"`, + `href="/ui/projects/acme/web"`, + ) + // 4.17.20 is used but not cached. + start := strings.Index(page, "Used by") + if start < 0 { + t.Fatal("no Used by section") + } + used := page[start:] + if i := strings.Index(used, "4.17.20"); i < 0 || !strings.Contains(used[i:], "not cached") { + t.Error("the uncached version 4.17.20 is not listed as not cached") + } + if strings.Index(used, "4.17.21") > strings.Index(used, "4.17.20") { + t.Error("versions are not listed highest first") + } +} + +func TestDashboardShowsVulnerableVersionsInUse(t *testing.T) { + ts := newProjectsTestServer(t) + seedLodash(t, ts.db, false) + ts.upload(t, "/api/projects/acme/web/sbom", syftSBOM(t)) + + if page := ts.page(t, "/ui/"); strings.Contains(page, "Vulnerable versions in use") { + t.Error("dashboard shows the vulnerable versions card without vulnerabilities") + } + assertContains(t, ts.page(t, "/ui/"), `href="/ui/projects"`) + + seedLodash(t, ts.db, true) + page := ts.page(t, "/ui/") + assertContains(t, page, "Vulnerable versions in use", "@4.17.20", `href="/ui/projects/acme/web"`, ">high<") + + // A project on the fixed version doesn't count. + ts.upload(t, "/api/projects/acme/web/sbom", []byte(`{"bomFormat":"CycloneDX","specVersion":"1.5","components":[ + {"type":"library","name":"lodash","version":"4.17.21","purl":"pkg:npm/lodash@4.17.21"}]}`)) + if page := ts.page(t, "/ui/"); strings.Contains(page, "Vulnerable versions in use") { + t.Error("dashboard lists a version the vulnerability is fixed in") + } +} diff --git a/internal/server/server.go b/internal/server/server.go index 12619a6..d96b9dc 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -39,6 +39,7 @@ // - /ui/packages - List all cached packages // - /ui/search - Search packages // - /ui/package/... - Package and version detail pages +// - /ui/projects/... - Project inventory (when projects.enabled) // - /ui/api/browse/... - Archive browsing (used by the UI) // - /ui/api/compare/... - Archive diffing (used by the UI) // @@ -50,6 +51,12 @@ // - POST /api/outdated - Check outdated packages // - POST /api/bulk - Bulk package lookup // - GET /api/packages - List cached packages (JSON) +// +// Project inventory endpoints (when projects.enabled): +// - PUT /api/projects/{project}/sbom - Upload a project's SBOM (token) +// - DELETE /api/projects/{project} - Delete a project (token) +// - GET /api/projects - List projects, or those using ?purl= +// - GET /api/projects/{project} - A project's package versions package server import ( @@ -357,6 +364,7 @@ func (s *Server) serve(listener net.Listener) error { if s.cfg.CacheDelete { ui.Delete("/api/cache/{ecosystem}/*", s.handleCacheDelete) } + s.mountProjectUI(ui) }) r.Get("/", func(w http.ResponseWriter, r *http.Request) { http.Redirect(w, r, "/ui/", http.StatusFound) @@ -404,6 +412,10 @@ func (s *Server) serve(listener net.Listener) error { registerHostedRoutes(r, hostedAPI, s.cfg.Hosted.TokenExpanded()) } + // Project inventory endpoints (opt-in via projects.enabled or PROXY_PROJECTS_ENABLED) + s.mountProjectAPI(r) + s.startProjectPruning(bgCtx) + s.http = &http.Server{ Addr: s.cfg.Listen, Handler: r, @@ -833,6 +845,7 @@ func (s *Server) handleRoot(w http.ResponseWriter, r *http.Request) { }, EnrichmentStats: enrichmentStatsView(enrichStats), } + data.VulnerableInUse = s.vulnerableVersionsInUse(dashboardTopN) for _, p := range popular { pkgInfo := PackageInfo{ @@ -1170,6 +1183,7 @@ func (s *Server) showPackage(w http.ResponseWriter, r *http.Request, ecosystem, Vulnerabilities: vulns, LicenseCategory: categorizeLicense(pkg.License), } + data.UsedBy = s.packageUsedBy(pkg, versions) if err := s.templates.Render(w, "package_show", data); err != nil { s.logger.Error("failed to render package show", "error", err) diff --git a/internal/server/templates/components/severity_badge.html b/internal/server/templates/components/severity_badge.html new file mode 100644 index 0000000..a060ed2 --- /dev/null +++ b/internal/server/templates/components/severity_badge.html @@ -0,0 +1,3 @@ +{{define "severity_badge"}} +{{if .}}{{.}}{{else}}unknown{{end}} +{{end}} diff --git a/internal/server/templates/layout/header.html b/internal/server/templates/layout/header.html index 560fe2b..8b4d94f 100644 --- a/internal/server/templates/layout/header.html +++ b/internal/server/templates/layout/header.html @@ -54,6 +54,7 @@ {{end}} {{define "nav_links"}} +{{if .Projects}}Projects{{end}} Analytics Install Health diff --git a/internal/server/templates/pages/dashboard.html b/internal/server/templates/pages/dashboard.html index e48e205..ff9bc1b 100644 --- a/internal/server/templates/pages/dashboard.html +++ b/internal/server/templates/pages/dashboard.html @@ -25,6 +25,33 @@ {{template "security_overview" .EnrichmentStats}} {{end}} +{{if .VulnerableInUse}} +
+
+

Vulnerable versions in use

+
+
+ {{range .VulnerableInUse}} +
+
+ {{template "ecosystem_badge" .Ecosystem}} + {{.Name}} + @{{.Version}} + {{template "severity_badge" .Severity}} + {{.VulnCount}} vulns +
+
+ {{range .Projects}}{{.}}{{end}} +
+
+ {{end}} +
+
+ See all projects → +
+
+{{end}} +
diff --git a/internal/server/templates/pages/package_show.html b/internal/server/templates/pages/package_show.html index a2724a8..4c5a0fc 100644 --- a/internal/server/templates/pages/package_show.html +++ b/internal/server/templates/pages/package_show.html @@ -49,6 +49,34 @@ {{end}}
+{{if .UsedBy}} +
+
+

Used by

+

Projects whose latest SBOM lists this package

+
+
+ {{range .UsedBy}} +
+
+ {{if .Cached}} + {{if .Version}}{{.Version}}{{else}}unknown version{{end}} + {{else}} + {{if .Version}}{{.Version}}{{else}}unknown version{{end}} + not cached + {{end}} +
+
+ {{range .Projects}} + {{.}} + {{end}} +
+
+ {{end}} +
+
+{{end}} + {{if .Versions}}
diff --git a/internal/server/templates/pages/project_show.html b/internal/server/templates/pages/project_show.html new file mode 100644 index 0000000..0125f9c --- /dev/null +++ b/internal/server/templates/pages/project_show.html @@ -0,0 +1,53 @@ +{{define "title"}}{{.Project.Name}} - git-pkgs proxy{{end}} + +{{define "content"}} +
+ ← Projects +

{{.Project.Name}}

+

+ {{.Project.Components}} packages, uploaded {{.Project.UploadedAt}}{{if .Project.Branch}} from {{.Project.Branch}}{{end}}{{if .Project.Ref}} at {{.Project.Ref}}{{end}} +

+ {{if or .Vulnerable .Outdated}} +
+ {{if .Vulnerable}}{{.Vulnerable}} vulnerable{{end}} + {{if .Outdated}}{{.Outdated}} outdated{{end}} +
+ {{end}} +
+ +{{if .Components}} +
+
+ {{range .Components}} +
+
+ {{template "ecosystem_badge" .Ecosystem}} + {{if .Cached}} + {{.Name}} + {{else}} + {{.Name}} + {{end}} + {{.Version}} +
+
+ {{if .VulnCount}} + {{template "severity_badge" .Severity}} + {{.VulnCount}} vulns + {{end}} + {{if .Outdated}} + outdated, latest {{.LatestVersion}} + {{end}} + {{if not .Cached}} + not cached + {{end}} +
+
+ {{end}} +
+
+{{else}} +
+

The latest SBOM lists no packages the proxy serves

+
+{{end}} +{{end}} diff --git a/internal/server/templates/pages/projects_list.html b/internal/server/templates/pages/projects_list.html new file mode 100644 index 0000000..f5ce62d --- /dev/null +++ b/internal/server/templates/pages/projects_list.html @@ -0,0 +1,33 @@ +{{define "title"}}Projects - git-pkgs proxy{{end}} + +{{define "content"}} +
+

Projects

+

{{len .Projects}} projects, as of their latest SBOM upload

+
+ +{{if .Projects}} +
+
+ {{range .Projects}} +
+
+ {{.Name}} + {{if or .Branch .Ref}} +
{{.Branch}}{{if and .Branch .Ref}} @ {{end}}{{.Ref}}
+ {{end}} +
+
+ {{.Components}} packages + {{.UploadedAt}} +
+
+ {{end}} +
+
+{{else}} +
+

No project has uploaded an SBOM yet

+
+{{end}} +{{end}} From 895f22a601b47f31539fa2a9d57e05a2da57b523 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Tue, 27 Oct 2026 19:43:43 +0100 Subject: [PATCH 467/633] Skip the project's own packages in uploaded SBOMs A syft SBOM of a repository lists the repository itself: the module in go.mod and the root of package-lock.json, both without a version or with the version from package.json, and the module its Go binaries were built from. The project page showed them as dependencies. Entries without a version are now skipped, as are lock-file roots (lock entries that depend on others while nothing depends on them) and modules syft marks as a binary's main module. --- docs/configuration.md | 4 +- internal/inventory/inventory.go | 67 ++++++++++++++++++++++++++-- internal/inventory/inventory_test.go | 37 +++++++++++++-- internal/server/projects_api_test.go | 6 +-- internal/server/projects_ui_test.go | 2 +- 5 files changed, 103 insertions(+), 13 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index c5c6f7a..3a5579d 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1545,10 +1545,10 @@ curl -X PUT --data-binary @sbom.json \ The response counts the recorded package versions and the skipped entries: ```json -{"project":"acme/shop/web","components":10,"skipped":5} +{"project":"acme/shop/web","components":9,"skipped":6} ``` -Entries are recorded by their package URL. Entries without one (files, the operating system) and package types the proxy doesn't serve (such as `pkg:github`) are skipped. Duplicates are recorded once. +Entries are recorded by their package URL. Entries without one (files, the operating system), package types the proxy doesn't serve (such as `pkg:github`) and entries without a version are skipped. So is the project's own package: the root of a lock file such as `package-lock.json`, and with syft, the Go module its binaries were built from. Duplicates are recorded once. A GitLab CI job on the default branch could look like this: diff --git a/internal/inventory/inventory.go b/internal/inventory/inventory.go index 954d46e..9df6ce4 100644 --- a/internal/inventory/inventory.go +++ b/internal/inventory/inventory.go @@ -91,18 +91,33 @@ func FromPURL(s string) (Component, error) { return Component{Ecosystem: ecosystem, Name: name, Version: p.Version, PackagePURL: packagePURL}, nil } +// syftMainModule is the property syft sets on modules found in a Go binary, +// naming the module the binary was built from. +const syftMainModule = "syft:metadata:mainModule" + +// syftMetadataType is the property naming where syft read a package from. +const syftMetadataType = "syft:package:metadataType" + +// unknownVersion is what syft gives a package without a resolved version, +// such as the project's own module in go.mod or the root of a lock file. +const unknownVersion = "UNKNOWN" + // Parse reads a CycloneDX (JSON or XML) or SPDX SBOM and returns the package -// versions it lists, without duplicates. Packages without a package URL, or -// of a type the proxy doesn't serve, are counted in skipped. +// versions it lists, without duplicates. Packages without a package URL, of a +// type the proxy doesn't serve, or without a version are counted in skipped. +// So are the project's own packages: Go modules its binaries were built from, +// and the root of a lock file. func Parse(data []byte) (components []Component, skipped int, err error) { doc, err := sbom.Parse(data) if err != nil { return nil, 0, err } + ownModules := mainModules(doc.Packages) + lockRoots := lockFileRoots(doc) seen := make(map[[2]string]bool, len(doc.Packages)) for _, pkg := range doc.Packages { c, ok := componentOf(pkg) - if !ok { + if !ok || lockRoots[pkg.ID] || (c.Ecosystem == "golang" && ownModules[c.Name]) { skipped++ continue } @@ -128,9 +143,55 @@ func componentOf(pkg sbom.Package) (Component, bool) { if c.Version == "" { c.Version = pkg.Version } + // A package without a version is the project itself (the module in + // go.mod, the root of package-lock.json), not something it depends on. + if c.Version == "" || c.Version == unknownVersion { + return Component{}, false + } return c, true } +// lockFileRoots returns the IDs of packages read from a lock file that depend +// on others while nothing depends on them: the project's own entry, such as +// the root of package-lock.json, which carries the version from package.json. +func lockFileRoots(doc *sbom.SBOM) map[string]bool { + dependents := map[string]bool{} + dependencies := map[string]bool{} + for _, r := range doc.Relationships { + if r.Type != sbom.RelDependsOn { + continue + } + dependencies[r.SourceID] = true + dependents[r.TargetID] = true + } + roots := map[string]bool{} + for _, pkg := range doc.Packages { + if pkg.ID == "" || !dependencies[pkg.ID] || dependents[pkg.ID] { + continue + } + for _, prop := range pkg.Properties { + if prop.Name == syftMetadataType && strings.Contains(prop.Value, "lock") { + roots[pkg.ID] = true + } + } + } + return roots +} + +// mainModules returns the Go modules the SBOM's binaries were built from, +// which are the project's own. +func mainModules(pkgs []sbom.Package) map[string]bool { + own := map[string]bool{} + for _, pkg := range pkgs { + for _, prop := range pkg.Properties { + if prop.Name == syftMainModule && prop.Value != "" { + own[prop.Value] = true + } + } + } + return own +} + // ValidateProjectName checks a project name such as "group/sub/repo": one or // more segments of letters, digits, ".", "_" and "-", none of them "." or // "..". diff --git a/internal/inventory/inventory_test.go b/internal/inventory/inventory_test.go index 3d55d50..b4c6cc3 100644 --- a/internal/inventory/inventory_test.go +++ b/internal/inventory/inventory_test.go @@ -27,14 +27,14 @@ func TestParseSyftCycloneDX(t *testing.T) { {Ecosystem: "composer", Name: "monolog/monolog", Version: "3.5.0", PackagePURL: "pkg:composer/monolog/monolog"}, {Ecosystem: "alpine", Name: "musl", Version: "1.2.4-r2", PackagePURL: "pkg:apk/alpine/musl"}, {Ecosystem: "composer", Name: "symfony/console", Version: "v6.4.1", PackagePURL: "pkg:composer/symfony/console"}, - {Ecosystem: "npm", Name: "webapp", Version: "1.0.0", PackagePURL: "pkg:npm/webapp"}, } if !slices.Equal(components, want) { t.Errorf("components =\n%v\nwant\n%v", components, want) } - // The operating system and the four lock files syft lists carry no PURL. - if skipped != 5 { - t.Errorf("skipped = %d, want 5", skipped) + // The operating system and the four lock files syft lists carry no PURL, + // and webapp is the root of package-lock.json, the project itself. + if skipped != 6 { + t.Errorf("skipped = %d, want 6", skipped) } } @@ -131,3 +131,32 @@ func TestOutdatedAndAffected(t *testing.T) { t.Error("a vulnerability without a fix should affect every version") } } + +func TestParseSkipsTheProjectsOwnPackages(t *testing.T) { + doc := `{"bomFormat":"CycloneDX","specVersion":"1.6","components":[ + {"bom-ref":"mod","type":"library","name":"github.com/acme/app","version":"UNKNOWN","purl":"pkg:golang/github.com/acme/app"}, + {"bom-ref":"bin","type":"library","name":"github.com/acme/app","version":"v0.0.0-20261027162038-a3e9dd9fd3f4+dirty","purl":"pkg:golang/github.com/acme/app@v0.0.0-20261027162038-a3e9dd9fd3f4%2Bdirty", + "properties":[{"name":"syft:metadata:mainModule","value":"github.com/acme/app"}]}, + {"bom-ref":"cobra","type":"library","name":"github.com/spf13/cobra","version":"v1.8.0","purl":"pkg:golang/github.com/spf13/cobra@v1.8.0", + "properties":[{"name":"syft:metadata:mainModule","value":"github.com/acme/app"}]}, + {"bom-ref":"root","type":"library","name":"app-frontend","version":"2.1.0","purl":"pkg:npm/app-frontend@2.1.0", + "properties":[{"name":"syft:package:metadataType","value":"javascript-npm-package-lock-entry"}]}, + {"bom-ref":"nover","type":"library","name":"app-tools","purl":"pkg:npm/app-tools"}, + {"bom-ref":"jsdom","type":"library","name":"jsdom","version":"30.1.3","purl":"pkg:npm/jsdom@30.1.3", + "properties":[{"name":"syft:package:metadataType","value":"javascript-npm-package-lock-entry"}]} + ],"dependencies":[{"ref":"root","dependsOn":["jsdom"]}]}` + components, skipped, err := Parse([]byte(doc)) + if err != nil { + t.Fatal(err) + } + want := []Component{ + {Ecosystem: "golang", Name: "github.com/spf13/cobra", Version: "v1.8.0", PackagePURL: "pkg:golang/github.com/spf13/cobra"}, + {Ecosystem: "npm", Name: "jsdom", Version: "30.1.3", PackagePURL: "pkg:npm/jsdom"}, + } + if !slices.Equal(components, want) { + t.Errorf("components =\n%v\nwant\n%v", components, want) + } + if skipped != 4 { + t.Errorf("skipped = %d, want 4", skipped) + } +} diff --git a/internal/server/projects_api_test.go b/internal/server/projects_api_test.go index 880a4cf..9a7d9b9 100644 --- a/internal/server/projects_api_test.go +++ b/internal/server/projects_api_test.go @@ -104,14 +104,14 @@ func TestProjectUploadAndGet(t *testing.T) { ts := newProjectsTestServer(t) resp := ts.upload(t, "/api/projects/acme/shop/web/sbom?ref=0123abcd&branch=main", syftSBOM(t)) - if resp != (ProjectUploadResponse{Project: "acme/shop/web", Components: 10, Skipped: 5}) { + if resp != (ProjectUploadResponse{Project: "acme/shop/web", Components: 9, Skipped: 6}) { t.Errorf("upload response = %+v", resp) } // The same project, URL-encoded as one segment. var project ProjectResponse decodeJSON(t, ts.do(t, http.MethodGet, "/api/projects/acme%2Fshop%2Fweb", "", nil), &project) - if project.Name != "acme/shop/web" || project.Ref != "0123abcd" || project.Branch != "main" || project.ComponentCount != 10 { + if project.Name != "acme/shop/web" || project.Ref != "0123abcd" || project.Branch != "main" || project.ComponentCount != 9 { t.Errorf("project = %+v", project.ProjectSummary) } var purls []string @@ -126,7 +126,7 @@ func TestProjectUploadAndGet(t *testing.T) { var list ProjectListResponse decodeJSON(t, ts.do(t, http.MethodGet, "/api/projects", "", nil), &list) - if len(list.Projects) != 1 || list.Projects[0].Name != "acme/shop/web" || list.Projects[0].ComponentCount != 10 { + if len(list.Projects) != 1 || list.Projects[0].Name != "acme/shop/web" || list.Projects[0].ComponentCount != 9 { t.Errorf("list = %+v", list) } } diff --git a/internal/server/projects_ui_test.go b/internal/server/projects_ui_test.go index 18fc5c1..46dbaa4 100644 --- a/internal/server/projects_ui_test.go +++ b/internal/server/projects_ui_test.go @@ -83,7 +83,7 @@ func TestProjectPageShowsComponents(t *testing.T) { } list := ts.page(t, "/ui/projects") - assertContains(t, list, `href="/ui/projects/acme/web"`, "10 packages") + assertContains(t, list, `href="/ui/projects/acme/web"`, "9 packages") } func TestPackagePageShowsUsedBy(t *testing.T) { From bd05533bfff6e6876f08dac21786dd84f1900636 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Tue, 27 Oct 2026 19:46:51 +0100 Subject: [PATCH 468/633] Delete projects from the project page The page asks for the upload token and sends it to DELETE /api/projects/{project}, so /ui gains no unauthenticated way to delete. The token is kept in session storage for the next delete. --- docs/configuration.md | 2 +- internal/server/projects_ui_test.go | 2 + internal/server/static/project-delete.js | 64 +++++++++++++++++++ .../server/templates/pages/project_show.html | 11 ++++ 4 files changed, 78 insertions(+), 1 deletion(-) create mode 100644 internal/server/static/project-delete.js diff --git a/docs/configuration.md b/docs/configuration.md index 3a5579d..e44b28a 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1578,7 +1578,7 @@ curl -X DELETE -H "Authorization: Bearer $PROXY_PROJECTS_TOKEN" \ https://proxy.example.com/api/projects/acme/shop/web ``` -Reading needs no token, like the rest of `/api`. +Reading needs no token, like the rest of `/api`. A project's page in the UI also has a delete button; it asks for the token and sends it to `DELETE /api/projects/{project}`, keeping it for the browser session. ### What it shows diff --git a/internal/server/projects_ui_test.go b/internal/server/projects_ui_test.go index 46dbaa4..cb1eb5d 100644 --- a/internal/server/projects_ui_test.go +++ b/internal/server/projects_ui_test.go @@ -67,6 +67,8 @@ func TestProjectPageShowsComponents(t *testing.T) { "1 vulnerable", "symfony/console", // not cached: listed without a link "not cached", + `data-project="acme/web"`, // delete control, asks for the token + `/ui/static/project-delete.js`, ) if strings.Contains(page, `href="/ui/package/composer/symfony/console"`) { t.Error("links to the package page of a package the proxy has no record of") diff --git a/internal/server/static/project-delete.js b/internal/server/static/project-delete.js new file mode 100644 index 0000000..b7422d7 --- /dev/null +++ b/internal/server/static/project-delete.js @@ -0,0 +1,64 @@ +// Deleting a project needs the upload token: /ui has no authentication of its +// own, so the page sends the token the user enters to the API, the same as an +// upload. The token is kept for the browser session only. +(() => { + const root = document.getElementById('project-delete'); + if (!root) { + return; + } + const start = document.getElementById('project-delete-start'); + const form = document.getElementById('project-delete-form'); + const input = document.getElementById('project-delete-token'); + const cancel = document.getElementById('project-delete-cancel'); + const errorBox = document.getElementById('project-delete-error'); + const submit = form.querySelector('button[type="submit"]'); + const storageKey = 'projects-token'; + + const show = (open) => { + start.classList.toggle('hidden', open); + form.classList.toggle('hidden', !open); + form.classList.toggle('flex', open); + errorBox.classList.add('hidden'); + }; + + start.addEventListener('click', () => { + try { + input.value = sessionStorage.getItem(storageKey) || ''; + } catch (e) { + input.value = ''; + } + show(true); + input.focus(); + }); + cancel.addEventListener('click', () => show(false)); + + form.addEventListener('submit', async (event) => { + event.preventDefault(); + const url = '/api/projects/' + encodeURIComponent(root.dataset.project); + submit.disabled = true; + errorBox.classList.add('hidden'); + try { + const resp = await fetch(url, { + method: 'DELETE', + headers: { Authorization: 'Bearer ' + input.value }, + }); + if (resp.status === 401) { + throw new Error('the token was rejected'); + } + if (!resp.ok) { + const body = await resp.json().catch(() => ({})); + throw new Error(body.message || body.error || `HTTP ${resp.status}`); + } + try { + sessionStorage.setItem(storageKey, input.value); + } catch (e) { + // Without storage the token is just asked for again. + } + window.location.href = '/ui/projects'; + } catch (err) { + errorBox.textContent = 'Delete failed: ' + err.message; + errorBox.classList.remove('hidden'); + submit.disabled = false; + } + }); +})(); diff --git a/internal/server/templates/pages/project_show.html b/internal/server/templates/pages/project_show.html index 0125f9c..9e0cf9e 100644 --- a/internal/server/templates/pages/project_show.html +++ b/internal/server/templates/pages/project_show.html @@ -13,7 +13,18 @@ {{if .Outdated}}{{.Outdated}} outdated{{end}}
{{end}} +
+ + + +
+ {{if .Components}}
From b5eec9c696ded690a5fa9e95f85c0166aa56fb3f Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Tue, 27 Oct 2026 20:15:50 +0100 Subject: [PATCH 469/633] Drop vulnerability markers from the project inventory The vulnerabilities table is only written by tests: nothing in the proxy calls UpsertVulnerability, so the dashboard card, the per-project count and the per-component markers were always empty. Remove them along with GetComponentVulnerabilities and inventory.Affected, and keep the outdated marker, which uses the latest version the handlers record. --- README.md | 2 +- docs/architecture.md | 2 +- docs/configuration.md | 5 +- internal/database/projects.go | 39 ------- internal/database/projects_test.go | 21 ---- internal/inventory/inventory.go | 12 -- internal/inventory/inventory_test.go | 23 ++-- internal/server/dashboard.go | 3 - internal/server/projects_ui.go | 108 ------------------ internal/server/projects_ui_test.go | 41 +------ internal/server/server.go | 1 - .../templates/components/severity_badge.html | 3 - .../server/templates/pages/dashboard.html | 27 ----- .../server/templates/pages/project_show.html | 9 +- 14 files changed, 20 insertions(+), 276 deletions(-) delete mode 100644 internal/server/templates/components/severity_badge.html diff --git a/README.md b/README.md index 61edf9d..7060960 100644 --- a/README.md +++ b/README.md @@ -1514,7 +1514,7 @@ The proxy serves a web UI under `/ui`. No separate frontend build is needed -- t - **Version detail** (`/ui/package/{ecosystem}/{name}/{version}`) -- per-version metadata, integrity hash, artifact cache status, and hit counts. - **Source browser** (`/ui/package/{ecosystem}/{name}/{version}/browse`) -- browse files inside cached archives with syntax highlighting for text files and image previews. - **Version diff** (`/ui/package/{ecosystem}/{name}/compare/{v1}...{v2}`) -- side-by-side diff of two cached versions showing added, removed, and changed files. -- **Projects** (`/ui/projects`) -- with the [project inventory](docs/configuration.md#project-inventory) enabled, the package versions each repository's uploaded SBOM lists, marked when outdated or vulnerable. Package pages then show which projects use each version, and the dashboard lists vulnerable versions in use. +- **Projects** (`/ui/projects`) -- with the [project inventory](docs/configuration.md#project-inventory) enabled, the package versions each repository's uploaded SBOM lists, marked when outdated. Package pages then show which projects use each version. ## Monitoring diff --git a/docs/architecture.md b/docs/architecture.md index 0105d3f..f1b5e32 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -365,7 +365,7 @@ The package also provides a `MetadataCache` for storing raw upstream metadata bl ### `internal/inventory` -Turns an SBOM uploaded to `/api/projects` into the package versions recorded for the project. Package URLs are mapped to the ecosystem names and unversioned PURLs the handlers record packages under, so a project's components join with `packages` by `purl`. Also compares versions for the outdated and vulnerability markers on the project pages. +Turns an SBOM uploaded to `/api/projects` into the package versions recorded for the project. Package URLs are mapped to the ecosystem names and unversioned PURLs the handlers record packages under, so a project's components join with `packages` by `purl`. Also compares versions for the outdated marker on the project pages. ### `internal/config` diff --git a/docs/configuration.md b/docs/configuration.md index e44b28a..9bc11df 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1508,9 +1508,8 @@ Pushed archives are stored like cached ones but marked as local. The cache size The project inventory answers "which repository uses which version of a package?". Each repository uploads an SBOM of its default branch, and the proxy records the package versions it lists. The UI then shows: -- `/ui/projects`: every project, and per project its package versions, marked when the proxy knows a newer version or vulnerabilities for that version. +- `/ui/projects`: every project, and per project its package versions, marked when the proxy knows a newer version. - On a package page: which projects use each version, including versions the proxy hasn't cached. -- On the dashboard: vulnerable versions in use and the projects using them. It is disabled by default. Uploads and deletes require a token: @@ -1582,7 +1581,7 @@ Reading needs no token, like the rest of `/api`. A project's page in the UI also ### What it shows -The inventory reflects what the projects' SBOMs list, not what was downloaded through the proxy: a project that installs from another registry still shows up, and the proxy may not have cached any of its packages. Outdated and vulnerability markers use what the proxy knows about a package: its recorded latest version and the vulnerabilities recorded for it, matched against the version a project uses by the vulnerability's fixed version. A package the proxy has no record of has neither. +The inventory reflects what the projects' SBOMs list, not what was downloaded through the proxy: a project that installs from another registry still shows up, and the proxy may not have cached any of its packages. The outdated marker compares the version a project uses with the latest version the proxy has recorded for the package; a package the proxy has no record of has none. Project names and the packages they use are shown on `/ui` and `/api/projects` without authentication. If that is sensitive, restrict both at your ingress or reverse proxy. diff --git a/internal/database/projects.go b/internal/database/projects.go index b00596b..729e09f 100644 --- a/internal/database/projects.go +++ b/internal/database/projects.go @@ -42,20 +42,6 @@ type ProjectUsage struct { Version string `db:"version"` } -// ComponentVulnerability is a vulnerability recorded for the package of a -// component a project uses. It is not yet filtered by version: whether it -// affects Version depends on FixedVersion. -type ComponentVulnerability struct { - Project string `db:"project"` - ProjectComponent - CachedEcosystem string `db:"cached_ecosystem"` - CachedName string `db:"cached_name"` - VulnID string `db:"vuln_id"` - Severity sql.NullString `db:"severity"` - FixedVersion sql.NullString `db:"fixed_version"` - CVSSScore sql.NullFloat64 `db:"cvss_score"` -} - const projectColumns = ` SELECT pr.id, pr.name, pr.ref, pr.branch, pr.uploaded_at, (SELECT COUNT(*) FROM project_components pc WHERE pc.project_id = pr.id) AS components` @@ -227,31 +213,6 @@ func (db *DB) GetProjectsUsingPackage(packagePURL, version string) ([]ProjectUsa return usages, nil } -// GetComponentVulnerabilities returns the vulnerabilities the proxy holds for -// packages that projects use, for one project or, with an empty name, for all -// of them. Vulnerabilities are recorded per package, so only packages the -// proxy has a record of are matched. -func (db *DB) GetComponentVulnerabilities(project string) ([]ComponentVulnerability, error) { - query := ` - SELECT pr.name AS project, c.package_purl, c.ecosystem, c.name, c.version, - p.ecosystem AS cached_ecosystem, p.name AS cached_name, - v.vuln_id, v.severity, v.fixed_version, v.cvss_score - FROM project_components c - JOIN projects pr ON pr.id = c.project_id - JOIN packages p ON p.purl = c.package_purl - JOIN vulnerabilities v ON v.ecosystem = p.ecosystem AND v.package_name = p.name` - var args []any - if project != "" { - query += ` WHERE pr.name = ?` - args = append(args, project) - } - vulns := []ComponentVulnerability{} - if err := db.Select(&vulns, db.Rebind(query+` ORDER BY c.package_purl, c.version, pr.name, v.vuln_id`), args...); err != nil { - return nil, err - } - return vulns, nil -} - func nullString(s string) sql.NullString { return sql.NullString{String: s, Valid: s != ""} } diff --git a/internal/database/projects_test.go b/internal/database/projects_test.go index 4eaaf50..2a5dded 100644 --- a/internal/database/projects_test.go +++ b/internal/database/projects_test.go @@ -159,13 +159,6 @@ func TestProjectComponentsJoinPackages(t *testing.T) { }); err != nil { t.Fatal(err) } - if err := db.UpsertVulnerability(&Vulnerability{ - VulnID: "GHSA-35jh-r3h4-6jhm", Ecosystem: "npm", PackageName: "lodash", - Severity: sql.NullString{String: "high", Valid: true}, - FixedVersion: sql.NullString{String: "4.17.21", Valid: true}, - }); err != nil { - t.Fatal(err) - } saveProject(t, db, "acme/web", time.Now(), lodash20, console) saveProject(t, db, "acme/api", time.Now(), lodash21) @@ -188,20 +181,6 @@ func TestProjectComponentsJoinPackages(t *testing.T) { t.Errorf("cached component = %+v", components[1]) } - vulns, err := db.GetComponentVulnerabilities("acme/web") - if err != nil { - t.Fatal(err) - } - if len(vulns) != 1 || vulns[0].Version != "4.17.20" || vulns[0].VulnID != "GHSA-35jh-r3h4-6jhm" { - t.Errorf("acme/web vulnerabilities = %+v", vulns) - } - all, err := db.GetComponentVulnerabilities("") - if err != nil { - t.Fatal(err) - } - if len(all) != 2 { - t.Errorf("all vulnerabilities = %+v, want one row per project", all) - } }) } diff --git a/internal/inventory/inventory.go b/internal/inventory/inventory.go index 9df6ce4..cfacdb8 100644 --- a/internal/inventory/inventory.go +++ b/internal/inventory/inventory.go @@ -232,15 +232,3 @@ func Outdated(ecosystem, version, latest string) bool { } return CompareVersions(ecosystem, version, latest) < 0 } - -// Affected reports whether a vulnerability fixed in fixedVersion affects -// version. A vulnerability without a fixed version affects every version. -func Affected(ecosystem, version, fixedVersion string) bool { - if fixedVersion == "" { - return true - } - if version == "" { - return false - } - return CompareVersions(ecosystem, version, fixedVersion) < 0 -} diff --git a/internal/inventory/inventory_test.go b/internal/inventory/inventory_test.go index b4c6cc3..998c93b 100644 --- a/internal/inventory/inventory_test.go +++ b/internal/inventory/inventory_test.go @@ -100,36 +100,29 @@ func TestValidateProjectName(t *testing.T) { } } -func TestOutdatedAndAffected(t *testing.T) { +func TestOutdated(t *testing.T) { tests := []struct { name string ecosystem string version string other string outdated bool - affected bool }{ - {"npm older", "npm", "4.17.20", "4.17.21", true, true}, - {"npm same", "npm", "4.17.21", "4.17.21", false, false}, - {"npm newer", "npm", "4.18.0", "4.17.21", false, false}, - {"composer v prefix", "composer", "v6.4.1", "v6.4.2", true, true}, - {"golang", "golang", "v1.8.0", "v1.10.0", true, true}, - {"alpine revisions", "alpine", "1.36.1-r15", "1.36.1-r2", false, false}, - {"empty version", "npm", "", "1.0.0", false, false}, + {"npm older", "npm", "4.17.20", "4.17.21", true}, + {"npm same", "npm", "4.17.21", "4.17.21", false}, + {"npm newer", "npm", "4.18.0", "4.17.21", false}, + {"composer v prefix", "composer", "v6.4.1", "v6.4.2", true}, + {"golang", "golang", "v1.8.0", "v1.10.0", true}, + {"alpine revisions", "alpine", "1.36.1-r15", "1.36.1-r2", false}, + {"empty version", "npm", "", "1.0.0", false}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { if got := Outdated(tt.ecosystem, tt.version, tt.other); got != tt.outdated { t.Errorf("Outdated = %v, want %v", got, tt.outdated) } - if got := Affected(tt.ecosystem, tt.version, tt.other); got != tt.affected { - t.Errorf("Affected = %v, want %v", got, tt.affected) - } }) } - if !Affected("npm", "1.0.0", "") { - t.Error("a vulnerability without a fix should affect every version") - } } func TestParseSkipsTheProjectsOwnPackages(t *testing.T) { diff --git a/internal/server/dashboard.go b/internal/server/dashboard.go index ebcc333..56bfddc 100644 --- a/internal/server/dashboard.go +++ b/internal/server/dashboard.go @@ -15,9 +15,6 @@ type DashboardData struct { EnrichmentStats EnrichmentStatsView RecentPackages []PackageInfo PopularPackages []PackageInfo - // VulnerableInUse lists vulnerable package versions projects use, when - // the project inventory is enabled. - VulnerableInUse []VulnerableVersion } // DashboardStats contains cache statistics for the dashboard. diff --git a/internal/server/projects_ui.go b/internal/server/projects_ui.go index 76ae8bd..0de4601 100644 --- a/internal/server/projects_ui.go +++ b/internal/server/projects_ui.go @@ -1,7 +1,6 @@ package server import ( - "cmp" "net/http" "net/url" "slices" @@ -31,7 +30,6 @@ type ProjectShowData struct { Layout Project ProjectRow Components []ProjectComponentRow - Vulnerable int Outdated int } @@ -48,8 +46,6 @@ type ProjectComponentRow struct { CachedName string LatestVersion string Outdated bool - VulnCount int - Severity string } // VersionUsage lists the projects using one version of a package. @@ -61,17 +57,6 @@ type VersionUsage struct { Projects []string } -// VulnerableVersion is a package version with known vulnerabilities that -// projects use. -type VulnerableVersion struct { - Ecosystem string - Name string - Version string - Severity string - VulnCount int - Projects []string -} - // mountProjectUI registers the project inventory pages when // projects.enabled is set. func (s *Server) mountProjectUI(ui chi.Router) { @@ -122,11 +107,6 @@ func (s *Server) handleProjectShow(w http.ResponseWriter, r *http.Request) { s.logger.Error("failed to get project components", "project", name, "error", err) components = nil } - vulns, err := s.db.GetComponentVulnerabilities(name) - if err != nil { - s.logger.Error("failed to get project vulnerabilities", "project", name, "error", err) - } - affecting := affectingVulnerabilities(vulns) data := ProjectShowData{Layout: s.layoutFor(r), Project: projectRow(*project)} for _, c := range components { @@ -140,10 +120,6 @@ func (s *Server) handleProjectShow(w http.ResponseWriter, r *http.Request) { LatestVersion: c.LatestVersion.String, Outdated: inventory.Outdated(c.Ecosystem, c.Version, c.LatestVersion.String), } - if v, ok := affecting[componentKey{c.PackagePURL, c.Version}]; ok { - row.VulnCount, row.Severity = len(v.ids), v.severity - data.Vulnerable++ - } if row.Outdated { data.Outdated++ } @@ -201,87 +177,3 @@ func (s *Server) packageUsedBy(pkg *database.Package, versions []database.Versio } return out } - -// vulnerableVersionsInUse returns the package versions projects use that -// have vulnerabilities, most severe and most used first, at most limit. It -// returns nothing when the project inventory is disabled. -func (s *Server) vulnerableVersionsInUse(limit int) []VulnerableVersion { - if !s.cfg.Projects.Enabled { - return nil - } - vulns, err := s.db.GetComponentVulnerabilities("") - if err != nil { - s.logger.Error("failed to get vulnerable versions in use", "error", err) - return nil - } - affecting := affectingVulnerabilities(vulns) - result := make([]VulnerableVersion, 0, len(affecting)) - for _, v := range affecting { - projects := make([]string, 0, len(v.projects)) - for p := range v.projects { - projects = append(projects, p) - } - slices.Sort(projects) - result = append(result, VulnerableVersion{ - Ecosystem: v.ecosystem, Name: v.name, Version: v.version, - Severity: v.severity, VulnCount: len(v.ids), Projects: projects, - }) - } - slices.SortFunc(result, func(a, b VulnerableVersion) int { - return cmp.Or( - cmp.Compare(severityRank(b.Severity), severityRank(a.Severity)), - cmp.Compare(len(b.Projects), len(a.Projects)), - cmp.Compare(a.Name, b.Name), - cmp.Compare(a.Version, b.Version), - ) - }) - if len(result) > limit { - result = result[:limit] - } - return result -} - -type componentKey struct{ packagePURL, version string } - -// affectingVersion collects the vulnerabilities affecting one package -// version, and the projects using it. -type affectingVersion struct { - ecosystem, name, version string - severity string - ids map[string]bool - projects map[string]bool -} - -// affectingVulnerabilities keeps the vulnerabilities that affect the version -// each component uses, grouped by package version. -func affectingVulnerabilities(vulns []database.ComponentVulnerability) map[componentKey]*affectingVersion { - out := map[componentKey]*affectingVersion{} - for _, v := range vulns { - if !inventory.Affected(v.Ecosystem, v.Version, v.FixedVersion.String) { - continue - } - key := componentKey{v.PackagePURL, v.Version} - a, ok := out[key] - if !ok { - a = &affectingVersion{ - ecosystem: v.CachedEcosystem, name: v.CachedName, version: v.Version, - ids: map[string]bool{}, projects: map[string]bool{}, - } - out[key] = a - } - a.ids[v.VulnID] = true - a.projects[v.Project] = true - if severityRank(v.Severity.String) > severityRank(a.severity) { - a.severity = v.Severity.String - } - } - return out -} - -// severityOrder lists severities from least to most severe. -var severityOrder = []string{"low", "medium", "high", "critical"} - -// severityRank orders severities; unknown ones rank below "low". -func severityRank(severity string) int { - return slices.Index(severityOrder, severity) + 1 -} diff --git a/internal/server/projects_ui_test.go b/internal/server/projects_ui_test.go index cb1eb5d..e8031d4 100644 --- a/internal/server/projects_ui_test.go +++ b/internal/server/projects_ui_test.go @@ -10,9 +10,8 @@ import ( ) // seedLodash records lodash with 4.17.21 as its latest and only cached -// version, and a vulnerability fixed in 4.17.21. The syft fixture uses -// 4.17.20. -func seedLodash(t *testing.T, db *database.DB, withVuln bool) { +// version. The syft fixture uses 4.17.20. +func seedLodash(t *testing.T, db *database.DB) { t.Helper() if err := db.UpsertPackage(&database.Package{ PURL: "pkg:npm/lodash", Ecosystem: "npm", Name: "lodash", @@ -23,16 +22,6 @@ func seedLodash(t *testing.T, db *database.DB, withVuln bool) { if err := db.UpsertVersion(&database.Version{PURL: "pkg:npm/lodash@4.17.21", PackagePURL: "pkg:npm/lodash"}); err != nil { t.Fatal(err) } - if !withVuln { - return - } - if err := db.UpsertVulnerability(&database.Vulnerability{ - VulnID: "GHSA-35jh-r3h4-6jhm", Ecosystem: "npm", PackageName: "lodash", - Severity: sql.NullString{String: "high", Valid: true}, - FixedVersion: sql.NullString{String: "4.17.21", Valid: true}, - }); err != nil { - t.Fatal(err) - } } func (ts projectsTestServer) page(t *testing.T, target string) string { @@ -55,7 +44,7 @@ func assertContains(t *testing.T, page string, want ...string) { func TestProjectPageShowsComponents(t *testing.T) { ts := newProjectsTestServer(t) - seedLodash(t, ts.db, true) + seedLodash(t, ts.db) ts.upload(t, "/api/projects/acme/web/sbom?branch=main", syftSBOM(t)) page := ts.page(t, "/ui/projects/acme/web") @@ -63,8 +52,7 @@ func TestProjectPageShowsComponents(t *testing.T) { "acme/web", `href="/ui/package/npm/lodash"`, // cached, so linked "outdated, latest 4.17.21", - "1 vulns", - "1 vulnerable", + "1 outdated", "symfony/console", // not cached: listed without a link "not cached", `data-project="acme/web"`, // delete control, asks for the token @@ -90,7 +78,7 @@ func TestProjectPageShowsComponents(t *testing.T) { func TestPackagePageShowsUsedBy(t *testing.T) { ts := newProjectsTestServer(t) - seedLodash(t, ts.db, false) + seedLodash(t, ts.db) ts.upload(t, "/api/projects/acme/web/sbom", syftSBOM(t)) ts.upload(t, "/api/projects/acme/api/sbom", []byte(`{"bomFormat":"CycloneDX","specVersion":"1.5","components":[ {"type":"library","name":"lodash","version":"4.17.21","purl":"pkg:npm/lodash@4.17.21"}]}`)) @@ -116,24 +104,7 @@ func TestPackagePageShowsUsedBy(t *testing.T) { } } -func TestDashboardShowsVulnerableVersionsInUse(t *testing.T) { +func TestNavLinksToProjects(t *testing.T) { ts := newProjectsTestServer(t) - seedLodash(t, ts.db, false) - ts.upload(t, "/api/projects/acme/web/sbom", syftSBOM(t)) - - if page := ts.page(t, "/ui/"); strings.Contains(page, "Vulnerable versions in use") { - t.Error("dashboard shows the vulnerable versions card without vulnerabilities") - } assertContains(t, ts.page(t, "/ui/"), `href="/ui/projects"`) - - seedLodash(t, ts.db, true) - page := ts.page(t, "/ui/") - assertContains(t, page, "Vulnerable versions in use", "@4.17.20", `href="/ui/projects/acme/web"`, ">high<") - - // A project on the fixed version doesn't count. - ts.upload(t, "/api/projects/acme/web/sbom", []byte(`{"bomFormat":"CycloneDX","specVersion":"1.5","components":[ - {"type":"library","name":"lodash","version":"4.17.21","purl":"pkg:npm/lodash@4.17.21"}]}`)) - if page := ts.page(t, "/ui/"); strings.Contains(page, "Vulnerable versions in use") { - t.Error("dashboard lists a version the vulnerability is fixed in") - } } diff --git a/internal/server/server.go b/internal/server/server.go index d96b9dc..88897f7 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -845,7 +845,6 @@ func (s *Server) handleRoot(w http.ResponseWriter, r *http.Request) { }, EnrichmentStats: enrichmentStatsView(enrichStats), } - data.VulnerableInUse = s.vulnerableVersionsInUse(dashboardTopN) for _, p := range popular { pkgInfo := PackageInfo{ diff --git a/internal/server/templates/components/severity_badge.html b/internal/server/templates/components/severity_badge.html deleted file mode 100644 index a060ed2..0000000 --- a/internal/server/templates/components/severity_badge.html +++ /dev/null @@ -1,3 +0,0 @@ -{{define "severity_badge"}} -{{if .}}{{.}}{{else}}unknown{{end}} -{{end}} diff --git a/internal/server/templates/pages/dashboard.html b/internal/server/templates/pages/dashboard.html index ff9bc1b..e48e205 100644 --- a/internal/server/templates/pages/dashboard.html +++ b/internal/server/templates/pages/dashboard.html @@ -25,33 +25,6 @@ {{template "security_overview" .EnrichmentStats}} {{end}} -{{if .VulnerableInUse}} -
-
-

Vulnerable versions in use

-
-
- {{range .VulnerableInUse}} -
-
- {{template "ecosystem_badge" .Ecosystem}} - {{.Name}} - @{{.Version}} - {{template "severity_badge" .Severity}} - {{.VulnCount}} vulns -
-
- {{range .Projects}}{{.}}{{end}} -
-
- {{end}} -
-
- See all projects → -
-
-{{end}} -
diff --git a/internal/server/templates/pages/project_show.html b/internal/server/templates/pages/project_show.html index 9e0cf9e..2d1b56a 100644 --- a/internal/server/templates/pages/project_show.html +++ b/internal/server/templates/pages/project_show.html @@ -7,10 +7,9 @@

{{.Project.Components}} packages, uploaded {{.Project.UploadedAt}}{{if .Project.Branch}} from {{.Project.Branch}}{{end}}{{if .Project.Ref}} at {{.Project.Ref}}{{end}}

- {{if or .Vulnerable .Outdated}} + {{if .Outdated}}
- {{if .Vulnerable}}{{.Vulnerable}} vulnerable{{end}} - {{if .Outdated}}{{.Outdated}} outdated{{end}} + {{.Outdated}} outdated
{{end}}
@@ -41,10 +40,6 @@ {{.Version}}
- {{if .VulnCount}} - {{template "severity_badge" .Severity}} - {{.VulnCount}} vulns - {{end}} {{if .Outdated}} outdated, latest {{.LatestVersion}} {{end}} From 8855c86fc54f84663fef740fb091cde88ea4e102 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Tue, 27 Oct 2026 20:18:50 +0100 Subject: [PATCH 470/633] Match more SBOM names to cached packages FromPURL now names packages the way their handlers store them: - NuGet IDs are lowercased, as the NuGet handler stores them. - PyPI names are normalized as in PEP 503. The PyPI handler takes the name from the file name, which spells "." as "_", so pkg:pypi/zope.interface has to match the cached zope-interface. - Conda package URLs without a channel qualifier are skipped: the conda handler records packages as channel/name, so they can never match. - The Go main-module check ignores case. syft keeps the module path's case in its mainModule property, while golang package URLs are lowercased. A table test checks every mapped PURL type against the name its handler passes when caching. --- docs/configuration.md | 2 +- internal/inventory/inventory.go | 76 +++++++++++++++--- internal/inventory/inventory_test.go | 110 +++++++++++++++++++++++++++ internal/server/projects_api.go | 5 +- 4 files changed, 178 insertions(+), 15 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 9bc11df..b8e9ae0 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1547,7 +1547,7 @@ The response counts the recorded package versions and the skipped entries: {"project":"acme/shop/web","components":9,"skipped":6} ``` -Entries are recorded by their package URL. Entries without one (files, the operating system), package types the proxy doesn't serve (such as `pkg:github`) and entries without a version are skipped. So is the project's own package: the root of a lock file such as `package-lock.json`, and with syft, the Go module its binaries were built from. Duplicates are recorded once. +Entries are recorded by their package URL, with the package named the way the proxy caches it: NuGet IDs lowercased, PyPI names normalized as in PEP 503 (`zope.interface` becomes `zope-interface`), the distribution dropped from `deb`, `rpm` and `apk` package URLs. Entries without a package URL (files, the operating system), package types the proxy doesn't serve (such as `pkg:github`), entries without a version, and conda entries without a `channel` qualifier (the proxy records conda packages as `channel/name`) are skipped. So is the project's own package: the root of a lock file such as `package-lock.json`, and with syft, the Go module its binaries were built from. Duplicates are recorded once. A GitLab CI job on the default branch could look like this: diff --git a/internal/inventory/inventory.go b/internal/inventory/inventory.go index cfacdb8..ccd9ecd 100644 --- a/internal/inventory/inventory.go +++ b/internal/inventory/inventory.go @@ -7,6 +7,7 @@ import ( "errors" "fmt" "strings" + "unicode" "github.com/git-pkgs/proxy/internal/packageurl" "github.com/git-pkgs/purl" @@ -66,8 +67,13 @@ var distroTypes = map[string]bool{"alpm": true, "apk": true, "bsd": true, "deb": // doesn't serve. var ErrUnsupportedPURL = errors.New("package type is not served by the proxy") -// FromPURL maps a package URL to a component. Qualifiers and subpath are -// dropped; the version is kept as given. +// ErrNoCondaChannel is returned for a conda package URL without a channel +// qualifier: the proxy records conda packages per channel. +var ErrNoCondaChannel = errors.New("conda package URL has no channel qualifier") + +// FromPURL maps a package URL to a component, with the package named the way +// the handler for its type records it. Qualifiers and subpath are dropped; +// the version is kept as given. func FromPURL(s string) (Component, error) { p, err := purl.Parse(s) if err != nil { @@ -77,12 +83,9 @@ func FromPURL(s string) (Component, error) { if !ok { return Component{}, fmt.Errorf("%w: %s", ErrUnsupportedPURL, p.Type) } - name := p.FullName() - switch { - case distroTypes[p.Type]: - name = p.Name - case p.Type == "conda" && p.Namespace == "" && p.Qualifier("channel") != "": - name = p.Qualifier("channel") + "/" + p.Name + name, err := handlerName(p) + if err != nil { + return Component{}, err } packagePURL, _ := packageurl.MakeCacheStrings(ecosystem, name, "") if packagePURL == "" { @@ -91,6 +94,53 @@ func FromPURL(s string) (Component, error) { return Component{Ecosystem: ecosystem, Name: name, Version: p.Version, PackagePURL: packagePURL}, nil } +// handlerName returns the package name the handler for p's type stores. +func handlerName(p *purl.PURL) (string, error) { + switch { + case distroTypes[p.Type]: + // The namespace is the distribution; the handlers store the bare + // package name. + return p.Name, nil + case p.Type == "conda" && p.Namespace == "": + // The conda handler stores "channel/name". + channel := p.Qualifier("channel") + if channel == "" { + return "", ErrNoCondaChannel + } + return channel + "/" + p.Name, nil + case p.Type == "nuget": + // The NuGet handler lowercases package IDs. + return strings.ToLower(p.FullName()), nil + case p.Type == "pypi": + return normalizePyPIName(p.FullName()), nil + } + return p.FullName(), nil +} + +// normalizePyPIName normalizes a PyPI project name as PEP 503 does: runs of +// "-", "_" and "." become one "-", and letters are lowercased. The PyPI +// handler stores the name from the file name, which wheels and current sdists +// spell with "_" for any of these, so "zope.interface" is cached as +// "zope-interface". (An old sdist named "zope.interface-5.0.tar.gz" is +// stored with the dot and doesn't match.) +func normalizePyPIName(name string) string { + var b strings.Builder + b.Grow(len(name)) + separator := false + for _, r := range name { + if r == '-' || r == '_' || r == '.' { + separator = true + continue + } + if separator && b.Len() > 0 { + b.WriteByte('-') + } + separator = false + b.WriteRune(unicode.ToLower(r)) + } + return b.String() +} + // syftMainModule is the property syft sets on modules found in a Go binary, // naming the module the binary was built from. const syftMainModule = "syft:metadata:mainModule" @@ -104,7 +154,8 @@ const unknownVersion = "UNKNOWN" // Parse reads a CycloneDX (JSON or XML) or SPDX SBOM and returns the package // versions it lists, without duplicates. Packages without a package URL, of a -// type the proxy doesn't serve, or without a version are counted in skipped. +// type the proxy doesn't serve, without a version, or conda packages without +// a channel are counted in skipped. // So are the project's own packages: Go modules its binaries were built from, // and the root of a lock file. func Parse(data []byte) (components []Component, skipped int, err error) { @@ -117,7 +168,7 @@ func Parse(data []byte) (components []Component, skipped int, err error) { seen := make(map[[2]string]bool, len(doc.Packages)) for _, pkg := range doc.Packages { c, ok := componentOf(pkg) - if !ok || lockRoots[pkg.ID] || (c.Ecosystem == "golang" && ownModules[c.Name]) { + if !ok || lockRoots[pkg.ID] || (c.Ecosystem == "golang" && ownModules[strings.ToLower(c.Name)]) { skipped++ continue } @@ -179,13 +230,14 @@ func lockFileRoots(doc *sbom.SBOM) map[string]bool { } // mainModules returns the Go modules the SBOM's binaries were built from, -// which are the project's own. +// which are the project's own, lowercased: syft keeps the module path's case +// while FromPURL lowercases golang names. func mainModules(pkgs []sbom.Package) map[string]bool { own := map[string]bool{} for _, pkg := range pkgs { for _, prop := range pkg.Properties { if prop.Name == syftMainModule && prop.Value != "" { - own[prop.Value] = true + own[strings.ToLower(prop.Value)] = true } } } diff --git a/internal/inventory/inventory_test.go b/internal/inventory/inventory_test.go index 998c93b..e5b6225 100644 --- a/internal/inventory/inventory_test.go +++ b/internal/inventory/inventory_test.go @@ -6,6 +6,8 @@ import ( "slices" "strings" "testing" + + "github.com/git-pkgs/proxy/internal/packageurl" ) func TestParseSyftCycloneDX(t *testing.T) { @@ -81,6 +83,96 @@ func TestFromPURLUnsupported(t *testing.T) { } } +// TestFromPURLMatchesHandlerNames checks, for every PURL type the inventory +// maps, that an SBOM's package URL ends up as the package URL the handler +// records when it caches that package. handlerName is the name the handler +// passes to GetOrFetchArtifactFromURL, taken from the handler's code. +func TestFromPURLMatchesHandlerNames(t *testing.T) { + tests := []struct { + purlType string + sbomPURL string + handlerName string + }{ + {"alpm", "pkg:alpm/arch/glibc@2.39-1?arch=x86_64", "glibc"}, + {"apk", "pkg:apk/alpine/busybox@1.36.1-r15?arch=x86_64", "busybox"}, + {"bsd", "pkg:bsd/freebsd/curl@8.7.1", "curl"}, + {"cargo", "pkg:cargo/serde@1.0.200", "serde"}, + {"chef", "pkg:chef/nginx@12.0.0", "nginx"}, + {"composer", "pkg:composer/symfony/console@v6.4.1", "symfony/console"}, + {"conan", "pkg:conan/zlib@1.3.1", "zlib"}, + {"conda", "pkg:conda/numpy@1.26.4?channel=conda-forge&subdir=linux-64", "conda-forge/numpy"}, + {"cran", "pkg:cran/ggplot2@3.5.1", "ggplot2"}, + {"deb", "pkg:deb/debian/libc6@2.36-9?arch=amd64&distro=debian-12", "libc6"}, + {"gem", "pkg:gem/rails@7.1.3", "rails"}, + // The go handler stores the module path with its case; PURLs of + // golang modules are lowercased on both sides. + {"golang", "pkg:golang/github.com/Azure/azure-sdk-for-go@v68.0.0", "github.com/Azure/azure-sdk-for-go"}, + {"hex", "pkg:hex/phoenix@1.7.12", "phoenix"}, + {"julia", "pkg:julia/Example@0.5.3?uuid=7876af07-990d-54b4-ab0e-23690620f79a", "Example"}, + {"maven", "pkg:maven/org.slf4j/slf4j-api@2.0.13?type=jar", "org.slf4j:slf4j-api"}, + {"npm", "pkg:npm/%40babel/core@7.23.0", "@babel/core"}, + // The NuGet handler lowercases the package ID. + {"nuget", "pkg:nuget/Newtonsoft.Json@13.0.3", "newtonsoft.json"}, + {"pub", "pkg:pub/http@1.2.1", "http"}, + // The PyPI handler takes the name from the wheel or sdist file name, + // which spells "." and "-" as "_". + {"pypi", "pkg:pypi/zope.interface@6.4", "zope_interface"}, + {"rpm", "pkg:rpm/fedora/curl@8.6.0-7.fc40?arch=x86_64", "curl"}, + {"swift", "pkg:swift/github.com/apple/swift-argument-parser@1.3.1", "github.com/apple/swift-argument-parser"}, + } + covered := map[string]bool{} + for _, tt := range tests { + t.Run(tt.purlType, func(t *testing.T) { + covered[tt.purlType] = true + c, err := FromPURL(tt.sbomPURL) + if err != nil { + t.Fatalf("FromPURL(%q): %v", tt.sbomPURL, err) + } + want, _ := packageurl.MakeCacheStrings(ecosystems[tt.purlType], tt.handlerName, "") + if want == "" { + t.Fatalf("handler name %q has no package URL", tt.handlerName) + } + if c.PackagePURL != want { + t.Errorf("PackagePURL = %q, want %q as the handler records it", c.PackagePURL, want) + } + }) + } + for purlType := range ecosystems { + if !covered[purlType] { + t.Errorf("no case for PURL type %q", purlType) + } + } +} + +func TestFromPURLNormalizesPyPINames(t *testing.T) { + for _, s := range []string{"pkg:pypi/zope.interface", "pkg:pypi/Zope_Interface", "pkg:pypi/zope--interface", "pkg:pypi/zope._-interface"} { + c, err := FromPURL(s) + if err != nil { + t.Fatal(err) + } + if c.Name != "zope-interface" || c.PackagePURL != "pkg:pypi/zope-interface" { + t.Errorf("FromPURL(%q) = %+v, want zope-interface", s, c) + } + } +} + +func TestFromPURLCondaNeedsChannel(t *testing.T) { + if _, err := FromPURL("pkg:conda/numpy@1.26.4"); !errors.Is(err, ErrNoCondaChannel) { + t.Errorf("err = %v, want ErrNoCondaChannel", err) + } + doc := `{"bomFormat":"CycloneDX","specVersion":"1.5","components":[ + {"type":"library","name":"numpy","version":"1.26.4","purl":"pkg:conda/numpy@1.26.4"}, + {"type":"library","name":"numpy","version":"1.26.4","purl":"pkg:conda/numpy@1.26.4?channel=conda-forge"} + ]}` + components, skipped, err := Parse([]byte(doc)) + if err != nil { + t.Fatal(err) + } + if len(components) != 1 || components[0].Name != "conda-forge/numpy" || skipped != 1 { + t.Errorf("components = %+v, skipped = %d; want conda-forge/numpy and 1 skipped", components, skipped) + } +} + func TestValidateProjectName(t *testing.T) { valid := []string{"repo", "group/repo", "group/sub/my_repo-2.0", strings.Repeat("a", MaxProjectNameLength)} for _, name := range valid { @@ -153,3 +245,21 @@ func TestParseSkipsTheProjectsOwnPackages(t *testing.T) { t.Errorf("skipped = %d, want 4", skipped) } } + +func TestParseSkipsMainModuleRegardlessOfCase(t *testing.T) { + // syft keeps the module path's case in mainModule; package URLs of Go + // modules are lowercased. + doc := `{"bomFormat":"CycloneDX","specVersion":"1.6","components":[ + {"bom-ref":"bin","type":"library","name":"github.com/Azure/foo","version":"v1.2.0","purl":"pkg:golang/github.com/Azure/foo@v1.2.0", + "properties":[{"name":"syft:metadata:mainModule","value":"github.com/Azure/foo"}]}, + {"bom-ref":"cobra","type":"library","name":"github.com/spf13/cobra","version":"v1.8.0","purl":"pkg:golang/github.com/spf13/cobra@v1.8.0", + "properties":[{"name":"syft:metadata:mainModule","value":"github.com/Azure/foo"}]} + ]}` + components, skipped, err := Parse([]byte(doc)) + if err != nil { + t.Fatal(err) + } + if len(components) != 1 || components[0].Name != "github.com/spf13/cobra" || skipped != 1 { + t.Errorf("components = %+v, skipped = %d; want only cobra, 1 skipped", components, skipped) + } +} diff --git a/internal/server/projects_api.go b/internal/server/projects_api.go index a007319..7417830 100644 --- a/internal/server/projects_api.go +++ b/internal/server/projects_api.go @@ -43,8 +43,9 @@ type ProjectUploadResponse struct { Project string `json:"project"` // Components is the number of package versions recorded. Components int `json:"components"` - // Skipped is the number of SBOM entries without a package URL or of a - // package type the proxy doesn't serve. + // Skipped is the number of SBOM entries not recorded: without a package + // URL or a version, of a package type the proxy doesn't serve, conda + // entries without a channel, and the project's own packages. Skipped int `json:"skipped"` } From 1c706fffd0dfe3f0ac589a72988a31c1c8d50da8 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Tue, 27 Oct 2026 20:23:14 +0100 Subject: [PATCH 471/633] Review fixes for the project inventory - GetProjectsUsingPackage no longer counts each project's components per row; the package page only needs names and versions. The ?purl= listing leaves component_count out accordingly. - The delete button sends to DELETE /ui/api/projects/{project}, the same handler as the API and still behind the token, so it sits under /ui like the cache deletes. - The project page answers 500 when its components can't be read, instead of showing an empty project. - DELETE of a path ending in /sbom that names no project answers 400 with a hint, since it's most likely the upload path. - The upload endpoint lists JSON, XML and plain text (SPDX tag-value) as accepted. - The docs say that versions are compared as the SBOM spells them. --- docs/configuration.md | 6 ++- docs/swagger/docs.go | 14 +++--- docs/swagger/swagger.json | 14 +++--- internal/database/projects.go | 10 ++-- internal/server/projects_api.go | 29 +++++++---- internal/server/projects_api_test.go | 62 +++++++++++++++++++++++- internal/server/projects_ui.go | 10 +++- internal/server/projects_ui_test.go | 12 ++++- internal/server/server.go | 1 + internal/server/static/project-delete.js | 6 +-- 10 files changed, 129 insertions(+), 35 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index b8e9ae0..e6019fd 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1577,11 +1577,13 @@ curl -X DELETE -H "Authorization: Bearer $PROXY_PROJECTS_TOKEN" \ https://proxy.example.com/api/projects/acme/shop/web ``` -Reading needs no token, like the rest of `/api`. A project's page in the UI also has a delete button; it asks for the token and sends it to `DELETE /api/projects/{project}`, keeping it for the browser session. +Reading needs no token, like the rest of `/api`. With `?purl=`, each project carries the `version` it uses and no `component_count`. A delete names the project itself: `DELETE /api/projects/acme/shop/web/sbom` is answered with 400 unless a project `acme/shop/web/sbom` exists. + +A project's page in the UI also has a delete button. It asks for the token and sends it to `DELETE /ui/api/projects/{project}`, which works like the API endpoint and also requires the token; the token is kept for the browser session. Like `/ui/api/cache`, it sits under `/ui`, so rules for the UI at an ingress cover it. ### What it shows -The inventory reflects what the projects' SBOMs list, not what was downloaded through the proxy: a project that installs from another registry still shows up, and the proxy may not have cached any of its packages. The outdated marker compares the version a project uses with the latest version the proxy has recorded for the package; a package the proxy has no record of has none. +The inventory reflects what the projects' SBOMs list, not what was downloaded through the proxy: a project that installs from another registry still shows up, and the proxy may not have cached any of its packages. The outdated marker compares the version a project uses with the latest version the proxy has recorded for the package; a package the proxy has no record of has none. Versions are compared as the SBOM gives them: a version spelled differently from how the proxy caches it, such as Composer's `v6.4.1` for `6.4.1` or a Debian version with or without its epoch, shows as "not cached" on the package page. Project names and the packages they use are shown on `/ui` and `/api/projects` without authentication. If that is sensitive, restrict both at your ingress or reverse proxy. diff --git a/docs/swagger/docs.go b/docs/swagger/docs.go index 7ddf630..1888289 100644 --- a/docs/swagger/docs.go +++ b/docs/swagger/docs.go @@ -540,7 +540,7 @@ const docTemplate = `{ "BearerAuth": [] } ], - "description": "Removes the project and the package versions recorded for it.", + "description": "Removes the project and the package versions recorded for it. The path names the project itself, without the /sbom of the upload path: a path ending in /sbom that names no project is answered with 400.", "tags": [ "projects" ], @@ -586,9 +586,11 @@ const docTemplate = `{ "BearerAuth": [] } ], - "description": "Replaces the package versions recorded for the project with the ones the SBOM lists. CycloneDX (JSON or XML) and SPDX (JSON or tag-value) are accepted, e.g. the output of ` + "`" + `syft dir:. -o cyclonedx-json` + "`" + `. Entries without a package URL, or of a package type the proxy doesn't serve, are skipped. The project may be given as path segments or URL-encoded as one segment.", + "description": "Replaces the package versions recorded for the project with the ones the SBOM lists. CycloneDX (JSON or XML) and SPDX (JSON or tag-value) are accepted, e.g. the output of ` + "`" + `syft dir:. -o cyclonedx-json` + "`" + `. Entries without a package URL or version, of a package type the proxy doesn't serve, and the project's own packages are skipped and counted. The project may be given as path segments or URL-encoded as one segment.", "consumes": [ - "application/json" + "application/json", + "application/xml", + "text/plain" ], "produces": [ "application/json" @@ -1341,7 +1343,7 @@ const docTemplate = `{ "type": "string" }, "component_count": { - "description": "ComponentCount is the number of package versions recorded.", + "description": "ComponentCount is the number of package versions recorded. It is not\nset when listing the projects using a package.", "type": "integer" }, "components": { @@ -1372,7 +1374,7 @@ const docTemplate = `{ "type": "string" }, "component_count": { - "description": "ComponentCount is the number of package versions recorded.", + "description": "ComponentCount is the number of package versions recorded. It is not\nset when listing the projects using a package.", "type": "integer" }, "name": { @@ -1401,7 +1403,7 @@ const docTemplate = `{ "type": "string" }, "skipped": { - "description": "Skipped is the number of SBOM entries without a package URL or of a\npackage type the proxy doesn't serve.", + "description": "Skipped is the number of SBOM entries not recorded: without a package\nURL or a version, of a package type the proxy doesn't serve, conda\nentries without a channel, and the project's own packages.", "type": "integer" } } diff --git a/docs/swagger/swagger.json b/docs/swagger/swagger.json index eb6f3b9..a76f868 100644 --- a/docs/swagger/swagger.json +++ b/docs/swagger/swagger.json @@ -533,7 +533,7 @@ "BearerAuth": [] } ], - "description": "Removes the project and the package versions recorded for it.", + "description": "Removes the project and the package versions recorded for it. The path names the project itself, without the /sbom of the upload path: a path ending in /sbom that names no project is answered with 400.", "tags": [ "projects" ], @@ -579,9 +579,11 @@ "BearerAuth": [] } ], - "description": "Replaces the package versions recorded for the project with the ones the SBOM lists. CycloneDX (JSON or XML) and SPDX (JSON or tag-value) are accepted, e.g. the output of `syft dir:. -o cyclonedx-json`. Entries without a package URL, or of a package type the proxy doesn't serve, are skipped. The project may be given as path segments or URL-encoded as one segment.", + "description": "Replaces the package versions recorded for the project with the ones the SBOM lists. CycloneDX (JSON or XML) and SPDX (JSON or tag-value) are accepted, e.g. the output of `syft dir:. -o cyclonedx-json`. Entries without a package URL or version, of a package type the proxy doesn't serve, and the project's own packages are skipped and counted. The project may be given as path segments or URL-encoded as one segment.", "consumes": [ - "application/json" + "application/json", + "application/xml", + "text/plain" ], "produces": [ "application/json" @@ -1334,7 +1336,7 @@ "type": "string" }, "component_count": { - "description": "ComponentCount is the number of package versions recorded.", + "description": "ComponentCount is the number of package versions recorded. It is not\nset when listing the projects using a package.", "type": "integer" }, "components": { @@ -1365,7 +1367,7 @@ "type": "string" }, "component_count": { - "description": "ComponentCount is the number of package versions recorded.", + "description": "ComponentCount is the number of package versions recorded. It is not\nset when listing the projects using a package.", "type": "integer" }, "name": { @@ -1394,7 +1396,7 @@ "type": "string" }, "skipped": { - "description": "Skipped is the number of SBOM entries without a package URL or of a\npackage type the proxy doesn't serve.", + "description": "Skipped is the number of SBOM entries not recorded: without a package\nURL or a version, of a package type the proxy doesn't serve, conda\nentries without a channel, and the project's own packages.", "type": "integer" } } diff --git a/internal/database/projects.go b/internal/database/projects.go index 729e09f..c14b0d6 100644 --- a/internal/database/projects.go +++ b/internal/database/projects.go @@ -38,8 +38,11 @@ type ProjectComponentInfo struct { // ProjectUsage is a project using a version of a package. type ProjectUsage struct { - Project - Version string `db:"version"` + Name string `db:"name"` + Ref sql.NullString `db:"ref"` + Branch sql.NullString `db:"branch"` + UploadedAt time.Time `db:"uploaded_at"` + Version string `db:"version"` } const projectColumns = ` @@ -197,7 +200,8 @@ func (db *DB) GetProjectComponents(projectID int64) ([]ProjectComponentInfo, err // GetProjectsUsingPackage returns the projects using a package, or only // those using version when it is not empty, ordered by project and version. func (db *DB) GetProjectsUsingPackage(packagePURL, version string) ([]ProjectUsage, error) { - query := projectColumns + `, c.version + query := ` + SELECT pr.name, pr.ref, pr.branch, pr.uploaded_at, c.version FROM project_components c JOIN projects pr ON pr.id = c.project_id WHERE c.package_purl = ?` diff --git a/internal/server/projects_api.go b/internal/server/projects_api.go index 7417830..f297a9a 100644 --- a/internal/server/projects_api.go +++ b/internal/server/projects_api.go @@ -55,8 +55,9 @@ type ProjectSummary struct { Ref string `json:"ref,omitempty"` Branch string `json:"branch,omitempty"` UploadedAt time.Time `json:"uploaded_at"` - // ComponentCount is the number of package versions recorded. - ComponentCount int64 `json:"component_count"` + // ComponentCount is the number of package versions recorded. It is not + // set when listing the projects using a package. + ComponentCount *int64 `json:"component_count,omitempty"` // Version is set when listing the projects using a package: the version // of it this project uses. Version string `json:"version,omitempty"` @@ -107,9 +108,9 @@ func registerProjectRoutes(r chi.Router, h *ProjectsAPIHandler, token string) { // HandleUpload handles PUT /api/projects/{project}/sbom. // @Summary Upload a project's SBOM -// @Description Replaces the package versions recorded for the project with the ones the SBOM lists. CycloneDX (JSON or XML) and SPDX (JSON or tag-value) are accepted, e.g. the output of `syft dir:. -o cyclonedx-json`. Entries without a package URL, or of a package type the proxy doesn't serve, are skipped. The project may be given as path segments or URL-encoded as one segment. +// @Description Replaces the package versions recorded for the project with the ones the SBOM lists. CycloneDX (JSON or XML) and SPDX (JSON or tag-value) are accepted, e.g. the output of `syft dir:. -o cyclonedx-json`. Entries without a package URL or version, of a package type the proxy doesn't serve, and the project's own packages are skipped and counted. The project may be given as path segments or URL-encoded as one segment. // @Tags projects -// @Accept json +// @Accept json,application/xml,plain // @Produce json // @Security BearerAuth // @Param project path string true "Project path, e.g. group/sub/repo" @@ -176,9 +177,10 @@ func (h *ProjectsAPIHandler) HandleUpload(w http.ResponseWriter, r *http.Request writeJSON(w, ProjectUploadResponse{Project: name, Components: len(records), Skipped: skipped}) } -// HandleDelete handles DELETE /api/projects/{project}. +// HandleDelete handles DELETE /api/projects/{project}, and the same under +// /ui/api/projects for the delete button on a project's page. // @Summary Delete a project -// @Description Removes the project and the package versions recorded for it. +// @Description Removes the project and the package versions recorded for it. The path names the project itself, without the /sbom of the upload path: a path ending in /sbom that names no project is answered with 400. // @Tags projects // @Security BearerAuth // @Param project path string true "Project path, e.g. group/sub/repo" @@ -197,6 +199,10 @@ func (h *ProjectsAPIHandler) HandleDelete(w http.ResponseWriter, r *http.Request case err != nil: h.logger.Error("failed to delete project", "project", name, "error", err) internalError(w, "failed to delete project") + case !deleted && (name == "sbom" || strings.HasSuffix(name, sbomPathSuffix)): + // Most likely the upload path; a project whose last segment is + // "sbom" is deleted above when it exists. + badRequest(w, "project not found; to delete a project, send DELETE /api/projects/{project} without /sbom") case !deleted: notFound(w, "project not found") default: @@ -246,8 +252,13 @@ func (h *ProjectsAPIHandler) listUsages(w http.ResponseWriter, query string) { } resp := ProjectListResponse{Projects: make([]ProjectSummary, len(usages))} for i, u := range usages { - resp.Projects[i] = projectSummary(u.Project) - resp.Projects[i].Version = u.Version + resp.Projects[i] = ProjectSummary{ + Name: u.Name, + Ref: u.Ref.String, + Branch: u.Branch.String, + UploadedAt: u.UploadedAt, + Version: u.Version, + } } writeJSON(w, resp) } @@ -301,7 +312,7 @@ func projectSummary(p database.Project) ProjectSummary { Ref: p.Ref.String, Branch: p.Branch.String, UploadedAt: p.UploadedAt, - ComponentCount: p.Components, + ComponentCount: &p.Components, } } diff --git a/internal/server/projects_api_test.go b/internal/server/projects_api_test.go index 9a7d9b9..ba68dae 100644 --- a/internal/server/projects_api_test.go +++ b/internal/server/projects_api_test.go @@ -100,6 +100,14 @@ func decodeJSON(t *testing.T, rec *httptest.ResponseRecorder, v any) { } } +// componentCount returns a project's component_count, or -1 if not set. +func componentCount(p ProjectSummary) int64 { + if p.ComponentCount == nil { + return -1 + } + return *p.ComponentCount +} + func TestProjectUploadAndGet(t *testing.T) { ts := newProjectsTestServer(t) @@ -111,7 +119,7 @@ func TestProjectUploadAndGet(t *testing.T) { // The same project, URL-encoded as one segment. var project ProjectResponse decodeJSON(t, ts.do(t, http.MethodGet, "/api/projects/acme%2Fshop%2Fweb", "", nil), &project) - if project.Name != "acme/shop/web" || project.Ref != "0123abcd" || project.Branch != "main" || project.ComponentCount != 9 { + if project.Name != "acme/shop/web" || project.Ref != "0123abcd" || project.Branch != "main" || componentCount(project.ProjectSummary) != 9 { t.Errorf("project = %+v", project.ProjectSummary) } var purls []string @@ -126,7 +134,7 @@ func TestProjectUploadAndGet(t *testing.T) { var list ProjectListResponse decodeJSON(t, ts.do(t, http.MethodGet, "/api/projects", "", nil), &list) - if len(list.Projects) != 1 || list.Projects[0].Name != "acme/shop/web" || list.Projects[0].ComponentCount != 9 { + if len(list.Projects) != 1 || list.Projects[0].Name != "acme/shop/web" || componentCount(list.Projects[0]) != 9 { t.Errorf("list = %+v", list) } } @@ -169,6 +177,9 @@ func TestProjectLookupByPURL(t *testing.T) { var got []string for _, p := range list.Projects { got = append(got, p.Name+"@"+p.Version) + if p.ComponentCount != nil { + t.Errorf("%s has component_count set when listing by purl", p.Name) + } } if strings.Join(got, ",") != strings.Join(tt.want, ",") { t.Errorf("projects = %v, want %v", got, tt.want) @@ -212,6 +223,40 @@ func TestProjectDelete(t *testing.T) { } } +func TestProjectDeleteOfUploadPath(t *testing.T) { + ts := newProjectsTestServer(t) + ts.upload(t, "/api/projects/acme/web/sbom", syftSBOM(t)) + rec := ts.do(t, http.MethodDelete, "/api/projects/acme/web/sbom", testProjectsToken, nil) + if rec.Code != http.StatusBadRequest || !strings.Contains(rec.Body.String(), "without /sbom") { + t.Errorf("DELETE of the upload path = %d: %s, want 400 with a hint", rec.Code, rec.Body) + } + if rec := ts.do(t, http.MethodGet, "/api/projects/acme/web", "", nil); rec.Code != http.StatusOK { + t.Errorf("project is gone after DELETE of the upload path: %d", rec.Code) + } + + // A project whose last segment is "sbom" is deleted as usual. + ts.upload(t, "/api/projects/acme/sbom/sbom", syftSBOM(t)) + if rec := ts.do(t, http.MethodDelete, "/api/projects/acme/sbom", testProjectsToken, nil); rec.Code != http.StatusNoContent { + t.Errorf("DELETE of project acme/sbom = %d: %s", rec.Code, rec.Body) + } +} + +func TestProjectDeleteFromUI(t *testing.T) { + ts := newProjectsTestServer(t) + ts.upload(t, "/api/projects/acme/web/sbom", syftSBOM(t)) + for _, token := range []string{"", "wrong-token"} { + if rec := ts.do(t, http.MethodDelete, "/ui/api/projects/acme%2Fweb", token, nil); rec.Code != http.StatusUnauthorized { + t.Errorf("UI delete with token %q status = %d, want 401", token, rec.Code) + } + } + if rec := ts.do(t, http.MethodDelete, "/ui/api/projects/acme%2Fweb", testProjectsToken, nil); rec.Code != http.StatusNoContent { + t.Fatalf("UI delete status = %d: %s", rec.Code, rec.Body) + } + if rec := ts.do(t, http.MethodGet, "/api/projects/acme/web", "", nil); rec.Code != http.StatusNotFound { + t.Errorf("get after UI delete status = %d, want 404", rec.Code) + } +} + func TestProjectPathsAreValidated(t *testing.T) { ts := newProjectsTestServer(t) uploads := []string{ @@ -331,6 +376,9 @@ func TestProjectRoutesFollowConfig(t *testing.T) { if status, _ := request(t, http.MethodPut, disabled+"/api/projects/acme/web/sbom", testProjectsToken, syftSBOM(t)); status != http.StatusNotFound { t.Errorf("upload with projects disabled = %d, want 404", status) } + if status, _ := request(t, http.MethodDelete, disabled+"/ui/api/projects/acme%2Fweb", testProjectsToken, nil); status != http.StatusNotFound { + t.Errorf("UI delete with projects disabled = %d, want 404", status) + } if _, body := request(t, http.MethodGet, disabled+"/ui/", "", nil); strings.Contains(body, `href="/ui/projects"`) { t.Error("navigation links to projects while they are disabled") } @@ -351,4 +399,14 @@ func TestProjectRoutesFollowConfig(t *testing.T) { if _, body := request(t, http.MethodGet, enabled+"/ui/", "", nil); !strings.Contains(body, `href="/ui/projects"`) { t.Error("navigation does not link to projects") } + if status, _ := request(t, http.MethodDelete, enabled+"/ui/api/projects/acme%2Fweb", "", nil); status != http.StatusUnauthorized { + t.Errorf("UI delete without token = %d, want 401", status) + } + if status, body := request(t, http.MethodDelete, enabled+"/ui/api/projects/acme%2Fweb", testProjectsToken, nil); status != http.StatusNoContent { + t.Errorf("UI delete = %d: %s", status, body) + } + // The delete button sends its request there, too. + if _, body := request(t, http.MethodGet, enabled+"/ui/static/project-delete.js", "", nil); !strings.Contains(body, "'/ui/api/projects/'") { + t.Error("project-delete.js does not send to /ui/api/projects") + } } diff --git a/internal/server/projects_ui.go b/internal/server/projects_ui.go index 0de4601..5bafb39 100644 --- a/internal/server/projects_ui.go +++ b/internal/server/projects_ui.go @@ -58,13 +58,18 @@ type VersionUsage struct { } // mountProjectUI registers the project inventory pages when -// projects.enabled is set. +// projects.enabled is set, and DELETE /ui/api/projects/{project} for the +// delete button on a project's page. Like DELETE /api/projects/{project} it +// requires the bearer token; it sits under /ui so that an ingress can treat +// the UI's requests together. func (s *Server) mountProjectUI(ui chi.Router) { if !s.cfg.Projects.Enabled { return } ui.Get("/projects", s.handleProjectsList) ui.Get("/projects/*", s.handleProjectShow) + api := NewProjectsAPIHandler(s.db, s.logger) + ui.With(requireBearerToken(s.cfg.Projects.TokenExpanded())).Delete("/api/projects/*", api.HandleDelete) } func (s *Server) handleProjectsList(w http.ResponseWriter, r *http.Request) { @@ -105,7 +110,8 @@ func (s *Server) handleProjectShow(w http.ResponseWriter, r *http.Request) { components, err := s.db.GetProjectComponents(project.ID) if err != nil { s.logger.Error("failed to get project components", "project", name, "error", err) - components = nil + http.Error(w, "failed to get project components", http.StatusInternalServerError) + return } data := ProjectShowData{Layout: s.layoutFor(r), Project: projectRow(*project)} diff --git a/internal/server/projects_ui_test.go b/internal/server/projects_ui_test.go index e8031d4..5be2224 100644 --- a/internal/server/projects_ui_test.go +++ b/internal/server/projects_ui_test.go @@ -104,7 +104,15 @@ func TestPackagePageShowsUsedBy(t *testing.T) { } } -func TestNavLinksToProjects(t *testing.T) { +func TestProjectPageFailsWhenComponentsCannotBeRead(t *testing.T) { ts := newProjectsTestServer(t) - assertContains(t, ts.page(t, "/ui/"), `href="/ui/projects"`) + ts.upload(t, "/api/projects/acme/web/sbom", syftSBOM(t)) + // The components query joins packages; without the table it fails while + // the project itself can still be read. + if _, err := ts.db.Exec("ALTER TABLE packages RENAME TO packages_gone"); err != nil { + t.Fatal(err) + } + if rec := ts.do(t, http.MethodGet, "/ui/projects/acme/web", "", nil); rec.Code != http.StatusInternalServerError { + t.Errorf("status = %d, want 500 rather than an empty project", rec.Code) + } } diff --git a/internal/server/server.go b/internal/server/server.go index 88897f7..6946e5c 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -42,6 +42,7 @@ // - /ui/projects/... - Project inventory (when projects.enabled) // - /ui/api/browse/... - Archive browsing (used by the UI) // - /ui/api/compare/... - Archive diffing (used by the UI) +// - /ui/api/projects/... - Project deletion (used by the UI, needs the token) // // API endpoints for enrichment data: // - GET /api/package/{ecosystem}/{name} - Package metadata diff --git a/internal/server/static/project-delete.js b/internal/server/static/project-delete.js index b7422d7..637cdc1 100644 --- a/internal/server/static/project-delete.js +++ b/internal/server/static/project-delete.js @@ -1,6 +1,6 @@ // Deleting a project needs the upload token: /ui has no authentication of its -// own, so the page sends the token the user enters to the API, the same as an -// upload. The token is kept for the browser session only. +// own, so the page sends the token the user enters with the request, the same +// as an upload. The token is kept for the browser session only. (() => { const root = document.getElementById('project-delete'); if (!root) { @@ -34,7 +34,7 @@ form.addEventListener('submit', async (event) => { event.preventDefault(); - const url = '/api/projects/' + encodeURIComponent(root.dataset.project); + const url = '/ui/api/projects/' + encodeURIComponent(root.dataset.project); submit.disabled = true; errorBox.classList.add('hidden'); try { From 9037d937bf6acca95c597c7868003379d5774a6f Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Wed, 28 Oct 2026 11:39:28 +0000 Subject: [PATCH 472/633] Document using proxy in Docker builds --- README.md | 3 +- docs/docker-builds.md | 165 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 167 insertions(+), 1 deletion(-) create mode 100644 docs/docker-builds.md diff --git a/README.md b/README.md index 20c6d5d..0403cd4 100644 --- a/README.md +++ b/README.md @@ -138,7 +138,8 @@ go build -o proxy ./cmd/proxy ./proxy -listen :3000 -base-url https://proxy.example.com ``` -The proxy is now running. Configure your package managers to use it. +The proxy is now running. Configure your package managers to use it. For image +builds, see [Using proxy in Docker builds](docs/docker-builds.md). ## OpenAPI (Swagger) diff --git a/docs/docker-builds.md b/docs/docker-builds.md new file mode 100644 index 0000000..14570fd --- /dev/null +++ b/docs/docker-builds.md @@ -0,0 +1,165 @@ +# Using proxy in Docker builds + +Docker builds can share proxy's package cache even when their builders and local +package caches are new. Configure the package manager inside the build to use +proxy. Cached archives still need to be downloaded from proxy and installed; +native extensions still need to be compiled. + +## Run a persistent proxy + +Run proxy outside the image build and keep its database and artifact storage +between builds. For a proxy running on your development machine: + +```sh +PROXY_CACHE_METADATA=true proxy serve \ + -listen :8080 \ + -base-url http://host.docker.internal:8080 \ + -storage-url "file://$PWD/proxy-cache/artifacts" \ + -database-path ./proxy-cache/proxy.db +``` + +Docker Desktop provides `host.docker.internal` for reaching the host. On Linux +Docker Engine, add the host mapping when building: + +```sh +docker build \ + --add-host host.docker.internal=host-gateway \ + --build-arg PROXY_URL=http://host.docker.internal:8080 \ + -t app . +``` + +`localhost` inside a build refers to the build container. Both the build's proxy +URL and proxy's `base_url` must be reachable from that container, since some +registry responses contain download URLs derived from `base_url`. The proxy +must listen on an interface reachable from the builder, with access restricted +to the machines that should use it. + +For CI or a remote builder, use a persistent service reachable from the builder, +such as `https://proxy.example.com`, and set proxy's `base_url` to that address. +Use the same service across jobs and pass its URL with `--build-arg PROXY_URL=...`. +Creating an empty proxy for every job loses the shared cache unless its database +and storage persist. If proxy runs in a container, mount persistent storage for +both. See [configuration](configuration.md) for storage and deployment options. + +Metadata caching is optional, but avoids repeated metadata fetches within +`metadata_ttl`. Artifact caching is enabled by default. Proxy's configured +cooldown, denylist and scanning policies also apply to Docker builds. + +## npm + +In your Node build stage, copy the manifests before application source so an +unchanged dependency set can reuse Docker's installation layer: + +```dockerfile +ARG PROXY_URL +COPY package.json package-lock.json ./ +RUN npm ci --registry="${PROXY_URL}/npm/" +COPY . . +``` + +Pass `PROXY_URL` without a trailing slash. With Docker Desktop: + +```sh +docker build \ + --build-arg PROXY_URL=http://host.docker.internal:8080 \ + -t app . +``` + +Keep `node_modules` out of the build context so `COPY . .` does not overwrite +the installed dependencies. Lockfiles with custom registry or download URLs +may still direct requests to those hosts; check proxy's access log when using +private registries or non-registry dependencies. npm audit requests are forwarded +upstream even when the package archives are cached. + +## Ruby / Bundler + +Keep `https://rubygems.org` as the source in your Gemfile and configure its mirror +inside the Ruby build stage. Install any system packages needed to compile +native extensions before this step: + +```dockerfile +ARG PROXY_URL +COPY Gemfile Gemfile.lock ./ +RUN bundle config set --local mirror.https://rubygems.org "${PROXY_URL}/gem" \ + && bundle install \ + && bundle config unset --local mirror.https://rubygems.org +COPY . . +``` + +Use the same build command as the npm example. The mirror setting is removed +after installation so it is not retained in the image. Exclude host Bundler +configuration and installed gems from the build context. Copy any local path +dependencies required by the Gemfile before `bundle install`. + +The RubyGems mirror covers gems fetched from that source. Git dependencies and +other gem sources need their own configuration. Bundler can fall back to the +original source when a mirror is unavailable; this setup does not enforce +network isolation. + +## Layer caches and package caches + +Keep existing Docker caching optimizations. If Docker reuses the installation +layer, the package manager does not run and proxy receives no requests. A +BuildKit cache mount can also preserve downloaded packages when that layer must +be rebuilt. For example, the npm step can use: + +```dockerfile +RUN --mount=type=cache,target=/root/.npm \ + npm ci --registry="${PROXY_URL}/npm/" +``` + +Cache mounts are stored by the builder and are excluded from ordinary +`--cache-to` exports. Proxy can supply packages when a fresh builder has neither +the installation layer nor the package-manager cache. See Docker's +[cache guidance](https://docs.docker.com/build/cache/optimize/) and +[cache-mount reference](https://docs.docker.com/reference/dockerfile/#run---mounttypecache). + +To check the configuration, enable proxy's JSONL log with `-access-log` and +rebuild with `docker build --no-cache`. Look for requests to `/npm/` or `/gem/`, +then repeat and check `proxy_cache_hits_total` on `/metrics`. Cache mounts still +retain their contents with `--no-cache`; omit them when testing proxy cache +hits. Upstream metadata revalidation and audit requests can still occur. + +## Base-image pulls + +Package-manager configuration affects commands such as `npm ci` and +`bundle install`. To cache images referenced by `FROM`, configure a registry +mirror in the builder as well. For a proxy served at +`https://proxy.example.com`, put this in `buildkitd.toml`: + +```toml +[registry."docker.io"] + mirrors = ["proxy.example.com"] +``` + +Create a Buildx builder with that configuration and select it explicitly: + +```sh +docker buildx create --name proxy-builder \ + --driver docker-container \ + --buildkitd-config ./buildkitd.toml + +docker buildx build --builder proxy-builder \ + --build-arg PROXY_URL=https://proxy.example.com \ + --load -t app . +``` + +For local Docker Desktop testing over HTTP, the configuration is: + +```toml +[registry."docker.io"] + mirrors = ["host.docker.internal:8080"] + +[registry."host.docker.internal:8080"] + http = true +``` + +The registry mirror address must be reachable from the BuildKit daemon itself. +The build's `--add-host` flag only adds a mapping inside build containers. A +separate Buildx builder has its own registry configuration; setting Docker +Engine's `registry-mirrors` does not configure that builder. See Docker's +[BuildKit configuration guide](https://docs.docker.com/build/buildkit/configure/). + +Proxy's OCI endpoint is pull-only. BuildKit registry-cache exports with +`--cache-to type=registry` need a separate writable registry. Other cache export +backends can be used alongside proxy. From 16bd0e5a59371c13773f9871cb76a7a0b3b3da53 Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Wed, 28 Oct 2026 12:50:57 +0000 Subject: [PATCH 473/633] Drop FillVersionPublishedAt until it has a caller --- internal/database/database_test.go | 42 ------------------------------ internal/database/queries.go | 30 --------------------- 2 files changed, 72 deletions(-) diff --git a/internal/database/database_test.go b/internal/database/database_test.go index 09f5a93..f8956a6 100644 --- a/internal/database/database_test.go +++ b/internal/database/database_test.go @@ -1112,48 +1112,6 @@ func BenchmarkMigrateSchemaFullyMigrated(b *testing.B) { } } -func TestFillVersionPublishedAt(t *testing.T) { - runWithBothDatabases(t, func(t *testing.T, db *DB) { - const purl, pkg = "pkg:npm/leftpad@1.0.0", "pkg:npm/leftpad" - first := time.Date(2020, 1, 2, 3, 4, 5, 0, time.UTC) - publishedAt := func() sql.NullTime { - t.Helper() - got, err := db.GetVersionByPURL(purl) - if err != nil || got == nil { - t.Fatalf("GetVersionByPURL: %v, %v", got, err) - } - return got.PublishedAt - } - - // A row without a time, as the artifact cache writes it, is filled. - if err := db.UpsertVersion(&Version{PURL: purl, PackagePURL: pkg}); err != nil { - t.Fatalf("UpsertVersion: %v", err) - } - if err := db.FillVersionPublishedAt(purl, pkg, first.In(time.FixedZone("CEST", 2*60*60))); err != nil { - t.Fatalf("FillVersionPublishedAt: %v", err) - } - if got := publishedAt(); !got.Valid || !got.Time.Equal(first) { - t.Fatalf("PublishedAt = %v, want %v", got, first) - } - - // A stored time is kept. - if err := db.FillVersionPublishedAt(purl, pkg, first.Add(time.Hour)); err != nil { - t.Fatalf("FillVersionPublishedAt again: %v", err) - } - if got := publishedAt(); !got.Time.Equal(first) { - t.Fatalf("PublishedAt after a second fill = %v, want %v kept", got.Time, first) - } - - // A version the proxy hasn't seen gets a row. - if err := db.FillVersionPublishedAt("pkg:npm/leftpad@2.0.0", pkg, first); err != nil { - t.Fatalf("FillVersionPublishedAt new: %v", err) - } - if got, err := db.GetVersionByPURL("pkg:npm/leftpad@2.0.0"); err != nil || got == nil || !got.PublishedAt.Time.Equal(first) { - t.Fatalf("new version = %+v, %v, want published %v", got, err, first) - } - }) -} - func TestVersionPublishedAtPreserved(t *testing.T) { runWithBothDatabases(t, func(t *testing.T, db *DB) { publishedAt := time.Date(2020, 1, 2, 3, 4, 5, 0, time.UTC) diff --git a/internal/database/queries.go b/internal/database/queries.go index 0bd5e82..d7150e8 100644 --- a/internal/database/queries.go +++ b/internal/database/queries.go @@ -223,36 +223,6 @@ func (db *DB) SetVersionPublishedAt(versionPURL, packagePURL string, publishedAt return nil } -// FillVersionPublishedAt records a version's publish time, in UTC, only where -// none is stored yet, creating the versions row if needed. Callers that see -// the same version on every metadata request write it once without reading -// first. -func (db *DB) FillVersionPublishedAt(versionPURL, packagePURL string, publishedAt time.Time) error { - now := time.Now() - query := ` - INSERT INTO versions (purl, package_purl, published_at, created_at, updated_at) - VALUES (?, ?, ?, ?, ?) - ON CONFLICT(purl) DO UPDATE SET - published_at = excluded.published_at, - updated_at = excluded.updated_at - WHERE versions.published_at IS NULL - ` - if db.dialect == DialectPostgres { - query = ` - INSERT INTO versions (purl, package_purl, published_at, created_at, updated_at) - VALUES ($1, $2, $3, $4, $5) - ON CONFLICT(purl) DO UPDATE SET - published_at = EXCLUDED.published_at, - updated_at = EXCLUDED.updated_at - WHERE versions.published_at IS NULL - ` - } - if _, err := db.Exec(query, versionPURL, packagePURL, publishedAt.UTC(), now, now); err != nil { - return fmt.Errorf("filling version publish time: %w", err) - } - return nil -} - // Artifact queries func (db *DB) GetArtifact(versionPURL, filename string) (*Artifact, error) { From 7eed8bc135d459dcb4afcecf3f5ffd33e0653d35 Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Wed, 28 Oct 2026 12:58:27 +0000 Subject: [PATCH 474/633] Refer to govulncheck's -db flag in comments --- internal/config/config.go | 2 +- internal/handler/go.go | 4 ++-- internal/handler/go_vulndb.go | 5 ++--- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/internal/config/config.go b/internal/config/config.go index 0a49349..3adb7fd 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -622,7 +622,7 @@ type UpstreamConfig struct { GoSumDBsOffline bool `json:"go_sumdbs_offline" yaml:"go_sumdbs_offline"` // GoVulnDB is the Go vulnerability database served at /go/vulndb, for - // govulncheck with GOVULNDB pointing at the proxy. + // govulncheck -db pointing at the proxy. // Default: https://vuln.go.dev GoVulnDB string `json:"go_vulndb" yaml:"go_vulndb"` diff --git a/internal/handler/go.go b/internal/handler/go.go index 59c0937..1af00ed 100644 --- a/internal/handler/go.go +++ b/internal/handler/go.go @@ -63,7 +63,7 @@ func (h *GoHandler) Routes() http.Handler { // Go module paths can contain slashes, so just use the handler directly return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { // govulncheck sends HEAD for index/modules.json.gz to check that a - // GOVULNDB other than vuln.go.dev is a vulnerability database. + // -db other than vuln.go.dev is a vulnerability database. if r.Method != http.MethodGet && (r.Method != http.MethodHead || !strings.HasPrefix(r.URL.Path, "/vulndb/")) { http.Error(w, "method not allowed", http.StatusMethodNotAllowed) return @@ -83,7 +83,7 @@ func (h *GoHandler) handleRequest(w http.ResponseWriter, r *http.Request) { } // So do vulnerability database requests, which govulncheck sends when - // GOVULNDB points at /go/vulndb. A module path's first element holds a + // -db points at /go/vulndb. A module path's first element holds a // dot, so neither prefix can be a module. if rest, ok := strings.CutPrefix(path, "vulndb/"); ok { h.handleVulnDB(w, r, rest) diff --git a/internal/handler/go_vulndb.go b/internal/handler/go_vulndb.go index 8531171..d3c1598 100644 --- a/internal/handler/go_vulndb.go +++ b/internal/handler/go_vulndb.go @@ -14,8 +14,7 @@ import ( ) const ( - // defaultGoVulnDBURL is the database govulncheck uses when GOVULNDB is - // not set. + // defaultGoVulnDBURL is the database govulncheck uses without -db. defaultGoVulnDBURL = "https://vuln.go.dev" // goVulnDBMaxIDLength bounds an entry ID in a request path. Go IDs look @@ -34,7 +33,7 @@ func (h *GoHandler) SetVulnDB(vulnDBURL string) { h.vulnDBURL = configuredUpstreamURL(vulnDBURL, defaultGoVulnDBURL) } -// handleVulnDB serves the Go vulnerability database (GOVULNDB) at +// handleVulnDB serves the Go vulnerability database at // /vulndb/{path}. Only the files the database API defines are passed on: // index/db.json, index/modules.json, index/vulns.json and ID/{id}.json, each // also as .json.gz, which is what govulncheck fetches. Every file can change, From 7a3b79b10a0d57856d507aafc5395a0bbd947e6d Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Wed, 28 Oct 2026 14:07:33 +0100 Subject: [PATCH 475/633] Skip the monthly release while the last one is a draft Building on a draft's tag would release its breaking changes as a patch release, without the upgrade notes the draft waits for. --- .github/workflows/monthly-release.yml | 8 ++++++++ CONTRIBUTING.md | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/workflows/monthly-release.yml b/.github/workflows/monthly-release.yml index 6e1c28f..b0da481 100644 --- a/.github/workflows/monthly-release.yml +++ b/.github/workflows/monthly-release.yml @@ -60,6 +60,14 @@ jobs: sha=$(git rev-parse HEAD) last=$(git describe --tags --abbrev=0 --match 'v[0-9]*.[0-9]*.[0-9]*' --exclude '*-*') + # A breaking release waits as a draft for its upgrade notes. Building + # on its tag would publish those changes without them. Older tags + # were released by hand and may have no release at all. + draft=$(gh release view "$last" -R "$REPO" --json isDraft --jq .isDraft 2>/dev/null || echo false) + if [ "$draft" = true ]; then + echo "::warning::$last is still a draft, publish it first" + exit 0 + fi merged=$(git rev-list --count "$last..$sha") if [ "$merged" -eq 0 ]; then echo "Nothing merged since $last" diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 8cee063..6674b8c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -224,7 +224,7 @@ Open an issue before a bigger change: anything that changes behaviour users rely A release is cut on the 1st of each month by `.github/workflows/monthly-release.yml` when `main` is green and something has merged since the last tag. It is a patch release, or a new minor version if a pull request labelled `breaking` was merged since then (a new major version from 1.0 on), so label pull requests that change behaviour users rely on. -A patch release is published on its own. A breaking release is created as a draft, without images or a chart: write the upgrade notes in it and publish it, which starts `publish.yml` for the images and the chart. +A patch release is published on its own. A breaking release is created as a draft, without images or a chart: write the upgrade notes in it and publish it, which starts `publish.yml` for the images and the chart. Until then the monthly run releases nothing. Scheduled runs only report what they would release until the repository variable `MONTHLY_RELEASE` is set to `true`. The workflow can also be started by hand from the Actions tab, as a dry run by default. From 9240f9edef2152a2873bf578dad97db28b67e056 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Wed, 28 Oct 2026 14:10:48 +0100 Subject: [PATCH 476/633] Keep the upstream Composer index in memory Serving packages.json fetched the upstream's on every request with cache_metadata off, and after each TTL with it on, so every Composer command waited on the upstream and on http_timeout when it was down. The parsed index is now kept for the metadata TTL, or an hour when it is 0, and a failed fetch is not retried within that time. --- docs/configuration.md | 2 +- internal/handler/composer.go | 6 +- internal/handler/composer_advisories.go | 72 +++++++++++++++-- internal/handler/composer_advisories_test.go | 85 ++++++++++++++++++++ 4 files changed, 157 insertions(+), 8 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 7eb8485..1fe9629 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -347,7 +347,7 @@ upstream: composer_repository: "https://upstream-proxy.example.com/composer" ``` -The proxy reads `packages.json` from `upstream.composer_repository`, cached for the metadata TTL, and passes on its `security-advisories` and `filter` entries (Packagist's malware list) with URLs pointing at the proxy. An upstream that offers neither is still usable; `composer audit` then reports nothing from it. A filter `api-url` is not passed on; Composer reads filter entries from package metadata instead. +The proxy reads `packages.json` from `upstream.composer_repository` and keeps it in memory for the metadata TTL, or an hour when the TTL is 0, whether or not `cache_metadata` is on. A failed fetch is not retried within that time: the last good copy is used, or the entries are left out. It passes on its `security-advisories` and `filter` entries (Packagist's malware list) with URLs pointing at the proxy. An upstream that offers neither is still usable; `composer audit` then reports nothing from it. A filter `api-url` is not passed on; Composer reads filter entries from package metadata instead. `upstream.hex_api` is used for cooldown timestamps and must expose Hex's `/api/packages/{name}` JSON endpoint. diff --git a/internal/handler/composer.go b/internal/handler/composer.go index bf04da0..2084ae6 100644 --- a/internal/handler/composer.go +++ b/internal/handler/composer.go @@ -55,6 +55,7 @@ type ComposerHandler struct { repoURL string proxyURL string routes packageRoutes + indexCache composerIndexCache } // NewComposerHandler creates a new Composer protocol handler. @@ -110,8 +111,9 @@ func (h *ComposerHandler) Routes() http.Handler { // handleServiceIndex returns the Composer repository service index. It // carries the security advisories and filter list entries of the default -// repository's own index when that has them; if the repository cannot be -// reached and nothing is cached, the index goes out without them. +// repository's own index when that has them, kept in memory for the metadata +// TTL; if the repository has not been reached, the index goes out without +// them. func (h *ComposerHandler) handleServiceIndex(w http.ResponseWriter, r *http.Request) { index := map[string]any{ "packages": map[string]any{}, diff --git a/internal/handler/composer_advisories.go b/internal/handler/composer_advisories.go index e77da37..ed8f4bc 100644 --- a/internal/handler/composer_advisories.go +++ b/internal/handler/composer_advisories.go @@ -8,6 +8,10 @@ import ( "net/http" "net/url" "strings" + "sync" + "time" + + "golang.org/x/sync/singleflight" ) const ( @@ -32,18 +36,76 @@ type composerUpstreamIndex struct { filterSummaryURL string } -// upstreamIndex fetches the default repository's packages.json, cached for -// the metadata TTL. ok is false when the repository could not be reached and -// nothing is cached; a document without the fields yields an empty index. +// composerIndexFallbackTTL is how long the upstream index is kept when the +// metadata TTL is 0. +const composerIndexFallbackTTL = time.Hour + +// composerIndexCache keeps the default repository's parsed packages.json in +// memory, whether or not metadata caching is on, so that serving the proxy's +// own packages.json does not wait on the upstream. A failed fetch is +// remembered for the same interval: the last good copy, or none, is used +// until then instead of retrying on every request. The zero value is ready +// to use. +type composerIndexCache struct { + mu sync.Mutex + index composerUpstreamIndex + ok bool + fetchedAt time.Time + group singleflight.Group + now func() time.Time // for tests; nil means time.Now +} + +func (c *composerIndexCache) clock() time.Time { + if c.now != nil { + return c.now() + } + return time.Now() +} + +// upstreamIndex returns the default repository's packages.json fields. ok is +// false when the repository has not been reached since the proxy started; a +// document without the fields yields an empty index. func (h *ComposerHandler) upstreamIndex(ctx context.Context) (composerUpstreamIndex, bool) { + c := &h.indexCache + ttl := h.proxy.MetadataTTL + if ttl <= 0 { + ttl = composerIndexFallbackTTL + } + c.mu.Lock() + if !c.fetchedAt.IsZero() && c.clock().Sub(c.fetchedAt) < ttl { + index, ok := c.index, c.ok + c.mu.Unlock() + return index, ok + } + c.mu.Unlock() + + // One fetch serves every waiting request. It is detached from the + // request that started it, so a client hanging up neither fails the + // others nor gets remembered as an upstream failure. + _, _, _ = c.group.Do("index", func() (any, error) { + index, err := h.fetchUpstreamIndex(context.WithoutCancel(ctx)) + c.mu.Lock() + defer c.mu.Unlock() + if err == nil { + c.index, c.ok = index, true + } + c.fetchedAt = c.clock() + return nil, nil + }) + c.mu.Lock() + defer c.mu.Unlock() + return c.index, c.ok +} + +func (h *ComposerHandler) fetchUpstreamIndex(ctx context.Context) (composerUpstreamIndex, error) { indexURL := h.repoURL + "/packages.json" cacheKey := hashedCacheKey(h.repoURL, "/packages.json") res := h.proxy.fetchMetadataDocument(ctx, "composer", cacheKey, indexURL, contentTypeJSON, nil) if res.err != nil { h.proxy.Logger.Warn("failed to fetch upstream composer packages.json", "url", indexURL, "error", res.err) - return composerUpstreamIndex{}, false + return composerUpstreamIndex{}, res.err } - return parseComposerUpstreamIndex(indexURL, res.body), true + return parseComposerUpstreamIndex(indexURL, res.body), nil } func parseComposerUpstreamIndex(indexURL string, body []byte) composerUpstreamIndex { diff --git a/internal/handler/composer_advisories_test.go b/internal/handler/composer_advisories_test.go index 6adc7d2..5e1a204 100644 --- a/internal/handler/composer_advisories_test.go +++ b/internal/handler/composer_advisories_test.go @@ -9,7 +9,9 @@ import ( "slices" "strings" "sync" + "sync/atomic" "testing" + "time" ) // composerAdvisoryUpstream serves a packages.json advertising security @@ -18,6 +20,7 @@ import ( type composerAdvisoryUpstream struct { *httptest.Server index string + down atomic.Bool // answer every request with 503 mu sync.Mutex requests []*recordedRequest @@ -43,6 +46,10 @@ func newComposerAdvisoryUpstream(t *testing.T) *composerAdvisoryUpstream { u.mu.Lock() u.requests = append(u.requests, &recordedRequest{r.Method, r.URL.Path, r.URL.Query(), string(body), r.Header.Clone()}) u.mu.Unlock() + if u.down.Load() { + w.WriteHeader(http.StatusServiceUnavailable) + return + } w.Header().Set("Content-Type", "application/json") switch r.URL.Path { case "/packages.json": @@ -164,6 +171,84 @@ func TestComposerServiceIndexUpstreamDown(t *testing.T) { } } +func advertisesAdvisories(index map[string]any) bool { + _, ok := index["security-advisories"] + return ok +} + +func TestComposerServiceIndexUpstreamDownNotRetried(t *testing.T) { + upstream := newComposerAdvisoryUpstream(t) + upstream.down.Store(true) + h := newAdvisoryComposerHandler(t, upstream) + + serveComposerIndex(t, h) + fetched := len(upstream.requestsTo("/packages.json")) + if fetched == 0 { + t.Fatal("first packages.json made no upstream request") + } + index := serveComposerIndex(t, h) + if got := len(upstream.requestsTo("/packages.json")); got != fetched { + t.Errorf("second packages.json made %d upstream requests, want none", got-fetched) + } + if advertisesAdvisories(index) { + t.Errorf("security-advisories advertised without an upstream index: %v", index["security-advisories"]) + } +} + +func TestComposerServiceIndexKeptInMemory(t *testing.T) { + upstream := newComposerAdvisoryUpstream(t) + h := newAdvisoryComposerHandler(t, upstream) + h.proxy.MetadataTTL = 0 + now := time.Now() + h.indexCache.now = func() time.Time { return now } + + if !advertisesAdvisories(serveComposerIndex(t, h)) { + t.Fatal("security-advisories not advertised") + } + if got := len(upstream.requestsTo("/packages.json")); got != 1 { + t.Fatalf("upstream packages.json requests = %d, want 1", got) + } + + // Within the interval the copy in memory is served without asking. + upstream.down.Store(true) + now = now.Add(composerIndexFallbackTTL - time.Minute) + if !advertisesAdvisories(serveComposerIndex(t, h)) { + t.Error("security-advisories dropped within the interval") + } + if got := len(upstream.requestsTo("/packages.json")); got != 1 { + t.Errorf("upstream packages.json requests = %d within the interval, want 1", got) + } + + // After it the index is fetched again; the failure keeps the last good + // copy and is not retried until the next interval. + now = now.Add(2 * time.Minute) + if !advertisesAdvisories(serveComposerIndex(t, h)) { + t.Error("last good copy dropped after a failed fetch") + } + fetched := len(upstream.requestsTo("/packages.json")) + if fetched < 2 { + t.Fatalf("upstream packages.json requests = %d after the interval, want a new fetch", fetched) + } + serveComposerIndex(t, h) + if got := len(upstream.requestsTo("/packages.json")); got != fetched { + t.Errorf("failed fetch retried %d times within the interval", got-fetched) + } +} + +func TestComposerServiceIndexFetchedOnce(t *testing.T) { + upstream := newComposerAdvisoryUpstream(t) + h := newAdvisoryComposerHandler(t, upstream) + + var wg sync.WaitGroup + for range 10 { + wg.Go(func() { serveComposer(h, "/packages.json") }) + } + wg.Wait() + if got := len(upstream.requestsTo("/packages.json")); got != 1 { + t.Errorf("upstream packages.json requests = %d for parallel requests, want 1", got) + } +} + func serveAdvisories(h *ComposerHandler, method, target, body string) *httptest.ResponseRecorder { var reader io.Reader if body != "" { From 93744379603eb097f450f6b5dda8162fcbce2fc6 Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Wed, 28 Oct 2026 13:45:11 +0000 Subject: [PATCH 477/633] Bump git-pkgs/registries to v0.12.0 --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 94fbe6c..dd44b27 100644 --- a/go.mod +++ b/go.mod @@ -17,7 +17,7 @@ require ( github.com/git-pkgs/integrity v0.2.0 github.com/git-pkgs/magic v0.5.0 github.com/git-pkgs/purl v0.1.24 - github.com/git-pkgs/registries v0.11.1 + github.com/git-pkgs/registries v0.12.0 github.com/git-pkgs/sbom v0.1.7 github.com/git-pkgs/spdx v0.3.3 github.com/go-chi/chi/v5 v5.5.0 diff --git a/go.sum b/go.sum index 28c811f..3b6c5c5 100644 --- a/go.sum +++ b/go.sum @@ -135,8 +135,8 @@ github.com/git-pkgs/pom v0.1.8 h1:mXCuWw6XkrKKZb6IurRZyIznYLPhV5AYg5ZO38Uj/l8= github.com/git-pkgs/pom v0.1.8/go.mod h1:sQBeYRzjCUn77fcABPn6I/a1aBHFxm55jTmkNxYtGtw= github.com/git-pkgs/purl v0.1.24 h1:JMEB1uNWI+baEpl6AeVx+Kfq1c+xkou+THxa5KJ3MrU= github.com/git-pkgs/purl v0.1.24/go.mod h1:l8rCfvt5iXVrFIcoQ8+vCLOaHhrtEW2id0LENP+IzMw= -github.com/git-pkgs/registries v0.11.1 h1:AkVQ15Rnrho4tbTnocYuHQuzNQ1m9jEjWKTA8P3xnPI= -github.com/git-pkgs/registries v0.11.1/go.mod h1:bKxgdy/PwnzQXL2FnOktmSQHTKk6B85MA4VtvvYItPc= +github.com/git-pkgs/registries v0.12.0 h1:fhxYaR5iR9Zlx6doLkjlXurVqPxkYcXmNFiIrFeV3/k= +github.com/git-pkgs/registries v0.12.0/go.mod h1:bKxgdy/PwnzQXL2FnOktmSQHTKk6B85MA4VtvvYItPc= github.com/git-pkgs/sbom v0.1.7 h1:4eboy/Yed/SC9ZcC/xT50iywHPzWbmCU7azDb5T1RbA= github.com/git-pkgs/sbom v0.1.7/go.mod h1:KIkZX0v31l62SeFsi2qsQXcZRVrAXvIdPQ4x4en+G2M= github.com/git-pkgs/spdx v0.3.3 h1:xXlUoZzjDSkwxC6mWHfaW+kc8a0+LH6d+gGHuZWftqI= From f5251d38344e3beb97e16cb5154b9165decec786 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 25 Oct 2026 19:40:13 +0100 Subject: [PATCH 478/633] Mirror CRAN source packages proxy mirror pkg:cran/@ now fetches the source tarball from upstream.cran and caches it under the key the CRAN handler reads, ("cran", name, version, name_version.tar.gz), which serves both /src/contrib/{file} and /src/contrib/Archive/{name}/{file}. A current version comes from src/contrib and is checked against the MD5sum in src/contrib/PACKAGES.gz, read once per five minutes. Other versions come from src/contrib/Archive/{name}/, checked against the MD5sum in that directory's PACKAGES.rds, an R-serialized character matrix read by a small decoder. Without a PACKAGES.rds the archived tarball is fetched unchecked, as the handler fetches it. Listing versions dropped duplicates: the CRAN client lists each archived version twice. --- docs/configuration.md | 2 + internal/mirror/cran.go | 260 ++++++++++++++ internal/mirror/cran_rds.go | 332 ++++++++++++++++++ internal/mirror/cran_test.go | 61 ++++ internal/mirror/mirror.go | 6 + .../mirror/testdata/cran-archive-PACKAGES.rds | Bin 0 -> 370 bytes internal/mirror/versions.go | 12 +- internal/server/mirror_cran_test.go | 179 ++++++++++ 8 files changed, 849 insertions(+), 3 deletions(-) create mode 100644 internal/mirror/cran.go create mode 100644 internal/mirror/cran_rds.go create mode 100644 internal/mirror/cran_test.go create mode 100644 internal/mirror/testdata/cran-archive-PACKAGES.rds create mode 100644 internal/server/mirror_cran_test.go diff --git a/docs/configuration.md b/docs/configuration.md index 832860c..7cdbe64 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1625,6 +1625,8 @@ A Julia package is mirrored with the source tarball `upstream.julia` serves at ` A Swift release is mirrored from `upstream.swift` with its source archive, checked against the SHA-256 checksum in the release metadata and cached under the key `/swift/{scope}/{name}/{version}.zip` serves. Give the registry scope as the namespace, for example `pkg:swift/apple/swift-argument-parser@1.5.0` for `apple.swift-argument-parser`, or the source repository, as SBOM tools write it: `pkg:swift/github.com/apple/swift-argument-parser@1.5.0` is looked up through the upstream's `/identifiers`, and the first identity it lists is mirrored. The release metadata, the package's release list and the release's `Package.swift` with its version-specific manifests are stored in the metadata cache too, so `swift package resolve` works with the upstream unreachable. Serving them, and the archive, offline needs `cache_metadata` on the server, since the archive is served with the checksum from the release metadata. +A CRAN version is mirrored as its source package from `upstream.cran`, for example `pkg:cran/glue@1.8.1`, and cached under the file name, so it is served for both `/cran/src/contrib/{file}` and `/cran/src/contrib/Archive/{name}/{file}`, where CRAN moves it once a newer version is published. The current version is fetched from `src/contrib` and checked against the MD5 sum in `src/contrib/PACKAGES.gz`; older versions from `src/contrib/Archive/{name}/`, checked against the MD5 sum in that directory's `PACKAGES.rds`, or unchecked when the repository has none there. Binary packages aren't mirrored, and neither are the `PACKAGES` indexes, so `install.packages` needs the upstream or an index the proxy already cached; installing the tarball by URL works without either. + | Flag | Default | Description | |------|---------|-------------| | `--sbom` | | Path to CycloneDX or SPDX SBOM file | diff --git a/internal/mirror/cran.go b/internal/mirror/cran.go new file mode 100644 index 0000000..b462ce6 --- /dev/null +++ b/internal/mirror/cran.go @@ -0,0 +1,260 @@ +package mirror + +import ( + "bufio" + "bytes" + "compress/gzip" + "context" + "errors" + "fmt" + "io" + "net/url" + "regexp" + "strings" + "sync" + "time" + + "github.com/git-pkgs/registries" +) + +const ( + cranDefaultURL = "https://cloud.r-project.org" + // cranIndexTTL is how long a fetched src/contrib/PACKAGES is reused, so + // mirroring many packages, or every version of one, reads it once. + cranIndexTTL = 5 * time.Minute + // cranMaxIndexSize limits the decompressed PACKAGES and PACKAGES.rds. + // CRAN's PACKAGES is about 7 MB. + cranMaxIndexSize = 64 << 20 + // cranMaxLine limits a line of PACKAGES. + cranMaxLine = 1 << 20 +) + +var ( + cranNamePattern = regexp.MustCompile(`^[A-Za-z][A-Za-z0-9.]*$`) + cranVersionPattern = regexp.MustCompile(`^[0-9]+[0-9.-]*$`) +) + +// cranIndex caches the MD5 sums src/contrib/PACKAGES lists for the current +// source packages of a CRAN repository, by file name. +type cranIndex struct { + mu sync.Mutex + baseURL string + fetched time.Time + md5s map[string]string +} + +// mirrorCRAN mirrors the source package of a CRAN version under the key the +// CRAN handler caches it under, ("cran", name, version, name_version.tar.gz), +// which both /src/contrib/{file} and /src/contrib/Archive/{name}/{file} are +// served from. A current version is fetched from src/contrib and checked +// against the MD5 sum src/contrib/PACKAGES lists; any other version from +// src/contrib/Archive, checked against the MD5 sum in that directory's +// PACKAGES.rds, or unchecked where the repository has none, as the handler +// fetches it. +func (m *Mirror) mirrorCRAN(ctx context.Context, pv PackageVersion, tracker *progressTracker) { + fail := func(err error) { m.recordFailure(tracker, pv, err) } + if !cranNamePattern.MatchString(pv.Name) { + fail(fmt.Errorf("invalid CRAN package name %q", pv.Name)) + return + } + if !cranVersionPattern.MatchString(pv.Version) { + fail(fmt.Errorf("invalid CRAN version %q", pv.Version)) + return + } + baseURL, client, err := m.cranRepository(pv.Name) + if err != nil { + fail(err) + return + } + filename := pv.Name + "_" + pv.Version + ".tar.gz" + + current, err := m.cranCurrentMD5s(ctx, baseURL, client) + if err != nil { + fail(fmt.Errorf("reading src/contrib/PACKAGES: %w", err)) + return + } + downloadURL := baseURL + "/src/contrib/" + filename + sum, ok := current[filename] + if !ok { + archive := baseURL + "/src/contrib/Archive/" + url.PathEscape(pv.Name) + "/" + downloadURL = archive + filename + if sum, err = cranArchivedMD5(ctx, client, archive+"PACKAGES.rds", pv.Version); err != nil { + fail(fmt.Errorf("reading the archive's PACKAGES.rds: %w", err)) + return + } + } + var declared string + if sum != "" { + declared = "md5-" + sum + } + result, err := m.proxy.GetOrFetchArtifactFromURLWithIntegrity(ctx, pv.Ecosystem, pv.Name, pv.Version, filename, downloadURL, declared) + if err != nil { + fail(err) + return + } + m.recordResult(tracker, pv, result) +} + +// cranRepository returns the repository to mirror a CRAN package from and +// the client to read its indexes with: those Versions lists versions with, +// which come from upstream.cran, or the public CRAN. +func (m *Mirror) cranRepository(name string) (string, *registries.Client, error) { + r, _ := m.Versions.(RegistryVersions) + baseURL := cranDefaultURL + if r.URL != nil { + configured, err := r.URL("cran", name) + if err != nil { + return "", nil, err + } + if configured != "" { + baseURL = strings.TrimSuffix(configured, "/") + } + } + client := r.Client + if client == nil { + client = registries.DefaultClient() + } + return baseURL, client, nil +} + +// cranCurrentMD5s returns the MD5 sums of the current source packages of the +// repository at baseURL, reading src/contrib/PACKAGES.gz unless it was read +// within cranIndexTTL. +func (m *Mirror) cranCurrentMD5s(ctx context.Context, baseURL string, client *registries.Client) (map[string]string, error) { + m.cran.mu.Lock() + defer m.cran.mu.Unlock() + if m.cran.md5s != nil && m.cran.baseURL == baseURL && time.Since(m.cran.fetched) < cranIndexTTL { + return m.cran.md5s, nil + } + body, err := client.GetBody(ctx, baseURL+"/src/contrib/PACKAGES.gz") + if err != nil { + return nil, err + } + data, err := gunzipLimited(body) + if err != nil { + return nil, err + } + sums, err := parseCRANPackages(data) + if err != nil { + return nil, err + } + m.cran.md5s = sums + m.cran.baseURL = baseURL + m.cran.fetched = time.Now() + return m.cran.md5s, nil +} + +// cranArchivedMD5 returns the MD5 sum the PACKAGES.rds at rdsURL lists for +// version, or "" when the repository has no PACKAGES.rds there or it doesn't +// list version. +func cranArchivedMD5(ctx context.Context, client *registries.Client, rdsURL, version string) (string, error) { + body, err := client.GetBody(ctx, rdsURL) + var httpErr *registries.HTTPError + if errors.As(err, &httpErr) && httpErr.IsNotFound() { + return "", nil + } + if err != nil { + return "", err + } + data, err := gunzipLimited(body) + if err != nil { + return "", err + } + sums, err := parseCRANArchiveRDS(data) + if err != nil { + return "", err + } + return sums[version], nil +} + +// gunzipLimited decompresses data, or returns it as is when it isn't +// gzipped, as R reads both. +func gunzipLimited(data []byte) ([]byte, error) { + if !bytes.HasPrefix(data, []byte{0x1f, 0x8b}) { + return data, nil + } + zr, err := gzip.NewReader(bytes.NewReader(data)) + if err != nil { + return nil, err + } + defer func() { _ = zr.Close() }() + out, err := io.ReadAll(io.LimitReader(zr, cranMaxIndexSize+1)) + if err != nil { + return nil, err + } + if len(out) > cranMaxIndexSize { + return nil, fmt.Errorf("index is larger than %d bytes", cranMaxIndexSize) + } + return out, nil +} + +// parseCRANPackages returns the MD5sum of each source package a PACKAGES +// file lists, by file name. Entries without one are listed with "". +func parseCRANPackages(data []byte) (map[string]string, error) { + sums := map[string]string{} + var name, version, sum string + flush := func() { + if name != "" && version != "" { + sums[name+"_"+version+".tar.gz"] = sum + } + name, version, sum = "", "", "" + } + scanner := bufio.NewScanner(bytes.NewReader(data)) + scanner.Buffer(nil, cranMaxLine) + for scanner.Scan() { + line := scanner.Text() + if strings.TrimSpace(line) == "" { + flush() + continue + } + key, value, ok := strings.Cut(line, ":") + if !ok || strings.HasPrefix(line, " ") || strings.HasPrefix(line, "\t") { + continue + } + switch value = strings.TrimSpace(value); key { + case "Package": + name = value + case "Version": + version = value + case "MD5sum": + sum = value + } + } + flush() + return sums, scanner.Err() +} + +// parseCRANArchiveRDS returns the MD5sum of each version in a CRAN archive +// directory's PACKAGES.rds, a character matrix with a column per field, +// saved by R. +func parseCRANArchiveRDS(data []byte) (map[string]string, error) { + values, nrow, columns, err := readRDSStringMatrix(data) + if err != nil { + return nil, err + } + versionCol, sumCol := -1, -1 + for i, column := range columns { + switch column { + case "Version": + versionCol = i + case "MD5sum": + sumCol = i + } + } + if versionCol < 0 || sumCol < 0 { + return nil, errors.New("PACKAGES.rds has no Version or MD5sum column") + } + sums := make(map[string]string, nrow) + for row := range nrow { + version := values[versionCol*nrow+row] + if version != nil { + sum := values[sumCol*nrow+row] + if sum != nil { + sums[*version] = *sum + } else { + sums[*version] = "" + } + } + } + return sums, nil +} diff --git a/internal/mirror/cran_rds.go b/internal/mirror/cran_rds.go new file mode 100644 index 0000000..016c8b2 --- /dev/null +++ b/internal/mirror/cran_rds.go @@ -0,0 +1,332 @@ +package mirror + +import ( + "bytes" + "encoding/binary" + "errors" + "fmt" + "io" +) + +// R serialization types and flags, from R's serialize.c. Only those a +// character matrix and its dimnames are written with are read. +const ( + rdsSymSXP = 1 + rdsListSXP = 2 + rdsCharSXP = 9 + rdsLglSXP = 10 + rdsIntSXP = 13 + rdsStrSXP = 16 + rdsVecSXP = 19 + rdsNilValue = 254 + rdsRefSXP = 255 + + rdsTypeMask = 0xff + rdsRefShift = 8 + rdsHasAttr = 1 << 9 + rdsHasTag = 1 << 10 + + rdsMagic = "X\n" + rdsMatrix = 2 // dimensions of a matrix + rdsFormatV2 = 2 + rdsFormatV3 = 3 + + // rdsMaxDepth limits nesting; a matrix needs four levels. + rdsMaxDepth = 16 +) + +// rdsSymbol is a symbol, the tag of an attribute. +type rdsSymbol string + +// rdsAttr is one attribute of a vector. +type rdsAttr struct { + tag rdsSymbol + value any +} + +// rdsVector is an R vector: []*string (nil for NA) for a character vector, +// []int32 for an integer or logical one, []any for a list. +type rdsVector struct { + values any + attrs []rdsAttr +} + +// rdsReader reads R's XDR serialization format, as saveRDS writes it +// uncompressed. +type rdsReader struct { + r *bytes.Reader + refs []any +} + +// readRDSStringMatrix reads an R character matrix saved by saveRDS, +// uncompressed, returning its values in column-major order, nil for NA, its +// number of rows and its column names. +func readRDSStringMatrix(data []byte) (values []*string, nrow int, columns []string, err error) { + rd := &rdsReader{r: bytes.NewReader(data)} + if err := rd.header(); err != nil { + return nil, 0, nil, err + } + item, err := rd.item(0) + if err != nil { + return nil, 0, nil, fmt.Errorf("reading rds: %w", err) + } + vec, ok := item.(*rdsVector) + if !ok { + return nil, 0, nil, errors.New("rds is not a vector") + } + values, ok = vec.values.([]*string) + if !ok { + return nil, 0, nil, errors.New("rds is not a character vector") + } + var dim []int32 + var dimnames []any + for _, attr := range vec.attrs { + v, _ := attr.value.(*rdsVector) + if v == nil { + continue + } + switch attr.tag { + case "dim": + dim, _ = v.values.([]int32) + case "dimnames": + dimnames, _ = v.values.([]any) + } + } + if len(dim) != rdsMatrix || dim[0] < 0 || dim[1] < 0 || int(dim[0])*int(dim[1]) != len(values) { + return nil, 0, nil, errors.New("rds is not a matrix") + } + if len(dimnames) != rdsMatrix { + return nil, 0, nil, errors.New("rds matrix has no column names") + } + names, _ := dimnames[1].(*rdsVector) + if names == nil { + return nil, 0, nil, errors.New("rds matrix has no column names") + } + colnames, _ := names.values.([]*string) + if len(colnames) != int(dim[1]) { + return nil, 0, nil, errors.New("rds matrix has no column names") + } + columns = make([]string, len(colnames)) + for i, name := range colnames { + if name != nil { + columns[i] = *name + } + } + return values, int(dim[0]), columns, nil +} + +func (rd *rdsReader) header() error { + magic := make([]byte, len(rdsMagic)) + if _, err := io.ReadFull(rd.r, magic); err != nil || string(magic) != rdsMagic { + return errors.New("rds is not in R's XDR format") + } + version, err := rd.int() + if err != nil { + return err + } + // The R version that wrote it and the oldest that reads it. + for range 2 { + if _, err := rd.int(); err != nil { + return err + } + } + switch version { + case rdsFormatV2: + case rdsFormatV3: + // The native encoding, unused: CHARSXPs are UTF-8 or ASCII. + n, err := rd.int() + if err != nil { + return err + } + if n < 0 || int64(n) > int64(rd.r.Len()) { + return errors.New("rds header is truncated") + } + if _, err := rd.r.Seek(int64(n), io.SeekCurrent); err != nil { + return err + } + default: + return fmt.Errorf("unsupported rds format version %d", version) + } + return nil +} + +func (rd *rdsReader) int() (int32, error) { + var v int32 + if err := binary.Read(rd.r, binary.BigEndian, &v); err != nil { + return 0, fmt.Errorf("rds is truncated: %w", err) + } + return v, nil +} + +// length reads a vector length, which needs no more items than are left. +func (rd *rdsReader) length() (int, error) { + n, err := rd.int() + if err != nil { + return 0, err + } + if n < 0 { + return 0, errors.New("rds has a long vector") + } + // Each element takes at least four bytes. + if int64(n)*4 > int64(rd.r.Len()) { + return 0, errors.New("rds is truncated") + } + return int(n), nil +} + +func (rd *rdsReader) item(depth int) (any, error) { + if depth > rdsMaxDepth { + return nil, errors.New("rds is nested too deeply") + } + flags, err := rd.int() + if err != nil { + return nil, err + } + kind := flags & rdsTypeMask + hasAttr := flags&rdsHasAttr != 0 + switch kind { + case rdsNilValue: + return nil, nil + case rdsRefSXP: + index := int(flags >> rdsRefShift) + if index == 0 { + i, err := rd.int() + if err != nil { + return nil, err + } + index = int(i) + } + if index < 1 || index > len(rd.refs) { + return nil, fmt.Errorf("rds refers to unknown item %d", index) + } + return rd.refs[index-1], nil + case rdsSymSXP: + name, err := rd.item(depth + 1) + if err != nil { + return nil, err + } + s, ok := name.(*string) + if !ok || s == nil { + return nil, errors.New("rds symbol has no name") + } + sym := rdsSymbol(*s) + rd.refs = append(rd.refs, sym) + return sym, nil + case rdsCharSXP: + return rd.charsxp() + case rdsListSXP: + return rd.pairlist(flags, depth) + case rdsStrSXP, rdsIntSXP, rdsLglSXP, rdsVecSXP: + vec := &rdsVector{} + if vec.values, err = rd.vector(kind, depth); err != nil { + return nil, err + } + if hasAttr { + attrs, err := rd.item(depth + 1) + if err != nil { + return nil, err + } + vec.attrs, _ = attrs.([]rdsAttr) + } + return vec, nil + default: + return nil, fmt.Errorf("unsupported rds item type %d", kind) + } +} + +func (rd *rdsReader) charsxp() (*string, error) { + n, err := rd.int() + if err != nil { + return nil, err + } + if n == -1 { + return nil, nil // NA_character_ + } + if n < 0 || int64(n) > int64(rd.r.Len()) { + return nil, errors.New("rds is truncated") + } + b := make([]byte, n) + if _, err := io.ReadFull(rd.r, b); err != nil { + return nil, err + } + s := string(b) + return &s, nil +} + +func (rd *rdsReader) vector(kind int32, depth int) (any, error) { + n, err := rd.length() + if err != nil { + return nil, err + } + switch kind { + case rdsStrSXP: + values := make([]*string, n) + for i := range values { + flags, err := rd.int() + if err != nil { + return nil, err + } + if flags&rdsTypeMask != rdsCharSXP { + return nil, errors.New("rds character vector holds a non-string") + } + if values[i], err = rd.charsxp(); err != nil { + return nil, err + } + } + return values, nil + case rdsIntSXP, rdsLglSXP: + values := make([]int32, n) + for i := range values { + if values[i], err = rd.int(); err != nil { + return nil, err + } + } + return values, nil + default: + values := make([]any, n) + for i := range values { + if values[i], err = rd.item(depth + 1); err != nil { + return nil, err + } + } + return values, nil + } +} + +// pairlist reads an attribute pairlist, its first cell's flags already +// read, as its tags and values. +func (rd *rdsReader) pairlist(flags int32, depth int) ([]rdsAttr, error) { + var attrs []rdsAttr + for { + if flags&rdsHasAttr != 0 { + if _, err := rd.item(depth + 1); err != nil { + return nil, err + } + } + var attr rdsAttr + if flags&rdsHasTag != 0 { + tag, err := rd.item(depth + 1) + if err != nil { + return nil, err + } + attr.tag, _ = tag.(rdsSymbol) + } + value, err := rd.item(depth + 1) + if err != nil { + return nil, err + } + attr.value = value + attrs = append(attrs, attr) + // The rest of the list follows as the next cell, or ends. + if flags, err = rd.int(); err != nil { + return nil, err + } + switch flags & rdsTypeMask { + case rdsNilValue: + return attrs, nil + case rdsListSXP: + default: + return nil, errors.New("rds pairlist is malformed") + } + } +} diff --git a/internal/mirror/cran_test.go b/internal/mirror/cran_test.go new file mode 100644 index 0000000..451af00 --- /dev/null +++ b/internal/mirror/cran_test.go @@ -0,0 +1,61 @@ +package mirror + +import ( + "maps" + "os" + "testing" +) + +// testdata/cran-archive-PACKAGES.rds was written by R 4.4.1 with saveRDS, +// shaped like the PACKAGES.rds in CRAN's src/contrib/Archive/{name}: a +// character matrix of fields, NA where a version has none. +func TestParseCRANArchiveRDS(t *testing.T) { + data, err := os.ReadFile("testdata/cran-archive-PACKAGES.rds") + if err != nil { + t.Fatal(err) + } + data, err = gunzipLimited(data) + if err != nil { + t.Fatal(err) + } + + sums, err := parseCRANArchiveRDS(data) + if err != nil { + t.Fatal(err) + } + + want := map[string]string{ + "0.9": "b7f01da919c10e36d6777fc337f48dfa", + "1.0": "e51e3cbb209131a84f3ff5938dd93602", + } + if !maps.Equal(sums, want) { + t.Errorf("sums = %v, want %v", sums, want) + } + + for i := range data { + if _, err := parseCRANArchiveRDS(data[:i]); err == nil { + t.Fatalf("truncated to %d bytes: no error", i) + } + } +} + +func TestParseCRANPackages(t *testing.T) { + data := []byte(`Package: A3 +Version: 1.0.0 +Depends: R (>= 2.15.0), xtable, + pbapply +MD5sum: 027ebdd8affce8f0effaecfcd5f5ade2 +NeedsCompilation: no + +Package: abc +Version: 2.2.1 +NeedsCompilation: no +`) + want := map[string]string{ + "A3_1.0.0.tar.gz": "027ebdd8affce8f0effaecfcd5f5ade2", + "abc_2.2.1.tar.gz": "", + } + if got, err := parseCRANPackages(data); err != nil || !maps.Equal(got, want) { + t.Errorf("parseCRANPackages = %v, %v, want %v", got, err, want) + } +} diff --git a/internal/mirror/mirror.go b/internal/mirror/mirror.go index 9b55a8c..6f74e22 100644 --- a/internal/mirror/mirror.go +++ b/internal/mirror/mirror.go @@ -43,6 +43,8 @@ type Mirror struct { // RefusePackages, when set, fails every item Images does not take with // this reason instead of fetching it, so nothing is cached. RefusePackages string + + cran cranIndex } // New creates a new Mirror with the given dependencies. @@ -255,6 +257,10 @@ func (m *Mirror) mirrorOne(ctx context.Context, pv PackageVersion, tracker *prog m.mirrorSwift(ctx, pv, tracker) return } + if pv.Ecosystem == "cran" { + m.mirrorCRAN(ctx, pv, tracker) + return + } result, err := m.proxy.GetOrFetchArtifact(ctx, pv.Ecosystem, pv.Name, pv.Version, "") if err != nil { m.recordFailure(tracker, pv, err) diff --git a/internal/mirror/testdata/cran-archive-PACKAGES.rds b/internal/mirror/testdata/cran-archive-PACKAGES.rds new file mode 100644 index 0000000000000000000000000000000000000000..9cc5a61abb09e00cde420421b11a10752b422fba GIT binary patch literal 370 zcmV-&0ge72iwFP!000001Fcb8OT#b}&bH27KsFR#rH_g@+GShY2L;&$2D@RW_*Sws zsbOncnhxBbS2NcXQSm_x9KP?%`OZ1HJkO|BddsrtQsaeJe~R`>Foy`^(&pS4bdm#{&40F zf64sMOnqfh!}O73I@mM_?D;**vM7eoq8;0(Ncl+DBoMDwZNo94iR=!Aly)6x`@REv zMmzHrdGv_^tteSRJ*v+_3CH_n7L6~KsrkMjVs8e!?)}LJe= 4.0)\nMD5sum: "+hex.EncodeToString(sum[:])+"\nNeedsCompilation: no\n") + _ = zw.Close() + + u := &cranUpstream{ + files: map[string]string{ + cranIndexPath: index.String(), + cranArchiveRDS: string(rds), + cranCurrentPath: current, + cranArchivedPath: "example 1.0 source", + "/src/contrib/Archive/example/example_0.9.tar.gz": "example 0.9 source", + "/web/packages/example/DESCRIPTION": "Package: example\nVersion: 1.1\n", + "/src/contrib/Archive/example/": `example_0.9.tar.gz +example_1.0.tar.gz`, + }, + hits: map[string]int{}, + } + for path, body := range tampered { + u.files[path] = body + } + u.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + u.mu.Lock() + body, ok := u.files[r.URL.Path] + u.hits[r.URL.Path]++ + u.mu.Unlock() + if !ok { + http.NotFound(w, r) + return + } + _, _ = io.WriteString(w, body) + })) + t.Cleanup(u.Close) + return u +} + +func (u *cranUpstream) hitsFor(path string) int { + u.mu.Lock() + defer u.mu.Unlock() + return u.hits[path] +} + +func cranMirrorConfig(upstream *cranUpstream) *config.Config { + cfg := config.Default() + cfg.Upstream.CRAN = upstream.URL + return cfg +} + +// getCRAN requests path from a CRAN handler for upstream. +func getCRAN(proxy *handler.Proxy, upstream *cranUpstream, path string) *httptest.ResponseRecorder { + rec := httptest.NewRecorder() + cran := handler.NewCRANHandlerWithUpstream(proxy, "http://proxy.test", upstream.URL) + cran.Routes().ServeHTTP(rec, httptest.NewRequest(http.MethodGet, path, nil)) + return rec +} + +func TestMirrorCRANSourcePackagesAreServedFromCache(t *testing.T) { + upstream := newCRANUpstream(t, nil) + + progress, proxy := mirrorWithConfig(t, cranMirrorConfig(upstream), "pkg:cran/example@1.1", "pkg:cran/example@1.0") + + if progress.Completed != 2 || progress.Failed != 0 { + t.Fatalf("completed = %d, failed = %d, errors = %v", progress.Completed, progress.Failed, progress.Errors) + } + for _, path := range []string{cranCurrentPath, cranArchivedPath} { + if got := upstream.hitsFor(path); got != 1 { + t.Errorf("%s fetched %d times, want 1", path, got) + } + } + + // R gets both from the cache once the upstream is down, the current + // version also from Archive/, where CRAN moves it once a newer one is + // published. + upstream.Close() + for path, want := range map[string]string{ + cranCurrentPath: "example 1.1 source", + "/src/contrib/Archive/example/example_1.1.tar.gz": "example 1.1 source", + cranArchivedPath: "example 1.0 source", + } { + if rec := getCRAN(proxy, upstream, path); rec.Code != http.StatusOK || rec.Body.String() != want { + t.Errorf("%s from cache: status = %d, body = %q, want 200 and %q", path, rec.Code, rec.Body, want) + } + } +} + +func TestMirrorCRANRefusesMD5Mismatch(t *testing.T) { + upstream := newCRANUpstream(t, map[string]string{ + cranCurrentPath: "tampered 1.1", + cranArchivedPath: "tampered 1.0", + }) + + progress, proxy := mirrorWithConfig(t, cranMirrorConfig(upstream), "pkg:cran/example@1.1", "pkg:cran/example@1.0") + + if progress.Failed != 2 || progress.Completed != 0 { + t.Fatalf("completed = %d, failed = %d, errors = %v; want both refused", progress.Completed, progress.Failed, progress.Errors) + } + upstream.Close() + for _, path := range []string{cranCurrentPath, cranArchivedPath} { + if rec := getCRAN(proxy, upstream, path); rec.Code == http.StatusOK { + t.Errorf("%s was cached although its MD5 doesn't match", path) + } + } +} + +func TestMirrorCRANArchiveWithoutPackagesRDS(t *testing.T) { + // A repository without the archive's PACKAGES.rds publishes no checksum + // for archived versions; they are fetched unchecked, as R gets them + // through the handler. + upstream := newCRANUpstream(t, nil) + delete(upstream.files, cranArchiveRDS) + + progress, _ := mirrorWithConfig(t, cranMirrorConfig(upstream), "pkg:cran/example@1.0") + + if progress.Completed != 1 || upstream.hitsFor(cranArchivedPath) != 1 { + t.Errorf("completed = %d, hits = %d, errors = %v", progress.Completed, upstream.hitsFor(cranArchivedPath), progress.Errors) + } +} + +func TestMirrorCRANAllVersions(t *testing.T) { + upstream := newCRANUpstream(t, nil) + + progress, _ := mirrorWithConfig(t, cranMirrorConfig(upstream), "pkg:cran/example") + + // The archive listing names each file twice, in the link and its text. + if progress.Total != 3 || progress.Completed != 3 || progress.Failed != 0 { + t.Fatalf("total = %d, completed = %d, failed = %d, errors = %v; want 0.9, 1.0 and 1.1 once each", progress.Total, progress.Completed, progress.Failed, progress.Errors) + } + if got := upstream.hitsFor(cranIndexPath); got != 1 { + t.Errorf("PACKAGES.gz fetched %d times, want once", got) + } +} From 3fd79ccb3666864d6cbf4639eaed82e0b7b8cf97 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 25 Oct 2026 19:44:07 +0100 Subject: [PATCH 479/633] Fuzz the PACKAGES.rds decoder It reads files from the network, so malformed input must fail with an error. Seeded with the real PACKAGES.rds; 3.7M executions found nothing. --- internal/mirror/cran_test.go | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/internal/mirror/cran_test.go b/internal/mirror/cran_test.go index 451af00..2dc130f 100644 --- a/internal/mirror/cran_test.go +++ b/internal/mirror/cran_test.go @@ -59,3 +59,25 @@ NeedsCompilation: no t.Errorf("parseCRANPackages = %v, %v, want %v", got, err, want) } } + +// FuzzReadRDSStringMatrix feeds the decoder arbitrary bytes, seeded with a +// real PACKAGES.rds: it reads files from the network, so it must reject +// anything malformed with an error rather than panic or allocate without +// bound. +func FuzzReadRDSStringMatrix(f *testing.F) { + data, err := os.ReadFile("testdata/cran-archive-PACKAGES.rds") + if err != nil { + f.Fatal(err) + } + if data, err = gunzipLimited(data); err != nil { + f.Fatal(err) + } + f.Add(data) + f.Add(data[:len(data)/2]) + f.Fuzz(func(t *testing.T, b []byte) { + values, nrow, columns, err := readRDSStringMatrix(b) + if err == nil && nrow > 0 && len(values) != nrow*len(columns) { + t.Errorf("%d values for %d rows and %d columns", len(values), nrow, len(columns)) + } + }) +} From 842fa21c0a406436e2d33adb6e244d8b12df46fa Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Mon, 26 Oct 2026 16:42:08 +0100 Subject: [PATCH 480/633] Fetch archived CRAN versions unchecked and drop the version dedup PACKAGES.rds comes from the same host as the tarball, so checking against it doesn't justify a decoder for R serialization. The duplicate archived versions are tracked in git-pkgs/registries#119. --- internal/mirror/cran.go | 81 +---- internal/mirror/cran_rds.go | 332 ------------------ internal/mirror/cran_test.go | 56 --- .../mirror/testdata/cran-archive-PACKAGES.rds | Bin 370 -> 0 bytes internal/mirror/versions.go | 12 +- internal/server/mirror_cran_test.go | 49 +-- 6 files changed, 19 insertions(+), 511 deletions(-) delete mode 100644 internal/mirror/cran_rds.go delete mode 100644 internal/mirror/testdata/cran-archive-PACKAGES.rds diff --git a/internal/mirror/cran.go b/internal/mirror/cran.go index b462ce6..02d26bd 100644 --- a/internal/mirror/cran.go +++ b/internal/mirror/cran.go @@ -5,7 +5,6 @@ import ( "bytes" "compress/gzip" "context" - "errors" "fmt" "io" "net/url" @@ -22,7 +21,7 @@ const ( // cranIndexTTL is how long a fetched src/contrib/PACKAGES is reused, so // mirroring many packages, or every version of one, reads it once. cranIndexTTL = 5 * time.Minute - // cranMaxIndexSize limits the decompressed PACKAGES and PACKAGES.rds. + // cranMaxIndexSize limits the decompressed PACKAGES. // CRAN's PACKAGES is about 7 MB. cranMaxIndexSize = 64 << 20 // cranMaxLine limits a line of PACKAGES. @@ -43,14 +42,9 @@ type cranIndex struct { md5s map[string]string } -// mirrorCRAN mirrors the source package of a CRAN version under the key the -// CRAN handler caches it under, ("cran", name, version, name_version.tar.gz), -// which both /src/contrib/{file} and /src/contrib/Archive/{name}/{file} are -// served from. A current version is fetched from src/contrib and checked -// against the MD5 sum src/contrib/PACKAGES lists; any other version from -// src/contrib/Archive, checked against the MD5 sum in that directory's -// PACKAGES.rds, or unchecked where the repository has none, as the handler -// fetches it. +// mirrorCRAN mirrors a CRAN source package under ("cran", name, version, +// name_version.tar.gz), the one entry the handler serves for both +// src/contrib and src/contrib/Archive. func (m *Mirror) mirrorCRAN(ctx context.Context, pv PackageVersion, tracker *progressTracker) { fail := func(err error) { m.recordFailure(tracker, pv, err) } if !cranNamePattern.MatchString(pv.Name) { @@ -76,12 +70,7 @@ func (m *Mirror) mirrorCRAN(ctx context.Context, pv PackageVersion, tracker *pro downloadURL := baseURL + "/src/contrib/" + filename sum, ok := current[filename] if !ok { - archive := baseURL + "/src/contrib/Archive/" + url.PathEscape(pv.Name) + "/" - downloadURL = archive + filename - if sum, err = cranArchivedMD5(ctx, client, archive+"PACKAGES.rds", pv.Version); err != nil { - fail(fmt.Errorf("reading the archive's PACKAGES.rds: %w", err)) - return - } + downloadURL = baseURL + "/src/contrib/Archive/" + url.PathEscape(pv.Name) + "/" + filename } var declared string if sum != "" { @@ -144,31 +133,8 @@ func (m *Mirror) cranCurrentMD5s(ctx context.Context, baseURL string, client *re return m.cran.md5s, nil } -// cranArchivedMD5 returns the MD5 sum the PACKAGES.rds at rdsURL lists for -// version, or "" when the repository has no PACKAGES.rds there or it doesn't -// list version. -func cranArchivedMD5(ctx context.Context, client *registries.Client, rdsURL, version string) (string, error) { - body, err := client.GetBody(ctx, rdsURL) - var httpErr *registries.HTTPError - if errors.As(err, &httpErr) && httpErr.IsNotFound() { - return "", nil - } - if err != nil { - return "", err - } - data, err := gunzipLimited(body) - if err != nil { - return "", err - } - sums, err := parseCRANArchiveRDS(data) - if err != nil { - return "", err - } - return sums[version], nil -} - // gunzipLimited decompresses data, or returns it as is when it isn't -// gzipped, as R reads both. +// gzipped. func gunzipLimited(data []byte) ([]byte, error) { if !bytes.HasPrefix(data, []byte{0x1f, 0x8b}) { return data, nil @@ -223,38 +189,3 @@ func parseCRANPackages(data []byte) (map[string]string, error) { flush() return sums, scanner.Err() } - -// parseCRANArchiveRDS returns the MD5sum of each version in a CRAN archive -// directory's PACKAGES.rds, a character matrix with a column per field, -// saved by R. -func parseCRANArchiveRDS(data []byte) (map[string]string, error) { - values, nrow, columns, err := readRDSStringMatrix(data) - if err != nil { - return nil, err - } - versionCol, sumCol := -1, -1 - for i, column := range columns { - switch column { - case "Version": - versionCol = i - case "MD5sum": - sumCol = i - } - } - if versionCol < 0 || sumCol < 0 { - return nil, errors.New("PACKAGES.rds has no Version or MD5sum column") - } - sums := make(map[string]string, nrow) - for row := range nrow { - version := values[versionCol*nrow+row] - if version != nil { - sum := values[sumCol*nrow+row] - if sum != nil { - sums[*version] = *sum - } else { - sums[*version] = "" - } - } - } - return sums, nil -} diff --git a/internal/mirror/cran_rds.go b/internal/mirror/cran_rds.go deleted file mode 100644 index 016c8b2..0000000 --- a/internal/mirror/cran_rds.go +++ /dev/null @@ -1,332 +0,0 @@ -package mirror - -import ( - "bytes" - "encoding/binary" - "errors" - "fmt" - "io" -) - -// R serialization types and flags, from R's serialize.c. Only those a -// character matrix and its dimnames are written with are read. -const ( - rdsSymSXP = 1 - rdsListSXP = 2 - rdsCharSXP = 9 - rdsLglSXP = 10 - rdsIntSXP = 13 - rdsStrSXP = 16 - rdsVecSXP = 19 - rdsNilValue = 254 - rdsRefSXP = 255 - - rdsTypeMask = 0xff - rdsRefShift = 8 - rdsHasAttr = 1 << 9 - rdsHasTag = 1 << 10 - - rdsMagic = "X\n" - rdsMatrix = 2 // dimensions of a matrix - rdsFormatV2 = 2 - rdsFormatV3 = 3 - - // rdsMaxDepth limits nesting; a matrix needs four levels. - rdsMaxDepth = 16 -) - -// rdsSymbol is a symbol, the tag of an attribute. -type rdsSymbol string - -// rdsAttr is one attribute of a vector. -type rdsAttr struct { - tag rdsSymbol - value any -} - -// rdsVector is an R vector: []*string (nil for NA) for a character vector, -// []int32 for an integer or logical one, []any for a list. -type rdsVector struct { - values any - attrs []rdsAttr -} - -// rdsReader reads R's XDR serialization format, as saveRDS writes it -// uncompressed. -type rdsReader struct { - r *bytes.Reader - refs []any -} - -// readRDSStringMatrix reads an R character matrix saved by saveRDS, -// uncompressed, returning its values in column-major order, nil for NA, its -// number of rows and its column names. -func readRDSStringMatrix(data []byte) (values []*string, nrow int, columns []string, err error) { - rd := &rdsReader{r: bytes.NewReader(data)} - if err := rd.header(); err != nil { - return nil, 0, nil, err - } - item, err := rd.item(0) - if err != nil { - return nil, 0, nil, fmt.Errorf("reading rds: %w", err) - } - vec, ok := item.(*rdsVector) - if !ok { - return nil, 0, nil, errors.New("rds is not a vector") - } - values, ok = vec.values.([]*string) - if !ok { - return nil, 0, nil, errors.New("rds is not a character vector") - } - var dim []int32 - var dimnames []any - for _, attr := range vec.attrs { - v, _ := attr.value.(*rdsVector) - if v == nil { - continue - } - switch attr.tag { - case "dim": - dim, _ = v.values.([]int32) - case "dimnames": - dimnames, _ = v.values.([]any) - } - } - if len(dim) != rdsMatrix || dim[0] < 0 || dim[1] < 0 || int(dim[0])*int(dim[1]) != len(values) { - return nil, 0, nil, errors.New("rds is not a matrix") - } - if len(dimnames) != rdsMatrix { - return nil, 0, nil, errors.New("rds matrix has no column names") - } - names, _ := dimnames[1].(*rdsVector) - if names == nil { - return nil, 0, nil, errors.New("rds matrix has no column names") - } - colnames, _ := names.values.([]*string) - if len(colnames) != int(dim[1]) { - return nil, 0, nil, errors.New("rds matrix has no column names") - } - columns = make([]string, len(colnames)) - for i, name := range colnames { - if name != nil { - columns[i] = *name - } - } - return values, int(dim[0]), columns, nil -} - -func (rd *rdsReader) header() error { - magic := make([]byte, len(rdsMagic)) - if _, err := io.ReadFull(rd.r, magic); err != nil || string(magic) != rdsMagic { - return errors.New("rds is not in R's XDR format") - } - version, err := rd.int() - if err != nil { - return err - } - // The R version that wrote it and the oldest that reads it. - for range 2 { - if _, err := rd.int(); err != nil { - return err - } - } - switch version { - case rdsFormatV2: - case rdsFormatV3: - // The native encoding, unused: CHARSXPs are UTF-8 or ASCII. - n, err := rd.int() - if err != nil { - return err - } - if n < 0 || int64(n) > int64(rd.r.Len()) { - return errors.New("rds header is truncated") - } - if _, err := rd.r.Seek(int64(n), io.SeekCurrent); err != nil { - return err - } - default: - return fmt.Errorf("unsupported rds format version %d", version) - } - return nil -} - -func (rd *rdsReader) int() (int32, error) { - var v int32 - if err := binary.Read(rd.r, binary.BigEndian, &v); err != nil { - return 0, fmt.Errorf("rds is truncated: %w", err) - } - return v, nil -} - -// length reads a vector length, which needs no more items than are left. -func (rd *rdsReader) length() (int, error) { - n, err := rd.int() - if err != nil { - return 0, err - } - if n < 0 { - return 0, errors.New("rds has a long vector") - } - // Each element takes at least four bytes. - if int64(n)*4 > int64(rd.r.Len()) { - return 0, errors.New("rds is truncated") - } - return int(n), nil -} - -func (rd *rdsReader) item(depth int) (any, error) { - if depth > rdsMaxDepth { - return nil, errors.New("rds is nested too deeply") - } - flags, err := rd.int() - if err != nil { - return nil, err - } - kind := flags & rdsTypeMask - hasAttr := flags&rdsHasAttr != 0 - switch kind { - case rdsNilValue: - return nil, nil - case rdsRefSXP: - index := int(flags >> rdsRefShift) - if index == 0 { - i, err := rd.int() - if err != nil { - return nil, err - } - index = int(i) - } - if index < 1 || index > len(rd.refs) { - return nil, fmt.Errorf("rds refers to unknown item %d", index) - } - return rd.refs[index-1], nil - case rdsSymSXP: - name, err := rd.item(depth + 1) - if err != nil { - return nil, err - } - s, ok := name.(*string) - if !ok || s == nil { - return nil, errors.New("rds symbol has no name") - } - sym := rdsSymbol(*s) - rd.refs = append(rd.refs, sym) - return sym, nil - case rdsCharSXP: - return rd.charsxp() - case rdsListSXP: - return rd.pairlist(flags, depth) - case rdsStrSXP, rdsIntSXP, rdsLglSXP, rdsVecSXP: - vec := &rdsVector{} - if vec.values, err = rd.vector(kind, depth); err != nil { - return nil, err - } - if hasAttr { - attrs, err := rd.item(depth + 1) - if err != nil { - return nil, err - } - vec.attrs, _ = attrs.([]rdsAttr) - } - return vec, nil - default: - return nil, fmt.Errorf("unsupported rds item type %d", kind) - } -} - -func (rd *rdsReader) charsxp() (*string, error) { - n, err := rd.int() - if err != nil { - return nil, err - } - if n == -1 { - return nil, nil // NA_character_ - } - if n < 0 || int64(n) > int64(rd.r.Len()) { - return nil, errors.New("rds is truncated") - } - b := make([]byte, n) - if _, err := io.ReadFull(rd.r, b); err != nil { - return nil, err - } - s := string(b) - return &s, nil -} - -func (rd *rdsReader) vector(kind int32, depth int) (any, error) { - n, err := rd.length() - if err != nil { - return nil, err - } - switch kind { - case rdsStrSXP: - values := make([]*string, n) - for i := range values { - flags, err := rd.int() - if err != nil { - return nil, err - } - if flags&rdsTypeMask != rdsCharSXP { - return nil, errors.New("rds character vector holds a non-string") - } - if values[i], err = rd.charsxp(); err != nil { - return nil, err - } - } - return values, nil - case rdsIntSXP, rdsLglSXP: - values := make([]int32, n) - for i := range values { - if values[i], err = rd.int(); err != nil { - return nil, err - } - } - return values, nil - default: - values := make([]any, n) - for i := range values { - if values[i], err = rd.item(depth + 1); err != nil { - return nil, err - } - } - return values, nil - } -} - -// pairlist reads an attribute pairlist, its first cell's flags already -// read, as its tags and values. -func (rd *rdsReader) pairlist(flags int32, depth int) ([]rdsAttr, error) { - var attrs []rdsAttr - for { - if flags&rdsHasAttr != 0 { - if _, err := rd.item(depth + 1); err != nil { - return nil, err - } - } - var attr rdsAttr - if flags&rdsHasTag != 0 { - tag, err := rd.item(depth + 1) - if err != nil { - return nil, err - } - attr.tag, _ = tag.(rdsSymbol) - } - value, err := rd.item(depth + 1) - if err != nil { - return nil, err - } - attr.value = value - attrs = append(attrs, attr) - // The rest of the list follows as the next cell, or ends. - if flags, err = rd.int(); err != nil { - return nil, err - } - switch flags & rdsTypeMask { - case rdsNilValue: - return attrs, nil - case rdsListSXP: - default: - return nil, errors.New("rds pairlist is malformed") - } - } -} diff --git a/internal/mirror/cran_test.go b/internal/mirror/cran_test.go index 2dc130f..3eae797 100644 --- a/internal/mirror/cran_test.go +++ b/internal/mirror/cran_test.go @@ -2,43 +2,9 @@ package mirror import ( "maps" - "os" "testing" ) -// testdata/cran-archive-PACKAGES.rds was written by R 4.4.1 with saveRDS, -// shaped like the PACKAGES.rds in CRAN's src/contrib/Archive/{name}: a -// character matrix of fields, NA where a version has none. -func TestParseCRANArchiveRDS(t *testing.T) { - data, err := os.ReadFile("testdata/cran-archive-PACKAGES.rds") - if err != nil { - t.Fatal(err) - } - data, err = gunzipLimited(data) - if err != nil { - t.Fatal(err) - } - - sums, err := parseCRANArchiveRDS(data) - if err != nil { - t.Fatal(err) - } - - want := map[string]string{ - "0.9": "b7f01da919c10e36d6777fc337f48dfa", - "1.0": "e51e3cbb209131a84f3ff5938dd93602", - } - if !maps.Equal(sums, want) { - t.Errorf("sums = %v, want %v", sums, want) - } - - for i := range data { - if _, err := parseCRANArchiveRDS(data[:i]); err == nil { - t.Fatalf("truncated to %d bytes: no error", i) - } - } -} - func TestParseCRANPackages(t *testing.T) { data := []byte(`Package: A3 Version: 1.0.0 @@ -59,25 +25,3 @@ NeedsCompilation: no t.Errorf("parseCRANPackages = %v, %v, want %v", got, err, want) } } - -// FuzzReadRDSStringMatrix feeds the decoder arbitrary bytes, seeded with a -// real PACKAGES.rds: it reads files from the network, so it must reject -// anything malformed with an error rather than panic or allocate without -// bound. -func FuzzReadRDSStringMatrix(f *testing.F) { - data, err := os.ReadFile("testdata/cran-archive-PACKAGES.rds") - if err != nil { - f.Fatal(err) - } - if data, err = gunzipLimited(data); err != nil { - f.Fatal(err) - } - f.Add(data) - f.Add(data[:len(data)/2]) - f.Fuzz(func(t *testing.T, b []byte) { - values, nrow, columns, err := readRDSStringMatrix(b) - if err == nil && nrow > 0 && len(values) != nrow*len(columns) { - t.Errorf("%d values for %d rows and %d columns", len(values), nrow, len(columns)) - } - }) -} diff --git a/internal/mirror/testdata/cran-archive-PACKAGES.rds b/internal/mirror/testdata/cran-archive-PACKAGES.rds deleted file mode 100644 index 9cc5a61abb09e00cde420421b11a10752b422fba..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 370 zcmV-&0ge72iwFP!000001Fcb8OT#b}&bH27KsFR#rH_g@+GShY2L;&$2D@RW_*Sws zsbOncnhxBbS2NcXQSm_x9KP?%`OZ1HJkO|BddsrtQsaeJe~R`>Foy`^(&pS4bdm#{&40F zf64sMOnqfh!}O73I@mM_?D;**vM7eoq8;0(Ncl+DBoMDwZNo94iR=!Aly)6x`@REv zMmzHrdGv_^tteSRJ*v+_3CH_n7L6~KsrkMjVs8e!?)}LJeexample_0.9.tar.gz -example_1.0.tar.gz`, + "/src/contrib/Archive/example/": `0.9 +1.0`, }, hits: map[string]int{}, } @@ -132,35 +123,16 @@ func TestMirrorCRANSourcePackagesAreServedFromCache(t *testing.T) { } func TestMirrorCRANRefusesMD5Mismatch(t *testing.T) { - upstream := newCRANUpstream(t, map[string]string{ - cranCurrentPath: "tampered 1.1", - cranArchivedPath: "tampered 1.0", - }) + upstream := newCRANUpstream(t, map[string]string{cranCurrentPath: "tampered 1.1"}) - progress, proxy := mirrorWithConfig(t, cranMirrorConfig(upstream), "pkg:cran/example@1.1", "pkg:cran/example@1.0") + progress, proxy := mirrorWithConfig(t, cranMirrorConfig(upstream), "pkg:cran/example@1.1") - if progress.Failed != 2 || progress.Completed != 0 { - t.Fatalf("completed = %d, failed = %d, errors = %v; want both refused", progress.Completed, progress.Failed, progress.Errors) + if progress.Failed != 1 || progress.Completed != 0 { + t.Fatalf("completed = %d, failed = %d, errors = %v; want it refused", progress.Completed, progress.Failed, progress.Errors) } upstream.Close() - for _, path := range []string{cranCurrentPath, cranArchivedPath} { - if rec := getCRAN(proxy, upstream, path); rec.Code == http.StatusOK { - t.Errorf("%s was cached although its MD5 doesn't match", path) - } - } -} - -func TestMirrorCRANArchiveWithoutPackagesRDS(t *testing.T) { - // A repository without the archive's PACKAGES.rds publishes no checksum - // for archived versions; they are fetched unchecked, as R gets them - // through the handler. - upstream := newCRANUpstream(t, nil) - delete(upstream.files, cranArchiveRDS) - - progress, _ := mirrorWithConfig(t, cranMirrorConfig(upstream), "pkg:cran/example@1.0") - - if progress.Completed != 1 || upstream.hitsFor(cranArchivedPath) != 1 { - t.Errorf("completed = %d, hits = %d, errors = %v", progress.Completed, upstream.hitsFor(cranArchivedPath), progress.Errors) + if rec := getCRAN(proxy, upstream, cranCurrentPath); rec.Code == http.StatusOK { + t.Errorf("%s was cached although its MD5 doesn't match", cranCurrentPath) } } @@ -169,7 +141,6 @@ func TestMirrorCRANAllVersions(t *testing.T) { progress, _ := mirrorWithConfig(t, cranMirrorConfig(upstream), "pkg:cran/example") - // The archive listing names each file twice, in the link and its text. if progress.Total != 3 || progress.Completed != 3 || progress.Failed != 0 { t.Fatalf("total = %d, completed = %d, failed = %d, errors = %v; want 0.9, 1.0 and 1.1 once each", progress.Total, progress.Completed, progress.Failed, progress.Errors) } From 2a8ceb93893de119ce90437e5ba24dc3bd2b5d6d Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 25 Oct 2026 21:21:19 +0100 Subject: [PATCH 481/633] Mirror Homebrew bottles pkg:brew PURLs failed with "unsupported ecosystem: brew", although listing their versions worked. Homebrew publishes a formula's bottles as an image on its artifact upstream, which proxy mirror can already fetch, so a brew PURL is now resolved to that image and mirrored like one: the formula's JSON API gives the tag, which carries the revision and the bottles' rebuild, and --platform selects bottles like image platforms. Listing a formula's versions returned the versioned siblings the Homebrew client adds (openssl@3.6 for openssl@3) as versions of it; they are now left out. --- README.md | 2 +- cmd/proxy/main.go | 1 + docs/configuration.md | 16 +++ internal/handler/homebrew_bottles.go | 96 +++++++++++++++++ internal/handler/homebrew_bottles_test.go | 72 +++++++++++++ internal/mirror/brew_test.go | 123 ++++++++++++++++++++++ internal/mirror/image.go | 40 ++++++- internal/mirror/mirror.go | 9 ++ internal/mirror/source.go | 8 +- internal/mirror/versions.go | 12 ++- internal/server/mirror_versions.go | 1 + 11 files changed, 372 insertions(+), 8 deletions(-) create mode 100644 internal/handler/homebrew_bottles.go create mode 100644 internal/handler/homebrew_bottles_test.go create mode 100644 internal/mirror/brew_test.go diff --git a/README.md b/README.md index 48a575a..32a817a 100644 --- a/README.md +++ b/README.md @@ -258,7 +258,7 @@ export HOMEBREW_ARTIFACT_DOMAIN=http://localhost:8080 The artifact domain proxies manifests and bottle blobs under `/v2/homebrew/core/`. GHCR routing is limited to that repository. Source archives, cask application downloads, custom tap artifacts, and legacy flat-file bottle mirrors use Homebrew's normal fallback URLs. Keep fallback enabled by leaving `HOMEBREW_ARTIFACT_DOMAIN_NO_FALLBACK` unset. -Enable `cache_metadata` or set `PROXY_CACHE_METADATA=true` to retain Homebrew JSON API responses for offline fallback. Bottle blobs and their OCI manifests are cached without this setting. +Enable `cache_metadata` or set `PROXY_CACHE_METADATA=true` to retain Homebrew JSON API responses for offline fallback. Bottle blobs and their OCI manifests are cached without this setting. `proxy mirror pkg:brew/` pre-fetches a formula's bottles (see [Homebrew bottles](docs/configuration.md#homebrew-bottles)). The upstreams default to `https://formulae.brew.sh/api` for the JSON API and `https://ghcr.io` for artifacts. To chain this proxy to another proxy, configure its Homebrew endpoints as the upstreams: diff --git a/cmd/proxy/main.go b/cmd/proxy/main.go index 525a4f6..21b6388 100644 --- a/cmd/proxy/main.go +++ b/cmd/proxy/main.go @@ -619,6 +619,7 @@ func newCommandMirror(cfg *config.Config, proxy *handler.Proxy, images *handler. db *database.DB, store storage.Storage, logger *slog.Logger, workers int) *mirror.Mirror { m := mirror.New(proxy, db, store, logger, workers) m.Images = images + m.Bottles = handler.NewHomebrewHandler(proxy, cfg.Upstream.HomebrewAPI) m.Charts = handler.NewHelmHandler(proxy, cfg.BaseURL, cfg.Upstream.Helm) server.UseMirrorUpstreams(m, cfg.Upstream, proxy.HTTPClient) if cfg.Scanning.Enabled { diff --git a/docs/configuration.md b/docs/configuration.md index 832860c..55e421f 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1665,6 +1665,22 @@ proxy mirror 'pkg:oci/debian@sha256:...?repository_url=docker.io/library/debian& - Non-distributable (foreign) layers are skipped. - While `scanning.enabled` is set, the command refuses images, because scanners fetch staged layers from the running server. Use the mirror API instead. +### Homebrew bottles + +`pkg:brew` PURLs mirror a homebrew/core formula's bottles, which Homebrew publishes as an image on `upstream.homebrew_artifact`. They are mirrored like that image, under the keys brew reads through `HOMEBREW_ARTIFACT_DOMAIN`, so `brew install` is served from cache, also when the upstream is unreachable: + +```bash +proxy mirror --platform linux/arm64 pkg:brew/hello@2.12.3 pkg:brew/openssl%403 +``` + +- A formula name with `@` is written with `%40`, since `@` starts the version: `pkg:brew/openssl%403` is `openssl@3`, and `pkg:brew/openssl@3` is version 3 of `openssl`. + +- The formula's JSON API on `upstream.homebrew_api` gives the image tag for the current version, which carries the formula's revision and the bottles' rebuild (`hello` 2.12.3 is tagged `2.12.3-1`). Only the current version is described there; another version is used as the tag as given, so give the full tag, such as `2.12.2` or `2.12.3-1`, for older bottles. +- A PURL without a version mirrors the current version. +- `--platform` selects bottles like image platforms: `linux/arm64`, `linux/amd64`, or `darwin/arm64` for every macOS release's arm64 bottle. Without it every bottle of the version is mirrored. +- Dependencies aren't mirrored with a formula; list them too. brew also needs the JSON API, which the proxy keeps with `cache_metadata`. +- Casks, source archives and taps other than homebrew/core aren't mirrored. + ### Scheduled refresh Tags such as `alpine:3.20` move. The server can re-mirror them on its own, so the cache follows the tag even when nobody pulls it or the proxy is offline later: diff --git a/internal/handler/homebrew_bottles.go b/internal/handler/homebrew_bottles.go new file mode 100644 index 0000000..37ab450 --- /dev/null +++ b/internal/handler/homebrew_bottles.go @@ -0,0 +1,96 @@ +package handler + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "regexp" + "strconv" + "strings" +) + +var ( + homebrewFormulaPattern = regexp.MustCompile(`^[a-z0-9][a-z0-9@+._-]*$`) + homebrewVersionPattern = regexp.MustCompile(`^[0-9A-Za-z][0-9A-Za-z._+-]*$`) +) + +// homebrewMaxFormulaJSON bounds one formula's JSON API document, a few KB. +const homebrewMaxFormulaJSON = 1 << 20 + +// homebrewFormula is the part of a formula's JSON API document that names +// its bottles' image tag. +type homebrewFormula struct { + Versions struct { + Stable string `json:"stable"` + } `json:"versions"` + Revision int `json:"revision"` + Bottle struct { + Stable struct { + Rebuild int `json:"rebuild"` + Files map[string]any `json:"files"` + } `json:"stable"` + } `json:"bottle"` +} + +// BottleImage returns the image Homebrew publishes the bottles of a +// homebrew/core formula version under, as brew pulls them through +// HOMEBREW_ARTIFACT_DOMAIN: openssl@3 is homebrew/core/openssl/3, and the tag +// carries the formula's revision and the bottles' rebuild, so hello 2.12.3 +// with rebuild 1 is tagged 2.12.3-1. Only the current version is described by +// the JSON API; any other version is used as the tag as given, which also +// takes a tag such as 2.12.3-1 directly. +func (h *HomebrewHandler) BottleImage(ctx context.Context, formula, version string) (ImageRef, error) { + if !homebrewFormulaPattern.MatchString(formula) { + return ImageRef{}, fmt.Errorf("invalid Homebrew formula name %q", formula) + } + if !homebrewVersionPattern.MatchString(version) { + return ImageRef{}, fmt.Errorf("invalid Homebrew version %q", version) + } + f, err := h.fetchFormula(ctx, formula) + if err != nil { + return ImageRef{}, err + } + tag := version + if version == f.Versions.Stable { + if len(f.Bottle.Stable.Files) == 0 { + return ImageRef{}, fmt.Errorf("homebrew formula %s %s has no bottles", formula, version) + } + if f.Revision > 0 { + tag += "_" + strconv.Itoa(f.Revision) + } + if f.Bottle.Stable.Rebuild > 0 { + tag += "-" + strconv.Itoa(f.Bottle.Stable.Rebuild) + } + } + name := strings.NewReplacer("@", "/", "+", "x").Replace(formula) + return ImageRef{Name: homebrewArtifactRepository + "/" + name, References: []string{tag}}, nil +} + +func (h *HomebrewHandler) fetchFormula(ctx context.Context, formula string) (*homebrewFormula, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, h.apiUpstream+"/formula/"+formula+".json", nil) + if err != nil { + return nil, err + } + resp, err := h.proxy.HTTPClient.Do(req) + if err != nil { + return nil, fmt.Errorf("reading the Homebrew formula: %w", err) + } + defer func() { _ = resp.Body.Close() }() + switch { + case resp.StatusCode == http.StatusNotFound: + return nil, fmt.Errorf("homebrew/core has no formula %q", formula) + case resp.StatusCode != http.StatusOK: + return nil, fmt.Errorf("reading the Homebrew formula: upstream answered %d", resp.StatusCode) + } + var f homebrewFormula + if err := json.NewDecoder(io.LimitReader(resp.Body, homebrewMaxFormulaJSON)).Decode(&f); err != nil { + return nil, fmt.Errorf("reading the Homebrew formula: %w", err) + } + if f.Versions.Stable == "" { + return nil, errors.New("reading the Homebrew formula: it has no stable version") + } + return &f, nil +} diff --git a/internal/handler/homebrew_bottles_test.go b/internal/handler/homebrew_bottles_test.go new file mode 100644 index 0000000..a2ddb64 --- /dev/null +++ b/internal/handler/homebrew_bottles_test.go @@ -0,0 +1,72 @@ +package handler + +import ( + "context" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" +) + +func TestHomebrewBottleImage(t *testing.T) { + formulas := map[string]string{ + "/formula/hello.json": `{"versions":{"stable":"2.12.3"},"revision":0, + "bottle":{"stable":{"rebuild":1,"files":{"arm64_linux":{}}}}}`, + "/formula/openssl@3.json": `{"versions":{"stable":"3.5.0"},"revision":2, + "bottle":{"stable":{"rebuild":0,"files":{"arm64_sequoia":{}}}}}`, + "/formula/libsigc++.json": `{"versions":{"stable":"3.6.0"},"revision":0, + "bottle":{"stable":{"rebuild":0,"files":{"x86_64_linux":{}}}}}`, + "/formula/nobottle.json": `{"versions":{"stable":"1.0"},"bottle":{}}`, + } + var requests atomic.Int32 + api := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests.Add(1) + body, ok := formulas[r.URL.Path] + if !ok { + http.NotFound(w, r) + return + } + _, _ = w.Write([]byte(body)) + })) + defer api.Close() + proxy, _, _, _ := setupTestProxy(t) + proxy.HTTPClient = api.Client() + h := NewHomebrewHandler(proxy, api.URL) + + tests := []struct { + formula, version string + image, tag, err string + }{ + {formula: "hello", version: "2.12.3", image: "homebrew/core/hello", tag: "2.12.3-1"}, + {formula: "openssl@3", version: "3.5.0", image: "homebrew/core/openssl/3", tag: "3.5.0_2"}, + {formula: "libsigc++", version: "3.6.0", image: "homebrew/core/libsigcxx", tag: "3.6.0"}, + // Only the current version is described; another is the tag as given. + {formula: "hello", version: "2.12.2", image: "homebrew/core/hello", tag: "2.12.2"}, + {formula: "hello", version: "2.12.3-1", image: "homebrew/core/hello", tag: "2.12.3-1"}, + {formula: "nobottle", version: "1.0", err: "has no bottles"}, + {formula: "missing", version: "1.0", err: `no formula "missing"`}, + } + for _, tt := range tests { + ref, err := h.BottleImage(context.Background(), tt.formula, tt.version) + if tt.err != "" { + if err == nil || !strings.Contains(err.Error(), tt.err) { + t.Errorf("%s@%s: err = %v, want %q", tt.formula, tt.version, err, tt.err) + } + continue + } + if err != nil || ref.Name != tt.image || len(ref.References) != 1 || ref.References[0] != tt.tag { + t.Errorf("%s@%s = %+v, %v; want %s:%s", tt.formula, tt.version, ref, err, tt.image, tt.tag) + } + } + + before := requests.Load() + for _, bad := range [][2]string{{"../hello", "1.0"}, {"Hello", "1.0"}, {"hello", "1.0/../x"}, {"hello", ""}} { + if _, err := h.BottleImage(context.Background(), bad[0], bad[1]); err == nil { + t.Errorf("%q@%q accepted", bad[0], bad[1]) + } + } + if requests.Load() != before { + t.Error("an invalid name or version reached the API") + } +} diff --git a/internal/mirror/brew_test.go b/internal/mirror/brew_test.go new file mode 100644 index 0000000..ea50e41 --- /dev/null +++ b/internal/mirror/brew_test.go @@ -0,0 +1,123 @@ +package mirror + +import ( + "context" + "errors" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + + "github.com/git-pkgs/proxy/internal/handler" + "github.com/git-pkgs/registries" +) + +// fakeBottles resolves every formula version to homebrew/core/ tagged +// -1, like a formula whose bottles were rebuilt once. +type fakeBottles struct { + err error +} + +func (f fakeBottles) BottleImage(_ context.Context, formula, version string) (handler.ImageRef, error) { + if f.err != nil { + return handler.ImageRef{}, f.err + } + return handler.ImageRef{Name: "homebrew/core/" + formula, References: []string{version + "-1"}}, nil +} + +// platformImages records the images mirrored and the platforms asked for. +type platformImages struct { + mu sync.Mutex + refs []handler.ImageRef + platforms [][]handler.Platform +} + +func (p *platformImages) MirrorImage(_ context.Context, ref handler.ImageRef, platforms []handler.Platform) (handler.ImageResult, error) { + p.mu.Lock() + defer p.mu.Unlock() + p.refs = append(p.refs, ref) + p.platforms = append(p.platforms, platforms) + return handler.ImageResult{Fetched: 2, Bytes: 10}, nil +} + +type stubVersions map[string][]string + +func (s stubVersions) ListVersions(_ context.Context, ecosystem, name string) ([]string, error) { + return s[ecosystem+"/"+name], nil +} + +func TestMirrorBrewMirrorsBottleImage(t *testing.T) { + m := setupTestMirror(t, 1) + images := &platformImages{} + m.Images, m.Bottles = images, fakeBottles{} + + source := &PURLSource{ + PURLs: []string{"pkg:brew/hello@2.12.3", "pkg:brew/jq"}, + Platforms: []string{"linux/arm64"}, + Versions: stubVersions{"brew/jq": {"1.8.1"}}, + } + progress, err := m.Run(context.Background(), source) + if err != nil { + t.Fatal(err) + } + if progress.Completed != 2 || progress.Failed != 0 { + t.Fatalf("progress = %+v, errors %v", progress, progress.Errors) + } + got := map[string]string{} + for i, ref := range images.refs { + if len(images.platforms[i]) != 1 || images.platforms[i][0].OS != "linux" || images.platforms[i][0].Architecture != "arm64" { + t.Errorf("%s: platforms = %+v, want linux/arm64", ref.Name, images.platforms[i]) + } + got[ref.Name] = strings.Join(ref.References, ",") + } + if got["homebrew/core/hello"] != "2.12.3-1" || got["homebrew/core/jq"] != "1.8.1-1" { + t.Errorf("images = %v", got) + } +} + +func TestMirrorBrewFailures(t *testing.T) { + m := setupTestMirror(t, 1) + progress, err := m.Run(context.Background(), &PURLSource{PURLs: []string{"pkg:brew/hello@2.12.3"}}) + if err != nil { + t.Fatal(err) + } + if progress.Failed != 1 || len(progress.Errors) != 1 || !strings.Contains(progress.Errors[0].Error, "isn't available") { + t.Errorf("without a bottle resolver: %+v", progress) + } + + images := &platformImages{} + m.Images, m.Bottles = images, fakeBottles{err: errors.New(`homebrew/core has no formula "nope"`)} + progress, err = m.Run(context.Background(), &PURLSource{PURLs: []string{"pkg:brew/nope@1.0"}}) + if err != nil { + t.Fatal(err) + } + if progress.Failed != 1 || len(images.refs) != 0 { + t.Errorf("unknown formula: %+v, mirrored %v", progress, images.refs) + } +} + +// The Homebrew client lists a formula's versioned siblings as versions; +// listing keeps only the formula's own version. +func TestListBrewVersionsSkipsSiblingFormulae(t *testing.T) { + api := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/api/formula/openssl@3.json" { + http.NotFound(w, r) + return + } + _, _ = w.Write([]byte(`{"name":"openssl@3","versions":{"stable":"3.5.9"}, + "versioned_formulae":["openssl@3.6","openssl@3.0"]}`)) + })) + defer api.Close() + lister := RegistryVersions{ + Client: registries.NewClient(registries.WithHTTPClient(api.Client())), + URL: func(string, string) (string, error) { return api.URL, nil }, + } + versions, err := lister.ListVersions(context.Background(), "brew", "openssl@3") + if err != nil { + t.Fatal(err) + } + if len(versions) != 1 || versions[0] != "3.5.9" { + t.Errorf("versions = %v, want [3.5.9]", versions) + } +} diff --git a/internal/mirror/image.go b/internal/mirror/image.go index 8b16893..5116f26 100644 --- a/internal/mirror/image.go +++ b/internal/mirror/image.go @@ -27,6 +27,18 @@ func isImageEcosystem(ecosystem string) bool { return ecosystem == "docker" || ecosystem == "oci" } +// BottleResolver resolves a Homebrew formula version to the image its +// bottles are published under. +type BottleResolver interface { + BottleImage(ctx context.Context, formula, version string) (handler.ImageRef, error) +} + +// takesPlatforms reports whether --platform applies to an ecosystem: images, +// and Homebrew bottles, which are published as images. +func takesPlatforms(ecosystem string) bool { + return isImageEcosystem(ecosystem) || ecosystem == "brew" +} + // ParsePlatforms parses os/architecture[/variant] values. func ParsePlatforms(values []string) ([]handler.Platform, error) { var platforms []handler.Platform @@ -98,6 +110,29 @@ func imageRef(pv PackageVersion) (handler.ImageRef, []handler.Platform, error) { } func (m *Mirror) mirrorImage(ctx context.Context, pv PackageVersion, tracker *progressTracker) { + ref, platforms, err := imageRef(pv) + m.mirrorImageRef(ctx, pv, ref, platforms, err, tracker) +} + +// mirrorBrew mirrors the bottles of a Homebrew formula version, which brew +// pulls as an image from homebrew/core on its artifact domain. +func (m *Mirror) mirrorBrew(ctx context.Context, pv PackageVersion, tracker *progressTracker) { + if m.Images == nil || m.Bottles == nil { + m.recordFailure(tracker, pv, errors.New("mirroring Homebrew bottles isn't available here")) + return + } + platforms, err := ParsePlatforms(pv.Platforms) + if err != nil { + m.recordFailure(tracker, pv, err) + return + } + ref, err := m.Bottles.BottleImage(ctx, pv.Name, pv.Version) + m.mirrorImageRef(ctx, pv, ref, platforms, err, tracker) +} + +// mirrorImageRef mirrors the image ref for pv, or records refErr, the error +// resolving it. +func (m *Mirror) mirrorImageRef(ctx context.Context, pv PackageVersion, ref handler.ImageRef, platforms []handler.Platform, refErr error, tracker *progressTracker) { fail := func(err error) { tracker.failed.Add(1) tracker.addError(pv.Ecosystem, pv.Name, pv.Version, err.Error()) @@ -107,9 +142,8 @@ func (m *Mirror) mirrorImage(ctx context.Context, pv PackageVersion, tracker *pr fail(fmt.Errorf("%w: %s", handler.ErrVersionDenied, pv)) return } - ref, platforms, err := imageRef(pv) - if err != nil { - fail(err) + if refErr != nil { + fail(refErr) return } result, err := m.Images.MirrorImage(ctx, ref, platforms) diff --git a/internal/mirror/mirror.go b/internal/mirror/mirror.go index 9b55a8c..482f46f 100644 --- a/internal/mirror/mirror.go +++ b/internal/mirror/mirror.go @@ -32,6 +32,11 @@ type Mirror struct { // Charts mirrors helm charts. Without it they fail. Charts ChartMirrorer + // Bottles resolves Homebrew formula versions to the images their + // bottles are published under, which Images mirrors. Without it brew + // PURLs fail. + Bottles BottleResolver + // Versions lists the versions of unversioned PURLs in mirror jobs and // schedules. Nil lists them from the public registries. Versions VersionLister @@ -228,6 +233,10 @@ func (m *Mirror) mirrorOne(ctx context.Context, pv PackageVersion, tracker *prog m.mirrorImage(ctx, pv, tracker) return } + if pv.Ecosystem == "brew" { + m.mirrorBrew(ctx, pv, tracker) + return + } if m.RefusePackages != "" { tracker.failed.Add(1) tracker.addError(pv.Ecosystem, pv.Name, pv.Version, m.RefusePackages) diff --git a/internal/mirror/source.go b/internal/mirror/source.go index b742e00..1dba9b6 100644 --- a/internal/mirror/source.go +++ b/internal/mirror/source.go @@ -86,6 +86,8 @@ func (s *PURLSource) Enumerate(ctx context.Context, fn func(PackageVersion) erro pv := PackageVersion{Ecosystem: ecosystem, Name: name, Version: p.Version, Qualifiers: qualifiers} if isImageEcosystem(ecosystem) { pv.Qualifiers = p.Qualifiers.Map() + } + if takesPlatforms(ecosystem) { pv.Platforms = s.Platforms } if err := fn(pv); err != nil { @@ -100,7 +102,11 @@ func (s *PURLSource) Enumerate(ctx context.Context, fn func(PackageVersion) erro return fmt.Errorf("fetching versions for %s/%s: %w", ecosystem, name, err) } for _, v := range listed { - if err := fn(PackageVersion{Ecosystem: ecosystem, Name: name, Version: v, Qualifiers: qualifiers}); err != nil { + pv := PackageVersion{Ecosystem: ecosystem, Name: name, Version: v, Qualifiers: qualifiers} + if takesPlatforms(ecosystem) { + pv.Platforms = s.Platforms + } + if err := fn(pv); err != nil { return err } } diff --git a/internal/mirror/versions.go b/internal/mirror/versions.go index 8e10eda..07b031f 100644 --- a/internal/mirror/versions.go +++ b/internal/mirror/versions.go @@ -42,9 +42,15 @@ func (r RegistryVersions) ListVersions(ctx context.Context, ecosystem, name stri if err != nil { return nil, err } - result := make([]string, len(versions)) - for i, v := range versions { - result[i] = v.Number + result := make([]string, 0, len(versions)) + for _, v := range versions { + // The Homebrew client also lists a formula's versioned siblings, + // openssl@3.6 for openssl@3, as versions marked with the sibling's + // name. They are other formulae, not versions of this one. + if _, sibling := v.Metadata["formula"]; sibling { + continue + } + result = append(result, v.Number) } return result, nil } diff --git a/internal/server/mirror_versions.go b/internal/server/mirror_versions.go index 59c6c3f..27b3d51 100644 --- a/internal/server/mirror_versions.go +++ b/internal/server/mirror_versions.go @@ -30,6 +30,7 @@ func NewMirrorVersions(upstream config.UpstreamConfig, client *http.Client) mirr func (s *Server) newMirror(proxy *handler.Proxy, workers int) *mirror.Mirror { m := mirror.New(proxy, s.db, s.storage, s.logger, workers) m.Images = s.newContainerHandler(proxy) + m.Bottles = handler.NewHomebrewHandler(proxy, s.cfg.Upstream.HomebrewAPI) m.Charts = handler.NewHelmHandler(proxy, s.cfg.BaseURL, s.cfg.Upstream.Helm) UseMirrorUpstreams(m, s.cfg.Upstream, proxy.HTTPClient) return m From db8a60e17513ff4e44e4f6ef05c715300db848d3 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Mon, 26 Oct 2026 16:57:00 +0100 Subject: [PATCH 482/633] Drop the Homebrew sibling formulae filter git-pkgs/registries#120 stops listing versioned formulae as versions. --- internal/mirror/brew_test.go | 28 ---------------------------- internal/mirror/versions.go | 12 +++--------- 2 files changed, 3 insertions(+), 37 deletions(-) diff --git a/internal/mirror/brew_test.go b/internal/mirror/brew_test.go index ea50e41..80e5a26 100644 --- a/internal/mirror/brew_test.go +++ b/internal/mirror/brew_test.go @@ -3,14 +3,11 @@ package mirror import ( "context" "errors" - "net/http" - "net/http/httptest" "strings" "sync" "testing" "github.com/git-pkgs/proxy/internal/handler" - "github.com/git-pkgs/registries" ) // fakeBottles resolves every formula version to homebrew/core/ tagged @@ -96,28 +93,3 @@ func TestMirrorBrewFailures(t *testing.T) { t.Errorf("unknown formula: %+v, mirrored %v", progress, images.refs) } } - -// The Homebrew client lists a formula's versioned siblings as versions; -// listing keeps only the formula's own version. -func TestListBrewVersionsSkipsSiblingFormulae(t *testing.T) { - api := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.URL.Path != "/api/formula/openssl@3.json" { - http.NotFound(w, r) - return - } - _, _ = w.Write([]byte(`{"name":"openssl@3","versions":{"stable":"3.5.9"}, - "versioned_formulae":["openssl@3.6","openssl@3.0"]}`)) - })) - defer api.Close() - lister := RegistryVersions{ - Client: registries.NewClient(registries.WithHTTPClient(api.Client())), - URL: func(string, string) (string, error) { return api.URL, nil }, - } - versions, err := lister.ListVersions(context.Background(), "brew", "openssl@3") - if err != nil { - t.Fatal(err) - } - if len(versions) != 1 || versions[0] != "3.5.9" { - t.Errorf("versions = %v, want [3.5.9]", versions) - } -} diff --git a/internal/mirror/versions.go b/internal/mirror/versions.go index 07b031f..8e10eda 100644 --- a/internal/mirror/versions.go +++ b/internal/mirror/versions.go @@ -42,15 +42,9 @@ func (r RegistryVersions) ListVersions(ctx context.Context, ecosystem, name stri if err != nil { return nil, err } - result := make([]string, 0, len(versions)) - for _, v := range versions { - // The Homebrew client also lists a formula's versioned siblings, - // openssl@3.6 for openssl@3, as versions marked with the sibling's - // name. They are other formulae, not versions of this one. - if _, sibling := v.Metadata["formula"]; sibling { - continue - } - result = append(result, v.Number) + result := make([]string, len(versions)) + for i, v := range versions { + result[i] = v.Number } return result, nil } From f4423ab9564128761abfd255face08b9e877fbc1 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Tue, 27 Oct 2026 11:07:45 +0100 Subject: [PATCH 483/633] Move the PURL's mirror name and qualifiers out of Enumerate Enumerate went over the cognitive complexity limit after main added the Julia uuid qualifier. --- internal/mirror/source.go | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/internal/mirror/source.go b/internal/mirror/source.go index 1dba9b6..1c8fa7b 100644 --- a/internal/mirror/source.go +++ b/internal/mirror/source.go @@ -68,17 +68,7 @@ func (s *PURLSource) Enumerate(ctx context.Context, fn func(PackageVersion) erro return fmt.Errorf("parsing PURL %q: %w", purlStr, err) } - ecosystem := handlerEcosystem(p.Type) - name := p.Name - if ecosystem == "conda" { - name = condaName(p.Namespace, p.Name, p.Qualifier("channel")) - } else if p.Namespace != "" { - name = p.Namespace + "/" + p.Name - } - qualifiers := fileQualifiers(ecosystem, p.Qualifiers.Map()) - if ecosystem == "julia" { - qualifiers = map[string]string{"uuid": p.Qualifier("uuid")} - } + ecosystem, name, qualifiers := mirrorTarget(p) // Images are never expanded to all tags, and conda packages without // a channel aren't listed; both fail when they are mirrored. @@ -114,6 +104,23 @@ func (s *PURLSource) Enumerate(ctx context.Context, fn func(PackageVersion) erro return nil } +// mirrorTarget returns the ecosystem, name and file qualifiers a PURL is +// mirrored under. +func mirrorTarget(p *purl.PURL) (ecosystem, name string, qualifiers map[string]string) { + ecosystem = handlerEcosystem(p.Type) + name = p.Name + if ecosystem == "conda" { + name = condaName(p.Namespace, p.Name, p.Qualifier("channel")) + } else if p.Namespace != "" { + name = p.Namespace + "/" + p.Name + } + qualifiers = fileQualifiers(ecosystem, p.Qualifiers.Map()) + if ecosystem == "julia" { + qualifiers = map[string]string{"uuid": p.Qualifier("uuid")} + } + return ecosystem, name, qualifiers +} + // SBOMSource extracts package versions from CycloneDX or SPDX SBOM data. type SBOMSource struct { Data []byte From 2ea13b00d61fbe4404175d9ae920f484ff064e4d Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Wed, 28 Oct 2026 16:25:08 +0100 Subject: [PATCH 484/633] Show cooldown only for ecosystems that apply it The package page showed the default cooldown for every ecosystem, also for Debian, Alpine, OCI and generic files, which have no publish time to check. It now shows none there, and the dashboard marks an ecosystem override for one as not supported. The docs list Helm, which applies cooldown from the index's created times. --- docs/architecture.md | 2 +- docs/configuration.md | 6 ++- internal/handler/cooldown_supported_test.go | 46 +++++++++++++++++++ internal/handler/handler.go | 14 ++++++ internal/server/cooldown_view.go | 20 +++++--- internal/server/cooldown_view_test.go | 20 ++++++++ .../server/templates/pages/dashboard.html | 2 +- 7 files changed, 100 insertions(+), 10 deletions(-) create mode 100644 internal/handler/cooldown_supported_test.go diff --git a/docs/architecture.md b/docs/architecture.md index b6c654f..24af2fb 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -355,7 +355,7 @@ Prometheus metrics for cache performance, upstream latency, storage operations, ### Cooldown -Version age filtering for supply chain attack mitigation, provided by [github.com/git-pkgs/cooldown](https://github.com/git-pkgs/cooldown). Configurable at global, ecosystem, and per-package levels. Supported by npm, PyPI, pub.dev, and Composer handlers. +Version age filtering for supply chain attack mitigation, provided by [github.com/git-pkgs/cooldown](https://github.com/git-pkgs/cooldown). Configurable at global, ecosystem, and per-package levels. Supported by the npm, PyPI, pub.dev, Composer, Cargo, NuGet, Conda, RubyGems, Hex, Go, Maven and Helm handlers. ### `internal/enrichment` diff --git a/docs/configuration.md b/docs/configuration.md index 832860c..2f8d3c1 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1124,9 +1124,9 @@ Exact `packages` entries take precedence over patterns. When several patterns ma Resolution order: exact package override, then package pattern, then ecosystem override, then global default. This lets you set a conservative default while exempting trusted package families. -The web UI's dashboard shows the default and the ecosystem overrides, and how many package overrides and patterns there are; their names aren't shown, since the UI has no authentication. A package's page shows the cooldown that applies to it and which kind of rule it comes from. +The web UI's dashboard shows the default and the ecosystem overrides, and how many package overrides and patterns there are; their names aren't shown, since the UI has no authentication. A package's page shows the cooldown that applies to it and which kind of rule it comes from; it shows none for ecosystems without cooldown support, and the dashboard marks an ecosystem override for one as not supported. -Currently supported for npm, PyPI, pub.dev, Composer, Cargo, NuGet, Conda, RubyGems, Hex, Go and Maven. Most of these ecosystems include publish timestamps in their metadata; Go and Maven are described below. +Currently supported for npm, PyPI, pub.dev, Composer, Cargo, NuGet, Conda, RubyGems, Hex, Go, Maven and Helm. Most of these ecosystems include publish timestamps in their metadata; Go and Maven are described below. Go has no publish time. The module proxy's `.info` gives the commit time of the revision a version points at, so a version tagged on an old commit counts as old. The proxy reads each listed version's `.info` once, stores the time, and leaves versions in cooldown out of `@v/list` and `@latest`. When upstream's `@latest` is in cooldown, it serves the newest release out of cooldown instead. Requests for a version in cooldown, including its `.info`, `.mod` and `.zip`, and a branch or commit query that resolves to one, get 403 rather than 404, because the go command moves on to the next `GOPROXY` entry, such as `direct`, after a 404. If a version's `.info` can't be fetched, the list or download fails with 502 instead of letting the version through. @@ -1139,6 +1139,8 @@ Maven caveats: - Maven Central lists versions in version order, not in the order they were published. A patch to an older release line published after a newer release that is out of cooldown stays listed while it is in cooldown, but its files are still refused. - SNAPSHOT versions are not held back. +Helm uses each release's `created` time from the repository's `index.yaml`. Releases in cooldown are left out of the index the proxy serves, and their archives aren't served; a release without `created` is let through. Charts pulled over OCI have no publish time and aren't held back. + RubyGems, Hex and PyPI keep their publish times outside the metadata clients read: the RubyGems versions API, the Hex HTTP API (`upstream.hex_api`) and the PyPI JSON API. The proxy fetches them alongside each compact index, package or simple page it filters, and for PyPI downloads. Cooldown fails closed: when it applies to a package and those times can't be fetched or parsed, or the metadata can't be rewritten, the request fails with 502, or with 503 and `Retry-After` while the upstream is rate limiting the proxy, instead of serving the metadata unfiltered. The same goes for npm, Composer and pub.dev metadata that can't be rewritten, and for npm tarball downloads whose publish time can't be read. A PyPI file or npm tarball that is already cached is still served when its publish time can't be read; only a download that would have to be fetched fails. A PyPI release's time is the upload time of its newest file. The proxy stores it when a download looks it up, and serves files of that release that are already cached against the stored time without reading the JSON API again. A file that isn't cached yet is always looked up, since it may have been added to the release since, and the newer time is stored. Packages exempted with a duration of `0` are served as before and don't need the times. A PyPI package the JSON API returns 404 for has no times to go by and is served unfiltered. For RubyGems and Hex, both documents go through the metadata cache. With [`cache_metadata`](#metadata-caching) they are served from it within `metadata_ttl` without asking the upstream, revalidated with their ETags after it, and filtered from the last cached copies, marked stale, while the upstream is down; without it they are still fetched once for concurrent requests. Each document has its own TTL, so the compact index or Hex package can be newer than the cached times and list a version they don't have. The times are then fetched again before filtering, and if that fails, such versions are left out until the times can be read. The filtered index or package carries an ETag of its own, so Bundler and Mix revalidate it with `304 Not Modified`. Publish times are compared with the current time on each request, but the filtered result is kept for up to a minute (see [rewritten metadata cache](#rewritten-metadata-cache)), so a version leaving cooldown can show up a minute late; a newly published version shows up within `metadata_ttl`, as with other cached metadata. Upstream errors are answered as for other cached metadata (404, or 502) rather than relayed. diff --git a/internal/handler/cooldown_supported_test.go b/internal/handler/cooldown_supported_test.go new file mode 100644 index 0000000..ac73a3b --- /dev/null +++ b/internal/handler/cooldown_supported_test.go @@ -0,0 +1,46 @@ +package handler + +import ( + "os" + "path/filepath" + "regexp" + "strings" + "testing" +) + +// TestCooldownSupportedCoversHandlers checks that every ecosystem a handler +// checks cooldown for is listed in cooldownEcosystems, so the UI shows its +// cooldown. +func TestCooldownSupportedCoversHandlers(t *testing.T) { + calls := regexp.MustCompile(`(?:IsAllowed|Evaluate)\("([a-z]+)"|withheld\(h\.proxy, "([a-z]+)"`) + files, err := filepath.Glob("*.go") + if err != nil { + t.Fatal(err) + } + found := map[string]bool{helmMetadataEcosystem: true} + for _, f := range files { + if strings.HasSuffix(f, "_test.go") { + continue + } + src, err := os.ReadFile(f) + if err != nil { + t.Fatal(err) + } + for _, m := range calls.FindAllStringSubmatch(string(src), -1) { + found[m[1]+m[2]] = true + } + } + if len(found) < 10 { + t.Fatalf("found only %d ecosystems checking cooldown, the pattern no longer matches the handlers", len(found)) + } + for eco := range found { + if !CooldownSupported(eco) { + t.Errorf("%s checks cooldown but is missing from cooldownEcosystems", eco) + } + } + for eco := range cooldownEcosystems { + if !found[eco] { + t.Errorf("%s is in cooldownEcosystems but no handler checks cooldown for it", eco) + } + } +} diff --git a/internal/handler/handler.go b/internal/handler/handler.go index b3ca1c4..571a466 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -283,6 +283,20 @@ type Proxy struct { started time.Time } +// cooldownEcosystems are the ecosystems whose handlers apply cooldown. Others +// have no publish time to check, so a configured cooldown has no effect there. +var cooldownEcosystems = map[string]bool{ + "cargo": true, "composer": true, "conda": true, "gem": true, + "golang": true, "helm": true, "hex": true, "maven": true, + "npm": true, "nuget": true, "pub": true, "pypi": true, +} + +// CooldownSupported reports whether the proxy applies cooldown to an +// ecosystem. +func CooldownSupported(ecosystem string) bool { + return cooldownEcosystems[ecosystem] +} + // CooldownPolicy supplies version-age filtering and package-specific durations. type CooldownPolicy interface { IsAllowed(ecosystem, packagePURL string, publishedAt time.Time) bool diff --git a/internal/server/cooldown_view.go b/internal/server/cooldown_view.go index 57a64ec..f244c85 100644 --- a/internal/server/cooldown_view.go +++ b/internal/server/cooldown_view.go @@ -7,6 +7,7 @@ import ( "github.com/git-pkgs/cooldown" "github.com/git-pkgs/proxy/internal/config" + "github.com/git-pkgs/proxy/internal/handler" "github.com/git-pkgs/proxy/internal/packageurl" ) @@ -21,10 +22,12 @@ type CooldownSummary struct { PackagePatterns int } -// CooldownOverride is one ecosystem's cooldown. +// CooldownOverride is one ecosystem's cooldown. Unsupported marks an +// ecosystem the proxy applies no cooldown to, so the setting has no effect. type CooldownOverride struct { - Ecosystem string - Duration string + Ecosystem string + Duration string + Unsupported bool } // PackageCooldown is the cooldown that applies to one package, and the kind @@ -73,15 +76,20 @@ func (v *cooldownView) summary() CooldownSummary { PackagePatterns: len(v.cfg.PackagePatterns), } for eco, d := range v.cfg.Ecosystems { - s.Ecosystems = append(s.Ecosystems, CooldownOverride{Ecosystem: eco, Duration: formatConfigDuration(d)}) + s.Ecosystems = append(s.Ecosystems, CooldownOverride{ + Ecosystem: eco, + Duration: formatConfigDuration(d), + Unsupported: !handler.CooldownSupported(eco), + }) } sort.Slice(s.Ecosystems, func(i, j int) bool { return s.Ecosystems[i].Ecosystem < s.Ecosystems[j].Ecosystem }) return s } -// forPackage returns the cooldown for a package, or nil with cooldown off. +// forPackage returns the cooldown for a package, or nil with cooldown off or +// for an ecosystem the proxy applies no cooldown to. func (v *cooldownView) forPackage(ecosystem, name string) *PackageCooldown { - if !v.enabled() { + if !v.enabled() || !handler.CooldownSupported(ecosystem) { return nil } purl := packageurl.MakeString(ecosystem, name, "") diff --git a/internal/server/cooldown_view_test.go b/internal/server/cooldown_view_test.go index 41ff789..2adbdae 100644 --- a/internal/server/cooldown_view_test.go +++ b/internal/server/cooldown_view_test.go @@ -3,6 +3,7 @@ package server import ( "net/http" "net/http/httptest" + "slices" "strings" "testing" "time" @@ -78,6 +79,25 @@ func TestCooldownViewOnlyPattern(t *testing.T) { } } +// TestCooldownViewUnsupportedEcosystem checks that ecosystems the proxy +// applies no cooldown to get no cooldown on their package pages, and that +// an override for one is marked on the dashboard. +func TestCooldownViewUnsupportedEcosystem(t *testing.T) { + v := testCooldownView(t, config.CooldownConfig{ + Default: "3d", + Ecosystems: map[string]string{"deb": "0", "npm": "7d"}, + }) + for _, eco := range []string{"deb", "alpine", "oci", "generic"} { + if got := v.forPackage(eco, "pkg"); got != nil { + t.Errorf("forPackage(%s) = %+v, want nil", eco, got) + } + } + want := []CooldownOverride{{"deb", "off", true}, {"npm", "7d", false}} + if got := v.summary().Ecosystems; !slices.Equal(got, want) { + t.Errorf("summary ecosystems = %+v, want %+v", got, want) + } +} + func TestFormatCooldown(t *testing.T) { for d, want := range map[time.Duration]string{ 0: "off", diff --git a/internal/server/templates/pages/dashboard.html b/internal/server/templates/pages/dashboard.html index eb4c786..0a90aed 100644 --- a/internal/server/templates/pages/dashboard.html +++ b/internal/server/templates/pages/dashboard.html @@ -32,7 +32,7 @@
Default:
{{.Default}}
{{range .Ecosystems}} -
{{.Ecosystem}}:
{{.Duration}}
+
{{.Ecosystem}}:
{{.Duration}}
{{if .Unsupported}} (not supported){{end}}
{{end}} {{if .PackageOverrides}}
Package overrides:
{{.PackageOverrides}}
{{end}} {{if .PackagePatterns}}
Package patterns:
{{.PackagePatterns}}
{{end}} From 28ce5f49fc08a110dd23d7a52b2eb70847489956 Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Wed, 28 Oct 2026 15:57:06 +0000 Subject: [PATCH 485/633] Make the slow-client storage failure test deterministic --- internal/handler/stream_miss_test.go | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/internal/handler/stream_miss_test.go b/internal/handler/stream_miss_test.go index 172be1b..2746588 100644 --- a/internal/handler/stream_miss_test.go +++ b/internal/handler/stream_miss_test.go @@ -536,10 +536,26 @@ func TestCacheMissSlowClientFailsWithoutStoredCopy(t *testing.T) { if err != nil { t.Fatal(err) } + defer func() { _ = res.Reader.Close() }() + + // Leave the stream unread until the fetch ends so the feed must overflow. + proxy.fetchMu.Lock() + var done <-chan struct{} + for _, f := range proxy.inFlight { + done = f.done + } + proxy.fetchMu.Unlock() + if done != nil { + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatal("fetch did not finish while the client was idle") + } + } + body, err := io.ReadAll(res.Reader) - _ = res.Reader.Close() - if err == nil { - t.Fatalf("read %d bytes without error; want the response to fail", len(body)) + if !errors.Is(err, store.storeErr) { + t.Fatalf("read %d bytes with error %v; want %v", len(body), err, store.storeErr) } } From 10bc2f330907ea4391504d7a5bf4b42ed9725adc Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Thu, 29 Oct 2026 10:52:39 +0100 Subject: [PATCH 486/633] Keep rewritten metadata gzip-only and longer with cooldown pnpm 12 checks every lockfile entry against the registry on each install unless trustLockfile is set, so each install reads the packument of every package in the lockfile. The rewrite cache kept each document twice, plain and gzipped, and with cooldown on for a minute only, so these packuments were rewritten and compressed again on every install. Keep only the gzip copy of a cached rewrite, about an eighth of the document, and decompress it for clients and handlers that need the plain bytes. Keep rewrites made with cooldown on for an hour. Answer HEAD for packuments, which pnpm 12 sends before each GET. --- docs/configuration.md | 6 +- internal/handler/gem.go | 2 +- internal/handler/hex.go | 2 +- internal/handler/maven_cooldown.go | 4 +- internal/handler/metadata_gzip_test.go | 21 ++++-- internal/handler/metadata_validators_test.go | 4 +- internal/handler/npm.go | 15 +++- internal/handler/npm_dist_tags.go | 2 +- internal/handler/npm_rewrite_test.go | 4 +- internal/handler/npm_test.go | 28 ++++++++ internal/handler/rewrite_cache.go | 72 ++++++++++++++++---- internal/handler/rewrite_cache_test.go | 14 ++-- 12 files changed, 135 insertions(+), 39 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 832860c..cf21075 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1420,9 +1420,11 @@ Or via environment variable: `PROXY_ARTIFACT_MAX_SIZE=2GB`. ### Rewritten metadata cache -The npm, Composer, PyPI and Helm handlers rewrite every metadata document they serve, so that download URLs point at the proxy. That means decoding the whole document and encoding it again, and Composer's minified documents are also expanded, which together cost milliseconds per typical package and far more for very large ones; a large Helm `index.yaml` takes over a hundred milliseconds. The proxy keeps rewritten documents in memory, so each distinct upstream document is rewritten once and requests arriving while it is being rewritten wait for that rewrite. New bytes from upstream are rewritten again. A PyPI simple page is cached per representation (HTML or JSON) and per set of versions filtered out of it. With version cooldown enabled, a cached rewrite is kept for one minute only, because cooldown filtering depends on the current time: a version leaving cooldown shows up at most a minute late. Cached documents carry an `ETag` of their own, and are dated like any other, so clients that revalidate with `If-None-Match` or `If-Modified-Since`, such as Composer, get `304 Not Modified`. The RubyGems compact index and Hex packages filtered by cooldown are kept the same way; they are never served gzip-compressed. +The npm, Composer, PyPI and Helm handlers rewrite every metadata document they serve, so that download URLs point at the proxy. That means decoding the whole document and encoding it again, and Composer's minified documents are also expanded, which together cost milliseconds per typical package and far more for very large ones; a large Helm `index.yaml` takes over a hundred milliseconds. The proxy keeps rewritten documents in memory, so each distinct upstream document is rewritten once and requests arriving while it is being rewritten wait for that rewrite. New bytes from upstream are rewritten again. A PyPI simple page is cached per representation (HTML or JSON) and per set of versions filtered out of it. With version cooldown enabled, a cached rewrite is kept for an hour only, because cooldown filtering depends on the current time: a version leaving cooldown shows up at most an hour late. Cached documents carry an `ETag` of their own, and are dated like any other, so clients that revalidate with `If-None-Match` or `If-Modified-Since`, such as Composer, get `304 Not Modified`. The RubyGems compact index and Hex packages filtered by cooldown are kept the same way; they are never served gzip-compressed. -Each cached document is also kept gzip-compressed, when that makes it smaller, and served with `Content-Encoding: gzip` to clients that accept it. A large packument is compressed once rather than on every request, by the proxy or by a reverse proxy in front of it. The compressed copy counts against the cache size and typically adds about an eighth to each entry. Documents rewritten outside the cache are served uncompressed. +Each cached document is kept gzip-compressed, when that makes it smaller, and served with `Content-Encoding: gzip` to clients that accept it, as npm, pnpm, Yarn and Composer do. A large packument is compressed once rather than on every request, by the proxy or by a reverse proxy in front of it. Only the compressed copy is kept, typically an eighth of the document, and it is decompressed for the clients that don't accept gzip. Documents rewritten outside the cache are served uncompressed. + +Size the cache for the metadata of the packages one install asks for. pnpm 12 checks every lockfile entry against the registry on each install unless `trustLockfile` is set, so each install reads the packument of every package in the lockfile; when they don't all fit, each install evicts what the next one needs and rewrites them all again. A lockfile of about 550 npm packages takes about 50 MB. ```yaml metadata_rewrite_cache_size: "256MB" # default; "0" rewrites on every request diff --git a/internal/handler/gem.go b/internal/handler/gem.go index edae77d..6e57559 100644 --- a/internal/handler/gem.go +++ b/internal/handler/gem.go @@ -195,7 +195,7 @@ func (h *GemHandler) handleCompactIndex(w http.ResponseWriter, r *http.Request) // Served as identity, as upstream's index was: Bundler checks digest // headers against the bytes it receives, and asks for ranges of it. - m.gzipped = nil + m = m.identity() contentType := docs.doc.contentType if contentType == "" || contentType == contentTypeJSON { contentType = "text/plain; charset=utf-8" diff --git a/internal/handler/hex.go b/internal/handler/hex.go index a46a7e4..f2a93ad 100644 --- a/internal/handler/hex.go +++ b/internal/handler/hex.go @@ -183,7 +183,7 @@ func serveHexPackage(w http.ResponseWriter, r *http.Request, m rewrittenMetadata w.Header().Set(headerContentType, "application/octet-stream") w.Header().Set(headerContentEncoding, "gzip") w.WriteHeader(http.StatusOK) - _, _ = w.Write(m.body) + _, _ = w.Write(m.content()) } // timedPackage describes a package's registry file and the Hex API entry diff --git a/internal/handler/maven_cooldown.go b/internal/handler/maven_cooldown.go index d67d9c9..4705f70 100644 --- a/internal/handler/maven_cooldown.go +++ b/internal/handler/maven_cooldown.go @@ -277,10 +277,10 @@ func (h *MavenHandler) serveCooldownMetadata(w http.ResponseWriter, r *http.Requ serveRewrittenMetadataAs(w, r, m, contentType) case mavenChecksums[suffix] != nil: sum := mavenChecksums[suffix]() - _, _ = sum.Write(m.body) + _, _ = sum.Write(m.content()) w.Header().Set(headerContentType, "text/plain") _, _ = io.WriteString(w, hex.EncodeToString(sum.Sum(nil))) - case bytes.Equal(m.body, body): + case bytes.Equal(m.content(), body): return false default: // A signature of the upstream metadata doesn't match what is served. diff --git a/internal/handler/metadata_gzip_test.go b/internal/handler/metadata_gzip_test.go index 98a3a4c..b5939c7 100644 --- a/internal/handler/metadata_gzip_test.go +++ b/internal/handler/metadata_gzip_test.go @@ -173,9 +173,10 @@ func TestRewrittenMetadataGzipOnlyFromRewriteCache(t *testing.T) { } } -// TestRewriteCache_KeepsGzipCopyOnlyWhenSmaller checks that the cache keeps a -// compressed copy of a rewrite, counted against its size, unless compressing -// does not make the rewrite smaller. +// TestRewriteCache_KeepsGzipCopyOnlyWhenSmaller checks that the cache keeps +// only a compressed copy of a rewrite, counted against its size, unless +// compressing does not make the rewrite smaller, and that the plain document +// is still there for callers that need it. func TestRewriteCache_KeepsGzipCopyOnlyWhenSmaller(t *testing.T) { c := newRewriteCache(1 << 20) identity := func(b []byte) ([]byte, error) { return b, nil } @@ -192,11 +193,17 @@ func TestRewriteCache_KeepsGzipCopyOnlyWhenSmaller(t *testing.T) { if !ok || m.gzipped == nil { t.Fatalf("large rewrite: cached %v, gzip copy %d bytes; want a gzip copy", ok, len(m.gzipped)) } - if c.size != int64(len(large)+len(m.gzipped)+3) { - t.Errorf("cache size = %d, want both rewrites and the gzip copy (%d)", c.size, len(large)+len(m.gzipped)+3) + if m.plain != nil { + t.Errorf("large rewrite: cached %d plain bytes besides the gzip copy, want none", len(m.plain)) } - if small, ok := c.get(rewriteCacheKey("npm", "http://proxy", "abc", []byte("abc"))); !ok || small.gzipped != nil { - t.Errorf("small rewrite: cached %v, gzip copy %d bytes; want none", ok, len(small.gzipped)) + if c.size != int64(len(m.gzipped)+3) { + t.Errorf("cache size = %d, want the gzip copy and the small rewrite (%d)", c.size, len(m.gzipped)+3) + } + if !bytes.Equal(m.content(), large) || m.size != len(large) { + t.Errorf("large rewrite content = %d bytes, size %d; want the %d bytes rewritten", len(m.content()), m.size, len(large)) + } + if small, ok := c.get(rewriteCacheKey("npm", "http://proxy", "abc", []byte("abc"))); !ok || small.gzipped != nil || string(small.content()) != "abc" { + t.Errorf("small rewrite: cached %v, gzip copy %d bytes, content %q; want abc uncompressed", ok, len(small.gzipped), small.content()) } } diff --git a/internal/handler/metadata_validators_test.go b/internal/handler/metadata_validators_test.go index f778c5f..a6e1bae 100644 --- a/internal/handler/metadata_validators_test.go +++ b/internal/handler/metadata_validators_test.go @@ -501,8 +501,8 @@ func TestRewriteCacheETag(t *testing.T) { key := rewriteCacheKey("npm", "http://proxy", "demo", []byte("doc")) m, err := c.rewrite(t.Context(), key, []byte("doc"), 0, exclaim) - if err != nil || string(m.body) != "doc!" { - t.Fatalf("rewrite = %q, %v", m.body, err) + if err != nil || string(m.content()) != "doc!" { + t.Fatalf("rewrite = %q, %v", m.content(), err) } if want := metadataETag([]byte("doc!")); m.etag != want { t.Errorf("rewrite ETag = %q, want %q", m.etag, want) diff --git a/internal/handler/npm.go b/internal/handler/npm.go index 59ea27e..537c4b3 100644 --- a/internal/handler/npm.go +++ b/internal/handler/npm.go @@ -85,6 +85,13 @@ func (h *NPMHandler) Routes() http.Handler { return } + // pnpm 12 checks packuments with HEAD before fetching them, so + // packuments also answer HEAD; net/http leaves out the body. + if r.Method == http.MethodHead && isPackumentPath(r.URL.Path) { + h.handlePackageMetadata(w, r) + return + } + if r.Method != http.MethodGet { http.Error(w, "method not allowed", http.StatusMethodNotAllowed) return @@ -125,6 +132,12 @@ func (h *NPMHandler) Routes() http.Handler { }) } +// isPackumentPath reports whether a request path names package metadata +// rather than a tarball or a registry endpoint under /-/. +func isPackumentPath(urlPath string) bool { + return !strings.HasPrefix(urlPath, "/-/") && !strings.Contains(strings.TrimPrefix(urlPath, "/"), "/-/") +} + // handleSecurity relays the npm audit endpoints to upstream. The request body // is the dependency tree being audited, so no two requests share a cache key, // and the response carries no tarball URLs to rewrite. @@ -391,7 +404,7 @@ func (h *NPMHandler) handleFilteredNPMVersionMetadata(w http.ResponseWriter, r * w.Header().Set(headerContentType, contentTypeJSON) w.WriteHeader(http.StatusOK) - _, _ = w.Write(rewritten.body) + _, _ = w.Write(rewritten.content()) } // npmMetadataCacheKey qualifies routed-package metadata cache entries with the diff --git a/internal/handler/npm_dist_tags.go b/internal/handler/npm_dist_tags.go index eb3741c..210b45f 100644 --- a/internal/handler/npm_dist_tags.go +++ b/internal/handler/npm_dist_tags.go @@ -98,7 +98,7 @@ func (h *NPMHandler) servedPackument(ctx context.Context, packageName string) (b // fails and nothing has to be filtered out. return upstream.body, upstream.stale, nil } - return rewritten.body, upstream.stale, nil + return rewritten.content(), upstream.stale, nil } // validNPMPackageName reports whether name is a plain or scoped package diff --git a/internal/handler/npm_rewrite_test.go b/internal/handler/npm_rewrite_test.go index 1386503..b36b229 100644 --- a/internal/handler/npm_rewrite_test.go +++ b/internal/handler/npm_rewrite_test.go @@ -153,7 +153,7 @@ func TestNPMStoredRewriteRecordsMetadataHit(t *testing.T) { if diff := artifactHits() - artifactBefore; diff != 0 { t.Errorf("artifact hits delta = %.0f, want 0", diff) } - if diff := served() - servedBefore; diff != float64(len(m.body)) { - t.Errorf("metadata served bytes delta = %.0f, want %d", diff, len(m.body)) + if diff := served() - servedBefore; diff != float64(len(m.content())) { + t.Errorf("metadata served bytes delta = %.0f, want %d", diff, len(m.content())) } } diff --git a/internal/handler/npm_test.go b/internal/handler/npm_test.go index 0e083a3..20fda4e 100644 --- a/internal/handler/npm_test.go +++ b/internal/handler/npm_test.go @@ -1401,3 +1401,31 @@ func TestNPMPingRelaysUpstream(t *testing.T) { t.Errorf("upstream request = %q?%q, want /-/ping?write=true", gotPath, gotQuery) } } + +// TestNPMHeadPackument checks that packuments answer HEAD, which pnpm 12 +// sends before fetching each one, while tarballs and registry endpoints +// still only answer GET. +func TestNPMHeadPackument(t *testing.T) { + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/testpkg" { + w.WriteHeader(http.StatusNotFound) + return + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"name":"testpkg","versions":{"1.0.0":{"name":"testpkg","version":"1.0.0","dist":{"tarball":"https://registry.npmjs.org/testpkg/-/testpkg-1.0.0.tgz"}}}}`)) + })) + defer upstream.Close() + h := &NPMHandler{proxy: testProxy(), upstreamURL: upstream.URL, proxyURL: "http://proxy.local"} + + get := serveNPM(t, h, http.MethodGet, "/testpkg", nil, nil) + head := serveNPM(t, h, http.MethodHead, "/testpkg", nil, nil) + if head.Code != http.StatusOK || head.Header().Get("ETag") == "" || head.Header().Get("ETag") != get.Header().Get("ETag") { + t.Errorf("HEAD /testpkg = %d, ETag %q; want 200 with GET's ETag %q", head.Code, head.Header().Get("ETag"), get.Header().Get("ETag")) + } + + for _, path := range []string{"/testpkg/-/testpkg-1.0.0.tgz", "/-/ping", "/-/v1/search", "/-/package/testpkg/dist-tags"} { + if w := serveNPM(t, h, http.MethodHead, path, nil, nil); w.Code != http.StatusMethodNotAllowed { + t.Errorf("HEAD %s = %d, want %d", path, w.Code, http.StatusMethodNotAllowed) + } + } +} diff --git a/internal/handler/rewrite_cache.go b/internal/handler/rewrite_cache.go index 8daf6aa..debe2b1 100644 --- a/internal/handler/rewrite_cache.go +++ b/internal/handler/rewrite_cache.go @@ -67,12 +67,15 @@ type rewriteEntry struct { // cooldownRewriteTTL is how long a rewrite made with cooldown on is served. // Versions leave cooldown as time passes, so a cached rewrite would otherwise -// keep hiding them. -const cooldownRewriteTTL = time.Minute +// keep hiding them. Cooldowns run to days, and a package manager that checks +// every lockfile entry on each install, as pnpm 12 does, would otherwise +// have the whole lockfile rewritten again on every install more than a +// minute after the last. +const cooldownRewriteTTL = time.Hour // size is how much of the cache's room the entry takes. func (e *rewriteEntry) size() int64 { - return int64(len(e.body) + len(e.gzipped)) + return int64(len(e.plain) + len(e.gzipped)) } // inflightRewrite is one rewrite that concurrent callers share. m and err are @@ -93,11 +96,50 @@ type metadataValidators struct { // rewrittenMetadata is a rewritten metadata document with its validators. // gzipped is the document gzip-compressed, or nil when there is no -// compressed copy to serve. +// compressed copy to serve. The cache keeps only gzipped when there is one, +// so plain is nil then; content returns the document either way. type rewrittenMetadata struct { - body []byte + plain []byte metadataValidators gzipped []byte + // size is the length of the uncompressed document. + size int +} + +// content returns the document uncompressed. A cached document is kept +// only gzip-compressed, and is decompressed for each caller that needs the +// plain bytes: clients not accepting gzip, and handlers reading the document. +// The compressed copy was made in memory by gzipIfSmaller, so reading it back +// fails only on a bug; content returns nil then. +func (m rewrittenMetadata) content() []byte { + if m.plain != nil || m.gzipped == nil { + return m.plain + } + zr, err := gzip.NewReader(bytes.NewReader(m.gzipped)) + if err != nil { + return nil + } + out := bytes.NewBuffer(make([]byte, 0, m.size)) + if _, err := io.Copy(out, zr); err != nil { + return nil + } + return out.Bytes() +} + +// identity returns m to be served without its gzip copy. +func (m rewrittenMetadata) identity() rewrittenMetadata { + m.plain = m.content() + m.gzipped = nil + return m +} + +// compact returns m as the cache keeps it: only the gzip copy when there is +// one. +func (m rewrittenMetadata) compact() rewrittenMetadata { + if m.gzipped != nil { + m.plain = nil + } + return m } // metadataETag returns the strong ETag of a rewritten document. @@ -248,14 +290,14 @@ func (c *rewriteCache) rewrite(ctx context.Context, key string, in []byte, ttl t c.mu.Lock() delete(c.inFlight, key) if f.err == nil { - c.add(&rewriteEntry{key: key, rewrittenMetadata: f.m, expires: expires}) + c.add(&rewriteEntry{key: key, rewrittenMetadata: f.m.compact(), expires: expires}) } c.mu.Unlock() close(f.done) }() f.m, f.err = rewriteWithETag(in, rewrite) if f.err == nil { - f.m.gzipped = gzipIfSmaller(f.m.body) + f.m.gzipped = gzipIfSmaller(f.m.plain) } return f.m, f.err } @@ -266,7 +308,7 @@ func rewriteWithETag(in []byte, rewrite func([]byte) ([]byte, error)) (rewritten if err != nil { return rewrittenMetadata{}, err } - return rewrittenMetadata{body: out, metadataValidators: metadataValidators{etag: metadataETag(out)}}, nil + return rewrittenMetadata{plain: out, metadataValidators: metadataValidators{etag: metadataETag(out)}, size: len(out)}, nil } // gzipIfSmaller returns body gzip-compressed, or nil when compressing does @@ -342,7 +384,7 @@ func (p *Proxy) cachedRewrite(ctx context.Context, ecosystem, proxyURL, name str if err != nil { return rewrittenMetadata{}, err } - return rewrittenMetadata{body: out, metadataValidators: metadataValidators{etag: keptVersionsETag(digest, proxyURL, name, kept)}}, nil + return rewrittenMetadata{plain: out, metadataValidators: metadataValidators{etag: keptVersionsETag(digest, proxyURL, name, kept)}, size: len(out)}, nil } m, err := rewriteWithETag(in, rewriteOnly) if err == nil && !cooldown { @@ -359,7 +401,7 @@ func (p *Proxy) uncachedRewrite(ecosystem, proxyURL, name string, out []byte) re if !p.cooldownEnabled() { v.lastModified = p.metadataDate(ecosystem, proxyURL, name, v.etag) } - return rewrittenMetadata{body: out, metadataValidators: v} + return rewrittenMetadata{plain: out, metadataValidators: v, size: len(out)} } func (p *Proxy) cooldownEnabled() bool { @@ -410,7 +452,7 @@ func (p *Proxy) storedRewriteVariant(ecosystem, cacheKey, proxyURL, name string, return rewrittenMetadata{}, "", false } m.lastModified = p.metadataDate(ecosystem, proxyURL, name, m.etag) - metrics.RecordMetadataCacheHit(ecosystem, len(m.body)) + metrics.RecordMetadataCacheHit(ecosystem, m.size) p.recordMetadataRead(entry) return m, contentType, true } @@ -447,10 +489,10 @@ func serveRewrittenMetadata(w http.ResponseWriter, r *http.Request, m rewrittenM // serveRewrittenMetadataAs is serveRewrittenMetadata for a document of // another content type. func serveRewrittenMetadataAs(w http.ResponseWriter, r *http.Request, m rewrittenMetadata, contentType string) { - body, etag := m.body, m.etag + etag := m.etag compressed := m.gzipped != nil && acceptsGzip(r) if compressed { - body, etag = m.gzipped, gzipETag(m.etag) + etag = gzipETag(m.etag) } ensureVary(w.Header(), headerAcceptEncoding) w.Header().Set(headerETag, etag) @@ -461,6 +503,10 @@ func serveRewrittenMetadataAs(w http.ResponseWriter, r *http.Request, m rewritte w.WriteHeader(http.StatusNotModified) return } + body := m.gzipped + if !compressed { + body = m.content() + } w.Header().Set(headerContentType, contentType) if compressed { w.Header().Set(headerContentEncoding, "gzip") diff --git a/internal/handler/rewrite_cache_test.go b/internal/handler/rewrite_cache_test.go index 373e845..8096bd6 100644 --- a/internal/handler/rewrite_cache_test.go +++ b/internal/handler/rewrite_cache_test.go @@ -45,8 +45,8 @@ func TestRewriteCache_RepeatServesCachedRewrite(t *testing.T) { for range 3 { m, err := c.rewrite(context.Background(), key, []byte("doc"), 0, r.fn) - if err != nil || string(m.body) != "DOC" { - t.Fatalf("rewrite = %q, %v", m.body, err) + if err != nil || string(m.content()) != "DOC" { + t.Fatalf("rewrite = %q, %v", m.content(), err) } } if got := r.calls.Load(); got != 1 { @@ -61,8 +61,8 @@ func TestRewriteCache_NewUpstreamBytesRewriteAgain(t *testing.T) { for _, doc := range []string{"v1", "v2"} { key := rewriteCacheKey("npm", "http://proxy", "left-pad", []byte(doc)) m, err := c.rewrite(context.Background(), key, []byte(doc), 0, r.fn) - if err != nil || string(m.body) != strings.ToUpper(doc) { - t.Fatalf("rewrite(%s) = %q, %v", doc, m.body, err) + if err != nil || string(m.content()) != strings.ToUpper(doc) { + t.Fatalf("rewrite(%s) = %q, %v", doc, m.content(), err) } } if got := r.calls.Load(); got != 2 { @@ -86,7 +86,7 @@ func TestRewriteCache_ConcurrentRequestsShareRewrite(t *testing.T) { if err != nil { t.Error(err) } - outs <- string(m.body) + outs <- string(m.content()) } go run() <-r.entered @@ -128,8 +128,8 @@ func TestRewriteCache_WaiterLeavingKeepsRewrite(t *testing.T) { if err := <-first; err != nil { t.Fatal(err) } - if m, err := c.rewrite(context.Background(), key, []byte("doc"), 0, r.fn); err != nil || string(m.body) != "DOC" { - t.Errorf("cached rewrite = %q, %v", m.body, err) + if m, err := c.rewrite(context.Background(), key, []byte("doc"), 0, r.fn); err != nil || string(m.content()) != "DOC" { + t.Errorf("cached rewrite = %q, %v", m.content(), err) } if got := r.calls.Load(); got != 1 { t.Errorf("rewrites = %d, want 1", got) From 96bd0f4a593efa8e5450ab3797d4041eec377821 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 29 Oct 2026 15:13:56 +0000 Subject: [PATCH 487/633] Bump github.com/git-pkgs/magic from 0.5.0 to 0.6.0 Bumps [github.com/git-pkgs/magic](https://github.com/git-pkgs/magic) from 0.5.0 to 0.6.0. - [Release notes](https://github.com/git-pkgs/magic/releases) - [Commits](https://github.com/git-pkgs/magic/compare/v0.5.0...v0.6.0) --- updated-dependencies: - dependency-name: github.com/git-pkgs/magic dependency-version: 0.6.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index dd44b27..b9b821e 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,7 @@ require ( github.com/git-pkgs/enrichment v0.7.4 github.com/git-pkgs/gcs v0.1.0 github.com/git-pkgs/integrity v0.2.0 - github.com/git-pkgs/magic v0.5.0 + github.com/git-pkgs/magic v0.6.0 github.com/git-pkgs/purl v0.1.24 github.com/git-pkgs/registries v0.12.0 github.com/git-pkgs/sbom v0.1.7 diff --git a/go.sum b/go.sum index 3b6c5c5..f1410b6 100644 --- a/go.sum +++ b/go.sum @@ -129,8 +129,8 @@ github.com/git-pkgs/gcs v0.1.0 h1:E3awGtsO0xZyHT9FUfEwMHjMkRxw41Bh+c7lgTmPvBo= github.com/git-pkgs/gcs v0.1.0/go.mod h1:bdkCFD66ryaWnU8MBhokVA3WkJfyAEchm9qec5woBpE= github.com/git-pkgs/integrity v0.2.0 h1:sxpAk7EUllZnTYttOoPTW/Yfe5J2imglpuJiuBUZOxM= github.com/git-pkgs/integrity v0.2.0/go.mod h1:vPLb2RB+hR/iCMIgqph0PZbQ7iktXDTK+uupoHQikKk= -github.com/git-pkgs/magic v0.5.0 h1:Pm+4fkHR+K6kg9pwJM4hnwWUuBlkYdLmF1yIAI8lU9Q= -github.com/git-pkgs/magic v0.5.0/go.mod h1:SXOqcsNmbmpZjJZHEEWnwxprbsFvpwWgTAZrgXp4Jm4= +github.com/git-pkgs/magic v0.6.0 h1:fINPe67FEIaFRVlZzhOIDClMkm+gftBroBENa8bHSxQ= +github.com/git-pkgs/magic v0.6.0/go.mod h1:SXOqcsNmbmpZjJZHEEWnwxprbsFvpwWgTAZrgXp4Jm4= github.com/git-pkgs/pom v0.1.8 h1:mXCuWw6XkrKKZb6IurRZyIznYLPhV5AYg5ZO38Uj/l8= github.com/git-pkgs/pom v0.1.8/go.mod h1:sQBeYRzjCUn77fcABPn6I/a1aBHFxm55jTmkNxYtGtw= github.com/git-pkgs/purl v0.1.24 h1:JMEB1uNWI+baEpl6AeVx+Kfq1c+xkou+THxa5KJ3MrU= From ce6061139195c4450cabdb24fa69a85c21347304 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 29 Oct 2026 15:14:17 +0000 Subject: [PATCH 488/633] Bump github.com/klauspost/compress from 1.20.1 to 1.20.3 Bumps [github.com/klauspost/compress](https://github.com/klauspost/compress) from 1.20.1 to 1.20.3. - [Release notes](https://github.com/klauspost/compress/releases) - [Commits](https://github.com/klauspost/compress/compare/v1.20.1...v1.20.3) --- updated-dependencies: - dependency-name: github.com/klauspost/compress dependency-version: 1.20.3 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index dd44b27..6a2d234 100644 --- a/go.mod +++ b/go.mod @@ -22,7 +22,7 @@ require ( github.com/git-pkgs/spdx v0.3.3 github.com/go-chi/chi/v5 v5.5.0 github.com/jmoiron/sqlx v1.4.0 - github.com/klauspost/compress v1.20.1 + github.com/klauspost/compress v1.20.3 github.com/lib/pq v1.12.3 github.com/opencontainers/go-digest v1.0.0 github.com/prometheus/client_golang v1.25.0 diff --git a/go.sum b/go.sum index 3b6c5c5..505e414 100644 --- a/go.sum +++ b/go.sum @@ -202,8 +202,8 @@ github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFF github.com/juju/gnuflag v0.0.0-20171113085948-2ce1bb71843d/go.mod h1:2PavIy+JPciBPrBUjwbNvtwB6RQlve+hkpll6QSNmOE= github.com/keybase/go-keychain v0.0.1 h1:way+bWYa6lDppZoZcgMbYsvC7GxljxrskdNInRtuthU= github.com/keybase/go-keychain v0.0.1/go.mod h1:PdEILRW3i9D8JcdM+FmY6RwkHGnhHxXwkPPMeUgOK1k= -github.com/klauspost/compress v1.20.1 h1:T7kKElXUMXrUJ2E9QhQhxFtcK5rPyLdsGZvdbLMPdiQ= -github.com/klauspost/compress v1.20.1/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= +github.com/klauspost/compress v1.20.3 h1:zWNoFqq/lSuYQRim3dUXlXeA0IqwuZGicdG5ZTfY5j4= +github.com/klauspost/compress v1.20.3/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= From 359ba32866a3fb5fe983d6b31acb2d5c1c20b026 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 29 Oct 2026 15:14:22 +0000 Subject: [PATCH 489/633] Bump github.com/git-pkgs/archives from 0.9.0 to 0.10.0 Bumps [github.com/git-pkgs/archives](https://github.com/git-pkgs/archives) from 0.9.0 to 0.10.0. - [Commits](https://github.com/git-pkgs/archives/compare/v0.9.0...v0.10.0) --- updated-dependencies: - dependency-name: github.com/git-pkgs/archives dependency-version: 0.10.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index dd44b27..3bfa70f 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/Masterminds/semver/v3 v3.5.0 github.com/aws/aws-sdk-go-v2/config v1.33.9 github.com/aws/aws-sdk-go-v2/service/ecr v1.66.5 - github.com/git-pkgs/archives v0.9.0 + github.com/git-pkgs/archives v0.10.0 github.com/git-pkgs/artifacts v0.2.1 github.com/git-pkgs/cooldown v0.3.0 github.com/git-pkgs/enrichment v0.7.4 diff --git a/go.sum b/go.sum index 3b6c5c5..20a3f25 100644 --- a/go.sum +++ b/go.sum @@ -117,8 +117,8 @@ github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a h1:yDWHCSQ40h88yi github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a/go.mod h1:7Ga40egUymuWXxAe151lTNnCv97MddSOVsjpPPkityA= github.com/felixge/httpsnoop v1.1.0 h1:3YtUj32ZZkqZtt3sZZsClsymw/QDuVfpNhoA31zeORc= github.com/felixge/httpsnoop v1.1.0/go.mod h1:Zqxgdd+1Rkcz8euOqdr7lqgCRJztwr5hp9vDSi5UZCE= -github.com/git-pkgs/archives v0.9.0 h1:j5+GtZLrcsefIG5QxAQ2dcdCFf57uizcn03PyGRFA3U= -github.com/git-pkgs/archives v0.9.0/go.mod h1:GJ/h7NPTUYiPinOUxpYNsHFCUikYZcJXvugBvP1VGNc= +github.com/git-pkgs/archives v0.10.0 h1:/qgk0MI6VK2sKOoVTd0B4zs5hwTo+St0X8cYYftzumc= +github.com/git-pkgs/archives v0.10.0/go.mod h1:GJ/h7NPTUYiPinOUxpYNsHFCUikYZcJXvugBvP1VGNc= github.com/git-pkgs/artifacts v0.2.1 h1:VwdxR4yTDaqBZ34h0slxQBVyr2Xfa+QGOKCKviMx2xk= github.com/git-pkgs/artifacts v0.2.1/go.mod h1:Otosgq52pXT5UNN7lh6s/lszpWKVDO8XrOjN1M70/IA= github.com/git-pkgs/cooldown v0.3.0 h1:eAl2CQFwqXxluoVP9hHmropAf1VoV3gSxLMkN+NEu8A= From b31ee702a5ca7f9833eec5a4a8ccf25d21fae937 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 29 Oct 2026 15:14:31 +0000 Subject: [PATCH 490/633] Bump modernc.org/sqlite from 1.60.1 to 1.61.0 Bumps [modernc.org/sqlite](https://gitlab.com/cznic/sqlite) from 1.60.1 to 1.61.0. - [Changelog](https://gitlab.com/cznic/sqlite/blob/master/CHANGELOG.md) - [Commits](https://gitlab.com/cznic/sqlite/compare/v1.60.1...v1.61.0) --- updated-dependencies: - dependency-name: modernc.org/sqlite dependency-version: 1.61.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- go.mod | 10 +++++----- go.sum | 36 ++++++++++++++++++------------------ 2 files changed, 23 insertions(+), 23 deletions(-) diff --git a/go.mod b/go.mod index dd44b27..9cac691 100644 --- a/go.mod +++ b/go.mod @@ -35,7 +35,7 @@ require ( golang.org/x/sync v0.24.0 google.golang.org/protobuf v1.36.12 gopkg.in/yaml.v3 v3.0.1 - modernc.org/sqlite v1.60.1 + modernc.org/sqlite v1.61.0 ) require ( @@ -72,7 +72,7 @@ require ( github.com/beorn7/perks v1.0.1 // indirect github.com/cenk/backoff v2.2.1+incompatible // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect - github.com/dustin/go-humanize v1.0.1 // indirect + github.com/dustin/go-humanize v1.1.0 // indirect github.com/ecosyste-ms/ecosystems-go v0.5.0 // indirect github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a // indirect github.com/git-pkgs/pom v0.1.8 // indirect @@ -96,7 +96,7 @@ require ( github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-isatty v0.0.24 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect - github.com/ncruces/go-strftime v1.0.0 // indirect + github.com/ncruces/go-strftime v1.1.0 // indirect github.com/oapi-codegen/nullable v1.2.0 // indirect github.com/oapi-codegen/runtime v1.7.0 // indirect github.com/package-url/packageurl-go v0.1.7 // indirect @@ -121,13 +121,13 @@ require ( golang.org/x/oauth2 v0.37.0 // indirect golang.org/x/sys v0.49.0 // indirect golang.org/x/text v0.43.0 // indirect - golang.org/x/tools v0.51.0 // indirect + golang.org/x/tools v0.52.0 // indirect golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect google.golang.org/api v0.297.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20260904194346-d0f1323225a4 // indirect google.golang.org/grpc v1.83.2 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect - modernc.org/libc v1.77.1 // indirect + modernc.org/libc v1.79.0 // indirect modernc.org/mathutil v1.7.1 // indirect modernc.org/memory v1.12.1 // indirect ) diff --git a/go.sum b/go.sum index 3b6c5c5..fd6fb83 100644 --- a/go.sum +++ b/go.sum @@ -104,8 +104,8 @@ github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2/go.mod h1:qwXFYgsP6T7X github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= -github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/dustin/go-humanize v1.1.0 h1:dbKTrvD0klcbBV/h4AWJdMuZogJACoMlvWIWZ5b2xWg= +github.com/dustin/go-humanize v1.1.0/go.mod h1:hc1CvRkJMsgxqjmjMQF3QNRAZBwY8AXBAzKYoSX9sFI= github.com/ecosyste-ms/ecosystems-go v0.5.0 h1:qZqgar3Do9RrtW327OXlsrGKiRvNdeX8CmXb1aX4Njo= github.com/ecosyste-ms/ecosystems-go v0.5.0/go.mod h1:IhsYBZfxkofxya0O1O2DlhDt281utWJ6VQO0mPVb99I= github.com/envoyproxy/go-control-plane v0.14.0 h1:hbG2kr4RuFj222B6+7T83thSPqLjwBIfQawTkC++2HA= @@ -181,8 +181,8 @@ github.com/google/go-replayers/httpreplay v1.2.0 h1:VM1wEyyjaoU53BwrOnaf9VhAyQQE github.com/google/go-replayers/httpreplay v1.2.0/go.mod h1:WahEFFZZ7a1P4VM1qEeHy+tME4bwyqPcwWbNlUI1Mcg= github.com/google/martian/v3 v3.3.3 h1:DIhPTQrbPkgs2yJYdXU/eNACCG5DVQjySNRNlflZ9Fc= github.com/google/martian/v3 v3.3.3/go.mod h1:iEPrYcgCF7jA9OtScMFQyAlZZ4YXTKEtJ1E6RWzmBA0= -github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3 h1:LMLX+LgTNWpfvCBdFebv6EsYotImrt/Ppc5cXIriCSo= -github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3/go.mod h1:jl5iWTm0/hd5PjEYEOuwAJ57L/CibdZfrqZ5XA5GrCk= +github.com/google/pprof v0.0.0-20261016150608-1facf8f91b50 h1:PlNWJRxu5APSV0pWqfIsZ0qt9OOGHSWfZQ4zvx+WgWs= +github.com/google/pprof v0.0.0-20261016150608-1facf8f91b50/go.mod h1:yUo1AMMpsPIDFGuwDK8nLcf8+vvAd5l2O6JDZ6hpwrM= github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0= github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= @@ -227,8 +227,8 @@ github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w= -github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= +github.com/ncruces/go-strftime v1.1.0 h1:vtPmPWMkb539RGV9WpWT9EwJGvrOCKvMNI4obWL47Hk= +github.com/ncruces/go-strftime v1.1.0/go.mod h1:yVaq7iwCN09vHRrKBrrflXT0M9tTMG2KmZ9Fj7xeG/g= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/oapi-codegen/nullable v1.2.0 h1:VflFkDW980KhBPiFF7nWSyjg+r4Obqj8lXipV0UkP5w= github.com/oapi-codegen/nullable v1.2.0/go.mod h1:KUZ3vUzkmEKY90ksAmit2+5juDIhIZhfDl+0PwOQlFY= @@ -332,8 +332,8 @@ golang.org/x/text v0.43.0/go.mod h1:hbSIYA/amXcRXTFSZqh8tWHEoqRvvbKPJ2uz/KU+a+s= golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.51.0 h1:k4Xc/1Om9jwkBJBo4NVLMSARBoWtK10mx+W5BnXCeAI= -golang.org/x/tools v0.51.0/go.mod h1:9eEncMayCV6zRMGhR5eZEC2iBx98qWcF1HZ9Z7wJOoA= +golang.org/x/tools v0.52.0 h1:ryxg4U63oaUYk7uOLTF37n43f4qgtGvqV4Ioit1BaEs= +golang.org/x/tools v0.52.0/go.mod h1:ItcP5UNaVGG+Yc4eIYmaFLNJ4soEopkx5Y2d4lcEurs= golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da h1:noIWHXmPHxILtqtCOPIhSt0ABwskkZKjD3bXGnZGpNY= golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= @@ -362,20 +362,20 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -modernc.org/cc/v4 v4.29.7 h1:q+NXGJ0bK3b4TXFYQQVr9pYETGnmwFWkrUzJnMya/Tg= -modernc.org/cc/v4 v4.29.7/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI= -modernc.org/ccgo/v4 v4.36.1 h1:ZNIUZAryN0UgnJwtyxrdEzcFc3yD4Cu4AzjfPXsLsIE= -modernc.org/ccgo/v4 v4.36.1/go.mod h1:rrtGc2QkS239nYb/mQNuBMyjq3/y3ZXWbBjPoV3wqzA= +modernc.org/cc/v4 v4.30.3 h1:aDq8gIX1X7+VFgO/JYYji6pLWL2H4tv3g8WatT3KCys= +modernc.org/cc/v4 v4.30.3/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI= +modernc.org/ccgo/v4 v4.37.0 h1:BAuJCUtIHIxFIYaFhqXIw5YaoMBlNCUlFovejHfMip4= +modernc.org/ccgo/v4 v4.37.0/go.mod h1:/8627VhwWtAbqUXmPORxFQU5SgLrAx2jl9seLw9GBhY= modernc.org/fileutil v1.4.0 h1:j6ZzNTftVS054gi281TyLjHPp6CPHr2KCxEXjEbD6SM= modernc.org/fileutil v1.4.0/go.mod h1:EqdKFDxiByqxLk8ozOxObDSfcVOv/54xDs/DUHdvCUU= modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI= modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito= -modernc.org/gc/v3 v3.1.5 h1:21ldfPfRYE31Tb7B3mwAK8gy1AxP4+dKjrOQPfqakoc= -modernc.org/gc/v3 v3.1.5/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY= +modernc.org/gc/v3 v3.2.0 h1:/Y3LGDMQaoYf55PebP1qsZ5FDGgAZyG1RYKEHeQzh1o= +modernc.org/gc/v3 v3.2.0/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY= modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks= modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI= -modernc.org/libc v1.77.1 h1:Ct8j47QtiZ1Enj2DtFXQtUqrPCAjdCmPjtCuvrYQ0Hs= -modernc.org/libc v1.77.1/go.mod h1:87/pZ4L6nD1zqW4nItuS12YO7hN1igAah34xjnQo/W0= +modernc.org/libc v1.79.0 h1:9Dvvv++bAMSkSuBzuOFAatbTV5OIoq6Y6FohmD3e7q8= +modernc.org/libc v1.79.0/go.mod h1:uT3kR61NsbeKJSLqq83GJwSnZMVLlSjyAz4DfTaBy1M= modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= modernc.org/memory v1.12.1 h1:nFMiWrpStgZczNl6XI9GnIk/rWhYIyHGUaR04pGbp9g= @@ -384,8 +384,8 @@ modernc.org/opt v0.2.0 h1:tGyef5ApycA7FSEOMraay9SaTk5zmbx7Tu+cJs4QKZg= modernc.org/opt v0.2.0/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns= modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w= modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE= -modernc.org/sqlite v1.60.1 h1:/blz53O951KWFOso4QQvEs/Fq6cDBKLtMVrYNSeJVKw= -modernc.org/sqlite v1.60.1/go.mod h1:1dIoEagfDE72QytD5scH1lxARtaUgKgHC/NuApA27r0= +modernc.org/sqlite v1.61.0 h1:sQi63oLQ9uquAUA4CzOisH35Q3pEszkPBuHM0tFL8CY= +modernc.org/sqlite v1.61.0/go.mod h1:f4gr5tbpx2umklaT/QVtbi/50hNdM+4ng+2zMo8KXJ0= modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0= modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A= modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= From 58e87c2571bd23a0cf3d5d987a03777e73a3e873 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Thu, 29 Oct 2026 19:19:12 +0100 Subject: [PATCH 491/633] Cache cooldown rewrites for a 24th of the shortest cooldown An hour is long next to a cooldown of a few hours. The TTL is now a 24th of the shortest cooldown configured, between a minute and an hour, and the RubyGems and Hex section no longer says a minute. --- docs/configuration.md | 4 +- internal/config/config.go | 23 ++++++++++ internal/config/config_test.go | 20 +++++++++ internal/handler/handler.go | 9 ++-- internal/handler/metadata_validators_test.go | 4 +- internal/handler/rewrite_cache.go | 44 +++++++++++++++----- internal/handler/rewrite_cache_test.go | 23 +++++++++- internal/server/server.go | 1 + 8 files changed, 108 insertions(+), 20 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index cf21075..941bc0f 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1141,7 +1141,7 @@ Maven caveats: RubyGems, Hex and PyPI keep their publish times outside the metadata clients read: the RubyGems versions API, the Hex HTTP API (`upstream.hex_api`) and the PyPI JSON API. The proxy fetches them alongside each compact index, package or simple page it filters, and for PyPI downloads. Cooldown fails closed: when it applies to a package and those times can't be fetched or parsed, or the metadata can't be rewritten, the request fails with 502, or with 503 and `Retry-After` while the upstream is rate limiting the proxy, instead of serving the metadata unfiltered. The same goes for npm, Composer and pub.dev metadata that can't be rewritten, and for npm tarball downloads whose publish time can't be read. A PyPI file or npm tarball that is already cached is still served when its publish time can't be read; only a download that would have to be fetched fails. A PyPI release's time is the upload time of its newest file. The proxy stores it when a download looks it up, and serves files of that release that are already cached against the stored time without reading the JSON API again. A file that isn't cached yet is always looked up, since it may have been added to the release since, and the newer time is stored. Packages exempted with a duration of `0` are served as before and don't need the times. A PyPI package the JSON API returns 404 for has no times to go by and is served unfiltered. -For RubyGems and Hex, both documents go through the metadata cache. With [`cache_metadata`](#metadata-caching) they are served from it within `metadata_ttl` without asking the upstream, revalidated with their ETags after it, and filtered from the last cached copies, marked stale, while the upstream is down; without it they are still fetched once for concurrent requests. Each document has its own TTL, so the compact index or Hex package can be newer than the cached times and list a version they don't have. The times are then fetched again before filtering, and if that fails, such versions are left out until the times can be read. The filtered index or package carries an ETag of its own, so Bundler and Mix revalidate it with `304 Not Modified`. Publish times are compared with the current time on each request, but the filtered result is kept for up to a minute (see [rewritten metadata cache](#rewritten-metadata-cache)), so a version leaving cooldown can show up a minute late; a newly published version shows up within `metadata_ttl`, as with other cached metadata. Upstream errors are answered as for other cached metadata (404, or 502) rather than relayed. +For RubyGems and Hex, both documents go through the metadata cache. With [`cache_metadata`](#metadata-caching) they are served from it within `metadata_ttl` without asking the upstream, revalidated with their ETags after it, and filtered from the last cached copies, marked stale, while the upstream is down; without it they are still fetched once for concurrent requests. Each document has its own TTL, so the compact index or Hex package can be newer than the cached times and list a version they don't have. The times are then fetched again before filtering, and if that fails, such versions are left out until the times can be read. The filtered index or package carries an ETag of its own, so Bundler and Mix revalidate it with `304 Not Modified`. Publish times are compared with the current time on each request, but the filtered result is kept for up to an hour (see [rewritten metadata cache](#rewritten-metadata-cache)), so a version leaving cooldown can show up that much later; a newly published version shows up within `metadata_ttl`, as with other cached metadata. Upstream errors are answered as for other cached metadata (404, or 502) rather than relayed. With cooldown on, the proxy asks npm for full packuments, since the abbreviated ones it uses otherwise carry no publish times. Abbreviated packuments cached before cooldown was turned on are fetched again in full rather than served, including as a stale copy when the upstream is down. When `latest` is in cooldown, the packument tags the newest version out of cooldown instead, and `npm dist-tag ls` shows the same tags. @@ -1420,7 +1420,7 @@ Or via environment variable: `PROXY_ARTIFACT_MAX_SIZE=2GB`. ### Rewritten metadata cache -The npm, Composer, PyPI and Helm handlers rewrite every metadata document they serve, so that download URLs point at the proxy. That means decoding the whole document and encoding it again, and Composer's minified documents are also expanded, which together cost milliseconds per typical package and far more for very large ones; a large Helm `index.yaml` takes over a hundred milliseconds. The proxy keeps rewritten documents in memory, so each distinct upstream document is rewritten once and requests arriving while it is being rewritten wait for that rewrite. New bytes from upstream are rewritten again. A PyPI simple page is cached per representation (HTML or JSON) and per set of versions filtered out of it. With version cooldown enabled, a cached rewrite is kept for an hour only, because cooldown filtering depends on the current time: a version leaving cooldown shows up at most an hour late. Cached documents carry an `ETag` of their own, and are dated like any other, so clients that revalidate with `If-None-Match` or `If-Modified-Since`, such as Composer, get `304 Not Modified`. The RubyGems compact index and Hex packages filtered by cooldown are kept the same way; they are never served gzip-compressed. +The npm, Composer, PyPI and Helm handlers rewrite every metadata document they serve, so that download URLs point at the proxy. That means decoding the whole document and encoding it again, and Composer's minified documents are also expanded, which together cost milliseconds per typical package and far more for very large ones; a large Helm `index.yaml` takes over a hundred milliseconds. The proxy keeps rewritten documents in memory, so each distinct upstream document is rewritten once and requests arriving while it is being rewritten wait for that rewrite. New bytes from upstream are rewritten again. A PyPI simple page is cached per representation (HTML or JSON) and per set of versions filtered out of it. With version cooldown enabled, a cached rewrite expires, because cooldown filtering depends on the current time: it is kept for a 24th of the shortest cooldown configured, at least a minute and at most an hour, so a version shows up at most that long after its cooldown ends. With a cooldown of a day or more that is an hour; with `"2h"` somewhere, five minutes. Cached documents carry an `ETag` of their own, and are dated like any other, so clients that revalidate with `If-None-Match` or `If-Modified-Since`, such as Composer, get `304 Not Modified`. The RubyGems compact index and Hex packages filtered by cooldown are kept the same way; they are never served gzip-compressed. Each cached document is kept gzip-compressed, when that makes it smaller, and served with `Content-Encoding: gzip` to clients that accept it, as npm, pnpm, Yarn and Composer do. A large packument is compressed once rather than on every request, by the proxy or by a reverse proxy in front of it. Only the compressed copy is kept, typically an eighth of the document, and it is decompressed for the clients that don't accept gzip. Documents rewritten outside the cache are served uncompressed. diff --git a/internal/config/config.go b/internal/config/config.go index 3adb7fd..5e90150 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -75,6 +75,7 @@ import ( "strings" "time" + "github.com/git-pkgs/cooldown" "github.com/git-pkgs/proxy/internal/denylist" "github.com/git-pkgs/proxy/internal/retention" "github.com/git-pkgs/purl" @@ -270,6 +271,28 @@ func (c *CooldownConfig) NormalizedPackages() map[string]string { return normalized } +// Shortest returns the shortest cooldown configured anywhere, by default, +// ecosystem, package or package pattern, or 0 when none is. Unparsable +// durations are left out; Validate reports them. +func (c *CooldownConfig) Shortest() time.Duration { + if c == nil { + return 0 + } + var shortest time.Duration + consider := func(s string) { + if d, err := cooldown.ParseDuration(s); err == nil && d > 0 && (shortest == 0 || d < shortest) { + shortest = d + } + } + consider(c.Default) + for _, m := range []map[string]string{c.Ecosystems, c.Packages, c.PackagePatterns} { + for _, v := range m { + consider(v) + } + } + return shortest +} + // ScanningConfig configures pre-cache artifact scanning (e.g. trivy, // ClamAV, Wiz, or a custom service) to mitigate supply chain attacks. // Unlike Cooldown, which only looks at a version's publish timestamp, diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 2f89de2..a0132cf 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -2056,3 +2056,23 @@ upstream: t.Errorf("generic_paths = %v", cfg.Upstream.GenericPaths) } } + +func TestCooldownShortest(t *testing.T) { + for _, tc := range []struct { + name string + cfg *CooldownConfig + want time.Duration + }{ + {"nil", nil, 0}, + {"none", &CooldownConfig{Default: "0"}, 0}, + {"default", &CooldownConfig{Default: "3d"}, 72 * time.Hour}, + {"ecosystem", &CooldownConfig{Default: "3d", Ecosystems: map[string]string{"npm": "2d", "deb": "0"}}, 48 * time.Hour}, + {"package", &CooldownConfig{Default: "3d", Packages: map[string]string{"pkg:npm/lodash": "6h"}}, 6 * time.Hour}, + {"pattern", &CooldownConfig{Default: "0", PackagePatterns: map[string]string{"pkg:npm/@acme/*": "90m"}}, 90 * time.Minute}, + {"invalid ignored", &CooldownConfig{Default: "3d", Ecosystems: map[string]string{"npm": "soon"}}, 72 * time.Hour}, + } { + if got := tc.cfg.Shortest(); got != tc.want { + t.Errorf("%s: Shortest() = %v, want %v", tc.name, got, tc.want) + } + } +} diff --git a/internal/handler/handler.go b/internal/handler/handler.go index b3ca1c4..5693421 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -210,9 +210,12 @@ type Proxy struct { // RouteFor returns how to fetch a package that doesn't come from the // ecosystem's default upstream, such as an npm package in // upstream.npm_routes, or nil to use Resolver. - RouteFor func(ecosystem, name string) *ArtifactRoute - Logger *slog.Logger - Cooldown CooldownPolicy + RouteFor func(ecosystem, name string) *ArtifactRoute + Logger *slog.Logger + Cooldown CooldownPolicy + // ShortestCooldown is the shortest cooldown configured anywhere. It + // bounds how long a rewrite filtered by cooldown is cached. + ShortestCooldown time.Duration Denylist *denylist.Policy CacheMetadata bool MetadataTTL time.Duration diff --git a/internal/handler/metadata_validators_test.go b/internal/handler/metadata_validators_test.go index a6e1bae..f7c1475 100644 --- a/internal/handler/metadata_validators_test.go +++ b/internal/handler/metadata_validators_test.go @@ -720,14 +720,14 @@ func TestRewrittenMetadataETagWithCooldownFollowsKeptVersions(t *testing.T) { if rec := serve(handler, c.path, http.Header{"If-None-Match": {etag}}); rec.Code != http.StatusNotModified { t.Errorf("within the rewrite's lifetime: status %d, want 304 from the cached rewrite", rec.Code) } - clock.advance(cooldownRewriteTTL) + clock.advance(maxCooldownRewriteTTL) rec := serve(handler, c.path, http.Header{"If-None-Match": {etag}}) if rec.Code != http.StatusOK || rec.Header().Get("ETag") == etag { t.Errorf("after a version left cooldown: status %d, ETag %q; want 200 with a new ETag", rec.Code, rec.Header().Get("ETag")) } proxy.Cooldown = &cooldown.Config{Default: "1d"} - clock.advance(cooldownRewriteTTL) + clock.advance(maxCooldownRewriteTTL) expectNotModified(t, serve(handler, c.path, http.Header{"If-None-Match": {etag}}), etag) }) } diff --git a/internal/handler/rewrite_cache.go b/internal/handler/rewrite_cache.go index debe2b1..b2dc992 100644 --- a/internal/handler/rewrite_cache.go +++ b/internal/handler/rewrite_cache.go @@ -35,8 +35,8 @@ import ( // depends on is fixed for the life of the process: the proxy URL is part of // the key, and the denylist is loaded at startup. The exception is cooldown, // which depends on the time of the request: with cooldown on, entries expire -// after cooldownRewriteTTL, so a version is held back at most that long past -// its cooldown. +// after Proxy.cooldownRewriteTTL, so a version is held back at most that long +// past its cooldown. // // Each entry keeps the ETag of its output, so clients can revalidate their // copy without the output being hashed again on every request. Dates are not @@ -65,13 +65,35 @@ type rewriteEntry struct { expires time.Time } -// cooldownRewriteTTL is how long a rewrite made with cooldown on is served. -// Versions leave cooldown as time passes, so a cached rewrite would otherwise -// keep hiding them. Cooldowns run to days, and a package manager that checks -// every lockfile entry on each install, as pnpm 12 does, would otherwise -// have the whole lockfile rewritten again on every install more than a -// minute after the last. -const cooldownRewriteTTL = time.Hour +// maxCooldownRewriteTTL is the longest a rewrite made with cooldown on is +// served. Versions leave cooldown as time passes, so a cached rewrite would +// otherwise keep hiding them. Cooldowns run to days, and a package manager +// that checks every lockfile entry on each install, as pnpm 12 does, would +// otherwise have the whole lockfile rewritten again on every install more +// than a minute after the last. +const maxCooldownRewriteTTL = time.Hour + +// minCooldownRewriteTTL is the shortest a rewrite made with cooldown on is +// served, however short the cooldown. +const minCooldownRewriteTTL = time.Minute + +// cooldownRewriteShare is the share of the shortest cooldown a rewrite made +// with cooldown on is served for, as a divisor. +const cooldownRewriteShare = 24 + +// cooldownRewriteTTL returns how long a rewrite made with cooldown on is +// served: a 24th of the shortest cooldown configured, so a version is held +// back at most about 4% past its cooldown, kept between a minute and an hour. +func (p *Proxy) cooldownRewriteTTL() time.Duration { + ttl := p.ShortestCooldown / cooldownRewriteShare + switch { + case ttl <= 0 || ttl > maxCooldownRewriteTTL: + return maxCooldownRewriteTTL + case ttl < minCooldownRewriteTTL: + return minCooldownRewriteTTL + } + return ttl +} // size is how much of the cache's room the entry takes. func (e *rewriteEntry) size() int64 { @@ -348,7 +370,7 @@ func (c *rewriteCache) add(e *rewriteEntry) { // metadata cache stored it, and is empty otherwise; when known it keys the // cache, so the document isn't hashed again. // -// With cooldown on, the cached rewrite expires after cooldownRewriteTTL. Its +// With cooldown on, the cached rewrite expires after p.cooldownRewriteTTL. Its // ETag is the hash of its output, and it is dated like any other, so clients // revalidate with either validator and get the gzip copy. // @@ -370,7 +392,7 @@ func (p *Proxy) cachedRewrite(ctx context.Context, ecosystem, proxyURL, name str } var ttl time.Duration if cooldown { - ttl = cooldownRewriteTTL + ttl = p.cooldownRewriteTTL() } m, err := p.rewrites.rewrite(ctx, key, in, ttl, rewriteOnly) if err == nil { diff --git a/internal/handler/rewrite_cache_test.go b/internal/handler/rewrite_cache_test.go index 8096bd6..7188634 100644 --- a/internal/handler/rewrite_cache_test.go +++ b/internal/handler/rewrite_cache_test.go @@ -200,7 +200,7 @@ func TestCachedRewrite_DisabledRewritesEveryTime(t *testing.T) { } // TestCachedRewrite_CooldownExpires asserts that with cooldown on a rewrite -// is cached for cooldownRewriteTTL only: cooldown filtering depends on the +// is cached for maxCooldownRewriteTTL only: cooldown filtering depends on the // current time, so a cached rewrite would otherwise keep hiding a version // after its cooldown ends. func TestCachedRewrite_CooldownExpires(t *testing.T) { @@ -218,7 +218,7 @@ func TestCachedRewrite_CooldownExpires(t *testing.T) { t.Errorf("rewrites within the TTL = %d, want 1", got) } - clock.advance(cooldownRewriteTTL) + clock.advance(maxCooldownRewriteTTL) _, _ = proxy.cachedRewrite(context.Background(), "npm", "http://proxy", "left-pad", []byte("doc"), "", r.keeping) if got := r.calls.Load(); got != 2 { t.Errorf("rewrites after the TTL = %d, want 2", got) @@ -301,3 +301,22 @@ func (c *rewriteCache) len() int { defer c.mu.Unlock() return len(c.entries) } + +// TestCooldownRewriteTTL checks that a rewrite filtered by cooldown is cached +// for a 24th of the shortest cooldown, between a minute and an hour. +func TestCooldownRewriteTTL(t *testing.T) { + for _, tc := range []struct { + shortest, want time.Duration + }{ + {0, time.Hour}, + {72 * time.Hour, time.Hour}, + {24 * time.Hour, time.Hour}, + {2 * time.Hour, 5 * time.Minute}, + {10 * time.Minute, time.Minute}, + } { + p := &Proxy{ShortestCooldown: tc.shortest} + if got := p.cooldownRewriteTTL(); got != tc.want { + t.Errorf("shortest cooldown %v: TTL = %v, want %v", tc.shortest, got, tc.want) + } + } +} diff --git a/internal/server/server.go b/internal/server/server.go index d119fcb..2f0bd68 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -255,6 +255,7 @@ func (s *Server) serve(listener net.Listener) error { return fmt.Errorf("configuring cooldown policy: %w", err) } proxy.Cooldown = cooldownPolicy + proxy.ShortestCooldown = s.cfg.Cooldown.Shortest() if s.cooldown, err = newCooldownView(s.cfg.Cooldown, cooldownPolicy); err != nil { return err } From 50e65a31d8d9524e9ae193a5939ded0241d22236 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Tue, 27 Oct 2026 16:16:11 +0100 Subject: [PATCH 492/633] Serve npm attestations through the proxy Packuments list each version's provenance at dist.attestations.url on registry.npmjs.org. npm takes only the path of that URL and resolves it against the configured registry, so behind the proxy `npm audit signatures` asked for /-/npm/v1/attestations/... at the proxy's root, outside /npm, and could not verify attestations. The rewrite now points dist.attestations.url at /npm/-/npm/v1/attestations/{name}@{version}, and the handler serves that path. It reads the version's attestations URL from the packument and fetches it from the package's registry, the route's for routed packages; a URL on another host is not followed and the registry's standard path is asked instead. Attestations never change once published, so they are cached as immutable metadata. Versions blocked by the denylist get 403 and versions held back by cooldown 404, as their tarballs do. Pushed versions have no upstream attestations and get 404. --- README.md | 6 +- docs/configuration.md | 1 + internal/handler/npm.go | 36 ++-- internal/handler/npm_attestations.go | 122 ++++++++++++++ internal/handler/npm_attestations_test.go | 191 ++++++++++++++++++++++ internal/metadata/npm/metadata.go | 96 ++++++++--- internal/metadata/npm/metadata_test.go | 29 ++++ 7 files changed, 447 insertions(+), 34 deletions(-) create mode 100644 internal/handler/npm_attestations.go create mode 100644 internal/handler/npm_attestations_test.go diff --git a/README.md b/README.md index 48a575a..3313324 100644 --- a/README.md +++ b/README.md @@ -186,7 +186,10 @@ through the proxy: the audit and signing-key endpoints are passed through to the configured upstream registry, with upstream authentication applied. Advisories therefore come from upstream's database, not from the proxy's own vulnerability data, and versions withheld by [cooldown](#version-cooldown) are not excluded -from the report. +from the report. Packuments point each version's attestations (provenance) at +the proxy, which fetches them from the package's registry and caches them; +attestations of versions held back by cooldown or blocked by the denylist are +not served. To serve private packages through the same registry URL, see [Private package routes](docs/configuration.md#private-package-routes). @@ -1262,6 +1265,7 @@ Recently cached: | `POST /npm/-/npm/v1/security/*` | npm/pnpm/Yarn audit endpoints, passed through to upstream | | `GET /npm/-/npm/v1/keys` | npm registry signing keys, passed through to upstream | | `GET /npm/-/package/{name}/dist-tags` | `npm dist-tag ls`, answered from the packument the proxy serves (cooldown and pushed versions applied); changing tags is not supported | +| `GET /npm/-/npm/v1/attestations/{name}@{version}` | npm provenance attestations of a version, fetched from the package's registry and cached | | `GET /npm/-/v1/search`, `GET /npm/-/ping` | `npm search` and `npm ping`, passed through to upstream uncached | | `GET /cargo/*` | Cargo sparse index protocol | | `GET /gem/*` | RubyGems protocol | diff --git a/docs/configuration.md b/docs/configuration.md index 941bc0f..57b3edf 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -388,6 +388,7 @@ upstream: - Composer metadata of routed packages carries an empty `notification-url`, so Composer does not report their installs to the default upstream's `notify-batch` endpoint. - Use `upstream.auth` for the registry's credentials. The proxy has no authentication of its own, so every client that can reach it can download routed packages. - Search (`/composer/search.json`) and the package list (`/composer/packages/list.json`) still come from the default upstream only. +- Attestations of a routed npm version (`/npm/-/npm/v1/attestations/...`) come from the route, like its tarballs. - npm audit, signing-key, search and ping requests still go to `upstream.npm`. An audit request lists every package in the dependency tree, routed ones included. - Routes can only be set in the configuration file, not with environment variables. diff --git a/internal/handler/npm.go b/internal/handler/npm.go index 537c4b3..96294fc 100644 --- a/internal/handler/npm.go +++ b/internal/handler/npm.go @@ -29,6 +29,10 @@ const ( // npmKeysPath serves the registry signing keys `npm audit signatures` reads. npmKeysPath = "/-/npm/v1/keys" + // npmAttestationsPrefix serves a version's provenance and publish + // attestations, as /-/npm/v1/attestations/{name}@{version}. + npmAttestationsPrefix = "/-/npm/v1/attestations/" + // npmSearchPath is what `npm search` queries, with the terms in the query // string. npmPingPath is what `npm ping` checks. npmSearchPath = "/-/v1/search" @@ -97,6 +101,11 @@ func (h *NPMHandler) Routes() http.Handler { return } + if strings.HasPrefix(r.URL.Path, npmAttestationsPrefix) { + h.handleAttestations(w, r) + return + } + switch r.URL.Path { case npmKeysPath: h.proxy.ProxyUpstream(w, r, h.upstreamURL+npmKeysPath, @@ -634,26 +643,33 @@ func npmVersionTarball(body []byte, version string) string { } func validateTarballURL(raw, registryURL string) (string, error) { - tarball, err := url.Parse(raw) + return validateRegistryURL("tarball", raw, registryURL) +} + +// validateRegistryURL checks that a URL a packument lists for a version's +// file is on the package's registry, under its base path, and returns it. +// what names the file in errors. +func validateRegistryURL(what, raw, registryURL string) (string, error) { + parsed, err := url.Parse(raw) if err != nil { - return "", fmt.Errorf("parsing npm tarball URL: %w", err) + return "", fmt.Errorf("parsing npm %s URL: %w", what, err) } upstream, err := url.Parse(registryURL) if err != nil { return "", fmt.Errorf("parsing npm upstream URL: %w", err) } - if tarball.User != nil || tarball.Fragment != "" || - tarball.Scheme != upstream.Scheme || !strings.EqualFold(tarball.Host, upstream.Host) { - return "", errors.New("npm tarball URL does not match upstream registry") + if parsed.User != nil || parsed.Fragment != "" || + parsed.Scheme != upstream.Scheme || !strings.EqualFold(parsed.Host, upstream.Host) { + return "", fmt.Errorf("npm %s URL does not match upstream registry", what) } - if containsPathTraversal(tarball.Path) || strings.Contains(tarball.Path, "\\") { - return "", errors.New("npm tarball URL contains path traversal") + if containsPathTraversal(parsed.Path) || strings.Contains(parsed.Path, "\\") { + return "", fmt.Errorf("npm %s URL contains path traversal", what) } basePath := strings.TrimRight(upstream.Path, "/") - if basePath != "" && !strings.HasPrefix(tarball.Path, basePath+"/") { - return "", errors.New("npm tarball URL is outside upstream base path") + if basePath != "" && !strings.HasPrefix(parsed.Path, basePath+"/") { + return "", fmt.Errorf("npm %s URL is outside upstream base path", what) } - return tarball.String(), nil + return parsed.String(), nil } // versionInCooldown reports whether a version is still inside the cooldown diff --git a/internal/handler/npm_attestations.go b/internal/handler/npm_attestations.go new file mode 100644 index 0000000..613a488 --- /dev/null +++ b/internal/handler/npm_attestations.go @@ -0,0 +1,122 @@ +package handler + +import ( + "errors" + "net/http" + "net/url" + "strings" + "sync" + + "github.com/git-pkgs/proxy/internal/jsonscan" +) + +// errNoAttestations answers a request for a version whose packument entry +// lists no attestations. +var errNoAttestations = errors.New("attestations not found") + +// handleAttestations serves /-/npm/v1/attestations/{name}@{version}, which +// `npm audit signatures` reads to verify provenance. Packuments point +// dist.attestations.url here (see npm.AttestationsURL). +// +// The document is fetched from the package's registry, the route's for a +// routed package, at the URL its packument lists for the version when that is +// on the same registry, and otherwise at the registry's standard path. A +// version's attestations never change once published, so they are cached +// like other immutable metadata and served without revalidation. +// +// Versions the denylist blocks get 403 and versions cooldown holds back 404, +// as their tarballs do. Pushed versions carry no upstream attestations, so +// they and packages in hosted-only scopes get 404. +func (h *NPMHandler) handleAttestations(w http.ResponseWriter, r *http.Request) { + packageName, version, ok := parseNPMAttestationsPath(strings.TrimPrefix(r.URL.Path, npmAttestationsPrefix)) + if !ok { + JSONError(w, http.StatusBadRequest, "invalid package name") + return + } + + if h.proxy.versionDenied("npm", packageName, version) { + JSONError(w, http.StatusForbidden, ErrVersionDenied.Error()+": "+canonicalVersionPURL("npm", packageName, version)) + return + } + local, err := h.npmLocalEntries(packageName) + if err != nil { + h.proxy.Logger.Error("failed to read pushed npm versions", "package", packageName, "error", err) + JSONError(w, http.StatusInternalServerError, "failed to read pushed versions") + return + } + if _, pushed := findNPMLocalEntry(local, version); pushed || h.proxy.hostedOnly("npm", packageName) { + JSONError(w, http.StatusNotFound, errNoAttestations.Error()) + return + } + + registryURL, cacheKey := h.sourceFor(packageName) + metadata := sync.OnceValues(func() ([]byte, error) { + upstreamURL := registryURL + "/" + url.PathEscape(packageName) + body, _, _, err := h.proxy.fetchOrCacheMetadata(r.Context(), "npm", cacheKey, upstreamURL, "", h.packumentValidator(), h.packumentAccept()) + return body, err + }) + + inCooldown, err := h.versionInCooldown(packageName, version, metadata) + if err != nil { + if writeUpstreamHeld(w, err) { + return + } + h.proxy.Logger.Warn("cooldown: could not read npm publish time", + "package", packageName, "version", version, "error", err) + JSONError(w, http.StatusBadGateway, "failed to check cooldown") + return + } + if inCooldown { + JSONError(w, http.StatusNotFound, ErrVersionNotFound.Error()) + return + } + + packument, err := metadata() + if err != nil { + if errors.Is(err, ErrUpstreamNotFound) { + JSONError(w, http.StatusNotFound, ErrPackageNotFound.Error()) + return + } + if writeUpstreamHeld(w, err) { + return + } + h.proxy.Logger.Error("failed to fetch npm metadata", "error", err) + JSONError(w, http.StatusBadGateway, "failed to fetch from upstream") + return + } + listed, _, _ := jsonscan.LookupString(packument, "versions", version, "dist", "attestations", "url") + if listed == "" { + JSONError(w, http.StatusNotFound, errNoAttestations.Error()) + return + } + upstreamURL, err := validateRegistryURL("attestations", listed, registryURL) + if err != nil { + // Mirrors of npmjs can keep its URLs in packuments. The proxy only + // ever asks the package's own registry. + upstreamURL = registryURL + "/-/npm/v1/attestations/" + url.PathEscape(packageName) + "@" + url.PathEscape(version) + } + + h.proxy.proxyImmutable(w, r, upstreamURL, "npm", cacheKey+"/-/attestations/"+version, contentTypeJSON) +} + +// parseNPMAttestationsPath splits "{name}@{version}" as the attestations +// endpoint takes it, where name may be "@scope/name" with the slash encoded +// or not. It rejects anything that is not one package and one version. +func parseNPMAttestationsPath(spec string) (packageName, version string, ok bool) { + at := strings.LastIndex(spec, "@") + if at <= 0 { + return "", "", false + } + packageName, version = spec[:at], spec[at+1:] + if version == "" || strings.ContainsAny(version, "/\\") || containsPathTraversal(packageName) || containsPathTraversal(version) { + return "", "", false + } + scope, name, scoped := strings.Cut(packageName, "/") + switch { + case scoped: + ok = len(scope) > 1 && scope[0] == '@' && name != "" && !strings.ContainsAny(name, "/@") + default: + ok = !strings.ContainsAny(packageName, "@\\") + } + return packageName, version, ok +} diff --git a/internal/handler/npm_attestations_test.go b/internal/handler/npm_attestations_test.go new file mode 100644 index 0000000..3717eaf --- /dev/null +++ b/internal/handler/npm_attestations_test.go @@ -0,0 +1,191 @@ +package handler + +import ( + "net/http" + "slices" + "strings" + "testing" + "time" + + "github.com/git-pkgs/cooldown" +) + +const testAttestations = `{"attestations":[{"predicateType":"https://slsa.dev/provenance/v1","bundle":{}}]}` + +// attestedPackument returns a packument whose version lists attestations at +// attestationsURL, resolved against the upstream's URL when it starts with +// "/", and was published age ago. +func attestedPackument(name, version, attestationsURL string, age time.Duration) func(string) string { + published := time.Now().Add(-age).UTC().Format(time.RFC3339) + return func(baseURL string) string { + if strings.HasPrefix(attestationsURL, "/") { + attestationsURL = baseURL + attestationsURL + } + dist := `{"tarball":"` + baseURL + "/" + name + `/-/x-` + version + `.tgz"` + if attestationsURL != "" { + dist += `,"attestations":{"url":"` + attestationsURL + `","provenance":{"predicateType":"https://slsa.dev/provenance/v1"}}` + } + dist += "}" + return `{"name":"` + name + `","dist-tags":{"latest":"` + version + `"},` + + `"versions":{"` + version + `":{"version":"` + version + `","dist":` + dist + `}},` + + `"time":{"` + version + `":"` + published + `"}}` + } +} + +func attestationsFile(string) string { return testAttestations } + +func newAttestationsProxy(t *testing.T) *Proxy { + t.Helper() + proxy, _, _, _ := setupTestProxy(t) + proxy.HTTPClient = http.DefaultClient + proxy.CacheMetadata = true + proxy.MetadataTTL = time.Hour + return proxy +} + +func TestNPMAttestationsServedAndCached(t *testing.T) { + upstream := newNPMUpstream(t, map[string]func(string) string{ + "/sigstore": attestedPackument("sigstore", "3.0.0", "/-/npm/v1/attestations/sigstore@3.0.0", 30*24*time.Hour), + "/-/npm/v1/attestations/sigstore@3.0.0": attestationsFile, + }, http.StatusOK) + h := NewNPMHandler(newAttestationsProxy(t), "http://proxy.test", upstream.URL) + + for range 2 { + w := serveNPM(t, h, http.MethodGet, "/-/npm/v1/attestations/sigstore@3.0.0", nil, nil) + if w.Code != http.StatusOK || w.Body.String() != testAttestations { + t.Fatalf("status = %d, body = %s", w.Code, w.Body.String()) + } + } + fetched := 0 + for _, p := range upstream.requested() { + if p == "/-/npm/v1/attestations/sigstore@3.0.0" { + fetched++ + } + } + if fetched != 1 { + t.Errorf("attestations fetched %d times, want once: %v", fetched, upstream.requested()) + } +} + +func TestNPMAttestationsRoutedPackageStaysOnRoute(t *testing.T) { + public := newNPMUpstream(t, map[string]func(string) string{}, http.StatusOK) + private := newNPMUpstream(t, map[string]func(string) string{ + routedNPMPath: attestedPackument("@example/widgets", "1.0.0", "/registry/-/npm/v1/attestations/@example%2fwidgets@1.0.0", 0), + "/registry/-/npm/v1/attestations/@example/widgets@1.0.0": attestationsFile, + }, http.StatusOK) + h := newRoutedNPMHandler(newAttestationsProxy(t), public, private) + + for _, target := range []string{ + "/-/npm/v1/attestations/@example%2fwidgets@1.0.0", + "/-/npm/v1/attestations/@example/widgets@1.0.0", + } { + w := serveNPM(t, h, http.MethodGet, target, nil, nil) + if w.Code != http.StatusOK || w.Body.String() != testAttestations { + t.Errorf("%s: status = %d, body = %s", target, w.Code, w.Body.String()) + } + } + if got := public.requested(); len(got) != 0 { + t.Errorf("default registry was queried for a routed package: %v", got) + } +} + +// TestNPMAttestationsOnlyAskThePackageRegistry checks that an attestations +// URL on another host is not followed: the proxy asks the package's registry +// at its standard path instead. +func TestNPMAttestationsOnlyAskThePackageRegistry(t *testing.T) { + other := newNPMUpstream(t, map[string]func(string) string{"/-/npm/v1/attestations/demo@1.0.0": attestationsFile}, http.StatusOK) + upstream := newNPMUpstream(t, map[string]func(string) string{ + "/demo": attestedPackument("demo", "1.0.0", other.URL+"/-/npm/v1/attestations/demo@1.0.0", 0), + "/-/npm/v1/attestations/demo@1.0.0": attestationsFile, + }, http.StatusOK) + h := NewNPMHandler(newAttestationsProxy(t), "http://proxy.test", upstream.URL) + + w := serveNPM(t, h, http.MethodGet, "/-/npm/v1/attestations/demo@1.0.0", nil, nil) + if w.Code != http.StatusOK { + t.Fatalf("status = %d, body = %s", w.Code, w.Body.String()) + } + if got := other.requested(); len(got) != 0 { + t.Errorf("host named in the packument was queried: %v", got) + } + if !slices.Contains(upstream.requested(), "/-/npm/v1/attestations/demo@1.0.0") { + t.Errorf("registry not asked for attestations: %v", upstream.requested()) + } +} + +func TestNPMAttestationsWithheld(t *testing.T) { + tests := []struct { + name string + packument func(string) string + setup func(t *testing.T, p *Proxy) + wantStatus int + }{ + {"version has none", attestedPackument("demo", "1.0.0", "", 0), nil, http.StatusNotFound}, + {"version in cooldown", attestedPackument("demo", "1.0.0", "/-/npm/v1/attestations/demo@1.0.0", time.Hour), + func(_ *testing.T, p *Proxy) { p.Cooldown = &cooldown.Config{Default: "7d"} }, http.StatusNotFound}, + {"version denied", attestedPackument("demo", "1.0.0", "/-/npm/v1/attestations/demo@1.0.0", 0), + func(t *testing.T, p *Proxy) { setTestDenylist(t, p, "pkg:npm/demo@1.0.0") }, http.StatusForbidden}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + upstream := newNPMUpstream(t, map[string]func(string) string{ + "/demo": tt.packument, + "/-/npm/v1/attestations/demo@1.0.0": attestationsFile, + }, http.StatusOK) + proxy := newAttestationsProxy(t) + if tt.setup != nil { + tt.setup(t, proxy) + } + h := NewNPMHandler(proxy, "http://proxy.test", upstream.URL) + + w := serveNPM(t, h, http.MethodGet, "/-/npm/v1/attestations/demo@1.0.0", nil, nil) + if w.Code != tt.wantStatus { + t.Errorf("status = %d, want %d; body: %s", w.Code, tt.wantStatus, w.Body.String()) + } + if slices.Contains(upstream.requested(), "/-/npm/v1/attestations/demo@1.0.0") { + t.Error("attestations were fetched for a withheld version") + } + }) + } +} + +func TestNPMAttestationsRejectsInvalidPaths(t *testing.T) { + upstream := newNPMUpstream(t, map[string]func(string) string{}, http.StatusOK) + h := NewNPMHandler(newAttestationsProxy(t), "http://proxy.test", upstream.URL) + + for _, target := range []string{ + "/-/npm/v1/attestations/demo", + "/-/npm/v1/attestations/demo@", + "/-/npm/v1/attestations/@demo@1.0.0", + "/-/npm/v1/attestations/@scope/a/b@1.0.0", + "/-/npm/v1/attestations/demo@1.0.0/x", + "/-/npm/v1/attestations/..%2f..%2fetc@1.0.0", + } { + w := serveNPM(t, h, http.MethodGet, target, nil, nil) + if w.Code != http.StatusBadRequest { + t.Errorf("%s: status = %d, want 400", target, w.Code) + } + } + if got := upstream.requested(); len(got) != 0 { + t.Errorf("upstream queried for invalid paths: %v", got) + } +} + +func TestNPMMetadataRewritesAttestationsURL(t *testing.T) { + const attestations = "/registry/-/npm/v1/attestations/@example%2fwidgets@1.0.0" + upstream := newNPMUpstream(t, map[string]func(string) string{ + routedNPMPath: attestedPackument("@example/widgets", "1.0.0", attestations, 0), + routedNPMPath + "/1.0.0": func(baseURL string) string { + return `{"name":"@example/widgets","version":"1.0.0","dist":{"attestations":{"url":"` + baseURL + attestations + + `"},"tarball":"` + baseURL + `/registry/@example/widgets/-/widgets-1.0.0.tgz"}}` + }, + }, http.StatusOK) + h := newRoutedNPMHandler(newAttestationsProxy(t), upstream, upstream) + + for _, target := range []string{"/@example%2fwidgets", "/@example%2fwidgets/1.0.0"} { + w := serveNPM(t, h, http.MethodGet, target, nil, nil) + want := `"url":"http://proxy.test/npm/-/npm/v1/attestations/@example%2Fwidgets@1.0.0"` + if w.Code != http.StatusOK || !strings.Contains(w.Body.String(), want) { + t.Errorf("%s: status %d, attestations URL not rewritten: %s", target, w.Code, w.Body.String()) + } + } +} diff --git a/internal/metadata/npm/metadata.go b/internal/metadata/npm/metadata.go index 62e531f..6b66e6a 100644 --- a/internal/metadata/npm/metadata.go +++ b/internal/metadata/npm/metadata.go @@ -52,7 +52,8 @@ type Options struct { type transformer struct{ Options } -// Rewrite filters upstream versions and rewrites their tarball URLs, preserving +// Rewrite filters upstream versions and points their tarball and attestation +// URLs at the proxy, preserving // untouched JSON values verbatim. Hosted entries override upstream versions. // The returned version set is unordered. The document, denylist and hosted // entries are not modified. @@ -316,9 +317,10 @@ func (h *transformer) writeNPMVersions(out *bytes.Buffer, packageName string, ve return err } -// writeNPMVersion writes one version entry with its dist.tarball pointing at -// this proxy, and reports whether it rewrote the tarball. An entry without a -// string tarball is written unchanged. +// writeNPMVersion writes one version entry with its dist.tarball and +// dist.attestations.url pointing at this proxy, and reports whether it +// rewrote the tarball. Members that are missing or not strings are written +// unchanged. func (h *transformer) writeNPMVersion(out *bytes.Buffer, packageName, version string, raw []byte) bool { dist, ok, err := jsonscan.FindMember(raw, "dist") if err != nil || !ok { @@ -326,27 +328,68 @@ func (h *transformer) writeNPMVersion(out *bytes.Buffer, packageName, version st return false } distRaw := dist.Value(raw) - tarball, ok, err := jsonscan.FindMember(distRaw, "tarball") - if err != nil || !ok || distRaw[tarball.ValStart] != '"' { - out.Write(raw) - return false - } - var oldTarball string - if err := json.Unmarshal(tarball.Value(distRaw), &oldTarball); err != nil { - out.Write(raw) - return false - } - newTarball := TarballURL(h.ProxyURL, packageName, version, oldTarball) - encoded, err := json.Marshal(newTarball) - if err != nil { - out.Write(raw) - return false + var buf [2]urlEdit + edits := buf[:0] + tarball, ok := stringMember(distRaw, "tarball") + if ok { + tarball.replacement = TarballURL(h.ProxyURL, packageName, version, tarball.old) + edits = append(edits, tarball.shift(dist.ValStart)) } - out.Write(raw[:dist.ValStart+tarball.ValStart]) - out.Write(encoded) - out.Write(raw[dist.ValStart+tarball.ValEnd:]) - return true + if attestations, found, err := jsonscan.FindMember(distRaw, "attestations"); err == nil && found { + if att, ok := stringMember(attestations.Value(distRaw), "url"); ok { + att.replacement = AttestationsURL(h.ProxyURL, packageName, version) + edits = append(edits, att.shift(dist.ValStart+attestations.ValStart)) + } + } + writeEdits(out, raw, edits) + return tarball.replacement != "" +} + +// urlEdit replaces the string value at raw[start:end], which decodes to old, +// with replacement. +type urlEdit struct { + start, end int + old string + replacement string +} + +func (e urlEdit) shift(offset int) urlEdit { + e.start += offset + e.end += offset + return e +} + +// stringMember finds the string member name in obj. +func stringMember(obj []byte, name string) (urlEdit, bool) { + m, ok, err := jsonscan.FindMember(obj, name) + if err != nil || !ok || obj[m.ValStart] != '"' { + return urlEdit{}, false + } + var old string + if err := json.Unmarshal(m.Value(obj), &old); err != nil { + return urlEdit{}, false + } + return urlEdit{start: m.ValStart, end: m.ValEnd, old: old}, true +} + +// writeEdits writes raw with the edits applied. There are at most two, which +// may come in either order. +func writeEdits(out *bytes.Buffer, raw []byte, edits []urlEdit) { + if len(edits) == 2 && edits[1].start < edits[0].start { + edits[0], edits[1] = edits[1], edits[0] + } + pos := 0 + for _, e := range edits { + encoded, err := json.Marshal(e.replacement) + if err != nil { + continue + } + out.Write(raw[pos:e.start]) + out.Write(encoded) + pos = e.end + } + out.Write(raw[pos:]) } // tarballsRewritten reports once per document how many tarball URLs it @@ -434,6 +477,13 @@ func TarballURL(proxyURL, packageName, version, tarball string) string { return fmt.Sprintf("%s/npm/%s/-/%s", proxyURL, url.PathEscape(packageName), filename) } +// AttestationsURL returns the proxy URL that serves the attestations of a +// version, in the form the npm registry uses: the package name with its scope +// slash encoded, "@", and the version. +func AttestationsURL(proxyURL, packageName, version string) string { + return fmt.Sprintf("%s/npm/-/npm/v1/attestations/%s@%s", proxyURL, url.PathEscape(packageName), url.PathEscape(version)) +} + // findNewestVersion returns the version string with the most recent timestamp // from the remaining versions, using the time map. func (h *transformer) findNewestVersion(versions map[string]any, timeMap map[string]any) string { diff --git a/internal/metadata/npm/metadata_test.go b/internal/metadata/npm/metadata_test.go index 5d68dd6..28e9669 100644 --- a/internal/metadata/npm/metadata_test.go +++ b/internal/metadata/npm/metadata_test.go @@ -165,3 +165,32 @@ func TestAbbreviated(t *testing.T) { } } } + +// TestNPMRewriteAttestationsURL checks that dist.attestations.url points at +// the proxy whichever order dist lists its members in, and that nothing else +// in dist changes. +func TestNPMRewriteAttestationsURL(t *testing.T) { + options := Options{ProxyURL: "http://proxy"} + tests := []struct{ name, version, dist, want string }{ + {"demo", "1.0.0", + `{"tarball":"https://registry.npmjs.org/demo/-/demo-1.0.0.tgz","attestations":{"url":"https://registry.npmjs.org/-/npm/v1/attestations/demo@1.0.0","provenance":{"predicateType":"p"}}}`, + `{"tarball":"http://proxy/npm/demo/-/demo-1.0.0.tgz","attestations":{"url":"http://proxy/npm/-/npm/v1/attestations/demo@1.0.0","provenance":{"predicateType":"p"}}}`}, + {"@scope/demo", "2.0.0-rc.1", + `{"attestations":{"provenance":{},"url":"https://registry.npmjs.org/-/npm/v1/attestations/@scope%2fdemo@2.0.0-rc.1"},"tarball":"https://registry.npmjs.org/@scope/demo/-/demo-2.0.0-rc.1.tgz"}`, + `{"attestations":{"provenance":{},"url":"http://proxy/npm/-/npm/v1/attestations/@scope%2Fdemo@2.0.0-rc.1"},"tarball":"http://proxy/npm/@scope%2Fdemo/-/demo-2.0.0-rc.1.tgz"}`}, + {"demo", "3.0.0", + `{"attestations":{"url":7}}`, + `{"attestations":{"url":7}}`}, + } + for _, tt := range tests { + input := `{"versions":{"` + tt.version + `":{"dist":` + tt.dist + `}}}` + out, _, err := Rewrite(tt.name, []byte(input), options) + if err != nil { + t.Fatal(err) + } + want := `{"versions":{"` + tt.version + `":{"dist":` + tt.want + `}}}` + if string(out) != want { + t.Errorf("%s@%s: rewrite =\n%s\nwant\n%s", tt.name, tt.version, out, want) + } + } +} From e0e212942f69cae98a5f35a8da027dba9fb54310 Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Fri, 30 Oct 2026 08:29:33 +0000 Subject: [PATCH 493/633] Trim npm attestations doc comments --- internal/handler/npm_attestations.go | 10 ---------- internal/metadata/npm/metadata.go | 7 +++---- 2 files changed, 3 insertions(+), 14 deletions(-) diff --git a/internal/handler/npm_attestations.go b/internal/handler/npm_attestations.go index 613a488..2694af2 100644 --- a/internal/handler/npm_attestations.go +++ b/internal/handler/npm_attestations.go @@ -17,16 +17,6 @@ var errNoAttestations = errors.New("attestations not found") // handleAttestations serves /-/npm/v1/attestations/{name}@{version}, which // `npm audit signatures` reads to verify provenance. Packuments point // dist.attestations.url here (see npm.AttestationsURL). -// -// The document is fetched from the package's registry, the route's for a -// routed package, at the URL its packument lists for the version when that is -// on the same registry, and otherwise at the registry's standard path. A -// version's attestations never change once published, so they are cached -// like other immutable metadata and served without revalidation. -// -// Versions the denylist blocks get 403 and versions cooldown holds back 404, -// as their tarballs do. Pushed versions carry no upstream attestations, so -// they and packages in hosted-only scopes get 404. func (h *NPMHandler) handleAttestations(w http.ResponseWriter, r *http.Request) { packageName, version, ok := parseNPMAttestationsPath(strings.TrimPrefix(r.URL.Path, npmAttestationsPrefix)) if !ok { diff --git a/internal/metadata/npm/metadata.go b/internal/metadata/npm/metadata.go index 6b66e6a..3a434c5 100644 --- a/internal/metadata/npm/metadata.go +++ b/internal/metadata/npm/metadata.go @@ -53,10 +53,9 @@ type Options struct { type transformer struct{ Options } // Rewrite filters upstream versions and points their tarball and attestation -// URLs at the proxy, preserving -// untouched JSON values verbatim. Hosted entries override upstream versions. -// The returned version set is unordered. The document, denylist and hosted -// entries are not modified. +// URLs at the proxy, preserving untouched JSON values verbatim. Hosted entries +// override upstream versions. The returned version set is unordered. The +// document, denylist and hosted entries are not modified. func Rewrite(packageName string, body []byte, options Options) ([]byte, []string, error) { return (&transformer{options}).rewriteMetadataWithLocal(packageName, body, options.LocalVersions) } From 39d178f62938a5dba2bf4b5a05c9de14d7bf60e1 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Fri, 30 Oct 2026 09:33:56 +0100 Subject: [PATCH 494/633] Expire cooldown rewrites when a withheld version becomes available A cached rewrite made with cooldown on stays right until a version it held back leaves cooldown, so it now expires then instead of after a fixed time. Handlers ask cooldown through Proxy.cooldown, which reports each withheld version to the rewrites running at that moment. A rewrite that held nothing back is kept for the shortest cooldown, at most a day. --- docs/configuration.md | 4 +- internal/handler/cargo.go | 2 +- internal/handler/composer.go | 4 +- internal/handler/conda.go | 4 +- internal/handler/cooldown_expiry.go | 164 +++++++++++++++++ internal/handler/cooldown_expiry_test.go | 178 +++++++++++++++++++ internal/handler/cooldown_times.go | 2 +- internal/handler/go_cooldown.go | 2 +- internal/handler/handler.go | 4 + internal/handler/helm.go | 2 +- internal/handler/maven_cooldown.go | 6 +- internal/handler/metadata_access_test.go | 2 +- internal/handler/metadata_gzip_test.go | 2 +- internal/handler/metadata_validators_test.go | 2 +- internal/handler/npm.go | 6 +- internal/handler/npm_rewrite_test.go | 4 +- internal/handler/nuget_cooldown.go | 2 +- internal/handler/pub.go | 2 +- internal/handler/pypi.go | 8 +- internal/handler/rewrite_cache.go | 62 ++----- internal/handler/rewrite_cache_test.go | 43 +++-- 21 files changed, 413 insertions(+), 92 deletions(-) create mode 100644 internal/handler/cooldown_expiry.go create mode 100644 internal/handler/cooldown_expiry_test.go diff --git a/docs/configuration.md b/docs/configuration.md index 941bc0f..7f62207 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1141,7 +1141,7 @@ Maven caveats: RubyGems, Hex and PyPI keep their publish times outside the metadata clients read: the RubyGems versions API, the Hex HTTP API (`upstream.hex_api`) and the PyPI JSON API. The proxy fetches them alongside each compact index, package or simple page it filters, and for PyPI downloads. Cooldown fails closed: when it applies to a package and those times can't be fetched or parsed, or the metadata can't be rewritten, the request fails with 502, or with 503 and `Retry-After` while the upstream is rate limiting the proxy, instead of serving the metadata unfiltered. The same goes for npm, Composer and pub.dev metadata that can't be rewritten, and for npm tarball downloads whose publish time can't be read. A PyPI file or npm tarball that is already cached is still served when its publish time can't be read; only a download that would have to be fetched fails. A PyPI release's time is the upload time of its newest file. The proxy stores it when a download looks it up, and serves files of that release that are already cached against the stored time without reading the JSON API again. A file that isn't cached yet is always looked up, since it may have been added to the release since, and the newer time is stored. Packages exempted with a duration of `0` are served as before and don't need the times. A PyPI package the JSON API returns 404 for has no times to go by and is served unfiltered. -For RubyGems and Hex, both documents go through the metadata cache. With [`cache_metadata`](#metadata-caching) they are served from it within `metadata_ttl` without asking the upstream, revalidated with their ETags after it, and filtered from the last cached copies, marked stale, while the upstream is down; without it they are still fetched once for concurrent requests. Each document has its own TTL, so the compact index or Hex package can be newer than the cached times and list a version they don't have. The times are then fetched again before filtering, and if that fails, such versions are left out until the times can be read. The filtered index or package carries an ETag of its own, so Bundler and Mix revalidate it with `304 Not Modified`. Publish times are compared with the current time on each request, but the filtered result is kept for up to an hour (see [rewritten metadata cache](#rewritten-metadata-cache)), so a version leaving cooldown can show up that much later; a newly published version shows up within `metadata_ttl`, as with other cached metadata. Upstream errors are answered as for other cached metadata (404, or 502) rather than relayed. +For RubyGems and Hex, both documents go through the metadata cache. With [`cache_metadata`](#metadata-caching) they are served from it within `metadata_ttl` without asking the upstream, revalidated with their ETags after it, and filtered from the last cached copies, marked stale, while the upstream is down; without it they are still fetched once for concurrent requests. Each document has its own TTL, so the compact index or Hex package can be newer than the cached times and list a version they don't have. The times are then fetched again before filtering, and if that fails, such versions are left out until the times can be read. The filtered index or package carries an ETag of its own, so Bundler and Mix revalidate it with `304 Not Modified`. Publish times are compared with the current time on each request, and the filtered result is kept until the first version it held back leaves cooldown (see [rewritten metadata cache](#rewritten-metadata-cache)); a newly published version shows up within `metadata_ttl`, as with other cached metadata. Upstream errors are answered as for other cached metadata (404, or 502) rather than relayed. With cooldown on, the proxy asks npm for full packuments, since the abbreviated ones it uses otherwise carry no publish times. Abbreviated packuments cached before cooldown was turned on are fetched again in full rather than served, including as a stale copy when the upstream is down. When `latest` is in cooldown, the packument tags the newest version out of cooldown instead, and `npm dist-tag ls` shows the same tags. @@ -1420,7 +1420,7 @@ Or via environment variable: `PROXY_ARTIFACT_MAX_SIZE=2GB`. ### Rewritten metadata cache -The npm, Composer, PyPI and Helm handlers rewrite every metadata document they serve, so that download URLs point at the proxy. That means decoding the whole document and encoding it again, and Composer's minified documents are also expanded, which together cost milliseconds per typical package and far more for very large ones; a large Helm `index.yaml` takes over a hundred milliseconds. The proxy keeps rewritten documents in memory, so each distinct upstream document is rewritten once and requests arriving while it is being rewritten wait for that rewrite. New bytes from upstream are rewritten again. A PyPI simple page is cached per representation (HTML or JSON) and per set of versions filtered out of it. With version cooldown enabled, a cached rewrite expires, because cooldown filtering depends on the current time: it is kept for a 24th of the shortest cooldown configured, at least a minute and at most an hour, so a version shows up at most that long after its cooldown ends. With a cooldown of a day or more that is an hour; with `"2h"` somewhere, five minutes. Cached documents carry an `ETag` of their own, and are dated like any other, so clients that revalidate with `If-None-Match` or `If-Modified-Since`, such as Composer, get `304 Not Modified`. The RubyGems compact index and Hex packages filtered by cooldown are kept the same way; they are never served gzip-compressed. +The npm, Composer, PyPI and Helm handlers rewrite every metadata document they serve, so that download URLs point at the proxy. That means decoding the whole document and encoding it again, and Composer's minified documents are also expanded, which together cost milliseconds per typical package and far more for very large ones; a large Helm `index.yaml` takes over a hundred milliseconds. The proxy keeps rewritten documents in memory, so each distinct upstream document is rewritten once and requests arriving while it is being rewritten wait for that rewrite. New bytes from upstream are rewritten again. A PyPI simple page is cached per representation (HTML or JSON) and per set of versions filtered out of it. With version cooldown enabled, a cached rewrite expires when the first version it held back leaves cooldown, since versions it let through stay allowed and a new upstream version changes the document. A rewrite that held nothing back is kept for the shortest cooldown configured, at least a minute and at most a day. Cached documents carry an `ETag` of their own, and are dated like any other, so clients that revalidate with `If-None-Match` or `If-Modified-Since`, such as Composer, get `304 Not Modified`. The RubyGems compact index and Hex packages filtered by cooldown are kept the same way; they are never served gzip-compressed. Each cached document is kept gzip-compressed, when that makes it smaller, and served with `Content-Encoding: gzip` to clients that accept it, as npm, pnpm, Yarn and Composer do. A large packument is compressed once rather than on every request, by the proxy or by a reverse proxy in front of it. Only the compressed copy is kept, typically an eighth of the document, and it is decompressed for the clients that don't accept gzip. Documents rewritten outside the cache are served uncompressed. diff --git a/internal/handler/cargo.go b/internal/handler/cargo.go index 578c9ec..0c26551 100644 --- a/internal/handler/cargo.go +++ b/internal/handler/cargo.go @@ -171,7 +171,7 @@ func (h *CargoHandler) applyCooldownFiltering(downstreamResponse http.ResponseWr cratePURL := canonicalPackagePURL("cargo", crate.Name) - if !h.proxy.Cooldown.IsAllowed("cargo", cratePURL, publishedAt) { + if !h.proxy.cooldown().IsAllowed("cargo", cratePURL, publishedAt) { h.proxy.Logger.Info("cooldown: filtering cargo version", "crate", crate.Name, "version", crate.Version, "published", crate.PublishTime) diff --git a/internal/handler/composer.go b/internal/handler/composer.go index 116d106..ccb42bb 100644 --- a/internal/handler/composer.go +++ b/internal/handler/composer.go @@ -394,7 +394,7 @@ func (h *ComposerHandler) shouldFilterVersion(packagePURL, packageName, version return false } - if !h.proxy.Cooldown.IsAllowed("composer", packagePURL, publishedAt) { + if !h.proxy.cooldown().IsAllowed("composer", packagePURL, publishedAt) { h.proxy.Logger.Info("cooldown: filtering composer version", "package", packageName, "version", version) return true @@ -731,7 +731,7 @@ func (h *ComposerHandler) serveCachedRelease(w http.ResponseWriter, r *http.Requ if err != nil || ver == nil || !ver.PublishedAt.Valid { return false } - if !h.proxy.Cooldown.IsAllowed("composer", canonicalPackagePURL("composer", packageName), ver.PublishedAt.Time) { + if !h.proxy.cooldown().IsAllowed("composer", canonicalPackagePURL("composer", packageName), ver.PublishedAt.Time) { h.proxy.Logger.Info("cooldown: withholding composer archive", "package", packageName, "version", version) http.Error(w, "version not found", http.StatusNotFound) diff --git a/internal/handler/conda.go b/internal/handler/conda.go index 76a43f2..3cbaf93 100644 --- a/internal/handler/conda.go +++ b/internal/handler/conda.go @@ -493,7 +493,7 @@ func (f *repodataFilter) inCooldown(filename string, raw []byte) bool { return false } publishedAt := time.Unix(int64(entry.Timestamp)/condaTimestampDivisor, 0) - decision := f.h.proxy.Cooldown.Evaluate("conda", canonicalPackagePURL("conda", entry.Name), publishedAt, f.now) + decision := f.h.proxy.cooldown().Evaluate("conda", canonicalPackagePURL("conda", entry.Name), publishedAt, f.now) if decision.Allowed { return false } @@ -517,7 +517,7 @@ func (h *CondaHandler) recordInCooldown(filename, name, version string, timestam return false } publishedAt := time.Unix(int64(timestamp)/condaTimestampDivisor, 0) - if h.proxy.Cooldown.IsAllowed("conda", canonicalPackagePURL("conda", name), publishedAt) { + if h.proxy.cooldown().IsAllowed("conda", canonicalPackagePURL("conda", name), publishedAt) { return false } h.proxy.Logger.Info("cooldown: filtering conda package", diff --git a/internal/handler/cooldown_expiry.go b/internal/handler/cooldown_expiry.go new file mode 100644 index 0000000..1163bde --- /dev/null +++ b/internal/handler/cooldown_expiry.go @@ -0,0 +1,164 @@ +package handler + +import ( + "sync" + "time" + + "github.com/git-pkgs/cooldown" +) + +// A rewrite made with cooldown on stays right until a version it held back +// leaves cooldown: versions it let through stay allowed as time passes, and +// new versions upstream change the document and so the cache key. So the +// cached rewrite expires when the earliest version it withheld becomes +// available, instead of after a fixed time. +// +// Handlers ask cooldown through Proxy.cooldown, which tells every rewrite +// running at that moment when a withheld version becomes available. A +// rewrite may hear of versions another rewrite withheld at the same time; +// that only makes it expire earlier than it has to, never later. + +// maxCooldownRewriteTTL is the longest a rewrite made with cooldown on is +// served, even when it held nothing back. +const maxCooldownRewriteTTL = 24 * time.Hour + +// minCooldownRewriteTTL is the shortest cap on a rewrite made with cooldown +// on, however short the cooldown. +const minCooldownRewriteTTL = time.Minute + +// cooldownRewriteTTL returns the longest a rewrite made with cooldown on is +// served: the shortest cooldown configured, between a minute and a day. It +// only matters when a rewrite held nothing back, or held back a version +// whose availability is unknown. +func (p *Proxy) cooldownRewriteTTL() time.Duration { + ttl := p.ShortestCooldown + switch { + case ttl <= 0 || ttl > maxCooldownRewriteTTL: + return maxCooldownRewriteTTL + case ttl < minCooldownRewriteTTL: + return minCooldownRewriteTTL + } + return ttl +} + +// cooldownWatch holds the rewrites running with cooldown on. +type cooldownWatch struct { + mu sync.Mutex + active map[*cooldownRecorder]struct{} +} + +// cooldownRecorder collects when the versions withheld during one rewrite +// become available. +type cooldownRecorder struct { + // earliest is when the first withheld version becomes available, or + // zero if none was withheld. + earliest time.Time + // unknown is set when a version was withheld without a time it becomes + // available. + unknown bool +} + +// watchCooldown starts recording the versions cooldown holds back for a +// rewrite about to run. unwatchCooldown ends it. +func (p *Proxy) watchCooldown() *cooldownRecorder { + rec := &cooldownRecorder{} + w := &p.cooldownWatch + w.mu.Lock() + defer w.mu.Unlock() + if w.active == nil { + w.active = make(map[*cooldownRecorder]struct{}) + } + w.active[rec] = struct{}{} + return rec +} + +func (p *Proxy) unwatchCooldown(rec *cooldownRecorder) { + w := &p.cooldownWatch + w.mu.Lock() + defer w.mu.Unlock() + delete(w.active, rec) +} + +// cooldownExpiry returns when a rewrite that started at start and recorded +// into rec should expire: when the first version it held back leaves +// cooldown, and no later than cooldownRewriteTTL after start. +func (p *Proxy) cooldownExpiry(rec *cooldownRecorder, start time.Time) time.Time { + w := &p.cooldownWatch + w.mu.Lock() + earliest, unknown := rec.earliest, rec.unknown + w.mu.Unlock() + + limit := start.Add(p.cooldownRewriteTTL()) + if unknown || earliest.IsZero() || earliest.After(limit) { + return limit + } + return earliest +} + +// withheld tells the running rewrites that a version is held back until +// availableAt, zero when that is unknown. +func (p *Proxy) withheld(availableAt time.Time) { + w := &p.cooldownWatch + w.mu.Lock() + defer w.mu.Unlock() + for rec := range w.active { + switch { + case availableAt.IsZero(): + rec.unknown = true + case rec.earliest.IsZero() || availableAt.Before(rec.earliest): + rec.earliest = availableAt + } + } +} + +// cooldown returns the proxy's cooldown policy, telling the running rewrites +// about each version it holds back. Handlers use it rather than p.Cooldown. +func (p *Proxy) cooldown() CooldownPolicy { //nolint:ireturn // nil when cooldown is not configured, as p.Cooldown is + if p.Cooldown == nil { + return nil + } + return recordingCooldown{p: p} +} + +// recordingCooldown is the proxy's cooldown policy, reporting the versions it +// holds back to the running rewrites. +type recordingCooldown struct { + p *Proxy +} + +func (r recordingCooldown) IsAllowed(ecosystem, packagePURL string, publishedAt time.Time) bool { + if r.p.Cooldown.IsAllowed(ecosystem, packagePURL, publishedAt) { + return true + } + r.p.withheld(r.availableAt(ecosystem, packagePURL, publishedAt)) + return false +} + +func (r recordingCooldown) Evaluate(ecosystem, packagePURL string, publishedAt, evaluatedAt time.Time) cooldown.Decision { + d := r.p.Cooldown.Evaluate(ecosystem, packagePURL, publishedAt, evaluatedAt) + if !d.Allowed { + r.p.withheld(d.AvailableAt) + } + return d +} + +func (r recordingCooldown) For(ecosystem, packagePURL string) time.Duration { + return r.p.Cooldown.For(ecosystem, packagePURL) +} + +func (r recordingCooldown) Enabled() bool { + return r.p.Cooldown.Enabled() +} + +// availableAt is when a version IsAllowed held back leaves cooldown, or zero +// when that is unknown. +func (r recordingCooldown) availableAt(ecosystem, packagePURL string, publishedAt time.Time) time.Time { + if publishedAt.IsZero() { + return time.Time{} + } + d := r.p.Cooldown.For(ecosystem, packagePURL) + if d <= 0 { + return time.Time{} + } + return publishedAt.Add(d) +} diff --git a/internal/handler/cooldown_expiry_test.go b/internal/handler/cooldown_expiry_test.go new file mode 100644 index 0000000..fd55845 --- /dev/null +++ b/internal/handler/cooldown_expiry_test.go @@ -0,0 +1,178 @@ +package handler + +import ( + "context" + "net/http" + "os" + "path/filepath" + "regexp" + "strings" + "testing" + "time" + + "github.com/git-pkgs/cooldown" +) + +// TestCachedRewrite_ExpiresWhenWithheldVersionIsAvailable checks that a +// rewrite which held a version back is cached until that version leaves +// cooldown, and no longer. +func TestCachedRewrite_ExpiresWhenWithheldVersionIsAvailable(t *testing.T) { + proxy, _, _, _ := setupTestProxy(t) + clock := &testClock{t: time.Now()} + proxy.now = clock.now + proxy.SetMetadataRewriteCacheSize(1 << 20) + proxy.Cooldown = &cooldown.Config{Default: "3d"} + purl := canonicalPackagePURL("npm", "left-pad") + // One version is 3d-5h old: it leaves cooldown in five hours. + published := clock.now().Add(-72*time.Hour + 5*time.Hour) + + var calls int + var rewrite keepingRewrite = func(b []byte) ([]byte, []string, error) { + calls++ + if proxy.cooldown().IsAllowed("npm", purl, published) { + return b, []string{"1.0.0", "1.1.0"}, nil + } + return b, []string{"1.0.0"}, nil + } + get := func() { + if _, err := proxy.cachedRewrite(context.Background(), "npm", "http://proxy", "left-pad", []byte("doc"), "", rewrite); err != nil { + t.Fatal(err) + } + } + + get() + clock.advance(5*time.Hour - time.Minute) + get() + if calls != 1 { + t.Errorf("rewrites before the withheld version is available = %d, want 1", calls) + } + clock.advance(time.Minute) + get() + if calls != 2 { + t.Errorf("rewrites once the withheld version is available = %d, want 2", calls) + } +} + +// TestCachedRewrite_NothingWithheldCachedForCap checks that a rewrite which +// held nothing back is cached for cooldownRewriteTTL. +func TestCachedRewrite_NothingWithheldCachedForCap(t *testing.T) { + proxy, _, _, _ := setupTestProxy(t) + clock := &testClock{t: time.Date(2026, 10, 10, 12, 0, 0, 0, time.UTC)} + proxy.now = clock.now + proxy.SetMetadataRewriteCacheSize(1 << 20) + proxy.Cooldown = &cooldown.Config{Default: "2h"} + proxy.ShortestCooldown = 2 * time.Hour + var r countingRewrite + + get := func() { + _, _ = proxy.cachedRewrite(context.Background(), "npm", "http://proxy", "left-pad", []byte("doc"), "", r.keeping) + } + get() + clock.advance(2*time.Hour - time.Second) + get() + if got := r.calls.Load(); got != 1 { + t.Errorf("rewrites within the cap = %d, want 1", got) + } + clock.advance(time.Second) + get() + if got := r.calls.Load(); got != 2 { + t.Errorf("rewrites after the cap = %d, want 2", got) + } +} + +// TestCooldownExpiry checks how one rewrite's recorded decisions set its +// expiry, including decisions made for other rewrites running at the same +// time, which may only make it expire earlier. +func TestCooldownExpiry(t *testing.T) { + p := &Proxy{Cooldown: &cooldown.Config{Default: "3d"}, ShortestCooldown: 72 * time.Hour} + // IsAllowed compares with the wall clock, so the times are relative to it. + start := time.Now() + limit := start.Add(24 * time.Hour) + purl := canonicalPackagePURL("npm", "left-pad") + + rec := p.watchCooldown() + if got := p.cooldownExpiry(rec, start); !got.Equal(limit) { + t.Errorf("nothing withheld: expiry = %v, want the cap %v", got, limit) + } + + other := p.watchCooldown() + p.cooldown().IsAllowed("npm", purl, start.Add(-60*time.Hour)) // available in 12h + p.unwatchCooldown(other) + p.cooldown().IsAllowed("npm", purl, start.Add(-66*time.Hour)) // available in 6h, after other ended + if got, want := p.cooldownExpiry(rec, start), start.Add(6*time.Hour); !got.Equal(want) { + t.Errorf("rec expiry = %v, want the earliest availability %v", got, want) + } + if got, want := p.cooldownExpiry(other, start), start.Add(12*time.Hour); !got.Equal(want) { + t.Errorf("other expiry = %v, want %v: decisions after it ended don't count", got, want) + } + + p.cooldown().Evaluate("npm", purl, start.Add(-71*time.Hour), start) // available in 1h + if got, want := p.cooldownExpiry(rec, start), start.Add(time.Hour); !got.Equal(want) { + t.Errorf("after Evaluate: expiry = %v, want %v", got, want) + } + p.unwatchCooldown(rec) + if len(p.cooldownWatch.active) != 0 { + t.Errorf("%d recorders left after unwatch", len(p.cooldownWatch.active)) + } +} + +// TestHandlersAskCooldownThroughRecorder checks that handlers make cooldown +// decisions through Proxy.cooldown. A decision made on Proxy.Cooldown +// directly would not reach the rewrites running, so a cached rewrite could +// keep hiding a version past its cooldown. +func TestHandlersAskCooldownThroughRecorder(t *testing.T) { + direct := regexp.MustCompile(`\.Cooldown\.(IsAllowed|Evaluate)\(|Cooldown:\s+\w+(\.\w+)*\.Cooldown\b`) + files, err := filepath.Glob("*.go") + if err != nil { + t.Fatal(err) + } + for _, f := range files { + if strings.HasSuffix(f, "_test.go") || f == "cooldown_expiry.go" { + continue + } + src, err := os.ReadFile(f) + if err != nil { + t.Fatal(err) + } + for i, line := range strings.Split(string(src), "\n") { + if direct.MatchString(line) { + t.Errorf("%s:%d asks Proxy.Cooldown directly; use Proxy.cooldown(): %s", f, i+1, strings.TrimSpace(line)) + } + } + } +} + +// TestHandlerRewriteExpiresWithWithheldVersion checks, through the handlers, +// that a cached rewrite expires when the version cooldown held back from it +// becomes available. +func TestHandlerRewriteExpiresWithWithheldVersion(t *testing.T) { + for _, c := range validatorCases { + t.Run(c.ecosystem, func(t *testing.T) { + u := cooldownUpstream(t) // the newest version was published 2h ago + proxy, _, clock := validatorProxyWithDB(t, u, time.Hour) + // Cooldown compares with the wall clock, so the proxy's clock + // follows it here. + clock.t = time.Now() + proxy.Cooldown = &cooldown.Config{Default: "3h"} + proxy.ShortestCooldown = 3 * time.Hour + handler := c.handler(proxy, u) + + if rec := serve(handler, c.path, nil); rec.Code != http.StatusOK { + t.Fatalf("status %d", rec.Code) + } + proxy.rewrites.mu.Lock() + defer proxy.rewrites.mu.Unlock() + if len(proxy.rewrites.entries) != 1 { + t.Fatalf("%d cached rewrites, want 1", len(proxy.rewrites.entries)) + } + for _, el := range proxy.rewrites.entries { + expires := el.Value.(*rewriteEntry).expires + // Published 2h ago with a 3h cooldown: available in an hour, + // to the second the publish time was given in. + if want := time.Now().Add(time.Hour); expires.Before(want.Add(-2*time.Second)) || expires.After(want.Add(time.Second)) { + t.Errorf("rewrite expires %v, want about %v when the withheld version is available", expires, want) + } + } + }) + } +} diff --git a/internal/handler/cooldown_times.go b/internal/handler/cooldown_times.go index 88532cd..5d95d16 100644 --- a/internal/handler/cooldown_times.go +++ b/internal/handler/cooldown_times.go @@ -126,7 +126,7 @@ func (d timedDocuments) withheld(p *Proxy, ecosystem, packagePURL, version strin if published.IsZero() { return false } - return !p.Cooldown.IsAllowed(ecosystem, packagePURL, published) + return !p.cooldown().IsAllowed(ecosystem, packagePURL, published) } // decodedMetadataBody returns res's body without its Content-Encoding. The diff --git a/internal/handler/go_cooldown.go b/internal/handler/go_cooldown.go index 90189d4..892e77b 100644 --- a/internal/handler/go_cooldown.go +++ b/internal/handler/go_cooldown.go @@ -42,7 +42,7 @@ func (h *GoHandler) allowedAt(module string, published time.Time) bool { if published.IsZero() { return true } - return h.proxy.Cooldown.IsAllowed("golang", canonicalPackagePURL("golang", module), published) + return h.proxy.cooldown().IsAllowed("golang", canonicalPackagePURL("golang", module), published) } // writeGoMetadataError answers a failed upstream metadata fetch. diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 5693421..1bc5ef7 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -284,6 +284,10 @@ type Proxy struct { // the process started; see metadataDate. now func() time.Time started time.Time + + // cooldownWatch holds the rewrites recording which versions cooldown + // holds back; see cooldown_expiry.go. + cooldownWatch cooldownWatch } // CooldownPolicy supplies version-age filtering and package-specific durations. diff --git a/internal/handler/helm.go b/internal/handler/helm.go index 07e2f74..fdc000e 100644 --- a/internal/handler/helm.go +++ b/internal/handler/helm.go @@ -378,7 +378,7 @@ func (h *HelmHandler) buildChartLookup(upstreamURL string, body []byte) (*helmCh func (h *HelmHandler) chartOnCooldown(chartName string, created time.Time) bool { return !created.IsZero() && h.proxy.Cooldown != nil && h.proxy.Cooldown.Enabled() && - !h.proxy.Cooldown.IsAllowed(helmMetadataEcosystem, canonicalPackagePURL(helmMetadataEcosystem, chartName), created) + !h.proxy.cooldown().IsAllowed(helmMetadataEcosystem, canonicalPackagePURL(helmMetadataEcosystem, chartName), created) } type helmChartDownload struct { diff --git a/internal/handler/maven_cooldown.go b/internal/handler/maven_cooldown.go index 4705f70..4da839e 100644 --- a/internal/handler/maven_cooldown.go +++ b/internal/handler/maven_cooldown.go @@ -222,7 +222,7 @@ func (h *MavenHandler) checkCooldown(w http.ResponseWriter, r *http.Request, sou http.Error(w, "failed to check cooldown", http.StatusBadGateway) return true } - if !h.proxy.Cooldown.IsAllowed("maven", mavenPackagePURL(group, artifact), published) { + if !h.proxy.cooldown().IsAllowed("maven", mavenPackagePURL(group, artifact), published) { h.proxy.Logger.Info("cooldown: withholding maven version", "group", group, "artifact", artifact, "version", version) http.Error(w, errMavenCooldown+": "+canonicalVersionPURL("maven", group+":"+artifact, version), http.StatusForbidden) @@ -307,7 +307,7 @@ func (h *MavenHandler) filterMetadata(ctx context.Context, source mavenSource, a packagePURL := mavenPackagePURL(group, artifact) // No version can be newer than the metadata listing it. if lastUpdated, err := time.Parse(mavenLastUpdatedLayout, doc.lastUpdated); err == nil && - h.proxy.Cooldown.IsAllowed("maven", packagePURL, lastUpdated) { + h.proxy.cooldown().IsAllowed("maven", packagePURL, lastUpdated) { return body, nil } @@ -324,7 +324,7 @@ func (h *MavenHandler) filterMetadata(ctx context.Context, source mavenSource, a if published.IsZero() { continue } - if h.proxy.Cooldown.IsAllowed("maven", packagePURL, published) { + if h.proxy.cooldown().IsAllowed("maven", packagePURL, published) { break } h.proxy.Logger.Info("cooldown: filtering maven version", "group", group, "artifact", artifact, "version", v) diff --git a/internal/handler/metadata_access_test.go b/internal/handler/metadata_access_test.go index 0a64b80..7b5e1cf 100644 --- a/internal/handler/metadata_access_test.go +++ b/internal/handler/metadata_access_test.go @@ -143,7 +143,7 @@ func TestStoredRewrite_MarksAccess(t *testing.T) { body := []byte(`{}`) proxy.cacheMetadataBlob(t.Context(), "npm", "demo", metadataStoragePath("npm", "demo"), &upstreamMetadata{body: body}) key := rewriteCacheKey("npm", "http://proxy", "demo", body) - if _, err := proxy.rewrites.rewrite(t.Context(), key, body, 0, func(b []byte) ([]byte, error) { return b, nil }); err != nil { + if _, err := proxy.rewrites.rewrite(t.Context(), key, body, nil, func(b []byte) ([]byte, error) { return b, nil }); err != nil { t.Fatal(err) } old := ageMetadataAccess(t, db, "npm", "demo") diff --git a/internal/handler/metadata_gzip_test.go b/internal/handler/metadata_gzip_test.go index b5939c7..57d20cc 100644 --- a/internal/handler/metadata_gzip_test.go +++ b/internal/handler/metadata_gzip_test.go @@ -184,7 +184,7 @@ func TestRewriteCache_KeepsGzipCopyOnlyWhenSmaller(t *testing.T) { for _, doc := range [][]byte{large, []byte("abc")} { key := rewriteCacheKey("npm", "http://proxy", string(doc[:3]), doc) - if _, err := c.rewrite(t.Context(), key, doc, 0, identity); err != nil { + if _, err := c.rewrite(t.Context(), key, doc, nil, identity); err != nil { t.Fatal(err) } } diff --git a/internal/handler/metadata_validators_test.go b/internal/handler/metadata_validators_test.go index f7c1475..340e3db 100644 --- a/internal/handler/metadata_validators_test.go +++ b/internal/handler/metadata_validators_test.go @@ -500,7 +500,7 @@ func TestRewriteCacheETag(t *testing.T) { exclaim := func(b []byte) ([]byte, error) { return []byte(string(b) + "!"), nil } key := rewriteCacheKey("npm", "http://proxy", "demo", []byte("doc")) - m, err := c.rewrite(t.Context(), key, []byte("doc"), 0, exclaim) + m, err := c.rewrite(t.Context(), key, []byte("doc"), nil, exclaim) if err != nil || string(m.content()) != "doc!" { t.Fatalf("rewrite = %q, %v", m.content(), err) } diff --git a/internal/handler/npm.go b/internal/handler/npm.go index 537c4b3..7263b3b 100644 --- a/internal/handler/npm.go +++ b/internal/handler/npm.go @@ -467,7 +467,7 @@ func (h *NPMHandler) rewriteMetadataWithLocal(packageName string, body []byte, l func (h *NPMHandler) npmRewriteOptions(packageName string, local []npmLocalEntry) npm.Options { return npm.Options{ ProxyURL: h.proxyURL, - Cooldown: h.proxy.Cooldown, + Cooldown: h.proxy.cooldown(), DeniedVersions: h.proxy.Denylist.Versions(canonicalPackagePURL("npm", packageName)), EvaluatedAt: time.Now(), LocalVersions: local, @@ -685,7 +685,7 @@ func (h *NPMHandler) versionInCooldown(packageName, version string, metadata fun versionPURL := canonicalVersionPURL("npm", packageName, version) if !routed { if ver, err := h.proxy.DB.GetVersionByPURL(versionPURL); err == nil && ver != nil && ver.PublishedAt.Valid { - return !h.proxy.Cooldown.IsAllowed("npm", canonicalPackagePURL("npm", packageName), ver.PublishedAt.Time), nil + return !h.proxy.cooldown().IsAllowed("npm", canonicalPackagePURL("npm", packageName), ver.PublishedAt.Time), nil } } @@ -717,7 +717,7 @@ func (h *NPMHandler) versionInCooldown(packageName, version string, metadata fun } } - return !h.proxy.Cooldown.IsAllowed("npm", canonicalPackagePURL("npm", packageName), publishedAt), nil + return !h.proxy.cooldown().IsAllowed("npm", canonicalPackagePURL("npm", packageName), publishedAt), nil } // npmMetadataPath returns the package name and the encoded registry path. diff --git a/internal/handler/npm_rewrite_test.go b/internal/handler/npm_rewrite_test.go index b36b229..9b8e35b 100644 --- a/internal/handler/npm_rewrite_test.go +++ b/internal/handler/npm_rewrite_test.go @@ -94,7 +94,7 @@ func TestNPMStoredRewriteSkippedWhenStale(t *testing.T) { body := []byte(`{}`) proxy.cacheMetadataBlob(t.Context(), "npm", "demo", metadataStoragePath("npm", "demo"), &upstreamMetadata{body: body}) key := rewriteCacheKey("npm", "http://proxy", "demo", body) - if _, err := proxy.rewrites.rewrite(t.Context(), key, body, 0, func(b []byte) ([]byte, error) { return b, nil }); err != nil { + if _, err := proxy.rewrites.rewrite(t.Context(), key, body, nil, func(b []byte) ([]byte, error) { return b, nil }); err != nil { t.Fatal(err) } if _, ok := proxy.storedRewrite("npm", "demo", "http://proxy", "demo"); !ok { @@ -129,7 +129,7 @@ func TestNPMStoredRewriteRecordsMetadataHit(t *testing.T) { body := []byte(`{"name":"demo"}`) proxy.cacheMetadataBlob(t.Context(), "npm", "hit-demo", metadataStoragePath("npm", "hit-demo"), &upstreamMetadata{body: body}) key := rewriteCacheKey("npm", "http://proxy", "hit-demo", body) - if _, err := proxy.rewrites.rewrite(t.Context(), key, body, 0, func(b []byte) ([]byte, error) { return b, nil }); err != nil { + if _, err := proxy.rewrites.rewrite(t.Context(), key, body, nil, func(b []byte) ([]byte, error) { return b, nil }); err != nil { t.Fatal(err) } diff --git a/internal/handler/nuget_cooldown.go b/internal/handler/nuget_cooldown.go index afb147d..b9345c5 100644 --- a/internal/handler/nuget_cooldown.go +++ b/internal/handler/nuget_cooldown.go @@ -204,7 +204,7 @@ func (h *NuGetHandler) nugetLeafAllowed(leaf map[string]any, id string) bool { if err != nil { return true } - return h.proxy.Cooldown.IsAllowed("nuget", canonicalPackagePURL("nuget", strings.ToLower(id)), when) + return h.proxy.cooldown().IsAllowed("nuget", canonicalPackagePURL("nuget", strings.ToLower(id)), when) } func nugetCatalogEntry(leaf map[string]any) map[string]any { diff --git a/internal/handler/pub.go b/internal/handler/pub.go index 3c81298..24fdd02 100644 --- a/internal/handler/pub.go +++ b/internal/handler/pub.go @@ -247,7 +247,7 @@ func (h *PubHandler) shouldFilterVersion(packagePURL, name, version string, vmap return false } - if !h.proxy.Cooldown.IsAllowed("pub", packagePURL, publishedAt) { + if !h.proxy.cooldown().IsAllowed("pub", packagePURL, publishedAt) { h.proxy.Logger.Info("cooldown: filtering pub version", "package", name, "version", version) return true diff --git a/internal/handler/pypi.go b/internal/handler/pypi.go index fe018d8..ebb3082 100644 --- a/internal/handler/pypi.go +++ b/internal/handler/pypi.go @@ -318,7 +318,7 @@ func (h *PyPIHandler) fetchFilteredVersions(r *http.Request, name string) (map[s filtered := make(map[string]bool) for version, publishedAt := range times { - if !publishedAt.IsZero() && !h.proxy.Cooldown.IsAllowed("pypi", packagePURL, publishedAt) { + if !publishedAt.IsZero() && !h.proxy.cooldown().IsAllowed("pypi", packagePURL, publishedAt) { filtered[version] = true } } @@ -611,7 +611,7 @@ func (h *PyPIHandler) shouldFilterRelease(packagePURL string, files any) bool { } publishedAt := h.newestUploadTime(filesArr) - return !publishedAt.IsZero() && !h.proxy.Cooldown.IsAllowed("pypi", packagePURL, publishedAt) + return !publishedAt.IsZero() && !h.proxy.cooldown().IsAllowed("pypi", packagePURL, publishedAt) } // versionInCooldown reports whether a version is still inside the cooldown @@ -636,7 +636,7 @@ func (h *PyPIHandler) versionInCooldown(r *http.Request, name, version string) ( return false, nil } h.storeUploadTime(name, version, publishedAt) - return !h.proxy.Cooldown.IsAllowed("pypi", canonicalPackagePURL("pypi", name), publishedAt), nil + return !h.proxy.cooldown().IsAllowed("pypi", canonicalPackagePURL("pypi", name), publishedAt), nil } // storeUploadTime records a release's newest upload time under the version @@ -675,7 +675,7 @@ func (h *PyPIHandler) storedCooldown(r *http.Request, name, version, filename st if err != nil || ver == nil || !ver.PublishedAt.Valid { return false, false } - if !h.proxy.Cooldown.IsAllowed("pypi", packagePURL, ver.PublishedAt.Time) { + if !h.proxy.cooldown().IsAllowed("pypi", packagePURL, ver.PublishedAt.Time) { return true, true } cached, err := h.proxy.HasCachedArtifact(r.Context(), "pypi", name, version, filename) diff --git a/internal/handler/rewrite_cache.go b/internal/handler/rewrite_cache.go index b2dc992..fd455a7 100644 --- a/internal/handler/rewrite_cache.go +++ b/internal/handler/rewrite_cache.go @@ -35,8 +35,8 @@ import ( // depends on is fixed for the life of the process: the proxy URL is part of // the key, and the denylist is loaded at startup. The exception is cooldown, // which depends on the time of the request: with cooldown on, entries expire -// after Proxy.cooldownRewriteTTL, so a version is held back at most that long -// past its cooldown. +// when the first version they held back leaves cooldown (see +// cooldown_expiry.go). // // Each entry keeps the ETag of its output, so clients can revalidate their // copy without the output being hashed again on every request. Dates are not @@ -65,36 +65,6 @@ type rewriteEntry struct { expires time.Time } -// maxCooldownRewriteTTL is the longest a rewrite made with cooldown on is -// served. Versions leave cooldown as time passes, so a cached rewrite would -// otherwise keep hiding them. Cooldowns run to days, and a package manager -// that checks every lockfile entry on each install, as pnpm 12 does, would -// otherwise have the whole lockfile rewritten again on every install more -// than a minute after the last. -const maxCooldownRewriteTTL = time.Hour - -// minCooldownRewriteTTL is the shortest a rewrite made with cooldown on is -// served, however short the cooldown. -const minCooldownRewriteTTL = time.Minute - -// cooldownRewriteShare is the share of the shortest cooldown a rewrite made -// with cooldown on is served for, as a divisor. -const cooldownRewriteShare = 24 - -// cooldownRewriteTTL returns how long a rewrite made with cooldown on is -// served: a 24th of the shortest cooldown configured, so a version is held -// back at most about 4% past its cooldown, kept between a minute and an hour. -func (p *Proxy) cooldownRewriteTTL() time.Duration { - ttl := p.ShortestCooldown / cooldownRewriteShare - switch { - case ttl <= 0 || ttl > maxCooldownRewriteTTL: - return maxCooldownRewriteTTL - case ttl < minCooldownRewriteTTL: - return minCooldownRewriteTTL - } - return ttl -} - // size is how much of the cache's room the entry takes. func (e *rewriteEntry) size() int64 { return int64(len(e.plain) + len(e.gzipped)) @@ -279,7 +249,7 @@ func (c *rewriteCache) remove(el *list.Element) { // shared. A caller waiting on another's rewrite leaves when ctx ends; the // rewrite itself always runs to completion, so the result is cached for the // next request. -func (c *rewriteCache) rewrite(ctx context.Context, key string, in []byte, ttl time.Duration, rewrite func([]byte) ([]byte, error)) (rewrittenMetadata, error) { +func (c *rewriteCache) rewrite(ctx context.Context, key string, in []byte, expiry func(start time.Time) time.Time, rewrite func([]byte) ([]byte, error)) (rewrittenMetadata, error) { if c == nil { return rewriteWithETag(in, rewrite) } @@ -302,16 +272,17 @@ func (c *rewriteCache) rewrite(ctx context.Context, key string, in []byte, ttl t c.inFlight[key] = f c.mu.Unlock() - // The entry expires counting from before the rewrite, so it never - // outlives the time its cooldown decisions were made at by more than ttl. - var expires time.Time - if ttl > 0 { - expires = c.clock().Add(ttl) - } + // expiry is given the time from before the rewrite, so an entry never + // outlives the time its cooldown decisions were made at. + start := c.clock() defer func() { c.mu.Lock() delete(c.inFlight, key) if f.err == nil { + var expires time.Time + if expiry != nil { + expires = expiry(start) + } c.add(&rewriteEntry{key: key, rewrittenMetadata: f.m.compact(), expires: expires}) } c.mu.Unlock() @@ -370,7 +341,8 @@ func (c *rewriteCache) add(e *rewriteEntry) { // metadata cache stored it, and is empty otherwise; when known it keys the // cache, so the document isn't hashed again. // -// With cooldown on, the cached rewrite expires after p.cooldownRewriteTTL. Its +// With cooldown on, the cached rewrite expires when the first version it +// held back leaves cooldown, and after p.cooldownRewriteTTL at the latest. Its // ETag is the hash of its output, and it is dated like any other, so clients // revalidate with either validator and get the gzip copy. // @@ -390,11 +362,15 @@ func (p *Proxy) cachedRewrite(ctx context.Context, ecosystem, proxyURL, name str if digest == "" { key = rewriteCacheKey(ecosystem, proxyURL, name, in) } - var ttl time.Duration + var expiry func(time.Time) time.Time if cooldown { - ttl = p.cooldownRewriteTTL() + // Recording runs from before the rewrite until after it, so it + // hears of every version the rewrite holds back. + rec := p.watchCooldown() + defer p.unwatchCooldown(rec) + expiry = func(start time.Time) time.Time { return p.cooldownExpiry(rec, start) } } - m, err := p.rewrites.rewrite(ctx, key, in, ttl, rewriteOnly) + m, err := p.rewrites.rewrite(ctx, key, in, expiry, rewriteOnly) if err == nil { m.lastModified = p.metadataDate(ecosystem, proxyURL, name, m.etag) } diff --git a/internal/handler/rewrite_cache_test.go b/internal/handler/rewrite_cache_test.go index 7188634..3f4c062 100644 --- a/internal/handler/rewrite_cache_test.go +++ b/internal/handler/rewrite_cache_test.go @@ -44,7 +44,7 @@ func TestRewriteCache_RepeatServesCachedRewrite(t *testing.T) { key := rewriteCacheKey("npm", "http://proxy", "left-pad", []byte("doc")) for range 3 { - m, err := c.rewrite(context.Background(), key, []byte("doc"), 0, r.fn) + m, err := c.rewrite(context.Background(), key, []byte("doc"), nil, r.fn) if err != nil || string(m.content()) != "DOC" { t.Fatalf("rewrite = %q, %v", m.content(), err) } @@ -60,7 +60,7 @@ func TestRewriteCache_NewUpstreamBytesRewriteAgain(t *testing.T) { for _, doc := range []string{"v1", "v2"} { key := rewriteCacheKey("npm", "http://proxy", "left-pad", []byte(doc)) - m, err := c.rewrite(context.Background(), key, []byte(doc), 0, r.fn) + m, err := c.rewrite(context.Background(), key, []byte(doc), nil, r.fn) if err != nil || string(m.content()) != strings.ToUpper(doc) { t.Fatalf("rewrite(%s) = %q, %v", doc, m.content(), err) } @@ -82,7 +82,7 @@ func TestRewriteCache_ConcurrentRequestsShareRewrite(t *testing.T) { outs := make(chan string, n) run := func() { - m, err := c.rewrite(context.Background(), key, []byte("doc"), 0, r.fn) + m, err := c.rewrite(context.Background(), key, []byte("doc"), nil, r.fn) if err != nil { t.Error(err) } @@ -115,20 +115,20 @@ func TestRewriteCache_WaiterLeavingKeepsRewrite(t *testing.T) { first := make(chan error, 1) go func() { - _, err := c.rewrite(context.Background(), key, []byte("doc"), 0, r.fn) + _, err := c.rewrite(context.Background(), key, []byte("doc"), nil, r.fn) first <- err }() <-r.entered ctx, cancel := context.WithCancel(context.Background()) cancel() - if _, err := c.rewrite(ctx, key, []byte("doc"), 0, r.fn); !errors.Is(err, context.Canceled) { + if _, err := c.rewrite(ctx, key, []byte("doc"), nil, r.fn); !errors.Is(err, context.Canceled) { t.Errorf("waiter err = %v, want context.Canceled", err) } close(r.release) if err := <-first; err != nil { t.Fatal(err) } - if m, err := c.rewrite(context.Background(), key, []byte("doc"), 0, r.fn); err != nil || string(m.content()) != "DOC" { + if m, err := c.rewrite(context.Background(), key, []byte("doc"), nil, r.fn); err != nil || string(m.content()) != "DOC" { t.Errorf("cached rewrite = %q, %v", m.content(), err) } if got := r.calls.Load(); got != 1 { @@ -142,18 +142,18 @@ func TestRewriteCache_EvictsLeastRecentlyUsed(t *testing.T) { keyFor := func(doc string) string { return rewriteCacheKey("npm", "http://proxy", doc, []byte(doc)) } for _, doc := range []string{"aaa", "bbb"} { - _, _ = c.rewrite(context.Background(), keyFor(doc), []byte(doc), 0, r.fn) + _, _ = c.rewrite(context.Background(), keyFor(doc), []byte(doc), nil, r.fn) } - _, _ = c.rewrite(context.Background(), keyFor("aaa"), []byte("aaa"), 0, r.fn) // aaa is now the most recent - _, _ = c.rewrite(context.Background(), keyFor("ccc"), []byte("ccc"), 0, r.fn) // evicts bbb + _, _ = c.rewrite(context.Background(), keyFor("aaa"), []byte("aaa"), nil, r.fn) // aaa is now the most recent + _, _ = c.rewrite(context.Background(), keyFor("ccc"), []byte("ccc"), nil, r.fn) // evicts bbb before := r.calls.Load() - _, _ = c.rewrite(context.Background(), keyFor("aaa"), []byte("aaa"), 0, r.fn) - _, _ = c.rewrite(context.Background(), keyFor("ccc"), []byte("ccc"), 0, r.fn) + _, _ = c.rewrite(context.Background(), keyFor("aaa"), []byte("aaa"), nil, r.fn) + _, _ = c.rewrite(context.Background(), keyFor("ccc"), []byte("ccc"), nil, r.fn) if got := r.calls.Load() - before; got != 0 { t.Errorf("aaa and ccc rewritten %d times, want both still cached", got) } - _, _ = c.rewrite(context.Background(), keyFor("bbb"), []byte("bbb"), 0, r.fn) + _, _ = c.rewrite(context.Background(), keyFor("bbb"), []byte("bbb"), nil, r.fn) if got := r.calls.Load() - before; got != 1 { t.Errorf("bbb should have been evicted and rewritten") } @@ -164,7 +164,7 @@ func TestRewriteCache_OutputLargerThanCacheIsNotStored(t *testing.T) { var r countingRewrite key := rewriteCacheKey("npm", "http://proxy", "big", []byte("big")) - _, _ = c.rewrite(context.Background(), key, []byte("big"), 0, r.fn) + _, _ = c.rewrite(context.Background(), key, []byte("big"), nil, r.fn) if c.len() != 0 { t.Errorf("cached %d entries, want 0", c.len()) } @@ -177,7 +177,7 @@ func TestRewriteCache_ErrorsAreNotCached(t *testing.T) { key := rewriteCacheKey("npm", "http://proxy", "broken", []byte("doc")) for range 2 { - if _, err := c.rewrite(context.Background(), key, []byte("doc"), 0, failing); err == nil { + if _, err := c.rewrite(context.Background(), key, []byte("doc"), nil, failing); err == nil { t.Fatal("expected the rewrite error") } } @@ -232,7 +232,7 @@ func TestRewriteCacheExpiredEntryFreesRoom(t *testing.T) { clock := &testClock{t: time.Date(2026, 10, 10, 12, 0, 0, 0, time.UTC)} c.now = clock.now upper := func(b []byte) ([]byte, error) { return bytes.ToUpper(b), nil } - if _, err := c.rewrite(context.Background(), "k", []byte("doc"), time.Minute, upper); err != nil { + if _, err := c.rewrite(context.Background(), "k", []byte("doc"), func(start time.Time) time.Time { return start.Add(time.Minute) }, upper); err != nil { t.Fatal(err) } clock.advance(time.Minute) @@ -302,17 +302,16 @@ func (c *rewriteCache) len() int { return len(c.entries) } -// TestCooldownRewriteTTL checks that a rewrite filtered by cooldown is cached -// for a 24th of the shortest cooldown, between a minute and an hour. +// TestCooldownRewriteTTL checks that the cap on a rewrite filtered by +// cooldown is the shortest cooldown, between a minute and a day. func TestCooldownRewriteTTL(t *testing.T) { for _, tc := range []struct { shortest, want time.Duration }{ - {0, time.Hour}, - {72 * time.Hour, time.Hour}, - {24 * time.Hour, time.Hour}, - {2 * time.Hour, 5 * time.Minute}, - {10 * time.Minute, time.Minute}, + {0, 24 * time.Hour}, + {72 * time.Hour, 24 * time.Hour}, + {2 * time.Hour, 2 * time.Hour}, + {10 * time.Second, time.Minute}, } { p := &Proxy{ShortestCooldown: tc.shortest} if got := p.cooldownRewriteTTL(); got != tc.want { From 2928b5c27f511378a51e9cbd43a93af5d9809be5 Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Fri, 30 Oct 2026 08:36:39 +0000 Subject: [PATCH 495/633] Detect Helm's cooldown check in the handler coverage test --- internal/handler/cooldown_supported_test.go | 11 ++++++----- internal/server/cooldown_view_test.go | 3 --- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/internal/handler/cooldown_supported_test.go b/internal/handler/cooldown_supported_test.go index ac73a3b..7187ede 100644 --- a/internal/handler/cooldown_supported_test.go +++ b/internal/handler/cooldown_supported_test.go @@ -8,16 +8,13 @@ import ( "testing" ) -// TestCooldownSupportedCoversHandlers checks that every ecosystem a handler -// checks cooldown for is listed in cooldownEcosystems, so the UI shows its -// cooldown. func TestCooldownSupportedCoversHandlers(t *testing.T) { - calls := regexp.MustCompile(`(?:IsAllowed|Evaluate)\("([a-z]+)"|withheld\(h\.proxy, "([a-z]+)"`) + calls := regexp.MustCompile(`(?:IsAllowed|Evaluate)\("([a-z]+)"|withheld\(h\.proxy, "([a-z]+)"|IsAllowed\((helmMetadataEcosystem)`) files, err := filepath.Glob("*.go") if err != nil { t.Fatal(err) } - found := map[string]bool{helmMetadataEcosystem: true} + found := map[string]bool{} for _, f := range files { if strings.HasSuffix(f, "_test.go") { continue @@ -27,6 +24,10 @@ func TestCooldownSupportedCoversHandlers(t *testing.T) { t.Fatal(err) } for _, m := range calls.FindAllStringSubmatch(string(src), -1) { + if m[3] != "" { + found[helmMetadataEcosystem] = true + continue + } found[m[1]+m[2]] = true } } diff --git a/internal/server/cooldown_view_test.go b/internal/server/cooldown_view_test.go index 2adbdae..13e731c 100644 --- a/internal/server/cooldown_view_test.go +++ b/internal/server/cooldown_view_test.go @@ -79,9 +79,6 @@ func TestCooldownViewOnlyPattern(t *testing.T) { } } -// TestCooldownViewUnsupportedEcosystem checks that ecosystems the proxy -// applies no cooldown to get no cooldown on their package pages, and that -// an override for one is marked on the dashboard. func TestCooldownViewUnsupportedEcosystem(t *testing.T) { v := testCooldownView(t, config.CooldownConfig{ Default: "3d", From 37ecc3d8ce2298158a7840b9f73a69a607f5f463 Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Fri, 30 Oct 2026 08:40:35 +0000 Subject: [PATCH 496/633] Trim BottleImage doc comment and rejoin the brew docs list --- docs/configuration.md | 1 - internal/handler/homebrew_bottles.go | 10 +++------- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 55e421f..17d2d6c 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1674,7 +1674,6 @@ proxy mirror --platform linux/arm64 pkg:brew/hello@2.12.3 pkg:brew/openssl%403 ``` - A formula name with `@` is written with `%40`, since `@` starts the version: `pkg:brew/openssl%403` is `openssl@3`, and `pkg:brew/openssl@3` is version 3 of `openssl`. - - The formula's JSON API on `upstream.homebrew_api` gives the image tag for the current version, which carries the formula's revision and the bottles' rebuild (`hello` 2.12.3 is tagged `2.12.3-1`). Only the current version is described there; another version is used as the tag as given, so give the full tag, such as `2.12.2` or `2.12.3-1`, for older bottles. - A PURL without a version mirrors the current version. - `--platform` selects bottles like image platforms: `linux/arm64`, `linux/amd64`, or `darwin/arm64` for every macOS release's arm64 bottle. Without it every bottle of the version is mirrored. diff --git a/internal/handler/homebrew_bottles.go b/internal/handler/homebrew_bottles.go index 37ab450..169ed61 100644 --- a/internal/handler/homebrew_bottles.go +++ b/internal/handler/homebrew_bottles.go @@ -35,13 +35,9 @@ type homebrewFormula struct { } `json:"bottle"` } -// BottleImage returns the image Homebrew publishes the bottles of a -// homebrew/core formula version under, as brew pulls them through -// HOMEBREW_ARTIFACT_DOMAIN: openssl@3 is homebrew/core/openssl/3, and the tag -// carries the formula's revision and the bottles' rebuild, so hello 2.12.3 -// with rebuild 1 is tagged 2.12.3-1. Only the current version is described by -// the JSON API; any other version is used as the tag as given, which also -// takes a tag such as 2.12.3-1 directly. +// BottleImage returns the homebrew/core image and tag for a formula version. +// Only the current version gets its revision and rebuild from the JSON API; +// any other version is used as the tag as given. func (h *HomebrewHandler) BottleImage(ctx context.Context, formula, version string) (ImageRef, error) { if !homebrewFormulaPattern.MatchString(formula) { return ImageRef{}, fmt.Errorf("invalid Homebrew formula name %q", formula) From 9babab7380b132e1ee6ccaa4a5900ceb64a317ac Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Fri, 30 Oct 2026 08:51:35 +0000 Subject: [PATCH 497/633] Describe archived CRAN versions as fetched unchecked --- docs/configuration.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/configuration.md b/docs/configuration.md index 7cdbe64..75878cc 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1625,7 +1625,7 @@ A Julia package is mirrored with the source tarball `upstream.julia` serves at ` A Swift release is mirrored from `upstream.swift` with its source archive, checked against the SHA-256 checksum in the release metadata and cached under the key `/swift/{scope}/{name}/{version}.zip` serves. Give the registry scope as the namespace, for example `pkg:swift/apple/swift-argument-parser@1.5.0` for `apple.swift-argument-parser`, or the source repository, as SBOM tools write it: `pkg:swift/github.com/apple/swift-argument-parser@1.5.0` is looked up through the upstream's `/identifiers`, and the first identity it lists is mirrored. The release metadata, the package's release list and the release's `Package.swift` with its version-specific manifests are stored in the metadata cache too, so `swift package resolve` works with the upstream unreachable. Serving them, and the archive, offline needs `cache_metadata` on the server, since the archive is served with the checksum from the release metadata. -A CRAN version is mirrored as its source package from `upstream.cran`, for example `pkg:cran/glue@1.8.1`, and cached under the file name, so it is served for both `/cran/src/contrib/{file}` and `/cran/src/contrib/Archive/{name}/{file}`, where CRAN moves it once a newer version is published. The current version is fetched from `src/contrib` and checked against the MD5 sum in `src/contrib/PACKAGES.gz`; older versions from `src/contrib/Archive/{name}/`, checked against the MD5 sum in that directory's `PACKAGES.rds`, or unchecked when the repository has none there. Binary packages aren't mirrored, and neither are the `PACKAGES` indexes, so `install.packages` needs the upstream or an index the proxy already cached; installing the tarball by URL works without either. +A CRAN version is mirrored as its source package from `upstream.cran`, for example `pkg:cran/glue@1.8.1`, and cached under the file name, so it is served for both `/cran/src/contrib/{file}` and `/cran/src/contrib/Archive/{name}/{file}`, where CRAN moves it once a newer version is published. The current version is fetched from `src/contrib` and checked against the MD5 sum in `src/contrib/PACKAGES.gz`; older versions are fetched unchecked from `src/contrib/Archive/{name}/`, as the CRAN handler fetches them. Binary packages aren't mirrored, and neither are the `PACKAGES` indexes, so `install.packages` needs the upstream or an index the proxy already cached; installing the tarball by URL works without either. | Flag | Default | Description | |------|---------|-------------| From c4615ff56164f410211ede3732fadf07e4d6c244 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Fri, 30 Oct 2026 10:09:21 +0100 Subject: [PATCH 498/633] Read change feeds through git-pkgs/registries The feed readers move to registries' changes package. The proxy keeps the cursor and moves it past the changes only once their metadata is expired, so a failed expiry reads them again. --- internal/changefeed/changefeed.go | 261 ------------------------- internal/changefeed/changefeed_test.go | 198 ------------------- internal/server/change_feeds.go | 54 +++-- internal/server/change_feeds_test.go | 75 +++++-- internal/server/server.go | 48 +++-- 5 files changed, 124 insertions(+), 512 deletions(-) delete mode 100644 internal/changefeed/changefeed.go delete mode 100644 internal/changefeed/changefeed_test.go diff --git a/internal/changefeed/changefeed.go b/internal/changefeed/changefeed.go deleted file mode 100644 index 2e2d48f..0000000 --- a/internal/changefeed/changefeed.go +++ /dev/null @@ -1,261 +0,0 @@ -// Package changefeed reads the feeds in which public registries list the -// packages that changed since a given point, so the proxy can revalidate its -// cached metadata for them before metadata_ttl runs out. -package changefeed - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "io" - "net/http" - "net/url" - "slices" - "strconv" - "strings" -) - -// Feed is one registry's change feed. It remembers how far it has read, in -// memory only: after a restart it starts again from the registry's current -// position. -type Feed interface { - // Ecosystem is the metadata cache ecosystem the changed names belong to. - Ecosystem() string - // Changes returns what changed since the previous call. The first call - // only finds the feed's current position and reports no changes. On error - // the position stays where it was, so the next call asks again. - Changes(ctx context.Context) (Changes, error) -} - -// Changes is one read of a feed. -type Changes struct { - // Names are the cache keys of the documents that changed. - Names []string - // All is set when the feed can't say what changed, so every document of - // the ecosystem has to be treated as changed. - All bool -} - -// maxFeedResponse bounds a feed response read into memory. A full page of -// the npm feed is about 1.5 MB. -const maxFeedResponse = 64 << 20 - -// getJSON fetches u and decodes its JSON body into v. A status other than 200 -// is an error unless allowStatus also accepts it. -func getJSON(ctx context.Context, client *http.Client, u string, v any, allowStatus ...int) error { - req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil) - if err != nil { - return fmt.Errorf("creating request: %w", err) - } - req.Header.Set("Accept", "application/json") - resp, err := client.Do(req) - if err != nil { - return err - } - defer func() { _ = resp.Body.Close() }() - if resp.StatusCode != http.StatusOK && !slices.Contains(allowStatus, resp.StatusCode) { - _, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, maxFeedResponse)) - return fmt.Errorf("%s: unexpected status %d", redact(u), resp.StatusCode) - } - body, err := io.ReadAll(io.LimitReader(resp.Body, maxFeedResponse+1)) - if err != nil { - return fmt.Errorf("%s: reading response: %w", redact(u), err) - } - if len(body) > maxFeedResponse { - return fmt.Errorf("%s: response larger than %d bytes", redact(u), maxFeedResponse) - } - if err := json.Unmarshal(body, v); err != nil { - return fmt.Errorf("%s: decoding response: %w", redact(u), err) - } - return nil -} - -// redact drops credentials and the query from u for error messages. -func redact(u string) string { - parsed, err := url.Parse(u) - if err != nil { - return "feed" - } - parsed.User = nil - parsed.RawQuery = "" - return parsed.String() -} - -// withQuery returns base with the query parameters in q added. -func withQuery(base string, q url.Values) string { - sep := "?" - if strings.Contains(base, "?") { - sep = "&" - } - return base + sep + q.Encode() -} - -// ComposerFeedURL is where a Packagist instance at apiURL, such as -// https://packagist.org, publishes its metadata change feed. Packagist -// advertises it as "metadata-changes-url" in its packages.json. -func ComposerFeedURL(apiURL string) string { - return strings.TrimSuffix(apiURL, "/") + "/metadata/changes.json" -} - -// ComposerFeed reads Packagist's metadata change feed -// (https://packagist.org/apidoc#track-package-updates). Its position is a -// timestamp in units of 1/10000 s. Asked without one it answers 400 with the -// current timestamp; asked with one older than its 24 hours of history it -// answers with a single "resync" action. Each update names the file that -// changed, "vendor/package" or "vendor/package~dev", which is also the key -// the Composer handler caches it under; a delete names the package, and -// covers both files. -type ComposerFeed struct { - client *http.Client - url string - since int64 -} - -// NewComposerFeed returns the feed at feedURL, see ComposerFeedURL. -func NewComposerFeed(client *http.Client, feedURL string) *ComposerFeed { - return &ComposerFeed{client: client, url: feedURL} -} - -func (f *ComposerFeed) Ecosystem() string { return "composer" } - -type composerChanges struct { - Actions []struct { - Type string `json:"type"` - Package string `json:"package"` - } `json:"actions"` - Timestamp int64 `json:"timestamp"` - Error string `json:"error"` -} - -func (f *ComposerFeed) Changes(ctx context.Context) (Changes, error) { - if f.since == 0 { - var start composerChanges - if err := getJSON(ctx, f.client, f.url, &start, http.StatusBadRequest); err != nil { - return Changes{}, err - } - if start.Timestamp <= 0 { - return Changes{}, errors.New("composer change feed: no timestamp to start from") - } - f.since = start.Timestamp - return Changes{}, nil - } - - var page composerChanges - u := withQuery(f.url, url.Values{"since": {strconv.FormatInt(f.since, 10)}}) - if err := getJSON(ctx, f.client, u, &page); err != nil { - return Changes{}, err - } - if page.Error != "" || page.Timestamp <= 0 { - return Changes{}, fmt.Errorf("composer change feed: %q", page.Error) - } - var changes Changes - for _, a := range page.Actions { - switch a.Type { - case "update": - changes.Names = append(changes.Names, a.Package) - case "delete": - // A deleted package takes its dev versions with it. - changes.Names = append(changes.Names, a.Package, a.Package+"~dev") - case "resync": - // The position is too old for the feed to list what changed. - changes.All = true - } - } - f.since = page.Timestamp - return changes, nil -} - -// NPMFeedURL is the change feed of the public npm registry. -const NPMFeedURL = "https://replicate.npmjs.com/registry/_changes" - -// npmPageLimit is the most changes the npm feed returns in one response. -const npmPageLimit = 10000 - -// npmMaxPages bounds how many pages one Changes call reads. The rest is read -// on the next call. -const npmMaxPages = 10 - -// NPMFeed reads the npm registry's CouchDB-style changes feed at -// replicate.npmjs.com. Its position is a sequence, which npm asks followers -// to treat as opaque; the database document at the feed's parent URL, with -// its trailing slash (without it the URL is 404), gives the current one as -// update_seq. A page holds at most 10000 changes (1000 when no limit is -// given). Each result's id is a package name, which is also the key the npm -// handler caches its packument under. -type NPMFeed struct { - client *http.Client - url string - // since is the sequence exactly as the feed returned it. - since json.RawMessage -} - -// NewNPMFeed returns the feed at feedURL, see NPMFeedURL. -func NewNPMFeed(client *http.Client, feedURL string) *NPMFeed { - return &NPMFeed{client: client, url: feedURL} -} - -func (f *NPMFeed) Ecosystem() string { return "npm" } - -type npmChanges struct { - Results []struct { - ID string `json:"id"` - } `json:"results"` - LastSeq json.RawMessage `json:"last_seq"` -} - -func (f *NPMFeed) Changes(ctx context.Context) (Changes, error) { - if f.since == nil { - var info struct { - UpdateSeq json.RawMessage `json:"update_seq"` - } - if err := getJSON(ctx, f.client, strings.TrimSuffix(f.url, "_changes"), &info); err != nil { - return Changes{}, err - } - if len(info.UpdateSeq) == 0 { - return Changes{}, errors.New("npm change feed: no update_seq to start from") - } - f.since = info.UpdateSeq - return Changes{}, nil - } - - var changes Changes - for range npmMaxPages { - var page npmChanges - u := withQuery(f.url, url.Values{ - "since": {seqParam(f.since)}, - "limit": {strconv.Itoa(npmPageLimit)}, - }) - if err := getJSON(ctx, f.client, u, &page); err != nil { - // Changes already read are reported, and the position kept - // after them, so they aren't lost to a later page's failure. - if len(changes.Names) > 0 { - return changes, nil - } - return Changes{}, err - } - if len(page.LastSeq) == 0 { - return changes, errors.New("npm change feed: response without last_seq") - } - for _, r := range page.Results { - if r.ID != "" && !strings.HasPrefix(r.ID, "_design/") { - changes.Names = append(changes.Names, r.ID) - } - } - f.since = page.LastSeq - if len(page.Results) < npmPageLimit { - break - } - } - return changes, nil -} - -// seqParam returns a sequence as the since parameter takes it: a number as -// is, a string without its quotes. -func seqParam(seq json.RawMessage) string { - var s string - if json.Unmarshal(seq, &s) == nil { - return s - } - return string(seq) -} diff --git a/internal/changefeed/changefeed_test.go b/internal/changefeed/changefeed_test.go deleted file mode 100644 index 21a4765..0000000 --- a/internal/changefeed/changefeed_test.go +++ /dev/null @@ -1,198 +0,0 @@ -package changefeed - -import ( - "context" - "fmt" - "net/http" - "net/http/httptest" - "slices" - "strconv" - "strings" - "sync" - "sync/atomic" - "testing" -) - -// fakePackagist serves a change feed the way packagist.org does: without -// since, 400 with the current timestamp; with an old since, a resync; with a -// known one, the actions recorded after it. -type fakePackagist struct { - mu sync.Mutex - now int64 - actions map[int64]string // since -> JSON actions - asked []string -} - -func (p *fakePackagist) ServeHTTP(w http.ResponseWriter, r *http.Request) { - p.mu.Lock() - defer p.mu.Unlock() - since := r.URL.Query().Get("since") - p.asked = append(p.asked, since) - w.Header().Set("Content-Type", "application/json") - if since == "" { - w.WriteHeader(http.StatusBadRequest) - _, _ = fmt.Fprintf(w, `{"error":"Invalid or missing since query parameter","timestamp":%d}`, p.now) - return - } - n, _ := strconv.ParseInt(since, 10, 64) - actions, ok := p.actions[n] - if !ok { - actions = `[{"type":"resync","package":"*","time":1}]` - } - _, _ = fmt.Fprintf(w, `{"actions":%s,"timestamp":%d}`, actions, p.now) -} - -func TestComposerFeed(t *testing.T) { - p := &fakePackagist{now: 100, actions: map[int64]string{ - 100: `[{"type":"update","package":"acme/lib","time":1},{"type":"update","package":"acme/lib~dev","time":1},{"type":"delete","package":"acme/gone","time":1}]`, - 200: `[]`, - }} - srv := httptest.NewServer(p) - defer srv.Close() - feed := NewComposerFeed(srv.Client(), ComposerFeedURL(srv.URL+"/")) - ctx := context.Background() - - if c, err := feed.Changes(ctx); err != nil || c.Names != nil || c.All { - t.Fatalf("first read = %+v, %v; want the start position only", c, err) - } - p.mu.Lock() - p.now = 200 - p.mu.Unlock() - c, err := feed.Changes(ctx) - if err != nil { - t.Fatal(err) - } - if want := []string{"acme/lib", "acme/lib~dev", "acme/gone", "acme/gone~dev"}; !slices.Equal(c.Names, want) || c.All { - t.Errorf("changes = %+v, want %v", c, want) - } - if c, err := feed.Changes(ctx); err != nil || len(c.Names) != 0 || c.All { - t.Errorf("changes without news = %+v, %v", c, err) - } - - // A position the feed no longer has history for gets a resync. - feed.since = 50 - if c, err := feed.Changes(ctx); err != nil || !c.All { - t.Errorf("changes after a resync = %+v, %v, want All", c, err) - } - if want := []string{"", "100", "200", "50"}; !slices.Equal(p.asked, want) { - t.Errorf("asked since %q, want %q", p.asked, want) - } -} - -func TestComposerFeedErrorKeepsPosition(t *testing.T) { - var fail atomic.Bool - fail.Store(true) - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if fail.Load() { - w.WriteHeader(http.StatusServiceUnavailable) - return - } - if r.URL.Query().Get("since") != "7" { - t.Errorf("since = %q, want 7", r.URL.Query().Get("since")) - } - _, _ = w.Write([]byte(`{"actions":[{"type":"update","package":"acme/lib","time":1}],"timestamp":8}`)) - })) - defer srv.Close() - feed := &ComposerFeed{client: srv.Client(), url: srv.URL, since: 7} - if _, err := feed.Changes(context.Background()); err == nil { - t.Fatal("want an error from a failing feed") - } - if feed.since != 7 { - t.Fatalf("since = %d after an error, want 7", feed.since) - } - fail.Store(false) - if c, err := feed.Changes(context.Background()); err != nil || !slices.Equal(c.Names, []string{"acme/lib"}) { - t.Errorf("changes after recovery = %+v, %v", c, err) - } -} - -// fakeReplicate serves a CouchDB-style changes feed over seqs 1..len(ids). -type fakeReplicate struct { - mu sync.Mutex - ids []string - limit int - asked []string -} - -func (f *fakeReplicate) ServeHTTP(w http.ResponseWriter, r *http.Request) { - f.mu.Lock() - defer f.mu.Unlock() - w.Header().Set("Content-Type", "application/json") - if r.URL.Path == "/registry/" { - _, _ = fmt.Fprintf(w, `{"db_name":"registry","update_seq":%d}`, len(f.ids)) - return - } - if r.URL.Path != "/registry/_changes" { - w.WriteHeader(http.StatusNotFound) - return - } - since, _ := strconv.Atoi(r.URL.Query().Get("since")) - limit, _ := strconv.Atoi(r.URL.Query().Get("limit")) - f.asked = append(f.asked, r.URL.Query().Get("since")) - if f.limit > 0 { - limit = min(limit, f.limit) - } - var results []string - last := since - for seq := since + 1; seq <= len(f.ids) && len(results) < limit; seq++ { - results = append(results, fmt.Sprintf(`{"seq":%d,"id":%q,"changes":[{"rev":"1-a"}]}`, seq, f.ids[seq-1])) - last = seq - } - _, _ = fmt.Fprintf(w, `{"results":[%s],"last_seq":%d}`, strings.Join(results, ","), last) -} - -func TestNPMFeed(t *testing.T) { - f := &fakeReplicate{ids: []string{"old"}} - srv := httptest.NewServer(f) - defer srv.Close() - feed := NewNPMFeed(srv.Client(), srv.URL+"/registry/_changes") - ctx := context.Background() - - if c, err := feed.Changes(ctx); err != nil || c.Names != nil { - t.Fatalf("first read = %+v, %v; want the start position only", c, err) - } - f.mu.Lock() - f.ids = append(f.ids, "lodash", "@scope/pkg", "_design/app") - f.mu.Unlock() - c, err := feed.Changes(ctx) - if err != nil { - t.Fatal(err) - } - if want := []string{"lodash", "@scope/pkg"}; !slices.Equal(c.Names, want) { - t.Errorf("changes = %v, want %v", c.Names, want) - } - if c, err := feed.Changes(ctx); err != nil || len(c.Names) != 0 { - t.Errorf("changes without news = %+v, %v", c, err) - } - if want := []string{"1", "4"}; !slices.Equal(f.asked, want) { - t.Errorf("asked since %q, want %q", f.asked, want) - } -} - -// TestNPMFeedPages checks that a backlog longer than a page is read page by -// page, and that one read stops after npmMaxPages. -func TestNPMFeedPages(t *testing.T) { - f := &fakeReplicate{limit: npmPageLimit} - srv := httptest.NewServer(f) - defer srv.Close() - feed := &NPMFeed{client: srv.Client(), url: srv.URL + "/registry/_changes", since: []byte("0")} - f.ids = make([]string, npmPageLimit*(npmMaxPages+1)+5) - for i := range f.ids { - f.ids[i] = "pkg-" + strconv.Itoa(i) - } - - c, err := feed.Changes(context.Background()) - if err != nil { - t.Fatal(err) - } - if len(c.Names) != npmPageLimit*npmMaxPages { - t.Errorf("first read returned %d names, want %d", len(c.Names), npmPageLimit*npmMaxPages) - } - c, err = feed.Changes(context.Background()) - if err != nil { - t.Fatal(err) - } - if len(c.Names) != npmPageLimit+5 { - t.Errorf("second read returned %d names, want %d", len(c.Names), npmPageLimit+5) - } -} diff --git a/internal/server/change_feeds.go b/internal/server/change_feeds.go index 059bd2f..556650f 100644 --- a/internal/server/change_feeds.go +++ b/internal/server/change_feeds.go @@ -6,9 +6,10 @@ import ( "net/http" "time" - "github.com/git-pkgs/proxy/internal/changefeed" "github.com/git-pkgs/proxy/internal/database" "github.com/git-pkgs/proxy/internal/metrics" + "github.com/git-pkgs/registries/changes" + "github.com/git-pkgs/registries/client" ) // changeFeedInterval is how often an upstream change feed is read. @@ -20,13 +21,14 @@ const changeFeedMaxBackoff = 15 * time.Minute // startChangeFeeds follows the change feeds the configuration turns on. Each // instance sharing a database follows them itself: the reads are small, and // expiring a row twice does no harm. -func (s *Server) startChangeFeeds(ctx context.Context, client *http.Client, metadataTTL time.Duration) { - var feeds []changefeed.Feed +func (s *Server) startChangeFeeds(ctx context.Context, httpClient *http.Client, metadataTTL time.Duration) { + c := changes.WithClient(client.NewClient(client.WithHTTPClient(httpClient))) + var feeds []changes.Feed if s.cfg.Upstream.NPMChangeFeed { - feeds = append(feeds, changefeed.NewNPMFeed(client, changefeed.NPMFeedURL)) + feeds = append(feeds, changes.NewNPM(c)) } if s.cfg.Upstream.ComposerChangeFeed { - feeds = append(feeds, changefeed.NewComposerFeed(client, changefeed.ComposerFeedURL(s.cfg.Upstream.Composer))) + feeds = append(feeds, changes.NewPackagist(c, changes.WithURL(changes.PackagistFeedURL(s.cfg.Upstream.Composer)))) } if len(feeds) == 0 { return @@ -43,11 +45,15 @@ func (s *Server) startChangeFeeds(ctx context.Context, client *http.Client, meta // followChangeFeed reads feed every interval until ctx ends and expires the // cached metadata it lists. After a failed read it waits twice as long as -// before, up to changeFeedMaxBackoff. -func followChangeFeed(ctx context.Context, db *database.DB, logger *slog.Logger, feed changefeed.Feed, interval time.Duration) { +// before, up to changeFeedMaxBackoff. The position in the feed is kept in +// memory: after a restart the feed is followed from its current position. +func followChangeFeed(ctx context.Context, db *database.DB, logger *slog.Logger, feed changes.Feed, interval time.Duration) { wait := interval + var cursor changes.Cursor for { - if err := readChangeFeed(ctx, db, logger, feed); err != nil { + next, err := readChangeFeed(ctx, db, logger, feed, cursor) + cursor = next + if err != nil { if ctx.Err() != nil { return } @@ -66,29 +72,37 @@ func followChangeFeed(ctx context.Context, db *database.DB, logger *slog.Logger, } } -// readChangeFeed reads feed once and expires the cached metadata of what it -// lists, so the next request for it revalidates upstream. Only rows fetched -// before the read finished are expired: a row fetched after it was fetched -// after every change the read returned. If expiring fails, the feed has -// already moved past the changes, and those rows wait out metadata_ttl. -func readChangeFeed(ctx context.Context, db *database.DB, logger *slog.Logger, feed changefeed.Feed) error { - changes, err := feed.Changes(ctx) +// readChangeFeed reads feed once after cursor, or finds its current position +// when cursor is empty, and expires the cached metadata of what it lists, so +// the next request for it revalidates upstream. Only rows fetched before the +// read finished are expired: a row fetched after it was fetched after every +// change the read returned. It returns the cursor to read on from, which +// moves past the changes only once they are expired, so a failed expiry +// reads them again. +func readChangeFeed(ctx context.Context, db *database.DB, logger *slog.Logger, feed changes.Feed, cursor changes.Cursor) (changes.Cursor, error) { + if cursor == "" { + return feed.Start(ctx) + } + found, next, err := feed.Changes(ctx, cursor) if err != nil { - return err + return cursor, err } cutoff := time.Now() ecosystem := feed.Ecosystem() var expired []string switch { - case changes.All: + case found.All: logger.Info("change feed: upstream asked for a full resync, revalidating all cached metadata", "ecosystem", ecosystem) expired, err = db.ExpireAllMetadataCache(ecosystem, cutoff) - case len(changes.Names) > 0: - expired, err = db.ExpireMetadataCache(ecosystem, changes.Names, cutoff) + case len(found.Names) > 0: + expired, err = db.ExpireMetadataCache(ecosystem, found.Names, cutoff) } for _, name := range expired { logger.Debug("change feed: cached metadata changed upstream", "ecosystem", ecosystem, "key", name) } metrics.RecordMetadataExpiredByChangeFeed(ecosystem, len(expired)) - return err + if err != nil { + return cursor, err + } + return next, nil } diff --git a/internal/server/change_feeds_test.go b/internal/server/change_feeds_test.go index 90441ac..366c1ae 100644 --- a/internal/server/change_feeds_test.go +++ b/internal/server/change_feeds_test.go @@ -9,17 +9,19 @@ import ( "log/slog" "net/http" "net/http/httptest" + "path/filepath" "strings" "sync" "sync/atomic" "testing" "time" - "github.com/git-pkgs/proxy/internal/changefeed" "github.com/git-pkgs/proxy/internal/config" "github.com/git-pkgs/proxy/internal/database" "github.com/git-pkgs/proxy/internal/handler" "github.com/git-pkgs/proxy/internal/storage" + "github.com/git-pkgs/registries/changes" + "github.com/git-pkgs/registries/client" "github.com/git-pkgs/registries/fetch" "github.com/go-chi/chi/v5" ) @@ -186,14 +188,18 @@ func TestChangeFeedRevalidatesChangedMetadata(t *testing.T) { reg.set("/npm/left-pad", npmDoc("1.0.0")) h, db := changeFeedRouter(t, reg) logger := slog.New(slog.NewTextHandler(io.Discard, nil)) - feeds := []changefeed.Feed{ - changefeed.NewComposerFeed(reg.Client(), changefeed.ComposerFeedURL(reg.URL)), - changefeed.NewNPMFeed(reg.Client(), reg.URL+"/replicate/registry/_changes"), + c := changes.WithClient(client.NewClient(client.WithHTTPClient(reg.Client()), client.WithMaxRetries(0))) + feeds := []changes.Feed{ + changes.NewPackagist(c, changes.WithURL(changes.PackagistFeedURL(reg.URL))), + changes.NewNPM(c, changes.WithURL(reg.URL+"/replicate/registry/_changes")), } - for _, feed := range feeds { - if err := readChangeFeed(context.Background(), db, logger, feed); err != nil { - t.Fatalf("starting %s feed: %v", feed.Ecosystem(), err) + cursors := make([]changes.Cursor, len(feeds)) + for i, feed := range feeds { + cursor, err := readChangeFeed(context.Background(), db, logger, feed, "") + if err != nil || cursor == "" { + t.Fatalf("starting %s feed: cursor %q, %v", feed.Ecosystem(), cursor, err) } + cursors[i] = cursor } getBody(t, h, "/composer/p2/acme/lib.json") @@ -209,9 +215,10 @@ func TestChangeFeedRevalidatesChangedMetadata(t *testing.T) { t.Fatal("npm packument revalidated before the feed listed it; the test proves nothing") } - for _, feed := range feeds { - if err := readChangeFeed(context.Background(), db, logger, feed); err != nil { - t.Fatalf("reading %s feed: %v", feed.Ecosystem(), err) + for i, feed := range feeds { + next, err := readChangeFeed(context.Background(), db, logger, feed, cursors[i]) + if err != nil || next == cursors[i] { + t.Fatalf("reading %s feed: cursor %q -> %q, %v", feed.Ecosystem(), cursors[i], next, err) } } @@ -238,11 +245,51 @@ type countingFeed struct { func (f *countingFeed) Ecosystem() string { return "npm" } -func (f *countingFeed) Changes(context.Context) (changefeed.Changes, error) { - if f.reads.Add(1) <= f.failures { - return changefeed.Changes{}, errors.New("feed down") +func (f *countingFeed) Start(context.Context) (changes.Cursor, error) { + return f.read() +} + +func (f *countingFeed) Changes(_ context.Context, cursor changes.Cursor) (changes.Changes, changes.Cursor, error) { + if _, err := f.read(); err != nil { + return changes.Changes{}, cursor, err + } + return changes.Changes{}, cursor, nil +} + +func (f *countingFeed) read() (changes.Cursor, error) { + if f.reads.Add(1) <= f.failures { + return "", errors.New("feed down") + } + return "1", nil +} + +// fixedFeed reports the same names on every read. +type fixedFeed struct{ names []string } + +func (fixedFeed) Ecosystem() string { return "npm" } +func (fixedFeed) Start(context.Context) (changes.Cursor, error) { return "1", nil } +func (f fixedFeed) Changes(context.Context, changes.Cursor) (changes.Changes, changes.Cursor, error) { + return changes.Changes{Names: f.names}, "2", nil +} + +// TestReadChangeFeedKeepsCursorWhenExpiryFails checks that the cursor moves +// past the changes only once they are expired, so changes whose expiry +// failed are read again. +func TestReadChangeFeedKeepsCursorWhenExpiryFails(t *testing.T) { + db, err := database.Create(filepath.Join(t.TempDir(), "proxy.db")) + if err != nil { + t.Fatal(err) + } + logger := slog.New(slog.NewTextHandler(io.Discard, nil)) + feed := fixedFeed{names: []string{"left-pad"}} + + if next, err := readChangeFeed(context.Background(), db, logger, feed, "1"); err != nil || next != "2" { + t.Fatalf("read = %q, %v; want cursor 2", next, err) + } + _ = db.Close() + if next, err := readChangeFeed(context.Background(), db, logger, feed, "1"); err == nil || next != "1" { + t.Errorf("read with a failing expiry = %q, %v; want an error and cursor 1", next, err) } - return changefeed.Changes{}, nil } // TestFollowChangeFeedStopsWithContext checks that a failing feed is retried diff --git a/internal/server/server.go b/internal/server/server.go index a5c7b07..574ce3a 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -234,6 +234,33 @@ func (s *Server) Start(listeners ...net.Listener) error { return s.serve(listener) } +// configureGitCache sets up the cache of upstream git repositories, when any +// are configured. +func (s *Server) configureGitCache(client *http.Client) error { + if len(s.cfg.Upstream.Git) == 0 { + return nil + } + upstreamDirs := make(map[string]string, len(s.cfg.Upstream.Git)) + for name, upstream := range s.cfg.Upstream.Git { + upstreamDirs[name] = gitcache.UpstreamDir(name, upstream.URL) + } + gitCache, err := gitcache.New(gitcache.Options{ + CacheDir: s.cfg.Git.CacheDir, + MaxSize: s.cfg.Git.MaxSizeBytes(), + MaxRepoSize: s.cfg.Git.MaxRepoSizeBytes(), + RefreshWait: s.cfg.Git.RefreshWaitDuration(), + StaleMaxAge: s.cfg.Git.StaleMaxAgeDuration(), + Client: client, + Logger: s.logger, + Upstreams: upstreamDirs, + }) + if err != nil { + return fmt.Errorf("configuring git cache: %w", err) + } + s.git = gitCache + return nil +} + func (s *Server) serve(listener net.Listener) error { metadataClient, artifactClient := s.newUpstreamClients(s.cfg.ParseArtifactTimeout()) @@ -283,25 +310,8 @@ func (s *Server) serve(listener net.Listener) error { proxy.DirectServeBaseURL = s.cfg.Storage.DirectServeBaseURL proxy.StreamArtifacts = !s.cfg.Storage.CacheArtifacts - if len(s.cfg.Upstream.Git) > 0 { - upstreamDirs := make(map[string]string, len(s.cfg.Upstream.Git)) - for name, upstream := range s.cfg.Upstream.Git { - upstreamDirs[name] = gitcache.UpstreamDir(name, upstream.URL) - } - gitCache, err := gitcache.New(gitcache.Options{ - CacheDir: s.cfg.Git.CacheDir, - MaxSize: s.cfg.Git.MaxSizeBytes(), - MaxRepoSize: s.cfg.Git.MaxRepoSizeBytes(), - RefreshWait: s.cfg.Git.RefreshWaitDuration(), - StaleMaxAge: s.cfg.Git.StaleMaxAgeDuration(), - Client: artifactClient, - Logger: s.logger, - Upstreams: upstreamDirs, - }) - if err != nil { - return fmt.Errorf("configuring git cache: %w", err) - } - s.git = gitCache + if err := s.configureGitCache(artifactClient); err != nil { + return err } // Create router with Chi From a06b20fd008bdb344060efdb7a7362207ca8425d Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Fri, 30 Oct 2026 09:10:44 +0000 Subject: [PATCH 499/633] Address Docker guide review points --- docs/docker-builds.md | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/docs/docker-builds.md b/docs/docker-builds.md index 14570fd..8509b9d 100644 --- a/docs/docker-builds.md +++ b/docs/docker-builds.md @@ -53,7 +53,7 @@ unchanged dependency set can reuse Docker's installation layer: ```dockerfile ARG PROXY_URL COPY package.json package-lock.json ./ -RUN npm ci --registry="${PROXY_URL}/npm/" +RUN npm ci --no-audit --no-fund --registry="${PROXY_URL:?PROXY_URL build arg required}/npm/" COPY . . ``` @@ -68,8 +68,8 @@ docker build \ Keep `node_modules` out of the build context so `COPY . .` does not overwrite the installed dependencies. Lockfiles with custom registry or download URLs may still direct requests to those hosts; check proxy's access log when using -private registries or non-registry dependencies. npm audit requests are forwarded -upstream even when the package archives are cached. +private registries or non-registry dependencies. `--no-audit` skips the audit +request, which proxy would otherwise forward upstream. ## Ruby / Bundler @@ -80,21 +80,24 @@ native extensions before this step: ```dockerfile ARG PROXY_URL COPY Gemfile Gemfile.lock ./ -RUN bundle config set --local mirror.https://rubygems.org "${PROXY_URL}/gem" \ +RUN bundle config set --local mirror.https://rubygems.org "${PROXY_URL:?PROXY_URL build arg required}/gem" \ && bundle install \ && bundle config unset --local mirror.https://rubygems.org COPY . . ``` Use the same build command as the npm example. The mirror setting is removed -after installation so it is not retained in the image. Exclude host Bundler +after installation so it is not retained in the image, but the URL still +appears in `docker history` because Docker records the build args used by `RUN` +lines. Keep credentials out of `PROXY_URL`. Exclude host Bundler configuration and installed gems from the build context. Copy any local path dependencies required by the Gemfile before `bundle install`. The RubyGems mirror covers gems fetched from that source. Git dependencies and -other gem sources need their own configuration. Bundler can fall back to the -original source when a mirror is unavailable; this setup does not enforce -network isolation. +other gem sources need their own configuration. Without +`mirror.https://rubygems.org.fallback_timeout`, `bundle install` fails if proxy +is unreachable. Setting it lets Bundler fall back to rubygems.org, so this setup +does not enforce network isolation either way. ## Layer caches and package caches @@ -105,7 +108,7 @@ be rebuilt. For example, the npm step can use: ```dockerfile RUN --mount=type=cache,target=/root/.npm \ - npm ci --registry="${PROXY_URL}/npm/" + npm ci --no-audit --no-fund --registry="${PROXY_URL:?PROXY_URL build arg required}/npm/" ``` Cache mounts are stored by the builder and are excluded from ordinary @@ -118,7 +121,7 @@ To check the configuration, enable proxy's JSONL log with `-access-log` and rebuild with `docker build --no-cache`. Look for requests to `/npm/` or `/gem/`, then repeat and check `proxy_cache_hits_total` on `/metrics`. Cache mounts still retain their contents with `--no-cache`; omit them when testing proxy cache -hits. Upstream metadata revalidation and audit requests can still occur. +hits. Upstream metadata revalidation can still occur. ## Base-image pulls From 77525b0d78cb01ca4bbad82614233ace33a80dc1 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Fri, 30 Oct 2026 10:26:21 +0100 Subject: [PATCH 500/633] Store each snapshot recording at a path of its own Two requests recording the same index version in the same millisecond stored to one path, and the one whose row was not added deleted the blob the other's row points at. The path now ends in a random suffix. Also trims comments that restated the PR, notes in the docs that turning snapshots off leaves them in place, and moves the git cache setup out of serve, which lint found too long after the merge. --- docs/configuration.md | 4 +++ internal/database/metadata_snapshots.go | 11 +++--- internal/handler/apk.go | 3 +- internal/handler/apk_snapshots.go | 38 ++++++++++---------- internal/handler/apk_snapshots_test.go | 37 +++++++++++++++++++ internal/server/server.go | 48 +++++++++++++++---------- 6 files changed, 95 insertions(+), 46 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 1744431..2d7542e 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -662,6 +662,10 @@ Durations take a `d` suffix; empty or `"0"` disables snapshots. Snapshots need `cache_metadata`. Their files live under `_snapshots/` in storage and do not count towards `storage.max_size`. +Turning snapshots off, or `cache_metadata` off, leaves the kept versions in +place. To remove them, first set a short duration such as `"1h"` and let the +hourly prune delete everything but the newest version of each index. + Snapshots keep indexes, not packages. A package listed by an old snapshot can still be evicted by `storage.max_size` or [retention](#retention), and is then only available while upstream still has it. Retention counts from a package's diff --git a/internal/database/metadata_snapshots.go b/internal/database/metadata_snapshots.go index b174aab..95edacc 100644 --- a/internal/database/metadata_snapshots.go +++ b/internal/database/metadata_snapshots.go @@ -7,13 +7,10 @@ import ( "time" ) -// Metadata snapshots keep earlier versions of a metadata document after -// upstream replaced it, so a request for a past date can be answered with the -// document that was current then. Each row is one version, valid from -// valid_from_ms (Unix milliseconds, the time the proxy first stored it) until -// the next row for the same ecosystem and name. - -// MetadataSnapshot is one kept version of a metadata document. +// MetadataSnapshot is one kept version of a metadata document, so a request +// for a past date can be answered with the document that was current then. +// It is valid from ValidFromMS (Unix milliseconds, the time the proxy first +// stored it) until the next snapshot of the same ecosystem and name. type MetadataSnapshot struct { ID int64 `db:"id"` Ecosystem string `db:"ecosystem"` diff --git a/internal/handler/apk.go b/internal/handler/apk.go index 071ae3a..83fee97 100644 --- a/internal/handler/apk.go +++ b/internal/handler/apk.go @@ -53,8 +53,7 @@ type APKHandler struct { proxy *Proxy proxyURL string repositories map[string]string - // snapshots is how long replaced index versions are kept; 0 disables - // snapshots. See WithSnapshots. + // snapshots: see WithSnapshots. snapshots time.Duration } diff --git a/internal/handler/apk_snapshots.go b/internal/handler/apk_snapshots.go index cca04d6..0f29c85 100644 --- a/internal/handler/apk_snapshots.go +++ b/internal/handler/apk_snapshots.go @@ -2,6 +2,7 @@ package handler import ( "context" + "crypto/rand" "io" "net/http" "strconv" @@ -11,23 +12,13 @@ import ( "github.com/git-pkgs/proxy/internal/database" ) -// Dated snapshots of APK indexes. -// -// Alpine mirrors keep only the current build of each package, so once a -// package is upgraded the index no longer lists the old build and -// `apk add name=old-version` fails. With snapshots enabled, every version of an -// index (and of a detached signature) the proxy stores is kept, and -// /apk/{name}@{YYYY-MM-DD}/... serves the version that was current at the end -// of that day (UTC). The kept bytes are the signed upstream bytes, so apk -// verifies them as usual. Packages under a snapshot path are the same -// requests as under the plain path. - // apkSnapshotDateLayout is the date format of a snapshot path segment. const apkSnapshotDateLayout = "2006-01-02" // WithSnapshots keeps earlier versions of the indexes this handler serves and -// makes them available under /apk/{name}@{YYYY-MM-DD}/. keep is how long a -// version is kept after it was replaced; 0 disables snapshots. +// makes them available under /apk/{name}@{YYYY-MM-DD}/, as signed upstream +// bytes that apk verifies as usual. keep is how long a version is kept after +// it was replaced; 0 disables snapshots. func (h *APKHandler) WithSnapshots(keep time.Duration) *APKHandler { h.snapshots = keep return h @@ -125,7 +116,15 @@ func (h *APKHandler) recordSnapshot(ctx context.Context, cacheKey string) { return } - validFrom := time.Now() + h.storeSnapshot(ctx, cacheKey, entry, time.Now()) +} + +// storeSnapshot records the cached document entry as the version current +// from validFrom. Concurrent requests can both get here for the same +// version; each copies to a path of its own, so the one whose row is not +// added only deletes its own copy. +func (h *APKHandler) storeSnapshot(ctx context.Context, cacheKey string, entry *database.MetadataCacheEntry, validFrom time.Time) { + digest := entry.ContentDigest.String storagePath := apkSnapshotStoragePath(cacheKey, digest, validFrom) size, ok := h.copySnapshotBlob(ctx, entry.StoragePath, storagePath, digest) if !ok { @@ -178,10 +177,13 @@ func (h *APKHandler) discardSnapshotBlob(ctx context.Context, path string) { } } -// apkSnapshotStoragePath is where one kept version of the document cached -// under cacheKey is stored. The time it became current keeps a version that -// returns after an upgrade was rolled back apart from its earlier copy. +// apkSnapshotStoragePath returns a new path to store one kept version of the +// document cached under cacheKey at. The time it became current keeps a +// version that returns after an upgrade was rolled back apart from its +// earlier copy, and a random suffix keeps two requests recording the same +// version in the same millisecond apart. func apkSnapshotStoragePath(cacheKey, digest string, validFrom time.Time) string { return "_snapshots/" + apkEcosystem + "/" + cacheKey + "/" + - strconv.FormatInt(validFrom.UnixMilli(), 10) + "-" + strings.TrimPrefix(digest, "sha256:") + strconv.FormatInt(validFrom.UnixMilli(), 10) + "-" + strings.TrimPrefix(digest, "sha256:") + + "-" + rand.Text() } diff --git a/internal/handler/apk_snapshots_test.go b/internal/handler/apk_snapshots_test.go index 50291c1..52b5ff7 100644 --- a/internal/handler/apk_snapshots_test.go +++ b/internal/handler/apk_snapshots_test.go @@ -166,3 +166,40 @@ func TestAPKSnapshots_ConfiguredNameWins(t *testing.T) { expectAPKResponse(t, h, "/alpine@2026-01-10"+apkSnapshotIndexPath, http.StatusOK, "index v1") } + +// TestAPKSnapshots_SameVersionRecordedTwice checks that two requests +// recording the same version in the same millisecond, as coalesced fetches +// after an upstream update can, leave the recorded snapshot readable: the +// request whose row is not added must not delete the other's blob. +func TestAPKSnapshots_SameVersionRecordedTwice(t *testing.T) { + u := newAPKSnapshotUpstream(t) + h, db := newSnapshotAPKHandler(t, u.server.URL) + expectAPKResponse(t, h, "/alpine"+apkSnapshotIndexPath, http.StatusOK, "index v1") + + key := h.metadataCacheKey("alpine", u.server.URL, apkSnapshotIndexPath[1:]) + entry, err := db.GetMetadataCache(apkEcosystem, key) + if err != nil || entry == nil { + t.Fatalf("cached index row = %v, %v", entry, err) + } + if _, err := db.Exec(`DELETE FROM metadata_snapshots`); err != nil { + t.Fatal(err) + } + validFrom := time.Now() + h.storeSnapshot(t.Context(), key, entry, validFrom) + h.storeSnapshot(t.Context(), key, entry, validFrom) + + if n := snapshotCount(t, db); n != 1 { + t.Fatalf("%d snapshots recorded, want 1", n) + } + snap, err := db.LatestMetadataSnapshot(apkEcosystem, key) + if err != nil || snap == nil { + t.Fatalf("latest snapshot = %v, %v", snap, err) + } + body, err := h.proxy.Storage.Open(t.Context(), snap.StoragePath) + if err != nil { + t.Fatalf("recorded snapshot blob unreadable: %v", err) + } + _ = body.Close() + day := validFrom.UTC().Format(apkSnapshotDateLayout) + expectAPKResponse(t, h, "/alpine@"+day+apkSnapshotIndexPath, http.StatusOK, "index v1") +} diff --git a/internal/server/server.go b/internal/server/server.go index ff06d91..41c4c5a 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -234,6 +234,33 @@ func (s *Server) Start(listeners ...net.Listener) error { return s.serve(listener) } +// configureGitCache sets up the cache of upstream git repositories, when any +// are configured. +func (s *Server) configureGitCache(client *http.Client) error { + if len(s.cfg.Upstream.Git) == 0 { + return nil + } + upstreamDirs := make(map[string]string, len(s.cfg.Upstream.Git)) + for name, upstream := range s.cfg.Upstream.Git { + upstreamDirs[name] = gitcache.UpstreamDir(name, upstream.URL) + } + gitCache, err := gitcache.New(gitcache.Options{ + CacheDir: s.cfg.Git.CacheDir, + MaxSize: s.cfg.Git.MaxSizeBytes(), + MaxRepoSize: s.cfg.Git.MaxRepoSizeBytes(), + RefreshWait: s.cfg.Git.RefreshWaitDuration(), + StaleMaxAge: s.cfg.Git.StaleMaxAgeDuration(), + Client: client, + Logger: s.logger, + Upstreams: upstreamDirs, + }) + if err != nil { + return fmt.Errorf("configuring git cache: %w", err) + } + s.git = gitCache + return nil +} + func (s *Server) serve(listener net.Listener) error { metadataClient, artifactClient := s.newUpstreamClients(s.cfg.ParseArtifactTimeout()) @@ -283,25 +310,8 @@ func (s *Server) serve(listener net.Listener) error { proxy.DirectServeBaseURL = s.cfg.Storage.DirectServeBaseURL proxy.StreamArtifacts = !s.cfg.Storage.CacheArtifacts - if len(s.cfg.Upstream.Git) > 0 { - upstreamDirs := make(map[string]string, len(s.cfg.Upstream.Git)) - for name, upstream := range s.cfg.Upstream.Git { - upstreamDirs[name] = gitcache.UpstreamDir(name, upstream.URL) - } - gitCache, err := gitcache.New(gitcache.Options{ - CacheDir: s.cfg.Git.CacheDir, - MaxSize: s.cfg.Git.MaxSizeBytes(), - MaxRepoSize: s.cfg.Git.MaxRepoSizeBytes(), - RefreshWait: s.cfg.Git.RefreshWaitDuration(), - StaleMaxAge: s.cfg.Git.StaleMaxAgeDuration(), - Client: artifactClient, - Logger: s.logger, - Upstreams: upstreamDirs, - }) - if err != nil { - return fmt.Errorf("configuring git cache: %w", err) - } - s.git = gitCache + if err := s.configureGitCache(artifactClient); err != nil { + return err } // Create router with Chi From 1205fa759e60f3c484c2646be7e75ac6b036e1f8 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Fri, 30 Oct 2026 13:54:24 +0100 Subject: [PATCH 501/633] Show versions held back by cooldown on the package page and in the API Part of #617. The package page marks versions in cooldown with when they become available, and lists a version only seen in metadata (no artifact row) only while it is held back. /api/package returns the package's cooldown and available_at on a held-back version. FillVersionPublishedAt stores a publish time without overwriting one, for the cooldown filters to call once they record the versions they hold back. --- docs/configuration.md | 2 +- docs/swagger/docs.go | 4 + docs/swagger/swagger.json | 4 + internal/database/database_test.go | 76 +++++++++++++ internal/database/queries.go | 54 +++++++++ internal/server/api.go | 41 ++++++- internal/server/cooldown_view.go | 45 +++++++- internal/server/cooldown_view_test.go | 104 ++++++++++++++++++ internal/server/dashboard.go | 2 +- internal/server/server.go | 13 ++- internal/server/server_test.go | 5 + .../server/templates/pages/package_show.html | 1 + internal/server/templates_test.go | 11 +- 13 files changed, 352 insertions(+), 10 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 0b9c779..9c67e98 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1125,7 +1125,7 @@ Exact `packages` entries take precedence over patterns. When several patterns ma Resolution order: exact package override, then package pattern, then ecosystem override, then global default. This lets you set a conservative default while exempting trusted package families. -The web UI's dashboard shows the default and the ecosystem overrides, and how many package overrides and patterns there are; their names aren't shown, since the UI has no authentication. A package's page shows the cooldown that applies to it and which kind of rule it comes from; it shows none for ecosystems without cooldown support, and the dashboard marks an ecosystem override for one as not supported. +The web UI's dashboard shows the default and the ecosystem overrides, and how many package overrides and patterns there are; their names aren't shown, since the UI has no authentication. A package's page shows the cooldown that applies to it and which kind of rule it comes from; it shows none for ecosystems without cooldown support, and the dashboard marks an ecosystem override for one as not supported. The version list marks versions held back with when they become available, and leaves out versions the proxy only saw in metadata once they're out of cooldown. `/api/package/{ecosystem}/{name}` returns the package's `cooldown` in the configuration's syntax, and a held-back version has `available_at`. Currently supported for npm, PyPI, pub.dev, Composer, Cargo, NuGet, Conda, RubyGems, Hex, Go, Maven and Helm. Most of these ecosystems include publish timestamps in their metadata; Go and Maven are described below. diff --git a/docs/swagger/docs.go b/docs/swagger/docs.go index e35a86f..57a896f 100644 --- a/docs/swagger/docs.go +++ b/docs/swagger/docs.go @@ -1062,6 +1062,10 @@ const docTemplate = `{ "server.PackageResponse": { "type": "object", "properties": { + "cooldown": { + "description": "Cooldown is the cooldown that applies to the package, in the\nconfiguration's syntax (\"3d\", \"0\").", + "type": "string" + }, "description": { "type": "string" }, diff --git a/docs/swagger/swagger.json b/docs/swagger/swagger.json index 09e4f11..1a95e14 100644 --- a/docs/swagger/swagger.json +++ b/docs/swagger/swagger.json @@ -1055,6 +1055,10 @@ "server.PackageResponse": { "type": "object", "properties": { + "cooldown": { + "description": "Cooldown is the cooldown that applies to the package, in the\nconfiguration's syntax (\"3d\", \"0\").", + "type": "string" + }, "description": { "type": "string" }, diff --git a/internal/database/database_test.go b/internal/database/database_test.go index f8956a6..30c6bff 100644 --- a/internal/database/database_test.go +++ b/internal/database/database_test.go @@ -1164,3 +1164,79 @@ func TestVersionPublishedAtPreserved(t *testing.T) { } }) } + +func TestFillVersionPublishedAt(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + const pkg, held, known = "pkg:npm/leftpad", "pkg:npm/leftpad@2.0.0", "pkg:npm/leftpad@1.0.0" + stored := time.Date(2020, 1, 2, 3, 4, 5, 0, time.UTC) + filtered := time.Date(2021, 6, 7, 8, 9, 10, 0, time.UTC) + + if err := db.SetVersionPublishedAt(known, pkg, stored); err != nil { + t.Fatalf("SetVersionPublishedAt: %v", err) + } + if err := db.UpsertVersion(&Version{PURL: held, PackagePURL: pkg}); err != nil { + t.Fatalf("UpsertVersion: %v", err) + } + + for _, purl := range []string{known, held, "pkg:npm/leftpad@3.0.0"} { + if err := db.FillVersionPublishedAt(purl, pkg, filtered); err != nil { + t.Fatalf("FillVersionPublishedAt(%s): %v", purl, err) + } + } + + want := map[string]time.Time{ + known: stored, + held: filtered, + "pkg:npm/leftpad@3.0.0": filtered, + } + for purl, at := range want { + got, err := db.GetVersionByPURL(purl) + if err != nil || got == nil { + t.Fatalf("GetVersionByPURL(%s): %v", purl, err) + } + if !got.PublishedAt.Valid || !got.PublishedAt.Time.Equal(at) { + t.Errorf("%s PublishedAt = %v, want %v", purl, got.PublishedAt, at) + } + } + }) +} + +func TestGetFetchedVersionPURLs(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + const pkg = "pkg:npm/leftpad" + for _, v := range []string{"1.0.0", "2.0.0", "3.0.0"} { + if err := db.UpsertVersion(&Version{PURL: pkg + "@" + v, PackagePURL: pkg}); err != nil { + t.Fatalf("UpsertVersion: %v", err) + } + } + if err := db.UpsertVersion(&Version{PURL: "pkg:npm/other@1.0.0", PackagePURL: "pkg:npm/other"}); err != nil { + t.Fatalf("UpsertVersion: %v", err) + } + artifacts := []Artifact{ + {VersionPURL: pkg + "@1.0.0", Filename: "leftpad-1.0.0.tgz", UpstreamURL: "https://example.com/1", + StoragePath: sql.NullString{String: "npm/leftpad-1.0.0.tgz", Valid: true}}, + // Evicted: the row stays without a storage path. + {VersionPURL: pkg + "@2.0.0", Filename: "leftpad-2.0.0.tgz", UpstreamURL: "https://example.com/2"}, + {VersionPURL: "pkg:npm/other@1.0.0", Filename: "other-1.0.0.tgz", UpstreamURL: "https://example.com/3"}, + } + for i := range artifacts { + if err := db.UpsertArtifact(&artifacts[i]); err != nil { + t.Fatalf("UpsertArtifact: %v", err) + } + } + + got, err := db.GetFetchedVersionPURLs(pkg) + if err != nil { + t.Fatalf("GetFetchedVersionPURLs: %v", err) + } + want := map[string]bool{pkg + "@1.0.0": true, pkg + "@2.0.0": true} + if len(got) != len(want) { + t.Fatalf("got %v, want %v", got, want) + } + for purl := range want { + if !got[purl] { + t.Errorf("missing %s in %v", purl, got) + } + } + }) +} diff --git a/internal/database/queries.go b/internal/database/queries.go index d7150e8..5ba8edf 100644 --- a/internal/database/queries.go +++ b/internal/database/queries.go @@ -223,6 +223,60 @@ func (db *DB) SetVersionPublishedAt(versionPURL, packagePURL string, publishedAt return nil } +// FillVersionPublishedAt records a version's publish time like +// SetVersionPublishedAt, but keeps a time already stored. The cooldown +// filters call it for each version they hold back, on every uncached +// metadata request, so it writes without reading first. +func (db *DB) FillVersionPublishedAt(versionPURL, packagePURL string, publishedAt time.Time) error { + now := time.Now() + var query string + + if db.dialect == DialectPostgres { + query = ` + INSERT INTO versions (purl, package_purl, published_at, created_at, updated_at) + VALUES ($1, $2, $3, $4, $5) + ON CONFLICT(purl) DO UPDATE SET + published_at = EXCLUDED.published_at, + updated_at = EXCLUDED.updated_at + WHERE versions.published_at IS NULL + ` + } else { + query = ` + INSERT INTO versions (purl, package_purl, published_at, created_at, updated_at) + VALUES (?, ?, ?, ?, ?) + ON CONFLICT(purl) DO UPDATE SET + published_at = excluded.published_at, + updated_at = excluded.updated_at + WHERE versions.published_at IS NULL + ` + } + + _, err := db.Exec(query, versionPURL, packagePURL, publishedAt, now, now) + if err != nil { + return fmt.Errorf("filling version publish time: %w", err) + } + return nil +} + +// GetFetchedVersionPURLs returns the versions of a package that have an +// artifact row, cached now or evicted since. Versions without one were only +// seen in metadata. +func (db *DB) GetFetchedVersionPURLs(packagePURL string) (map[string]bool, error) { + var purls []string + query := db.Rebind(` + SELECT DISTINCT version_purl FROM artifacts + WHERE version_purl IN (SELECT purl FROM versions WHERE package_purl = ?) + `) + if err := db.Select(&purls, query, packagePURL); err != nil { + return nil, fmt.Errorf("listing fetched versions: %w", err) + } + fetched := make(map[string]bool, len(purls)) + for _, p := range purls { + fetched[p] = true + } + return fetched, nil +} + // Artifact queries func (db *DB) GetArtifact(versionPURL, filename string) (*Artifact, error) { diff --git a/internal/server/api.go b/internal/server/api.go index b204367..ca1e626 100644 --- a/internal/server/api.go +++ b/internal/server/api.go @@ -4,6 +4,7 @@ import ( "encoding/json" "net/http" "strings" + "time" shared "github.com/git-pkgs/enrichment" "github.com/git-pkgs/proxy/internal/database" @@ -23,6 +24,8 @@ type APIHandler struct { enrichment *enrichment.Service ecosystems *shared.EcosystemsClient db DBSearcher + // cooldown is nil with cooldown off. + cooldown *cooldownView } // DBSearcher defines the interface for database search operations. @@ -46,6 +49,13 @@ func NewAPIHandler(svc *enrichment.Service, db DBSearcher) *APIHandler { return h } +// newAPIHandler returns the server's API handler, with its cooldown policy. +func (s *Server) newAPIHandler(svc *enrichment.Service) *APIHandler { + h := NewAPIHandler(svc, s.db) + h.cooldown = s.cooldown + return h +} + // PackageResponse contains enriched package metadata. type PackageResponse struct { Ecosystem string `json:"ecosystem"` @@ -57,6 +67,9 @@ type PackageResponse struct { Homepage string `json:"homepage,omitempty"` Repository string `json:"repository,omitempty"` RegistryURL string `json:"registry_url,omitempty"` + // Cooldown is the cooldown that applies to the package, in the + // configuration's syntax ("3d", "0"). + Cooldown string `json:"cooldown,omitempty"` } // VersionResponse contains enriched version metadata. @@ -66,11 +79,16 @@ type VersionResponse struct { Version string `json:"version"` License string `json:"license,omitempty"` PublishedAt string `json:"published_at,omitempty"` + // AvailableAt is when a version held back by cooldown leaves it. + AvailableAt string `json:"available_at,omitempty"` Integrity string `json:"integrity,omitempty"` Yanked bool `json:"yanked"` IsOutdated bool `json:"is_outdated"` } +// apiTimeFormat is the format of the API's timestamps. +const apiTimeFormat = "2006-01-02T15:04:05Z" + // VulnResponse contains vulnerability information. type VulnResponse struct { ID string `json:"id"` @@ -179,6 +197,7 @@ func (h *APIHandler) HandlePackagePath(w http.ResponseWriter, r *http.Request) { Homepage: info.Homepage, Repository: info.Repository, RegistryURL: info.RegistryURL, + Cooldown: h.packageCooldown(ecosystem, fullName), } writeJSON(w, resp) return @@ -212,6 +231,7 @@ func (h *APIHandler) getPackage(w http.ResponseWriter, r *http.Request, ecosyste Homepage: info.Homepage, Repository: info.Repository, RegistryURL: info.RegistryURL, + Cooldown: h.packageCooldown(ecosystem, name), } writeJSON(w, resp) @@ -240,6 +260,7 @@ func (h *APIHandler) getVersion(w http.ResponseWriter, r *http.Request, ecosyste Homepage: result.Package.Homepage, Repository: result.Package.Repository, RegistryURL: result.Package.RegistryURL, + Cooldown: h.packageCooldown(ecosystem, name), } } @@ -253,8 +274,11 @@ func (h *APIHandler) getVersion(w http.ResponseWriter, r *http.Request, ecosyste Yanked: result.Version.Yanked, IsOutdated: result.IsOutdated, } - if !result.Version.PublishedAt.IsZero() { - resp.Version.PublishedAt = result.Version.PublishedAt.Format("2006-01-02T15:04:05Z") + if published := result.Version.PublishedAt; !published.IsZero() { + resp.Version.PublishedAt = published.UTC().Format(apiTimeFormat) + if at := h.cooldown.availableAt(ecosystem, name, published, time.Now()); !at.IsZero() { + resp.Version.AvailableAt = at.UTC().Format(apiTimeFormat) + } } } @@ -272,6 +296,19 @@ func (h *APIHandler) getVersion(w http.ResponseWriter, r *http.Request, ecosyste writeJSON(w, resp) } +// packageCooldown returns the cooldown for a package in the configuration's +// syntax, or "" with cooldown off or not applied to the ecosystem. +func (h *APIHandler) packageCooldown(ecosystem, name string) string { + c := h.cooldown.forPackage(ecosystem, name) + switch { + case c == nil: + return "" + case c.Duration == cooldownOff: + return "0" + } + return c.Duration +} + // HandleVulnsPath dispatches /api/vulns/{ecosystem}/* to the vulns handler. // Supports both {name} and {name}/{version} paths with namespaced package names. func (h *APIHandler) HandleVulnsPath(w http.ResponseWriter, r *http.Request) { diff --git a/internal/server/cooldown_view.go b/internal/server/cooldown_view.go index f244c85..415d708 100644 --- a/internal/server/cooldown_view.go +++ b/internal/server/cooldown_view.go @@ -7,6 +7,7 @@ import ( "github.com/git-pkgs/cooldown" "github.com/git-pkgs/proxy/internal/config" + "github.com/git-pkgs/proxy/internal/database" "github.com/git-pkgs/proxy/internal/handler" "github.com/git-pkgs/proxy/internal/packageurl" ) @@ -30,6 +31,14 @@ type CooldownOverride struct { Unsupported bool } +// PackageVersion is a version on the package page. +type PackageVersion struct { + database.Version + // AvailableAt is when the version leaves cooldown, zero when it is not + // held back. + AvailableAt time.Time +} + // PackageCooldown is the cooldown that applies to one package, and the kind // of rule it comes from. type PackageCooldown struct { @@ -104,6 +113,37 @@ func (v *cooldownView) forPackage(ecosystem, name string) *PackageCooldown { return &PackageCooldown{Duration: formatCooldown(v.policy.For(ecosystem, purl)), Rule: rule} } +// availableAt returns when a version published at publishedAt leaves +// cooldown, or zero when it is not held back at now. +func (v *cooldownView) availableAt(ecosystem, name string, publishedAt, now time.Time) time.Time { + if !v.enabled() || !handler.CooldownSupported(ecosystem) || publishedAt.IsZero() { + return time.Time{} + } + d := v.policy.Evaluate(ecosystem, packageurl.MakeString(ecosystem, name, ""), publishedAt, now) + if d.Allowed { + return time.Time{} + } + return d.AvailableAt +} + +// packageVersions returns the versions to list on a package page. A version +// without an artifact row was only seen in metadata, by a cooldown filter or +// a download check, and is listed only while it is held back. +func (v *cooldownView) packageVersions(pkg *database.Package, versions []database.Version, fetched map[string]bool, now time.Time) []PackageVersion { + rows := make([]PackageVersion, 0, len(versions)) + for _, ver := range versions { + var at time.Time + if ver.PublishedAt.Valid { + at = v.availableAt(pkg.Ecosystem, pkg.Name, ver.PublishedAt.Time, now) + } + if at.IsZero() && !fetched[ver.PURL] { + continue + } + rows = append(rows, PackageVersion{Version: ver, AvailableAt: at}) + } + return rows +} + func formatConfigDuration(s string) string { d, err := cooldown.ParseDuration(s) if err != nil { @@ -112,13 +152,16 @@ func formatConfigDuration(s string) string { return formatCooldown(d) } +// cooldownOff is how formatCooldown writes a zero cooldown. +const cooldownOff = "off" + // formatCooldown writes d as the configuration does: "3d", "12h", "30m", // and "off" for zero. func formatCooldown(d time.Duration) string { const day = 24 * time.Hour switch { case d <= 0: - return "off" + return cooldownOff case d%day == 0: return fmt.Sprintf("%dd", d/day) case d%time.Hour == 0: diff --git a/internal/server/cooldown_view_test.go b/internal/server/cooldown_view_test.go index 13e731c..0a347a8 100644 --- a/internal/server/cooldown_view_test.go +++ b/internal/server/cooldown_view_test.go @@ -1,6 +1,7 @@ package server import ( + "database/sql" "net/http" "net/http/httptest" "slices" @@ -152,3 +153,106 @@ func TestCooldownPages(t *testing.T) { t.Error("package page doesn't show the 7d ecosystem override") } } + +func TestPackagePageWithheldVersions(t *testing.T) { + ts := newTestServer(t) + defer ts.close() + ts.server.cooldown = testCooldownView(t, config.CooldownConfig{Default: "3d"}) + + const pkg = "pkg:npm/lodash" + if err := ts.db.UpsertPackage(&database.Package{PURL: pkg, Ecosystem: "npm", Name: "lodash"}); err != nil { + t.Fatalf("UpsertPackage: %v", err) + } + now := time.Now() + held := now.Add(-time.Hour) + versions := map[string]time.Time{ + "1.0.0": now.Add(-30 * 24 * time.Hour), // fetched + "1.1.0": now.Add(-time.Hour), // fetched while held back + "2.0.0": now.Add(-20 * 24 * time.Hour), // only seen, left cooldown + "2.1.0": held, // only seen, held back + } + for v, at := range versions { + if err := ts.db.SetVersionPublishedAt(pkg+"@"+v, pkg, at); err != nil { + t.Fatalf("SetVersionPublishedAt: %v", err) + } + } + // No publish time: the download checks never stored one. + if err := ts.db.UpsertVersion(&database.Version{PURL: pkg + "@0.9.0", PackagePURL: pkg}); err != nil { + t.Fatalf("UpsertVersion: %v", err) + } + for _, v := range []string{"1.0.0", "1.1.0"} { + if err := ts.db.UpsertArtifact(&database.Artifact{ + VersionPURL: pkg + "@" + v, Filename: "lodash-" + v + ".tgz", UpstreamURL: "https://example.com/" + v, + }); err != nil { + t.Fatalf("UpsertArtifact: %v", err) + } + } + + w := httptest.NewRecorder() + ts.handler.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/ui/package/npm/lodash", nil)) + if w.Code != http.StatusOK { + t.Fatalf("status %d", w.Code) + } + page := w.Body.String() + + for _, v := range []string{"1.0.0", "1.1.0", "2.1.0"} { + if !strings.Contains(page, "lodash@"+v) { + t.Errorf("page is missing %s", v) + } + } + for _, v := range []string{"2.0.0", "0.9.0"} { + if strings.Contains(page, "lodash@"+v) { + t.Errorf("page lists %s, which nobody fetched and cooldown no longer holds back", v) + } + } + until := held.Add(3 * 24 * time.Hour).UTC().Format("2006-01-02 15:04") + if got := strings.Count(page, "held back until"); got != 2 { + t.Errorf("page marks %d versions as held back, want 2 (1.1.0 and 2.1.0)", got) + } + if !strings.Contains(page, "held back until "+until+" UTC") { + t.Errorf("page is missing the availability of 2.1.0, %s", until) + } +} + +func TestPackageVersionsCooldownOff(t *testing.T) { + var off *cooldownView + pkg := &database.Package{PURL: "pkg:npm/lodash", Ecosystem: "npm", Name: "lodash"} + recent := sql.NullTime{Time: time.Now().Add(-time.Hour), Valid: true} + versions := []database.Version{ + {PURL: "pkg:npm/lodash@1.0.0", PublishedAt: recent}, + {PURL: "pkg:npm/lodash@2.0.0", PublishedAt: recent}, + } + got := off.packageVersions(pkg, versions, map[string]bool{"pkg:npm/lodash@1.0.0": true}, time.Now()) + if len(got) != 1 || got[0].PURL != "pkg:npm/lodash@1.0.0" || !got[0].AvailableAt.IsZero() { + t.Errorf("packageVersions with cooldown off = %+v, want only the fetched 1.0.0", got) + } +} + +func TestAPIPackageCooldown(t *testing.T) { + h := &APIHandler{cooldown: testCooldownView(t, config.CooldownConfig{ + Default: "3d", + Packages: map[string]string{"pkg:npm/leftpad": "0"}, + })} + tests := []struct{ ecosystem, name, want string }{ + {"npm", "lodash", "3d"}, + {"npm", "leftpad", "0"}, + {"deb", "curl", ""}, + } + for _, tt := range tests { + if got := h.packageCooldown(tt.ecosystem, tt.name); got != tt.want { + t.Errorf("packageCooldown(%s, %s) = %q, want %q", tt.ecosystem, tt.name, got, tt.want) + } + } + if got := (&APIHandler{}).packageCooldown("npm", "lodash"); got != "" { + t.Errorf("packageCooldown with cooldown off = %q, want empty", got) + } + + now := time.Now() + published := now.Add(-time.Hour) + if got, want := h.cooldown.availableAt("npm", "lodash", published, now), published.Add(72*time.Hour); !got.Equal(want) { + t.Errorf("availableAt = %v, want %v", got, want) + } + if got := h.cooldown.availableAt("npm", "lodash", now.Add(-96*time.Hour), now); !got.IsZero() { + t.Errorf("availableAt after cooldown = %v, want zero", got) + } +} diff --git a/internal/server/dashboard.go b/internal/server/dashboard.go index 7e14af6..08cda58 100644 --- a/internal/server/dashboard.go +++ b/internal/server/dashboard.go @@ -66,7 +66,7 @@ type RegistryConfig struct { type PackageShowData struct { Layout Package *database.Package - Versions []database.Version + Versions []PackageVersion Vulnerabilities []database.Vulnerability LicenseCategory string // Cooldown is nil with cooldown off. diff --git a/internal/server/server.go b/internal/server/server.go index 2f0bd68..5ced4bc 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -372,7 +372,7 @@ func (s *Server) serve(listener net.Listener) error { // API endpoints for enrichment data enrichSvc := enrichment.New(s.logger) - apiHandler := NewAPIHandler(enrichSvc, s.db) + apiHandler := s.newAPIHandler(enrichSvc) r.Get("/api/package/{ecosystem}/*", apiHandler.HandlePackagePath) r.Get("/api/vulns/{ecosystem}/*", apiHandler.HandleVulnsPath) @@ -1163,6 +1163,15 @@ func (s *Server) showPackage(w http.ResponseWriter, r *http.Request, ecosystem, s.logger.Error("failed to get versions", "error", err) versions = []database.Version{} } + fetched, err := s.db.GetFetchedVersionPURLs(pkg.PURL) + if err != nil { + // List every version rather than hide fetched ones. + s.logger.Error("failed to get fetched versions", "error", err) + fetched = make(map[string]bool, len(versions)) + for _, v := range versions { + fetched[v.PURL] = true + } + } vulns, err := s.db.GetVulnerabilitiesForPackage(ecosystem, name) if err != nil { @@ -1173,7 +1182,7 @@ func (s *Server) showPackage(w http.ResponseWriter, r *http.Request, ecosystem, data := PackageShowData{ Layout: s.layoutFor(r), Package: pkg, - Versions: versions, + Versions: s.cooldown.packageVersions(pkg, versions, fetched, time.Now()), Vulnerabilities: vulns, LicenseCategory: categorizeLicense(pkg.License), Cooldown: s.cooldown.forPackage(pkg.Ecosystem, pkg.Name), diff --git a/internal/server/server_test.go b/internal/server/server_test.go index 292643c..aad9037 100644 --- a/internal/server/server_test.go +++ b/internal/server/server_test.go @@ -1098,6 +1098,11 @@ func TestVersionShowPage_PlusInVersion(t *testing.T) { }); err != nil { t.Fatalf("failed to upsert version: %v", err) } + if err := ts.db.UpsertArtifact(&database.Artifact{ + VersionPURL: versionPURL, Filename: "nmap.deb", UpstreamURL: "https://example.com/nmap.deb", + }); err != nil { + t.Fatalf("failed to upsert artifact: %v", err) + } // The package page must link to and display the decoded version. req := httptest.NewRequest("GET", "/ui/package/deb/nmap", nil) diff --git a/internal/server/templates/pages/package_show.html b/internal/server/templates/pages/package_show.html index b0dd29a..977ff6e 100644 --- a/internal/server/templates/pages/package_show.html +++ b/internal/server/templates/pages/package_show.html @@ -69,6 +69,7 @@ {{.DisplayPURL}} {{if .Yanked}}yanked{{end}} + {{if not .AvailableAt.IsZero}}held back until {{.AvailableAt.UTC.Format "2006-01-02 15:04"}} UTC{{end}}
{{if .PublishedAt.Valid}}{{.PublishedAt.Time.Format "2006-01-02"}}{{end}}
diff --git a/internal/server/templates_test.go b/internal/server/templates_test.go index 708f02c..c6fa492 100644 --- a/internal/server/templates_test.go +++ b/internal/server/templates_test.go @@ -8,6 +8,7 @@ import ( "regexp" "strings" "testing" + "time" "github.com/git-pkgs/proxy/internal/database" ) @@ -122,8 +123,12 @@ func TestTemplatesRenderAllPages(t *testing.T) { LatestVersion: sql.NullString{String: "4.17.21", Valid: true}, License: sql.NullString{String: "MIT", Valid: true}, }, - Versions: []database.Version{ - {PURL: "pkg:npm/lodash@4.17.21", PackagePURL: "pkg:npm/lodash"}, + Versions: []PackageVersion{ + {Version: database.Version{PURL: "pkg:npm/lodash@4.17.21", PackagePURL: "pkg:npm/lodash"}}, + { + Version: database.Version{PURL: "pkg:npm/lodash@4.17.22", PackagePURL: "pkg:npm/lodash"}, + AvailableAt: time.Date(2026, 1, 2, 3, 4, 0, 0, time.UTC), + }, }, Vulnerabilities: []database.Vulnerability{}, LicenseCategory: "permissive", @@ -134,7 +139,7 @@ func TestTemplatesRenderAllPages(t *testing.T) { Ecosystem: "npm", Name: "minimal", }, - Versions: []database.Version{}, + Versions: []PackageVersion{}, Vulnerabilities: []database.Vulnerability{}, LicenseCategory: "unknown", }}, From 5b57994c09ffccc09d43a26eddf9b4e0e48578ba Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Fri, 30 Oct 2026 14:14:10 +0100 Subject: [PATCH 502/633] Stream npm packuments through storage when rewriting them Closes #525. With metadata caching and the rewrite cache on, a packument is streamed from upstream into storage and rewritten from there in two passes: the first collects the time map, dist-tags and version names, the second copies each kept version and rewrites its tarball URL. Neither the packument nor its rewrite is held in memory uncompressed, so metadata_max_size no longer limits packuments. Both passes check the stored bytes against the digest in the cache row, so a document replaced in between is not rewritten from mixed bytes. Documents the stream can't handle (abbreviated, no versions object) and rewrite errors fall back to the in-memory rewrite. --- docs/architecture.md | 2 +- docs/configuration.md | 2 +- internal/handler/conda.go | 3 +- internal/handler/handler.go | 87 ++++++--- internal/handler/metadata_stored.go | 207 ++++++++++++++++++++ internal/handler/npm.go | 80 ++++++++ internal/handler/npm_stream_test.go | 243 +++++++++++++++++++++++ internal/handler/rewrite_cache.go | 94 ++++++++- internal/metadata/npm/imports_test.go | 1 + internal/metadata/npm/stream.go | 270 ++++++++++++++++++++++++++ internal/metadata/npm/stream_test.go | 162 ++++++++++++++++ 11 files changed, 1111 insertions(+), 40 deletions(-) create mode 100644 internal/handler/metadata_stored.go create mode 100644 internal/handler/npm_stream_test.go create mode 100644 internal/metadata/npm/stream.go create mode 100644 internal/metadata/npm/stream_test.go diff --git a/docs/architecture.md b/docs/architecture.md index b6c654f..c859f33 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -316,7 +316,7 @@ HTTP protocol handlers for each registry type. ### `internal/metadata/npm` -The npm metadata rewrite, separated from the handler so it can run where the proxy's infrastructure can't, such as WASI. It takes a packument's bytes and everything else as explicit inputs (proxy URL, cooldown policy, the time cooldown is evaluated at, denied versions, pushed versions) and returns the rewritten document with the versions it kept. It filters versions by cooldown and the denylist, moves `latest` off a withheld version, lists pushed versions in place of upstream ones, and points each tarball URL at the proxy, copying everything else byte for byte. It does no I/O and never reads the clock; callbacks report what it did, which the handler logs. It imports no database, storage, metrics, HTTP or logging packages, which a test checks. +The npm metadata rewrite, separated from the handler so it can run where the proxy's infrastructure can't, such as WASI. It takes a packument's bytes and everything else as explicit inputs (proxy URL, cooldown policy, the time cooldown is evaluated at, denied versions, pushed versions) and returns the rewritten document with the versions it kept. It filters versions by cooldown and the denylist, moves `latest` off a withheld version, lists pushed versions in place of upstream ones, and points each tarball URL at the proxy, copying everything else byte for byte. It does no I/O and never reads the clock; callbacks report what it did, which the handler logs. `ScanPackument` and `RewriteStream` do the same rewrite over a reader in two passes, the first collecting the time map, dist-tags and version names and the second copying each kept version, so the handler can rewrite a packument from storage without holding it in memory. It imports no database, storage, metrics, HTTP or logging packages, which a test checks. ### `internal/jsonscan` diff --git a/docs/configuration.md b/docs/configuration.md index 7f62207..6adfa38 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1400,7 +1400,7 @@ When upstream is unreachable and the cached entry is past its TTL, the proxy ser ### Metadata size limit -Upstream metadata responses are buffered in memory before being rewritten and served. `metadata_max_size` caps that buffer to protect against OOM from a misbehaving upstream. Some npm packages with thousands of versions (for example `renovate`) exceed the 100 MB default, so raise this if you see `metadata response exceeds size limit` in the logs. Conda `repodata.json` with cooldown enabled is filtered as it streams and isn't held in memory decompressed. With `cache_metadata` the limit applies to the compressed document as it does without cooldown; one over it is filtered as it streams from the upstream and not cached. +Upstream metadata responses are buffered in memory before being rewritten and served. `metadata_max_size` caps that buffer to protect against OOM from a misbehaving upstream. Some npm packages with thousands of versions (for example `renovate`) exceed the 100 MB default, so raise this if you see `metadata response exceeds size limit` in the logs. Conda `repodata.json` with cooldown enabled is filtered as it streams and isn't held in memory decompressed. With `cache_metadata` the limit applies to the compressed document as it does without cooldown; one over it is filtered as it streams from the upstream and not cached. With `cache_metadata` and the metadata rewrite cache on, an npm packument is streamed from the upstream into storage and rewritten from there in two passes, so it isn't held in memory and the limit doesn't apply to it; the rewritten copy is kept gzip-compressed. Requests for a single version (`//`) still read the packument into memory. ```yaml metadata_max_size: "100MB" # default diff --git a/internal/handler/conda.go b/internal/handler/conda.go index 3cbaf93..93ef0f0 100644 --- a/internal/handler/conda.go +++ b/internal/handler/conda.go @@ -409,7 +409,8 @@ func (h *CondaHandler) rememberRepodata(cacheKey, digest string, filter *repodat h.repodataStates[cacheKey] = repodataState{digest: digest, etag: etag, until: filter.until} } -// countingWriter records whether anything has reached the response yet. +// countingWriter counts the bytes written through it, such as whether +// anything has reached the response yet. type countingWriter struct { w io.Writer n int64 diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 1bc5ef7..b2db3c5 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -1796,6 +1796,10 @@ type metadataResult struct { // fromCache is set when the body is a stored copy within its TTL, read // without asking upstream. fromCache bool + // entry is the cache row of the stored document, for a lookup that + // leaves the document in storage (see storedMetadataDocument). body is + // nil then. + entry *database.MetadataCacheEntry } // describedBy returns res with the digest that entry, the cache row its body @@ -1935,24 +1939,9 @@ type upstreamMetadata struct { // alongside for the caller to store and replay. An empty acceptEncoding leaves Go to negotiate and // decompress transparently. func (p *Proxy) fetchUpstreamMetadata(ctx context.Context, upstreamURL string, entry *database.MetadataCacheEntry, accept, acceptEncoding string) (*upstreamMetadata, error) { - req, err := http.NewRequestWithContext(ctx, http.MethodGet, upstreamURL, nil) + resp, err := p.requestUpstreamMetadata(ctx, upstreamURL, entry, accept, acceptEncoding) if err != nil { - return nil, fmt.Errorf("creating request: %w", err) - } - req.Header.Set("Accept", accept) - if acceptEncoding != "" { - req.Header.Set(headerAcceptEncoding, acceptEncoding) - } - - if entry != nil && entry.ETag.Valid { - req.Header.Set("If-None-Match", entry.ETag.String) - } else if entry != nil && entry.LastModified.Valid { - req.Header.Set("If-Modified-Since", entry.LastModified.Time.UTC().Format(http.TimeFormat)) - } - - resp, err := p.HTTPClient.Do(req) - if err != nil { - return nil, fmt.Errorf("fetching metadata: %w", err) + return nil, err } defer func() { _ = resp.Body.Close() }() @@ -1980,12 +1969,8 @@ func (p *Proxy) fetchUpstreamMetadata(ctx context.Context, upstreamURL string, e return meta, nil } - if resp.StatusCode == http.StatusNotFound { - body, _ := io.ReadAll(io.LimitReader(resp.Body, upstreamNotFoundBodyLimit)) - return nil, &upstreamNotFoundError{body: body} - } - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("upstream returned %d", resp.StatusCode) + if err := upstreamMetadataStatus(resp); err != nil { + return nil, err } // ContentLength is -1 when the transport decompresses the body, and @@ -2010,6 +1995,44 @@ func (p *Proxy) fetchUpstreamMetadata(ctx context.Context, upstreamURL string, e return meta, nil } +// requestUpstreamMetadata sends the request for a metadata document, +// revalidating entry when it is given. +func (p *Proxy) requestUpstreamMetadata(ctx context.Context, upstreamURL string, entry *database.MetadataCacheEntry, accept, acceptEncoding string) (*http.Response, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, upstreamURL, nil) + if err != nil { + return nil, fmt.Errorf("creating request: %w", err) + } + req.Header.Set("Accept", accept) + if acceptEncoding != "" { + req.Header.Set(headerAcceptEncoding, acceptEncoding) + } + + if entry != nil && entry.ETag.Valid { + req.Header.Set("If-None-Match", entry.ETag.String) + } else if entry != nil && entry.LastModified.Valid { + req.Header.Set("If-Modified-Since", entry.LastModified.Time.UTC().Format(http.TimeFormat)) + } + + resp, err := p.HTTPClient.Do(req) + if err != nil { + return nil, fmt.Errorf("fetching metadata: %w", err) + } + return resp, nil +} + +// upstreamMetadataStatus returns the error for a metadata response other +// than 200. Callers handle a 304 they asked for before. +func upstreamMetadataStatus(resp *http.Response) error { + switch resp.StatusCode { + case http.StatusOK: + return nil + case http.StatusNotFound: + body, _ := io.ReadAll(io.LimitReader(resp.Body, upstreamNotFoundBodyLimit)) + return &upstreamNotFoundError{body: body} + } + return fmt.Errorf("upstream returned %d", resp.StatusCode) +} + // refreshMetadataBlob records that upstream answered 304 for the cached // document meta was read from, by moving the row's fetched_at on. The bytes in // storage are unchanged, so they are neither written nor hashed again. It @@ -2037,14 +2060,22 @@ func (p *Proxy) refreshMetadataBlob(ecosystem, cacheKey string, meta *upstreamMe // database. It returns the digest it recorded for the bytes, or "" when they // were not cached. func (p *Proxy) cacheMetadataBlob(ctx context.Context, ecosystem, cacheKey, storagePath string, meta *upstreamMetadata) string { + digest, _ := p.storeMetadataBlob(ctx, ecosystem, cacheKey, storagePath, bytes.NewReader(meta.body), meta) + return digest +} + +// storeMetadataBlob is cacheMetadataBlob for a document read from body, +// which meta describes. It also returns the error that kept it from caching +// the document. +func (p *Proxy) storeMetadataBlob(ctx context.Context, ecosystem, cacheKey, storagePath string, body io.Reader, meta *upstreamMetadata) (string, error) { if p.DB == nil || p.Storage == nil { - return "" + return "", errMetadataNotStored } - size, hash, err := p.Storage.Store(ctx, storagePath, bytes.NewReader(meta.body)) + size, hash, err := p.Storage.Store(ctx, storagePath, body) if err != nil { p.Logger.Warn("failed to cache metadata", "ecosystem", ecosystem, "key", cacheKey, "error", err) - return "" + return "", err } var digest string if hash != "" { @@ -2074,9 +2105,9 @@ func (p *Proxy) cacheMetadataBlob(ctx context.Context, ecosystem, cacheKey, stor if delErr := p.Storage.Delete(ctx, storagePath); delErr != nil { p.Logger.Warn("failed to discard metadata blob", "ecosystem", ecosystem, "key", cacheKey, "error", delErr) } - return "" + return "", err } - return digest + return digest, nil } // currentMetadataEntry re-reads the metadata cache row and returns it, or diff --git a/internal/handler/metadata_stored.go b/internal/handler/metadata_stored.go new file mode 100644 index 0000000..e46239f --- /dev/null +++ b/internal/handler/metadata_stored.go @@ -0,0 +1,207 @@ +package handler + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "hash" + "io" + "net/http" + "time" + + "github.com/git-pkgs/proxy/internal/database" + "github.com/git-pkgs/proxy/internal/metrics" +) + +// A metadata document can be rewritten from storage instead of from memory: +// storedMetadataDocument streams it from upstream into storage, and the +// rewrite reads it back as often as it needs. Nothing holds the whole +// document, so its size is not limited by metadata_max_size or by memory. + +// errMetadataNotStored is returned when metadata is not cached in storage, so +// a document has to be read into memory. +var errMetadataNotStored = errors.New("metadata is not cached in storage") + +// errStoredMetadataChanged is returned when a stored document no longer has +// the digest its cache row records: a fetch replaced it while it was read. +var errStoredMetadataChanged = errors.New("stored metadata changed while it was read") + +// storedMetadataDocument returns the cache row of a metadata document stored +// uncompressed with its digest: the stored copy while it is within its TTL, +// or else the document fetched from upstream and stored, or the stored copy +// when upstream fails. The document is streamed from upstream into storage +// without being held in memory, so a fetch takes no metadata_max_size limit. +// Concurrent calls for one document share a fetch, and instances sharing the +// database take turns as for other metadata. +// +// It returns errMetadataNotStored when the proxy doesn't cache metadata. +func (p *Proxy) storedMetadataDocument(ctx context.Context, ecosystem, cacheKey, upstreamURL, accept string) (*database.MetadataCacheEntry, error) { + if !p.CacheMetadata || p.DB == nil || p.Storage == nil || p.MetadataTTL <= 0 { + return nil, errMetadataNotStored + } + if containsPathTraversal(cacheKey) { + return nil, fmt.Errorf("invalid cache key: %q", cacheKey) + } + if entry, fresh := p.storedMetadataEntryState(ctx, ecosystem, cacheKey); fresh { + metrics.RecordMetadataCacheHit(ecosystem, int(entry.Size.Int64)) + p.recordMetadataRead(entry) + return entry, nil + } + p.recordMetadataCacheMiss(ecosystem) + + key := metadataCoalesceKey(ecosystem, cacheKey, upstreamURL, accept, "", false) + "\x00stored" + res, _ := p.coalesceMetadata(ctx, key, func(fetchCtx context.Context) metadataResult { + entry, fresh := p.storedMetadataEntryState(fetchCtx, ecosystem, cacheKey) + if fresh { + return metadataResult{entry: entry} + } + release, outcome, err := p.awaitFetchLease(fetchCtx, metrics.CacheKindMetadata, key, func() bool { + entry, fresh = p.storedMetadataEntryState(fetchCtx, ecosystem, cacheKey) + return fresh + }) + if err != nil { + return metadataResult{err: err} + } + switch outcome { + case leaseStored: + return metadataResult{entry: entry} + case leaseFailed: + // Serve the stored copy, as the holder did, rather than trying + // the failing upstream again. + p.reloadUpstreamHolds() + if usableStoredMetadata(entry) { + return p.staleStoredMetadata(ecosystem, cacheKey, entry, errFetchFailedElsewhere) + } + case leaseFetch: + } + res := metadataResult{err: errSharedFetchAbandoned} + defer func() { release(res.err != nil || res.stale) }() + res = p.fetchStoredMetadata(fetchCtx, ecosystem, cacheKey, upstreamURL, accept, entry) + return res + }) + return res.entry, res.err +} + +// storedMetadataEntryState reads the cache row of a document and reports +// whether it is within its TTL and usable by storedMetadataDocument. +func (p *Proxy) storedMetadataEntryState(ctx context.Context, ecosystem, cacheKey string) (*database.MetadataCacheEntry, bool) { + entry, _ := p.DB.GetMetadataCache(ecosystem, cacheKey) + entry = p.migrateLegacyMetadataEntry(ctx, ecosystem, cacheKey, entry) + fresh := usableStoredMetadata(entry) && entry.FetchedAt.Valid && time.Since(entry.FetchedAt.Time) < p.MetadataTTL + return entry, fresh +} + +// usableStoredMetadata reports whether entry describes a document stored +// uncompressed with its digest, which a rewrite can read back and check. +func usableStoredMetadata(entry *database.MetadataCacheEntry) bool { + return entry != nil && entry.ContentDigest.Valid && entry.ContentEncoding.String == "" +} + +// fetchStoredMetadata fetches a document from upstream into storage, +// revalidating entry when there is one, and falls back to entry when +// upstream fails. +func (p *Proxy) fetchStoredMetadata(ctx context.Context, ecosystem, cacheKey, upstreamURL, accept string, entry *database.MetadataCacheEntry) metadataResult { + revalidate := entry + if !usableStoredMetadata(entry) { + revalidate = nil + } + stored, err := p.storeUpstreamMetadata(ctx, ecosystem, cacheKey, upstreamURL, accept, revalidate) + if errors.Is(err, errStale304) { + stored, err = p.storeUpstreamMetadata(ctx, ecosystem, cacheKey, upstreamURL, accept, nil) + } + if err == nil { + return metadataResult{entry: stored} + } + if !usableStoredMetadata(entry) || errors.Is(err, ErrUpstreamNotFound) { + return metadataResult{err: err} + } + return p.staleStoredMetadata(ecosystem, cacheKey, entry, err) +} + +// staleStoredMetadata falls back to the stored document entry describes after +// the upstream fetch failed with err. +func (p *Proxy) staleStoredMetadata(ecosystem, cacheKey string, entry *database.MetadataCacheEntry, err error) metadataResult { + p.Logger.Warn("upstream metadata fetch failed, serving cached copy", + "ecosystem", ecosystem, "key", cacheKey, "error", err) + entry = p.currentMetadataEntry(ecosystem, cacheKey, entry) + if !usableStoredMetadata(entry) { + return metadataResult{err: fmt.Errorf("upstream failed and no usable cached metadata: %w", err)} + } + return metadataResult{entry: entry, stale: true} +} + +// storeUpstreamMetadata fetches a document and streams it into storage, and +// returns its new cache row. On a 304 for entry it only moves the row's +// fetched_at on; errStale304 means the row was replaced meanwhile and the +// caller should fetch without validators. +func (p *Proxy) storeUpstreamMetadata(ctx context.Context, ecosystem, cacheKey, upstreamURL, accept string, entry *database.MetadataCacheEntry) (*database.MetadataCacheEntry, error) { + resp, err := p.requestUpstreamMetadata(ctx, upstreamURL, entry, accept, "") + if err != nil { + return nil, err + } + defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode == http.StatusNotModified && entry != nil { + refreshed, err := p.DB.RefreshMetadataCache(ecosystem, cacheKey, entry.ContentDigest.String, time.Now()) + if err != nil || !refreshed { + return nil, errStale304 + } + return p.currentMetadataEntry(ecosystem, cacheKey, entry), nil + } + if err := upstreamMetadataStatus(resp); err != nil { + return nil, err + } + + meta := &upstreamMetadata{ + contentType: resp.Header.Get(headerContentType), + etag: resp.Header.Get(headerETag), + } + if meta.contentType == "" { + meta.contentType = contentTypeJSON + } + if lm := resp.Header.Get(headerLastModified); lm != "" { + meta.lastModified, _ = http.ParseTime(lm) + } + // Go decompresses the body, since no Accept-Encoding was set, so it is + // stored uncompressed. + if _, err := p.storeMetadataBlob(ctx, ecosystem, cacheKey, metadataStoragePath(ecosystem, cacheKey), resp.Body, meta); err != nil { + return nil, err + } + stored, err := p.DB.GetMetadataCache(ecosystem, cacheKey) + if err != nil || !usableStoredMetadata(stored) { + return nil, fmt.Errorf("reading the stored metadata row: %w", errors.Join(err, errMetadataNotStored)) + } + return stored, nil +} + +// openStoredMetadata opens the document entry describes. Reading it to the +// end fails with errStoredMetadataChanged when its bytes don't have the +// digest entry records. +func (p *Proxy) openStoredMetadata(ctx context.Context, entry *database.MetadataCacheEntry) (io.ReadCloser, error) { + r, err := p.Storage.Open(ctx, entry.StoragePath) + if err != nil { + return nil, err + } + return &digestReader{r: r, h: sha256.New(), want: entry.ContentDigest.String}, nil +} + +// digestReader checks at the end of its reader that what it read has the +// digest want, in the form the metadata cache records it. +type digestReader struct { + r io.ReadCloser + h hash.Hash + want string +} + +func (d *digestReader) Read(b []byte) (int, error) { + n, err := d.r.Read(b) + d.h.Write(b[:n]) + if errors.Is(err, io.EOF) && "sha256:"+hex.EncodeToString(d.h.Sum(nil)) != d.want { + return n, errStoredMetadataChanged + } + return n, err +} + +func (d *digestReader) Close() error { return d.r.Close() } diff --git a/internal/handler/npm.go b/internal/handler/npm.go index 7263b3b..99c12ae 100644 --- a/internal/handler/npm.go +++ b/internal/handler/npm.go @@ -255,6 +255,10 @@ func (h *NPMHandler) handlePackageMetadata(w http.ResponseWriter, r *http.Reques return } + if !versionLookup && h.serveStreamedPackument(w, r, packageName, upstreamURL, accept, metadataCacheKey) { + return + } + var validate func([]byte) error if !versionLookup { validate = h.packumentValidator() @@ -308,6 +312,82 @@ func (h *NPMHandler) handlePackageMetadata(w http.ResponseWriter, r *http.Reques serveRewrittenMetadata(w, r, rewritten) } +// serveStreamedPackument serves a packument rewritten from storage: the +// document is streamed from upstream into storage, then read twice, once to +// collect the time map and version names cooldown and the denylist decide +// on, and once to copy each kept version. registry.npmjs.org writes +// dist-tags before versions and time after them, so one pass can't do it. +// Neither the packument nor its rewrite is held in memory uncompressed, and +// packuments larger than metadata_max_size can be served. +// +// It reports false when it leaves the request to the in-memory rewrite: +// without metadata or rewrite caching, for a document it can't stream, such +// as an abbreviated packument where the full one is required, and when the +// rewrite fails. The document is stored by then, so the in-memory path reads +// it from storage. +func (h *NPMHandler) serveStreamedPackument(w http.ResponseWriter, r *http.Request, packageName, upstreamURL, accept, metadataCacheKey string) bool { + if h.proxy.rewrites == nil { + return false + } + entry, err := h.proxy.storedMetadataDocument(r.Context(), "npm", metadataCacheKey, upstreamURL, accept) + switch { + case err == nil: + case errors.Is(err, errMetadataNotStored): + return false + case errors.Is(err, ErrUpstreamNotFound): + JSONError(w, http.StatusNotFound, ErrPackageNotFound.Error()) + return true + case writeUpstreamHeld(w, err): + return true + default: + h.proxy.Logger.Error("failed to fetch npm metadata", "error", err) + JSONError(w, http.StatusBadGateway, "failed to fetch from upstream") + return true + } + + fullRequired := h.packumentValidator() != nil + options := h.npmRewriteOptions(packageName, nil) + rewritten, err := h.proxy.streamedRewrite(r.Context(), "npm", h.proxyURL, metadataCacheKey, entry, func(open func() (io.ReadCloser, error), out io.Writer) error { + var scan *npm.Scan + err := readStored(open, func(in io.Reader) (err error) { + scan, err = npm.ScanPackument(in) + return err + }) + if err != nil { + return err + } + if fullRequired && scan.Abbreviated() { + return npm.ErrNotStreamable + } + return readStored(open, func(in io.Reader) error { + _, err := npm.RewriteStream(packageName, scan, in, out, options) + return err + }) + }) + if err != nil { + if r.Context().Err() != nil { + return true // the client left while waiting on a shared rewrite + } + if !errors.Is(err, npm.ErrNotStreamable) { + h.proxy.Logger.Warn("failed to rewrite stored npm metadata, rewriting in memory", + "package", packageName, "error", err) + } + return false + } + serveRewrittenMetadata(w, r, rewritten) + return true +} + +// readStored opens a stored document and reads it with read. +func readStored(open func() (io.ReadCloser, error), read func(io.Reader) error) error { + in, err := open() + if err != nil { + return err + } + defer func() { _ = in.Close() }() + return read(in) +} + // packumentAccept returns the Accept header packuments are requested with. // It prefers the smaller abbreviated packument format but includes // application/json as a fallback so upstreams that reject the abbreviated type diff --git a/internal/handler/npm_stream_test.go b/internal/handler/npm_stream_test.go new file mode 100644 index 0000000..565f062 --- /dev/null +++ b/internal/handler/npm_stream_test.go @@ -0,0 +1,243 @@ +package handler + +import ( + "bytes" + "compress/gzip" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/git-pkgs/cooldown" +) + +// bigPackument returns a packument laid out as registry.npmjs.org writes +// them, dist-tags before versions and time after, with n old versions padded +// so the document is large, and a latest published an hour ago. +func bigPackument(n int) string { + var b strings.Builder + b.WriteString(`{"_id":"big","name":"big","dist-tags":{"latest":"9.0.0","old":"1.0.0"},"versions":{`) + times := []string{`"created":"2015-01-01T00:00:00Z"`} + for i := range n { + v := fmt.Sprintf("1.0.%d", i) + if i == 0 { + v = "1.0.0" + } + fmt.Fprintf(&b, `%q:{"name":"big","version":%q,"readme":%q,"dist":{"tarball":"https://registry.npmjs.org/big/-/big-%s.tgz"}},`, + v, v, strings.Repeat("x", 512), v) + times = append(times, fmt.Sprintf(`%q:"2016-01-01T00:%02d:%02dZ"`, v, i/60%60, i%60)) + } + recent := time.Now().Add(-time.Hour).UTC().Format(time.RFC3339) + b.WriteString(`"9.0.0":{"name":"big","version":"9.0.0","dist":{"tarball":"https://registry.npmjs.org/big/-/big-9.0.0.tgz"}}},`) + times = append(times, `"9.0.0":"`+recent+`"`) + b.WriteString(`"time":{` + strings.Join(times, ",") + `},"readme":"big"}`) + return b.String() +} + +type streamUpstream struct { + *httptest.Server + body atomic.Value // string + fail atomic.Bool + requests atomic.Int32 +} + +func newStreamUpstream(t *testing.T, body string) *streamUpstream { + t.Helper() + u := &streamUpstream{} + u.body.Store(body) + u.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + u.requests.Add(1) + switch { + case u.fail.Load(): + http.Error(w, "down", http.StatusBadGateway) + case r.URL.Path == "/big" && r.Header.Get("If-None-Match") == `"v1"`: + w.WriteHeader(http.StatusNotModified) + case r.URL.Path == "/big": + w.Header().Set("Content-Type", "application/json") + w.Header().Set("ETag", `"v1"`) + _, _ = io.WriteString(w, u.body.Load().(string)) + default: + http.NotFound(w, r) + } + })) + t.Cleanup(u.Close) + return u +} + +func streamProxy(t *testing.T, u *streamUpstream) (*Proxy, *mockStorage, http.Handler) { + t.Helper() + proxy, _, store, _ := setupTestProxy(t) + proxy.HTTPClient = u.Client() + proxy.CacheMetadata = true + proxy.MetadataTTL = time.Hour + proxy.SetMetadataRewriteCacheSize(64 << 20) + proxy.Cooldown = &cooldown.Config{Default: "3d"} + return proxy, store, NewNPMHandler(proxy, "http://proxy.example", u.URL).Routes() +} + +type packumentDoc struct { + DistTags map[string]string `json:"dist-tags"` + Versions map[string]json.RawMessage `json:"versions"` + Time map[string]string `json:"time"` +} + +func decodeStreamed(t *testing.T, rec *httptest.ResponseRecorder) packumentDoc { + t.Helper() + if rec.Code != http.StatusOK { + t.Fatalf("status %d: %s", rec.Code, rec.Body.String()) + } + body := rec.Body.Bytes() + if rec.Header().Get("Content-Encoding") == "gzip" { + zr, err := gzip.NewReader(bytes.NewReader(body)) + if err != nil { + t.Fatal(err) + } + if body, err = io.ReadAll(zr); err != nil { + t.Fatal(err) + } + } + var doc packumentDoc + if err := json.Unmarshal(body, &doc); err != nil { + t.Fatalf("response is not JSON: %v", err) + } + return doc +} + +// TestNPMStreamedPackument requests a packument larger than +// metadata_max_size with cooldown on, its time map after its versions and +// its latest inside cooldown. The in-memory rewrite can't read it at all. +func TestNPMStreamedPackument(t *testing.T) { + const versions = 400 + body := bigPackument(versions) + u := newStreamUpstream(t, body) + proxy, _, handler := streamProxy(t, u) + proxy.MetadataMaxSize = int64(len(body) / 4) + + for _, header := range []http.Header{nil, {"Accept-Encoding": {"gzip"}}} { + doc := decodeStreamed(t, serve(handler, "/big", header)) + if len(doc.Versions) != versions { + t.Errorf("%d versions, want %d without 9.0.0", len(doc.Versions), versions) + } + if _, ok := doc.Versions["9.0.0"]; ok { + t.Error("9.0.0 is listed although it is inside cooldown") + } + if _, ok := doc.Time["9.0.0"]; ok { + t.Error("time still lists 9.0.0") + } + if doc.DistTags["latest"] != fmt.Sprintf("1.0.%d", versions-1) || doc.DistTags["old"] != "1.0.0" { + t.Errorf("dist-tags = %v, want latest moved to the newest version left", doc.DistTags) + } + if !strings.Contains(string(doc.Versions["1.0.0"]), `"tarball":"http://proxy.example/npm/big/-/big-1.0.0.tgz"`) { + t.Errorf("tarball not rewritten: %s", doc.Versions["1.0.0"]) + } + } + if n := u.requests.Load(); n != 1 { + t.Errorf("upstream asked %d times, want once", n) + } + + // Without the rewrite cache the in-memory path runs, which the size + // limit stops. + proxy.SetMetadataRewriteCacheSize(0) + if rec := serve(handler, "/big", nil); rec.Code == http.StatusOK { + t.Error("in-memory rewrite served a packument over metadata_max_size; the test no longer covers the stream") + } +} + +// TestNPMStreamedPackumentMatchesInMemory checks that both paths serve the +// same document and ETag. +func TestNPMStreamedPackumentMatchesInMemory(t *testing.T) { + body := bigPackument(50) + u := newStreamUpstream(t, body) + _, _, streamed := streamProxy(t, u) + inMemory, _, handler := streamProxy(t, u) + inMemory.CacheMetadata = false + + a, b := serve(streamed, "/big", nil), serve(handler, "/big", nil) + if a.Code != http.StatusOK || b.Code != http.StatusOK { + t.Fatalf("status %d and %d", a.Code, b.Code) + } + var ca, cb bytes.Buffer + if err := json.Compact(&ca, a.Body.Bytes()); err != nil { + t.Fatal(err) + } + if err := json.Compact(&cb, b.Body.Bytes()); err != nil { + t.Fatal(err) + } + if !bytes.Equal(ca.Bytes(), cb.Bytes()) { + t.Error("streamed and in-memory rewrites differ") + } +} + +// TestNPMStreamedPackumentStale serves the stored copy when upstream fails +// after the TTL, and 404 for a package upstream doesn't have. +func TestNPMStreamedPackumentStale(t *testing.T) { + u := newStreamUpstream(t, bigPackument(10)) + proxy, _, handler := streamProxy(t, u) + decodeStreamed(t, serve(handler, "/big", nil)) + + proxy.MetadataTTL = time.Nanosecond + proxy.SetMetadataRewriteCacheSize(64 << 20) // drop the cached rewrite + u.fail.Store(true) + if doc := decodeStreamed(t, serve(handler, "/big", nil)); len(doc.Versions) != 10 { + t.Errorf("stale copy has %d versions, want 10", len(doc.Versions)) + } + + u.fail.Store(false) + if rec := serve(handler, "/missing", nil); rec.Code != http.StatusNotFound { + t.Errorf("missing package: status %d, want 404", rec.Code) + } +} + +// TestNPMStreamedPackumentChangedInStorage checks that a document replaced +// in storage between the two passes isn't rewritten from mixed bytes. +func TestNPMStreamedPackumentChangedInStorage(t *testing.T) { + u := newStreamUpstream(t, bigPackument(10)) + proxy, store, _ := streamProxy(t, u) + entry, err := proxy.storedMetadataDocument(context.Background(), "npm", "big", u.URL+"/big", contentTypeJSON) + if err != nil { + t.Fatal(err) + } + store.mu.Lock() + store.files[entry.StoragePath] = []byte(bigPackument(11)) + store.mu.Unlock() + + r, err := proxy.openStoredMetadata(context.Background(), entry) + if err != nil { + t.Fatal(err) + } + defer func() { _ = r.Close() }() + if _, err := io.ReadAll(r); !errors.Is(err, errStoredMetadataChanged) { + t.Errorf("read error %v, want errStoredMetadataChanged", err) + } +} + +// TestNPMStreamedPackumentRevalidates checks that a 304 after the TTL keeps +// the stored document and its cached rewrite, without storing it again. +func TestNPMStreamedPackumentRevalidates(t *testing.T) { + u := newStreamUpstream(t, bigPackument(10)) + proxy, store, handler := streamProxy(t, u) + first := serve(handler, "/big", nil) + decodeStreamed(t, first) + + proxy.MetadataTTL = time.Nanosecond + second := serve(handler, "/big", nil) + decodeStreamed(t, second) + if second.Header().Get("ETag") != first.Header().Get("ETag") { + t.Errorf("ETag changed from %s to %s", first.Header().Get("ETag"), second.Header().Get("ETag")) + } + if n := u.requests.Load(); n != 2 { + t.Errorf("upstream asked %d times, want 2", n) + } + store.mu.Lock() + defer store.mu.Unlock() + if store.stores != 1 { + t.Errorf("document stored %d times, want once", store.stores) + } +} diff --git a/internal/handler/rewrite_cache.go b/internal/handler/rewrite_cache.go index fd455a7..a624dba 100644 --- a/internal/handler/rewrite_cache.go +++ b/internal/handler/rewrite_cache.go @@ -18,6 +18,7 @@ import ( "sync" "time" + "github.com/git-pkgs/proxy/internal/database" "github.com/git-pkgs/proxy/internal/metrics" ) @@ -253,6 +254,18 @@ func (c *rewriteCache) rewrite(ctx context.Context, key string, in []byte, expir if c == nil { return rewriteWithETag(in, rewrite) } + return c.produce(ctx, key, expiry, func() (rewrittenMetadata, error) { + m, err := rewriteWithETag(in, rewrite) + if err == nil { + m.gzipped = gzipIfSmaller(m.plain) + } + return m, err + }) +} + +// produce is rewrite for a document that build makes with its validators +// and gzip copy. +func (c *rewriteCache) produce(ctx context.Context, key string, expiry func(start time.Time) time.Time, build func() (rewrittenMetadata, error)) (rewrittenMetadata, error) { c.mu.Lock() if e, ok := c.lookup(key, c.clock()); ok { @@ -288,10 +301,7 @@ func (c *rewriteCache) rewrite(ctx context.Context, key string, in []byte, expir c.mu.Unlock() close(f.done) }() - f.m, f.err = rewriteWithETag(in, rewrite) - if f.err == nil { - f.m.gzipped = gzipIfSmaller(f.m.plain) - } + f.m, f.err = build() return f.m, f.err } @@ -391,6 +401,60 @@ func (p *Proxy) cachedRewrite(ctx context.Context, ecosystem, proxyURL, name str return m, err } +// streamedRewrite rewrites a document stored in storage, through the +// proxy's rewrite cache, without holding the document or its rewrite in +// memory uncompressed. rewrite is given a way to open the stored document, +// as often as it needs to read it, and writes the rewritten document to w. +// The result is cached under the same key cachedRewrite uses for the +// document, and expires the same way with cooldown on. +// +// It returns errMetadataNotStored without the rewrite cache, which the +// result has to be kept in. +func (p *Proxy) streamedRewrite(ctx context.Context, ecosystem, proxyURL, name string, entry *database.MetadataCacheEntry, rewrite func(open func() (io.ReadCloser, error), w io.Writer) error) (rewrittenMetadata, error) { + if p.rewrites == nil { + return rewrittenMetadata{}, errMetadataNotStored + } + var expiry func(time.Time) time.Time + if p.cooldownEnabled() { + rec := p.watchCooldown() + defer p.unwatchCooldown(rec) + expiry = func(start time.Time) time.Time { return p.cooldownExpiry(rec, start) } + } + key := rewriteCacheKeyForDigest(ecosystem, proxyURL, name, entry.ContentDigest.String) + m, err := p.rewrites.produce(ctx, key, expiry, func() (rewrittenMetadata, error) { + // The rewrite runs to completion for the cache even when the + // caller leaves. + readCtx := context.WithoutCancel(ctx) + return gzipRewrite(func(w io.Writer) error { + return rewrite(func() (io.ReadCloser, error) { return p.openStoredMetadata(readCtx, entry) }, w) + }) + }) + if err == nil { + m.lastModified = p.metadataDate(ecosystem, proxyURL, name, m.etag) + } + return m, err +} + +// gzipRewrite runs write and keeps what it writes gzip-compressed only, with +// the ETag and length of the uncompressed output. +func gzipRewrite(write func(w io.Writer) error) (rewrittenMetadata, error) { + var buf bytes.Buffer + zw := gzip.NewWriter(&buf) + sum := sha256.New() + n := &countingWriter{w: io.Discard} + if err := write(io.MultiWriter(zw, sum, n)); err != nil { + return rewrittenMetadata{}, err + } + if err := zw.Close(); err != nil { + return rewrittenMetadata{}, err + } + return rewrittenMetadata{ + metadataValidators: metadataValidators{etag: `"` + hex.EncodeToString(sum.Sum(nil)) + `"`}, + gzipped: buf.Bytes(), + size: int(n.n), + }, nil +} + // uncachedRewrite gives a rewrite made outside the cache its validators: an // ETag of the output itself and, as cachedRewrite does, a date that replicas // agree on. @@ -501,16 +565,28 @@ func serveRewrittenMetadataAs(w http.ResponseWriter, r *http.Request, m rewritte w.WriteHeader(http.StatusNotModified) return } - body := m.gzipped - if !compressed { - body = m.content() - } w.Header().Set(headerContentType, contentType) if compressed { w.Header().Set(headerContentEncoding, "gzip") + w.WriteHeader(http.StatusOK) + _, _ = w.Write(m.gzipped) + return + } + if m.plain == nil && m.gzipped != nil { + // Decompressed as it is sent, rather than into a copy of the + // whole document. + zr, err := gzip.NewReader(bytes.NewReader(m.gzipped)) + if err != nil { + http.Error(w, "failed to read cached metadata", http.StatusInternalServerError) + return + } + w.Header().Set(headerContentLength, strconv.Itoa(m.size)) + w.WriteHeader(http.StatusOK) + _, _ = io.Copy(w, zr) + return } w.WriteHeader(http.StatusOK) - _, _ = w.Write(body) + _, _ = w.Write(m.plain) } // ensureVary adds field to the Vary header unless it is listed already. diff --git a/internal/metadata/npm/imports_test.go b/internal/metadata/npm/imports_test.go index 0f04bc1..19f828c 100644 --- a/internal/metadata/npm/imports_test.go +++ b/internal/metadata/npm/imports_test.go @@ -23,6 +23,7 @@ func TestPortableImports(t *testing.T) { "github.com/Masterminds/semver/v3", "github.com/git-pkgs/cooldown", "github.com/git-pkgs/proxy/internal/jsonscan", + "github.com/git-pkgs/proxy/internal/jsonstream", "github.com/git-pkgs/proxy/internal/metadata/npm", "github.com/git-pkgs/proxy/internal/packageurl", "github.com/git-pkgs/purl", diff --git a/internal/metadata/npm/stream.go b/internal/metadata/npm/stream.go new file mode 100644 index 0000000..cc70c52 --- /dev/null +++ b/internal/metadata/npm/stream.go @@ -0,0 +1,270 @@ +package npm + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "io" + "maps" + "reflect" + "slices" + + "github.com/git-pkgs/proxy/internal/jsonstream" +) + +// ErrNotStreamable is returned by RewriteStream for a document it leaves to +// Rewrite: one without a versions object, such as a single version's +// metadata, or a rewrite that lists pushed versions. +var ErrNotStreamable = errors.New("npm metadata can't be rewritten as a stream") + +// Scan is what a first pass over a packument learns: the version names, and +// the time map and dist-tags the filters work on. It holds no version +// entries, so it stays small whatever the size of the packument. +type Scan struct { + versions []string + versionsObject bool + time any + distTags any + // occurrences counts each filtered key, so the second pass can drop the + // earlier duplicates, as Rewrite does. + occurrences map[string]int + hasModified bool +} + +// Abbreviated reports whether the scanned document is an abbreviated +// packument, as the package function Abbreviated does for a document in +// memory. +func (s *Scan) Abbreviated() bool { + return s.hasModified && s.occurrences["time"] == 0 +} + +// ScanPackument reads a packument once, keeping only what RewriteStream +// needs to decide which versions remain. Version entries are read one at a +// time and dropped. +func ScanPackument(r io.Reader) (*Scan, error) { + s := &Scan{occurrences: map[string]int{}} + dec := json.NewDecoder(r) + if err := expectDelim(dec, '{'); err != nil { + return nil, err + } + for dec.More() { + key, err := objectKey(dec) + if err != nil { + return nil, err + } + switch key { + case "versions": + s.occurrences[key]++ + s.versions, s.versionsObject, err = objectKeys(dec) + case "time": + s.occurrences[key]++ + s.time = nil + err = dec.Decode(&s.time) + case "dist-tags": + s.occurrences[key]++ + s.distTags = nil + err = dec.Decode(&s.distTags) + default: + if key == "modified" { + s.hasModified = true + } + err = skipValue(dec) + } + if err != nil { + return nil, err + } + } + if err := expectDelim(dec, '}'); err != nil { + return nil, err + } + if _, err := dec.Token(); err != io.EOF { + if err == nil { + err = errors.New("unexpected data after the top-level object") + } + return nil, err + } + return s, nil +} + +// RewriteStream is Rewrite for a packument too large to hold in memory. scan +// is the result of ScanPackument over the same bytes r reads again; the +// caller checks they are the same. The rewritten document is written to w +// with the bytes of untouched values copied as they are, and its output is +// the same as Rewrite's for the same document and options. It returns the +// versions it kept. For a document without a versions object, or options +// listing pushed versions, it returns ErrNotStreamable before writing. +func RewriteStream(packageName string, scan *Scan, r io.Reader, w io.Writer, options Options) ([]string, error) { + if !scan.versionsObject || len(options.LocalVersions) > 0 { + return nil, ErrNotStreamable + } + h := &transformer{options} + + versions := make(map[string]any, len(scan.versions)) + for _, v := range scan.versions { + versions[v] = nil + } + metadata := map[string]any{} + timeMap := scan.objectValue(metadata, "time", scan.time) + distTags := scan.objectValue(metadata, "dist-tags", scan.distTags) + timeEntries := len(timeMap) + originalTags := maps.Clone(distTags) + + h.applyCooldownFiltering(metadata, versions, packageName) + h.applyDenylistFiltering(metadata, versions) + + sw := &streamWriter{ + transformer: h, + packageName: packageName, + scan: scan, + versions: versions, + timeMap: timeMap, + distTags: distTags, + timeChanged: timeMap != nil && len(timeMap) != timeEntries, + tagsChanged: distTags != nil && !reflect.DeepEqual(distTags, originalTags), + seen: map[string]int{}, + } + if err := jsonstream.CopyObject(json.NewDecoder(r), w, sw.member); err != nil { + return nil, err + } + h.tarballsRewritten(sw.rewritten) + return slices.Collect(maps.Keys(versions)), nil +} + +// streamWriter writes the second pass of RewriteStream from the filtered +// state the first pass left. +type streamWriter struct { + *transformer + packageName string + scan *Scan + versions map[string]any + timeMap, distTags map[string]any + timeChanged, tagsChanged bool + seen map[string]int + entry bytes.Buffer + rewritten int +} + +// member writes one top-level member. Members the filters don't touch are +// left unconsumed, so CopyObject copies them as they are. +func (s *streamWriter) member(m *jsonstream.Member) error { + total, filtered := s.scan.occurrences[m.Key] + if !filtered { + return nil + } + s.seen[m.Key]++ + if s.seen[m.Key] < total { + _, err := m.Raw() // an earlier duplicate, dropped + return err + } + switch { + case m.Key == "versions": + return m.Object(s.version) + case m.Key == "time" && s.timeChanged: + return m.Object(s.time) + case m.Key == "dist-tags" && s.tagsChanged: + if _, err := m.Raw(); err != nil { + return err + } + encoded, err := json.Marshal(s.distTags) + if err != nil { + return err + } + return m.Write(encoded) + } + return nil +} + +// version writes a kept version with its tarball pointing at the proxy. +func (s *streamWriter) version(v *jsonstream.Member) error { + raw, err := v.Raw() + if err != nil { + return err + } + if _, keep := s.versions[v.Key]; !keep { + return nil + } + s.entry.Reset() + if s.writeNPMVersion(&s.entry, s.packageName, v.Key, raw) { + s.rewritten++ + } + return v.Write(s.entry.Bytes()) +} + +// time writes a time entry the filters kept. +func (s *streamWriter) time(t *jsonstream.Member) error { + if _, keep := s.timeMap[t.Key]; keep { + return t.Copy() + } + _, err := t.Raw() + return err +} + +// objectValue records value under name in metadata, as Rewrite's +// decodeObject does, and returns it when it is an object. +func (s *Scan) objectValue(metadata map[string]any, name string, value any) map[string]any { + if s.occurrences[name] == 0 { + return nil + } + metadata[name] = value + object, _ := value.(map[string]any) + return object +} + +func expectDelim(dec *json.Decoder, want json.Delim) error { + tok, err := dec.Token() + if err != nil { + return err + } + if tok != want { + return fmt.Errorf("npm metadata: expected %v, got %v", want, tok) + } + return nil +} + +func objectKey(dec *json.Decoder) (string, error) { + tok, err := dec.Token() + if err != nil { + return "", err + } + key, ok := tok.(string) + if !ok { + return "", fmt.Errorf("npm metadata: object key is %T, not a string", tok) + } + return key, nil +} + +// objectKeys reads a value and returns its keys when it is an object. +func objectKeys(dec *json.Decoder) ([]string, bool, error) { + tok, err := dec.Token() + if err != nil { + return nil, false, err + } + delim, ok := tok.(json.Delim) + if !ok { + return nil, false, nil // a scalar, read whole by Token + } + var keys []string + for dec.More() { + if delim == '{' { + key, err := objectKey(dec) + if err != nil { + return nil, false, err + } + keys = append(keys, key) + } + if err := skipValue(dec); err != nil { + return nil, false, err + } + } + if _, err := dec.Token(); err != nil { + return nil, false, err + } + return keys, delim == '{', nil +} + +// skipValue reads the next value without keeping it. +func skipValue(dec *json.Decoder) error { + var raw json.RawMessage + return dec.Decode(&raw) +} diff --git a/internal/metadata/npm/stream_test.go b/internal/metadata/npm/stream_test.go new file mode 100644 index 0000000..ef2c954 --- /dev/null +++ b/internal/metadata/npm/stream_test.go @@ -0,0 +1,162 @@ +package npm + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "slices" + "strings" + "testing" + "time" + + "github.com/git-pkgs/cooldown" +) + +// rewriteStream runs both passes of RewriteStream over body. +func rewriteStream(packageName string, body []byte, options Options) ([]byte, []string, error) { + scan, err := ScanPackument(bytes.NewReader(body)) + if err != nil { + return nil, nil, err + } + var out bytes.Buffer + kept, err := RewriteStream(packageName, scan, bytes.NewReader(body), &out, options) + return out.Bytes(), kept, err +} + +// TestRewriteStreamMatchesRewrite checks that the streamed rewrite writes the +// same bytes and keeps the same versions as Rewrite, over documents with the +// filtered members in either order, duplicates, and values Rewrite copies +// verbatim. +func TestRewriteStreamMatchesRewrite(t *testing.T) { + now := time.Date(2026, 10, 18, 12, 0, 0, 0, time.UTC) + ts := func(age time.Duration) string { return now.Add(-age).Format(time.RFC3339) } + version := func(v string) string { + return fmt.Sprintf(`%q:{"version":%q,"dist":{"tarball":"https://registry.npmjs.org/demo/-/demo-%s.tgz"}}`, v, v, v) + } + versions := `"versions":{` + version("1.0.0") + `,` + version("2.0.0") + `,` + version("3.0.0") + `}` + times := `"time":{"created":"` + ts(1000*time.Hour) + `","1.0.0":"` + ts(900*time.Hour) + `","2.0.0":"` + ts(800*time.Hour) + `","3.0.0":"` + ts(time.Hour) + `"}` + tags := `"dist-tags":{"latest":"3.0.0","beta":"2.0.0","next":"1.0.0"}` + + docs := map[string]string{ + "testdata": string(packument), + "tags, versions, time": `{"name":"demo",` + tags + `,` + versions + `,` + times + `}`, + "time first": `{` + times + `,"name":"demo",` + versions + `,` + tags + `}`, + "no time": `{"name":"demo",` + tags + `,` + versions + `}`, + "time null": `{"time":null,` + tags + `,` + versions + `}`, + "tags not an object": `{"dist-tags":"x",` + versions + `,` + times + `}`, + "duplicates": `{"versions":{` + version("9.0.0") + `},` + tags + `,"time":{"9.0.0":"` + ts(time.Hour) + `"},` + + versions + `,` + times + `,"dist-tags":{"latest":"3.0.0"}}`, + "verbatim values": `{"_big":12345678901234567890,"readme":"&","nested":{"a":[1, 2 ,{"b":null}]},` + versions + `}`, + "unusual entries": `{"versions":{"1.0.0":"not an object","2.0.0":{"dist":"x"},"3.0.0":{"dist":{"tarball":7}},` + + `"4.0.0":{"dist":{"tarball":"https://registry.npmjs.org/demo/-/renamed.tgz"}},"5.0.0":{"dist":{}}}}`, + "whitespace": "{ \"name\" : \"demo\" ,\n\t\"versions\" : {\n \"1.0.0\" : { \"dist\" : { \"tarball\" : \"https://registry.npmjs.org/demo/-/demo-1.0.0.tgz\" } } } }", + "empty versions": `{"name":"demo","versions":{}}`, + } + options := map[string]Options{ + "plain": {ProxyURL: "http://proxy"}, + "cooldown": {ProxyURL: "http://proxy", Cooldown: &cooldown.Config{Default: "3d"}, EvaluatedAt: now}, + "denylist": {ProxyURL: "http://proxy", DeniedVersions: map[string]bool{"2.0.0": true}}, + "both": {ProxyURL: "http://proxy", Cooldown: &cooldown.Config{Default: "3d"}, EvaluatedAt: now, + DeniedVersions: map[string]bool{"1.0.0": true}}, + } + for docName, doc := range docs { + for optName, opts := range options { + t.Run(docName+"/"+optName, func(t *testing.T) { + want, wantKept, err := Rewrite("demo", []byte(doc), opts) + if err != nil { + t.Fatalf("Rewrite: %v", err) + } + got, gotKept, err := rewriteStream("demo", []byte(doc), opts) + if err != nil { + t.Fatalf("RewriteStream: %v", err) + } + // Rewrite keeps the whitespace inside the members of a + // filtered time map, which the stream doesn't see. + if !bytes.Equal(compact(t, got), compact(t, want)) { + t.Errorf("RewriteStream =\n%s\nRewrite =\n%s", got, want) + } + if docName == "verbatim values" && !bytes.Contains(got, []byte(`"_big":12345678901234567890,"readme":"&","nested":{"a":[1, 2 ,{"b":null}]}`)) { + t.Errorf("values not copied verbatim: %s", got) + } + slices.Sort(gotKept) + slices.Sort(wantKept) + if !slices.Equal(gotKept, wantKept) { + t.Errorf("kept %v, Rewrite kept %v", gotKept, wantKept) + } + }) + } + } +} + +func compact(t *testing.T, doc []byte) []byte { + t.Helper() + var out bytes.Buffer + if err := json.Compact(&out, doc); err != nil { + t.Fatalf("output is not JSON: %v\n%s", err, doc) + } + return out.Bytes() +} + +func TestRewriteStreamCallbacks(t *testing.T) { + var filtered []string + var rewritten []int + options := Options{ProxyURL: "http://proxy", Cooldown: &cooldown.Config{Default: "7d"}, + EvaluatedAt: time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC), + OnCooldownFilter: func(version, _ string) { filtered = append(filtered, version) }, + OnTarballsRewritten: func(count int) { rewritten = append(rewritten, count) }, + } + if _, _, err := rewriteStream("@example/widgets", packument, options); err != nil { + t.Fatal(err) + } + if !slices.Equal(filtered, []string{"3.0.0"}) || !slices.Equal(rewritten, []int{2}) { + t.Errorf("filtered %v, rewritten %v; want [3.0.0] and [2]", filtered, rewritten) + } +} + +func TestRewriteStreamNotStreamable(t *testing.T) { + for name, tc := range map[string]struct { + doc string + options Options + }{ + "single version": {`{"name":"demo","version":"1.0.0","dist":{"tarball":"t"}}`, Options{}}, + "versions array": {`{"name":"demo","versions":[{"a":1}]}`, Options{}}, + "versions null": {`{"name":"demo","versions":null}`, Options{}}, + "pushed versions": {`{"versions":{}}`, Options{LocalVersions: []LocalVersion{{Version: "1.0.0", Entry: []byte(`{}`)}}}}, + } { + t.Run(name, func(t *testing.T) { + var out bytes.Buffer + scan, err := ScanPackument(strings.NewReader(tc.doc)) + if err != nil { + t.Fatal(err) + } + if _, err := RewriteStream("demo", scan, strings.NewReader(tc.doc), &out, tc.options); !errors.Is(err, ErrNotStreamable) { + t.Errorf("err = %v, want ErrNotStreamable", err) + } + if out.Len() != 0 { + t.Errorf("wrote %q before refusing", out.String()) + } + }) + } +} + +func TestScanPackument(t *testing.T) { + for _, body := range []string{`{"versions": {}`, `[1]`, `{"versions": {}} trailing`, `{"a":}`, ``} { + if _, err := ScanPackument(strings.NewReader(body)); err == nil { + t.Errorf("ScanPackument(%q): expected an error", body) + } + } + for body, want := range map[string]bool{ + `{"name":"demo","modified":"2024-01-01T00:00:00Z","versions":{}}`: true, + `{"name":"demo","modified":"2024-01-01T00:00:00Z","time":{},"versions":{}}`: false, + `{"name":"demo","versions":{}}`: false, + } { + scan, err := ScanPackument(strings.NewReader(body)) + if err != nil { + t.Fatal(err) + } + if scan.Abbreviated() != want || Abbreviated([]byte(body)) != want { + t.Errorf("Abbreviated(%s) = %v, want %v", body, scan.Abbreviated(), want) + } + } +} From ce9ab47eace39b8eee4fa4391fa2a68dda602238 Mon Sep 17 00:00:00 2001 From: simon chrzanowski Date: Fri, 30 Oct 2026 14:59:52 +0100 Subject: [PATCH 503/633] Add --latest to mirror only the newest version (#631) * Add --latest to mirror only the newest version An unversioned PURL mirrors every version the upstream lists. With --latest, or "latest": true in a mirror API job, only the newest is mirrored: the highest by the ecosystem's version ordering, without yanked or retracted versions and, unless there is nothing else, pre-releases. * Pick the latest version with registries.FetchLatestVersion --latest now uses the shared policy from git-pkgs/registries: the version the registry advertises as latest where it has one, otherwise the newest active version by publish time. Deprecated versions are left out too, and newestVersion and the direct vers dependency go away. --- README.md | 3 + cmd/proxy/main.go | 6 +- docs/configuration.md | 15 ++++ internal/mirror/job.go | 6 +- internal/mirror/latest_test.go | 127 +++++++++++++++++++++++++++++++++ internal/mirror/source.go | 43 +++++++++-- internal/mirror/versions.go | 41 +++++++++-- 7 files changed, 229 insertions(+), 12 deletions(-) create mode 100644 internal/mirror/latest_test.go diff --git a/README.md b/README.md index 0d111ce..a324c53 100644 --- a/README.md +++ b/README.md @@ -1156,6 +1156,9 @@ proxy mirror pkg:npm/lodash@4.17.21 pkg:cargo/serde@1.0.0 # Mirror all versions of a package proxy mirror pkg:npm/lodash +# Mirror only the newest version of a package +proxy mirror --latest pkg:npm/lodash + # Mirror from a CycloneDX or SPDX SBOM proxy mirror --sbom sbom.cdx.json diff --git a/cmd/proxy/main.go b/cmd/proxy/main.go index 21b6388..824f2e2 100644 --- a/cmd/proxy/main.go +++ b/cmd/proxy/main.go @@ -442,6 +442,7 @@ func runMirror() { concurrency := fs.Int("concurrency", 4, "Number of parallel downloads") //nolint:mnd // default concurrency dryRun := fs.Bool("dry-run", false, "Show what would be mirrored without downloading") platforms := fs.String("platform", "", "Comma-separated os/arch[/variant] list limiting which platforms of container images are mirrored") + latest := fs.Bool("latest", false, "Mirror only the newest version of PURLs without a version") fs.Usage = func() { fmt.Fprintf(os.Stderr, "git-pkgs proxy - Pre-populate cache\n\n") @@ -450,6 +451,7 @@ func runMirror() { fmt.Fprintf(os.Stderr, " proxy mirror pkg:npm/lodash@4.17.21\n") fmt.Fprintf(os.Stderr, " proxy mirror --sbom sbom.cdx.json\n") fmt.Fprintf(os.Stderr, " proxy mirror pkg:npm/lodash # all versions\n") + fmt.Fprintf(os.Stderr, " proxy mirror --latest pkg:npm/lodash # newest version\n") fmt.Fprintf(os.Stderr, " proxy mirror --platform linux/amd64 pkg:docker/library/alpine@3.20\n\n") fmt.Fprintf(os.Stderr, "Flags:\n") fs.PrintDefaults() @@ -475,9 +477,9 @@ func runMirror() { fmt.Fprintf(os.Stderr, "error reading SBOM %s: %v\n", *sbomPath, err) os.Exit(1) } - source = &mirror.SBOMSource{Data: data, Name: *sbomPath, Platforms: platformList} + source = &mirror.SBOMSource{Data: data, Name: *sbomPath, Latest: *latest, Platforms: platformList} case len(purls) > 0: - source = &mirror.PURLSource{PURLs: purls, Platforms: platformList} + source = &mirror.PURLSource{PURLs: purls, Latest: *latest, Platforms: platformList} default: fmt.Fprintf(os.Stderr, "error: provide PURLs or --sbom\n") fs.Usage() diff --git a/docs/configuration.md b/docs/configuration.md index 0b9c779..e25b040 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1589,6 +1589,14 @@ curl -X POST http://localhost:8080/api/mirror \ -d '{"purls":["pkg:docker/library/alpine@3.20"],"platforms":["linux/amd64"]}' ``` +`latest` mirrors only the newest version of PURLs without a version, like the command's `--latest` flag: + +```bash +curl -X POST http://localhost:8080/api/mirror \ + -H "Content-Type: application/json" \ + -d '{"purls":["pkg:npm/lodash"],"latest":true}' +``` + ## Cache Delete API Deleting cached artifacts is disabled by default. Enable it to register `DELETE /api/cache` and show Delete buttons on the version page of the web UI: @@ -1638,6 +1646,7 @@ A CRAN version is mirrored as its source package from `upstream.cran`, for examp | `--concurrency` | `4` | Number of parallel downloads | | `--dry-run` | `false` | Show what would be mirrored without downloading | | `--platform` | | Comma-separated `os/arch[/variant]` list limiting which platforms of container images are mirrored | +| `--latest` | `false` | Mirror only the newest version of PURLs without a version | | `--config` | | Path to configuration file | | `--storage-url` | | Storage URL | | `--database-driver` | | Database driver | @@ -1654,6 +1663,12 @@ proxy mirror pkg:npm/lodash@4.17.21 pkg:cargo/serde@1.0.0 A PURL without a version, here or in a mirror API job, mirrors every version the configured upstream lists, through the same client, so `upstream.auth` and `upstream.allow_private_hosts` apply. npm packages in `upstream.npm_routes` are listed from their route. Listing works for npm, gem, golang, hex (from `upstream.hex_api`), pub, pypi, maven, nuget, composer, cran, brew and swift. For cargo and conda the upstream doesn't serve the API used for listing, so versions come from the public registry while the upstream is the default one; with another upstream, and for Composer packages in `upstream.composer_routes` or any other ecosystem, give a version. Julia versions are always listed from the General registry. +With `--latest`, or `"latest": true` in a mirror API job, only the latest version is mirrored, as git-pkgs/registries picks it: the version the registry advertises as latest where it has one, such as npm's `dist-tags.latest`, and otherwise the newest by publish time, leaving out yanked, retracted and deprecated versions. PURLs with a version, including those in an SBOM, are mirrored as given. Run it from cron to keep the newest releases of a list of packages cached: + +```bash +proxy mirror --latest pkg:npm/lodash pkg:pypi/requests pkg:cargo/serde +``` + ### Container images `pkg:docker` and `pkg:oci` PURLs mirror a container image the way a pull does: the manifest or index, the platform manifests, then config and layer blobs. Everything is stored under the keys a pull through `/v2` reads, so a later `docker pull` or containerd pull is served from cache, also when the upstream is unreachable. diff --git a/internal/mirror/job.go b/internal/mirror/job.go index acf0f25..841c7c9 100644 --- a/internal/mirror/job.go +++ b/internal/mirror/job.go @@ -42,6 +42,8 @@ type JobRequest struct { // Platforms limit which platforms of container images are mirrored, // as os/architecture[/variant]. Platforms []string `json:"platforms,omitempty"` + // Latest mirrors only the newest version of PURLs without a version. + Latest bool `json:"latest,omitempty"` } // JobStore manages in-memory mirror jobs. @@ -209,9 +211,9 @@ func (js *JobStore) sourceFromRequest(req JobRequest) (Source, error) { //nolint case len(req.PURLs) > 0 && len(req.SBOM) > 0: return nil, fmt.Errorf("request must include only one of purls or sbom") case len(req.PURLs) > 0: - return &PURLSource{PURLs: req.PURLs, Versions: js.mirror.Versions, Platforms: req.Platforms}, nil + return &PURLSource{PURLs: req.PURLs, Versions: js.mirror.Versions, Latest: req.Latest, Platforms: req.Platforms}, nil case len(req.SBOM) > 0: - return &SBOMSource{Data: req.SBOM, Versions: js.mirror.Versions, Platforms: req.Platforms}, nil + return &SBOMSource{Data: req.SBOM, Versions: js.mirror.Versions, Latest: req.Latest, Platforms: req.Platforms}, nil case req.Registry != "": return nil, fmt.Errorf("registry mirroring is not yet implemented; use purls instead") default: diff --git a/internal/mirror/latest_test.go b/internal/mirror/latest_test.go new file mode 100644 index 0000000..8135116 --- /dev/null +++ b/internal/mirror/latest_test.go @@ -0,0 +1,127 @@ +package mirror + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "reflect" + "testing" +) + +func TestPURLSourceLatestMirrorsNewestVersion(t *testing.T) { + versions := &fakeVersions{versions: []string{"1.0.0", "1.10.0", "1.2.0"}} + source := &PURLSource{PURLs: []string{"pkg:npm/left-pad", "pkg:npm/is-odd@3.0.0"}, Versions: versions, Latest: true} + + items := enumerate(t, source) + + want := []PackageVersion{ + {Ecosystem: "npm", Name: "left-pad", Version: "1.10.0"}, + {Ecosystem: "npm", Name: "is-odd", Version: "3.0.0"}, + } + if !reflect.DeepEqual(items, want) { + t.Errorf("items = %v, want %v", items, want) + } +} + +func TestPURLSourceLatestWithoutVersions(t *testing.T) { + source := &PURLSource{PURLs: []string{"pkg:npm/left-pad"}, Versions: &fakeVersions{}, Latest: true} + if items := enumerate(t, source); len(items) != 0 { + t.Errorf("items = %v, want none", items) + } +} + +func TestSBOMSourceLatest(t *testing.T) { + bom := map[string]any{ + "bomFormat": "CycloneDX", + "specVersion": "1.4", + "components": []map[string]any{{"type": "library", "name": "left-pad", "purl": "pkg:npm/left-pad"}}, + } + source := &SBOMSource{Data: marshalJSON(t, bom), Versions: &fakeVersions{versions: []string{"1.0.0", "1.3.0"}}, Latest: true} + + items := enumerate(t, source) + + if len(items) != 1 || items[0].Version != "1.3.0" { + t.Errorf("items = %v, want only left-pad@1.3.0", items) + } +} + +// TestRegistryVersionsLatestSkipsYanked checks that without an advertised +// latest version the newest version by publish time that isn't yanked wins. +func TestRegistryVersionsLatestSkipsYanked(t *testing.T) { + registry := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/api/v1/crates/demo" { + http.NotFound(w, r) + return + } + _ = json.NewEncoder(w).Encode(map[string]any{ + "crate": map[string]any{"name": "demo"}, + "versions": []map[string]any{ + {"num": "1.3.0", "yanked": true, "created_at": "2026-03-01T00:00:00Z"}, + {"num": "1.2.0", "created_at": "2026-02-01T00:00:00Z"}, + {"num": "1.10.0-alpha.1", "created_at": "2026-01-01T00:00:00Z"}, + }, + }) + })) + defer registry.Close() + versions := RegistryVersions{URL: func(string, string) (string, error) { return registry.URL, nil }} + + got, err := versions.LatestVersion(context.Background(), "cargo", "demo") + if err != nil { + t.Fatalf("LatestVersion() error = %v", err) + } + if got != "1.2.0" { + t.Errorf("LatestVersion() = %q, want 1.2.0", got) + } +} + +// TestRegistryVersionsLatestPrefersAdvertised checks that the version the +// registry advertises as latest wins over a higher one, here a 4.0.0 +// published by mistake and deprecated. +func TestRegistryVersionsLatestPrefersAdvertised(t *testing.T) { + registry := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/left-pad" { + http.NotFound(w, r) + return + } + _ = json.NewEncoder(w).Encode(map[string]any{ + "name": "left-pad", + "dist-tags": map[string]any{"latest": "3.4.0"}, + "versions": map[string]any{ + "3.4.0": map[string]any{"name": "left-pad", "version": "3.4.0"}, + "4.0.0": map[string]any{"name": "left-pad", "version": "4.0.0", "deprecated": "published by mistake"}, + }, + "time": map[string]any{"3.4.0": "2026-01-01T00:00:00Z", "4.0.0": "2026-02-01T00:00:00Z"}, + }) + })) + defer registry.Close() + versions := RegistryVersions{URL: func(string, string) (string, error) { return registry.URL, nil }} + + got, err := versions.LatestVersion(context.Background(), "npm", "left-pad") + if err != nil { + t.Fatalf("LatestVersion() error = %v", err) + } + if got != "3.4.0" { + t.Errorf("LatestVersion() = %q, want the advertised 3.4.0", got) + } +} + +func TestSourceFromRequestPassesLatest(t *testing.T) { + js := NewJobStore(context.Background(), setupTestMirror(t, 1)) + + source, err := js.sourceFromRequest(JobRequest{PURLs: []string{"pkg:npm/left-pad"}, Latest: true}) + if err != nil { + t.Fatalf("sourceFromRequest() error = %v", err) + } + if !source.(*PURLSource).Latest { + t.Error("PURL source mirrors every version, want only the newest") + } + + source, err = js.sourceFromRequest(JobRequest{SBOM: json.RawMessage(`{"bomFormat":"CycloneDX","components":[]}`), Latest: true}) + if err != nil { + t.Fatalf("sourceFromRequest() error = %v", err) + } + if !source.(*SBOMSource).Latest { + t.Error("SBOM source mirrors every version, want only the newest") + } +} diff --git a/internal/mirror/source.go b/internal/mirror/source.go index 1c8fa7b..ecc3a0c 100644 --- a/internal/mirror/source.go +++ b/internal/mirror/source.go @@ -6,6 +6,7 @@ import ( "strings" "github.com/git-pkgs/purl" + "github.com/git-pkgs/registries" "github.com/git-pkgs/sbom" ) @@ -46,12 +47,14 @@ type Source interface { // PURLSource yields packages from PURL strings. // Versioned PURLs produce a single item. Unversioned PURLs produce one item -// per version Versions lists. +// per version Versions lists, or only the newest with Latest. type PURLSource struct { PURLs []string // Versions lists the versions of unversioned PURLs. Nil lists them from // the public registries. Versions VersionLister + // Latest mirrors only the newest version of unversioned PURLs. + Latest bool // Platforms limit which platforms of container images are mirrored. Platforms []string } @@ -86,8 +89,8 @@ func (s *PURLSource) Enumerate(ctx context.Context, fn func(PackageVersion) erro continue } - // Unversioned: enumerate all versions - listed, err := versions.ListVersions(ctx, ecosystem, name) + // Unversioned: enumerate all versions, or the newest + listed, err := s.listVersions(ctx, versions, ecosystem, name) if err != nil { return fmt.Errorf("fetching versions for %s/%s: %w", ecosystem, name, err) } @@ -104,6 +107,36 @@ func (s *PURLSource) Enumerate(ctx context.Context, fn func(PackageVersion) erro return nil } +func (s *PURLSource) listVersions(ctx context.Context, versions VersionLister, ecosystem, name string) ([]string, error) { + if !s.Latest { + return versions.ListVersions(ctx, ecosystem, name) + } + var latest string + if finder, ok := versions.(LatestFinder); ok { + v, err := finder.LatestVersion(ctx, ecosystem, name) + if err != nil { + return nil, err + } + latest = v + } else { + listed, err := versions.ListVersions(ctx, ecosystem, name) + if err != nil { + return nil, err + } + all := make([]registries.Version, len(listed)) + for i, v := range listed { + all[i] = registries.Version{Number: v} + } + if v := registries.SelectLatestVersion(all, purl.EcosystemToPURLType(ecosystem), ""); v != nil { + latest = v.Number + } + } + if latest == "" { + return nil, nil + } + return []string{latest}, nil +} + // mirrorTarget returns the ecosystem, name and file qualifiers a PURL is // mirrored under. func mirrorTarget(p *purl.PURL) (ecosystem, name string, qualifiers map[string]string) { @@ -128,6 +161,8 @@ type SBOMSource struct { // Versions lists the versions of unversioned PURLs. Nil lists them from // the public registries. Versions VersionLister + // Latest mirrors only the newest version of unversioned PURLs. + Latest bool // Platforms limit which platforms of container images are mirrored. Platforms []string } @@ -141,7 +176,7 @@ func (s *SBOMSource) Enumerate(ctx context.Context, fn func(PackageVersion) erro return fmt.Errorf("parsing SBOM: %w", err) } - inner := &PURLSource{PURLs: purls, Versions: s.Versions, Platforms: s.Platforms} + inner := &PURLSource{PURLs: purls, Versions: s.Versions, Latest: s.Latest, Platforms: s.Platforms} return inner.Enumerate(ctx, fn) } diff --git a/internal/mirror/versions.go b/internal/mirror/versions.go index 8e10eda..04353b4 100644 --- a/internal/mirror/versions.go +++ b/internal/mirror/versions.go @@ -36,6 +36,29 @@ type VersionFinder interface { FindVersion(ctx context.Context, ecosystem, name, version string) (registries.Version, error) } +// LatestFinder picks the latest version of a package, for unversioned PURLs +// when only the latest is mirrored. Listers without it fall back to +// registries.SelectLatestVersion over ListVersions. +type LatestFinder interface { + LatestVersion(ctx context.Context, ecosystem, name string) (string, error) +} + +// LatestVersion implements LatestFinder with registries.FetchLatestVersion: +// the version the registry advertises as latest where it has one, such as +// npm's dist-tags.latest, and otherwise the newest version by publish time +// that isn't yanked, retracted or deprecated. +func (r RegistryVersions) LatestVersion(ctx context.Context, ecosystem, name string) (string, error) { + reg, name, err := r.registryFor(ctx, ecosystem, name) + if err != nil { + return "", err + } + v, err := registries.FetchLatestVersion(ctx, reg, name) + if err != nil || v == nil { + return "", err + } + return v.Number, nil +} + // ListVersions implements VersionLister. func (r RegistryVersions) ListVersions(ctx context.Context, ecosystem, name string) ([]string, error) { versions, err := r.fetchVersions(ctx, ecosystem, name) @@ -64,10 +87,20 @@ func (r RegistryVersions) FindVersion(ctx context.Context, ecosystem, name, vers } func (r RegistryVersions) fetchVersions(ctx context.Context, ecosystem, name string) ([]registries.Version, error) { - reg, err := r.registry(ecosystem, name) + reg, name, err := r.registryFor(ctx, ecosystem, name) if err != nil { return nil, err } + return reg.FetchVersions(ctx, name) +} + +// registryFor returns the client for the registry name is listed in, and +// the name the registry knows the package by. +func (r RegistryVersions) registryFor(ctx context.Context, ecosystem, name string) (registries.Registry, string, error) { //nolint:ireturn // registries.New returns the interface + reg, err := r.registry(ecosystem, name) + if err != nil { + return nil, "", err + } switch purl.EcosystemToPURLType(ecosystem) { case "helm": // The namespace names the repository, which URL resolves to the @@ -76,13 +109,13 @@ func (r RegistryVersions) fetchVersions(ctx context.Context, ecosystem, name str case "swift": var baseURL string if baseURL, err = r.baseURL("swift", name); err != nil { - return nil, err + return nil, "", err } if name, err = swiftIdentity(ctx, r.client(), baseURL, name); err != nil { - return nil, err + return nil, "", err } } - return reg.FetchVersions(ctx, name) + return reg, name, nil } // registry returns the client for the registry name is listed in. From cb566d5df6af63d2a3d481cfe1151cff68eb4f10 Mon Sep 17 00:00:00 2001 From: simon chrzanowski Date: Fri, 30 Oct 2026 15:06:42 +0100 Subject: [PATCH 504/633] Expire cooldown rewrites when a withheld version becomes available (#638) A cached rewrite made with cooldown on stays right until a version it held back leaves cooldown, so it now expires then instead of after a fixed time. Handlers ask cooldown through Proxy.cooldown, which reports each withheld version to the rewrites running at that moment. A rewrite that held nothing back is kept for the shortest cooldown, at most a day. --- docs/configuration.md | 4 +- internal/handler/cargo.go | 2 +- internal/handler/composer.go | 4 +- internal/handler/conda.go | 4 +- internal/handler/cooldown_expiry.go | 164 +++++++++++++++++ internal/handler/cooldown_expiry_test.go | 178 +++++++++++++++++++ internal/handler/cooldown_times.go | 2 +- internal/handler/go_cooldown.go | 2 +- internal/handler/handler.go | 4 + internal/handler/helm.go | 2 +- internal/handler/maven_cooldown.go | 6 +- internal/handler/metadata_access_test.go | 2 +- internal/handler/metadata_gzip_test.go | 2 +- internal/handler/metadata_validators_test.go | 2 +- internal/handler/npm.go | 6 +- internal/handler/npm_rewrite_test.go | 4 +- internal/handler/nuget_cooldown.go | 2 +- internal/handler/pub.go | 2 +- internal/handler/pypi.go | 8 +- internal/handler/rewrite_cache.go | 62 ++----- internal/handler/rewrite_cache_test.go | 43 +++-- 21 files changed, 413 insertions(+), 92 deletions(-) create mode 100644 internal/handler/cooldown_expiry.go create mode 100644 internal/handler/cooldown_expiry_test.go diff --git a/docs/configuration.md b/docs/configuration.md index e25b040..e1a4f9e 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1144,7 +1144,7 @@ Helm uses each release's `created` time from the repository's `index.yaml`. Rele RubyGems, Hex and PyPI keep their publish times outside the metadata clients read: the RubyGems versions API, the Hex HTTP API (`upstream.hex_api`) and the PyPI JSON API. The proxy fetches them alongside each compact index, package or simple page it filters, and for PyPI downloads. Cooldown fails closed: when it applies to a package and those times can't be fetched or parsed, or the metadata can't be rewritten, the request fails with 502, or with 503 and `Retry-After` while the upstream is rate limiting the proxy, instead of serving the metadata unfiltered. The same goes for npm, Composer and pub.dev metadata that can't be rewritten, and for npm tarball downloads whose publish time can't be read. A PyPI file or npm tarball that is already cached is still served when its publish time can't be read; only a download that would have to be fetched fails. A PyPI release's time is the upload time of its newest file. The proxy stores it when a download looks it up, and serves files of that release that are already cached against the stored time without reading the JSON API again. A file that isn't cached yet is always looked up, since it may have been added to the release since, and the newer time is stored. Packages exempted with a duration of `0` are served as before and don't need the times. A PyPI package the JSON API returns 404 for has no times to go by and is served unfiltered. -For RubyGems and Hex, both documents go through the metadata cache. With [`cache_metadata`](#metadata-caching) they are served from it within `metadata_ttl` without asking the upstream, revalidated with their ETags after it, and filtered from the last cached copies, marked stale, while the upstream is down; without it they are still fetched once for concurrent requests. Each document has its own TTL, so the compact index or Hex package can be newer than the cached times and list a version they don't have. The times are then fetched again before filtering, and if that fails, such versions are left out until the times can be read. The filtered index or package carries an ETag of its own, so Bundler and Mix revalidate it with `304 Not Modified`. Publish times are compared with the current time on each request, but the filtered result is kept for up to an hour (see [rewritten metadata cache](#rewritten-metadata-cache)), so a version leaving cooldown can show up that much later; a newly published version shows up within `metadata_ttl`, as with other cached metadata. Upstream errors are answered as for other cached metadata (404, or 502) rather than relayed. +For RubyGems and Hex, both documents go through the metadata cache. With [`cache_metadata`](#metadata-caching) they are served from it within `metadata_ttl` without asking the upstream, revalidated with their ETags after it, and filtered from the last cached copies, marked stale, while the upstream is down; without it they are still fetched once for concurrent requests. Each document has its own TTL, so the compact index or Hex package can be newer than the cached times and list a version they don't have. The times are then fetched again before filtering, and if that fails, such versions are left out until the times can be read. The filtered index or package carries an ETag of its own, so Bundler and Mix revalidate it with `304 Not Modified`. Publish times are compared with the current time on each request, and the filtered result is kept until the first version it held back leaves cooldown (see [rewritten metadata cache](#rewritten-metadata-cache)); a newly published version shows up within `metadata_ttl`, as with other cached metadata. Upstream errors are answered as for other cached metadata (404, or 502) rather than relayed. With cooldown on, the proxy asks npm for full packuments, since the abbreviated ones it uses otherwise carry no publish times. Abbreviated packuments cached before cooldown was turned on are fetched again in full rather than served, including as a stale copy when the upstream is down. When `latest` is in cooldown, the packument tags the newest version out of cooldown instead, and `npm dist-tag ls` shows the same tags. @@ -1423,7 +1423,7 @@ Or via environment variable: `PROXY_ARTIFACT_MAX_SIZE=2GB`. ### Rewritten metadata cache -The npm, Composer, PyPI and Helm handlers rewrite every metadata document they serve, so that download URLs point at the proxy. That means decoding the whole document and encoding it again, and Composer's minified documents are also expanded, which together cost milliseconds per typical package and far more for very large ones; a large Helm `index.yaml` takes over a hundred milliseconds. The proxy keeps rewritten documents in memory, so each distinct upstream document is rewritten once and requests arriving while it is being rewritten wait for that rewrite. New bytes from upstream are rewritten again. A PyPI simple page is cached per representation (HTML or JSON) and per set of versions filtered out of it. With version cooldown enabled, a cached rewrite expires, because cooldown filtering depends on the current time: it is kept for a 24th of the shortest cooldown configured, at least a minute and at most an hour, so a version shows up at most that long after its cooldown ends. With a cooldown of a day or more that is an hour; with `"2h"` somewhere, five minutes. Cached documents carry an `ETag` of their own, and are dated like any other, so clients that revalidate with `If-None-Match` or `If-Modified-Since`, such as Composer, get `304 Not Modified`. The RubyGems compact index and Hex packages filtered by cooldown are kept the same way; they are never served gzip-compressed. +The npm, Composer, PyPI and Helm handlers rewrite every metadata document they serve, so that download URLs point at the proxy. That means decoding the whole document and encoding it again, and Composer's minified documents are also expanded, which together cost milliseconds per typical package and far more for very large ones; a large Helm `index.yaml` takes over a hundred milliseconds. The proxy keeps rewritten documents in memory, so each distinct upstream document is rewritten once and requests arriving while it is being rewritten wait for that rewrite. New bytes from upstream are rewritten again. A PyPI simple page is cached per representation (HTML or JSON) and per set of versions filtered out of it. With version cooldown enabled, a cached rewrite expires when the first version it held back leaves cooldown, since versions it let through stay allowed and a new upstream version changes the document. A rewrite that held nothing back is kept for the shortest cooldown configured, at least a minute and at most a day. Cached documents carry an `ETag` of their own, and are dated like any other, so clients that revalidate with `If-None-Match` or `If-Modified-Since`, such as Composer, get `304 Not Modified`. The RubyGems compact index and Hex packages filtered by cooldown are kept the same way; they are never served gzip-compressed. Each cached document is kept gzip-compressed, when that makes it smaller, and served with `Content-Encoding: gzip` to clients that accept it, as npm, pnpm, Yarn and Composer do. A large packument is compressed once rather than on every request, by the proxy or by a reverse proxy in front of it. Only the compressed copy is kept, typically an eighth of the document, and it is decompressed for the clients that don't accept gzip. Documents rewritten outside the cache are served uncompressed. diff --git a/internal/handler/cargo.go b/internal/handler/cargo.go index 578c9ec..0c26551 100644 --- a/internal/handler/cargo.go +++ b/internal/handler/cargo.go @@ -171,7 +171,7 @@ func (h *CargoHandler) applyCooldownFiltering(downstreamResponse http.ResponseWr cratePURL := canonicalPackagePURL("cargo", crate.Name) - if !h.proxy.Cooldown.IsAllowed("cargo", cratePURL, publishedAt) { + if !h.proxy.cooldown().IsAllowed("cargo", cratePURL, publishedAt) { h.proxy.Logger.Info("cooldown: filtering cargo version", "crate", crate.Name, "version", crate.Version, "published", crate.PublishTime) diff --git a/internal/handler/composer.go b/internal/handler/composer.go index 116d106..ccb42bb 100644 --- a/internal/handler/composer.go +++ b/internal/handler/composer.go @@ -394,7 +394,7 @@ func (h *ComposerHandler) shouldFilterVersion(packagePURL, packageName, version return false } - if !h.proxy.Cooldown.IsAllowed("composer", packagePURL, publishedAt) { + if !h.proxy.cooldown().IsAllowed("composer", packagePURL, publishedAt) { h.proxy.Logger.Info("cooldown: filtering composer version", "package", packageName, "version", version) return true @@ -731,7 +731,7 @@ func (h *ComposerHandler) serveCachedRelease(w http.ResponseWriter, r *http.Requ if err != nil || ver == nil || !ver.PublishedAt.Valid { return false } - if !h.proxy.Cooldown.IsAllowed("composer", canonicalPackagePURL("composer", packageName), ver.PublishedAt.Time) { + if !h.proxy.cooldown().IsAllowed("composer", canonicalPackagePURL("composer", packageName), ver.PublishedAt.Time) { h.proxy.Logger.Info("cooldown: withholding composer archive", "package", packageName, "version", version) http.Error(w, "version not found", http.StatusNotFound) diff --git a/internal/handler/conda.go b/internal/handler/conda.go index 76a43f2..3cbaf93 100644 --- a/internal/handler/conda.go +++ b/internal/handler/conda.go @@ -493,7 +493,7 @@ func (f *repodataFilter) inCooldown(filename string, raw []byte) bool { return false } publishedAt := time.Unix(int64(entry.Timestamp)/condaTimestampDivisor, 0) - decision := f.h.proxy.Cooldown.Evaluate("conda", canonicalPackagePURL("conda", entry.Name), publishedAt, f.now) + decision := f.h.proxy.cooldown().Evaluate("conda", canonicalPackagePURL("conda", entry.Name), publishedAt, f.now) if decision.Allowed { return false } @@ -517,7 +517,7 @@ func (h *CondaHandler) recordInCooldown(filename, name, version string, timestam return false } publishedAt := time.Unix(int64(timestamp)/condaTimestampDivisor, 0) - if h.proxy.Cooldown.IsAllowed("conda", canonicalPackagePURL("conda", name), publishedAt) { + if h.proxy.cooldown().IsAllowed("conda", canonicalPackagePURL("conda", name), publishedAt) { return false } h.proxy.Logger.Info("cooldown: filtering conda package", diff --git a/internal/handler/cooldown_expiry.go b/internal/handler/cooldown_expiry.go new file mode 100644 index 0000000..1163bde --- /dev/null +++ b/internal/handler/cooldown_expiry.go @@ -0,0 +1,164 @@ +package handler + +import ( + "sync" + "time" + + "github.com/git-pkgs/cooldown" +) + +// A rewrite made with cooldown on stays right until a version it held back +// leaves cooldown: versions it let through stay allowed as time passes, and +// new versions upstream change the document and so the cache key. So the +// cached rewrite expires when the earliest version it withheld becomes +// available, instead of after a fixed time. +// +// Handlers ask cooldown through Proxy.cooldown, which tells every rewrite +// running at that moment when a withheld version becomes available. A +// rewrite may hear of versions another rewrite withheld at the same time; +// that only makes it expire earlier than it has to, never later. + +// maxCooldownRewriteTTL is the longest a rewrite made with cooldown on is +// served, even when it held nothing back. +const maxCooldownRewriteTTL = 24 * time.Hour + +// minCooldownRewriteTTL is the shortest cap on a rewrite made with cooldown +// on, however short the cooldown. +const minCooldownRewriteTTL = time.Minute + +// cooldownRewriteTTL returns the longest a rewrite made with cooldown on is +// served: the shortest cooldown configured, between a minute and a day. It +// only matters when a rewrite held nothing back, or held back a version +// whose availability is unknown. +func (p *Proxy) cooldownRewriteTTL() time.Duration { + ttl := p.ShortestCooldown + switch { + case ttl <= 0 || ttl > maxCooldownRewriteTTL: + return maxCooldownRewriteTTL + case ttl < minCooldownRewriteTTL: + return minCooldownRewriteTTL + } + return ttl +} + +// cooldownWatch holds the rewrites running with cooldown on. +type cooldownWatch struct { + mu sync.Mutex + active map[*cooldownRecorder]struct{} +} + +// cooldownRecorder collects when the versions withheld during one rewrite +// become available. +type cooldownRecorder struct { + // earliest is when the first withheld version becomes available, or + // zero if none was withheld. + earliest time.Time + // unknown is set when a version was withheld without a time it becomes + // available. + unknown bool +} + +// watchCooldown starts recording the versions cooldown holds back for a +// rewrite about to run. unwatchCooldown ends it. +func (p *Proxy) watchCooldown() *cooldownRecorder { + rec := &cooldownRecorder{} + w := &p.cooldownWatch + w.mu.Lock() + defer w.mu.Unlock() + if w.active == nil { + w.active = make(map[*cooldownRecorder]struct{}) + } + w.active[rec] = struct{}{} + return rec +} + +func (p *Proxy) unwatchCooldown(rec *cooldownRecorder) { + w := &p.cooldownWatch + w.mu.Lock() + defer w.mu.Unlock() + delete(w.active, rec) +} + +// cooldownExpiry returns when a rewrite that started at start and recorded +// into rec should expire: when the first version it held back leaves +// cooldown, and no later than cooldownRewriteTTL after start. +func (p *Proxy) cooldownExpiry(rec *cooldownRecorder, start time.Time) time.Time { + w := &p.cooldownWatch + w.mu.Lock() + earliest, unknown := rec.earliest, rec.unknown + w.mu.Unlock() + + limit := start.Add(p.cooldownRewriteTTL()) + if unknown || earliest.IsZero() || earliest.After(limit) { + return limit + } + return earliest +} + +// withheld tells the running rewrites that a version is held back until +// availableAt, zero when that is unknown. +func (p *Proxy) withheld(availableAt time.Time) { + w := &p.cooldownWatch + w.mu.Lock() + defer w.mu.Unlock() + for rec := range w.active { + switch { + case availableAt.IsZero(): + rec.unknown = true + case rec.earliest.IsZero() || availableAt.Before(rec.earliest): + rec.earliest = availableAt + } + } +} + +// cooldown returns the proxy's cooldown policy, telling the running rewrites +// about each version it holds back. Handlers use it rather than p.Cooldown. +func (p *Proxy) cooldown() CooldownPolicy { //nolint:ireturn // nil when cooldown is not configured, as p.Cooldown is + if p.Cooldown == nil { + return nil + } + return recordingCooldown{p: p} +} + +// recordingCooldown is the proxy's cooldown policy, reporting the versions it +// holds back to the running rewrites. +type recordingCooldown struct { + p *Proxy +} + +func (r recordingCooldown) IsAllowed(ecosystem, packagePURL string, publishedAt time.Time) bool { + if r.p.Cooldown.IsAllowed(ecosystem, packagePURL, publishedAt) { + return true + } + r.p.withheld(r.availableAt(ecosystem, packagePURL, publishedAt)) + return false +} + +func (r recordingCooldown) Evaluate(ecosystem, packagePURL string, publishedAt, evaluatedAt time.Time) cooldown.Decision { + d := r.p.Cooldown.Evaluate(ecosystem, packagePURL, publishedAt, evaluatedAt) + if !d.Allowed { + r.p.withheld(d.AvailableAt) + } + return d +} + +func (r recordingCooldown) For(ecosystem, packagePURL string) time.Duration { + return r.p.Cooldown.For(ecosystem, packagePURL) +} + +func (r recordingCooldown) Enabled() bool { + return r.p.Cooldown.Enabled() +} + +// availableAt is when a version IsAllowed held back leaves cooldown, or zero +// when that is unknown. +func (r recordingCooldown) availableAt(ecosystem, packagePURL string, publishedAt time.Time) time.Time { + if publishedAt.IsZero() { + return time.Time{} + } + d := r.p.Cooldown.For(ecosystem, packagePURL) + if d <= 0 { + return time.Time{} + } + return publishedAt.Add(d) +} diff --git a/internal/handler/cooldown_expiry_test.go b/internal/handler/cooldown_expiry_test.go new file mode 100644 index 0000000..fd55845 --- /dev/null +++ b/internal/handler/cooldown_expiry_test.go @@ -0,0 +1,178 @@ +package handler + +import ( + "context" + "net/http" + "os" + "path/filepath" + "regexp" + "strings" + "testing" + "time" + + "github.com/git-pkgs/cooldown" +) + +// TestCachedRewrite_ExpiresWhenWithheldVersionIsAvailable checks that a +// rewrite which held a version back is cached until that version leaves +// cooldown, and no longer. +func TestCachedRewrite_ExpiresWhenWithheldVersionIsAvailable(t *testing.T) { + proxy, _, _, _ := setupTestProxy(t) + clock := &testClock{t: time.Now()} + proxy.now = clock.now + proxy.SetMetadataRewriteCacheSize(1 << 20) + proxy.Cooldown = &cooldown.Config{Default: "3d"} + purl := canonicalPackagePURL("npm", "left-pad") + // One version is 3d-5h old: it leaves cooldown in five hours. + published := clock.now().Add(-72*time.Hour + 5*time.Hour) + + var calls int + var rewrite keepingRewrite = func(b []byte) ([]byte, []string, error) { + calls++ + if proxy.cooldown().IsAllowed("npm", purl, published) { + return b, []string{"1.0.0", "1.1.0"}, nil + } + return b, []string{"1.0.0"}, nil + } + get := func() { + if _, err := proxy.cachedRewrite(context.Background(), "npm", "http://proxy", "left-pad", []byte("doc"), "", rewrite); err != nil { + t.Fatal(err) + } + } + + get() + clock.advance(5*time.Hour - time.Minute) + get() + if calls != 1 { + t.Errorf("rewrites before the withheld version is available = %d, want 1", calls) + } + clock.advance(time.Minute) + get() + if calls != 2 { + t.Errorf("rewrites once the withheld version is available = %d, want 2", calls) + } +} + +// TestCachedRewrite_NothingWithheldCachedForCap checks that a rewrite which +// held nothing back is cached for cooldownRewriteTTL. +func TestCachedRewrite_NothingWithheldCachedForCap(t *testing.T) { + proxy, _, _, _ := setupTestProxy(t) + clock := &testClock{t: time.Date(2026, 10, 10, 12, 0, 0, 0, time.UTC)} + proxy.now = clock.now + proxy.SetMetadataRewriteCacheSize(1 << 20) + proxy.Cooldown = &cooldown.Config{Default: "2h"} + proxy.ShortestCooldown = 2 * time.Hour + var r countingRewrite + + get := func() { + _, _ = proxy.cachedRewrite(context.Background(), "npm", "http://proxy", "left-pad", []byte("doc"), "", r.keeping) + } + get() + clock.advance(2*time.Hour - time.Second) + get() + if got := r.calls.Load(); got != 1 { + t.Errorf("rewrites within the cap = %d, want 1", got) + } + clock.advance(time.Second) + get() + if got := r.calls.Load(); got != 2 { + t.Errorf("rewrites after the cap = %d, want 2", got) + } +} + +// TestCooldownExpiry checks how one rewrite's recorded decisions set its +// expiry, including decisions made for other rewrites running at the same +// time, which may only make it expire earlier. +func TestCooldownExpiry(t *testing.T) { + p := &Proxy{Cooldown: &cooldown.Config{Default: "3d"}, ShortestCooldown: 72 * time.Hour} + // IsAllowed compares with the wall clock, so the times are relative to it. + start := time.Now() + limit := start.Add(24 * time.Hour) + purl := canonicalPackagePURL("npm", "left-pad") + + rec := p.watchCooldown() + if got := p.cooldownExpiry(rec, start); !got.Equal(limit) { + t.Errorf("nothing withheld: expiry = %v, want the cap %v", got, limit) + } + + other := p.watchCooldown() + p.cooldown().IsAllowed("npm", purl, start.Add(-60*time.Hour)) // available in 12h + p.unwatchCooldown(other) + p.cooldown().IsAllowed("npm", purl, start.Add(-66*time.Hour)) // available in 6h, after other ended + if got, want := p.cooldownExpiry(rec, start), start.Add(6*time.Hour); !got.Equal(want) { + t.Errorf("rec expiry = %v, want the earliest availability %v", got, want) + } + if got, want := p.cooldownExpiry(other, start), start.Add(12*time.Hour); !got.Equal(want) { + t.Errorf("other expiry = %v, want %v: decisions after it ended don't count", got, want) + } + + p.cooldown().Evaluate("npm", purl, start.Add(-71*time.Hour), start) // available in 1h + if got, want := p.cooldownExpiry(rec, start), start.Add(time.Hour); !got.Equal(want) { + t.Errorf("after Evaluate: expiry = %v, want %v", got, want) + } + p.unwatchCooldown(rec) + if len(p.cooldownWatch.active) != 0 { + t.Errorf("%d recorders left after unwatch", len(p.cooldownWatch.active)) + } +} + +// TestHandlersAskCooldownThroughRecorder checks that handlers make cooldown +// decisions through Proxy.cooldown. A decision made on Proxy.Cooldown +// directly would not reach the rewrites running, so a cached rewrite could +// keep hiding a version past its cooldown. +func TestHandlersAskCooldownThroughRecorder(t *testing.T) { + direct := regexp.MustCompile(`\.Cooldown\.(IsAllowed|Evaluate)\(|Cooldown:\s+\w+(\.\w+)*\.Cooldown\b`) + files, err := filepath.Glob("*.go") + if err != nil { + t.Fatal(err) + } + for _, f := range files { + if strings.HasSuffix(f, "_test.go") || f == "cooldown_expiry.go" { + continue + } + src, err := os.ReadFile(f) + if err != nil { + t.Fatal(err) + } + for i, line := range strings.Split(string(src), "\n") { + if direct.MatchString(line) { + t.Errorf("%s:%d asks Proxy.Cooldown directly; use Proxy.cooldown(): %s", f, i+1, strings.TrimSpace(line)) + } + } + } +} + +// TestHandlerRewriteExpiresWithWithheldVersion checks, through the handlers, +// that a cached rewrite expires when the version cooldown held back from it +// becomes available. +func TestHandlerRewriteExpiresWithWithheldVersion(t *testing.T) { + for _, c := range validatorCases { + t.Run(c.ecosystem, func(t *testing.T) { + u := cooldownUpstream(t) // the newest version was published 2h ago + proxy, _, clock := validatorProxyWithDB(t, u, time.Hour) + // Cooldown compares with the wall clock, so the proxy's clock + // follows it here. + clock.t = time.Now() + proxy.Cooldown = &cooldown.Config{Default: "3h"} + proxy.ShortestCooldown = 3 * time.Hour + handler := c.handler(proxy, u) + + if rec := serve(handler, c.path, nil); rec.Code != http.StatusOK { + t.Fatalf("status %d", rec.Code) + } + proxy.rewrites.mu.Lock() + defer proxy.rewrites.mu.Unlock() + if len(proxy.rewrites.entries) != 1 { + t.Fatalf("%d cached rewrites, want 1", len(proxy.rewrites.entries)) + } + for _, el := range proxy.rewrites.entries { + expires := el.Value.(*rewriteEntry).expires + // Published 2h ago with a 3h cooldown: available in an hour, + // to the second the publish time was given in. + if want := time.Now().Add(time.Hour); expires.Before(want.Add(-2*time.Second)) || expires.After(want.Add(time.Second)) { + t.Errorf("rewrite expires %v, want about %v when the withheld version is available", expires, want) + } + } + }) + } +} diff --git a/internal/handler/cooldown_times.go b/internal/handler/cooldown_times.go index 88532cd..5d95d16 100644 --- a/internal/handler/cooldown_times.go +++ b/internal/handler/cooldown_times.go @@ -126,7 +126,7 @@ func (d timedDocuments) withheld(p *Proxy, ecosystem, packagePURL, version strin if published.IsZero() { return false } - return !p.Cooldown.IsAllowed(ecosystem, packagePURL, published) + return !p.cooldown().IsAllowed(ecosystem, packagePURL, published) } // decodedMetadataBody returns res's body without its Content-Encoding. The diff --git a/internal/handler/go_cooldown.go b/internal/handler/go_cooldown.go index 90189d4..892e77b 100644 --- a/internal/handler/go_cooldown.go +++ b/internal/handler/go_cooldown.go @@ -42,7 +42,7 @@ func (h *GoHandler) allowedAt(module string, published time.Time) bool { if published.IsZero() { return true } - return h.proxy.Cooldown.IsAllowed("golang", canonicalPackagePURL("golang", module), published) + return h.proxy.cooldown().IsAllowed("golang", canonicalPackagePURL("golang", module), published) } // writeGoMetadataError answers a failed upstream metadata fetch. diff --git a/internal/handler/handler.go b/internal/handler/handler.go index cf93d98..3951016 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -284,6 +284,10 @@ type Proxy struct { // the process started; see metadataDate. now func() time.Time started time.Time + + // cooldownWatch holds the rewrites recording which versions cooldown + // holds back; see cooldown_expiry.go. + cooldownWatch cooldownWatch } // cooldownEcosystems are the ecosystems whose handlers apply cooldown. Others diff --git a/internal/handler/helm.go b/internal/handler/helm.go index 07e2f74..fdc000e 100644 --- a/internal/handler/helm.go +++ b/internal/handler/helm.go @@ -378,7 +378,7 @@ func (h *HelmHandler) buildChartLookup(upstreamURL string, body []byte) (*helmCh func (h *HelmHandler) chartOnCooldown(chartName string, created time.Time) bool { return !created.IsZero() && h.proxy.Cooldown != nil && h.proxy.Cooldown.Enabled() && - !h.proxy.Cooldown.IsAllowed(helmMetadataEcosystem, canonicalPackagePURL(helmMetadataEcosystem, chartName), created) + !h.proxy.cooldown().IsAllowed(helmMetadataEcosystem, canonicalPackagePURL(helmMetadataEcosystem, chartName), created) } type helmChartDownload struct { diff --git a/internal/handler/maven_cooldown.go b/internal/handler/maven_cooldown.go index 4705f70..4da839e 100644 --- a/internal/handler/maven_cooldown.go +++ b/internal/handler/maven_cooldown.go @@ -222,7 +222,7 @@ func (h *MavenHandler) checkCooldown(w http.ResponseWriter, r *http.Request, sou http.Error(w, "failed to check cooldown", http.StatusBadGateway) return true } - if !h.proxy.Cooldown.IsAllowed("maven", mavenPackagePURL(group, artifact), published) { + if !h.proxy.cooldown().IsAllowed("maven", mavenPackagePURL(group, artifact), published) { h.proxy.Logger.Info("cooldown: withholding maven version", "group", group, "artifact", artifact, "version", version) http.Error(w, errMavenCooldown+": "+canonicalVersionPURL("maven", group+":"+artifact, version), http.StatusForbidden) @@ -307,7 +307,7 @@ func (h *MavenHandler) filterMetadata(ctx context.Context, source mavenSource, a packagePURL := mavenPackagePURL(group, artifact) // No version can be newer than the metadata listing it. if lastUpdated, err := time.Parse(mavenLastUpdatedLayout, doc.lastUpdated); err == nil && - h.proxy.Cooldown.IsAllowed("maven", packagePURL, lastUpdated) { + h.proxy.cooldown().IsAllowed("maven", packagePURL, lastUpdated) { return body, nil } @@ -324,7 +324,7 @@ func (h *MavenHandler) filterMetadata(ctx context.Context, source mavenSource, a if published.IsZero() { continue } - if h.proxy.Cooldown.IsAllowed("maven", packagePURL, published) { + if h.proxy.cooldown().IsAllowed("maven", packagePURL, published) { break } h.proxy.Logger.Info("cooldown: filtering maven version", "group", group, "artifact", artifact, "version", v) diff --git a/internal/handler/metadata_access_test.go b/internal/handler/metadata_access_test.go index 0a64b80..7b5e1cf 100644 --- a/internal/handler/metadata_access_test.go +++ b/internal/handler/metadata_access_test.go @@ -143,7 +143,7 @@ func TestStoredRewrite_MarksAccess(t *testing.T) { body := []byte(`{}`) proxy.cacheMetadataBlob(t.Context(), "npm", "demo", metadataStoragePath("npm", "demo"), &upstreamMetadata{body: body}) key := rewriteCacheKey("npm", "http://proxy", "demo", body) - if _, err := proxy.rewrites.rewrite(t.Context(), key, body, 0, func(b []byte) ([]byte, error) { return b, nil }); err != nil { + if _, err := proxy.rewrites.rewrite(t.Context(), key, body, nil, func(b []byte) ([]byte, error) { return b, nil }); err != nil { t.Fatal(err) } old := ageMetadataAccess(t, db, "npm", "demo") diff --git a/internal/handler/metadata_gzip_test.go b/internal/handler/metadata_gzip_test.go index b5939c7..57d20cc 100644 --- a/internal/handler/metadata_gzip_test.go +++ b/internal/handler/metadata_gzip_test.go @@ -184,7 +184,7 @@ func TestRewriteCache_KeepsGzipCopyOnlyWhenSmaller(t *testing.T) { for _, doc := range [][]byte{large, []byte("abc")} { key := rewriteCacheKey("npm", "http://proxy", string(doc[:3]), doc) - if _, err := c.rewrite(t.Context(), key, doc, 0, identity); err != nil { + if _, err := c.rewrite(t.Context(), key, doc, nil, identity); err != nil { t.Fatal(err) } } diff --git a/internal/handler/metadata_validators_test.go b/internal/handler/metadata_validators_test.go index f7c1475..340e3db 100644 --- a/internal/handler/metadata_validators_test.go +++ b/internal/handler/metadata_validators_test.go @@ -500,7 +500,7 @@ func TestRewriteCacheETag(t *testing.T) { exclaim := func(b []byte) ([]byte, error) { return []byte(string(b) + "!"), nil } key := rewriteCacheKey("npm", "http://proxy", "demo", []byte("doc")) - m, err := c.rewrite(t.Context(), key, []byte("doc"), 0, exclaim) + m, err := c.rewrite(t.Context(), key, []byte("doc"), nil, exclaim) if err != nil || string(m.content()) != "doc!" { t.Fatalf("rewrite = %q, %v", m.content(), err) } diff --git a/internal/handler/npm.go b/internal/handler/npm.go index 96294fc..62bf440 100644 --- a/internal/handler/npm.go +++ b/internal/handler/npm.go @@ -476,7 +476,7 @@ func (h *NPMHandler) rewriteMetadataWithLocal(packageName string, body []byte, l func (h *NPMHandler) npmRewriteOptions(packageName string, local []npmLocalEntry) npm.Options { return npm.Options{ ProxyURL: h.proxyURL, - Cooldown: h.proxy.Cooldown, + Cooldown: h.proxy.cooldown(), DeniedVersions: h.proxy.Denylist.Versions(canonicalPackagePURL("npm", packageName)), EvaluatedAt: time.Now(), LocalVersions: local, @@ -701,7 +701,7 @@ func (h *NPMHandler) versionInCooldown(packageName, version string, metadata fun versionPURL := canonicalVersionPURL("npm", packageName, version) if !routed { if ver, err := h.proxy.DB.GetVersionByPURL(versionPURL); err == nil && ver != nil && ver.PublishedAt.Valid { - return !h.proxy.Cooldown.IsAllowed("npm", canonicalPackagePURL("npm", packageName), ver.PublishedAt.Time), nil + return !h.proxy.cooldown().IsAllowed("npm", canonicalPackagePURL("npm", packageName), ver.PublishedAt.Time), nil } } @@ -733,7 +733,7 @@ func (h *NPMHandler) versionInCooldown(packageName, version string, metadata fun } } - return !h.proxy.Cooldown.IsAllowed("npm", canonicalPackagePURL("npm", packageName), publishedAt), nil + return !h.proxy.cooldown().IsAllowed("npm", canonicalPackagePURL("npm", packageName), publishedAt), nil } // npmMetadataPath returns the package name and the encoded registry path. diff --git a/internal/handler/npm_rewrite_test.go b/internal/handler/npm_rewrite_test.go index b36b229..9b8e35b 100644 --- a/internal/handler/npm_rewrite_test.go +++ b/internal/handler/npm_rewrite_test.go @@ -94,7 +94,7 @@ func TestNPMStoredRewriteSkippedWhenStale(t *testing.T) { body := []byte(`{}`) proxy.cacheMetadataBlob(t.Context(), "npm", "demo", metadataStoragePath("npm", "demo"), &upstreamMetadata{body: body}) key := rewriteCacheKey("npm", "http://proxy", "demo", body) - if _, err := proxy.rewrites.rewrite(t.Context(), key, body, 0, func(b []byte) ([]byte, error) { return b, nil }); err != nil { + if _, err := proxy.rewrites.rewrite(t.Context(), key, body, nil, func(b []byte) ([]byte, error) { return b, nil }); err != nil { t.Fatal(err) } if _, ok := proxy.storedRewrite("npm", "demo", "http://proxy", "demo"); !ok { @@ -129,7 +129,7 @@ func TestNPMStoredRewriteRecordsMetadataHit(t *testing.T) { body := []byte(`{"name":"demo"}`) proxy.cacheMetadataBlob(t.Context(), "npm", "hit-demo", metadataStoragePath("npm", "hit-demo"), &upstreamMetadata{body: body}) key := rewriteCacheKey("npm", "http://proxy", "hit-demo", body) - if _, err := proxy.rewrites.rewrite(t.Context(), key, body, 0, func(b []byte) ([]byte, error) { return b, nil }); err != nil { + if _, err := proxy.rewrites.rewrite(t.Context(), key, body, nil, func(b []byte) ([]byte, error) { return b, nil }); err != nil { t.Fatal(err) } diff --git a/internal/handler/nuget_cooldown.go b/internal/handler/nuget_cooldown.go index afb147d..b9345c5 100644 --- a/internal/handler/nuget_cooldown.go +++ b/internal/handler/nuget_cooldown.go @@ -204,7 +204,7 @@ func (h *NuGetHandler) nugetLeafAllowed(leaf map[string]any, id string) bool { if err != nil { return true } - return h.proxy.Cooldown.IsAllowed("nuget", canonicalPackagePURL("nuget", strings.ToLower(id)), when) + return h.proxy.cooldown().IsAllowed("nuget", canonicalPackagePURL("nuget", strings.ToLower(id)), when) } func nugetCatalogEntry(leaf map[string]any) map[string]any { diff --git a/internal/handler/pub.go b/internal/handler/pub.go index 3c81298..24fdd02 100644 --- a/internal/handler/pub.go +++ b/internal/handler/pub.go @@ -247,7 +247,7 @@ func (h *PubHandler) shouldFilterVersion(packagePURL, name, version string, vmap return false } - if !h.proxy.Cooldown.IsAllowed("pub", packagePURL, publishedAt) { + if !h.proxy.cooldown().IsAllowed("pub", packagePURL, publishedAt) { h.proxy.Logger.Info("cooldown: filtering pub version", "package", name, "version", version) return true diff --git a/internal/handler/pypi.go b/internal/handler/pypi.go index fe018d8..ebb3082 100644 --- a/internal/handler/pypi.go +++ b/internal/handler/pypi.go @@ -318,7 +318,7 @@ func (h *PyPIHandler) fetchFilteredVersions(r *http.Request, name string) (map[s filtered := make(map[string]bool) for version, publishedAt := range times { - if !publishedAt.IsZero() && !h.proxy.Cooldown.IsAllowed("pypi", packagePURL, publishedAt) { + if !publishedAt.IsZero() && !h.proxy.cooldown().IsAllowed("pypi", packagePURL, publishedAt) { filtered[version] = true } } @@ -611,7 +611,7 @@ func (h *PyPIHandler) shouldFilterRelease(packagePURL string, files any) bool { } publishedAt := h.newestUploadTime(filesArr) - return !publishedAt.IsZero() && !h.proxy.Cooldown.IsAllowed("pypi", packagePURL, publishedAt) + return !publishedAt.IsZero() && !h.proxy.cooldown().IsAllowed("pypi", packagePURL, publishedAt) } // versionInCooldown reports whether a version is still inside the cooldown @@ -636,7 +636,7 @@ func (h *PyPIHandler) versionInCooldown(r *http.Request, name, version string) ( return false, nil } h.storeUploadTime(name, version, publishedAt) - return !h.proxy.Cooldown.IsAllowed("pypi", canonicalPackagePURL("pypi", name), publishedAt), nil + return !h.proxy.cooldown().IsAllowed("pypi", canonicalPackagePURL("pypi", name), publishedAt), nil } // storeUploadTime records a release's newest upload time under the version @@ -675,7 +675,7 @@ func (h *PyPIHandler) storedCooldown(r *http.Request, name, version, filename st if err != nil || ver == nil || !ver.PublishedAt.Valid { return false, false } - if !h.proxy.Cooldown.IsAllowed("pypi", packagePURL, ver.PublishedAt.Time) { + if !h.proxy.cooldown().IsAllowed("pypi", packagePURL, ver.PublishedAt.Time) { return true, true } cached, err := h.proxy.HasCachedArtifact(r.Context(), "pypi", name, version, filename) diff --git a/internal/handler/rewrite_cache.go b/internal/handler/rewrite_cache.go index b2dc992..fd455a7 100644 --- a/internal/handler/rewrite_cache.go +++ b/internal/handler/rewrite_cache.go @@ -35,8 +35,8 @@ import ( // depends on is fixed for the life of the process: the proxy URL is part of // the key, and the denylist is loaded at startup. The exception is cooldown, // which depends on the time of the request: with cooldown on, entries expire -// after Proxy.cooldownRewriteTTL, so a version is held back at most that long -// past its cooldown. +// when the first version they held back leaves cooldown (see +// cooldown_expiry.go). // // Each entry keeps the ETag of its output, so clients can revalidate their // copy without the output being hashed again on every request. Dates are not @@ -65,36 +65,6 @@ type rewriteEntry struct { expires time.Time } -// maxCooldownRewriteTTL is the longest a rewrite made with cooldown on is -// served. Versions leave cooldown as time passes, so a cached rewrite would -// otherwise keep hiding them. Cooldowns run to days, and a package manager -// that checks every lockfile entry on each install, as pnpm 12 does, would -// otherwise have the whole lockfile rewritten again on every install more -// than a minute after the last. -const maxCooldownRewriteTTL = time.Hour - -// minCooldownRewriteTTL is the shortest a rewrite made with cooldown on is -// served, however short the cooldown. -const minCooldownRewriteTTL = time.Minute - -// cooldownRewriteShare is the share of the shortest cooldown a rewrite made -// with cooldown on is served for, as a divisor. -const cooldownRewriteShare = 24 - -// cooldownRewriteTTL returns how long a rewrite made with cooldown on is -// served: a 24th of the shortest cooldown configured, so a version is held -// back at most about 4% past its cooldown, kept between a minute and an hour. -func (p *Proxy) cooldownRewriteTTL() time.Duration { - ttl := p.ShortestCooldown / cooldownRewriteShare - switch { - case ttl <= 0 || ttl > maxCooldownRewriteTTL: - return maxCooldownRewriteTTL - case ttl < minCooldownRewriteTTL: - return minCooldownRewriteTTL - } - return ttl -} - // size is how much of the cache's room the entry takes. func (e *rewriteEntry) size() int64 { return int64(len(e.plain) + len(e.gzipped)) @@ -279,7 +249,7 @@ func (c *rewriteCache) remove(el *list.Element) { // shared. A caller waiting on another's rewrite leaves when ctx ends; the // rewrite itself always runs to completion, so the result is cached for the // next request. -func (c *rewriteCache) rewrite(ctx context.Context, key string, in []byte, ttl time.Duration, rewrite func([]byte) ([]byte, error)) (rewrittenMetadata, error) { +func (c *rewriteCache) rewrite(ctx context.Context, key string, in []byte, expiry func(start time.Time) time.Time, rewrite func([]byte) ([]byte, error)) (rewrittenMetadata, error) { if c == nil { return rewriteWithETag(in, rewrite) } @@ -302,16 +272,17 @@ func (c *rewriteCache) rewrite(ctx context.Context, key string, in []byte, ttl t c.inFlight[key] = f c.mu.Unlock() - // The entry expires counting from before the rewrite, so it never - // outlives the time its cooldown decisions were made at by more than ttl. - var expires time.Time - if ttl > 0 { - expires = c.clock().Add(ttl) - } + // expiry is given the time from before the rewrite, so an entry never + // outlives the time its cooldown decisions were made at. + start := c.clock() defer func() { c.mu.Lock() delete(c.inFlight, key) if f.err == nil { + var expires time.Time + if expiry != nil { + expires = expiry(start) + } c.add(&rewriteEntry{key: key, rewrittenMetadata: f.m.compact(), expires: expires}) } c.mu.Unlock() @@ -370,7 +341,8 @@ func (c *rewriteCache) add(e *rewriteEntry) { // metadata cache stored it, and is empty otherwise; when known it keys the // cache, so the document isn't hashed again. // -// With cooldown on, the cached rewrite expires after p.cooldownRewriteTTL. Its +// With cooldown on, the cached rewrite expires when the first version it +// held back leaves cooldown, and after p.cooldownRewriteTTL at the latest. Its // ETag is the hash of its output, and it is dated like any other, so clients // revalidate with either validator and get the gzip copy. // @@ -390,11 +362,15 @@ func (p *Proxy) cachedRewrite(ctx context.Context, ecosystem, proxyURL, name str if digest == "" { key = rewriteCacheKey(ecosystem, proxyURL, name, in) } - var ttl time.Duration + var expiry func(time.Time) time.Time if cooldown { - ttl = p.cooldownRewriteTTL() + // Recording runs from before the rewrite until after it, so it + // hears of every version the rewrite holds back. + rec := p.watchCooldown() + defer p.unwatchCooldown(rec) + expiry = func(start time.Time) time.Time { return p.cooldownExpiry(rec, start) } } - m, err := p.rewrites.rewrite(ctx, key, in, ttl, rewriteOnly) + m, err := p.rewrites.rewrite(ctx, key, in, expiry, rewriteOnly) if err == nil { m.lastModified = p.metadataDate(ecosystem, proxyURL, name, m.etag) } diff --git a/internal/handler/rewrite_cache_test.go b/internal/handler/rewrite_cache_test.go index 7188634..3f4c062 100644 --- a/internal/handler/rewrite_cache_test.go +++ b/internal/handler/rewrite_cache_test.go @@ -44,7 +44,7 @@ func TestRewriteCache_RepeatServesCachedRewrite(t *testing.T) { key := rewriteCacheKey("npm", "http://proxy", "left-pad", []byte("doc")) for range 3 { - m, err := c.rewrite(context.Background(), key, []byte("doc"), 0, r.fn) + m, err := c.rewrite(context.Background(), key, []byte("doc"), nil, r.fn) if err != nil || string(m.content()) != "DOC" { t.Fatalf("rewrite = %q, %v", m.content(), err) } @@ -60,7 +60,7 @@ func TestRewriteCache_NewUpstreamBytesRewriteAgain(t *testing.T) { for _, doc := range []string{"v1", "v2"} { key := rewriteCacheKey("npm", "http://proxy", "left-pad", []byte(doc)) - m, err := c.rewrite(context.Background(), key, []byte(doc), 0, r.fn) + m, err := c.rewrite(context.Background(), key, []byte(doc), nil, r.fn) if err != nil || string(m.content()) != strings.ToUpper(doc) { t.Fatalf("rewrite(%s) = %q, %v", doc, m.content(), err) } @@ -82,7 +82,7 @@ func TestRewriteCache_ConcurrentRequestsShareRewrite(t *testing.T) { outs := make(chan string, n) run := func() { - m, err := c.rewrite(context.Background(), key, []byte("doc"), 0, r.fn) + m, err := c.rewrite(context.Background(), key, []byte("doc"), nil, r.fn) if err != nil { t.Error(err) } @@ -115,20 +115,20 @@ func TestRewriteCache_WaiterLeavingKeepsRewrite(t *testing.T) { first := make(chan error, 1) go func() { - _, err := c.rewrite(context.Background(), key, []byte("doc"), 0, r.fn) + _, err := c.rewrite(context.Background(), key, []byte("doc"), nil, r.fn) first <- err }() <-r.entered ctx, cancel := context.WithCancel(context.Background()) cancel() - if _, err := c.rewrite(ctx, key, []byte("doc"), 0, r.fn); !errors.Is(err, context.Canceled) { + if _, err := c.rewrite(ctx, key, []byte("doc"), nil, r.fn); !errors.Is(err, context.Canceled) { t.Errorf("waiter err = %v, want context.Canceled", err) } close(r.release) if err := <-first; err != nil { t.Fatal(err) } - if m, err := c.rewrite(context.Background(), key, []byte("doc"), 0, r.fn); err != nil || string(m.content()) != "DOC" { + if m, err := c.rewrite(context.Background(), key, []byte("doc"), nil, r.fn); err != nil || string(m.content()) != "DOC" { t.Errorf("cached rewrite = %q, %v", m.content(), err) } if got := r.calls.Load(); got != 1 { @@ -142,18 +142,18 @@ func TestRewriteCache_EvictsLeastRecentlyUsed(t *testing.T) { keyFor := func(doc string) string { return rewriteCacheKey("npm", "http://proxy", doc, []byte(doc)) } for _, doc := range []string{"aaa", "bbb"} { - _, _ = c.rewrite(context.Background(), keyFor(doc), []byte(doc), 0, r.fn) + _, _ = c.rewrite(context.Background(), keyFor(doc), []byte(doc), nil, r.fn) } - _, _ = c.rewrite(context.Background(), keyFor("aaa"), []byte("aaa"), 0, r.fn) // aaa is now the most recent - _, _ = c.rewrite(context.Background(), keyFor("ccc"), []byte("ccc"), 0, r.fn) // evicts bbb + _, _ = c.rewrite(context.Background(), keyFor("aaa"), []byte("aaa"), nil, r.fn) // aaa is now the most recent + _, _ = c.rewrite(context.Background(), keyFor("ccc"), []byte("ccc"), nil, r.fn) // evicts bbb before := r.calls.Load() - _, _ = c.rewrite(context.Background(), keyFor("aaa"), []byte("aaa"), 0, r.fn) - _, _ = c.rewrite(context.Background(), keyFor("ccc"), []byte("ccc"), 0, r.fn) + _, _ = c.rewrite(context.Background(), keyFor("aaa"), []byte("aaa"), nil, r.fn) + _, _ = c.rewrite(context.Background(), keyFor("ccc"), []byte("ccc"), nil, r.fn) if got := r.calls.Load() - before; got != 0 { t.Errorf("aaa and ccc rewritten %d times, want both still cached", got) } - _, _ = c.rewrite(context.Background(), keyFor("bbb"), []byte("bbb"), 0, r.fn) + _, _ = c.rewrite(context.Background(), keyFor("bbb"), []byte("bbb"), nil, r.fn) if got := r.calls.Load() - before; got != 1 { t.Errorf("bbb should have been evicted and rewritten") } @@ -164,7 +164,7 @@ func TestRewriteCache_OutputLargerThanCacheIsNotStored(t *testing.T) { var r countingRewrite key := rewriteCacheKey("npm", "http://proxy", "big", []byte("big")) - _, _ = c.rewrite(context.Background(), key, []byte("big"), 0, r.fn) + _, _ = c.rewrite(context.Background(), key, []byte("big"), nil, r.fn) if c.len() != 0 { t.Errorf("cached %d entries, want 0", c.len()) } @@ -177,7 +177,7 @@ func TestRewriteCache_ErrorsAreNotCached(t *testing.T) { key := rewriteCacheKey("npm", "http://proxy", "broken", []byte("doc")) for range 2 { - if _, err := c.rewrite(context.Background(), key, []byte("doc"), 0, failing); err == nil { + if _, err := c.rewrite(context.Background(), key, []byte("doc"), nil, failing); err == nil { t.Fatal("expected the rewrite error") } } @@ -232,7 +232,7 @@ func TestRewriteCacheExpiredEntryFreesRoom(t *testing.T) { clock := &testClock{t: time.Date(2026, 10, 10, 12, 0, 0, 0, time.UTC)} c.now = clock.now upper := func(b []byte) ([]byte, error) { return bytes.ToUpper(b), nil } - if _, err := c.rewrite(context.Background(), "k", []byte("doc"), time.Minute, upper); err != nil { + if _, err := c.rewrite(context.Background(), "k", []byte("doc"), func(start time.Time) time.Time { return start.Add(time.Minute) }, upper); err != nil { t.Fatal(err) } clock.advance(time.Minute) @@ -302,17 +302,16 @@ func (c *rewriteCache) len() int { return len(c.entries) } -// TestCooldownRewriteTTL checks that a rewrite filtered by cooldown is cached -// for a 24th of the shortest cooldown, between a minute and an hour. +// TestCooldownRewriteTTL checks that the cap on a rewrite filtered by +// cooldown is the shortest cooldown, between a minute and a day. func TestCooldownRewriteTTL(t *testing.T) { for _, tc := range []struct { shortest, want time.Duration }{ - {0, time.Hour}, - {72 * time.Hour, time.Hour}, - {24 * time.Hour, time.Hour}, - {2 * time.Hour, 5 * time.Minute}, - {10 * time.Minute, time.Minute}, + {0, 24 * time.Hour}, + {72 * time.Hour, 24 * time.Hour}, + {2 * time.Hour, 2 * time.Hour}, + {10 * time.Second, time.Minute}, } { p := &Proxy{ShortestCooldown: tc.shortest} if got := p.cooldownRewriteTTL(); got != tc.want { From 8c9e87f15ff8bc5c5022c5770176baf701e2d14d Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Fri, 30 Oct 2026 15:24:18 +0100 Subject: [PATCH 505/633] Stage fetched packuments, fall back on storage errors, stream version lookups A fetched packument is now stored under a staging path and checked before it replaces the cached copy, so an abbreviated one no longer overwrites a full copy; the full copy is served as when upstream fails. A failure to store falls back to the in-memory path, as caching is best effort there. Requests for a single version or dist-tag and the tarball download's cooldown and integrity lookup read the stored packument as a stream too, keeping only the one version entry they need. --- internal/handler/metadata_stored.go | 81 ++++++++-- internal/handler/npm.go | 223 +++++++++++++++++++++++---- internal/handler/npm_stream_test.go | 119 +++++++++++++- internal/handler/npm_test.go | 4 +- internal/metadata/npm/stream.go | 204 ++++++++++++++++++++---- internal/metadata/npm/stream_test.go | 73 ++++++++- 6 files changed, 617 insertions(+), 87 deletions(-) diff --git a/internal/handler/metadata_stored.go b/internal/handler/metadata_stored.go index e46239f..d831303 100644 --- a/internal/handler/metadata_stored.go +++ b/internal/handler/metadata_stored.go @@ -2,6 +2,7 @@ package handler import ( "context" + "crypto/rand" "crypto/sha256" "encoding/hex" "errors" @@ -36,8 +37,14 @@ var errStoredMetadataChanged = errors.New("stored metadata changed while it was // Concurrent calls for one document share a fetch, and instances sharing the // database take turns as for other metadata. // -// It returns errMetadataNotStored when the proxy doesn't cache metadata. -func (p *Proxy) storedMetadataDocument(ctx context.Context, ecosystem, cacheKey, upstreamURL, accept string) (*database.MetadataCacheEntry, error) { +// validate, when not nil, checks a fetched document before it replaces the +// stored one; a document it rejects is not stored, and the stored copy is +// served as when upstream fails. It does not check a stored copy within its +// TTL. +// +// It returns errMetadataNotStored when the proxy doesn't cache metadata or +// storing the document failed, and the caller reads it into memory instead. +func (p *Proxy) storedMetadataDocument(ctx context.Context, ecosystem, cacheKey, upstreamURL, accept string, validate func(io.Reader) error) (*database.MetadataCacheEntry, error) { if !p.CacheMetadata || p.DB == nil || p.Storage == nil || p.MetadataTTL <= 0 { return nil, errMetadataNotStored } @@ -78,7 +85,7 @@ func (p *Proxy) storedMetadataDocument(ctx context.Context, ecosystem, cacheKey, } res := metadataResult{err: errSharedFetchAbandoned} defer func() { release(res.err != nil || res.stale) }() - res = p.fetchStoredMetadata(fetchCtx, ecosystem, cacheKey, upstreamURL, accept, entry) + res = p.fetchStoredMetadata(fetchCtx, ecosystem, cacheKey, upstreamURL, accept, validate, entry) return res }) return res.entry, res.err @@ -101,20 +108,21 @@ func usableStoredMetadata(entry *database.MetadataCacheEntry) bool { // fetchStoredMetadata fetches a document from upstream into storage, // revalidating entry when there is one, and falls back to entry when -// upstream fails. -func (p *Proxy) fetchStoredMetadata(ctx context.Context, ecosystem, cacheKey, upstreamURL, accept string, entry *database.MetadataCacheEntry) metadataResult { +// upstream fails or validate rejects what it sent. A failure to store is +// returned as it is, for the caller to read the document into memory. +func (p *Proxy) fetchStoredMetadata(ctx context.Context, ecosystem, cacheKey, upstreamURL, accept string, validate func(io.Reader) error, entry *database.MetadataCacheEntry) metadataResult { revalidate := entry if !usableStoredMetadata(entry) { revalidate = nil } - stored, err := p.storeUpstreamMetadata(ctx, ecosystem, cacheKey, upstreamURL, accept, revalidate) + stored, err := p.storeUpstreamMetadata(ctx, ecosystem, cacheKey, upstreamURL, accept, validate, revalidate) if errors.Is(err, errStale304) { - stored, err = p.storeUpstreamMetadata(ctx, ecosystem, cacheKey, upstreamURL, accept, nil) + stored, err = p.storeUpstreamMetadata(ctx, ecosystem, cacheKey, upstreamURL, accept, validate, nil) } if err == nil { return metadataResult{entry: stored} } - if !usableStoredMetadata(entry) || errors.Is(err, ErrUpstreamNotFound) { + if !usableStoredMetadata(entry) || errors.Is(err, ErrUpstreamNotFound) || errors.Is(err, errMetadataNotStored) { return metadataResult{err: err} } return p.staleStoredMetadata(ecosystem, cacheKey, entry, err) @@ -136,7 +144,10 @@ func (p *Proxy) staleStoredMetadata(ecosystem, cacheKey string, entry *database. // returns its new cache row. On a 304 for entry it only moves the row's // fetched_at on; errStale304 means the row was replaced meanwhile and the // caller should fetch without validators. -func (p *Proxy) storeUpstreamMetadata(ctx context.Context, ecosystem, cacheKey, upstreamURL, accept string, entry *database.MetadataCacheEntry) (*database.MetadataCacheEntry, error) { +// +// With validate, the document is staged under a path of its own and checked +// there first, so one validate rejects never replaces the stored copy. +func (p *Proxy) storeUpstreamMetadata(ctx context.Context, ecosystem, cacheKey, upstreamURL, accept string, validate func(io.Reader) error, entry *database.MetadataCacheEntry) (*database.MetadataCacheEntry, error) { resp, err := p.requestUpstreamMetadata(ctx, upstreamURL, entry, accept, "") if err != nil { return nil, err @@ -166,8 +177,17 @@ func (p *Proxy) storeUpstreamMetadata(ctx context.Context, ecosystem, cacheKey, } // Go decompresses the body, since no Accept-Encoding was set, so it is // stored uncompressed. - if _, err := p.storeMetadataBlob(ctx, ecosystem, cacheKey, metadataStoragePath(ecosystem, cacheKey), resp.Body, meta); err != nil { - return nil, err + var body io.Reader = resp.Body + if validate != nil { + staged, err := p.stageMetadata(ctx, ecosystem, cacheKey, resp.Body, validate) + if err != nil { + return nil, err + } + defer func() { _ = staged.Close() }() + body = staged + } + if _, err := p.storeMetadataBlob(ctx, ecosystem, cacheKey, metadataStoragePath(ecosystem, cacheKey), body, meta); err != nil { + return nil, fmt.Errorf("%w: %w", errMetadataNotStored, err) } stored, err := p.DB.GetMetadataCache(ecosystem, cacheKey) if err != nil || !usableStoredMetadata(stored) { @@ -176,6 +196,45 @@ func (p *Proxy) storeUpstreamMetadata(ctx context.Context, ecosystem, cacheKey, return stored, nil } +// stageMetadata stores body under a staging path of its own, checks it with +// validate, and returns it opened for reading, to be stored in place of the +// cached copy. The staged copy is deleted when the reader is closed. A +// failure to store it is returned as errMetadataNotStored. +func (p *Proxy) stageMetadata(ctx context.Context, ecosystem, cacheKey string, body io.Reader, validate func(io.Reader) error) (io.ReadCloser, error) { + path := metadataStoragePath(ecosystem, cacheKey) + ".staged-" + rand.Text() + discard := func() { + if err := p.Storage.Delete(context.WithoutCancel(ctx), path); err != nil { + p.Logger.Warn("failed to delete staged metadata", "ecosystem", ecosystem, "key", cacheKey, "error", err) + } + } + if _, _, err := p.Storage.Store(ctx, path, body); err != nil { + discard() + return nil, fmt.Errorf("%w: staging: %w", errMetadataNotStored, err) + } + if err := readStored(func() (io.ReadCloser, error) { return p.Storage.Open(ctx, path) }, validate); err != nil { + discard() + return nil, err + } + staged, err := p.Storage.Open(ctx, path) + if err != nil { + discard() + return nil, fmt.Errorf("%w: reading staged copy: %w", errMetadataNotStored, err) + } + return &stagedReader{ReadCloser: staged, discard: discard}, nil +} + +// stagedReader deletes the staged copy it reads when closed. +type stagedReader struct { + io.ReadCloser + discard func() +} + +func (s *stagedReader) Close() error { + err := s.ReadCloser.Close() + s.discard() + return err +} + // openStoredMetadata opens the document entry describes. Reading it to the // end fails with errStoredMetadataChanged when its bytes don't have the // digest entry records. diff --git a/internal/handler/npm.go b/internal/handler/npm.go index 99c12ae..baddd97 100644 --- a/internal/handler/npm.go +++ b/internal/handler/npm.go @@ -2,6 +2,7 @@ package handler import ( "bytes" + "context" "errors" "fmt" "io" @@ -12,6 +13,7 @@ import ( "time" "github.com/git-pkgs/integrity" + "github.com/git-pkgs/proxy/internal/database" "github.com/git-pkgs/proxy/internal/jsonscan" "github.com/git-pkgs/proxy/internal/metadata/npm" "github.com/git-pkgs/proxy/internal/retention" @@ -329,19 +331,13 @@ func (h *NPMHandler) serveStreamedPackument(w http.ResponseWriter, r *http.Reque if h.proxy.rewrites == nil { return false } - entry, err := h.proxy.storedMetadataDocument(r.Context(), "npm", metadataCacheKey, upstreamURL, accept) + entry, err := h.storedPackument(r.Context(), upstreamURL, accept, metadataCacheKey) switch { case err == nil: - case errors.Is(err, errMetadataNotStored): + case errors.Is(err, errInMemory): return false - case errors.Is(err, ErrUpstreamNotFound): - JSONError(w, http.StatusNotFound, ErrPackageNotFound.Error()) - return true - case writeUpstreamHeld(w, err): - return true default: - h.proxy.Logger.Error("failed to fetch npm metadata", "error", err) - JSONError(w, http.StatusBadGateway, "failed to fetch from upstream") + h.writeFetchError(w, err) return true } @@ -378,6 +374,109 @@ func (h *NPMHandler) serveStreamedPackument(w http.ResponseWriter, r *http.Reque return true } +// npmFetchError is a failure to fetch a packument, answered as such rather +// than as a failure to filter it. +type npmFetchError struct{ err error } + +func (e *npmFetchError) Error() string { return e.err.Error() } +func (e *npmFetchError) Unwrap() error { return e.err } + +// writeFetchError answers a request whose packument could not be fetched. +func (h *NPMHandler) writeFetchError(w http.ResponseWriter, err error) { + switch { + case errors.Is(err, ErrUpstreamNotFound): + JSONError(w, http.StatusNotFound, ErrPackageNotFound.Error()) + case writeUpstreamHeld(w, err): + default: + h.proxy.Logger.Error("failed to fetch npm metadata", "error", err) + JSONError(w, http.StatusBadGateway, "failed to fetch from upstream") + } +} + +// errInMemory is returned when a packument has to be read into memory to +// answer the request: metadata is not cached in storage, or the stored +// document can't be read as a stream. +var errInMemory = errors.New("npm packument needs the in-memory path") + +// storedPackument returns the cache row of the packument stored for cacheKey, +// fetching it into storage first when needed. It returns errInMemory when +// metadata isn't cached in storage, and ErrUpstreamNotFound and upstream +// errors as they come. +func (h *NPMHandler) storedPackument(ctx context.Context, packumentURL, accept, cacheKey string) (*database.MetadataCacheEntry, error) { + entry, err := h.proxy.storedMetadataDocument(ctx, "npm", cacheKey, packumentURL, accept, h.packumentStreamValidator()) + if errors.Is(err, errMetadataNotStored) { + return nil, errInMemory + } + return entry, err +} + +// packumentStreamValidator is packumentValidator for a packument read from +// storage. +func (h *NPMHandler) packumentStreamValidator() func(io.Reader) error { + if h.packumentValidator() == nil { + return nil + } + return func(r io.Reader) error { + entry, err := npm.LookupVersion(r, "") + if err != nil { + return err + } + if entry.Abbreviated { + return errAbbreviatedPackument + } + return nil + } +} + +// streamedVersionMetadata resolves a version or dist-tag through the +// packument stored in storage, as rewriteNPMVersionFromPackument does in +// memory: one pass collects what cooldown and the denylist decide on, and a +// second finds the version entry. It returns errInMemory when the request +// has to take the in-memory path. +func (h *NPMHandler) streamedVersionMetadata(ctx context.Context, packageName, specifier, packumentURL, accept, cacheKey, metadataCacheKey string) (rewrittenMetadata, error) { + if h.proxy.rewrites == nil { + return rewrittenMetadata{}, errInMemory + } + entry, err := h.storedPackument(ctx, packumentURL, accept, cacheKey) + if errors.Is(err, errInMemory) { + return rewrittenMetadata{}, err + } + if err != nil { + return rewrittenMetadata{}, &npmFetchError{err} + } + fullRequired := h.packumentValidator() != nil + options := h.npmRewriteOptions(packageName, nil) + m, err := h.proxy.streamedRewrite(ctx, "npm", h.proxyURL, metadataCacheKey, entry, func(open func() (io.ReadCloser, error), out io.Writer) error { + var scan *npm.Scan + err := readStored(open, func(in io.Reader) (err error) { + scan, err = npm.ScanPackument(in) + return err + }) + if err != nil { + return err + } + if fullRequired && scan.Abbreviated() { + return npm.ErrNotStreamable + } + return readStored(open, func(in io.Reader) error { + version, err := npm.VersionStream(packageName, specifier, scan, in, options) + if err != nil { + return npmRewriteError(err) + } + _, err = out.Write(version) + return err + }) + }) + switch { + case err == nil, ctx.Err() != nil, errors.Is(err, ErrVersionDenied), errors.Is(err, ErrVersionNotFound): + return m, err + case !errors.Is(err, npm.ErrNotStreamable): + h.proxy.Logger.Warn("failed to read stored npm metadata, reading it in memory", + "package", packageName, "error", err) + } + return rewrittenMetadata{}, errInMemory +} + // readStored opens a stored document and reads it with read. func readStored(open func() (io.ReadCloser, error), read func(io.Reader) error) error { in, err := open() @@ -442,7 +541,19 @@ func requireFullPackument(body []byte) error { // filtered packument so cooldown and dist-tags match the root metadata response. func (h *NPMHandler) handleFilteredNPMVersionMetadata(w http.ResponseWriter, r *http.Request, packageName, specifier, accept, registryURL, cacheKey, metadataCacheKey string) { _, packumentPath := npmMetadataPath(packageName) - packument, _, _, err := h.proxy.fetchOrCacheMetadata(r.Context(), "npm", cacheKey, registryURL+"/"+packumentPath, "", h.packumentValidator(), accept) + packumentURL := registryURL + "/" + packumentPath + rewritten, err := h.streamedVersionMetadata(r.Context(), packageName, specifier, packumentURL, accept, cacheKey, metadataCacheKey) + var fetchErr *npmFetchError + switch { + case errors.As(err, &fetchErr): + h.writeFetchError(w, fetchErr.err) + return + case !errors.Is(err, errInMemory): + h.writeVersionMetadata(w, r, packageName, rewritten, err) + return + } + + packument, _, _, err := h.proxy.fetchOrCacheMetadata(r.Context(), "npm", cacheKey, packumentURL, "", h.packumentValidator(), accept) if err != nil { if errors.Is(err, ErrUpstreamNotFound) { JSONError(w, http.StatusNotFound, ErrPackageNotFound.Error()) @@ -456,10 +567,16 @@ func (h *NPMHandler) handleFilteredNPMVersionMetadata(w http.ResponseWriter, r * return } - rewritten, err := h.proxy.cachedRewrite(r.Context(), "npm", h.proxyURL, metadataCacheKey, packument, "", func(b []byte) ([]byte, []string, error) { + rewritten, err = h.proxy.cachedRewrite(r.Context(), "npm", h.proxyURL, metadataCacheKey, packument, "", func(b []byte) ([]byte, []string, error) { out, err := h.rewriteNPMVersionFromPackument(packageName, specifier, b) return out, nil, err }) + h.writeVersionMetadata(w, r, packageName, rewritten, err) +} + +// writeVersionMetadata answers a version metadata request with its rewrite +// or the error that kept it from being made. +func (h *NPMHandler) writeVersionMetadata(w http.ResponseWriter, r *http.Request, packageName string, rewritten rewrittenMetadata, err error) { if err != nil { if r.Context().Err() != nil { return @@ -620,10 +737,8 @@ func (h *NPMHandler) handleDownload(w http.ResponseWriter, r *http.Request) { return } registryURL, cacheKey := h.sourceFor(packageName) - metadata := sync.OnceValues(func() ([]byte, error) { - upstreamURL := registryURL + "/" + url.PathEscape(packageName) - body, _, _, err := h.proxy.fetchOrCacheMetadata(r.Context(), "npm", cacheKey, upstreamURL, "", h.packumentValidator(), h.packumentAccept()) - return body, err + metadata := sync.OnceValues(func() (npmVersionData, error) { + return h.versionData(r.Context(), registryURL+"/"+url.PathEscape(packageName), cacheKey, version) }) inCooldown, err := h.versionInCooldown(packageName, version, metadata) if err != nil { @@ -665,7 +780,7 @@ func (h *NPMHandler) handleDownload(w http.ResponseWriter, r *http.Request) { ServeArtifactRequest(w, r, result) } -func (h *NPMHandler) getTarball(r *http.Request, registryURL, packageName, version, filename string, metadata func() ([]byte, error)) (*CacheResult, error) { +func (h *NPMHandler) getTarball(r *http.Request, registryURL, packageName, version, filename string, metadata func() (npmVersionData, error)) (*CacheResult, error) { cacheFilename := h.routes.cacheFilename(packageName, filename) if cached, err := h.proxy.GetCachedArtifact(r.Context(), "npm", packageName, version, cacheFilename); err != nil || cached != nil { return cached, err @@ -673,35 +788,35 @@ func (h *NPMHandler) getTarball(r *http.Request, registryURL, packageName, versi // The packument says where the tarball is and what it must hash to. // Without it the tarball could not be verified, and an unverified copy // would be cached and served to everyone, so the download fails instead. - body, err := metadata() + data, err := metadata() if err != nil { return nil, fmt.Errorf("fetching npm metadata to verify tarball: %w", err) } downloadURL := fmt.Sprintf("%s/%s/-/%s", registryURL, escapeNPMDownloadPackage(packageName), url.PathEscape(filename)) - if tarball := npmVersionTarball(body, version); tarball != "" { + if tarball := npmEntryTarball(data.entry); tarball != "" { downloadURL, err = validateTarballURL(tarball, registryURL) if err != nil { return nil, err } } - sri := npmVersionIntegrity(body, version) + sri := npmEntryIntegrity(data.entry) return h.proxy.GetOrFetchArtifactFromURLWithIntegrity(r.Context(), "npm", packageName, version, cacheFilename, downloadURL, sri) } -// npmVersionIntegrity returns the integrity a packument declares for a -// version's tarball: dist.integrity when it is a Subresource Integrity string +// npmEntryIntegrity returns the integrity a version's packument entry +// declares for its tarball: dist.integrity when it is a Subresource Integrity string // the proxy can verify, or else the SHA-1 in dist.shasum, which is all // versions published before integrity existed carry, as "sha1-". It // returns "" when neither is usable, and the tarball is fetched unverified as // before. -func npmVersionIntegrity(body []byte, version string) string { - sri, _, _ := jsonscan.LookupString(body, "versions", version, "dist", "integrity") +func npmEntryIntegrity(entry []byte) string { + sri, _, _ := jsonscan.LookupString(entry, "dist", "integrity") if sri != "" { if _, err := integrity.ParseSRI(sri); err == nil { return sri } } - shasum, _, _ := jsonscan.LookupString(body, "versions", version, "dist", "shasum") + shasum, _, _ := jsonscan.LookupString(entry, "dist", "shasum") return sha1Checksum(shasum) } @@ -709,10 +824,57 @@ func npmVersionIntegrity(body []byte, version string) string { // or "" if it has none. It reads that one value in place rather than decoding // every version, since it runs on each tarball the proxy has not cached yet. func npmVersionTarball(body []byte, version string) string { - tarball, _, _ := jsonscan.LookupString(body, "versions", version, "dist", "tarball") + entry, _ := jsonscan.Lookup(body, "versions", version) + return npmEntryTarball(entry) +} + +// npmEntryTarball is npmVersionTarball for the version's entry. +func npmEntryTarball(entry []byte) string { + tarball, _, _ := jsonscan.LookupString(entry, "dist", "tarball") return tarball } +// npmVersionData is what a tarball download reads from the packument: the +// version's entry, nil when it has none, and its publish time from the time +// map, "" when it has none. +type npmVersionData struct { + entry []byte + published string +} + +// versionData reads what a tarball download needs about version from the +// packument: from storage in one pass when metadata is cached there, and +// otherwise from the packument read into memory. +func (h *NPMHandler) versionData(ctx context.Context, packumentURL, cacheKey, version string) (npmVersionData, error) { + accept := h.packumentAccept() + if entry, err := h.storedPackument(ctx, packumentURL, accept, cacheKey); err == nil { + var found npm.VersionEntry + err := readStored(func() (io.ReadCloser, error) { return h.proxy.openStoredMetadata(ctx, entry) }, func(in io.Reader) (err error) { + found, err = npm.LookupVersion(in, version) + return err + }) + switch { + case err == nil && (!found.Abbreviated || h.packumentValidator() == nil): + return npmVersionData{entry: found.Entry, published: found.Published}, nil + case err != nil: + h.proxy.Logger.Warn("failed to read stored npm metadata, reading it in memory", "url", packumentURL, "error", err) + } + } else if !errors.Is(err, errInMemory) { + return npmVersionData{}, err + } + + body, _, _, err := h.proxy.fetchOrCacheMetadata(ctx, "npm", cacheKey, packumentURL, "", h.packumentValidator(), accept) + if err != nil { + return npmVersionData{}, err + } + entry, _ := jsonscan.Lookup(body, "versions", version) + published, _, err := jsonscan.LookupString(body, "time", version) + if err != nil { + return npmVersionData{}, fmt.Errorf("parsing npm metadata: %w", err) + } + return npmVersionData{entry: entry, published: published}, nil +} + func validateTarballURL(raw, registryURL string) (string, error) { tarball, err := url.Parse(raw) if err != nil { @@ -756,7 +918,7 @@ func validateTarballURL(raw, registryURL string) (string, error) { // for a private version of the same name, a private time would hold back the // public version, and moving a route to another upstream would keep the old // one's times. -func (h *NPMHandler) versionInCooldown(packageName, version string, metadata func() ([]byte, error)) (bool, error) { +func (h *NPMHandler) versionInCooldown(packageName, version string, metadata func() (npmVersionData, error)) (bool, error) { if !h.proxy.cooldownApplies("npm", packageName) { return false, nil } @@ -769,7 +931,7 @@ func (h *NPMHandler) versionInCooldown(packageName, version string, metadata fun } } - body, err := metadata() + data, err := metadata() if errors.Is(err, ErrUpstreamNotFound) { return false, nil } @@ -777,11 +939,8 @@ func (h *NPMHandler) versionInCooldown(packageName, version string, metadata fun return false, err } - published, ok, err := jsonscan.LookupString(body, "time", version) - if err != nil { - return false, fmt.Errorf("parsing npm metadata: %w", err) - } - if !ok { + published := data.published + if published == "" { return false, nil } diff --git a/internal/handler/npm_stream_test.go b/internal/handler/npm_stream_test.go index 565f062..3442ce2 100644 --- a/internal/handler/npm_stream_test.go +++ b/internal/handler/npm_stream_test.go @@ -16,12 +16,18 @@ import ( "time" "github.com/git-pkgs/cooldown" + "github.com/git-pkgs/registries/fetch" ) // bigPackument returns a packument laid out as registry.npmjs.org writes // them, dist-tags before versions and time after, with n old versions padded // so the document is large, and a latest published an hour ago. func bigPackument(n int) string { + return bigPackumentAt("https://registry.npmjs.org", n) +} + +// bigPackumentAt is bigPackument with tarballs under base. +func bigPackumentAt(base string, n int) string { var b strings.Builder b.WriteString(`{"_id":"big","name":"big","dist-tags":{"latest":"9.0.0","old":"1.0.0"},"versions":{`) times := []string{`"created":"2015-01-01T00:00:00Z"`} @@ -30,12 +36,12 @@ func bigPackument(n int) string { if i == 0 { v = "1.0.0" } - fmt.Fprintf(&b, `%q:{"name":"big","version":%q,"readme":%q,"dist":{"tarball":"https://registry.npmjs.org/big/-/big-%s.tgz"}},`, - v, v, strings.Repeat("x", 512), v) + fmt.Fprintf(&b, `%q:{"name":"big","version":%q,"readme":%q,"dist":{"tarball":"%s/big/-/big-%s.tgz"}},`, + v, v, strings.Repeat("x", 512), base, v) times = append(times, fmt.Sprintf(`%q:"2016-01-01T00:%02d:%02dZ"`, v, i/60%60, i%60)) } recent := time.Now().Add(-time.Hour).UTC().Format(time.RFC3339) - b.WriteString(`"9.0.0":{"name":"big","version":"9.0.0","dist":{"tarball":"https://registry.npmjs.org/big/-/big-9.0.0.tgz"}}},`) + b.WriteString(`"9.0.0":{"name":"big","version":"9.0.0","dist":{"tarball":"` + base + `/big/-/big-9.0.0.tgz"}}},`) times = append(times, `"9.0.0":"`+recent+`"`) b.WriteString(`"time":{` + strings.Join(times, ",") + `},"readme":"big"}`) return b.String() @@ -200,7 +206,7 @@ func TestNPMStreamedPackumentStale(t *testing.T) { func TestNPMStreamedPackumentChangedInStorage(t *testing.T) { u := newStreamUpstream(t, bigPackument(10)) proxy, store, _ := streamProxy(t, u) - entry, err := proxy.storedMetadataDocument(context.Background(), "npm", "big", u.URL+"/big", contentTypeJSON) + entry, err := proxy.storedMetadataDocument(context.Background(), "npm", "big", u.URL+"/big", contentTypeJSON, nil) if err != nil { t.Fatal(err) } @@ -225,6 +231,9 @@ func TestNPMStreamedPackumentRevalidates(t *testing.T) { proxy, store, handler := streamProxy(t, u) first := serve(handler, "/big", nil) decodeStreamed(t, first) + store.mu.Lock() + stores := store.stores + store.mu.Unlock() proxy.MetadataTTL = time.Nanosecond second := serve(handler, "/big", nil) @@ -237,7 +246,105 @@ func TestNPMStreamedPackumentRevalidates(t *testing.T) { } store.mu.Lock() defer store.mu.Unlock() - if store.stores != 1 { - t.Errorf("document stored %d times, want once", store.stores) + if store.stores != stores { + t.Errorf("the 304 stored the document again (%d stores, %d before)", store.stores, stores) + } +} + +// TestNPMStreamedPackumentKeepsFullCopy checks that an abbreviated packument +// from upstream, where cooldown needs the full one, doesn't replace the full +// copy in storage, which is served instead as when upstream fails. +func TestNPMStreamedPackumentKeepsFullCopy(t *testing.T) { + full := bigPackument(10) + u := newStreamUpstream(t, full) + proxy, store, handler := streamProxy(t, u) + decodeStreamed(t, serve(handler, "/big", nil)) + + proxy.MetadataTTL = time.Nanosecond + proxy.SetMetadataRewriteCacheSize(64 << 20) + u.body.Store(`{"name":"big","modified":"2026-01-01T00:00:00Z","dist-tags":{"latest":"1.0.0"},"versions":{"1.0.0":{}}}`) + if doc := decodeStreamed(t, serve(handler, "/big", nil)); len(doc.Versions) != 10 { + t.Errorf("served %d versions, want the 10 of the full copy", len(doc.Versions)) + } + + store.mu.Lock() + defer store.mu.Unlock() + if got := string(store.files[metadataStoragePath("npm", "big")]); got != full { + t.Errorf("stored packument was replaced: %.80s", got) + } + for path := range store.files { + if strings.Contains(path, ".staged-") { + t.Errorf("staged copy %s left in storage", path) + } + } +} + +// TestNPMStreamedPackumentStorageFails checks that a packument is still +// served, read into memory, when storing it fails. +func TestNPMStreamedPackumentStorageFails(t *testing.T) { + u := newStreamUpstream(t, bigPackument(10)) + _, store, handler := streamProxy(t, u) + store.storeErr = errors.New("no space left on device") + doc := decodeStreamed(t, serve(handler, "/big", nil)) + if len(doc.Versions) != 10 || doc.DistTags["latest"] != "1.0.9" { + t.Errorf("served %d versions, latest %q; want 10 and 1.0.9", len(doc.Versions), doc.DistTags["latest"]) + } +} + +// TestNPMStreamedVersionMetadata resolves versions and dist-tags of a +// packument larger than metadata_max_size. +func TestNPMStreamedVersionMetadata(t *testing.T) { + body := bigPackument(100) + u := newStreamUpstream(t, body) + proxy, _, handler := streamProxy(t, u) + proxy.MetadataMaxSize = int64(len(body) / 4) + + for path, want := range map[string]string{ + "/big/1.0.5": "1.0.5", + "/big/latest": "1.0.99", // 9.0.0 is inside cooldown + "/big/old": "1.0.0", + } { + rec := serve(handler, path, nil) + var v struct { + Version string `json:"version"` + Dist struct { + Tarball string `json:"tarball"` + } `json:"dist"` + } + if rec.Code != http.StatusOK || json.Unmarshal(rec.Body.Bytes(), &v) != nil { + t.Errorf("%s: status %d: %.200s", path, rec.Code, rec.Body.String()) + continue + } + if v.Version != want || v.Dist.Tarball != "http://proxy.example/npm/big/-/big-"+want+".tgz" { + t.Errorf("%s = %s with tarball %s, want %s", path, v.Version, v.Dist.Tarball, want) + } + } + if rec := serve(handler, "/big/9.0.0", nil); rec.Code != http.StatusNotFound { + t.Errorf("version inside cooldown: status %d, want 404", rec.Code) + } + if n := u.requests.Load(); n != 1 { + t.Errorf("upstream asked %d times, want once", n) + } +} + +// TestNPMStreamedTarballCooldown checks the tarball download against a +// packument larger than metadata_max_size: the version's publish time and +// tarball URL are read from storage. +func TestNPMStreamedTarballCooldown(t *testing.T) { + u := newStreamUpstream(t, "") + u.body.Store(bigPackumentAt(u.URL, 100)) + proxy, _, handler := streamProxy(t, u) + proxy.MetadataMaxSize = 1024 + fetcher := proxy.Fetcher.(*mockFetcher) + fetcher.artifact = &fetch.Artifact{Body: io.NopCloser(strings.NewReader("package"))} + + if rec := serve(handler, "/big/-/big-9.0.0.tgz", nil); rec.Code != http.StatusNotFound { + t.Errorf("tarball inside cooldown: status %d, want 404", rec.Code) + } + if rec := serve(handler, "/big/-/big-1.0.5.tgz", nil); rec.Code != http.StatusOK || rec.Body.String() != "package" { + t.Errorf("tarball: status %d, body %q", rec.Code, rec.Body.String()) + } + if want := u.URL + "/big/-/big-1.0.5.tgz"; fetcher.fetchedURL != want { + t.Errorf("fetched %s, want %s from the packument", fetcher.fetchedURL, want) } } diff --git a/internal/handler/npm_test.go b/internal/handler/npm_test.go index 20fda4e..9b9a6c3 100644 --- a/internal/handler/npm_test.go +++ b/internal/handler/npm_test.go @@ -561,9 +561,9 @@ func TestNPMDownloadCooldownDisabled(t *testing.T) { proxy, _, _, _ := setupTestProxy(t) h := NewNPMHandler(proxy, "http://proxy.test", "") - if inCooldown, _ := h.versionInCooldown("leftpad", testVersion100, func() ([]byte, error) { + if inCooldown, _ := h.versionInCooldown("leftpad", testVersion100, func() (npmVersionData, error) { t.Fatal("cooldown must not request metadata when disabled") - return nil, nil + return npmVersionData{}, nil }); inCooldown { t.Error("versionInCooldown = true, want false when cooldown is not configured") } diff --git a/internal/metadata/npm/stream.go b/internal/metadata/npm/stream.go index cc70c52..3e86faa 100644 --- a/internal/metadata/npm/stream.go +++ b/internal/metadata/npm/stream.go @@ -99,50 +99,63 @@ func RewriteStream(packageName string, scan *Scan, r io.Reader, w io.Writer, opt return nil, ErrNotStreamable } h := &transformer{options} - - versions := make(map[string]any, len(scan.versions)) - for _, v := range scan.versions { - versions[v] = nil - } - metadata := map[string]any{} - timeMap := scan.objectValue(metadata, "time", scan.time) - distTags := scan.objectValue(metadata, "dist-tags", scan.distTags) - timeEntries := len(timeMap) - originalTags := maps.Clone(distTags) - - h.applyCooldownFiltering(metadata, versions, packageName) - h.applyDenylistFiltering(metadata, versions) - + f := scan.filter(h, packageName) sw := &streamWriter{ transformer: h, packageName: packageName, scan: scan, - versions: versions, - timeMap: timeMap, - distTags: distTags, - timeChanged: timeMap != nil && len(timeMap) != timeEntries, - tagsChanged: distTags != nil && !reflect.DeepEqual(distTags, originalTags), + filtered: f, seen: map[string]int{}, } if err := jsonstream.CopyObject(json.NewDecoder(r), w, sw.member); err != nil { return nil, err } h.tarballsRewritten(sw.rewritten) - return slices.Collect(maps.Keys(versions)), nil + return slices.Collect(maps.Keys(f.versions)), nil +} + +// filtered is what cooldown and the denylist leave of a scanned packument. +type filtered struct { + versions map[string]any + timeMap, distTags map[string]any + timeChanged, tagsChanged bool +} + +// filter runs the filters Rewrite runs over what scan collected. The scan is +// left as it is, so it can be filtered again. +func (s *Scan) filter(h *transformer, packageName string) filtered { + versions := make(map[string]any, len(s.versions)) + for _, v := range s.versions { + versions[v] = nil + } + metadata := map[string]any{} + timeMap := s.objectValue(metadata, "time", s.time) + distTags := s.objectValue(metadata, "dist-tags", s.distTags) + timeEntries := len(timeMap) + originalTags := maps.Clone(distTags) + + h.applyCooldownFiltering(metadata, versions, packageName) + h.applyDenylistFiltering(metadata, versions) + + return filtered{ + versions: versions, + timeMap: timeMap, + distTags: distTags, + timeChanged: timeMap != nil && len(timeMap) != timeEntries, + tagsChanged: distTags != nil && !reflect.DeepEqual(distTags, originalTags), + } } // streamWriter writes the second pass of RewriteStream from the filtered // state the first pass left. type streamWriter struct { *transformer - packageName string - scan *Scan - versions map[string]any - timeMap, distTags map[string]any - timeChanged, tagsChanged bool - seen map[string]int - entry bytes.Buffer - rewritten int + filtered + packageName string + scan *Scan + seen map[string]int + entry bytes.Buffer + rewritten int } // member writes one top-level member. Members the filters don't touch are @@ -201,13 +214,19 @@ func (s *streamWriter) time(t *jsonstream.Member) error { } // objectValue records value under name in metadata, as Rewrite's -// decodeObject does, and returns it when it is an object. +// decodeObject does, and returns it when it is an object. An object is +// copied, since the filters edit it. func (s *Scan) objectValue(metadata map[string]any, name string, value any) map[string]any { if s.occurrences[name] == 0 { return nil } - metadata[name] = value - object, _ := value.(map[string]any) + object, ok := value.(map[string]any) + if !ok { + metadata[name] = value + return nil + } + object = maps.Clone(object) + metadata[name] = object return object } @@ -268,3 +287,126 @@ func skipValue(dec *json.Decoder) error { var raw json.RawMessage return dec.Decode(&raw) } + +// VersionStream is Version for a packument too large to hold in memory. scan +// is the result of ScanPackument over the bytes r reads again; the second +// pass only looks for the one version entry it serves. For a document +// without a versions object it returns ErrNotStreamable. +func VersionStream(packageName, specifier string, scan *Scan, r io.Reader, options Options) ([]byte, error) { + if !scan.versionsObject || len(options.LocalVersions) > 0 { + return nil, ErrNotStreamable + } + h := &transformer{options} + f := scan.filter(h, packageName) + + target := specifier + if tag, ok := f.distTags[specifier].(string); ok { + target = tag + } + if options.DeniedVersions[target] { + return nil, ErrVersionDenied + } + if _, kept := f.versions[target]; !kept { + return nil, ErrVersionNotFound + } + entry, err := LookupVersion(r, target) + if err != nil { + return nil, err + } + if entry.Entry == nil { + return nil, ErrVersionNotFound + } + var out bytes.Buffer + h.writeNPMVersion(&out, packageName, target, entry.Entry) + return out.Bytes(), nil +} + +// VersionEntry is what a packument holds for one version. +type VersionEntry struct { + // Entry is versions[version], or nil when there is none. + Entry json.RawMessage + // Published is time[version], or "" when it is missing or not a string. + Published string + // Abbreviated is set for an abbreviated packument (see Abbreviated). + Abbreviated bool +} + +// LookupVersion reads a packument once and returns what it holds for +// version, keeping no other version's entry. Of repeated keys the last one +// counts, as with JSON.parse. +func LookupVersion(r io.Reader, version string) (VersionEntry, error) { + var e VersionEntry + var hasTime, hasModified bool + dec := json.NewDecoder(r) + if err := expectDelim(dec, '{'); err != nil { + return e, err + } + for dec.More() { + key, err := objectKey(dec) + if err != nil { + return e, err + } + switch key { + case "versions": + e.Entry, err = objectMember(dec, version) + case "time": + hasTime = true + var raw json.RawMessage + raw, err = objectMember(dec, version) + e.Published = "" + if err == nil && raw != nil { + _ = json.Unmarshal(raw, &e.Published) + } + default: + hasModified = hasModified || key == "modified" + err = skipValue(dec) + } + if err != nil { + return e, err + } + } + if err := expectDelim(dec, '}'); err != nil { + return e, err + } + if _, err := dec.Token(); err != io.EOF { + if err == nil { + err = errors.New("unexpected data after the top-level object") + } + return e, err + } + e.Abbreviated = hasModified && !hasTime + return e, nil +} + +// objectMember reads a value and returns its member key when it is an object +// that has one, the last if repeated, or nil. +func objectMember(dec *json.Decoder, key string) (json.RawMessage, error) { + tok, err := dec.Token() + if err != nil { + return nil, err + } + delim, ok := tok.(json.Delim) + if !ok { + return nil, nil + } + var found json.RawMessage + for dec.More() { + var k string + if delim == '{' { + if k, err = objectKey(dec); err != nil { + return nil, err + } + } + var raw json.RawMessage + if err := dec.Decode(&raw); err != nil { + return nil, err + } + if delim == '{' && k == key { + found = raw + } + } + if _, err := dec.Token(); err != nil { + return nil, err + } + return found, nil +} diff --git a/internal/metadata/npm/stream_test.go b/internal/metadata/npm/stream_test.go index ef2c954..0c8a4ce 100644 --- a/internal/metadata/npm/stream_test.go +++ b/internal/metadata/npm/stream_test.go @@ -24,11 +24,10 @@ func rewriteStream(packageName string, body []byte, options Options) ([]byte, [] return out.Bytes(), kept, err } -// TestRewriteStreamMatchesRewrite checks that the streamed rewrite writes the -// same bytes and keeps the same versions as Rewrite, over documents with the -// filtered members in either order, duplicates, and values Rewrite copies -// verbatim. -func TestRewriteStreamMatchesRewrite(t *testing.T) { +// streamCases returns packuments with the filtered members in either +// order, duplicates and values Rewrite copies verbatim, and the options to +// rewrite them with. +func streamCases() (map[string]string, map[string]Options) { now := time.Date(2026, 10, 18, 12, 0, 0, 0, time.UTC) ts := func(age time.Duration) string { return now.Add(-age).Format(time.RFC3339) } version := func(v string) string { @@ -60,6 +59,15 @@ func TestRewriteStreamMatchesRewrite(t *testing.T) { "both": {ProxyURL: "http://proxy", Cooldown: &cooldown.Config{Default: "3d"}, EvaluatedAt: now, DeniedVersions: map[string]bool{"1.0.0": true}}, } + return docs, options +} + +// TestRewriteStreamMatchesRewrite checks that the streamed rewrite writes the +// same bytes and keeps the same versions as Rewrite, over documents with the +// filtered members in either order, duplicates, and values Rewrite copies +// verbatim. +func TestRewriteStreamMatchesRewrite(t *testing.T) { + docs, options := streamCases() for docName, doc := range docs { for optName, opts := range options { t.Run(docName+"/"+optName, func(t *testing.T) { @@ -160,3 +168,58 @@ func TestScanPackument(t *testing.T) { } } } + +// TestVersionStreamMatchesVersion checks that a single version resolved from +// the stream is the one Version serves, with the same errors. +func TestVersionStreamMatchesVersion(t *testing.T) { + docs, options := streamCases() + specifiers := []string{"latest", "beta", "next", "old", "1.0.0", "2.0.0", "3.0.0", "9.0.0", "missing"} + for docName, doc := range docs { + scan, err := ScanPackument(strings.NewReader(doc)) + if err != nil { + t.Fatalf("%s: %v", docName, err) + } + for optName, opts := range options { + for _, spec := range specifiers { + want, wantErr := Version("demo", spec, []byte(doc), opts) + got, gotErr := VersionStream("demo", spec, scan, strings.NewReader(doc), opts) + name := docName + "/" + optName + "/" + spec + switch { + case wantErr != nil || gotErr != nil: + if !errors.Is(gotErr, ErrVersionDenied) == errors.Is(wantErr, ErrVersionDenied) || + !errors.Is(gotErr, ErrVersionNotFound) == errors.Is(wantErr, ErrVersionNotFound) || + (gotErr == nil) != (wantErr == nil) { + t.Errorf("%s: error %v, Version gave %v", name, gotErr, wantErr) + } + case !bytes.Equal(got, want): + t.Errorf("%s:\n%s\nVersion:\n%s", name, got, want) + } + } + } + } +} + +func TestLookupVersion(t *testing.T) { + doc := `{"modified":"x","versions":{"1.0.0":{"a":1}},"time":{"1.0.0":"2020-01-01T00:00:00Z","2.0.0":7},` + + `"versions":{"1.0.0":{"a":2},"2.0.0":{"b":1}}}` + for version, want := range map[string]VersionEntry{ + "1.0.0": {Entry: json.RawMessage(`{"a":2}`), Published: "2020-01-01T00:00:00Z"}, + "2.0.0": {Entry: json.RawMessage(`{"b":1}`)}, + "3.0.0": {}, + } { + got, err := LookupVersion(strings.NewReader(doc), version) + if err != nil { + t.Fatal(err) + } + if string(got.Entry) != string(want.Entry) || got.Published != want.Published || got.Abbreviated { + t.Errorf("LookupVersion(%s) = %+v, want %+v", version, got, want) + } + } + got, err := LookupVersion(strings.NewReader(`{"modified":"x","versions":{}}`), "1.0.0") + if err != nil || !got.Abbreviated { + t.Errorf("abbreviated packument: %+v, %v", got, err) + } + if _, err := LookupVersion(strings.NewReader(`{"versions":{}`), "1.0.0"); err == nil { + t.Error("expected an error for truncated JSON") + } +} From ddba210a60d082fedf2c98bac8dc1bd60a1d6b40 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Fri, 30 Oct 2026 15:31:31 +0100 Subject: [PATCH 506/633] Serialize snapshot recording per document on Postgres AddMetadataSnapshot takes a transaction-scoped advisory lock on the document before its check and insert, so two replicas recording the same version can't both read the previous digest and both insert. SQLite runs one write at a time, so it keeps the single statement. --- internal/database/metadata_snapshots.go | 36 ++++++++++-- internal/database/metadata_snapshots_test.go | 52 +++++++++++++++++ internal/handler/apk_snapshots_test.go | 60 ++++++++++++++++++++ 3 files changed, 143 insertions(+), 5 deletions(-) diff --git a/internal/database/metadata_snapshots.go b/internal/database/metadata_snapshots.go index 95edacc..fba774a 100644 --- a/internal/database/metadata_snapshots.go +++ b/internal/database/metadata_snapshots.go @@ -62,11 +62,25 @@ func (db *DB) metadataSnapshotBefore(ecosystem, name string, beforeMS int64) (*M // AddMetadataSnapshot records snap as the current version of its document, // unless the newest snapshot already has its digest, and reports whether it -// did. The check and the insert are one statement, so concurrent callers -// recording the same document add it once. The casts let Postgres type the -// parameters of the INSERT ... SELECT. +// did. On Postgres, replicas recording the same document take an advisory +// lock on it first, so their check and insert don't interleave: without it, +// two transactions can both read the previous digest and both insert. SQLite +// runs one write at a time, so the single statement is enough there. A +// document going back to an earlier digest is recorded again, since only the +// newest snapshot is compared. The casts let Postgres type the parameters of +// the INSERT ... SELECT. func (db *DB) AddMetadataSnapshot(snap *MetadataSnapshot) (bool, error) { - res, err := db.Exec(db.Rebind(` + tx, err := db.Beginx() + if err != nil { + return false, fmt.Errorf("recording metadata snapshot: %w", err) + } + defer func() { _ = tx.Rollback() }() + if db.dialect == DialectPostgres { + if _, err := tx.Exec("SELECT pg_advisory_xact_lock($1)", metadataSnapshotLockKey(snap.Ecosystem, snap.Name)); err != nil { + return false, fmt.Errorf("locking metadata snapshots: %w", err) + } + } + res, err := tx.Exec(db.Rebind(` INSERT INTO metadata_snapshots (ecosystem, name, storage_path, content_type, content_encoding, content_digest, size, valid_from_ms) SELECT CAST(? AS TEXT), CAST(? AS TEXT), CAST(? AS TEXT), CAST(? AS TEXT), CAST(? AS TEXT), @@ -84,7 +98,19 @@ func (db *DB) AddMetadataSnapshot(snap *MetadataSnapshot) (bool, error) { return false, fmt.Errorf("recording metadata snapshot: %w", err) } n, err := res.RowsAffected() - return n > 0, err + if err != nil { + return false, err + } + if err := tx.Commit(); err != nil { + return false, fmt.Errorf("recording metadata snapshot: %w", err) + } + return n > 0, nil +} + +// metadataSnapshotLockKey is the advisory lock AddMetadataSnapshot takes for +// a document. +func metadataSnapshotLockKey(ecosystem, name string) int64 { + return lockKey("metadata-snapshot:" + ecosystem + "\x00" + name) } // PruneMetadataSnapshots deletes the snapshots of ecosystem that stopped diff --git a/internal/database/metadata_snapshots_test.go b/internal/database/metadata_snapshots_test.go index ae859fd..0b4919f 100644 --- a/internal/database/metadata_snapshots_test.go +++ b/internal/database/metadata_snapshots_test.go @@ -106,3 +106,55 @@ func TestMetadataSnapshots_Prune(t *testing.T) { } }) } + +// TestMetadataSnapshots_ConcurrentPostgres records a version while another +// replica's transaction has recorded the same one and not committed yet, as +// two replicas can. The second must wait and then find it, so one row +// remains. Without the lock both inserts succeed. +func TestMetadataSnapshots_ConcurrentPostgres(t *testing.T) { + db := createTestPostgresDB(t) + if db == nil { + t.Skip("PROXY_DATABASE_URL not set, skipping postgres test") + } + defer func() { _ = db.Close() }() + const name = "v3.22/main/x86_64/APKINDEX.tar.gz" + + other, err := db.Beginx() + if err != nil { + t.Fatal(err) + } + defer func() { _ = other.Rollback() }() + if _, err := other.Exec("SELECT pg_advisory_xact_lock($1)", metadataSnapshotLockKey("alpine", name)); err != nil { + t.Fatal(err) + } + if _, err := other.Exec(`INSERT INTO metadata_snapshots (ecosystem, name, storage_path, content_digest, size, valid_from_ms) + VALUES ('alpine', $1, 'other', 'sha256:a', 1, $2)`, name, time.Now().UnixMilli()); err != nil { + t.Fatal(err) + } + + done := make(chan bool, 1) + go func() { done <- addSnapshot(t, db, name, "sha256:a", time.Now()) }() + select { + case added := <-done: + t.Fatalf("recorded without waiting for the other replica (added=%v)", added) + case <-time.After(200 * time.Millisecond): + } + if err := other.Commit(); err != nil { + t.Fatal(err) + } + if added := <-done; added { + t.Error("recorded a version the other replica had just recorded") + } + var n int + if err := db.Get(&n, `SELECT COUNT(*) FROM metadata_snapshots WHERE name = $1`, name); err != nil { + t.Fatal(err) + } + if n != 1 { + t.Errorf("%d snapshot rows, want 1", n) + } + + // Going back to an earlier digest is still recorded. + if !addSnapshot(t, db, name, "sha256:b", time.Now()) || !addSnapshot(t, db, name, "sha256:a", time.Now().Add(time.Millisecond)) { + t.Error("a change back to an earlier digest was not recorded") + } +} diff --git a/internal/handler/apk_snapshots_test.go b/internal/handler/apk_snapshots_test.go index 52b5ff7..38d4b08 100644 --- a/internal/handler/apk_snapshots_test.go +++ b/internal/handler/apk_snapshots_test.go @@ -3,6 +3,8 @@ package handler import ( "net/http" "net/http/httptest" + "strings" + "sync" "sync/atomic" "testing" "time" @@ -203,3 +205,61 @@ func TestAPKSnapshots_SameVersionRecordedTwice(t *testing.T) { day := validFrom.UTC().Format(apkSnapshotDateLayout) expectAPKResponse(t, h, "/alpine@"+day+apkSnapshotIndexPath, http.StatusOK, "index v1") } + +// TestAPKSnapshots_ConcurrentReplicas records the same index version from +// two replicas sharing a Postgres database, many times at once. One snapshot +// row must remain, with a readable blob, and the other copies are deleted. +func TestAPKSnapshots_ConcurrentReplicas(t *testing.T) { + a, b, db := sharedInstances(t) + u := newAPKSnapshotUpstream(t) + handlers := make([]*APKHandler, 0, 2) + for _, p := range []*Proxy{a, b} { + p.CacheMetadata = true + p.HTTPClient = http.DefaultClient + handlers = append(handlers, NewAPKHandler(p, "http://proxy.example", map[string]string{"alpine": u.server.URL}). + WithSnapshots(30*24*time.Hour)) + } + expectAPKResponse(t, handlers[0], "/alpine"+apkSnapshotIndexPath, http.StatusOK, "index v1") + + key := handlers[0].metadataCacheKey("alpine", u.server.URL, apkSnapshotIndexPath[1:]) + entry, err := db.GetMetadataCache(apkEcosystem, key) + if err != nil || entry == nil { + t.Fatalf("cached index row = %v, %v", entry, err) + } + if _, err := db.Exec(`DELETE FROM metadata_snapshots`); err != nil { + t.Fatal(err) + } + // The request above recorded a copy of its own, which the test leaves be. + store := a.Storage.(*mockStorage) + store.mu.Lock() + before := map[string]bool{} + for path := range store.files { + before[path] = true + } + store.mu.Unlock() + + validFrom := time.Now() + var wg sync.WaitGroup + for i := range 16 { + wg.Go(func() { handlers[i%2].storeSnapshot(t.Context(), key, entry, validFrom) }) + } + wg.Wait() + + if n := snapshotCount(t, db); n != 1 { + t.Fatalf("%d snapshots recorded, want 1", n) + } + snap, err := db.LatestMetadataSnapshot(apkEcosystem, key) + if err != nil || snap == nil { + t.Fatalf("latest snapshot = %v, %v", snap, err) + } + store.mu.Lock() + defer store.mu.Unlock() + if got := string(store.files[snap.StoragePath]); got != "index v1" { + t.Errorf("snapshot blob = %q, want the index", got) + } + for path := range store.files { + if strings.HasPrefix(path, "_snapshots/") && path != snap.StoragePath && !before[path] { + t.Errorf("copy %s of a snapshot that wasn't recorded is left in storage", path) + } + } +} From ba95de17f30d9068b2fa61b8dad8eb10a76495ad Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Fri, 30 Oct 2026 14:41:45 +0000 Subject: [PATCH 507/633] Handle Swift registry identities in cache and vulnerability APIs --- go.mod | 2 +- go.sum | 4 +- internal/enrichment/enrichment.go | 9 ++-- internal/enrichment/enrichment_test.go | 11 +++-- internal/packageurl/packageurl.go | 24 ++++------- internal/packageurl/packageurl_test.go | 25 +++++------ internal/server/api.go | 9 ++++ internal/server/swift_vulns_test.go | 57 ++++++++++++++++++++++++++ 8 files changed, 102 insertions(+), 39 deletions(-) create mode 100644 internal/server/swift_vulns_test.go diff --git a/go.mod b/go.mod index 4a164c2..79978b1 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/git-pkgs/archives v0.10.0 github.com/git-pkgs/artifacts v0.2.1 github.com/git-pkgs/cooldown v0.3.0 - github.com/git-pkgs/enrichment v0.7.4 + github.com/git-pkgs/enrichment v0.7.5 github.com/git-pkgs/gcs v0.1.0 github.com/git-pkgs/integrity v0.2.0 github.com/git-pkgs/magic v0.6.0 diff --git a/go.sum b/go.sum index 983c87e..7ed7c29 100644 --- a/go.sum +++ b/go.sum @@ -123,8 +123,8 @@ github.com/git-pkgs/artifacts v0.2.1 h1:VwdxR4yTDaqBZ34h0slxQBVyr2Xfa+QGOKCKviMx github.com/git-pkgs/artifacts v0.2.1/go.mod h1:Otosgq52pXT5UNN7lh6s/lszpWKVDO8XrOjN1M70/IA= github.com/git-pkgs/cooldown v0.3.0 h1:eAl2CQFwqXxluoVP9hHmropAf1VoV3gSxLMkN+NEu8A= github.com/git-pkgs/cooldown v0.3.0/go.mod h1:FnTg8wufEALATC9Sm/9AHjLdA+u9eVn96e9BqXXjGB0= -github.com/git-pkgs/enrichment v0.7.4 h1:owiCKjITG8tNxBikcyUCDuDf1i3C/+Yoa5yJseuqIjA= -github.com/git-pkgs/enrichment v0.7.4/go.mod h1:ZN/cH4PnmzsapEVPvmAwREP7+ox+RX0yfR6KvoB8xKE= +github.com/git-pkgs/enrichment v0.7.5 h1:jbfFzf7+HSS7el8AuMri4vHXq2izIIZDVIU0uYI5LFM= +github.com/git-pkgs/enrichment v0.7.5/go.mod h1:TbxC/nGQN96YCbdoiDs8a9z3gk2zXmLVS1989/jvnsU= github.com/git-pkgs/gcs v0.1.0 h1:E3awGtsO0xZyHT9FUfEwMHjMkRxw41Bh+c7lgTmPvBo= github.com/git-pkgs/gcs v0.1.0/go.mod h1:bdkCFD66ryaWnU8MBhokVA3WkJfyAEchm9qec5woBpE= github.com/git-pkgs/integrity v0.2.0 h1:sxpAk7EUllZnTYttOoPTW/Yfe5J2imglpuJiuBUZOxM= diff --git a/internal/enrichment/enrichment.go b/internal/enrichment/enrichment.go index 4d847af..2246e39 100644 --- a/internal/enrichment/enrichment.go +++ b/internal/enrichment/enrichment.go @@ -100,9 +100,6 @@ func (s *Service) BulkEnrichPackages(ctx context.Context, packages []struct{ Eco // CheckVulnerabilities queries for vulnerabilities affecting a package version. func (s *Service) CheckVulnerabilities(ctx context.Context, ecosystem, name, version string) ([]VulnInfo, error) { - if packageurl.Make(ecosystem, name, version) == nil { - return nil, nil - } return s.vulns.Check(ctx, ecosystem, name, version) } @@ -137,7 +134,8 @@ type EnrichmentResult struct { } // EnrichFull fetches package, version and vulnerability data for a package -// version concurrently. Failed lookups are logged and left empty. +// version concurrently. Unsupported vulnerability identities return an error; +// other failed lookups are logged and left empty. func (s *Service) EnrichFull(ctx context.Context, ecosystem, name, version string) (*EnrichmentResult, error) { result := &EnrichmentResult{} @@ -147,6 +145,9 @@ func (s *Service) EnrichFull(ctx context.Context, ecosystem, name, version strin wg.Go(func() { result.Version, verErr = s.EnrichVersion(ctx, ecosystem, name, version) }) wg.Go(func() { result.Vulnerabilities, vulnErr = s.CheckVulnerabilities(ctx, ecosystem, name, version) }) wg.Wait() + if errors.Is(vulnErr, shared.ErrUnsupportedVulnerabilityIdentity) { + return nil, vulnErr + } if pkgErr != nil { s.logger.Debug("failed to enrich package", "ecosystem", ecosystem, "name", name, "error", pkgErr) diff --git a/internal/enrichment/enrichment_test.go b/internal/enrichment/enrichment_test.go index a854e5a..f01bb29 100644 --- a/internal/enrichment/enrichment_test.go +++ b/internal/enrichment/enrichment_test.go @@ -2,10 +2,12 @@ package enrichment import ( "context" + "errors" "log/slog" "os" "testing" + shared "github.com/git-pkgs/enrichment" "github.com/git-pkgs/registries" ) @@ -26,7 +28,7 @@ func TestNew(t *testing.T) { } } -func TestSwiftRegistryIdentitySkipsPURLDependentLookups(t *testing.T) { +func TestSwiftRegistryIdentityLookups(t *testing.T) { svc := New(slog.New(slog.NewTextHandler(os.Stdout, nil))) ctx := context.Background() @@ -41,8 +43,11 @@ func TestSwiftRegistryIdentitySkipsPURLDependentLookups(t *testing.T) { } vulnerabilities, err := svc.CheckVulnerabilities(ctx, "swift", "apple/example", "1.2.3") - if err != nil || vulnerabilities != nil { - t.Errorf("CheckVulnerabilities() = %#v, %v; want nil, nil", vulnerabilities, err) + if !errors.Is(err, shared.ErrUnsupportedVulnerabilityIdentity) || vulnerabilities != nil { + t.Errorf("CheckVulnerabilities() = %#v, %v; want unsupported identity", vulnerabilities, err) + } + if _, err := svc.EnrichFull(ctx, "swift", "apple/example", "1.2.3"); !errors.Is(err, shared.ErrUnsupportedVulnerabilityIdentity) { + t.Errorf("EnrichFull error = %v, want unsupported identity", err) } latest, err := svc.GetLatestVersion(ctx, "swift", "apple/example") diff --git a/internal/packageurl/packageurl.go b/internal/packageurl/packageurl.go index 7110942..be0e9ab 100644 --- a/internal/packageurl/packageurl.go +++ b/internal/packageurl/packageurl.go @@ -29,25 +29,19 @@ func WithVersionString(packagePURL, version string) string { } // MakeCacheStrings returns package and version PURLs suitable for artifact -// cache records. Swift registry identities use an explicit generic PURL until -// their source repository has been resolved. The result is independent of the -// configured upstream so cache entries survive an upstream.swift change, -// matching every other ecosystem. +// cache records. Swift registry identities keep their generic storage PURLs. +// Cache keys are independent of the configured upstream. func MakeCacheStrings(ecosystem, name, version string) (packagePURL, versionPURL string) { + if purl.NormalizeEcosystem(ecosystem) == "swift" { + if identity, ok := swiftRegistryIdentity(name); ok { + pkg := purl.New("generic", "swift-registry", identity, "", nil) + return pkg.String(), pkg.WithVersion(version).String() + } + } if pkg := Make(ecosystem, name, ""); pkg != nil { return pkg.String(), pkg.WithVersion(version).String() } - if purl.NormalizeEcosystem(ecosystem) != "swift" { - return "", "" - } - - identity, ok := swiftRegistryIdentity(name) - if !ok { - return "", "" - } - - pkg := purl.New("generic", "swift-registry", identity, "", nil) - return pkg.String(), pkg.WithVersion(version).String() + return "", "" } func swiftRegistryIdentity(name string) (string, bool) { diff --git a/internal/packageurl/packageurl_test.go b/internal/packageurl/packageurl_test.go index 8982288..5a46776 100644 --- a/internal/packageurl/packageurl_test.go +++ b/internal/packageurl/packageurl_test.go @@ -2,17 +2,12 @@ package packageurl import "testing" -func TestMakeSwiftRegistryIdentityUnsupported(t *testing.T) { - identities := []string{"apple.swift-argument-parser", "apple/swift-argument-parser"} - for _, identity := range identities { - t.Run(identity, func(t *testing.T) { - if got := Make("swift", identity, "1.8.2"); got != nil { - t.Errorf("Make() = %q, want nil", got.String()) - } - if got := MakeString("swift", identity, "1.8.2"); got != "" { - t.Errorf("MakeString() = %q, want empty string", got) - } - }) +func TestMakeSwiftRegistryPathUnsupported(t *testing.T) { + if got := Make("swift", "apple/swift-argument-parser", "1.8.2"); got != nil { + t.Errorf("Make() = %q, want nil", got.String()) + } + if got := MakeString("swift", "apple/swift-argument-parser", "1.8.2"); got != "" { + t.Errorf("MakeString() = %q, want empty string", got) } } @@ -49,9 +44,11 @@ func TestMakeCacheStringsSwiftRegistryIdentity(t *testing.T) { t.Errorf("version PURL = %q, want %q", versionPURL, wantVersion) } - dottedPackage, dottedVersion := MakeCacheStrings("swift", "apple.example", "1.2.3") - if dottedPackage != packagePURL || dottedVersion != versionPURL { - t.Errorf("dotted identity cache PURLs = %q, %q; want %q, %q", dottedPackage, dottedVersion, packagePURL, versionPURL) + for _, identity := range []string{"apple.example", "APPLE.EXAMPLE", "apple/example"} { + dottedPackage, dottedVersion := MakeCacheStrings("Swift", identity, "1.2.3") + if dottedPackage != packagePURL || dottedVersion != versionPURL { + t.Errorf("%q cache PURLs = %q, %q; want %q, %q", identity, dottedPackage, dottedVersion, packagePURL, versionPURL) + } } } diff --git a/internal/server/api.go b/internal/server/api.go index b204367..22bfb8a 100644 --- a/internal/server/api.go +++ b/internal/server/api.go @@ -2,6 +2,7 @@ package server import ( "encoding/json" + "errors" "net/http" "strings" @@ -219,6 +220,10 @@ func (h *APIHandler) getPackage(w http.ResponseWriter, r *http.Request, ecosyste func (h *APIHandler) getVersion(w http.ResponseWriter, r *http.Request, ecosystem, name, version string) { result, err := h.enrichment.EnrichFull(r.Context(), ecosystem, name, version) + if errors.Is(err, shared.ErrUnsupportedVulnerabilityIdentity) { + badRequest(w, "package identity is not supported for vulnerability lookups") + return + } if err != nil { writeError(w, http.StatusBadGateway, ErrCodeUpstream, "failed to enrich version") return @@ -303,6 +308,10 @@ func (h *APIHandler) HandleVulnsPath(w http.ResponseWriter, r *http.Request) { } vulns, err := h.enrichment.CheckVulnerabilities(r.Context(), ecosystem, name, version) + if errors.Is(err, shared.ErrUnsupportedVulnerabilityIdentity) { + badRequest(w, "package identity is not supported for vulnerability lookups") + return + } if err != nil { writeError(w, http.StatusBadGateway, ErrCodeUpstream, "failed to check vulnerabilities") return diff --git a/internal/server/swift_vulns_test.go b/internal/server/swift_vulns_test.go new file mode 100644 index 0000000..f28caaf --- /dev/null +++ b/internal/server/swift_vulns_test.go @@ -0,0 +1,57 @@ +package server + +import ( + "encoding/json" + "fmt" + "log/slog" + "net/http" + "net/http/httptest" + "testing" + + "github.com/git-pkgs/proxy/internal/enrichment" + "github.com/go-chi/chi/v5" +) + +func TestSwiftRegistryVulnerabilityErrors(t *testing.T) { + h := NewAPIHandler(enrichment.New(slog.Default()), nil) + r := chi.NewRouter() + r.Get("/api/vulns/{ecosystem}/*", h.HandleVulnsPath) + r.Get("/api/package/{ecosystem}/*", h.HandlePackagePath) + transport := &swiftVulnsTransport{} + original := http.DefaultTransport + http.DefaultTransport = transport + t.Cleanup(func() { http.DefaultTransport = original }) + + for _, path := range []string{ + "/api/vulns/swift/apple.swift-nio", + "/api/vulns/swift/apple.swift-nio/2.0.0", + "/api/vulns/Swift/Apple.Swift-NIO/2.0.0", + "/api/package/swift/apple.swift-nio/2.0.0", + "/api/package/Swift/Apple.Swift-NIO/2.0.0", + } { + t.Run(path, func(t *testing.T) { + w := httptest.NewRecorder() + r.ServeHTTP(w, httptest.NewRequest(http.MethodGet, path, nil)) + if w.Code != http.StatusBadRequest { + t.Fatalf("status = %d, body = %s", w.Code, w.Body) + } + var response ErrorResponse + if err := json.Unmarshal(w.Body.Bytes(), &response); err != nil { + t.Fatal(err) + } + if response.Code != ErrCodeBadRequest || response.Message != "package identity is not supported for vulnerability lookups" { + t.Fatalf("unexpected response: %+v", response) + } + }) + } + if transport.requests != 0 { + t.Errorf("made %d upstream requests for unsupported identities", transport.requests) + } +} + +type swiftVulnsTransport struct{ requests int } + +func (s *swiftVulnsTransport) RoundTrip(r *http.Request) (*http.Response, error) { + s.requests++ + return nil, fmt.Errorf("unexpected upstream request: %s", r.URL) +} From f32b46b71ae1f1f9cfd92e21b2a9bfb3deaa72d2 Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Fri, 30 Oct 2026 18:02:30 +0000 Subject: [PATCH 508/633] Preserve Go cache and policy behavior when upgrading PURL --- docs/configuration.md | 3 ++ go.mod | 4 +- go.sum | 8 +-- internal/config/config.go | 3 +- internal/database/rekey_artifact.go | 40 ++++++++++++++ internal/database/rekey_artifact_test.go | 69 ++++++++++++++++++++++++ internal/denylist/denylist.go | 3 +- internal/handler/cooldown_expiry.go | 7 ++- internal/handler/cooldown_fail_closed.go | 2 +- internal/handler/denylist_test.go | 6 +++ internal/handler/go.go | 60 +++++++++++++++------ internal/handler/go_case_test.go | 69 ++++++++++++++++++++++++ internal/handler/go_cooldown.go | 8 +-- internal/handler/go_policy_test.go | 47 ++++++++++++++++ internal/handler/handler.go | 6 +++ internal/packageurl/packageurl.go | 9 ++++ internal/retention/retention.go | 11 ++-- internal/server/retention_golang_test.go | 4 +- 18 files changed, 320 insertions(+), 39 deletions(-) create mode 100644 internal/database/rekey_artifact.go create mode 100644 internal/database/rekey_artifact_test.go create mode 100644 internal/handler/go_policy_test.go diff --git a/docs/configuration.md b/docs/configuration.md index e1a4f9e..b20685f 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1119,6 +1119,8 @@ Durations support days (`7d`), hours (`48h`), and minutes (`30m`). Set to `0` to Package PURL keys are normalized to canonical form before matching, so `pkg:npm/@babel/core` and `pkg:npm/%40babel/core` are equivalent, as are `pkg:pypi/Django` and `pkg:pypi/django`. If both forms configure the same package, the canonical entry wins. +Go module names in exact cooldown overrides are matched case-insensitively. If both lowercase and mixed-case keys are configured, the lowercase entry wins. Go package patterns match lowercase module names. + `package_patterns` matches canonical, versionless PURLs using `*` for zero or more characters and `?` for one character. Neither wildcard crosses `/` separators. Character classes (`[...]`) and backslash escapes are rejected at startup. For example, `"pkg:npm/@example/*"` matches packages under the `@example` npm scope. Patterns accept `@` as an alias for `%40`; other characters must use their canonical PURL form. Equivalent patterns with different durations are rejected at startup. Equal durations, such as `1d` and `24h`, are accepted. Exact `packages` entries take precedence over patterns. When several patterns match, longer patterns win after excluding `*` and `?` from the length. Ties use lexical order of the normalized patterns. @@ -1180,6 +1182,7 @@ Names and versions are compared the way each ecosystem treats them: `pkg:composer/symfony/console@6.1.7` also covers `v6.1.7`. - Go versions are matched as written, `pkg:golang/example.com/mod@v1.0.0-RC1`, though download URLs escape uppercase letters (`v1.0.0-!r!c1`). + Go module names in denylist entries are matched case-insensitively. - A Swift entry names the registry scope and package, ignoring case: `pkg:swift/apple/swift-nio@2.62.0`. The stored form, `pkg:generic/swift-registry/apple.swift-nio@2.62.0`, works too. diff --git a/go.mod b/go.mod index 79978b1..2ce65e2 100644 --- a/go.mod +++ b/go.mod @@ -16,8 +16,8 @@ require ( github.com/git-pkgs/gcs v0.1.0 github.com/git-pkgs/integrity v0.2.0 github.com/git-pkgs/magic v0.6.0 - github.com/git-pkgs/purl v0.1.24 - github.com/git-pkgs/registries v0.12.0 + github.com/git-pkgs/purl v0.2.1 + github.com/git-pkgs/registries v0.13.0 github.com/git-pkgs/sbom v0.1.7 github.com/git-pkgs/spdx v0.3.3 github.com/go-chi/chi/v5 v5.5.0 diff --git a/go.sum b/go.sum index 7ed7c29..c322161 100644 --- a/go.sum +++ b/go.sum @@ -133,10 +133,10 @@ github.com/git-pkgs/magic v0.6.0 h1:fINPe67FEIaFRVlZzhOIDClMkm+gftBroBENa8bHSxQ= github.com/git-pkgs/magic v0.6.0/go.mod h1:SXOqcsNmbmpZjJZHEEWnwxprbsFvpwWgTAZrgXp4Jm4= github.com/git-pkgs/pom v0.1.8 h1:mXCuWw6XkrKKZb6IurRZyIznYLPhV5AYg5ZO38Uj/l8= github.com/git-pkgs/pom v0.1.8/go.mod h1:sQBeYRzjCUn77fcABPn6I/a1aBHFxm55jTmkNxYtGtw= -github.com/git-pkgs/purl v0.1.24 h1:JMEB1uNWI+baEpl6AeVx+Kfq1c+xkou+THxa5KJ3MrU= -github.com/git-pkgs/purl v0.1.24/go.mod h1:l8rCfvt5iXVrFIcoQ8+vCLOaHhrtEW2id0LENP+IzMw= -github.com/git-pkgs/registries v0.12.0 h1:fhxYaR5iR9Zlx6doLkjlXurVqPxkYcXmNFiIrFeV3/k= -github.com/git-pkgs/registries v0.12.0/go.mod h1:bKxgdy/PwnzQXL2FnOktmSQHTKk6B85MA4VtvvYItPc= +github.com/git-pkgs/purl v0.2.1 h1:b46hRlOMbrG6aXjjFJdYixGi5WTh0TLlAnH9iNeuOS0= +github.com/git-pkgs/purl v0.2.1/go.mod h1:l8rCfvt5iXVrFIcoQ8+vCLOaHhrtEW2id0LENP+IzMw= +github.com/git-pkgs/registries v0.13.0 h1:5i9dHnbiXseIbXLIZ61YnuFySJ9dQX+KEu1MrmwB2BU= +github.com/git-pkgs/registries v0.13.0/go.mod h1:1g/jXVi1M/3jWvW1Wom1NMnPrhglJ7UVEZ215l2Zh0M= github.com/git-pkgs/sbom v0.1.7 h1:4eboy/Yed/SC9ZcC/xT50iywHPzWbmCU7azDb5T1RbA= github.com/git-pkgs/sbom v0.1.7/go.mod h1:KIkZX0v31l62SeFsi2qsQXcZRVrAXvIdPQ4x4en+G2M= github.com/git-pkgs/spdx v0.3.3 h1:xXlUoZzjDSkwxC6mWHfaW+kc8a0+LH6d+gGHuZWftqI= diff --git a/internal/config/config.go b/internal/config/config.go index 5e90150..c09a1ba 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -77,6 +77,7 @@ import ( "github.com/git-pkgs/cooldown" "github.com/git-pkgs/proxy/internal/denylist" + "github.com/git-pkgs/proxy/internal/packageurl" "github.com/git-pkgs/proxy/internal/retention" "github.com/git-pkgs/purl" "gopkg.in/yaml.v3" @@ -261,7 +262,7 @@ func (c *CooldownConfig) NormalizedPackages() map[string]string { for _, key := range keys { canonical := key if parsed, err := purl.Parse(key); err == nil { - canonical = parsed.String() + canonical = packageurl.PolicyPackageKey(parsed.String()) } if _, exists := normalized[canonical]; exists && key != canonical { continue diff --git a/internal/database/rekey_artifact.go b/internal/database/rekey_artifact.go new file mode 100644 index 0000000..68a8dbd --- /dev/null +++ b/internal/database/rekey_artifact.go @@ -0,0 +1,40 @@ +package database + +import ( + "fmt" + "time" +) + +// RekeyCachedArtifact moves one unchanged upstream record without replacing +// an existing destination. Access history and stored bytes are preserved. +func (db *DB) RekeyCachedArtifact(old *Artifact, pkg *Package, version *Version) (bool, error) { + tx, err := db.Beginx() + if err != nil { + return false, err + } + defer func() { _ = tx.Rollback() }() + now := time.Now() + _, err = tx.Exec(db.Rebind(`INSERT INTO packages (purl, ecosystem, name, created_at, updated_at) + VALUES (?, ?, ?, ?, ?) ON CONFLICT(purl) DO NOTHING`), pkg.PURL, pkg.Ecosystem, pkg.Name, now, now) + if err != nil { + return false, fmt.Errorf("creating artifact package: %w", err) + } + _, err = tx.Exec(db.Rebind(`INSERT INTO versions (purl, package_purl, created_at, updated_at) + VALUES (?, ?, ?, ?) ON CONFLICT(purl) DO NOTHING`), version.PURL, version.PackagePURL, now, now) + if err != nil { + return false, fmt.Errorf("creating artifact version: %w", err) + } + result, err := tx.Exec(db.Rebind(`UPDATE artifacts SET version_purl = ?, updated_at = ? + WHERE id = ? AND version_purl = ? AND storage_path = ? AND upstream_url = ? AND source = ? + AND NOT EXISTS (SELECT 1 FROM artifacts WHERE version_purl = ? AND filename = ?)`), + version.PURL, now, old.ID, old.VersionPURL, old.StoragePath, old.UpstreamURL, SourceUpstream, + version.PURL, old.Filename) + if err != nil { + return false, fmt.Errorf("moving cached artifact: %w", err) + } + count, err := result.RowsAffected() + if err != nil || count == 0 { + return false, err + } + return true, tx.Commit() +} diff --git a/internal/database/rekey_artifact_test.go b/internal/database/rekey_artifact_test.go new file mode 100644 index 0000000..d5fd270 --- /dev/null +++ b/internal/database/rekey_artifact_test.go @@ -0,0 +1,69 @@ +package database + +import ( + "database/sql" + "testing" +) + +func TestRekeyCachedArtifact(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + for _, scenario := range []string{"move", "destination exists", "source changed", "local source"} { + t.Run(scenario, func(t *testing.T) { + oldPackage := &Package{PURL: "pkg:golang/example.com/old", Ecosystem: "golang", Name: "example.com/Old"} + oldVersion := &Version{PURL: oldPackage.PURL + "@v1.0.0", PackagePURL: oldPackage.PURL} + old := &Artifact{VersionPURL: oldVersion.PURL, Filename: scenario + ".zip", + UpstreamURL: "https://proxy.example/example.com/!old/@v/v1.0.0.zip", + StoragePath: sql.NullString{String: scenario + "/old.zip", Valid: true}} + if err := db.RecordCachedArtifact(oldPackage, oldVersion, old); err != nil { + t.Fatal(err) + } + old, err := db.GetArtifact(oldVersion.PURL, old.Filename) + if err != nil || old == nil { + t.Fatalf("source artifact = %+v, %v", old, err) + } + pkg := &Package{PURL: "pkg:golang/example.com/Old", Ecosystem: "golang", Name: "example.com/Old"} + version := &Version{PURL: pkg.PURL + "@v1.0.0", PackagePURL: pkg.PURL} + prepareRekeyConflict(t, db, scenario, old, pkg, version) + moved, err := db.RekeyCachedArtifact(old, pkg, version) + if err != nil || moved != (scenario == "move") { + t.Fatalf("RekeyCachedArtifact = %v, %v", moved, err) + } + checkRekeyResult(t, db, scenario, old, version) + }) + } + }) +} + +func checkRekeyResult(t *testing.T, db *DB, scenario string, old *Artifact, version *Version) { + t.Helper() + current, err := db.GetArtifact(version.PURL, old.Filename) + if err != nil { + t.Fatal(err) + } + if scenario == "move" && (current == nil || current.ID != old.ID || current.StoragePath != old.StoragePath) { + t.Errorf("moved artifact = %+v, old = %+v", current, old) + } + if scenario == "destination exists" && (current == nil || current.StoragePath.String != "destination.zip") { + t.Errorf("destination replaced: %+v", current) + } + if paths := allPendingDeletes(t, db); len(paths) != 0 { + t.Errorf("rekey queued stored bytes for deletion: %v", paths) + } +} + +func prepareRekeyConflict(t *testing.T, db *DB, scenario string, old *Artifact, pkg *Package, version *Version) { + t.Helper() + var err error + switch scenario { + case "destination exists": + err = db.RecordCachedArtifact(pkg, version, &Artifact{VersionPURL: version.PURL, Filename: old.Filename, + StoragePath: sql.NullString{String: "destination.zip", Valid: true}}) + case "source changed": + _, err = db.Exec(db.Rebind(`UPDATE artifacts SET storage_path = ? WHERE id = ?`), "replacement.zip", old.ID) + case "local source": + _, err = db.Exec(db.Rebind(`UPDATE artifacts SET source = ? WHERE id = ?`), SourceLocal, old.ID) + } + if err != nil { + t.Fatal(err) + } +} diff --git a/internal/denylist/denylist.go b/internal/denylist/denylist.go index 15bb9e5..76f2de5 100644 --- a/internal/denylist/denylist.go +++ b/internal/denylist/denylist.go @@ -5,6 +5,7 @@ import ( "fmt" "strings" + "github.com/git-pkgs/proxy/internal/packageurl" "github.com/git-pkgs/purl" ) @@ -138,5 +139,5 @@ func identify(pkg *purl.PURL) identity { channel = c.Qualifier("channel") } key := purl.New(c.Type, c.Namespace, c.Name, "", nil).String() - return identity{key: key, channel: channel, version: c.Version} + return identity{key: packageurl.PolicyPackageKey(key), channel: channel, version: c.Version} } diff --git a/internal/handler/cooldown_expiry.go b/internal/handler/cooldown_expiry.go index 1163bde..c31ace8 100644 --- a/internal/handler/cooldown_expiry.go +++ b/internal/handler/cooldown_expiry.go @@ -5,6 +5,7 @@ import ( "time" "github.com/git-pkgs/cooldown" + "github.com/git-pkgs/proxy/internal/packageurl" ) // A rewrite made with cooldown on stays right until a version it held back @@ -127,6 +128,7 @@ type recordingCooldown struct { } func (r recordingCooldown) IsAllowed(ecosystem, packagePURL string, publishedAt time.Time) bool { + packagePURL = packageurl.PolicyPackageKey(packagePURL) if r.p.Cooldown.IsAllowed(ecosystem, packagePURL, publishedAt) { return true } @@ -135,6 +137,7 @@ func (r recordingCooldown) IsAllowed(ecosystem, packagePURL string, publishedAt } func (r recordingCooldown) Evaluate(ecosystem, packagePURL string, publishedAt, evaluatedAt time.Time) cooldown.Decision { + packagePURL = packageurl.PolicyPackageKey(packagePURL) d := r.p.Cooldown.Evaluate(ecosystem, packagePURL, publishedAt, evaluatedAt) if !d.Allowed { r.p.withheld(d.AvailableAt) @@ -143,7 +146,7 @@ func (r recordingCooldown) Evaluate(ecosystem, packagePURL string, publishedAt, } func (r recordingCooldown) For(ecosystem, packagePURL string) time.Duration { - return r.p.Cooldown.For(ecosystem, packagePURL) + return r.p.Cooldown.For(ecosystem, packageurl.PolicyPackageKey(packagePURL)) } func (r recordingCooldown) Enabled() bool { @@ -156,7 +159,7 @@ func (r recordingCooldown) availableAt(ecosystem, packagePURL string, publishedA if publishedAt.IsZero() { return time.Time{} } - d := r.p.Cooldown.For(ecosystem, packagePURL) + d := r.For(ecosystem, packagePURL) if d <= 0 { return time.Time{} } diff --git a/internal/handler/cooldown_fail_closed.go b/internal/handler/cooldown_fail_closed.go index da600c1..0dd2c36 100644 --- a/internal/handler/cooldown_fail_closed.go +++ b/internal/handler/cooldown_fail_closed.go @@ -5,7 +5,7 @@ import "net/http" // cooldownApplies reports whether cooldown withholds new versions of the // package, taking package overrides and exemptions into account. func (p *Proxy) cooldownApplies(ecosystem, name string) bool { - return p.cooldownEnabled() && p.Cooldown.For(ecosystem, canonicalPackagePURL(ecosystem, name)) > 0 + return p.cooldownEnabled() && p.cooldown().For(ecosystem, canonicalPackagePURL(ecosystem, name)) > 0 } // mustFilterMetadata reports whether the package's metadata may have diff --git a/internal/handler/denylist_test.go b/internal/handler/denylist_test.go index bb41e38..663d0c3 100644 --- a/internal/handler/denylist_test.go +++ b/internal/handler/denylist_test.go @@ -288,6 +288,12 @@ func TestDenylistMatchesEcosystemSpellings(t *testing.T) { {"go case-escaped version", "pkg:golang/github.com/example/mod@v1.0.0-RC1", "/github.com/example/mod/@v/v1.0.0-!r!c1.zip", "golang", "github.com/example/mod", "v1.0.0-!r!c1", "mod@v1.0.0-!r!c1.zip", func(p *Proxy) http.Handler { return NewGoHandler(p, "http://proxy.test").Routes() }}, + {"go lowercase rule", "pkg:golang/github.com/example/mod@v1.0.0", "/github.com/!example/mod/@v/v1.0.0.zip", + "golang", "github.com/Example/mod", "v1.0.0", goZipFilename("github.com/Example/mod", "v1.0.0"), + func(p *Proxy) http.Handler { return NewGoHandler(p, "http://proxy.test").Routes() }}, + {"go mixed-case rule", "pkg:golang/github.com/Example/mod@v1.0.0", "/github.com/example/mod/@v/v1.0.0.zip", + "golang", "github.com/example/mod", "v1.0.0", "mod@v1.0.0.zip", + func(p *Proxy) http.Handler { return NewGoHandler(p, "http://proxy.test").Routes() }}, {"oci repository path", "pkg:oci/library/alpine@" + ociTestDigest, "/library/alpine/blobs/" + ociTestDigest, "oci", "library/alpine", ociTestDigest, ociTestDigest, func(p *Proxy) http.Handler { diff --git a/internal/handler/go.go b/internal/handler/go.go index 1af00ed..08dae57 100644 --- a/internal/handler/go.go +++ b/internal/handler/go.go @@ -4,11 +4,15 @@ import ( "crypto/sha256" "encoding/hex" "errors" + "fmt" "net/http" + "net/url" "strings" "github.com/git-pkgs/proxy/internal/database" + "github.com/git-pkgs/proxy/internal/packageurl" "github.com/git-pkgs/proxy/internal/retention" + "github.com/git-pkgs/purl" "github.com/git-pkgs/registries/fetch" gomodule "golang.org/x/mod/module" ) @@ -18,15 +22,13 @@ const ( asciiCaseOffset = 32 // difference between lowercase and uppercase ASCII letters ) -// Module zips are stored under pkg:golang/{module path}, which the PURL -// rules lowercase, so a package rule covers a module whatever case either -// is written in. A major version suffix (/v2) is part of the module path -// and so a module of its own. A key in the escaped form proxy URLs and the -// module cache use (github.com/!azure) does not survive the default -// mapping's round trip and is refused; the handler stores decoded paths, -// so such a key could never match. func init() { - retention.Default.Register(retention.Spec{Key: "golang", CanonicalPackage: retention.DefaultCanonical("golang")}) + retention.Default.Register(retention.Spec{Key: "golang", CanonicalPackage: golangRetentionKey}) +} + +func golangRetentionKey(p *purl.PURL) (string, bool) { + key, ok := retention.DefaultCanonical("golang")(p) + return packageurl.PolicyPackageKey(key), ok } // GoHandler handles Go module proxy protocol requests. @@ -200,13 +202,7 @@ func (h *GoHandler) handleDownload(w http.ResponseWriter, r *http.Request, modul ServeArtifactRequest(w, r, result) } -// goZipFilename is the name a module zip is cached under. Module paths are -// case-sensitive, but the zip is stored under the module's PURL, which -// lowercases golang names, so github.com/Sirupsen/logrus and -// github.com/sirupsen/logrus share one package. A path with uppercase -// letters therefore gets a hash of its exact spelling in the filename. An -// all-lowercase path keeps the plain name, so existing cache entries stay -// valid. +// Keep the module spelling hash used by caches with lowercased PURL keys. func goZipFilename(module, version string) string { base := lastComponent(module) + "@" + version if module == strings.ToLower(module) { @@ -220,6 +216,40 @@ func goZipFilename(module, version string) string { // filename carries: enough to tell apart paths that differ only in case. const goModuleHashBytes = 8 +// An old cache key may combine case-distinct modules. The upstream path +// identifies which module owns an individual archive. +func (p *Proxy) rekeyLegacyGoArtifact(pkgPURL, versionPURL, filename string) (bool, error) { + if !strings.HasPrefix(versionPURL, "pkg:golang/") { + return false, nil + } + parsed, err := purl.Parse(versionPURL) + if err != nil || parsed.FullName() == strings.ToLower(parsed.FullName()) { + return false, nil + } + name := parsed.FullName() + _, legacyVersion, err := packagePURLStrings("golang", strings.ToLower(name), parsed.Version) + if err != nil { + return false, err + } + record, err := p.DB.GetArtifact(legacyVersion, filename) + if err != nil { + return false, fmt.Errorf("reading legacy Go artifact: %w", err) + } + if record == nil || !record.StoragePath.Valid || record.Source != database.SourceUpstream { + return false, nil + } + escaped, err := gomodule.EscapePath(name) + if err != nil { + return false, nil + } + upstream, err := url.Parse(record.UpstreamURL) + if err != nil || !strings.HasSuffix(upstream.Path, "/"+escaped+"/@v/"+parsed.Version+".zip") { + return false, nil + } + return p.DB.RekeyCachedArtifact(record, &database.Package{PURL: pkgPURL, Ecosystem: "golang", Name: name}, + &database.Version{PURL: versionPURL, PackagePURL: pkgPURL}) +} + // discardOtherModulesZip drops a cached zip that another module stored // under filename. Before module paths kept their case, a module with // uppercase letters was cached under the plain filename of its lowercase diff --git a/internal/handler/go_case_test.go b/internal/handler/go_case_test.go index 9451caa..91bac44 100644 --- a/internal/handler/go_case_test.go +++ b/internal/handler/go_case_test.go @@ -9,6 +9,7 @@ import ( "sync/atomic" "testing" + "github.com/git-pkgs/proxy/internal/database" "github.com/git-pkgs/registries/fetch" ) @@ -115,6 +116,74 @@ func TestGoZipFromAnotherUpstreamHostIsKept(t *testing.T) { } } +func TestGoLegacyCacheKeepsCaseDistinctModules(t *testing.T) { + upstream, counts := goCaseUpstream(t) + h, proxy := newGoCaseHandler(t, upstream) + seedGoZip(t, proxy, upstream.URL+lowerLogrusPath, "zip of "+lowerLogrusPath) + old := seedLegacyUpperGoZip(t, proxy, "https://old-proxy.example"+upperLogrusPath, "zip of "+upperLogrusPath) + upstream.Close() + + for range 2 { + for _, path := range []string{upperLogrusPath, lowerLogrusPath} { + if w := getGo(h, path); w.Code != http.StatusOK || w.Body.String() != "zip of "+path { + t.Fatalf("GET %s: status = %d, body = %q", path, w.Code, w.Body) + } + } + } + current, err := proxy.DB.GetArtifact("pkg:golang/github.com/Sirupsen/logrus@v1.0.6", old.Filename) + if err != nil || current == nil { + t.Fatalf("moved artifact = %+v, %v", current, err) + } + if current.ID != old.ID || current.StoragePath != old.StoragePath || current.FetchedAt != old.FetchedAt || current.HitCount != old.HitCount+2 { + t.Errorf("artifact history changed: before=%+v, after=%+v", old, current) + } + legacy, err := proxy.DB.GetArtifact(old.VersionPURL, old.Filename) + if err != nil || legacy != nil { + t.Fatalf("legacy artifact = %+v, %v", legacy, err) + } + for path, count := range counts { + if count.Load() != 0 { + t.Errorf("unexpected upstream request: %s", path) + } + } +} + +func TestGoLegacyCacheRequiresExactModuleOrigin(t *testing.T) { + for _, origin := range []string{"", "https://proxy.example" + lowerLogrusPath} { + t.Run(origin, func(t *testing.T) { + upstream, counts := goCaseUpstream(t) + h, proxy := newGoCaseHandler(t, upstream) + seedLegacyUpperGoZip(t, proxy, origin, "another module") + if w := getGo(h, upperLogrusPath); w.Code != http.StatusOK || w.Body.String() != "zip of "+upperLogrusPath { + t.Fatalf("status = %d, body = %q", w.Code, w.Body) + } + if counts[upperLogrusPath].Load() != 1 { + t.Fatal("archive with unproven identity was reused") + } + }) + } +} + +func seedLegacyUpperGoZip(t *testing.T, proxy *Proxy, upstreamURL, content string) *database.Artifact { + t.Helper() + const packagePURL = "pkg:golang/github.com/sirupsen/logrus" + const versionPURL = packagePURL + "@v1.0.6" + const storagePath = "golang/logrus/v1.0.6/legacy/upper.zip" + filename := goZipFilename("github.com/Sirupsen/logrus", "v1.0.6") + if _, _, err := proxy.Storage.Store(context.Background(), storagePath, strings.NewReader(content)); err != nil { + t.Fatal(err) + } + artifact := testArtifact(content, versionPURL, filename, "application/zip") + if err := proxy.updateCacheDB("golang", "github.com/Sirupsen/logrus", packagePURL, upstreamURL, storagePath, artifact, ""); err != nil { + t.Fatal(err) + } + old, err := proxy.DB.GetArtifact(versionPURL, filename) + if err != nil || old == nil { + t.Fatalf("seeded artifact = %+v, %v", old, err) + } + return old +} + // seedGoZip caches content as github.com/sirupsen/logrus v1.0.6 under the // plain filename, recorded as fetched from upstreamURL. func seedGoZip(t *testing.T, proxy *Proxy, upstreamURL, content string) { diff --git a/internal/handler/go_cooldown.go b/internal/handler/go_cooldown.go index 892e77b..efcc1e1 100644 --- a/internal/handler/go_cooldown.go +++ b/internal/handler/go_cooldown.go @@ -71,12 +71,8 @@ func (h *GoHandler) fetchGoMetadata(ctx context.Context, escapedModule, module, return body, err } -// goTimeKey returns the versions-table PURLs a module's times are stored -// under. golang PURLs lowercase the name, so the key uses the escaped module -// path (github.com/!azure for github.com/Azure), which keeps modules that -// differ only in case apart and is the plain path for lowercase modules. -// Only canonical versions are stored: a query such as a branch name can -// resolve to another commit later. +// Keep escaped module paths in publish-time keys for cache compatibility. +// Only canonical versions are stored; branch queries can resolve differently. func (h *GoHandler) goTimeKey(module, version string) (versionPURL, packagePURL string, ok bool) { if h.proxy.DB == nil || gomodule.CanonicalVersion(version) != version { return "", "", false diff --git a/internal/handler/go_policy_test.go b/internal/handler/go_policy_test.go new file mode 100644 index 0000000..0ce981f --- /dev/null +++ b/internal/handler/go_policy_test.go @@ -0,0 +1,47 @@ +package handler + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/git-pkgs/cooldown" + "github.com/git-pkgs/proxy/internal/config" +) + +func TestGoCooldownOverridesIgnoreModuleCase(t *testing.T) { + for _, tc := range []struct { + name, rule, fallback, override string + status int + }{ + {"lowercase restriction", "example.com/module", "0", "7d", http.StatusForbidden}, + {"mixed-case restriction", "example.com/Module", "0", "7d", http.StatusForbidden}, + {"lowercase exemption", "example.com/module", "7d", "0", http.StatusOK}, + {"mixed-case exemption", "example.com/Module", "7d", "0", http.StatusOK}, + } { + t.Run(tc.name, func(t *testing.T) { + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if strings.HasSuffix(r.URL.Path, ".info") { + _ = json.NewEncoder(w).Encode(goInfo{Version: "v1.0.0", Time: time.Now()}) + return + } + _, _ = w.Write([]byte("module zip")) + })) + defer upstream.Close() + h, proxy := newGoCaseHandler(t, upstream) + cfg := config.CooldownConfig{Packages: map[string]string{"pkg:golang/" + tc.rule: tc.override}} + proxy.Cooldown = &cooldown.Config{Default: tc.fallback, Packages: cfg.NormalizedPackages()} + for _, path := range []string{ + "/example.com/!module/@v/v1.0.0.zip", "/example.com/module/@v/v1.0.0.zip", + "/example.com/!module/@v/v1.0.0.info", "/example.com/module/@v/v1.0.0.info", + } { + if response := getGo(h, path); response.Code != tc.status { + t.Errorf("GET %s = %d %s, want %d", path, response.Code, response.Body, tc.status) + } + } + }) + } +} diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 3951016..4b7ebe7 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -498,6 +498,12 @@ func (p *Proxy) lookupCache(ctx context.Context, pkgPURL, versionPURL, filename if p.StreamArtifacts { return nil, nil } + if cached, err := p.readStored(ctx, pkgPURL, versionPURL, filename); err != nil || cached != nil { + return cached, err + } + if moved, err := p.rekeyLegacyGoArtifact(pkgPURL, versionPURL, filename); err != nil || !moved { + return nil, err + } return p.readStored(ctx, pkgPURL, versionPURL, filename) } diff --git a/internal/packageurl/packageurl.go b/internal/packageurl/packageurl.go index be0e9ab..e01f4b2 100644 --- a/internal/packageurl/packageurl.go +++ b/internal/packageurl/packageurl.go @@ -18,6 +18,15 @@ func MakeString(ecosystem, name, version string) string { return purl.MakePURLString(ecosystem, name, version) } +// PolicyPackageKey maps a canonical versionless PURL to its policy key. +// Go policies match module names case-insensitively for compatibility. +func PolicyPackageKey(packagePURL string) string { + if strings.HasPrefix(packagePURL, "pkg:golang/") { + return strings.ToLower(packagePURL) + } + return packagePURL +} + // WithVersionString returns a package PURL with its version replaced. It // returns an empty string when packagePURL is invalid. func WithVersionString(packagePURL, version string) string { diff --git a/internal/retention/retention.go b/internal/retention/retention.go index a8ff835..df8e538 100644 --- a/internal/retention/retention.go +++ b/internal/retention/retention.go @@ -14,6 +14,7 @@ import ( "sync" "time" + "github.com/git-pkgs/proxy/internal/packageurl" "github.com/git-pkgs/purl" ) @@ -41,8 +42,8 @@ type Spec struct { // derived from Key that belong to this ecosystem. ExtraDBEcosystems []string - // CanonicalPackage turns a package PURL from the config into the - // package PURL the handler stores in the cache, reporting false when the + // CanonicalPackage turns a package PURL from the config into a + // policy key, reporting false when the // PURL does not belong to this ecosystem. Nil means the ecosystem accepts // no package overrides. CanonicalPackage func(p *purl.PURL) (string, bool) @@ -150,8 +151,7 @@ func (r *Registry) KeyForDBEcosystem(ecosystem string) (string, bool) { return key, ok } -// Canonical returns the stored package PURL a configured package PURL -// stands for, asking the registered specs in key order. +// Canonical returns the policy key for a configured package PURL. func (r *Registry) Canonical(p *purl.PURL) (string, bool) { for _, key := range r.Keys() { s, _ := r.Lookup(key) @@ -173,7 +173,7 @@ type Rules struct { Default time.Duration // Ecosystems holds rules keyed by registered ecosystem key. Ecosystems map[string]time.Duration - // Packages holds rules keyed by the package PURL stored in the cache. + // Packages holds rules keyed by package PURL, with Go keys lowercased. Packages map[string]time.Duration } @@ -181,6 +181,7 @@ type Rules struct { // stored package PURL: the package rule, else the ecosystem rule, else the // default. Generic artifacts require an explicit package or ecosystem rule. func (r Rules) For(key, packagePURL string) time.Duration { + packagePURL = packageurl.PolicyPackageKey(packagePURL) if d, ok := r.Packages[packagePURL]; ok { return d } diff --git a/internal/server/retention_golang_test.go b/internal/server/retention_golang_test.go index f74eaf4..8ff0357 100644 --- a/internal/server/retention_golang_test.go +++ b/internal/server/retention_golang_test.go @@ -188,8 +188,8 @@ func TestGolangRetentionPackageRules(t *testing.T) { if err := db.Get(&stored, db.Rebind(`SELECT package_purl FROM versions WHERE purl = ?`), zip.VersionPURL); err != nil { t.Fatalf("reading stored package PURL: %v", err) } - if stored != "pkg:golang/github.com/burntsushi/toml" { - t.Errorf("stored package PURL = %q, want pkg:golang/github.com/burntsushi/toml", stored) + if stored != "pkg:golang/github.com/BurntSushi/toml" { + t.Errorf("stored package PURL = %q, want pkg:golang/github.com/BurntSushi/toml", stored) } for _, keys := range []map[string]string{ From 9941d79561a750426a751de4660b30e65f3eb7ab Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Fri, 30 Oct 2026 19:19:01 +0100 Subject: [PATCH 509/633] Document the published container image and add a Docker health check --- Dockerfile | 4 ++++ README.md | 16 ++++++++++++++-- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 1a934ed..cc35e9b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -31,5 +31,9 @@ WORKDIR /data EXPOSE 8080 +# /health checks the database and storage and can take up to ten seconds. +HEALTHCHECK --interval=30s --timeout=11s --start-period=10s --retries=3 \ + CMD wget -q -O /dev/null http://127.0.0.1:8080/health || exit 1 + ENTRYPOINT ["proxy"] CMD ["serve"] diff --git a/README.md b/README.md index a324c53..6d84241 100644 --- a/README.md +++ b/README.md @@ -87,7 +87,11 @@ Cooldown requires publish timestamps in metadata. Registries without a "Yes" in brew install git-pkgs/git-pkgs/proxy ``` -Or download a binary from the [releases page](https://github.com/git-pkgs/proxy/releases). +Or download a binary from the [releases page](https://github.com/git-pkgs/proxy/releases), or run the container image: + +```bash +docker run -p 8080:8080 -v proxy-data:/data ghcr.io/git-pkgs/proxy:latest +``` ### Helm @@ -1715,13 +1719,21 @@ sudo systemctl start proxy ### Docker -A Dockerfile is included in the repo. Build and run: +Images for linux/amd64 and linux/arm64 are published to `ghcr.io/git-pkgs/proxy` for each release, tagged with the version and `latest`: + +```bash +docker run -p 8080:8080 -v proxy-data:/data ghcr.io/git-pkgs/proxy:latest +``` + +To build the image from the repo instead: ```bash docker build -t proxy . docker run -p 8080:8080 -v proxy-data:/data proxy ``` +The image has a health check that requests `/health` on port 8080. It checks the database and storage, so a container reports unhealthy when either fails. If you change the port with `PROXY_LISTEN`, override it with `--health-cmd`. + With `--read-only`, also pass `--tmpfs /tmp`: the proxy writes some uploads to a temporary file before reading them. With Postgres and S3: From 760623e617d7bed7d3b64b437fb942076eef4504 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Fri, 30 Oct 2026 19:25:09 +0100 Subject: [PATCH 510/633] Serve the stored npm packument on upstream 404 and refetch on a 304 for a missing blob --- internal/handler/metadata_stored.go | 11 +++-- internal/handler/npm_stream_test.go | 72 +++++++++++++++++++++++++++-- 2 files changed, 76 insertions(+), 7 deletions(-) diff --git a/internal/handler/metadata_stored.go b/internal/handler/metadata_stored.go index d831303..80cbaf4 100644 --- a/internal/handler/metadata_stored.go +++ b/internal/handler/metadata_stored.go @@ -122,7 +122,9 @@ func (p *Proxy) fetchStoredMetadata(ctx context.Context, ecosystem, cacheKey, up if err == nil { return metadataResult{entry: stored} } - if !usableStoredMetadata(entry) || errors.Is(err, ErrUpstreamNotFound) || errors.Is(err, errMetadataNotStored) { + // As in fetchMetadataFromUpstream, any upstream failure, a 404 included, + // falls back to the stored copy. + if !usableStoredMetadata(entry) || errors.Is(err, errMetadataNotStored) { return metadataResult{err: err} } return p.staleStoredMetadata(ecosystem, cacheKey, entry, err) @@ -142,8 +144,8 @@ func (p *Proxy) staleStoredMetadata(ecosystem, cacheKey string, entry *database. // storeUpstreamMetadata fetches a document and streams it into storage, and // returns its new cache row. On a 304 for entry it only moves the row's -// fetched_at on; errStale304 means the row was replaced meanwhile and the -// caller should fetch without validators. +// fetched_at on; errStale304 means the stored copy is gone or the row was +// replaced meanwhile, and the caller should fetch without validators. // // With validate, the document is staged under a path of its own and checked // there first, so one validate rejects never replaces the stored copy. @@ -155,6 +157,9 @@ func (p *Proxy) storeUpstreamMetadata(ctx context.Context, ecosystem, cacheKey, defer func() { _ = resp.Body.Close() }() if resp.StatusCode == http.StatusNotModified && entry != nil { + if ok, err := p.Storage.Exists(ctx, entry.StoragePath); err != nil || !ok { + return nil, errStale304 + } refreshed, err := p.DB.RefreshMetadataCache(ecosystem, cacheKey, entry.ContentDigest.String, time.Now()) if err != nil || !refreshed { return nil, errStale304 diff --git a/internal/handler/npm_stream_test.go b/internal/handler/npm_stream_test.go index 3442ce2..de5a17d 100644 --- a/internal/handler/npm_stream_test.go +++ b/internal/handler/npm_stream_test.go @@ -50,24 +50,38 @@ func bigPackumentAt(base string, n int) string { type streamUpstream struct { *httptest.Server body atomic.Value // string + etag atomic.Value // string fail atomic.Bool + gone atomic.Bool requests atomic.Int32 + // sent counts the responses that carried the document. + sent atomic.Int32 +} + +// replace makes upstream send body under a new ETag, so a revalidation +// gets it in a 200. +func (u *streamUpstream) replace(body, etag string) { + u.body.Store(body) + u.etag.Store(etag) } func newStreamUpstream(t *testing.T, body string) *streamUpstream { t.Helper() u := &streamUpstream{} - u.body.Store(body) + u.replace(body, `"v1"`) u.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { u.requests.Add(1) switch { case u.fail.Load(): http.Error(w, "down", http.StatusBadGateway) - case r.URL.Path == "/big" && r.Header.Get("If-None-Match") == `"v1"`: + case u.gone.Load(): + http.NotFound(w, r) + case r.URL.Path == "/big" && r.Header.Get("If-None-Match") == u.etag.Load().(string): w.WriteHeader(http.StatusNotModified) case r.URL.Path == "/big": + u.sent.Add(1) w.Header().Set("Content-Type", "application/json") - w.Header().Set("ETag", `"v1"`) + w.Header().Set("ETag", u.etag.Load().(string)) _, _ = io.WriteString(w, u.body.Load().(string)) default: http.NotFound(w, r) @@ -201,6 +215,53 @@ func TestNPMStreamedPackumentStale(t *testing.T) { } } +// TestNPMStreamedPackumentStaleOn404 checks that a stored copy past its TTL +// is served when upstream answers 404, as the in-memory path does. +func TestNPMStreamedPackumentStaleOn404(t *testing.T) { + u := newStreamUpstream(t, bigPackument(10)) + proxy, _, handler := streamProxy(t, u) + decodeStreamed(t, serve(handler, "/big", nil)) + + proxy.MetadataTTL = time.Nanosecond + u.gone.Store(true) + for _, size := range []int64{64 << 20, 0} { // streamed, then in memory + proxy.SetMetadataRewriteCacheSize(size) + if doc := decodeStreamed(t, serve(handler, "/big", nil)); len(doc.Versions) != 10 { + t.Errorf("rewrite cache %d: stale copy has %d versions, want 10", size, len(doc.Versions)) + } + } +} + +// TestNPMStreamedPackumentBlobMissing checks that a 304 for a row whose +// document is gone from storage fetches the document again, also when it is +// larger than metadata_max_size. +func TestNPMStreamedPackumentBlobMissing(t *testing.T) { + body := bigPackument(400) + u := newStreamUpstream(t, body) + proxy, store, handler := streamProxy(t, u) + proxy.MetadataMaxSize = int64(len(body) / 4) + decodeStreamed(t, serve(handler, "/big", nil)) + + path := metadataStoragePath("npm", "big") + store.mu.Lock() + delete(store.files, path) + store.mu.Unlock() + proxy.MetadataTTL = time.Nanosecond + proxy.SetMetadataRewriteCacheSize(64 << 20) + + if doc := decodeStreamed(t, serve(handler, "/big", nil)); len(doc.Versions) != 400 { + t.Errorf("served %d versions, want 400", len(doc.Versions)) + } + if n := u.sent.Load(); n != 2 { + t.Errorf("upstream sent the document %d times, want 2", n) + } + store.mu.Lock() + defer store.mu.Unlock() + if string(store.files[path]) != body { + t.Error("the document was not stored again") + } +} + // TestNPMStreamedPackumentChangedInStorage checks that a document replaced // in storage between the two passes isn't rewritten from mixed bytes. func TestNPMStreamedPackumentChangedInStorage(t *testing.T) { @@ -262,10 +323,13 @@ func TestNPMStreamedPackumentKeepsFullCopy(t *testing.T) { proxy.MetadataTTL = time.Nanosecond proxy.SetMetadataRewriteCacheSize(64 << 20) - u.body.Store(`{"name":"big","modified":"2026-01-01T00:00:00Z","dist-tags":{"latest":"1.0.0"},"versions":{"1.0.0":{}}}`) + u.replace(`{"name":"big","modified":"2026-01-01T00:00:00Z","dist-tags":{"latest":"1.0.0"},"versions":{"1.0.0":{}}}`, `"v2"`) if doc := decodeStreamed(t, serve(handler, "/big", nil)); len(doc.Versions) != 10 { t.Errorf("served %d versions, want the 10 of the full copy", len(doc.Versions)) } + if n := u.sent.Load(); n != 2 { + t.Fatalf("upstream sent a document %d times, want 2: the abbreviated one was not fetched", n) + } store.mu.Lock() defer store.mu.Unlock() From 3ce582c7330335a707df8a286e357e65f10cee89 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Fri, 30 Oct 2026 19:25:41 +0100 Subject: [PATCH 511/633] Keep the last good Composer index when upstream sends one that doesn't parse parseComposerUpstreamIndex now returns an error for a packages.json that is not a JSON object or has the advisories and filter fields in the wrong shape, and fetchUpstreamIndex passes it to the metadata cache as validate. A malformed 200 is no longer stored: with metadata caching the stored copy is served as when upstream fails, and without it the error keeps the copy in memory. --- internal/handler/composer_advisories.go | 31 ++++++-- internal/handler/composer_advisories_test.go | 77 ++++++++++++++++++++ 2 files changed, 103 insertions(+), 5 deletions(-) diff --git a/internal/handler/composer_advisories.go b/internal/handler/composer_advisories.go index ed8f4bc..aa0fa26 100644 --- a/internal/handler/composer_advisories.go +++ b/internal/handler/composer_advisories.go @@ -1,9 +1,11 @@ package handler import ( + "bytes" "context" "encoding/json" "errors" + "fmt" "io" "net/http" "net/url" @@ -97,18 +99,34 @@ func (h *ComposerHandler) upstreamIndex(ctx context.Context) (composerUpstreamIn return c.index, c.ok } +// fetchUpstreamIndex fetches and parses the default repository's +// packages.json. A document that doesn't parse is rejected before it is +// cached, so a broken response never replaces a good copy: with metadata +// caching the stored copy is used instead, and without it the error keeps +// the copy in memory. func (h *ComposerHandler) fetchUpstreamIndex(ctx context.Context) (composerUpstreamIndex, error) { indexURL := h.repoURL + "/packages.json" cacheKey := hashedCacheKey(h.repoURL, "/packages.json") - res := h.proxy.fetchMetadataDocument(ctx, "composer", cacheKey, indexURL, contentTypeJSON, nil) + validate := func(body []byte) error { + _, err := parseComposerUpstreamIndex(indexURL, body) + return err + } + res := h.proxy.fetchMetadataDocument(ctx, "composer", cacheKey, indexURL, contentTypeJSON, validate) if res.err != nil { h.proxy.Logger.Warn("failed to fetch upstream composer packages.json", "url", indexURL, "error", res.err) return composerUpstreamIndex{}, res.err } - return parseComposerUpstreamIndex(indexURL, res.body), nil + return parseComposerUpstreamIndex(indexURL, res.body) } -func parseComposerUpstreamIndex(indexURL string, body []byte) composerUpstreamIndex { +// errComposerIndexNotObject is returned for a packages.json that is valid +// JSON but not an object, such as null. +var errComposerIndexNotObject = errors.New("composer packages.json is not a JSON object") + +// parseComposerUpstreamIndex reads the advisories and filter fields of a +// packages.json. A document without them yields an empty index; one that is +// not a JSON object, or has them in the wrong shape, is an error. +func parseComposerUpstreamIndex(indexURL string, body []byte) (composerUpstreamIndex, error) { var doc struct { SecurityAdvisories struct { Metadata bool `json:"metadata"` @@ -121,8 +139,11 @@ func parseComposerUpstreamIndex(indexURL string, body []byte) composerUpstreamIn } `json:"filter"` } var index composerUpstreamIndex + if trimmed := bytes.TrimSpace(body); len(trimmed) == 0 || trimmed[0] != '{' { + return index, errComposerIndexNotObject + } if err := json.Unmarshal(body, &doc); err != nil { - return index + return index, fmt.Errorf("parsing composer packages.json: %w", err) } // Composer refuses advisories configured without an api-url unless the // repository lists its packages, which the proxy does not. @@ -137,7 +158,7 @@ func parseComposerUpstreamIndex(indexURL string, body []byte) composerUpstreamIn index.filterLists = doc.Filter.Lists index.filterSummaryURL, _ = resolveComposerURL(indexURL, doc.Filter.SummaryURL) } - return index + return index, nil } // resolveComposerURL resolves a URL from packages.json against the document's diff --git a/internal/handler/composer_advisories_test.go b/internal/handler/composer_advisories_test.go index 5e1a204..097d8cc 100644 --- a/internal/handler/composer_advisories_test.go +++ b/internal/handler/composer_advisories_test.go @@ -2,6 +2,7 @@ package handler import ( "encoding/json" + "fmt" "io" "net/http" "net/http/httptest" @@ -12,6 +13,8 @@ import ( "sync/atomic" "testing" "time" + + "github.com/git-pkgs/proxy/internal/database" ) // composerAdvisoryUpstream serves a packages.json advertising security @@ -21,6 +24,8 @@ type composerAdvisoryUpstream struct { *httptest.Server index string down atomic.Bool // answer every request with 503 + // malformed answers packages.json with an HTML page and status 200. + malformed atomic.Bool mu sync.Mutex requests []*recordedRequest @@ -53,6 +58,11 @@ func newComposerAdvisoryUpstream(t *testing.T) *composerAdvisoryUpstream { w.Header().Set("Content-Type", "application/json") switch r.URL.Path { case "/packages.json": + if u.malformed.Load() { + w.Header().Set("Content-Type", "text/html") + _, _ = io.WriteString(w, "Service temporarily unavailable") + return + } _, _ = io.WriteString(w, u.index) case "/api/security-advisories/": _, _ = io.WriteString(w, composerAdvisoriesResponse) @@ -235,6 +245,73 @@ func TestComposerServiceIndexKeptInMemory(t *testing.T) { } } +// assertComposerSecurityAdvertised checks that h advertises the upstream's +// advisories API and malware filter, and relays both. +func assertComposerSecurityAdvertised(t *testing.T, h *ComposerHandler, when string) { + t.Helper() + index := serveComposerIndex(t, h) + if !advertisesAdvisories(index) { + t.Errorf("%s: security-advisories not advertised: %v", when, index) + } + if _, ok := index["filter"]; !ok { + t.Errorf("%s: filter not advertised: %v", when, index) + } + w := serveAdvisories(h, http.MethodPost, "/api/security-advisories/", "packages%5B%5D=public%2Flibrary") + if w.Code != http.StatusOK || w.Body.String() != composerAdvisoriesResponse { + t.Errorf("%s: advisories = %d %q, want upstream's advisories", when, w.Code, w.Body) + } + if w := serveComposer(h, "/lists/summary.json"); w.Code != http.StatusOK || w.Body.String() != composerFilterSummary { + t.Errorf("%s: filter summary = %d %q, want upstream's summary", when, w.Code, w.Body) + } +} + +// A packages.json that doesn't parse, sent with status 200, must not replace +// the last good index, with metadata caching on or off: the advisories API +// and the malware filter stay advertised and relayed. +func TestComposerServiceIndexMalformedRefreshKeepsLastGood(t *testing.T) { + for _, cacheMetadata := range []bool{false, true} { + t.Run(fmt.Sprintf("cache_metadata=%v", cacheMetadata), func(t *testing.T) { + upstream := newComposerAdvisoryUpstream(t) + h := newAdvisoryComposerHandler(t, upstream) + h.proxy.CacheMetadata = cacheMetadata + h.proxy.MetadataTTL = time.Hour + now := time.Now() + h.indexCache.now = func() time.Time { return now } + cacheKey := hashedCacheKey(upstream.URL, "/packages.json") + + assertComposerSecurityAdvertised(t, h, "before the refresh") + var stored *database.MetadataCacheEntry + if cacheMetadata { + var err error + if stored, err = h.proxy.DB.GetMetadataCache("composer", cacheKey); err != nil || stored == nil { + t.Fatalf("packages.json not cached: %v, %v", stored, err) + } + expireMetadata(t, h.proxy.DB, "composer", cacheKey) + } + + upstream.malformed.Store(true) + now = now.Add(2 * time.Hour) + fetched := len(upstream.requestsTo("/packages.json")) + assertComposerSecurityAdvertised(t, h, "after a malformed refresh") + if got := len(upstream.requestsTo("/packages.json")); got == fetched { + t.Fatal("packages.json was not fetched again after the TTL") + } + + if !cacheMetadata { + return + } + after, err := h.proxy.DB.GetMetadataCache("composer", cacheKey) + if err != nil || after == nil || after.ContentDigest != stored.ContentDigest { + t.Errorf("cached packages.json replaced by the malformed refresh: %v, %v", after, err) + } + // A fresh handler, as after a restart, has nothing in memory and + // reads the stored copy. + restarted := NewComposerHandlerWithUpstreams(h.proxy, "http://proxy.test", upstream.URL, upstream.URL) + assertComposerSecurityAdvertised(t, restarted, "after a restart") + }) + } +} + func TestComposerServiceIndexFetchedOnce(t *testing.T) { upstream := newComposerAdvisoryUpstream(t) h := newAdvisoryComposerHandler(t, upstream) From 04f732e14744592bea58c1176a64b3f32c887d69 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Fri, 30 Oct 2026 19:26:03 +0100 Subject: [PATCH 512/633] Record APK snapshots from the document a request serves A snapshot was recorded after the response by reading the metadata cache again, so a refresh that replaced the cached copy in between lost the version the request served. Record it from the served bytes before responding instead, and compare a new snapshot with the version current at its own time, so one recorded late keeps its place in the history. --- internal/database/metadata_snapshots.go | 21 ++-- internal/database/metadata_snapshots_test.go | 30 ++++++ internal/handler/apk.go | 6 +- internal/handler/apk_snapshots.go | 104 +++++++++++++++---- internal/handler/apk_snapshots_test.go | 76 ++++++++++++++ 5 files changed, 207 insertions(+), 30 deletions(-) diff --git a/internal/database/metadata_snapshots.go b/internal/database/metadata_snapshots.go index fba774a..4df1fa1 100644 --- a/internal/database/metadata_snapshots.go +++ b/internal/database/metadata_snapshots.go @@ -60,15 +60,18 @@ func (db *DB) metadataSnapshotBefore(ecosystem, name string, beforeMS int64) (*M return &snap, nil } -// AddMetadataSnapshot records snap as the current version of its document, -// unless the newest snapshot already has its digest, and reports whether it -// did. On Postgres, replicas recording the same document take an advisory -// lock on it first, so their check and insert don't interleave: without it, -// two transactions can both read the previous digest and both insert. SQLite +// AddMetadataSnapshot records snap as the version of its document current +// from snap.ValidFromMS, unless the snapshot current at that time already +// has its digest, and reports whether it did. Comparing with the version +// current at that time rather than the newest one keeps a version a request +// recorded late, after a newer one, in its place in the history. On +// Postgres, replicas recording the same document take an advisory lock on it +// first, so their check and insert don't interleave: without it, two +// transactions can both read the previous digest and both insert. SQLite // runs one write at a time, so the single statement is enough there. A // document going back to an earlier digest is recorded again, since only the -// newest snapshot is compared. The casts let Postgres type the parameters of -// the INSERT ... SELECT. +// snapshot current at that time is compared. The casts let Postgres type the +// parameters of the INSERT ... SELECT. func (db *DB) AddMetadataSnapshot(snap *MetadataSnapshot) (bool, error) { tx, err := db.Beginx() if err != nil { @@ -87,13 +90,13 @@ func (db *DB) AddMetadataSnapshot(snap *MetadataSnapshot) (bool, error) { CAST(? AS TEXT), CAST(? AS BIGINT), CAST(? AS BIGINT) WHERE COALESCE(( SELECT content_digest FROM metadata_snapshots - WHERE ecosystem = ? AND name = ? + WHERE ecosystem = ? AND name = ? AND valid_from_ms <= CAST(? AS BIGINT) ORDER BY valid_from_ms DESC, id DESC LIMIT 1 ), '') <> CAST(? AS TEXT) `), snap.Ecosystem, snap.Name, snap.StoragePath, snap.ContentType, snap.ContentEncoding, snap.ContentDigest, snap.Size, snap.ValidFromMS, - snap.Ecosystem, snap.Name, snap.ContentDigest) + snap.Ecosystem, snap.Name, snap.ValidFromMS, snap.ContentDigest) if err != nil { return false, fmt.Errorf("recording metadata snapshot: %w", err) } diff --git a/internal/database/metadata_snapshots_test.go b/internal/database/metadata_snapshots_test.go index 0b4919f..71865b9 100644 --- a/internal/database/metadata_snapshots_test.go +++ b/internal/database/metadata_snapshots_test.go @@ -70,6 +70,36 @@ func TestMetadataSnapshots_AddAndLookup(t *testing.T) { }) } +// TestMetadataSnapshots_RecordedLate adds a version after a newer one was +// recorded, as a request that fetched it first but recorded it last does. It +// is compared with the version current at its own time, so it keeps its place +// in the history. +func TestMetadataSnapshots_RecordedLate(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + day := time.Date(2026, 3, 1, 0, 0, 0, 0, time.UTC) + addSnapshot(t, db, "idx", "sha256:a", day) + addSnapshot(t, db, "idx", "sha256:c", day.Add(2*time.Hour)) + if !addSnapshot(t, db, "idx", "sha256:b", day.Add(time.Hour)) { + t.Fatal("version recorded after a newer one not added") + } + if addSnapshot(t, db, "idx", "sha256:b", day.Add(90*time.Minute)) { + t.Error("version already current at that time added again") + } + for _, tc := range []struct { + at time.Time + want string + }{ + {day.Add(30 * time.Minute), "sha256:a"}, + {day.Add(100 * time.Minute), "sha256:b"}, + {day.Add(3 * time.Hour), "sha256:c"}, + } { + if got := snapshotDigestAt(t, db, "idx", tc.at); got != tc.want { + t.Errorf("snapshot at %s = %q, want %q", tc.at, got, tc.want) + } + } + }) +} + func TestMetadataSnapshots_Prune(t *testing.T) { runWithBothDatabases(t, func(t *testing.T, db *DB) { now := time.Date(2026, 6, 1, 0, 0, 0, 0, time.UTC) diff --git a/internal/handler/apk.go b/internal/handler/apk.go index 83fee97..5de909a 100644 --- a/internal/handler/apk.go +++ b/internal/handler/apk.go @@ -1,7 +1,6 @@ package handler import ( - "context" "crypto/sha256" "encoding/hex" "net/http" @@ -169,10 +168,11 @@ func (h *APKHandler) handlePackageDownload(w http.ResponseWriter, r *http.Reques // and detached signatures valid. func (h *APKHandler) handleMetadata(w http.ResponseWriter, r *http.Request, repository, upstreamURL, path string) { cacheKey := h.metadataCacheKey(repository, upstreamURL, path) - h.proxy.ProxyCached(w, r, upstreamURL+"/"+path, apkEcosystem, cacheKey, "*/*") if h.snapshotsEnabled() { - h.recordSnapshot(context.WithoutCancel(r.Context()), cacheKey) + h.serveMetadataWithSnapshot(w, r, upstreamURL+"/"+path, cacheKey) + return } + h.proxy.ProxyCached(w, r, upstreamURL+"/"+path, apkEcosystem, cacheKey, "*/*") } // metadataCacheKey derives the metadata cache key from the repository name, diff --git a/internal/handler/apk_snapshots.go b/internal/handler/apk_snapshots.go index 0f29c85..d581980 100644 --- a/internal/handler/apk_snapshots.go +++ b/internal/handler/apk_snapshots.go @@ -1,8 +1,12 @@ package handler import ( + "bytes" "context" "crypto/rand" + "crypto/sha256" + "database/sql" + "encoding/hex" "io" "net/http" "strconv" @@ -101,22 +105,80 @@ func (h *APKHandler) serveSnapshot(w http.ResponseWriter, r *http.Request, repos } } -// recordSnapshot keeps the document now cached under cacheKey as a snapshot -// when it differs from the newest one kept. It runs after the document was -// served, so a failure only means this version is recorded on a later -// request instead. -func (h *APKHandler) recordSnapshot(ctx context.Context, cacheKey string) { - entry, err := h.proxy.DB.GetMetadataCache(apkEcosystem, cacheKey) - if err != nil || entry == nil || !entry.ContentDigest.Valid { - return - } - digest := entry.ContentDigest.String - latest, err := h.proxy.DB.LatestMetadataSnapshot(apkEcosystem, cacheKey) - if err != nil || (latest != nil && latest.ContentDigest == digest) { - return +// serveMetadataWithSnapshot is ProxyCached for an index or signature that +// keeps each version it serves as a snapshot. A version is recorded from the +// document the request serves, not by reading the cache again afterwards: a +// concurrent refresh can replace the cached copy at any time, and the version +// it replaced would then never be recorded. +func (h *APKHandler) serveMetadataWithSnapshot(w http.ResponseWriter, r *http.Request, upstream, cacheKey string) { + p := h.proxy + ctx := context.WithoutCancel(r.Context()) + + // A cached copy within its TTL was recorded by the request that fetched + // it; this only catches one stored before snapshots were enabled, or + // whose recording failed. + if entry := p.freshMetadataEntry(apkEcosystem, cacheKey); entry != nil { + seen := time.Now() + if p.serveCachedMetadata(w, r, apkEcosystem, cacheKey) { + h.recordStoredSnapshot(ctx, cacheKey, entry, seen) + return + } } - h.storeSnapshot(ctx, cacheKey, entry, time.Now()) + body, contentType, contentEncoding, err := p.fetchOrCacheMetadata(r.Context(), apkEcosystem, cacheKey, upstream, "identity", nil, "*/*") + if err != nil { + p.writeMetadataFetchError(w, err) + return + } + // Record before responding, so the version is kept by the time a + // client has it. + h.recordServedSnapshot(ctx, cacheKey, body, contentType, contentEncoding, time.Now()) + p.writeMetadataCachedResponseWithEncoding(w, r, apkEcosystem, cacheKey, body, contentType, contentEncoding) +} + +// recordServedSnapshot keeps body, the document a request is serving, as the +// version current from observed, unless the version current then already has +// its bytes. +func (h *APKHandler) recordServedSnapshot(ctx context.Context, cacheKey string, body []byte, contentType, contentEncoding string, observed time.Time) { + sum := sha256.Sum256(body) + digest := "sha256:" + hex.EncodeToString(sum[:]) + if !h.snapshotNeeded(cacheKey, digest, observed) { + return + } + storagePath := apkSnapshotStoragePath(cacheKey, digest, observed) + if _, _, err := h.proxy.Storage.Store(ctx, storagePath, bytes.NewReader(body)); err != nil { + h.proxy.Logger.Warn("failed to store apk snapshot", "path", storagePath, "error", err) + h.discardSnapshotBlob(ctx, storagePath) + return + } + h.addSnapshot(ctx, &database.MetadataSnapshot{ + Ecosystem: apkEcosystem, + Name: cacheKey, + StoragePath: storagePath, + ContentType: sql.NullString{String: contentType, Valid: contentType != ""}, + ContentEncoding: sql.NullString{String: contentEncoding, Valid: contentEncoding != ""}, + ContentDigest: digest, + Size: int64(len(body)), + ValidFromMS: observed.UnixMilli(), + }) +} + +// recordStoredSnapshot keeps the cached document entry describes as the +// version current from observed, unless the version current then already +// has its digest. +func (h *APKHandler) recordStoredSnapshot(ctx context.Context, cacheKey string, entry *database.MetadataCacheEntry, observed time.Time) { + if !entry.ContentDigest.Valid || !h.snapshotNeeded(cacheKey, entry.ContentDigest.String, observed) { + return + } + h.storeSnapshot(ctx, cacheKey, entry, observed) +} + +// snapshotNeeded reports whether a version with digest seen at observed +// differs from the version current then. It is only a shortcut that saves +// copying the document: AddMetadataSnapshot makes the same check. +func (h *APKHandler) snapshotNeeded(cacheKey, digest string, observed time.Time) bool { + current, err := h.proxy.DB.MetadataSnapshotAt(apkEcosystem, cacheKey, observed.Add(time.Millisecond)) + return err == nil && (current == nil || current.ContentDigest != digest) } // storeSnapshot records the cached document entry as the version current @@ -130,7 +192,7 @@ func (h *APKHandler) storeSnapshot(ctx context.Context, cacheKey string, entry * if !ok { return } - added, err := h.proxy.DB.AddMetadataSnapshot(&database.MetadataSnapshot{ + h.addSnapshot(ctx, &database.MetadataSnapshot{ Ecosystem: apkEcosystem, Name: cacheKey, StoragePath: storagePath, @@ -140,12 +202,18 @@ func (h *APKHandler) storeSnapshot(ctx context.Context, cacheKey string, entry * Size: size, ValidFromMS: validFrom.UnixMilli(), }) +} + +// addSnapshot adds the row for a snapshot whose blob is stored, and deletes +// the blob when the row isn't added, as when another request recorded the +// same version first. +func (h *APKHandler) addSnapshot(ctx context.Context, snap *database.MetadataSnapshot) { + added, err := h.proxy.DB.AddMetadataSnapshot(snap) if err != nil { - h.proxy.Logger.Warn("failed to record apk snapshot", "key", cacheKey, "error", err) + h.proxy.Logger.Warn("failed to record apk snapshot", "key", snap.Name, "error", err) } if err != nil || !added { - // Another request recorded this version first. - h.discardSnapshotBlob(ctx, storagePath) + h.discardSnapshotBlob(ctx, snap.StoragePath) } } diff --git a/internal/handler/apk_snapshots_test.go b/internal/handler/apk_snapshots_test.go index 38d4b08..b0f109d 100644 --- a/internal/handler/apk_snapshots_test.go +++ b/internal/handler/apk_snapshots_test.go @@ -1,6 +1,7 @@ package handler import ( + "io" "net/http" "net/http/httptest" "strings" @@ -206,6 +207,81 @@ func TestAPKSnapshots_SameVersionRecordedTwice(t *testing.T) { expectAPKResponse(t, h, "/alpine@"+day+apkSnapshotIndexPath, http.StatusOK, "index v1") } +// pausingWriter blocks the first Write until release is closed, after +// signalling on writing. +type pausingWriter struct { + *httptest.ResponseRecorder + once sync.Once + writing chan struct{} + release chan struct{} +} + +func (w *pausingWriter) Write(b []byte) (int, error) { + w.once.Do(func() { + close(w.writing) + <-w.release + }) + return w.ResponseRecorder.Write(b) +} + +// TestAPKSnapshots_OverlappingRequests pauses a request while it writes +// index v1, lets a second request fetch and serve v2, and then lets the first +// one finish. Both versions must be kept, in the order they were fetched, +// with readable blobs. +func TestAPKSnapshots_OverlappingRequests(t *testing.T) { + u := newAPKSnapshotUpstream(t) + h, db := newSnapshotAPKHandler(t, u.server.URL) + target := "/alpine" + apkSnapshotIndexPath + + first := &pausingWriter{ + ResponseRecorder: httptest.NewRecorder(), + writing: make(chan struct{}), + release: make(chan struct{}), + } + done := make(chan struct{}) + go func() { + defer close(done) + h.Routes().ServeHTTP(first, httptest.NewRequest(http.MethodGet, target, nil)) + }() + select { + case <-first.writing: + case <-done: + t.Fatal("first request finished without writing a body") + } + + u.index.Store("index v2") + expectAPKResponse(t, h, target, http.StatusOK, "index v2") + close(first.release) + <-done + if first.Code != http.StatusOK || first.Body.String() != "index v1" { + t.Fatalf("first request = %d %q, want 200 with index v1", first.Code, first.Body.String()) + } + + var snaps []database.MetadataSnapshot + if err := db.Select(&snaps, `SELECT `+ + `id, ecosystem, name, storage_path, content_type, content_encoding, content_digest, size, valid_from_ms `+ + `FROM metadata_snapshots ORDER BY valid_from_ms, id`); err != nil { + t.Fatal(err) + } + if len(snaps) != 2 { + t.Fatalf("%d snapshots recorded, want 2", len(snaps)) + } + for i, want := range []string{"index v1", "index v2"} { + body, err := h.proxy.Storage.Open(t.Context(), snaps[i].StoragePath) + if err != nil { + t.Fatalf("snapshot %d blob unreadable: %v", i, err) + } + got, _ := io.ReadAll(body) + _ = body.Close() + if string(got) != want { + t.Errorf("snapshot %d = %q, want %q", i, got, want) + } + if snaps[i].Size != int64(len(want)) { + t.Errorf("snapshot %d size = %d, want %d", i, snaps[i].Size, len(want)) + } + } +} + // TestAPKSnapshots_ConcurrentReplicas records the same index version from // two replicas sharing a Postgres database, many times at once. One snapshot // row must remain, with a readable blob, and the other copies are deleted. From 9458dc84e1ccdeace927c990a22510e819bf644a Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Fri, 30 Oct 2026 19:29:11 +0100 Subject: [PATCH 513/633] Share one PURL-to-handler mapping between the mirror and the inventory packageurl.HandlerPackage returns the ecosystem and name the handlers store a package URL under. The mirror and the inventory both use it, which settles where they disagreed: distribution namespaces of apk, alpm, bsd, deb and rpm are dropped, PyPI and NuGet names use purl's comparison form, and Maven names are group:artifact as the handler stores them, where the mirror used group/artifact. --- internal/inventory/inventory.go | 93 ++++------------------------ internal/inventory/inventory_test.go | 4 +- internal/mirror/files.go | 14 ----- internal/mirror/source.go | 21 +------ internal/mirror/source_test.go | 27 +++++--- internal/packageurl/handler.go | 59 ++++++++++++++++++ internal/packageurl/handler_test.go | 55 ++++++++++++++++ 7 files changed, 150 insertions(+), 123 deletions(-) create mode 100644 internal/packageurl/handler.go create mode 100644 internal/packageurl/handler_test.go diff --git a/internal/inventory/inventory.go b/internal/inventory/inventory.go index ccd9ecd..9a13a8b 100644 --- a/internal/inventory/inventory.go +++ b/internal/inventory/inventory.go @@ -7,7 +7,6 @@ import ( "errors" "fmt" "strings" - "unicode" "github.com/git-pkgs/proxy/internal/packageurl" "github.com/git-pkgs/purl" @@ -33,36 +32,16 @@ type Component struct { PackagePURL string } -// ecosystems maps the PURL types the proxy serves to the ecosystem name its -// handlers store packages under. -var ecosystems = map[string]string{ - "alpm": "arch", - "apk": "alpine", - "bsd": "freebsd", - "cargo": "cargo", - "chef": "chef", - "composer": "composer", - "conan": "conan", - "conda": "conda", - "cran": "cran", - "deb": "deb", - "gem": "gem", - "golang": "golang", - "hex": "hex", - "julia": "julia", - "maven": "maven", - "npm": "npm", - "nuget": "nuget", - "pub": "pub", - "pypi": "pypi", - "rpm": "rpm", - "swift": "swift", +// servedTypes are the PURL types the proxy serves packages of and the +// inventory records. +var servedTypes = map[string]bool{ + "alpm": true, "apk": true, "bsd": true, "cargo": true, "chef": true, + "composer": true, "conan": true, "conda": true, "cran": true, "deb": true, + "gem": true, "golang": true, "hex": true, "julia": true, "maven": true, + "npm": true, "nuget": true, "pub": true, "pypi": true, "rpm": true, + "swift": true, } -// distroTypes are PURL types whose namespace names the distribution -// ("debian", "alpine"), not part of the package name the proxy stores. -var distroTypes = map[string]bool{"alpm": true, "apk": true, "bsd": true, "deb": true, "rpm": true} - // ErrUnsupportedPURL is returned for a package URL whose type the proxy // doesn't serve. var ErrUnsupportedPURL = errors.New("package type is not served by the proxy") @@ -79,13 +58,12 @@ func FromPURL(s string) (Component, error) { if err != nil { return Component{}, err } - ecosystem, ok := ecosystems[p.Type] - if !ok { + if !servedTypes[p.Type] { return Component{}, fmt.Errorf("%w: %s", ErrUnsupportedPURL, p.Type) } - name, err := handlerName(p) - if err != nil { - return Component{}, err + ecosystem, name := packageurl.HandlerPackage(p) + if ecosystem == "conda" && !strings.Contains(name, "/") { + return Component{}, ErrNoCondaChannel } packagePURL, _ := packageurl.MakeCacheStrings(ecosystem, name, "") if packagePURL == "" { @@ -94,53 +72,6 @@ func FromPURL(s string) (Component, error) { return Component{Ecosystem: ecosystem, Name: name, Version: p.Version, PackagePURL: packagePURL}, nil } -// handlerName returns the package name the handler for p's type stores. -func handlerName(p *purl.PURL) (string, error) { - switch { - case distroTypes[p.Type]: - // The namespace is the distribution; the handlers store the bare - // package name. - return p.Name, nil - case p.Type == "conda" && p.Namespace == "": - // The conda handler stores "channel/name". - channel := p.Qualifier("channel") - if channel == "" { - return "", ErrNoCondaChannel - } - return channel + "/" + p.Name, nil - case p.Type == "nuget": - // The NuGet handler lowercases package IDs. - return strings.ToLower(p.FullName()), nil - case p.Type == "pypi": - return normalizePyPIName(p.FullName()), nil - } - return p.FullName(), nil -} - -// normalizePyPIName normalizes a PyPI project name as PEP 503 does: runs of -// "-", "_" and "." become one "-", and letters are lowercased. The PyPI -// handler stores the name from the file name, which wheels and current sdists -// spell with "_" for any of these, so "zope.interface" is cached as -// "zope-interface". (An old sdist named "zope.interface-5.0.tar.gz" is -// stored with the dot and doesn't match.) -func normalizePyPIName(name string) string { - var b strings.Builder - b.Grow(len(name)) - separator := false - for _, r := range name { - if r == '-' || r == '_' || r == '.' { - separator = true - continue - } - if separator && b.Len() > 0 { - b.WriteByte('-') - } - separator = false - b.WriteRune(unicode.ToLower(r)) - } - return b.String() -} - // syftMainModule is the property syft sets on modules found in a Go binary, // naming the module the binary was built from. const syftMainModule = "syft:metadata:mainModule" diff --git a/internal/inventory/inventory_test.go b/internal/inventory/inventory_test.go index e5b6225..818129a 100644 --- a/internal/inventory/inventory_test.go +++ b/internal/inventory/inventory_test.go @@ -128,7 +128,7 @@ func TestFromPURLMatchesHandlerNames(t *testing.T) { if err != nil { t.Fatalf("FromPURL(%q): %v", tt.sbomPURL, err) } - want, _ := packageurl.MakeCacheStrings(ecosystems[tt.purlType], tt.handlerName, "") + want, _ := packageurl.MakeCacheStrings(packageurl.HandlerEcosystem(tt.purlType), tt.handlerName, "") if want == "" { t.Fatalf("handler name %q has no package URL", tt.handlerName) } @@ -137,7 +137,7 @@ func TestFromPURLMatchesHandlerNames(t *testing.T) { } }) } - for purlType := range ecosystems { + for purlType := range servedTypes { if !covered[purlType] { t.Errorf("no case for PURL type %q", purlType) } diff --git a/internal/mirror/files.go b/internal/mirror/files.go index 8cf61fe..7486e8c 100644 --- a/internal/mirror/files.go +++ b/internal/mirror/files.go @@ -116,17 +116,3 @@ func fileQualifiers(ecosystem string, qualifiers map[string]string) map[string]s } return kept } - -// condaName returns a conda package name with its channel, taken from the -// namespace or the channel qualifier. Without either it stays bare, and -// mirroring refuses it. -func condaName(namespace, name, channel string) string { - switch { - case namespace != "": - return namespace + "/" + name - case channel != "": - return channel + "/" + name - default: - return name - } -} diff --git a/internal/mirror/source.go b/internal/mirror/source.go index ecc3a0c..22812b6 100644 --- a/internal/mirror/source.go +++ b/internal/mirror/source.go @@ -5,6 +5,7 @@ import ( "fmt" "strings" + "github.com/git-pkgs/proxy/internal/packageurl" "github.com/git-pkgs/purl" "github.com/git-pkgs/registries" "github.com/git-pkgs/sbom" @@ -28,18 +29,6 @@ func (pv PackageVersion) String() string { return fmt.Sprintf("pkg:%s/%s@%s", pv.Ecosystem, pv.Name, pv.Version) } -// handlerEcosystem returns the ecosystem name the proxy's handlers store -// packages of a PURL type under. purl names gem and composer after their -// registries, rubygems and packagist; the handlers use the type. -func handlerEcosystem(purlType string) string { - switch purlType { - case "gem", "composer": - return purlType - default: - return purl.PURLTypeToEcosystem(purlType) - } -} - // Source produces PackageVersion items for mirroring. type Source interface { Enumerate(ctx context.Context, fn func(PackageVersion) error) error @@ -140,13 +129,7 @@ func (s *PURLSource) listVersions(ctx context.Context, versions VersionLister, e // mirrorTarget returns the ecosystem, name and file qualifiers a PURL is // mirrored under. func mirrorTarget(p *purl.PURL) (ecosystem, name string, qualifiers map[string]string) { - ecosystem = handlerEcosystem(p.Type) - name = p.Name - if ecosystem == "conda" { - name = condaName(p.Namespace, p.Name, p.Qualifier("channel")) - } else if p.Namespace != "" { - name = p.Namespace + "/" + p.Name - } + ecosystem, name = packageurl.HandlerPackage(p) qualifiers = fileQualifiers(ecosystem, p.Qualifiers.Map()) if ecosystem == "julia" { qualifiers = map[string]string{"uuid": p.Qualifier("uuid")} diff --git a/internal/mirror/source_test.go b/internal/mirror/source_test.go index 9c69afe..faddf35 100644 --- a/internal/mirror/source_test.go +++ b/internal/mirror/source_test.go @@ -3,6 +3,7 @@ package mirror import ( "context" "encoding/json" + "slices" "strings" "testing" ) @@ -229,26 +230,38 @@ func marshalJSON(t *testing.T, v any) []byte { return data } -// The mirror files packages under the ecosystem names the handlers use, so -// a mirrored package and the same package cached by a client share one name. +// The mirror files packages under the ecosystem and package names the +// handlers use, so a mirrored package and the same package cached by a client +// share one name. func TestPURLSourceUsesHandlerEcosystemNames(t *testing.T) { source := &PURLSource{PURLs: []string{ "pkg:gem/rake@13.2.1", "pkg:composer/psr/log@3.0.2", - "pkg:apk/curl@8.14.1-r2", + "pkg:apk/alpine/curl@8.14.1-r2", "pkg:npm/left-pad@1.3.0", + "pkg:maven/org.slf4j/slf4j-api@2.0.13", + "pkg:pypi/Zope.Interface@6.4", + "pkg:nuget/Newtonsoft.Json@13.0.3", }} var got []string if err := source.Enumerate(context.Background(), func(pv PackageVersion) error { - got = append(got, pv.Ecosystem) + got = append(got, pv.Ecosystem+" "+pv.Name) return nil }); err != nil { t.Fatalf("Enumerate() error = %v", err) } - want := []string{"gem", "composer", "alpine", "npm"} - if strings.Join(got, ",") != strings.Join(want, ",") { - t.Errorf("ecosystems = %v, want %v", got, want) + want := []string{ + "gem rake", + "composer psr/log", + "alpine curl", + "npm left-pad", + "maven org.slf4j:slf4j-api", + "pypi zope-interface", + "nuget newtonsoft.json", + } + if !slices.Equal(got, want) { + t.Errorf("packages = %q, want %q", got, want) } } diff --git a/internal/packageurl/handler.go b/internal/packageurl/handler.go new file mode 100644 index 0000000..6056fcb --- /dev/null +++ b/internal/packageurl/handler.go @@ -0,0 +1,59 @@ +package packageurl + +import ( + "github.com/git-pkgs/purl" +) + +// distroTypes are PURL types whose namespace names the distribution +// ("alpine", "debian"), not part of the package name the handlers store. +var distroTypes = map[string]bool{"alpm": true, "apk": true, "bsd": true, "deb": true, "rpm": true} + +// HandlerEcosystem returns the ecosystem name the proxy's handlers store +// packages of a PURL type under. purl names gem and composer after their +// registries, rubygems and packagist; the handlers use the type. +func HandlerEcosystem(purlType string) string { + switch purlType { + case "gem", "composer": + return purlType + default: + return purl.PURLTypeToEcosystem(purlType) + } +} + +// HandlerPackage returns the ecosystem and package name the proxy's handlers +// store p under, so a package named in a PURL, such as one from an SBOM, +// matches the package a client fetched through the proxy: +// +// - apk, alpm, bsd, deb and rpm: the namespace names the distribution and +// is dropped +// - conda: the channel, from the namespace or the channel qualifier, is +// part of the name ("conda-forge/numpy"); without either the name stays +// bare +// - pypi and nuget: the name in purl's comparison form, PEP 503 for PyPI +// and lowercased for NuGet, as the handlers store them +// +// Other types use purl's full name, which keeps the namespace: "@babel/core", +// or "org.slf4j:slf4j-api" for Maven, as its handler stores it. +func HandlerPackage(p *purl.PURL) (ecosystem, name string) { + ecosystem = HandlerEcosystem(p.Type) + switch { + case distroTypes[p.Type]: + return ecosystem, p.Name + case p.Type == "conda": + return ecosystem, condaName(p.Namespace, p.Name, p.Qualifier("channel")) + case p.Type == "pypi" || p.Type == "nuget": + return ecosystem, p.ComparisonForm().FullName() + } + return ecosystem, p.FullName() +} + +func condaName(namespace, name, channel string) string { + switch { + case namespace != "": + return namespace + "/" + name + case channel != "": + return channel + "/" + name + default: + return name + } +} diff --git a/internal/packageurl/handler_test.go b/internal/packageurl/handler_test.go new file mode 100644 index 0000000..5ae9443 --- /dev/null +++ b/internal/packageurl/handler_test.go @@ -0,0 +1,55 @@ +package packageurl + +import ( + "testing" + + "github.com/git-pkgs/purl" +) + +// HandlerPackage is used by both the mirror and the project inventory, so a +// mirrored package, a package listed in an SBOM and the same package fetched +// by a client end up under one name. +func TestHandlerPackage(t *testing.T) { + tests := []struct { + purl string + ecosystem string + name string + }{ + // purl names these after their registries; the handlers use the type. + {"pkg:gem/rake@13.2.1", "gem", "rake"}, + {"pkg:composer/psr/log@3.0.2", "composer", "psr/log"}, + // Distribution namespaces are dropped. + {"pkg:apk/alpine/busybox@1.36.1-r15?arch=x86_64", "alpine", "busybox"}, + {"pkg:apk/curl@8.14.1-r2", "alpine", "curl"}, + {"pkg:alpm/arch/glibc@2.39-1", "arch", "glibc"}, + {"pkg:bsd/freebsd/curl@8.7.1", "freebsd", "curl"}, + {"pkg:deb/debian/libc6@2.36-9?arch=amd64", "deb", "libc6"}, + {"pkg:rpm/fedora/curl@8.6.0-7.fc40", "rpm", "curl"}, + // The conda channel is part of the name. + {"pkg:conda/numpy@1.26.4?channel=conda-forge", "conda", "conda-forge/numpy"}, + {"pkg:conda/conda-forge/numpy@1.26.4", "conda", "conda-forge/numpy"}, + {"pkg:conda/numpy@1.26.4", "conda", "numpy"}, + // PyPI names are normalized as PEP 503 does, NuGet IDs lowercased. + {"pkg:pypi/Zope.Interface@6.4", "pypi", "zope-interface"}, + {"pkg:pypi/zope_interface", "pypi", "zope-interface"}, + {"pkg:nuget/Newtonsoft.Json@13.0.3", "nuget", "newtonsoft.json"}, + // Other namespaces stay part of the name. + {"pkg:npm/%40babel/core@7.23.0", "npm", "@babel/core"}, + // The Maven handler stores "group:artifact". + {"pkg:maven/org.slf4j/slf4j-api@2.0.13", "maven", "org.slf4j:slf4j-api"}, + {"pkg:docker/library/alpine@3.20", "docker", "library/alpine"}, + {"pkg:julia/Example@0.5.3?uuid=7876af07-990d-54b4-ab0e-23690620f79a", "julia", "Example"}, + } + for _, tt := range tests { + t.Run(tt.purl, func(t *testing.T) { + p, err := purl.Parse(tt.purl) + if err != nil { + t.Fatal(err) + } + ecosystem, name := HandlerPackage(p) + if ecosystem != tt.ecosystem || name != tt.name { + t.Errorf("HandlerPackage(%q) = %q, %q, want %q, %q", tt.purl, ecosystem, name, tt.ecosystem, tt.name) + } + }) + } +} From 479f26e09e6128e71f7c424e335406326d30562f Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Fri, 30 Oct 2026 19:29:11 +0100 Subject: [PATCH 514/633] Start project pruning with the project API and move enrichment routes out of serve --- internal/server/projects_api.go | 7 +++++-- internal/server/server.go | 25 +++++++++++++------------ 2 files changed, 18 insertions(+), 14 deletions(-) diff --git a/internal/server/projects_api.go b/internal/server/projects_api.go index f297a9a..750be17 100644 --- a/internal/server/projects_api.go +++ b/internal/server/projects_api.go @@ -1,6 +1,7 @@ package server import ( + "context" "errors" "io" "log/slog" @@ -82,11 +83,13 @@ type ProjectResponse struct { Components []ProjectComponent `json:"components"` } -// mountProjectAPI registers /api/projects when projects.enabled is set. -func (s *Server) mountProjectAPI(r chi.Router) { +// mountProjectAPI registers /api/projects when projects.enabled is set, and +// starts pruning projects that sent no SBOM within projects.max_age. +func (s *Server) mountProjectAPI(ctx context.Context, r chi.Router) { if s.cfg.Projects.Enabled { registerProjectRoutes(r, NewProjectsAPIHandler(s.db, s.logger), s.cfg.Projects.TokenExpanded()) } + s.startProjectPruning(ctx) } // registerProjectRoutes mounts /api/projects. Reading is open like the rest diff --git a/internal/server/server.go b/internal/server/server.go index 0ec6470..1812957 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -242,6 +242,17 @@ func (s *Server) Start(listeners ...net.Listener) error { return s.serve(listener) } +// mountEnrichmentAPI registers the API endpoints for enrichment data. +func (s *Server) mountEnrichmentAPI(r chi.Router) { + apiHandler := NewAPIHandler(enrichment.New(s.logger), s.db) + r.Get("/api/package/{ecosystem}/*", apiHandler.HandlePackagePath) + r.Get("/api/vulns/{ecosystem}/*", apiHandler.HandleVulnsPath) + r.Post("/api/outdated", apiHandler.HandleOutdated) + r.Post("/api/bulk", apiHandler.HandleBulkLookup) + r.Get("/api/search", apiHandler.HandleSearch) + r.Get("/api/packages", apiHandler.HandlePackagesList) +} + func (s *Server) serve(listener net.Listener) error { metadataClient, artifactClient := s.newUpstreamClients(s.cfg.ParseArtifactTimeout()) @@ -379,16 +390,7 @@ func (s *Server) serve(listener net.Listener) error { r.Delete("/api/cache/{ecosystem}/*", s.handleCacheDelete) } - // API endpoints for enrichment data - enrichSvc := enrichment.New(s.logger) - apiHandler := NewAPIHandler(enrichSvc, s.db) - - r.Get("/api/package/{ecosystem}/*", apiHandler.HandlePackagePath) - r.Get("/api/vulns/{ecosystem}/*", apiHandler.HandleVulnsPath) - r.Post("/api/outdated", apiHandler.HandleOutdated) - r.Post("/api/bulk", apiHandler.HandleBulkLookup) - r.Get("/api/search", apiHandler.HandleSearch) - r.Get("/api/packages", apiHandler.HandlePackagesList) + s.mountEnrichmentAPI(r) // Start background context (used by mirror jobs and cleanup) bgCtx, bgCancel := context.WithCancel(context.Background()) @@ -419,8 +421,7 @@ func (s *Server) serve(listener net.Listener) error { } // Project inventory endpoints (opt-in via projects.enabled or PROXY_PROJECTS_ENABLED) - s.mountProjectAPI(r) - s.startProjectPruning(bgCtx) + s.mountProjectAPI(bgCtx, r) s.http = &http.Server{ Addr: s.cfg.Listen, From 4ae534eba391826c6c60babf92632063b10e3357 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sat, 31 Oct 2026 07:58:51 +0100 Subject: [PATCH 515/633] Fetch an image's blobs concurrently when mirroring The image mirror fetched the config and layer blobs of a manifest one after another, so a run took as long as all layers together. It now fetches up to three at once, as many as a Docker pull does by default. For a Home Assistant image (550 MB in ten blobs) that would leave mostly the time of its largest layer. --- internal/handler/container_mirror.go | 46 +++++++++++++++++------ internal/handler/container_mirror_test.go | 36 ++++++++++++++++++ 2 files changed, 70 insertions(+), 12 deletions(-) diff --git a/internal/handler/container_mirror.go b/internal/handler/container_mirror.go index ddd3022..82a98cf 100644 --- a/internal/handler/container_mirror.go +++ b/internal/handler/container_mirror.go @@ -11,7 +11,10 @@ import ( "regexp" "slices" "strings" + "sync" "time" + + "golang.org/x/sync/errgroup" ) const ( @@ -19,6 +22,9 @@ const ( maxMirrorIndexDepth = 2 // mirrorErrorBodyLimit bounds how much of an error response is quoted. mirrorErrorBodyLimit = 200 + // mirrorBlobConcurrency is how many blobs of one manifest are fetched at + // once, as many as a Docker pull fetches by default. + mirrorBlobConcurrency = 3 dockerReferenceTypeAnnotation = "vnd.docker.reference.type" dockerReferenceDigestAnnotation = "vnd.docker.reference.digest" @@ -439,6 +445,7 @@ func (m *imageMirror) mirrorBlobs(ctx context.Context, manifest *ociManifest) er if manifest.Config != nil { blobs = append([]ociDescriptor{*manifest.Config}, blobs...) } + var digests []string for _, blob := range blobs { if len(blob.URLs) > 0 || strings.Contains(blob.MediaType, "nondistributable") || strings.Contains(blob.MediaType, "foreign") { m.result.Warnings = append(m.result.Warnings, fmt.Sprintf("skipped non-distributable layer %s", blob.Digest)) @@ -451,34 +458,49 @@ func (m *imageMirror) mirrorBlobs(ctx context.Context, manifest *ociManifest) er continue } m.seen[blob.Digest] = true - if err := m.mirrorBlob(ctx, blob.Digest); err != nil { - return err - } + digests = append(digests, blob.Digest) } - return nil + + // Large layers take most of the time, so they are fetched side by side. + var mu sync.Mutex + g, gctx := errgroup.WithContext(ctx) + g.SetLimit(mirrorBlobConcurrency) + for _, digest := range digests { + g.Go(func() error { + size, fetched, err := m.mirrorBlob(gctx, digest) + if err != nil || !fetched { + return err + } + mu.Lock() + m.result.Fetched++ + m.result.Bytes += size + mu.Unlock() + return nil + }) + } + return g.Wait() } -// mirrorBlob stores a blob under the same identity handleBlobDownload uses. -func (m *imageMirror) mirrorBlob(ctx context.Context, digest string) error { +// mirrorBlob stores a blob under the same identity handleBlobDownload uses, +// and returns its size when it had to be fetched. +func (m *imageMirror) mirrorBlob(ctx context.Context, digest string) (int64, bool, error) { proxy := m.h.proxy cached, err := proxy.HasCachedArtifact(ctx, "oci", m.cacheName, digest, digest) if err != nil { - return fmt.Errorf("blob %s: %w", digest, err) + return 0, false, fmt.Errorf("blob %s: %w", digest, err) } if cached { - return nil + return 0, false, nil } result, err := proxy.GetOrFetchArtifactFromURLWithDigest(ctx, "oci", m.cacheName, digest, digest, fmt.Sprintf("%s/v2/%s/blobs/%s", m.registryURL, m.upstreamName, digest), digest) if err != nil { - return fmt.Errorf("blob %s: %w", digest, err) + return 0, false, fmt.Errorf("blob %s: %w", digest, err) } if result.Reader != nil { _ = result.Reader.Close() } - m.result.Fetched++ - m.result.Bytes += result.Artifact.Size - return nil + return result.Artifact.Size, true, nil } // mirrorResponse collects the handler's manifest response in memory. diff --git a/internal/handler/container_mirror_test.go b/internal/handler/container_mirror_test.go index 149fb23..61b0ceb 100644 --- a/internal/handler/container_mirror_test.go +++ b/internal/handler/container_mirror_test.go @@ -10,8 +10,10 @@ import ( "net/url" "strconv" "strings" + "sync" "sync/atomic" "testing" + "time" "github.com/git-pkgs/registries/fetch" ) @@ -503,3 +505,37 @@ func TestMirrorImageRefetchesBlobMissingFromStorage(t *testing.T) { t.Errorf("second run fetched %d blobs, want the one missing from storage", got) } } + +func TestMirrorImageFetchesBlobsConcurrently(t *testing.T) { + registry, _ := newFakeImageRegistry(t) + proxy := newMirrorTestProxy(t, registry) + h := NewContainerHandlerWithRegistry(proxy, "http://proxy.example", registry.server.URL) + + // "single" has a config and a layer blob. Each request waits until the + // other is in flight too, or gives up after a while. + var inFlight atomic.Int32 + var concurrent atomic.Bool + both := make(chan struct{}) + var once sync.Once + registry.onBlob = func(string) { + defer inFlight.Add(-1) + if inFlight.Add(1) >= 2 { + concurrent.Store(true) + once.Do(func() { close(both) }) + } + select { + case <-both: + case <-time.After(2 * time.Second): + } + } + result, err := h.MirrorImage(context.Background(), ImageRef{Name: "team/app", References: []string{"single"}}, nil) + if err != nil { + t.Fatalf("MirrorImage: %v", err) + } + if !concurrent.Load() { + t.Error("blobs were fetched one after another, want them fetched concurrently") + } + if result.Fetched != 3 { + t.Errorf("fetched = %d, want the manifest and its two blobs", result.Fetched) + } +} From a79a58b04ef1a1e84c7d629ce7ec7e810d16e4ba Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sat, 31 Oct 2026 08:27:39 +0100 Subject: [PATCH 516/633] Guard the fake registry's manifests in mirror tests TestMirrorImageKeepsTagMovedByPullDuringWalk moves a tag while a blob is served. With blobs fetched concurrently, another blob request reads the manifest map at the same time, which the race detector caught on Windows. --- internal/handler/container_mirror_test.go | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/internal/handler/container_mirror_test.go b/internal/handler/container_mirror_test.go index 61b0ceb..d2cb54b 100644 --- a/internal/handler/container_mirror_test.go +++ b/internal/handler/container_mirror_test.go @@ -21,8 +21,10 @@ import ( // fakeImageRegistry serves manifests and blobs of a small multi-arch image, // a single-arch image and a moved tag. It fails with 503 once offline. type fakeImageRegistry struct { - server *httptest.Server - offline atomic.Bool + server *httptest.Server + offline atomic.Bool + // mu guards manifests against a test changing it while blobs are served. + mu sync.RWMutex manifests map[string]fakeManifest blobs map[string]string blobGets atomic.Int32 @@ -119,7 +121,10 @@ func newFakeImageRegistry(t *testing.T) (*fakeImageRegistry, fakeImage) { http.Error(w, "offline", http.StatusServiceUnavailable) return } - if doc, ok := r.manifests[req.URL.Path]; ok { + r.mu.RLock() + doc, ok := r.manifests[req.URL.Path] + r.mu.RUnlock() + if ok { w.Header().Set("Content-Type", doc.contentType) w.Header().Set("Docker-Content-Digest", sha256Digest([]byte(doc.body))) if req.Method == http.MethodGet && !strings.Contains(req.URL.Path, "/manifests/sha256:") { @@ -434,7 +439,9 @@ func TestMirrorImageKeepsTagMovedByPullDuringWalk(t *testing.T) { var once atomic.Bool registry.onBlob = func(string) { if once.CompareAndSwap(false, true) { + registry.mu.Lock() registry.manifests["/v2/team/app/manifests/single"] = moved + registry.mu.Unlock() if response := pull(t, h, "/team/app/manifests/single", dockerManifestAccept); response.Code != http.StatusOK { t.Errorf("client pull during the walk: status %d", response.Code) } From 3eadb3eb46b13d8217a554eeda0eeb1944b30eb1 Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Sat, 31 Oct 2026 09:49:35 +0000 Subject: [PATCH 517/633] Preserve Swift and Go compatibility when upgrading PURL (#642) * Handle Swift registry identities in cache and vulnerability APIs * Preserve Go cache and policy behavior when upgrading PURL --- docs/configuration.md | 3 ++ go.mod | 6 +-- go.sum | 12 ++--- internal/config/config.go | 3 +- internal/database/rekey_artifact.go | 40 ++++++++++++++ internal/database/rekey_artifact_test.go | 69 ++++++++++++++++++++++++ internal/denylist/denylist.go | 3 +- internal/enrichment/enrichment.go | 9 ++-- internal/enrichment/enrichment_test.go | 11 ++-- internal/handler/cooldown_expiry.go | 7 ++- internal/handler/cooldown_fail_closed.go | 2 +- internal/handler/denylist_test.go | 6 +++ internal/handler/go.go | 60 +++++++++++++++------ internal/handler/go_case_test.go | 69 ++++++++++++++++++++++++ internal/handler/go_cooldown.go | 8 +-- internal/handler/go_policy_test.go | 47 ++++++++++++++++ internal/handler/handler.go | 6 +++ internal/packageurl/packageurl.go | 33 ++++++------ internal/packageurl/packageurl_test.go | 25 ++++----- internal/retention/retention.go | 11 ++-- internal/server/api.go | 9 ++++ internal/server/retention_golang_test.go | 4 +- internal/server/swift_vulns_test.go | 57 ++++++++++++++++++++ 23 files changed, 422 insertions(+), 78 deletions(-) create mode 100644 internal/database/rekey_artifact.go create mode 100644 internal/database/rekey_artifact_test.go create mode 100644 internal/handler/go_policy_test.go create mode 100644 internal/server/swift_vulns_test.go diff --git a/docs/configuration.md b/docs/configuration.md index e1a4f9e..b20685f 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1119,6 +1119,8 @@ Durations support days (`7d`), hours (`48h`), and minutes (`30m`). Set to `0` to Package PURL keys are normalized to canonical form before matching, so `pkg:npm/@babel/core` and `pkg:npm/%40babel/core` are equivalent, as are `pkg:pypi/Django` and `pkg:pypi/django`. If both forms configure the same package, the canonical entry wins. +Go module names in exact cooldown overrides are matched case-insensitively. If both lowercase and mixed-case keys are configured, the lowercase entry wins. Go package patterns match lowercase module names. + `package_patterns` matches canonical, versionless PURLs using `*` for zero or more characters and `?` for one character. Neither wildcard crosses `/` separators. Character classes (`[...]`) and backslash escapes are rejected at startup. For example, `"pkg:npm/@example/*"` matches packages under the `@example` npm scope. Patterns accept `@` as an alias for `%40`; other characters must use their canonical PURL form. Equivalent patterns with different durations are rejected at startup. Equal durations, such as `1d` and `24h`, are accepted. Exact `packages` entries take precedence over patterns. When several patterns match, longer patterns win after excluding `*` and `?` from the length. Ties use lexical order of the normalized patterns. @@ -1180,6 +1182,7 @@ Names and versions are compared the way each ecosystem treats them: `pkg:composer/symfony/console@6.1.7` also covers `v6.1.7`. - Go versions are matched as written, `pkg:golang/example.com/mod@v1.0.0-RC1`, though download URLs escape uppercase letters (`v1.0.0-!r!c1`). + Go module names in denylist entries are matched case-insensitively. - A Swift entry names the registry scope and package, ignoring case: `pkg:swift/apple/swift-nio@2.62.0`. The stored form, `pkg:generic/swift-registry/apple.swift-nio@2.62.0`, works too. diff --git a/go.mod b/go.mod index 4a164c2..2ce65e2 100644 --- a/go.mod +++ b/go.mod @@ -12,12 +12,12 @@ require ( github.com/git-pkgs/archives v0.10.0 github.com/git-pkgs/artifacts v0.2.1 github.com/git-pkgs/cooldown v0.3.0 - github.com/git-pkgs/enrichment v0.7.4 + github.com/git-pkgs/enrichment v0.7.5 github.com/git-pkgs/gcs v0.1.0 github.com/git-pkgs/integrity v0.2.0 github.com/git-pkgs/magic v0.6.0 - github.com/git-pkgs/purl v0.1.24 - github.com/git-pkgs/registries v0.12.0 + github.com/git-pkgs/purl v0.2.1 + github.com/git-pkgs/registries v0.13.0 github.com/git-pkgs/sbom v0.1.7 github.com/git-pkgs/spdx v0.3.3 github.com/go-chi/chi/v5 v5.5.0 diff --git a/go.sum b/go.sum index 983c87e..c322161 100644 --- a/go.sum +++ b/go.sum @@ -123,8 +123,8 @@ github.com/git-pkgs/artifacts v0.2.1 h1:VwdxR4yTDaqBZ34h0slxQBVyr2Xfa+QGOKCKviMx github.com/git-pkgs/artifacts v0.2.1/go.mod h1:Otosgq52pXT5UNN7lh6s/lszpWKVDO8XrOjN1M70/IA= github.com/git-pkgs/cooldown v0.3.0 h1:eAl2CQFwqXxluoVP9hHmropAf1VoV3gSxLMkN+NEu8A= github.com/git-pkgs/cooldown v0.3.0/go.mod h1:FnTg8wufEALATC9Sm/9AHjLdA+u9eVn96e9BqXXjGB0= -github.com/git-pkgs/enrichment v0.7.4 h1:owiCKjITG8tNxBikcyUCDuDf1i3C/+Yoa5yJseuqIjA= -github.com/git-pkgs/enrichment v0.7.4/go.mod h1:ZN/cH4PnmzsapEVPvmAwREP7+ox+RX0yfR6KvoB8xKE= +github.com/git-pkgs/enrichment v0.7.5 h1:jbfFzf7+HSS7el8AuMri4vHXq2izIIZDVIU0uYI5LFM= +github.com/git-pkgs/enrichment v0.7.5/go.mod h1:TbxC/nGQN96YCbdoiDs8a9z3gk2zXmLVS1989/jvnsU= github.com/git-pkgs/gcs v0.1.0 h1:E3awGtsO0xZyHT9FUfEwMHjMkRxw41Bh+c7lgTmPvBo= github.com/git-pkgs/gcs v0.1.0/go.mod h1:bdkCFD66ryaWnU8MBhokVA3WkJfyAEchm9qec5woBpE= github.com/git-pkgs/integrity v0.2.0 h1:sxpAk7EUllZnTYttOoPTW/Yfe5J2imglpuJiuBUZOxM= @@ -133,10 +133,10 @@ github.com/git-pkgs/magic v0.6.0 h1:fINPe67FEIaFRVlZzhOIDClMkm+gftBroBENa8bHSxQ= github.com/git-pkgs/magic v0.6.0/go.mod h1:SXOqcsNmbmpZjJZHEEWnwxprbsFvpwWgTAZrgXp4Jm4= github.com/git-pkgs/pom v0.1.8 h1:mXCuWw6XkrKKZb6IurRZyIznYLPhV5AYg5ZO38Uj/l8= github.com/git-pkgs/pom v0.1.8/go.mod h1:sQBeYRzjCUn77fcABPn6I/a1aBHFxm55jTmkNxYtGtw= -github.com/git-pkgs/purl v0.1.24 h1:JMEB1uNWI+baEpl6AeVx+Kfq1c+xkou+THxa5KJ3MrU= -github.com/git-pkgs/purl v0.1.24/go.mod h1:l8rCfvt5iXVrFIcoQ8+vCLOaHhrtEW2id0LENP+IzMw= -github.com/git-pkgs/registries v0.12.0 h1:fhxYaR5iR9Zlx6doLkjlXurVqPxkYcXmNFiIrFeV3/k= -github.com/git-pkgs/registries v0.12.0/go.mod h1:bKxgdy/PwnzQXL2FnOktmSQHTKk6B85MA4VtvvYItPc= +github.com/git-pkgs/purl v0.2.1 h1:b46hRlOMbrG6aXjjFJdYixGi5WTh0TLlAnH9iNeuOS0= +github.com/git-pkgs/purl v0.2.1/go.mod h1:l8rCfvt5iXVrFIcoQ8+vCLOaHhrtEW2id0LENP+IzMw= +github.com/git-pkgs/registries v0.13.0 h1:5i9dHnbiXseIbXLIZ61YnuFySJ9dQX+KEu1MrmwB2BU= +github.com/git-pkgs/registries v0.13.0/go.mod h1:1g/jXVi1M/3jWvW1Wom1NMnPrhglJ7UVEZ215l2Zh0M= github.com/git-pkgs/sbom v0.1.7 h1:4eboy/Yed/SC9ZcC/xT50iywHPzWbmCU7azDb5T1RbA= github.com/git-pkgs/sbom v0.1.7/go.mod h1:KIkZX0v31l62SeFsi2qsQXcZRVrAXvIdPQ4x4en+G2M= github.com/git-pkgs/spdx v0.3.3 h1:xXlUoZzjDSkwxC6mWHfaW+kc8a0+LH6d+gGHuZWftqI= diff --git a/internal/config/config.go b/internal/config/config.go index 5e90150..c09a1ba 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -77,6 +77,7 @@ import ( "github.com/git-pkgs/cooldown" "github.com/git-pkgs/proxy/internal/denylist" + "github.com/git-pkgs/proxy/internal/packageurl" "github.com/git-pkgs/proxy/internal/retention" "github.com/git-pkgs/purl" "gopkg.in/yaml.v3" @@ -261,7 +262,7 @@ func (c *CooldownConfig) NormalizedPackages() map[string]string { for _, key := range keys { canonical := key if parsed, err := purl.Parse(key); err == nil { - canonical = parsed.String() + canonical = packageurl.PolicyPackageKey(parsed.String()) } if _, exists := normalized[canonical]; exists && key != canonical { continue diff --git a/internal/database/rekey_artifact.go b/internal/database/rekey_artifact.go new file mode 100644 index 0000000..68a8dbd --- /dev/null +++ b/internal/database/rekey_artifact.go @@ -0,0 +1,40 @@ +package database + +import ( + "fmt" + "time" +) + +// RekeyCachedArtifact moves one unchanged upstream record without replacing +// an existing destination. Access history and stored bytes are preserved. +func (db *DB) RekeyCachedArtifact(old *Artifact, pkg *Package, version *Version) (bool, error) { + tx, err := db.Beginx() + if err != nil { + return false, err + } + defer func() { _ = tx.Rollback() }() + now := time.Now() + _, err = tx.Exec(db.Rebind(`INSERT INTO packages (purl, ecosystem, name, created_at, updated_at) + VALUES (?, ?, ?, ?, ?) ON CONFLICT(purl) DO NOTHING`), pkg.PURL, pkg.Ecosystem, pkg.Name, now, now) + if err != nil { + return false, fmt.Errorf("creating artifact package: %w", err) + } + _, err = tx.Exec(db.Rebind(`INSERT INTO versions (purl, package_purl, created_at, updated_at) + VALUES (?, ?, ?, ?) ON CONFLICT(purl) DO NOTHING`), version.PURL, version.PackagePURL, now, now) + if err != nil { + return false, fmt.Errorf("creating artifact version: %w", err) + } + result, err := tx.Exec(db.Rebind(`UPDATE artifacts SET version_purl = ?, updated_at = ? + WHERE id = ? AND version_purl = ? AND storage_path = ? AND upstream_url = ? AND source = ? + AND NOT EXISTS (SELECT 1 FROM artifacts WHERE version_purl = ? AND filename = ?)`), + version.PURL, now, old.ID, old.VersionPURL, old.StoragePath, old.UpstreamURL, SourceUpstream, + version.PURL, old.Filename) + if err != nil { + return false, fmt.Errorf("moving cached artifact: %w", err) + } + count, err := result.RowsAffected() + if err != nil || count == 0 { + return false, err + } + return true, tx.Commit() +} diff --git a/internal/database/rekey_artifact_test.go b/internal/database/rekey_artifact_test.go new file mode 100644 index 0000000..d5fd270 --- /dev/null +++ b/internal/database/rekey_artifact_test.go @@ -0,0 +1,69 @@ +package database + +import ( + "database/sql" + "testing" +) + +func TestRekeyCachedArtifact(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + for _, scenario := range []string{"move", "destination exists", "source changed", "local source"} { + t.Run(scenario, func(t *testing.T) { + oldPackage := &Package{PURL: "pkg:golang/example.com/old", Ecosystem: "golang", Name: "example.com/Old"} + oldVersion := &Version{PURL: oldPackage.PURL + "@v1.0.0", PackagePURL: oldPackage.PURL} + old := &Artifact{VersionPURL: oldVersion.PURL, Filename: scenario + ".zip", + UpstreamURL: "https://proxy.example/example.com/!old/@v/v1.0.0.zip", + StoragePath: sql.NullString{String: scenario + "/old.zip", Valid: true}} + if err := db.RecordCachedArtifact(oldPackage, oldVersion, old); err != nil { + t.Fatal(err) + } + old, err := db.GetArtifact(oldVersion.PURL, old.Filename) + if err != nil || old == nil { + t.Fatalf("source artifact = %+v, %v", old, err) + } + pkg := &Package{PURL: "pkg:golang/example.com/Old", Ecosystem: "golang", Name: "example.com/Old"} + version := &Version{PURL: pkg.PURL + "@v1.0.0", PackagePURL: pkg.PURL} + prepareRekeyConflict(t, db, scenario, old, pkg, version) + moved, err := db.RekeyCachedArtifact(old, pkg, version) + if err != nil || moved != (scenario == "move") { + t.Fatalf("RekeyCachedArtifact = %v, %v", moved, err) + } + checkRekeyResult(t, db, scenario, old, version) + }) + } + }) +} + +func checkRekeyResult(t *testing.T, db *DB, scenario string, old *Artifact, version *Version) { + t.Helper() + current, err := db.GetArtifact(version.PURL, old.Filename) + if err != nil { + t.Fatal(err) + } + if scenario == "move" && (current == nil || current.ID != old.ID || current.StoragePath != old.StoragePath) { + t.Errorf("moved artifact = %+v, old = %+v", current, old) + } + if scenario == "destination exists" && (current == nil || current.StoragePath.String != "destination.zip") { + t.Errorf("destination replaced: %+v", current) + } + if paths := allPendingDeletes(t, db); len(paths) != 0 { + t.Errorf("rekey queued stored bytes for deletion: %v", paths) + } +} + +func prepareRekeyConflict(t *testing.T, db *DB, scenario string, old *Artifact, pkg *Package, version *Version) { + t.Helper() + var err error + switch scenario { + case "destination exists": + err = db.RecordCachedArtifact(pkg, version, &Artifact{VersionPURL: version.PURL, Filename: old.Filename, + StoragePath: sql.NullString{String: "destination.zip", Valid: true}}) + case "source changed": + _, err = db.Exec(db.Rebind(`UPDATE artifacts SET storage_path = ? WHERE id = ?`), "replacement.zip", old.ID) + case "local source": + _, err = db.Exec(db.Rebind(`UPDATE artifacts SET source = ? WHERE id = ?`), SourceLocal, old.ID) + } + if err != nil { + t.Fatal(err) + } +} diff --git a/internal/denylist/denylist.go b/internal/denylist/denylist.go index 15bb9e5..76f2de5 100644 --- a/internal/denylist/denylist.go +++ b/internal/denylist/denylist.go @@ -5,6 +5,7 @@ import ( "fmt" "strings" + "github.com/git-pkgs/proxy/internal/packageurl" "github.com/git-pkgs/purl" ) @@ -138,5 +139,5 @@ func identify(pkg *purl.PURL) identity { channel = c.Qualifier("channel") } key := purl.New(c.Type, c.Namespace, c.Name, "", nil).String() - return identity{key: key, channel: channel, version: c.Version} + return identity{key: packageurl.PolicyPackageKey(key), channel: channel, version: c.Version} } diff --git a/internal/enrichment/enrichment.go b/internal/enrichment/enrichment.go index 4d847af..2246e39 100644 --- a/internal/enrichment/enrichment.go +++ b/internal/enrichment/enrichment.go @@ -100,9 +100,6 @@ func (s *Service) BulkEnrichPackages(ctx context.Context, packages []struct{ Eco // CheckVulnerabilities queries for vulnerabilities affecting a package version. func (s *Service) CheckVulnerabilities(ctx context.Context, ecosystem, name, version string) ([]VulnInfo, error) { - if packageurl.Make(ecosystem, name, version) == nil { - return nil, nil - } return s.vulns.Check(ctx, ecosystem, name, version) } @@ -137,7 +134,8 @@ type EnrichmentResult struct { } // EnrichFull fetches package, version and vulnerability data for a package -// version concurrently. Failed lookups are logged and left empty. +// version concurrently. Unsupported vulnerability identities return an error; +// other failed lookups are logged and left empty. func (s *Service) EnrichFull(ctx context.Context, ecosystem, name, version string) (*EnrichmentResult, error) { result := &EnrichmentResult{} @@ -147,6 +145,9 @@ func (s *Service) EnrichFull(ctx context.Context, ecosystem, name, version strin wg.Go(func() { result.Version, verErr = s.EnrichVersion(ctx, ecosystem, name, version) }) wg.Go(func() { result.Vulnerabilities, vulnErr = s.CheckVulnerabilities(ctx, ecosystem, name, version) }) wg.Wait() + if errors.Is(vulnErr, shared.ErrUnsupportedVulnerabilityIdentity) { + return nil, vulnErr + } if pkgErr != nil { s.logger.Debug("failed to enrich package", "ecosystem", ecosystem, "name", name, "error", pkgErr) diff --git a/internal/enrichment/enrichment_test.go b/internal/enrichment/enrichment_test.go index a854e5a..f01bb29 100644 --- a/internal/enrichment/enrichment_test.go +++ b/internal/enrichment/enrichment_test.go @@ -2,10 +2,12 @@ package enrichment import ( "context" + "errors" "log/slog" "os" "testing" + shared "github.com/git-pkgs/enrichment" "github.com/git-pkgs/registries" ) @@ -26,7 +28,7 @@ func TestNew(t *testing.T) { } } -func TestSwiftRegistryIdentitySkipsPURLDependentLookups(t *testing.T) { +func TestSwiftRegistryIdentityLookups(t *testing.T) { svc := New(slog.New(slog.NewTextHandler(os.Stdout, nil))) ctx := context.Background() @@ -41,8 +43,11 @@ func TestSwiftRegistryIdentitySkipsPURLDependentLookups(t *testing.T) { } vulnerabilities, err := svc.CheckVulnerabilities(ctx, "swift", "apple/example", "1.2.3") - if err != nil || vulnerabilities != nil { - t.Errorf("CheckVulnerabilities() = %#v, %v; want nil, nil", vulnerabilities, err) + if !errors.Is(err, shared.ErrUnsupportedVulnerabilityIdentity) || vulnerabilities != nil { + t.Errorf("CheckVulnerabilities() = %#v, %v; want unsupported identity", vulnerabilities, err) + } + if _, err := svc.EnrichFull(ctx, "swift", "apple/example", "1.2.3"); !errors.Is(err, shared.ErrUnsupportedVulnerabilityIdentity) { + t.Errorf("EnrichFull error = %v, want unsupported identity", err) } latest, err := svc.GetLatestVersion(ctx, "swift", "apple/example") diff --git a/internal/handler/cooldown_expiry.go b/internal/handler/cooldown_expiry.go index 1163bde..c31ace8 100644 --- a/internal/handler/cooldown_expiry.go +++ b/internal/handler/cooldown_expiry.go @@ -5,6 +5,7 @@ import ( "time" "github.com/git-pkgs/cooldown" + "github.com/git-pkgs/proxy/internal/packageurl" ) // A rewrite made with cooldown on stays right until a version it held back @@ -127,6 +128,7 @@ type recordingCooldown struct { } func (r recordingCooldown) IsAllowed(ecosystem, packagePURL string, publishedAt time.Time) bool { + packagePURL = packageurl.PolicyPackageKey(packagePURL) if r.p.Cooldown.IsAllowed(ecosystem, packagePURL, publishedAt) { return true } @@ -135,6 +137,7 @@ func (r recordingCooldown) IsAllowed(ecosystem, packagePURL string, publishedAt } func (r recordingCooldown) Evaluate(ecosystem, packagePURL string, publishedAt, evaluatedAt time.Time) cooldown.Decision { + packagePURL = packageurl.PolicyPackageKey(packagePURL) d := r.p.Cooldown.Evaluate(ecosystem, packagePURL, publishedAt, evaluatedAt) if !d.Allowed { r.p.withheld(d.AvailableAt) @@ -143,7 +146,7 @@ func (r recordingCooldown) Evaluate(ecosystem, packagePURL string, publishedAt, } func (r recordingCooldown) For(ecosystem, packagePURL string) time.Duration { - return r.p.Cooldown.For(ecosystem, packagePURL) + return r.p.Cooldown.For(ecosystem, packageurl.PolicyPackageKey(packagePURL)) } func (r recordingCooldown) Enabled() bool { @@ -156,7 +159,7 @@ func (r recordingCooldown) availableAt(ecosystem, packagePURL string, publishedA if publishedAt.IsZero() { return time.Time{} } - d := r.p.Cooldown.For(ecosystem, packagePURL) + d := r.For(ecosystem, packagePURL) if d <= 0 { return time.Time{} } diff --git a/internal/handler/cooldown_fail_closed.go b/internal/handler/cooldown_fail_closed.go index da600c1..0dd2c36 100644 --- a/internal/handler/cooldown_fail_closed.go +++ b/internal/handler/cooldown_fail_closed.go @@ -5,7 +5,7 @@ import "net/http" // cooldownApplies reports whether cooldown withholds new versions of the // package, taking package overrides and exemptions into account. func (p *Proxy) cooldownApplies(ecosystem, name string) bool { - return p.cooldownEnabled() && p.Cooldown.For(ecosystem, canonicalPackagePURL(ecosystem, name)) > 0 + return p.cooldownEnabled() && p.cooldown().For(ecosystem, canonicalPackagePURL(ecosystem, name)) > 0 } // mustFilterMetadata reports whether the package's metadata may have diff --git a/internal/handler/denylist_test.go b/internal/handler/denylist_test.go index bb41e38..663d0c3 100644 --- a/internal/handler/denylist_test.go +++ b/internal/handler/denylist_test.go @@ -288,6 +288,12 @@ func TestDenylistMatchesEcosystemSpellings(t *testing.T) { {"go case-escaped version", "pkg:golang/github.com/example/mod@v1.0.0-RC1", "/github.com/example/mod/@v/v1.0.0-!r!c1.zip", "golang", "github.com/example/mod", "v1.0.0-!r!c1", "mod@v1.0.0-!r!c1.zip", func(p *Proxy) http.Handler { return NewGoHandler(p, "http://proxy.test").Routes() }}, + {"go lowercase rule", "pkg:golang/github.com/example/mod@v1.0.0", "/github.com/!example/mod/@v/v1.0.0.zip", + "golang", "github.com/Example/mod", "v1.0.0", goZipFilename("github.com/Example/mod", "v1.0.0"), + func(p *Proxy) http.Handler { return NewGoHandler(p, "http://proxy.test").Routes() }}, + {"go mixed-case rule", "pkg:golang/github.com/Example/mod@v1.0.0", "/github.com/example/mod/@v/v1.0.0.zip", + "golang", "github.com/example/mod", "v1.0.0", "mod@v1.0.0.zip", + func(p *Proxy) http.Handler { return NewGoHandler(p, "http://proxy.test").Routes() }}, {"oci repository path", "pkg:oci/library/alpine@" + ociTestDigest, "/library/alpine/blobs/" + ociTestDigest, "oci", "library/alpine", ociTestDigest, ociTestDigest, func(p *Proxy) http.Handler { diff --git a/internal/handler/go.go b/internal/handler/go.go index 1af00ed..08dae57 100644 --- a/internal/handler/go.go +++ b/internal/handler/go.go @@ -4,11 +4,15 @@ import ( "crypto/sha256" "encoding/hex" "errors" + "fmt" "net/http" + "net/url" "strings" "github.com/git-pkgs/proxy/internal/database" + "github.com/git-pkgs/proxy/internal/packageurl" "github.com/git-pkgs/proxy/internal/retention" + "github.com/git-pkgs/purl" "github.com/git-pkgs/registries/fetch" gomodule "golang.org/x/mod/module" ) @@ -18,15 +22,13 @@ const ( asciiCaseOffset = 32 // difference between lowercase and uppercase ASCII letters ) -// Module zips are stored under pkg:golang/{module path}, which the PURL -// rules lowercase, so a package rule covers a module whatever case either -// is written in. A major version suffix (/v2) is part of the module path -// and so a module of its own. A key in the escaped form proxy URLs and the -// module cache use (github.com/!azure) does not survive the default -// mapping's round trip and is refused; the handler stores decoded paths, -// so such a key could never match. func init() { - retention.Default.Register(retention.Spec{Key: "golang", CanonicalPackage: retention.DefaultCanonical("golang")}) + retention.Default.Register(retention.Spec{Key: "golang", CanonicalPackage: golangRetentionKey}) +} + +func golangRetentionKey(p *purl.PURL) (string, bool) { + key, ok := retention.DefaultCanonical("golang")(p) + return packageurl.PolicyPackageKey(key), ok } // GoHandler handles Go module proxy protocol requests. @@ -200,13 +202,7 @@ func (h *GoHandler) handleDownload(w http.ResponseWriter, r *http.Request, modul ServeArtifactRequest(w, r, result) } -// goZipFilename is the name a module zip is cached under. Module paths are -// case-sensitive, but the zip is stored under the module's PURL, which -// lowercases golang names, so github.com/Sirupsen/logrus and -// github.com/sirupsen/logrus share one package. A path with uppercase -// letters therefore gets a hash of its exact spelling in the filename. An -// all-lowercase path keeps the plain name, so existing cache entries stay -// valid. +// Keep the module spelling hash used by caches with lowercased PURL keys. func goZipFilename(module, version string) string { base := lastComponent(module) + "@" + version if module == strings.ToLower(module) { @@ -220,6 +216,40 @@ func goZipFilename(module, version string) string { // filename carries: enough to tell apart paths that differ only in case. const goModuleHashBytes = 8 +// An old cache key may combine case-distinct modules. The upstream path +// identifies which module owns an individual archive. +func (p *Proxy) rekeyLegacyGoArtifact(pkgPURL, versionPURL, filename string) (bool, error) { + if !strings.HasPrefix(versionPURL, "pkg:golang/") { + return false, nil + } + parsed, err := purl.Parse(versionPURL) + if err != nil || parsed.FullName() == strings.ToLower(parsed.FullName()) { + return false, nil + } + name := parsed.FullName() + _, legacyVersion, err := packagePURLStrings("golang", strings.ToLower(name), parsed.Version) + if err != nil { + return false, err + } + record, err := p.DB.GetArtifact(legacyVersion, filename) + if err != nil { + return false, fmt.Errorf("reading legacy Go artifact: %w", err) + } + if record == nil || !record.StoragePath.Valid || record.Source != database.SourceUpstream { + return false, nil + } + escaped, err := gomodule.EscapePath(name) + if err != nil { + return false, nil + } + upstream, err := url.Parse(record.UpstreamURL) + if err != nil || !strings.HasSuffix(upstream.Path, "/"+escaped+"/@v/"+parsed.Version+".zip") { + return false, nil + } + return p.DB.RekeyCachedArtifact(record, &database.Package{PURL: pkgPURL, Ecosystem: "golang", Name: name}, + &database.Version{PURL: versionPURL, PackagePURL: pkgPURL}) +} + // discardOtherModulesZip drops a cached zip that another module stored // under filename. Before module paths kept their case, a module with // uppercase letters was cached under the plain filename of its lowercase diff --git a/internal/handler/go_case_test.go b/internal/handler/go_case_test.go index 9451caa..91bac44 100644 --- a/internal/handler/go_case_test.go +++ b/internal/handler/go_case_test.go @@ -9,6 +9,7 @@ import ( "sync/atomic" "testing" + "github.com/git-pkgs/proxy/internal/database" "github.com/git-pkgs/registries/fetch" ) @@ -115,6 +116,74 @@ func TestGoZipFromAnotherUpstreamHostIsKept(t *testing.T) { } } +func TestGoLegacyCacheKeepsCaseDistinctModules(t *testing.T) { + upstream, counts := goCaseUpstream(t) + h, proxy := newGoCaseHandler(t, upstream) + seedGoZip(t, proxy, upstream.URL+lowerLogrusPath, "zip of "+lowerLogrusPath) + old := seedLegacyUpperGoZip(t, proxy, "https://old-proxy.example"+upperLogrusPath, "zip of "+upperLogrusPath) + upstream.Close() + + for range 2 { + for _, path := range []string{upperLogrusPath, lowerLogrusPath} { + if w := getGo(h, path); w.Code != http.StatusOK || w.Body.String() != "zip of "+path { + t.Fatalf("GET %s: status = %d, body = %q", path, w.Code, w.Body) + } + } + } + current, err := proxy.DB.GetArtifact("pkg:golang/github.com/Sirupsen/logrus@v1.0.6", old.Filename) + if err != nil || current == nil { + t.Fatalf("moved artifact = %+v, %v", current, err) + } + if current.ID != old.ID || current.StoragePath != old.StoragePath || current.FetchedAt != old.FetchedAt || current.HitCount != old.HitCount+2 { + t.Errorf("artifact history changed: before=%+v, after=%+v", old, current) + } + legacy, err := proxy.DB.GetArtifact(old.VersionPURL, old.Filename) + if err != nil || legacy != nil { + t.Fatalf("legacy artifact = %+v, %v", legacy, err) + } + for path, count := range counts { + if count.Load() != 0 { + t.Errorf("unexpected upstream request: %s", path) + } + } +} + +func TestGoLegacyCacheRequiresExactModuleOrigin(t *testing.T) { + for _, origin := range []string{"", "https://proxy.example" + lowerLogrusPath} { + t.Run(origin, func(t *testing.T) { + upstream, counts := goCaseUpstream(t) + h, proxy := newGoCaseHandler(t, upstream) + seedLegacyUpperGoZip(t, proxy, origin, "another module") + if w := getGo(h, upperLogrusPath); w.Code != http.StatusOK || w.Body.String() != "zip of "+upperLogrusPath { + t.Fatalf("status = %d, body = %q", w.Code, w.Body) + } + if counts[upperLogrusPath].Load() != 1 { + t.Fatal("archive with unproven identity was reused") + } + }) + } +} + +func seedLegacyUpperGoZip(t *testing.T, proxy *Proxy, upstreamURL, content string) *database.Artifact { + t.Helper() + const packagePURL = "pkg:golang/github.com/sirupsen/logrus" + const versionPURL = packagePURL + "@v1.0.6" + const storagePath = "golang/logrus/v1.0.6/legacy/upper.zip" + filename := goZipFilename("github.com/Sirupsen/logrus", "v1.0.6") + if _, _, err := proxy.Storage.Store(context.Background(), storagePath, strings.NewReader(content)); err != nil { + t.Fatal(err) + } + artifact := testArtifact(content, versionPURL, filename, "application/zip") + if err := proxy.updateCacheDB("golang", "github.com/Sirupsen/logrus", packagePURL, upstreamURL, storagePath, artifact, ""); err != nil { + t.Fatal(err) + } + old, err := proxy.DB.GetArtifact(versionPURL, filename) + if err != nil || old == nil { + t.Fatalf("seeded artifact = %+v, %v", old, err) + } + return old +} + // seedGoZip caches content as github.com/sirupsen/logrus v1.0.6 under the // plain filename, recorded as fetched from upstreamURL. func seedGoZip(t *testing.T, proxy *Proxy, upstreamURL, content string) { diff --git a/internal/handler/go_cooldown.go b/internal/handler/go_cooldown.go index 892e77b..efcc1e1 100644 --- a/internal/handler/go_cooldown.go +++ b/internal/handler/go_cooldown.go @@ -71,12 +71,8 @@ func (h *GoHandler) fetchGoMetadata(ctx context.Context, escapedModule, module, return body, err } -// goTimeKey returns the versions-table PURLs a module's times are stored -// under. golang PURLs lowercase the name, so the key uses the escaped module -// path (github.com/!azure for github.com/Azure), which keeps modules that -// differ only in case apart and is the plain path for lowercase modules. -// Only canonical versions are stored: a query such as a branch name can -// resolve to another commit later. +// Keep escaped module paths in publish-time keys for cache compatibility. +// Only canonical versions are stored; branch queries can resolve differently. func (h *GoHandler) goTimeKey(module, version string) (versionPURL, packagePURL string, ok bool) { if h.proxy.DB == nil || gomodule.CanonicalVersion(version) != version { return "", "", false diff --git a/internal/handler/go_policy_test.go b/internal/handler/go_policy_test.go new file mode 100644 index 0000000..0ce981f --- /dev/null +++ b/internal/handler/go_policy_test.go @@ -0,0 +1,47 @@ +package handler + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/git-pkgs/cooldown" + "github.com/git-pkgs/proxy/internal/config" +) + +func TestGoCooldownOverridesIgnoreModuleCase(t *testing.T) { + for _, tc := range []struct { + name, rule, fallback, override string + status int + }{ + {"lowercase restriction", "example.com/module", "0", "7d", http.StatusForbidden}, + {"mixed-case restriction", "example.com/Module", "0", "7d", http.StatusForbidden}, + {"lowercase exemption", "example.com/module", "7d", "0", http.StatusOK}, + {"mixed-case exemption", "example.com/Module", "7d", "0", http.StatusOK}, + } { + t.Run(tc.name, func(t *testing.T) { + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if strings.HasSuffix(r.URL.Path, ".info") { + _ = json.NewEncoder(w).Encode(goInfo{Version: "v1.0.0", Time: time.Now()}) + return + } + _, _ = w.Write([]byte("module zip")) + })) + defer upstream.Close() + h, proxy := newGoCaseHandler(t, upstream) + cfg := config.CooldownConfig{Packages: map[string]string{"pkg:golang/" + tc.rule: tc.override}} + proxy.Cooldown = &cooldown.Config{Default: tc.fallback, Packages: cfg.NormalizedPackages()} + for _, path := range []string{ + "/example.com/!module/@v/v1.0.0.zip", "/example.com/module/@v/v1.0.0.zip", + "/example.com/!module/@v/v1.0.0.info", "/example.com/module/@v/v1.0.0.info", + } { + if response := getGo(h, path); response.Code != tc.status { + t.Errorf("GET %s = %d %s, want %d", path, response.Code, response.Body, tc.status) + } + } + }) + } +} diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 3951016..4b7ebe7 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -498,6 +498,12 @@ func (p *Proxy) lookupCache(ctx context.Context, pkgPURL, versionPURL, filename if p.StreamArtifacts { return nil, nil } + if cached, err := p.readStored(ctx, pkgPURL, versionPURL, filename); err != nil || cached != nil { + return cached, err + } + if moved, err := p.rekeyLegacyGoArtifact(pkgPURL, versionPURL, filename); err != nil || !moved { + return nil, err + } return p.readStored(ctx, pkgPURL, versionPURL, filename) } diff --git a/internal/packageurl/packageurl.go b/internal/packageurl/packageurl.go index 7110942..e01f4b2 100644 --- a/internal/packageurl/packageurl.go +++ b/internal/packageurl/packageurl.go @@ -18,6 +18,15 @@ func MakeString(ecosystem, name, version string) string { return purl.MakePURLString(ecosystem, name, version) } +// PolicyPackageKey maps a canonical versionless PURL to its policy key. +// Go policies match module names case-insensitively for compatibility. +func PolicyPackageKey(packagePURL string) string { + if strings.HasPrefix(packagePURL, "pkg:golang/") { + return strings.ToLower(packagePURL) + } + return packagePURL +} + // WithVersionString returns a package PURL with its version replaced. It // returns an empty string when packagePURL is invalid. func WithVersionString(packagePURL, version string) string { @@ -29,25 +38,19 @@ func WithVersionString(packagePURL, version string) string { } // MakeCacheStrings returns package and version PURLs suitable for artifact -// cache records. Swift registry identities use an explicit generic PURL until -// their source repository has been resolved. The result is independent of the -// configured upstream so cache entries survive an upstream.swift change, -// matching every other ecosystem. +// cache records. Swift registry identities keep their generic storage PURLs. +// Cache keys are independent of the configured upstream. func MakeCacheStrings(ecosystem, name, version string) (packagePURL, versionPURL string) { + if purl.NormalizeEcosystem(ecosystem) == "swift" { + if identity, ok := swiftRegistryIdentity(name); ok { + pkg := purl.New("generic", "swift-registry", identity, "", nil) + return pkg.String(), pkg.WithVersion(version).String() + } + } if pkg := Make(ecosystem, name, ""); pkg != nil { return pkg.String(), pkg.WithVersion(version).String() } - if purl.NormalizeEcosystem(ecosystem) != "swift" { - return "", "" - } - - identity, ok := swiftRegistryIdentity(name) - if !ok { - return "", "" - } - - pkg := purl.New("generic", "swift-registry", identity, "", nil) - return pkg.String(), pkg.WithVersion(version).String() + return "", "" } func swiftRegistryIdentity(name string) (string, bool) { diff --git a/internal/packageurl/packageurl_test.go b/internal/packageurl/packageurl_test.go index 8982288..5a46776 100644 --- a/internal/packageurl/packageurl_test.go +++ b/internal/packageurl/packageurl_test.go @@ -2,17 +2,12 @@ package packageurl import "testing" -func TestMakeSwiftRegistryIdentityUnsupported(t *testing.T) { - identities := []string{"apple.swift-argument-parser", "apple/swift-argument-parser"} - for _, identity := range identities { - t.Run(identity, func(t *testing.T) { - if got := Make("swift", identity, "1.8.2"); got != nil { - t.Errorf("Make() = %q, want nil", got.String()) - } - if got := MakeString("swift", identity, "1.8.2"); got != "" { - t.Errorf("MakeString() = %q, want empty string", got) - } - }) +func TestMakeSwiftRegistryPathUnsupported(t *testing.T) { + if got := Make("swift", "apple/swift-argument-parser", "1.8.2"); got != nil { + t.Errorf("Make() = %q, want nil", got.String()) + } + if got := MakeString("swift", "apple/swift-argument-parser", "1.8.2"); got != "" { + t.Errorf("MakeString() = %q, want empty string", got) } } @@ -49,9 +44,11 @@ func TestMakeCacheStringsSwiftRegistryIdentity(t *testing.T) { t.Errorf("version PURL = %q, want %q", versionPURL, wantVersion) } - dottedPackage, dottedVersion := MakeCacheStrings("swift", "apple.example", "1.2.3") - if dottedPackage != packagePURL || dottedVersion != versionPURL { - t.Errorf("dotted identity cache PURLs = %q, %q; want %q, %q", dottedPackage, dottedVersion, packagePURL, versionPURL) + for _, identity := range []string{"apple.example", "APPLE.EXAMPLE", "apple/example"} { + dottedPackage, dottedVersion := MakeCacheStrings("Swift", identity, "1.2.3") + if dottedPackage != packagePURL || dottedVersion != versionPURL { + t.Errorf("%q cache PURLs = %q, %q; want %q, %q", identity, dottedPackage, dottedVersion, packagePURL, versionPURL) + } } } diff --git a/internal/retention/retention.go b/internal/retention/retention.go index a8ff835..df8e538 100644 --- a/internal/retention/retention.go +++ b/internal/retention/retention.go @@ -14,6 +14,7 @@ import ( "sync" "time" + "github.com/git-pkgs/proxy/internal/packageurl" "github.com/git-pkgs/purl" ) @@ -41,8 +42,8 @@ type Spec struct { // derived from Key that belong to this ecosystem. ExtraDBEcosystems []string - // CanonicalPackage turns a package PURL from the config into the - // package PURL the handler stores in the cache, reporting false when the + // CanonicalPackage turns a package PURL from the config into a + // policy key, reporting false when the // PURL does not belong to this ecosystem. Nil means the ecosystem accepts // no package overrides. CanonicalPackage func(p *purl.PURL) (string, bool) @@ -150,8 +151,7 @@ func (r *Registry) KeyForDBEcosystem(ecosystem string) (string, bool) { return key, ok } -// Canonical returns the stored package PURL a configured package PURL -// stands for, asking the registered specs in key order. +// Canonical returns the policy key for a configured package PURL. func (r *Registry) Canonical(p *purl.PURL) (string, bool) { for _, key := range r.Keys() { s, _ := r.Lookup(key) @@ -173,7 +173,7 @@ type Rules struct { Default time.Duration // Ecosystems holds rules keyed by registered ecosystem key. Ecosystems map[string]time.Duration - // Packages holds rules keyed by the package PURL stored in the cache. + // Packages holds rules keyed by package PURL, with Go keys lowercased. Packages map[string]time.Duration } @@ -181,6 +181,7 @@ type Rules struct { // stored package PURL: the package rule, else the ecosystem rule, else the // default. Generic artifacts require an explicit package or ecosystem rule. func (r Rules) For(key, packagePURL string) time.Duration { + packagePURL = packageurl.PolicyPackageKey(packagePURL) if d, ok := r.Packages[packagePURL]; ok { return d } diff --git a/internal/server/api.go b/internal/server/api.go index b204367..22bfb8a 100644 --- a/internal/server/api.go +++ b/internal/server/api.go @@ -2,6 +2,7 @@ package server import ( "encoding/json" + "errors" "net/http" "strings" @@ -219,6 +220,10 @@ func (h *APIHandler) getPackage(w http.ResponseWriter, r *http.Request, ecosyste func (h *APIHandler) getVersion(w http.ResponseWriter, r *http.Request, ecosystem, name, version string) { result, err := h.enrichment.EnrichFull(r.Context(), ecosystem, name, version) + if errors.Is(err, shared.ErrUnsupportedVulnerabilityIdentity) { + badRequest(w, "package identity is not supported for vulnerability lookups") + return + } if err != nil { writeError(w, http.StatusBadGateway, ErrCodeUpstream, "failed to enrich version") return @@ -303,6 +308,10 @@ func (h *APIHandler) HandleVulnsPath(w http.ResponseWriter, r *http.Request) { } vulns, err := h.enrichment.CheckVulnerabilities(r.Context(), ecosystem, name, version) + if errors.Is(err, shared.ErrUnsupportedVulnerabilityIdentity) { + badRequest(w, "package identity is not supported for vulnerability lookups") + return + } if err != nil { writeError(w, http.StatusBadGateway, ErrCodeUpstream, "failed to check vulnerabilities") return diff --git a/internal/server/retention_golang_test.go b/internal/server/retention_golang_test.go index f74eaf4..8ff0357 100644 --- a/internal/server/retention_golang_test.go +++ b/internal/server/retention_golang_test.go @@ -188,8 +188,8 @@ func TestGolangRetentionPackageRules(t *testing.T) { if err := db.Get(&stored, db.Rebind(`SELECT package_purl FROM versions WHERE purl = ?`), zip.VersionPURL); err != nil { t.Fatalf("reading stored package PURL: %v", err) } - if stored != "pkg:golang/github.com/burntsushi/toml" { - t.Errorf("stored package PURL = %q, want pkg:golang/github.com/burntsushi/toml", stored) + if stored != "pkg:golang/github.com/BurntSushi/toml" { + t.Errorf("stored package PURL = %q, want pkg:golang/github.com/BurntSushi/toml", stored) } for _, keys := range []map[string]string{ diff --git a/internal/server/swift_vulns_test.go b/internal/server/swift_vulns_test.go new file mode 100644 index 0000000..f28caaf --- /dev/null +++ b/internal/server/swift_vulns_test.go @@ -0,0 +1,57 @@ +package server + +import ( + "encoding/json" + "fmt" + "log/slog" + "net/http" + "net/http/httptest" + "testing" + + "github.com/git-pkgs/proxy/internal/enrichment" + "github.com/go-chi/chi/v5" +) + +func TestSwiftRegistryVulnerabilityErrors(t *testing.T) { + h := NewAPIHandler(enrichment.New(slog.Default()), nil) + r := chi.NewRouter() + r.Get("/api/vulns/{ecosystem}/*", h.HandleVulnsPath) + r.Get("/api/package/{ecosystem}/*", h.HandlePackagePath) + transport := &swiftVulnsTransport{} + original := http.DefaultTransport + http.DefaultTransport = transport + t.Cleanup(func() { http.DefaultTransport = original }) + + for _, path := range []string{ + "/api/vulns/swift/apple.swift-nio", + "/api/vulns/swift/apple.swift-nio/2.0.0", + "/api/vulns/Swift/Apple.Swift-NIO/2.0.0", + "/api/package/swift/apple.swift-nio/2.0.0", + "/api/package/Swift/Apple.Swift-NIO/2.0.0", + } { + t.Run(path, func(t *testing.T) { + w := httptest.NewRecorder() + r.ServeHTTP(w, httptest.NewRequest(http.MethodGet, path, nil)) + if w.Code != http.StatusBadRequest { + t.Fatalf("status = %d, body = %s", w.Code, w.Body) + } + var response ErrorResponse + if err := json.Unmarshal(w.Body.Bytes(), &response); err != nil { + t.Fatal(err) + } + if response.Code != ErrCodeBadRequest || response.Message != "package identity is not supported for vulnerability lookups" { + t.Fatalf("unexpected response: %+v", response) + } + }) + } + if transport.requests != 0 { + t.Errorf("made %d upstream requests for unsupported identities", transport.requests) + } +} + +type swiftVulnsTransport struct{ requests int } + +func (s *swiftVulnsTransport) RoundTrip(r *http.Request) (*http.Response, error) { + s.requests++ + return nil, fmt.Errorf("unexpected upstream request: %s", r.URL) +} From 7a77baf01a58f0bfc280fa63499aac18052c53d2 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sat, 31 Oct 2026 11:33:31 +0100 Subject: [PATCH 518/633] Keep namespace/name for Swift package URLs in the handler mapping With purl v0.2.1, FullName returns a Swift package as its registry identity, scope.name. The mirror splits namespace/name again to find the scope or the source repository, so HandlerPackage keeps that form, as mirrorTarget did before the two mappings were shared. --- internal/packageurl/handler.go | 5 +++++ internal/packageurl/handler_test.go | 7 +++++++ 2 files changed, 12 insertions(+) diff --git a/internal/packageurl/handler.go b/internal/packageurl/handler.go index 6056fcb..ee3005e 100644 --- a/internal/packageurl/handler.go +++ b/internal/packageurl/handler.go @@ -31,6 +31,9 @@ func HandlerEcosystem(purlType string) string { // bare // - pypi and nuget: the name in purl's comparison form, PEP 503 for PyPI // and lowercased for NuGet, as the handlers store them +// - swift: namespace/name, the registry scope or the source repository +// with the package, which the Swift mirror resolves; purl's full name +// is the registry identity scope.name // // Other types use purl's full name, which keeps the namespace: "@babel/core", // or "org.slf4j:slf4j-api" for Maven, as its handler stores it. @@ -43,6 +46,8 @@ func HandlerPackage(p *purl.PURL) (ecosystem, name string) { return ecosystem, condaName(p.Namespace, p.Name, p.Qualifier("channel")) case p.Type == "pypi" || p.Type == "nuget": return ecosystem, p.ComparisonForm().FullName() + case p.Type == "swift" && p.Namespace != "": + return ecosystem, p.Namespace + "/" + p.Name } return ecosystem, p.FullName() } diff --git a/internal/packageurl/handler_test.go b/internal/packageurl/handler_test.go index 5ae9443..7283b9f 100644 --- a/internal/packageurl/handler_test.go +++ b/internal/packageurl/handler_test.go @@ -39,6 +39,13 @@ func TestHandlerPackage(t *testing.T) { {"pkg:maven/org.slf4j/slf4j-api@2.0.13", "maven", "org.slf4j:slf4j-api"}, {"pkg:docker/library/alpine@3.20", "docker", "library/alpine"}, {"pkg:julia/Example@0.5.3?uuid=7876af07-990d-54b4-ab0e-23690620f79a", "julia", "Example"}, + // Swift keeps namespace/name, the scope or the source repository, + // which the Swift mirror splits again; purl's full name would be the + // registry identity "scope.name". + {"pkg:swift/Acme/Widget@1.0.0", "swift", "Acme/Widget"}, + {"pkg:swift/github.com/apple/swift-nio@2.62.0", "swift", "github.com/apple/swift-nio"}, + // Go module paths keep their case. + {"pkg:golang/github.com/Azure/foo@v1.2.0", "golang", "github.com/Azure/foo"}, } for _, tt := range tests { t.Run(tt.purl, func(t *testing.T) { From 1cadc4a7ba68980f8d1c6bd5a78490aa8ad68712 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sat, 31 Oct 2026 11:33:31 +0100 Subject: [PATCH 519/633] Compare Go main modules with their case purl v0.2.1 keeps the case of Go module paths, so the inventory no longer lowercases syft's mainModule. Module paths are case-sensitive: github.com/azure/foo is a dependency of github.com/Azure/foo, not part of it. --- internal/inventory/inventory.go | 8 ++++---- internal/inventory/inventory_test.go | 25 ++++++++++++++++++++++--- 2 files changed, 26 insertions(+), 7 deletions(-) diff --git a/internal/inventory/inventory.go b/internal/inventory/inventory.go index 9a13a8b..d115fb7 100644 --- a/internal/inventory/inventory.go +++ b/internal/inventory/inventory.go @@ -99,7 +99,7 @@ func Parse(data []byte) (components []Component, skipped int, err error) { seen := make(map[[2]string]bool, len(doc.Packages)) for _, pkg := range doc.Packages { c, ok := componentOf(pkg) - if !ok || lockRoots[pkg.ID] || (c.Ecosystem == "golang" && ownModules[strings.ToLower(c.Name)]) { + if !ok || lockRoots[pkg.ID] || (c.Ecosystem == "golang" && ownModules[c.Name]) { skipped++ continue } @@ -161,14 +161,14 @@ func lockFileRoots(doc *sbom.SBOM) map[string]bool { } // mainModules returns the Go modules the SBOM's binaries were built from, -// which are the project's own, lowercased: syft keeps the module path's case -// while FromPURL lowercases golang names. +// which are the project's own. Module paths are case-sensitive and package +// URLs keep their case, so they are compared as they are. func mainModules(pkgs []sbom.Package) map[string]bool { own := map[string]bool{} for _, pkg := range pkgs { for _, prop := range pkg.Properties { if prop.Name == syftMainModule && prop.Value != "" { - own[strings.ToLower(prop.Value)] = true + own[prop.Value] = true } } } diff --git a/internal/inventory/inventory_test.go b/internal/inventory/inventory_test.go index 818129a..3fbe113 100644 --- a/internal/inventory/inventory_test.go +++ b/internal/inventory/inventory_test.go @@ -246,9 +246,9 @@ func TestParseSkipsTheProjectsOwnPackages(t *testing.T) { } } -func TestParseSkipsMainModuleRegardlessOfCase(t *testing.T) { - // syft keeps the module path's case in mainModule; package URLs of Go - // modules are lowercased. +func TestParseSkipsMixedCaseMainModule(t *testing.T) { + // syft keeps the module path's case in mainModule, and so does the + // package URL. doc := `{"bomFormat":"CycloneDX","specVersion":"1.6","components":[ {"bom-ref":"bin","type":"library","name":"github.com/Azure/foo","version":"v1.2.0","purl":"pkg:golang/github.com/Azure/foo@v1.2.0", "properties":[{"name":"syft:metadata:mainModule","value":"github.com/Azure/foo"}]}, @@ -263,3 +263,22 @@ func TestParseSkipsMainModuleRegardlessOfCase(t *testing.T) { t.Errorf("components = %+v, skipped = %d; want only cobra, 1 skipped", components, skipped) } } + +func TestParseMainModuleKeepsCase(t *testing.T) { + // Go module paths are case-sensitive, and package URLs keep their case, + // so github.com/azure/foo is another module than the main module + // github.com/Azure/foo, and a dependency. + doc := `{"bomFormat":"CycloneDX","specVersion":"1.6","components":[ + {"bom-ref":"bin","type":"library","name":"github.com/Azure/foo","version":"v1.2.0","purl":"pkg:golang/github.com/Azure/foo@v1.2.0", + "properties":[{"name":"syft:metadata:mainModule","value":"github.com/Azure/foo"}]}, + {"bom-ref":"other","type":"library","name":"github.com/azure/foo","version":"v0.1.0","purl":"pkg:golang/github.com/azure/foo@v0.1.0", + "properties":[{"name":"syft:metadata:mainModule","value":"github.com/Azure/foo"}]} + ]}` + components, skipped, err := Parse([]byte(doc)) + if err != nil { + t.Fatal(err) + } + if len(components) != 1 || components[0].Name != "github.com/azure/foo" || skipped != 1 { + t.Errorf("components = %+v, skipped = %d; want only github.com/azure/foo, 1 skipped", components, skipped) + } +} From 1979fb098edd2ed720b143b0a34b3faf2c056372 Mon Sep 17 00:00:00 2001 From: simon chrzanowski Date: Sat, 31 Oct 2026 12:34:05 +0100 Subject: [PATCH 520/633] Accept Swift source repository PURLs in the denylist and retention rules (#641) * Accept Swift source repository PURLs in the denylist and retention rules Closes #613. pkg:swift/github.com/apple/swift-nio, as SBOM tools and GHSA write Swift packages, is resolved to its registry identity through the upstream's /identifiers: - Denylist entries are resolved at startup, and the proxy refuses to start when one can't be, since a skipped entry would let the version through. - Retention keys are accepted at validation and kept pending. The retention loop resolves them before each sweep, and Swift isn't swept while any is pending, so a rule that keeps a package is never skipped. The mirror's source URL helper moves to the handler as SwiftSourceURL. * Resolve Swift source denylist entries in proxy mirror too serve resolved source-form Swift denylist entries through the upstream, but proxy mirror built its denylist directly, so such an entry never matched a mirror download. Both now use server.ConfigureDenylist, and neither runs when an entry can't be resolved. * Apply Swift source policies to every listed identity A Swift registry may list several identities for one source repository, but a denylist entry or retention rule naming the repository only covered the first. Denying pkg:swift/github.com/acme/widget@1.0.0 let acme.widget through when the registry listed other.widget first, and a never-evict rule cleared the cached apple.swift-nio archive when other.swift-nio came first. Both now cover every identity the registry lists. A retention rule that conflicts with another rule for any of them stays pending, with none applied. Mirroring a source PURL still downloads the first identity. --- cmd/proxy/main.go | 14 +- cmd/proxy/mirror_swift_test.go | 186 ++++++++++++++++++++++++ docs/configuration.md | 9 +- internal/config/retention.go | 37 +++-- internal/handler/swift.go | 50 ++++++- internal/handler/swift_mirror.go | 54 +++++-- internal/mirror/swift.go | 12 +- internal/mirror/versions.go | 2 +- internal/retention/retention.go | 57 ++++++++ internal/retention/retention_test.go | 50 +++++++ internal/server/retention.go | 13 +- internal/server/retention_swift_test.go | 183 +++++++++++++++++++++-- internal/server/server.go | 12 +- internal/server/swift_sources.go | 136 +++++++++++++++++ 14 files changed, 745 insertions(+), 70 deletions(-) create mode 100644 cmd/proxy/mirror_swift_test.go create mode 100644 internal/server/swift_sources.go diff --git a/cmd/proxy/main.go b/cmd/proxy/main.go index 824f2e2..258fba3 100644 --- a/cmd/proxy/main.go +++ b/cmd/proxy/main.go @@ -124,7 +124,6 @@ import ( "github.com/git-pkgs/proxy/internal/config" "github.com/git-pkgs/proxy/internal/database" - "github.com/git-pkgs/proxy/internal/denylist" "github.com/git-pkgs/proxy/internal/handler" "github.com/git-pkgs/proxy/internal/memlimit" "github.com/git-pkgs/proxy/internal/mirror" @@ -552,15 +551,16 @@ func runMirror() { // Build proxy with the same upstream clients and authentication as serve proxy, images := server.NewMirrorProxy(cfg, db, store, logger) - proxy.Denylist, err = denylist.New(cfg.Denylist.Packages) - if err != nil { - _ = db.Close() - fmt.Fprintf(os.Stderr, "invalid denylist: %v\n", err) - os.Exit(1) //nolint:gocritic // db closed above - } proxy.CacheMetadata = true // mirror always caches metadata proxy.MetadataTTL = cfg.ParseMetadataTTL() proxy.MetadataMaxSize = cfg.ParseMetadataMaxSize() + // As serve does, resolve Swift source repositories in the denylist through + // the upstream, and refuse to run when one can't be. + if err := server.ConfigureDenylist(cfg, proxy); err != nil { + _ = db.Close() + fmt.Fprintf(os.Stderr, "error: %v\n", err) + os.Exit(1) //nolint:gocritic // db closed above + } m := newCommandMirror(cfg, proxy, images, db, store, logger, *concurrency) switch s := source.(type) { diff --git a/cmd/proxy/mirror_swift_test.go b/cmd/proxy/mirror_swift_test.go new file mode 100644 index 0000000..28fa8ba --- /dev/null +++ b/cmd/proxy/mirror_swift_test.go @@ -0,0 +1,186 @@ +package main + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync/atomic" + "testing" + + "github.com/git-pkgs/proxy/internal/database" +) + +const swiftWidgetArchive = "acme widget archive" + +// swiftMirrorUpstream serves acme.widget 1.0.0 the way a Swift package +// registry does, with https://github.com/acme/widget as its source +// repository, and counts archive downloads and identifier lookups. The +// repository's identifiers are ids, or only acme.widget. +func swiftMirrorUpstream(t *testing.T, ids ...string) (srv *httptest.Server, downloads, lookups *atomic.Int64) { + t.Helper() + if len(ids) == 0 { + ids = []string{"acme.widget"} + } + identifiers, err := json.Marshal(map[string][]string{"identifiers": ids}) + if err != nil { + t.Fatal(err) + } + downloads, lookups = new(atomic.Int64), new(atomic.Int64) + sum := sha256.Sum256([]byte(swiftWidgetArchive)) + srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Version", "1") + switch r.URL.Path { + case "/identifiers": + lookups.Add(1) + if r.URL.Query().Get("url") != "https://github.com/acme/widget" { + http.NotFound(w, r) + return + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write(identifiers) + case "/acme/widget": + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, `{"releases":{"1.0.0":{}}}`) + case "/acme/widget/1.0.0/Package.swift": + w.Header().Set("Content-Type", "text/x-swift") + _, _ = io.WriteString(w, "// swift-tools-version:5.9\n") + case "/acme/widget/1.0.0": + w.Header().Set("Content-Type", "application/json") + _, _ = fmt.Fprintf(w, `{"id":"acme.widget","version":"1.0.0","resources":[{"name":"source-archive","type":"application/zip","checksum":%q}]}`, hex.EncodeToString(sum[:])) + case "/acme/widget/1.0.0.zip": + downloads.Add(1) + w.Header().Set("Content-Type", "application/zip") + _, _ = io.WriteString(w, swiftWidgetArchive) + default: + http.NotFound(w, r) + } + })) + t.Cleanup(srv.Close) + return srv, downloads, lookups +} + +// writeSwiftMirrorConfig writes a config using upstream as the Swift +// registry, denying the denied package PURLs. +func writeSwiftMirrorConfig(t *testing.T, dir, upstream string, denied ...string) string { + t.Helper() + cfg := "storage:\n" + + " url: file://" + filepath.Join(dir, "artifacts") + "\n" + + "database:\n" + + " path: " + filepath.Join(dir, "proxy.db") + "\n" + + "upstream:\n" + + " swift: " + upstream + "\n" + + " allow_loopback: true\n" + if len(denied) > 0 { + cfg += "denylist:\n packages:\n" + for _, p := range denied { + cfg += " - " + p + "\n" + } + } + path := filepath.Join(dir, "config.yaml") + if err := os.WriteFile(path, []byte(cfg), 0o600); err != nil { + t.Fatalf("writing config: %v", err) + } + return path +} + +func cachedArtifacts(t *testing.T, dir string) int64 { + t.Helper() + db, err := database.Open(filepath.Join(dir, "proxy.db")) + if err != nil { + t.Fatalf("opening database: %v", err) + } + defer func() { _ = db.Close() }() + stats, err := db.GetCacheStats() + if err != nil { + t.Fatalf("GetCacheStats: %v", err) + } + return stats.TotalArtifacts +} + +// Without a denylist the fake registry mirrors the release, so the tests +// below fail for the denylist and not for the setup. +func TestMirrorSwiftRelease(t *testing.T) { + upstream, downloads, _ := swiftMirrorUpstream(t) + dir := t.TempDir() + configPath := writeSwiftMirrorConfig(t, dir, upstream.URL) + + output := runMirrorProcess(t, 0, "-config", configPath, "pkg:swift/acme/widget@1.0.0") + + if !strings.Contains(output, "1 downloaded") { + t.Errorf("output:\n%s", output) + } + if n := downloads.Load(); n != 1 { + t.Errorf("archive downloads = %d, want 1", n) + } + if n := cachedArtifacts(t, dir); n != 1 { + t.Errorf("cached artifacts = %d, want 1", n) + } +} + +// A denylist entry naming the package by its source repository blocks the +// mirror download of the registry identity it resolves to. +func TestMirrorDenylistSwiftSourceEntry(t *testing.T) { + upstream, downloads, _ := swiftMirrorUpstream(t) + dir := t.TempDir() + configPath := writeSwiftMirrorConfig(t, dir, upstream.URL, "pkg:swift/github.com/acme/widget@1.0.0") + + output := runMirrorProcess(t, 1, "-config", configPath, "pkg:swift/acme/widget@1.0.0") + + if !strings.Contains(output, "0 downloaded") || !strings.Contains(output, "1 failed") { + t.Errorf("output:\n%s", output) + } + if n := downloads.Load(); n != 0 { + t.Errorf("archive downloads = %d, want 0", n) + } + if n := cachedArtifacts(t, dir); n != 0 { + t.Errorf("cached artifacts = %d, want 0", n) + } +} + +// The registry may list several identities for one repository. A source +// entry denies all of them, not only the first. +func TestMirrorDenylistSwiftSourceEntryEveryIdentity(t *testing.T) { + upstream, downloads, _ := swiftMirrorUpstream(t, "other.widget", "acme.widget") + dir := t.TempDir() + configPath := writeSwiftMirrorConfig(t, dir, upstream.URL, "pkg:swift/github.com/acme/widget@1.0.0") + + output := runMirrorProcess(t, 1, "-config", configPath, "pkg:swift/acme/widget@1.0.0") + + if !strings.Contains(output, "0 downloaded") || !strings.Contains(output, "1 failed") { + t.Errorf("output:\n%s", output) + } + if n := downloads.Load(); n != 0 { + t.Errorf("archive downloads = %d, want 0", n) + } +} + +// A source repository entry the registry can't resolve stops the mirror +// before it downloads anything, as it stops serve: left out, it would let +// the version through. +func TestMirrorDenylistUnresolvedSwiftSource(t *testing.T) { + upstream, downloads, lookups := swiftMirrorUpstream(t) + dir := t.TempDir() + configPath := writeSwiftMirrorConfig(t, dir, upstream.URL, "pkg:swift/github.com/acme/unknown@1.0.0") + + output := runMirrorProcess(t, 1, "-config", configPath, "pkg:swift/acme/widget@1.0.0") + + if !strings.Contains(output, "configuring denylist") || !strings.Contains(output, "github.com/acme/unknown") { + t.Errorf("output does not report the unresolved entry:\n%s", output) + } + if strings.Contains(output, "Mirror complete") { + t.Errorf("mirror ran:\n%s", output) + } + if lookups.Load() == 0 { + t.Error("the entry was not looked up") + } + if n := downloads.Load(); n != 0 { + t.Errorf("archive downloads = %d, want 0", n) + } +} diff --git a/docs/configuration.md b/docs/configuration.md index b20685f..3e94531 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -150,7 +150,7 @@ The package rule covers every release and asset in that repository, but each cac | `pub` | yes | `pkg:pub/http`, the package name as published. | | `pypi` | yes | `pkg:pypi/requests`; case and single `_`/`-` don't matter. Dots and repeated separators are kept as written, so a project like `zope.interface` can be stored as `zope.interface` and `zope-interface`; add a rule for each. | | `rpm` | yes | `pkg:rpm/curl`, the name as packaged, case included (`pkg:rpm/NetworkManager`). A vendor namespace such as `pkg:rpm/fedora/curl` is accepted and dropped. Fedora and EPEL update repositories keep only the latest build of a package, so an evicted older one may not be fetchable again. | -| `swift` | yes | `pkg:swift/apple/swift-nio`, with the registry scope as the namespace, or the stored form `pkg:generic/swift-registry/apple.swift-nio`; case doesn't matter. A source repository key like `pkg:swift/github.com/apple/swift-nio` is refused, since it doesn't name a registry identity. | +| `swift` | yes | `pkg:swift/apple/swift-nio`, with the registry scope as the namespace, or the stored form `pkg:generic/swift-registry/apple.swift-nio`; case doesn't matter. A source repository key like `pkg:swift/github.com/apple/swift-nio` is looked up through `upstream.swift`'s `/identifiers` before each sweep and applies to every identity listed for it; until every such key resolves, Swift artifacts aren't swept, so a rule that keeps a package is never skipped. | An evicted artifact is fetched again from upstream on the next request. Packages preloaded with `proxy mirror` age from the time they were mirrored, so an unused mirrored package is evicted like any other. Running `proxy mirror` again over a package that is still cached counts as a download and restarts its clock, so packages a scheduled mirror job keeps listing never age out. A registry that removes old releases (Linux distribution archives in particular) cannot serve an evicted version again, so set those ecosystems' rules with that in mind. @@ -1185,7 +1185,12 @@ Names and versions are compared the way each ecosystem treats them: Go module names in denylist entries are matched case-insensitively. - A Swift entry names the registry scope and package, ignoring case: `pkg:swift/apple/swift-nio@2.62.0`. The stored form, - `pkg:generic/swift-registry/apple.swift-nio@2.62.0`, works too. + `pkg:generic/swift-registry/apple.swift-nio@2.62.0`, works too, and so does + the source repository, as SBOM tools and GHSA advisories write it: + `pkg:swift/github.com/apple/swift-nio@2.62.0`. The proxy looks a source + repository up through `upstream.swift`'s `/identifiers` when `serve` or + `proxy mirror` starts and denies every identity listed for it; neither runs + when one can't be resolved. - An OCI entry names the repository as it is pulled through the proxy and a digest or tag: `pkg:oci/library/alpine@sha256:` refuses the blob or manifest with that digest, including a manifest a tag resolves to, and diff --git a/internal/config/retention.go b/internal/config/retention.go index 3467166..9493d28 100644 --- a/internal/config/retention.go +++ b/internal/config/retention.go @@ -48,6 +48,7 @@ func (c *Config) RetentionRules(reg *retention.Registry) (retention.Rules, error rules := retention.Rules{ Ecosystems: map[string]time.Duration{}, Packages: map[string]time.Duration{}, + Sources: map[string]map[string]time.Duration{}, } var err error @@ -84,7 +85,7 @@ func (c *Config) RetentionRules(reg *retention.Registry) (retention.Rules, error for _, key := range sortedKeys(r.Packages) { field := fmt.Sprintf("storage.retention.packages[%q]", key) - canonical, err := retentionPackageKey(reg, key) + canonical, sourceOf, err := retentionPackageKey(reg, key) if err != nil { return retention.Rules{}, fmt.Errorf("invalid %s: %w", field, err) } @@ -92,6 +93,13 @@ func (c *Config) RetentionRules(reg *retention.Registry) (retention.Rules, error if err != nil { return retention.Rules{}, err } + if sourceOf != "" { + if rules.Sources[sourceOf] == nil { + rules.Sources[sourceOf] = map[string]time.Duration{} + } + rules.Sources[sourceOf][key] = d + continue + } if prev, ok := rules.Packages[canonical]; ok && prev != d { return retention.Rules{}, fmt.Errorf("invalid %s: conflicts with another key for package %s", field, canonical) } @@ -102,38 +110,41 @@ func (c *Config) RetentionRules(reg *retention.Registry) (retention.Rules, error } // retentionPackageKey returns the stored package PURL a configured package -// key stands for. -func retentionPackageKey(reg *retention.Registry, key string) (string, error) { +// key stands for, or, for a key that has to be resolved through the upstream +// first, the ecosystem key it belongs to. +func retentionPackageKey(reg *retention.Registry, key string) (canonical, sourceOf string, _ error) { // A julia PURL needs a uuid qualifier to parse at all, and the julia // handler stores none, so no key could ever match one of its packages. if strings.HasPrefix(strings.ToLower(key), "pkg:julia/") { - return "", fmt.Errorf("package rules are not supported for julia") + return "", "", fmt.Errorf("package rules are not supported for julia") } p, err := purl.Parse(key) if err != nil { - return "", fmt.Errorf("not a valid package PURL: %w", err) + return "", "", fmt.Errorf("not a valid package PURL: %w", err) } if p.Version != "" { - return "", fmt.Errorf("package PURL must not carry a version") + return "", "", fmt.Errorf("package PURL must not carry a version") } // A rule names a whole package; qualifiers and subpaths would be // dropped on the way to the stored PURL, so refuse them rather than // let "?type=pom" quietly cover every file of the package. if len(p.Qualifiers) > 0 || p.Subpath != "" { - return "", fmt.Errorf("package PURL must not carry qualifiers or a subpath") + return "", "", fmt.Errorf("package PURL must not carry qualifiers or a subpath") } - canonical, ok := reg.Canonical(p) - if ok { - return canonical, nil + if canonical, ok := reg.Canonical(p); ok { + return canonical, "", nil + } + if ecosystem, ok := reg.Source(p); ok { + return "", ecosystem, nil } ecosystem, registered := reg.KeyForDBEcosystem(purl.PURLTypeToEcosystem(p.Type)) if !registered { - return "", fmt.Errorf("retention is not supported for %s packages in this version", p.Type) + return "", "", fmt.Errorf("retention is not supported for %s packages in this version", p.Type) } if spec, _ := reg.Lookup(ecosystem); spec.CanonicalPackage == nil { - return "", fmt.Errorf("package rules are not supported for %s; use storage.retention.ecosystems.%s", ecosystem, ecosystem) + return "", "", fmt.Errorf("package rules are not supported for %s; use storage.retention.ecosystems.%s", ecosystem, ecosystem) } - return "", fmt.Errorf("does not match how %s packages are stored; see the package rule forms in docs/configuration.md", ecosystem) + return "", "", fmt.Errorf("does not match how %s packages are stored; see the package rule forms in docs/configuration.md", ecosystem) } func sortedKeys(m map[string]string) []string { diff --git a/internal/handler/swift.go b/internal/handler/swift.go index 7de0cd1..37f666e 100644 --- a/internal/handler/swift.go +++ b/internal/handler/swift.go @@ -31,10 +31,54 @@ const ( // Source archives are stored under the registry identity, // pkg:generic/swift-registry/{scope}.{name}, lowercased. Package rules write // the scope as the namespace, pkg:swift/apple/swift-nio, or use the stored -// form. A source repository URL such as pkg:swift/github.com/apple/swift-nio -// doesn't name a registry identity and is refused. +// form. A source repository such as pkg:swift/github.com/apple/swift-nio +// names no registry identity; it is resolved through the upstream's +// /identifiers before it applies. func init() { - retention.Default.Register(retention.Spec{Key: "swift", CanonicalPackage: swiftRetentionKey}) + retention.Default.Register(retention.Spec{ + Key: "swift", + CanonicalPackage: swiftRetentionKey, + SourcePackage: func(p *purl.PURL) bool { return SwiftSourcePURL(p) != "" }, + }) +} + +// SwiftSourceURL returns the repository URL of a pkg:swift name whose +// namespace starts with a host, as in github.com/apple/swift-nio, or "" for +// a name whose namespace is a registry scope. +func SwiftSourceURL(name string) string { + if host, _, ok := strings.Cut(name, "/"); ok && strings.Contains(host, ".") { + return "https://" + name + } + return "" +} + +// SwiftSourcePURL returns the repository URL of a pkg:swift PURL that names +// a package by its source repository, as SBOM tools and GHSA do, or "". +func SwiftSourcePURL(p *purl.PURL) string { + if p == nil || p.Type != "swift" || p.Namespace == "" { + return "" + } + return SwiftSourceURL(p.Namespace + "/" + p.Name) +} + +// ResolveSourcePURL returns p with a source repository namespace replaced by +// each registry scope and name the upstream lists for it, and p itself when +// it already names a registry identity. A policy for the repository applies +// to all of them, since a client may use any. +func (h *SwiftHandler) ResolveSourcePURL(ctx context.Context, p *purl.PURL) ([]*purl.PURL, error) { + source := SwiftSourcePURL(p) + if source == "" { + return []*purl.PURL{p}, nil + } + ids, err := h.ResolveIdentities(ctx, source) + if err != nil { + return nil, err + } + resolved := make([]*purl.PURL, 0, len(ids)) + for _, id := range ids { + resolved = append(resolved, purl.New("swift", id.Scope, id.Name, p.Version, nil)) + } + return resolved, nil } // swiftRetentionKey maps a package rule to the stored package PURL, refusing diff --git a/internal/handler/swift_mirror.go b/internal/handler/swift_mirror.go index d3944e3..5326c1d 100644 --- a/internal/handler/swift_mirror.go +++ b/internal/handler/swift_mirror.go @@ -121,37 +121,67 @@ func swiftLinkHasRel(params, rel string) bool { return false } -// ResolveIdentity returns the registry scope and name the upstream lists for -// the source repository at sourceURL. The answer is cached as the identifiers -// route caches it. +// SwiftIdentity is a package's identity in a Swift registry, scope.name. +type SwiftIdentity struct { + Scope, Name string +} + +// ResolveIdentity returns the first registry identity the upstream lists for +// the source repository at sourceURL, the one a download uses. func (h *SwiftHandler) ResolveIdentity(ctx context.Context, sourceURL string) (scope, name string, _ error) { + ids, err := h.ResolveIdentities(ctx, sourceURL) + if err != nil { + return "", "", err + } + return ids[0].Scope, ids[0].Name, nil +} + +// ResolveIdentities returns every registry identity the upstream lists for +// the source repository at sourceURL; the registry protocol allows several. +// The answer is cached as the identifiers route caches it. +func (h *SwiftHandler) ResolveIdentities(ctx context.Context, sourceURL string) ([]SwiftIdentity, error) { query := url.Values{"url": {sourceURL}}.Encode() body, _, err := h.proxy.FetchOrCacheMetadata(ctx, "swift", swiftMetadataCacheKey("identifiers", query), h.upstreamURL+"/identifiers?"+query, swiftAcceptJSON) if errors.Is(err, ErrUpstreamNotFound) { - return "", "", fmt.Errorf("the Swift registry has no package for %s", sourceURL) + return nil, fmt.Errorf("the Swift registry has no package for %s", sourceURL) } if err != nil { - return "", "", err + return nil, err } - return SwiftIdentityOf(body, sourceURL) + return SwiftIdentitiesOf(body, sourceURL) } // SwiftIdentityOf returns the scope and name of the first identity in body, // a registry's answer to an identifiers lookup for sourceURL. func SwiftIdentityOf(body []byte, sourceURL string) (scope, name string, _ error) { + ids, err := SwiftIdentitiesOf(body, sourceURL) + if err != nil { + return "", "", err + } + return ids[0].Scope, ids[0].Name, nil +} + +// SwiftIdentitiesOf returns every identity in body, a registry's answer to +// an identifiers lookup for sourceURL. One invalid identity fails the whole +// answer, so a policy never covers only some of them. +func SwiftIdentitiesOf(body []byte, sourceURL string) ([]SwiftIdentity, error) { var answer struct { Identifiers []string `json:"identifiers"` } if err := json.Unmarshal(body, &answer); err != nil { - return "", "", fmt.Errorf("parsing the Swift registry's identifiers for %s: %w", sourceURL, err) + return nil, fmt.Errorf("parsing the Swift registry's identifiers for %s: %w", sourceURL, err) } if len(answer.Identifiers) == 0 { - return "", "", fmt.Errorf("the Swift registry has no package for %s", sourceURL) + return nil, fmt.Errorf("the Swift registry has no package for %s", sourceURL) } - scope, name, ok := strings.Cut(answer.Identifiers[0], ".") - if !ok || scope == "" || name == "" { - return "", "", fmt.Errorf("the Swift registry answered %s with the invalid identity %q", sourceURL, answer.Identifiers[0]) + ids := make([]SwiftIdentity, 0, len(answer.Identifiers)) + for _, identifier := range answer.Identifiers { + scope, name, ok := strings.Cut(identifier, ".") + if !ok || scope == "" || name == "" { + return nil, fmt.Errorf("the Swift registry answered %s with the invalid identity %q", sourceURL, identifier) + } + ids = append(ids, SwiftIdentity{Scope: scope, Name: name}) } - return scope, name, nil + return ids, nil } diff --git a/internal/mirror/swift.go b/internal/mirror/swift.go index 4616c8f..153bb10 100644 --- a/internal/mirror/swift.go +++ b/internal/mirror/swift.go @@ -19,7 +19,7 @@ func (m *Mirror) mirrorSwift(ctx context.Context, pv PackageVersion, tracker *pr } h := handler.NewSwiftHandler(m.proxy, "", upstream) var scope, name string - if source := swiftSourceURL(pv.Name); source != "" { + if source := handler.SwiftSourceURL(pv.Name); source != "" { scope, name, err = h.ResolveIdentity(ctx, source) } else { scope, name, err = swiftScopeAndName(pv.Name) @@ -45,16 +45,6 @@ func (m *Mirror) registryURL(purlType, name string) (string, error) { return "", nil } -// swiftSourceURL returns the repository URL of a pkg:swift name whose -// namespace starts with a host, as in pkg:swift/github.com/apple/swift-nio, -// or "" for a name whose namespace is a registry scope. -func swiftSourceURL(name string) string { - if host, _, ok := strings.Cut(name, "/"); ok && strings.Contains(host, ".") { - return "https://" + name - } - return "" -} - // swiftScopeAndName splits the name of a pkg:swift PURL whose namespace is // a registry scope, as in pkg:swift/apple/swift-nio for the identity // apple.swift-nio. diff --git a/internal/mirror/versions.go b/internal/mirror/versions.go index 04353b4..0a7e1ba 100644 --- a/internal/mirror/versions.go +++ b/internal/mirror/versions.go @@ -145,7 +145,7 @@ func (r RegistryVersions) client() *registries.Client { // swiftIdentity returns the registry identity, scope.name, of the name of a // pkg:swift PURL, looking a source repository up on the registry at baseURL. func swiftIdentity(ctx context.Context, client *registries.Client, baseURL, name string) (string, error) { - source := swiftSourceURL(name) + source := handler.SwiftSourceURL(name) if source == "" { scope, pkg, err := swiftScopeAndName(name) if err != nil { diff --git a/internal/retention/retention.go b/internal/retention/retention.go index df8e538..156e410 100644 --- a/internal/retention/retention.go +++ b/internal/retention/retention.go @@ -47,6 +47,13 @@ type Spec struct { // PURL does not belong to this ecosystem. Nil means the ecosystem accepts // no package overrides. CanonicalPackage func(p *purl.PURL) (string, bool) + + // SourcePackage reports a package PURL that names the package by where + // its source lives, such as a Swift package by its repository. It has to + // be resolved through the upstream before it can be compared, so the + // config keeps it in Rules.Sources until then. Nil means the ecosystem + // has no such form. + SourcePackage func(p *purl.PURL) bool } // dbEcosystems returns every packages.ecosystem value that belongs to the @@ -165,6 +172,17 @@ func (r *Registry) Canonical(p *purl.PURL) (string, bool) { return "", false } +// Source returns the ecosystem key whose SourcePackage claims p. +func (r *Registry) Source(p *purl.PURL) (string, bool) { + for _, key := range r.Keys() { + s, _ := r.Lookup(key) + if s.SourcePackage != nil && s.SourcePackage(p) { + return key, true + } + } + return "", false +} + // Rules are resolved retention durations. A duration of zero means an // artifact is never evicted by age. type Rules struct { @@ -175,6 +193,40 @@ type Rules struct { Ecosystems map[string]time.Duration // Packages holds rules keyed by package PURL, with Go keys lowercased. Packages map[string]time.Duration + // Sources holds package rules not resolved to a stored package PURL + // yet, by ecosystem key and then by the configured PURL (see + // Spec.SourcePackage). An ecosystem with any is not swept, since one of + // them may keep an artifact the sweep would otherwise clear. + Sources map[string]map[string]time.Duration +} + +// Pending reports whether the ecosystem key has package rules still to be +// resolved, and must not be swept. +func (r Rules) Pending(key string) bool { + return len(r.Sources[key]) > 0 +} + +// Resolve moves the rule for source in ecosystem key to every stored package +// PURL it resolved to. A different rule already set for any of them is a +// conflict, and the source stays pending with none of them set. +func (r Rules) Resolve(key, source string, canonical []string) error { + d, ok := r.Sources[key][source] + if !ok { + return nil + } + if len(canonical) == 0 { + return fmt.Errorf("%s resolves to no stored package", source) + } + for _, c := range canonical { + if prev, set := r.Packages[c]; set && prev != d { + return fmt.Errorf("%s resolves to %s, which another rule sets differently", source, c) + } + } + for _, c := range canonical { + r.Packages[c] = d + } + delete(r.Sources[key], source) + return nil } // For returns the retention for an artifact of the given ecosystem key and @@ -211,5 +263,10 @@ func (r Rules) MinPositive() time.Duration { for _, d := range r.Packages { consider(d) } + for _, sources := range r.Sources { + for _, d := range sources { + consider(d) + } + } return minimum } diff --git a/internal/retention/retention_test.go b/internal/retention/retention_test.go index 171fd70..5036375 100644 --- a/internal/retention/retention_test.go +++ b/internal/retention/retention_test.go @@ -192,3 +192,53 @@ func TestRulesMinPositive(t *testing.T) { t.Errorf("MinPositive() = %v, want 2h", got) } } + +func TestRulesResolveSources(t *testing.T) { + rules := Rules{ + Packages: map[string]time.Duration{"pkg:x/kept": 0}, + Sources: map[string]map[string]time.Duration{"swift": { + "pkg:swift/github.com/a/one": 0, + "pkg:swift/github.com/a/two": time.Hour, + }}, + } + if got := rules.MinPositive(); got != time.Hour { + t.Errorf("MinPositive = %v, want the pending rule's hour", got) + } + if err := rules.Resolve("swift", "pkg:swift/github.com/a/two", []string{"pkg:x/kept"}); err == nil { + t.Error("a rule resolving to a package another rule sets differently was accepted") + } + if !rules.Pending("swift") { + t.Fatal("conflicting rule no longer pending") + } + if err := rules.Resolve("swift", "pkg:swift/github.com/a/one", []string{"pkg:x/kept"}); err != nil { + t.Errorf("same duration for the same package: %v", err) + } + if err := rules.Resolve("swift", "pkg:swift/github.com/a/two", []string{"pkg:x/two"}); err != nil { + t.Fatal(err) + } + if rules.Pending("swift") || rules.Packages["pkg:x/two"] != time.Hour { + t.Errorf("after resolving: sources %v, packages %v", rules.Sources, rules.Packages) + } +} + +// A source that resolves to several stored packages sets the rule for all +// of them, or, on a conflict with any, for none. +func TestRulesResolveSeveralPackages(t *testing.T) { + rules := Rules{ + Packages: map[string]time.Duration{"pkg:x/b": time.Hour}, + Sources: map[string]map[string]time.Duration{"swift": {"pkg:swift/github.com/a/one": 0}}, + } + if err := rules.Resolve("swift", "pkg:swift/github.com/a/one", []string{"pkg:x/a", "pkg:x/b"}); err == nil { + t.Error("a rule conflicting for one of its packages was accepted") + } + if _, set := rules.Packages["pkg:x/a"]; set || !rules.Pending("swift") { + t.Fatalf("conflicting rule partly applied: sources %v, packages %v", rules.Sources, rules.Packages) + } + delete(rules.Packages, "pkg:x/b") + if err := rules.Resolve("swift", "pkg:swift/github.com/a/one", []string{"pkg:x/a", "pkg:x/b"}); err != nil { + t.Fatal(err) + } + if rules.Pending("swift") || len(rules.Packages) != 2 { + t.Errorf("after resolving: sources %v, packages %v", rules.Sources, rules.Packages) + } +} diff --git a/internal/server/retention.go b/internal/server/retention.go index 520b4cc..ecd88de 100644 --- a/internal/server/retention.go +++ b/internal/server/retention.go @@ -19,7 +19,7 @@ const retentionWindow = 5000 // follow over several intervals. A variable so tests can lower it. var retentionMaxClears = 10000 -func (s *Server) startRetentionLoop(ctx context.Context) { +func (s *Server) startRetentionLoop(ctx context.Context, resolvers map[string]sourceResolver) { reg := retention.Default rules, err := s.cfg.RetentionRules(reg) if err != nil { @@ -47,14 +47,17 @@ func (s *Server) startRetentionLoop(ctx context.Context) { ticker := time.NewTicker(s.cfg.ParseRetentionSweepInterval()) defer ticker.Stop() - s.sweepRetention(ctx, reg, rules) - + sweep := func() { + resolveRetentionSources(ctx, s.logger, resolvers, rules) + s.sweepRetention(ctx, reg, rules) + } + sweep() for { select { case <-ctx.Done(): return case <-ticker.C: - s.sweepRetention(ctx, reg, rules) + sweep() } } } @@ -147,7 +150,7 @@ func sweepRetention(ctx context.Context, db *database.DB, logger *slog.Logger, func clearIfExpired(db *database.DB, logger *slog.Logger, reg *retention.Registry, rules retention.Rules, now time.Time, c database.RetentionCandidate) bool { key, ok := reg.KeyForDBEcosystem(c.Ecosystem) - if !ok { + if !ok || rules.Pending(key) { return false } d := rules.For(key, c.PackagePURL) diff --git a/internal/server/retention_swift_test.go b/internal/server/retention_swift_test.go index 441320a..3788d4b 100644 --- a/internal/server/retention_swift_test.go +++ b/internal/server/retention_swift_test.go @@ -4,6 +4,7 @@ import ( "context" "crypto/sha256" "encoding/hex" + "encoding/json" "fmt" "io" "net/http" @@ -15,6 +16,7 @@ import ( "github.com/git-pkgs/proxy/internal/config" "github.com/git-pkgs/proxy/internal/database" + "github.com/git-pkgs/proxy/internal/denylist" "github.com/git-pkgs/proxy/internal/handler" "github.com/git-pkgs/proxy/internal/retention" "github.com/git-pkgs/registries/fetch" @@ -154,18 +156,144 @@ func TestSwiftRetentionEvictsAndRefetches(t *testing.T) { } } -// A source repository key parses as a swift PURL but names no registry -// identity, so it is refused at validation. A scope/name key is accepted. -func TestSwiftRetentionRefusesRepositoryKeys(t *testing.T) { +// A source repository key is accepted at validation and kept pending until +// it is resolved through the upstream. A scope/name key resolves offline. +func TestSwiftRetentionSourceKeysPending(t *testing.T) { cfg := config.Default() - cfg.Storage.Retention.Packages = map[string]string{"pkg:swift/apple/swift-nio": "0"} - if _, err := cfg.RetentionRules(retention.Default); err != nil { - t.Fatalf("RetentionRules(pkg:swift/apple/swift-nio): %v", err) + cfg.Storage.Retention.Packages = map[string]string{ + "pkg:swift/apple/swift-nio": "0", + "pkg:swift/github.com/apple/swift-log": "0", } + rules, err := cfg.RetentionRules(retention.Default) + if err != nil { + t.Fatalf("RetentionRules: %v", err) + } + if _, ok := rules.Packages["pkg:generic/swift-registry/apple.swift-nio"]; !ok { + t.Errorf("scope/name key not resolved offline: %v", rules.Packages) + } + if _, ok := rules.Sources["swift"]["pkg:swift/github.com/apple/swift-log"]; !ok || !rules.Pending("swift") { + t.Errorf("source key not pending: %v", rules.Sources) + } +} + +// swiftIdentifiers answers /identifiers for the swift-nio and swift-log +// repositories while up is set, and fails otherwise. +func swiftIdentifiers(up *atomic.Bool, next http.Handler) http.Handler { + return swiftIdentifiersFor(up, map[string][]string{ + "https://github.com/apple/swift-nio": {"apple.swift-nio"}, + "https://github.com/apple/swift-log": {"apple.swift-log"}, + }, next) +} + +// swiftIdentifiersFor answers /identifiers from ids, by repository URL, while +// up is set, and fails otherwise. +func swiftIdentifiersFor(up *atomic.Bool, ids map[string][]string, next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/identifiers" { + next.ServeHTTP(w, r) + return + } + list, ok := ids[r.URL.Query().Get("url")] + switch { + case !up.Load(): + http.Error(w, "down", http.StatusBadGateway) + case !ok: + http.NotFound(w, r) + default: + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string][]string{"identifiers": list}) + } + }) +} + +// A never-evict rule keyed by the source repository keeps Swift from being +// swept while the upstream can't resolve it, and applies once it can. +func TestSwiftRetentionSourceRuleDefersSweep(t *testing.T) { + archives, _ := swiftRetentionUpstream(t) + var up atomic.Bool + identifiers := httptest.NewServer(swiftIdentifiers(&up, http.NotFoundHandler())) + t.Cleanup(identifiers.Close) + srv, db := newSwiftRetentionProxy(t, archives) + const nio, log = "swift-nio-2.65.0.zip", "swift-log-1.5.4.zip" + swiftGet(t, srv.URL+"/apple/swift-nio/2.65.0.zip") + swiftGet(t, srv.URL+"/apple/swift-log/1.5.4.zip") + old := time.Now().Add(-30 * retentionDay) + swiftBackdate(t, db, nio, old, old) + swiftBackdate(t, db, log, old, old) + + cfg := config.Default() + cfg.Storage.Retention.Ecosystems = map[string]string{"swift": "1d"} cfg.Storage.Retention.Packages = map[string]string{"pkg:swift/github.com/apple/swift-nio": "0"} - _, err := cfg.RetentionRules(retention.Default) - if err == nil || !strings.Contains(err.Error(), "does not match how swift packages are stored") { - t.Errorf("RetentionRules(pkg:swift/github.com/apple/swift-nio) error = %v, want the key refused", err) + rules, err := cfg.RetentionRules(retention.Default) + if err != nil { + t.Fatal(err) + } + proxy := handler.NewProxy(db, nil, nil, fetch.NewResolver(), discardLogger()) + proxy.HTTPClient = identifiers.Client() + resolvers := map[string]sourceResolver{ + "swift": swiftRetentionResolver(handler.NewSwiftHandler(proxy, "http://proxy.test", identifiers.URL), retention.Default), + } + + resolveRetentionSources(context.Background(), discardLogger(), resolvers, rules) + sweepRetention(context.Background(), db, discardLogger(), retention.Default, rules, time.Now()) + if !swiftArtifact(t, db, nio).StoragePath.Valid || !swiftArtifact(t, db, log).StoragePath.Valid { + t.Fatal("swift swept while a package rule was unresolved") + } + + up.Store(true) + resolveRetentionSources(context.Background(), discardLogger(), resolvers, rules) + if rules.Pending("swift") { + t.Fatalf("rule still pending once the upstream answers: %v", rules.Sources) + } + sweepRetention(context.Background(), db, discardLogger(), retention.Default, rules, time.Now()) + if !swiftArtifact(t, db, nio).StoragePath.Valid { + t.Error("swift-nio evicted despite its never-evict rule") + } + if swiftArtifact(t, db, log).StoragePath.Valid { + t.Error("swift-log kept past the swift rule") + } +} + +// Denylist entries naming a source repository are resolved to the registry +// identity with their version; one the upstream can't resolve is an error. +func TestResolveDenylistSources(t *testing.T) { + var up atomic.Bool + up.Store(true) + identifiers := httptest.NewServer(swiftIdentifiers(&up, http.NotFoundHandler())) + t.Cleanup(identifiers.Close) + db, store := setupEvictionTest(t) + proxy := handler.NewProxy(db, store, nil, fetch.NewResolver(), discardLogger()) + proxy.HTTPClient = identifiers.Client() + h := handler.NewSwiftHandler(proxy, "http://proxy.test", identifiers.URL) + + got, err := resolveDenylistSources(context.Background(), h, []string{ + "pkg:npm/left-pad@1.3.0", + "pkg:swift/github.com/apple/swift-nio@2.62.0", + "pkg:swift/apple/swift-log@1.5.4", + }) + if err != nil { + t.Fatal(err) + } + want := []string{"pkg:npm/left-pad@1.3.0", "pkg:swift/apple/swift-nio@2.62.0", "pkg:swift/apple/swift-log@1.5.4"} + if fmt.Sprint(got) != fmt.Sprint(want) { + t.Errorf("resolved %v, want %v", got, want) + } + policy, err := denylist.New(got) + if err != nil { + t.Fatal(err) + } + if !policy.Denied("pkg:generic/swift-registry/apple.swift-nio@2.62.0") { + t.Error("the resolved entry doesn't deny the stored archive") + } + + for _, entry := range []string{"pkg:swift/github.com/unknown/repo@1.0.0"} { + if _, err := resolveDenylistSources(context.Background(), h, []string{entry}); err == nil { + t.Errorf("%s: resolved a repository the upstream doesn't know", entry) + } + } + up.Store(false) + if _, err := resolveDenylistSources(context.Background(), h, []string{"pkg:swift/github.com/apple/swift-log@1.5.4"}); err == nil { + t.Error("resolved with the upstream down") } } @@ -210,3 +338,40 @@ func TestSwiftRetentionPackageRules(t *testing.T) { } } } + +// The registry may list several identities for one repository. A source rule +// covers all of them, so the stored archive is kept even when its identity +// is not the first one listed. +func TestSwiftRetentionSourceRuleCoversEveryIdentity(t *testing.T) { + archives, _ := swiftRetentionUpstream(t) + var up atomic.Bool + up.Store(true) + identifiers := httptest.NewServer(swiftIdentifiersFor(&up, map[string][]string{ + "https://github.com/apple/swift-nio": {"other.swift-nio", "apple.swift-nio"}, + }, http.NotFoundHandler())) + t.Cleanup(identifiers.Close) + srv, db := newSwiftRetentionProxy(t, archives) + const nio = "swift-nio-2.65.0.zip" + swiftGet(t, srv.URL+"/apple/swift-nio/2.65.0.zip") + old := time.Now().Add(-30 * retentionDay) + swiftBackdate(t, db, nio, old, old) + + cfg := config.Default() + cfg.Storage.Retention.Ecosystems = map[string]string{"swift": "1d"} + cfg.Storage.Retention.Packages = map[string]string{"pkg:swift/github.com/apple/swift-nio": "0"} + rules, err := cfg.RetentionRules(retention.Default) + if err != nil { + t.Fatal(err) + } + proxy := handler.NewProxy(db, nil, nil, fetch.NewResolver(), discardLogger()) + proxy.HTTPClient = identifiers.Client() + resolvers := map[string]sourceResolver{ + "swift": swiftRetentionResolver(handler.NewSwiftHandler(proxy, "http://proxy.test", identifiers.URL), retention.Default), + } + + resolveRetentionSources(context.Background(), discardLogger(), resolvers, rules) + sweepRetention(context.Background(), db, discardLogger(), retention.Default, rules, time.Now()) + if !swiftArtifact(t, db, nio).StoragePath.Valid { + t.Error("apple.swift-nio evicted although its repository has a never-evict rule") + } +} diff --git a/internal/server/server.go b/internal/server/server.go index 2f0bd68..69363ca 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -75,7 +75,6 @@ import ( "github.com/git-pkgs/proxy/internal/accesslog" "github.com/git-pkgs/proxy/internal/config" "github.com/git-pkgs/proxy/internal/database" - "github.com/git-pkgs/proxy/internal/denylist" "github.com/git-pkgs/proxy/internal/enrichment" "github.com/git-pkgs/proxy/internal/gitcache" "github.com/git-pkgs/proxy/internal/handler" @@ -259,11 +258,6 @@ func (s *Server) serve(listener net.Listener) error { if s.cooldown, err = newCooldownView(s.cfg.Cooldown, cooldownPolicy); err != nil { return err } - policy, err := denylist.New(s.cfg.Denylist.Packages) - if err != nil { - return fmt.Errorf("configuring denylist: %w", err) - } - proxy.Denylist = policy scanGroup, err := configureScanning(proxy, s.cfg.Scanning, s.cfg.BaseURL, s.logger) if err != nil { return fmt.Errorf("configuring scanners: %w", err) @@ -282,6 +276,9 @@ func (s *Server) serve(listener net.Listener) error { proxy.DirectServeTTL = s.cfg.ParseDirectServeTTL() proxy.DirectServeBaseURL = s.cfg.Storage.DirectServeBaseURL proxy.StreamArtifacts = !s.cfg.Storage.CacheArtifacts + if err := ConfigureDenylist(s.cfg, proxy); err != nil { + return err + } if len(s.cfg.Upstream.Git) > 0 { upstreamDirs := make(map[string]string, len(s.cfg.Upstream.Git)) @@ -429,7 +426,8 @@ func (s *Server) serve(listener net.Listener) error { s.background.Go(func() { s.updateCacheStatsMetrics(bgCtx) }) s.background.Go(func() { backfillMetadataAccess(bgCtx, s.db, s.logger, metadataBackfillBatch) }) s.background.Go(func() { s.startEvictionLoop(bgCtx) }) - s.background.Go(func() { s.startRetentionLoop(bgCtx) }) + retentionResolvers := s.retentionResolvers(proxy) + s.background.Go(func() { s.startRetentionLoop(bgCtx, retentionResolvers) }) s.background.Go(func() { s.startReclaimLoop(bgCtx) }) if listener != nil { diff --git a/internal/server/swift_sources.go b/internal/server/swift_sources.go new file mode 100644 index 0000000..b944e6a --- /dev/null +++ b/internal/server/swift_sources.go @@ -0,0 +1,136 @@ +package server + +import ( + "context" + "fmt" + "log/slog" + "maps" + "slices" + "strings" + "time" + + "github.com/git-pkgs/proxy/internal/config" + "github.com/git-pkgs/proxy/internal/denylist" + "github.com/git-pkgs/proxy/internal/handler" + "github.com/git-pkgs/proxy/internal/retention" + "github.com/git-pkgs/purl" +) + +// A Swift package can be configured by its source repository, as SBOM tools +// and GHSA name it (pkg:swift/github.com/apple/swift-nio), as well as by its +// registry identity. The proxy stores and compares registry identities, so +// source repositories are resolved through the upstream's /identifiers: for +// the denylist at startup, which fails when one can't be, and for retention +// before each sweep, which leaves Swift unswept until all are. + +// sourceResolveTimeout bounds resolving the denylist's source repositories at +// startup. +const sourceResolveTimeout = time.Minute + +// sourceResolver resolves a package PURL that names a package by its source +// to the stored package PURLs retention compares. +type sourceResolver func(ctx context.Context, p *purl.PURL) ([]string, error) + +// swiftRetentionResolver resolves Swift source repositories through h, to +// every identity the upstream lists. +func swiftRetentionResolver(h *handler.SwiftHandler, reg *retention.Registry) sourceResolver { + return func(ctx context.Context, p *purl.PURL) ([]string, error) { + resolved, err := h.ResolveSourcePURL(ctx, p) + if err != nil { + return nil, err + } + canonical := make([]string, 0, len(resolved)) + for _, r := range resolved { + c, ok := reg.Canonical(r) + if !ok { + return nil, fmt.Errorf("%s resolved to %s, which names no stored package", p, r) + } + canonical = append(canonical, c) + } + return canonical, nil + } +} + +// retentionResolvers returns how the retention loop resolves package rules +// that name a package by its source, by ecosystem key. +func (s *Server) retentionResolvers(proxy *handler.Proxy) map[string]sourceResolver { + return map[string]sourceResolver{ + "swift": swiftRetentionResolver(handler.NewSwiftHandler(proxy, s.cfg.BaseURL, s.cfg.Upstream.Swift), retention.Default), + } +} + +// resolveDenylistSources returns entries with each Swift source repository +// replaced by every registry identity the upstream lists for it, keeping the +// version. An entry that can't be resolved is an error: left out, it would +// let the version through. +func resolveDenylistSources(ctx context.Context, h *handler.SwiftHandler, entries []string) ([]string, error) { + resolved := make([]string, 0, len(entries)) + for _, entry := range entries { + p, err := purl.Parse(entry) + if err != nil || handler.SwiftSourcePURL(p) == "" { + resolved = append(resolved, entry) + continue + } + identities, err := h.ResolveSourcePURL(ctx, p) + if err != nil { + return nil, fmt.Errorf("resolving denylist package %q: %w", entry, err) + } + for _, identity := range identities { + resolved = append(resolved, identity.String()) + } + } + return resolved, nil +} + +// resolveRetentionSources resolves the package rules in rules.Sources it can, +// moving them to rules.Packages. A rule that fails stays pending, so its +// ecosystem isn't swept, and is tried again before the next sweep. +func resolveRetentionSources(ctx context.Context, logger *slog.Logger, resolvers map[string]sourceResolver, rules retention.Rules) { + for _, key := range slices.Sorted(maps.Keys(rules.Sources)) { + resolve := resolvers[key] + for _, source := range slices.Sorted(maps.Keys(rules.Sources[key])) { + if resolve == nil { + logger.Error("retention: no way to resolve package rule, not sweeping "+key, "package", source) + continue + } + p, err := purl.Parse(source) + if err != nil { + continue // config validation refuses these + } + canonical, err := resolve(ctx, p) + if err == nil { + err = rules.Resolve(key, source, canonical) + } + if err != nil { + logger.Warn("retention: could not resolve package rule, not sweeping "+key+" until it is", + "package", source, "error", err) + continue + } + logger.Info("retention: resolved package rule", "package", source, "stored", strings.Join(canonical, ", ")) + } + } +} + +// ConfigureDenylist sets proxy's denylist from cfg, resolving Swift source +// repositories through the upstream first. serve and proxy mirror call it +// once proxy can fetch metadata. An entry that can't be resolved is an +// error, so neither starts with a denylist that lets its versions through. +func ConfigureDenylist(cfg *config.Config, proxy *handler.Proxy) error { + entries := cfg.Denylist.Packages + // Report a malformed entry before asking the upstream about any. + if _, err := denylist.New(entries); err != nil { + return fmt.Errorf("configuring denylist: %w", err) + } + ctx, cancel := context.WithTimeout(context.Background(), sourceResolveTimeout) + defer cancel() + entries, err := resolveDenylistSources(ctx, handler.NewSwiftHandler(proxy, cfg.BaseURL, cfg.Upstream.Swift), entries) + if err != nil { + return fmt.Errorf("configuring denylist: %w", err) + } + policy, err := denylist.New(entries) + if err != nil { + return fmt.Errorf("configuring denylist: %w", err) + } + proxy.Denylist = policy + return nil +} From c4979b9e762096c13ce3a1420f2103c194b4fc7e Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sat, 31 Oct 2026 15:19:11 +0100 Subject: [PATCH 521/633] Count stored publish times under their ecosystem in the stats Cooldown stores the publish time of every version it looks up, also of packages nobody downloaded, which have no package row. The ecosystem stats counted those versions as unattributed. They now count under the type of their package PURL; only a PURL that can't be parsed stays unattributed. --- internal/database/analytics_test.go | 39 +++++++++++++++++++++++++++++ internal/database/queries.go | 36 +++++++++++++++++++++----- 2 files changed, 69 insertions(+), 6 deletions(-) diff --git a/internal/database/analytics_test.go b/internal/database/analytics_test.go index 269b9d2..e537694 100644 --- a/internal/database/analytics_test.go +++ b/internal/database/analytics_test.go @@ -373,3 +373,42 @@ func TestMergeAliasedEcosystemsBreaksSizeTiesByName(t *testing.T) { t.Errorf("input order changed the surviving spelling: %q vs %q", first, second) } } + +// Cooldown stores publish times of versions nobody downloaded, without a +// package row. They count under the ecosystem of their package PURL, not as +// unattributed. +func TestGetEcosystemStatsAttributesPublishTimesByPURL(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + seedArtifact(t, db, "npm", "lodash", "4.17.21", 1000, 2) + published := time.Now().Add(-time.Hour) + for _, v := range []struct{ version, pkg string }{ + {"pkg:npm/react@19.0.0", "pkg:npm/react"}, + {"pkg:golang/github.com/swaggo/swag@v1.16.3", "pkg:golang/github.com/swaggo/swag"}, + {"pkg:golang/github.com/swaggo/swag@v1.16.4", "pkg:golang/github.com/swaggo/swag"}, + {"pkg:composer/vendor/pkg@1.0.0", "pkg:composer/vendor/pkg"}, + {"not-a-purl@1", "not-a-purl"}, + } { + if err := db.SetVersionPublishedAt(v.version, v.pkg, published); err != nil { + t.Fatal(err) + } + } + + stats, err := db.GetEcosystemStats() + if err != nil { + t.Fatalf("GetEcosystemStats: %v", err) + } + versions := map[string]int64{} + for _, e := range stats { + versions[e.Ecosystem] = e.Versions + } + want := map[string]int64{"npm": 2, "golang": 2, "composer": 1, unattributedEcosystem: 1} + for ecosystem, n := range want { + if versions[ecosystem] != n { + t.Errorf("versions of %s = %d, want %d (all: %v)", ecosystem, versions[ecosystem], n, versions) + } + } + if len(versions) != len(want) { + t.Errorf("ecosystems = %v, want %v", versions, want) + } + }) +} diff --git a/internal/database/queries.go b/internal/database/queries.go index d7150e8..b3ec832 100644 --- a/internal/database/queries.go +++ b/internal/database/queries.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "sort" + "strings" "time" "github.com/git-pkgs/artifacts" @@ -1458,15 +1459,18 @@ func (db *DB) GetEcosystemStats() ([]EcosystemStats, error) { return nil, err } - // Left joined and bucketed for the same reason as artifacts below: a version - // whose package row is missing would otherwise vanish here while still - // counting in GetCacheStats' COUNT(*), leaving the two unable to reconcile. + // Left joined for the same reason as artifacts below: a version whose + // package row is missing would otherwise vanish here while still counting + // in GetCacheStats' COUNT(*), leaving the two unable to reconcile. Such + // versions are mostly publish times cooldown stored for versions nobody + // downloaded, so they count under the type of their package PURL, and + // only a PURL that can't be parsed is unattributed. if err := db.eachCount(` - SELECT COALESCE(p.ecosystem, '`+unattributedEcosystem+`'), COUNT(*) + SELECT COALESCE(p.ecosystem, '`+orphanVersionPrefix+`' || v.package_purl), COUNT(*) FROM versions v LEFT JOIN packages p ON p.purl = v.package_purl - GROUP BY COALESCE(p.ecosystem, '`+unattributedEcosystem+`') - `, func(ecosystem string, n int64) { get(ecosystem).Versions = n }); err != nil { + GROUP BY COALESCE(p.ecosystem, '`+orphanVersionPrefix+`' || v.package_purl) + `, func(key string, n int64) { get(versionEcosystem(key)).Versions += n }); err != nil { return nil, err } @@ -1549,6 +1553,26 @@ func mergeAliasedEcosystems(stats []EcosystemStats) []EcosystemStats { // Grafana. Bucketing them keeps the two reconcilable. const unattributedEcosystem = "unattributed" +// orphanVersionPrefix marks the package PURL of a version without a package +// row in GetEcosystemStats. No ecosystem name starts with it. +const orphanVersionPrefix = "~" + +// versionEcosystem returns the ecosystem a GetEcosystemStats version count +// is for: the package row's, or the type of the package PURL of a version +// without one. The ecosystems that store publish times are named like their +// PURL types. +func versionEcosystem(key string) string { + packagePURL, orphan := strings.CutPrefix(key, orphanVersionPrefix) + if !orphan { + return key + } + p, err := purl.Parse(packagePURL) + if err != nil { + return unattributedEcosystem + } + return p.Type +} + func (db *DB) eachArtifactStat(get func(string) *EcosystemStats) error { rows, err := db.Query(` SELECT COALESCE(p.ecosystem, '` + unattributedEcosystem + `'), From 11ff4cf314c1d7dbeaf45167d01b2559b6543472 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sat, 31 Oct 2026 16:55:15 +0100 Subject: [PATCH 522/633] Fetch an npm packument again when its stored copy is gone within the TTL A row within its TTL was served without checking its blob, so a missing blob failed the streamed read and the request fell back to the bounded in-memory path. A read that finds the stored copy missing now fetches the packument again into storage, ignoring that row, and reads it once more, for packument, version and tarball requests. --- internal/handler/metadata_stored.go | 28 +++++- internal/handler/npm.go | 149 +++++++++++++++++----------- internal/handler/npm_stream_test.go | 87 +++++++++++----- 3 files changed, 176 insertions(+), 88 deletions(-) diff --git a/internal/handler/metadata_stored.go b/internal/handler/metadata_stored.go index 80cbaf4..7d43d01 100644 --- a/internal/handler/metadata_stored.go +++ b/internal/handler/metadata_stored.go @@ -45,13 +45,27 @@ var errStoredMetadataChanged = errors.New("stored metadata changed while it was // It returns errMetadataNotStored when the proxy doesn't cache metadata or // storing the document failed, and the caller reads it into memory instead. func (p *Proxy) storedMetadataDocument(ctx context.Context, ecosystem, cacheKey, upstreamURL, accept string, validate func(io.Reader) error) (*database.MetadataCacheEntry, error) { + return p.storedMetadata(ctx, ecosystem, cacheKey, upstreamURL, accept, validate, "") +} + +// restoreStoredMetadata is storedMetadataDocument for a document whose row +// gone describes but whose stored copy was missing when read: the row is not +// served or revalidated, within its TTL or not, and the document is fetched +// again. +func (p *Proxy) restoreStoredMetadata(ctx context.Context, ecosystem, cacheKey, upstreamURL, accept string, validate func(io.Reader) error, gone *database.MetadataCacheEntry) (*database.MetadataCacheEntry, error) { + return p.storedMetadata(ctx, ecosystem, cacheKey, upstreamURL, accept, validate, gone.ContentDigest.String) +} + +// storedMetadata implements storedMetadataDocument. A row with the digest +// gone is treated as if there were none. +func (p *Proxy) storedMetadata(ctx context.Context, ecosystem, cacheKey, upstreamURL, accept string, validate func(io.Reader) error, gone string) (*database.MetadataCacheEntry, error) { if !p.CacheMetadata || p.DB == nil || p.Storage == nil || p.MetadataTTL <= 0 { return nil, errMetadataNotStored } if containsPathTraversal(cacheKey) { return nil, fmt.Errorf("invalid cache key: %q", cacheKey) } - if entry, fresh := p.storedMetadataEntryState(ctx, ecosystem, cacheKey); fresh { + if entry, fresh := p.storedMetadataEntryState(ctx, ecosystem, cacheKey, gone); fresh { metrics.RecordMetadataCacheHit(ecosystem, int(entry.Size.Int64)) p.recordMetadataRead(entry) return entry, nil @@ -60,12 +74,12 @@ func (p *Proxy) storedMetadataDocument(ctx context.Context, ecosystem, cacheKey, key := metadataCoalesceKey(ecosystem, cacheKey, upstreamURL, accept, "", false) + "\x00stored" res, _ := p.coalesceMetadata(ctx, key, func(fetchCtx context.Context) metadataResult { - entry, fresh := p.storedMetadataEntryState(fetchCtx, ecosystem, cacheKey) + entry, fresh := p.storedMetadataEntryState(fetchCtx, ecosystem, cacheKey, gone) if fresh { return metadataResult{entry: entry} } release, outcome, err := p.awaitFetchLease(fetchCtx, metrics.CacheKindMetadata, key, func() bool { - entry, fresh = p.storedMetadataEntryState(fetchCtx, ecosystem, cacheKey) + entry, fresh = p.storedMetadataEntryState(fetchCtx, ecosystem, cacheKey, gone) return fresh }) if err != nil { @@ -92,10 +106,14 @@ func (p *Proxy) storedMetadataDocument(ctx context.Context, ecosystem, cacheKey, } // storedMetadataEntryState reads the cache row of a document and reports -// whether it is within its TTL and usable by storedMetadataDocument. -func (p *Proxy) storedMetadataEntryState(ctx context.Context, ecosystem, cacheKey string) (*database.MetadataCacheEntry, bool) { +// whether it is within its TTL and usable by storedMetadataDocument. A row +// with the digest gone is returned as nil. +func (p *Proxy) storedMetadataEntryState(ctx context.Context, ecosystem, cacheKey, gone string) (*database.MetadataCacheEntry, bool) { entry, _ := p.DB.GetMetadataCache(ecosystem, cacheKey) entry = p.migrateLegacyMetadataEntry(ctx, ecosystem, cacheKey, entry) + if gone != "" && entry != nil && entry.ContentDigest.String == gone { + return nil, false + } fresh := usableStoredMetadata(entry) && entry.FetchedAt.Valid && time.Since(entry.FetchedAt.Time) < p.MetadataTTL return entry, fresh } diff --git a/internal/handler/npm.go b/internal/handler/npm.go index 05e77ea..054400b 100644 --- a/internal/handler/npm.go +++ b/internal/handler/npm.go @@ -17,6 +17,7 @@ import ( "github.com/git-pkgs/proxy/internal/jsonscan" "github.com/git-pkgs/proxy/internal/metadata/npm" "github.com/git-pkgs/proxy/internal/retention" + "github.com/git-pkgs/proxy/internal/storage" ) const ( @@ -340,35 +341,36 @@ func (h *NPMHandler) serveStreamedPackument(w http.ResponseWriter, r *http.Reque if h.proxy.rewrites == nil { return false } - entry, err := h.storedPackument(r.Context(), upstreamURL, accept, metadataCacheKey) - switch { - case err == nil: - case errors.Is(err, errInMemory): - return false - default: - h.writeFetchError(w, err) - return true - } - fullRequired := h.packumentValidator() != nil options := h.npmRewriteOptions(packageName, nil) - rewritten, err := h.proxy.streamedRewrite(r.Context(), "npm", h.proxyURL, metadataCacheKey, entry, func(open func() (io.ReadCloser, error), out io.Writer) error { - var scan *npm.Scan - err := readStored(open, func(in io.Reader) (err error) { - scan, err = npm.ScanPackument(in) - return err - }) - if err != nil { - return err - } - if fullRequired && scan.Abbreviated() { - return npm.ErrNotStreamable - } - return readStored(open, func(in io.Reader) error { - _, err := npm.RewriteStream(packageName, scan, in, out, options) - return err + var rewritten rewrittenMetadata + err, fetchErr := h.readStoredPackument(r.Context(), upstreamURL, accept, metadataCacheKey, func(entry *database.MetadataCacheEntry) (err error) { + rewritten, err = h.proxy.streamedRewrite(r.Context(), "npm", h.proxyURL, metadataCacheKey, entry, func(open func() (io.ReadCloser, error), out io.Writer) error { + var scan *npm.Scan + err := readStored(open, func(in io.Reader) (err error) { + scan, err = npm.ScanPackument(in) + return err + }) + if err != nil { + return err + } + if fullRequired && scan.Abbreviated() { + return npm.ErrNotStreamable + } + return readStored(open, func(in io.Reader) error { + _, err := npm.RewriteStream(packageName, scan, in, out, options) + return err + }) }) + return err }) + switch { + case errors.Is(fetchErr, errInMemory): + return false + case fetchErr != nil: + h.writeFetchError(w, fetchErr) + return true + } if err != nil { if r.Context().Err() != nil { return true // the client left while waiting on a shared rewrite @@ -419,6 +421,30 @@ func (h *NPMHandler) storedPackument(ctx context.Context, packumentURL, accept, return entry, err } +// readStoredPackument reads the packument stored for cacheKey with read, +// fetching it into storage first when needed. When the stored copy is gone, +// although its row says otherwise, the packument is fetched again and read +// once more. fetchErr is what storedPackument returned when the packument +// couldn't be fetched, and err is what read returned. +func (h *NPMHandler) readStoredPackument(ctx context.Context, packumentURL, accept, cacheKey string, read func(*database.MetadataCacheEntry) error) (err, fetchErr error) { + entry, fetchErr := h.storedPackument(ctx, packumentURL, accept, cacheKey) + if fetchErr != nil { + return nil, fetchErr + } + if err = read(entry); !errors.Is(err, storage.ErrNotFound) { + return err, nil + } + h.proxy.Logger.Warn("stored npm metadata is missing, fetching it again", "url", packumentURL) + entry, fetchErr = h.proxy.restoreStoredMetadata(ctx, "npm", cacheKey, packumentURL, accept, h.packumentStreamValidator(), entry) + if errors.Is(fetchErr, errMetadataNotStored) { + return nil, errInMemory + } + if fetchErr != nil { + return nil, fetchErr + } + return read(entry), nil +} + // packumentStreamValidator is packumentValidator for a packument read from // storage. func (h *NPMHandler) packumentStreamValidator() func(io.Reader) error { @@ -446,37 +472,40 @@ func (h *NPMHandler) streamedVersionMetadata(ctx context.Context, packageName, s if h.proxy.rewrites == nil { return rewrittenMetadata{}, errInMemory } - entry, err := h.storedPackument(ctx, packumentURL, accept, cacheKey) - if errors.Is(err, errInMemory) { - return rewrittenMetadata{}, err - } - if err != nil { - return rewrittenMetadata{}, &npmFetchError{err} - } fullRequired := h.packumentValidator() != nil options := h.npmRewriteOptions(packageName, nil) - m, err := h.proxy.streamedRewrite(ctx, "npm", h.proxyURL, metadataCacheKey, entry, func(open func() (io.ReadCloser, error), out io.Writer) error { - var scan *npm.Scan - err := readStored(open, func(in io.Reader) (err error) { - scan, err = npm.ScanPackument(in) - return err - }) - if err != nil { - return err - } - if fullRequired && scan.Abbreviated() { - return npm.ErrNotStreamable - } - return readStored(open, func(in io.Reader) error { - version, err := npm.VersionStream(packageName, specifier, scan, in, options) + var m rewrittenMetadata + err, fetchErr := h.readStoredPackument(ctx, packumentURL, accept, cacheKey, func(entry *database.MetadataCacheEntry) (err error) { + m, err = h.proxy.streamedRewrite(ctx, "npm", h.proxyURL, metadataCacheKey, entry, func(open func() (io.ReadCloser, error), out io.Writer) error { + var scan *npm.Scan + err := readStored(open, func(in io.Reader) (err error) { + scan, err = npm.ScanPackument(in) + return err + }) if err != nil { - return npmRewriteError(err) + return err } - _, err = out.Write(version) - return err + if fullRequired && scan.Abbreviated() { + return npm.ErrNotStreamable + } + return readStored(open, func(in io.Reader) error { + version, err := npm.VersionStream(packageName, specifier, scan, in, options) + if err != nil { + return npmRewriteError(err) + } + _, err = out.Write(version) + return err + }) }) + return err }) switch { + case errors.Is(fetchErr, errInMemory): + return rewrittenMetadata{}, fetchErr + case fetchErr != nil: + return rewrittenMetadata{}, &npmFetchError{fetchErr} + } + switch { case err == nil, ctx.Err() != nil, errors.Is(err, ErrVersionDenied), errors.Is(err, ErrVersionNotFound): return m, err case !errors.Is(err, npm.ErrNotStreamable): @@ -856,20 +885,20 @@ type npmVersionData struct { // otherwise from the packument read into memory. func (h *NPMHandler) versionData(ctx context.Context, packumentURL, cacheKey, version string) (npmVersionData, error) { accept := h.packumentAccept() - if entry, err := h.storedPackument(ctx, packumentURL, accept, cacheKey); err == nil { - var found npm.VersionEntry - err := readStored(func() (io.ReadCloser, error) { return h.proxy.openStoredMetadata(ctx, entry) }, func(in io.Reader) (err error) { + var found npm.VersionEntry + err, fetchErr := h.readStoredPackument(ctx, packumentURL, accept, cacheKey, func(entry *database.MetadataCacheEntry) error { + return readStored(func() (io.ReadCloser, error) { return h.proxy.openStoredMetadata(ctx, entry) }, func(in io.Reader) (err error) { found, err = npm.LookupVersion(in, version) return err }) - switch { - case err == nil && (!found.Abbreviated || h.packumentValidator() == nil): - return npmVersionData{entry: found.Entry, published: found.Published}, nil - case err != nil: - h.proxy.Logger.Warn("failed to read stored npm metadata, reading it in memory", "url", packumentURL, "error", err) - } - } else if !errors.Is(err, errInMemory) { - return npmVersionData{}, err + }) + switch { + case fetchErr == nil && err == nil && (!found.Abbreviated || h.packumentValidator() == nil): + return npmVersionData{entry: found.Entry, published: found.Published}, nil + case fetchErr == nil && err != nil: + h.proxy.Logger.Warn("failed to read stored npm metadata, reading it in memory", "url", packumentURL, "error", err) + case fetchErr != nil && !errors.Is(fetchErr, errInMemory): + return npmVersionData{}, fetchErr } body, _, _, err := h.proxy.fetchOrCacheMetadata(ctx, "npm", cacheKey, packumentURL, "", h.packumentValidator(), accept) diff --git a/internal/handler/npm_stream_test.go b/internal/handler/npm_stream_test.go index de5a17d..d63ab9b 100644 --- a/internal/handler/npm_stream_test.go +++ b/internal/handler/npm_stream_test.go @@ -232,33 +232,74 @@ func TestNPMStreamedPackumentStaleOn404(t *testing.T) { } } -// TestNPMStreamedPackumentBlobMissing checks that a 304 for a row whose -// document is gone from storage fetches the document again, also when it is -// larger than metadata_max_size. +// TestNPMStreamedPackumentBlobMissing checks that a row whose document is +// gone from storage, within its TTL or revalidated by a 304 after it, fetches +// the document again, also when it is larger than metadata_max_size. func TestNPMStreamedPackumentBlobMissing(t *testing.T) { - body := bigPackument(400) - u := newStreamUpstream(t, body) + for _, tt := range []struct { + name string + ttl time.Duration + }{ + {"fresh row", time.Hour}, + {"expired row", time.Nanosecond}, + } { + t.Run(tt.name, func(t *testing.T) { + body := bigPackument(400) + u := newStreamUpstream(t, body) + proxy, store, handler := streamProxy(t, u) + proxy.MetadataMaxSize = int64(len(body) / 4) + decodeStreamed(t, serve(handler, "/big", nil)) + + path := metadataStoragePath("npm", "big") + store.mu.Lock() + delete(store.files, path) + store.mu.Unlock() + proxy.MetadataTTL = tt.ttl + proxy.SetMetadataRewriteCacheSize(64 << 20) + + if doc := decodeStreamed(t, serve(handler, "/big", nil)); len(doc.Versions) != 400 { + t.Errorf("served %d versions, want 400", len(doc.Versions)) + } + if n := u.sent.Load(); n != 2 { + t.Errorf("upstream sent the document %d times, want 2", n) + } + store.mu.Lock() + defer store.mu.Unlock() + if string(store.files[path]) != body { + t.Error("the document was not stored again") + } + }) + } +} + +// TestNPMStreamedVersionBlobMissing checks that a version request and a +// tarball download fetch a packument again whose stored copy is gone while +// its row is within the TTL. +func TestNPMStreamedVersionBlobMissing(t *testing.T) { + u := newStreamUpstream(t, "") + u.body.Store(bigPackumentAt(u.URL, 100)) proxy, store, handler := streamProxy(t, u) - proxy.MetadataMaxSize = int64(len(body) / 4) - decodeStreamed(t, serve(handler, "/big", nil)) - + proxy.MetadataMaxSize = 1024 + fetcher := proxy.Fetcher.(*mockFetcher) + fetcher.artifact = &fetch.Artifact{Body: io.NopCloser(strings.NewReader("package"))} path := metadataStoragePath("npm", "big") - store.mu.Lock() - delete(store.files, path) - store.mu.Unlock() - proxy.MetadataTTL = time.Nanosecond - proxy.SetMetadataRewriteCacheSize(64 << 20) - if doc := decodeStreamed(t, serve(handler, "/big", nil)); len(doc.Versions) != 400 { - t.Errorf("served %d versions, want 400", len(doc.Versions)) - } - if n := u.sent.Load(); n != 2 { - t.Errorf("upstream sent the document %d times, want 2", n) - } - store.mu.Lock() - defer store.mu.Unlock() - if string(store.files[path]) != body { - t.Error("the document was not stored again") + for _, req := range []string{"/big/1.0.5", "/big/-/big-1.0.5.tgz"} { + decodeStreamed(t, serve(handler, "/big", nil)) + store.mu.Lock() + delete(store.files, path) + store.mu.Unlock() + proxy.SetMetadataRewriteCacheSize(64 << 20) + + if rec := serve(handler, req, nil); rec.Code != http.StatusOK { + t.Errorf("%s: status %d: %.200s", req, rec.Code, rec.Body.String()) + } + store.mu.Lock() + _, ok := store.files[path] + store.mu.Unlock() + if !ok { + t.Errorf("%s: the packument was not stored again", req) + } } } From f0c96e87a81ce6f2404ddd45c7527747c6d6dbaf Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sat, 31 Oct 2026 17:50:24 +0100 Subject: [PATCH 523/633] Keep the stored copy when upstream cuts a metadata document off A body that ended before its Content-Length was read as complete in memory: ReadMetadataSized took the transport's io.ErrUnexpectedEOF for the end of a body shorter than its size hint. The half document was served and cached, or failed the filter with a 502, instead of the stored copy being served. Upstream bodies are now read through a reader that marks their read errors, and such a failure is an upstream failure on both paths: streamed, it no longer counts as a failure to store, which had sent the request to the bounded in-memory path. A stored packument whose bytes no longer match its row's digest is now fetched again like a missing one. Tests also cover concurrent requests sharing one fetch, client revalidation and upstream failures with nothing cached. The stream tests build cooldown through cooldown.NewPolicy as the server does, since a bare cooldown.Config parses itself on first use without locking. --- internal/handler/handler.go | 9 +- internal/handler/metadata_stored.go | 36 +++- internal/handler/npm.go | 10 +- internal/handler/npm_stream_harden_test.go | 206 +++++++++++++++++++++ internal/handler/npm_stream_test.go | 27 ++- internal/handler/read_metadata_test.go | 16 ++ 6 files changed, 287 insertions(+), 17 deletions(-) create mode 100644 internal/handler/npm_stream_harden_test.go diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 96ae655..e0cede7 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -157,7 +157,9 @@ func (p *Proxy) ReadMetadata(r io.Reader) ([]byte, error) { // Reading into a buffer of that size avoids io.ReadAll growing its buffer // step by step, which peaks at more than twice the document size. size is // only a hint: a negative one, or one above the limit, reads as ReadMetadata -// does, and a body of a different length is still read correctly. +// does, and a body of a different length is still read correctly. A body +// read through upstreamBodyReader that upstream cuts off fails, even though +// it ends like a body shorter than size. func (p *Proxy) ReadMetadataSized(r io.Reader, size int64) ([]byte, error) { limit := p.MetadataMaxSize if limit <= 0 { @@ -170,7 +172,10 @@ func (p *Proxy) ReadMetadataSized(r io.Reader, size int64) ([]byte, error) { // the buffer and a longer one is noticed. buf := make([]byte, size+1) n, err := io.ReadFull(r, buf) + var cut *upstreamBodyError switch { + case errors.As(err, &cut): + return nil, err case errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF): if int64(n) < size/2 { // The hint was far off; don't keep the oversized buffer alive. @@ -1989,7 +1994,7 @@ func (p *Proxy) fetchUpstreamMetadata(ctx context.Context, upstreamURL string, e // ContentLength is -1 when the transport decompresses the body, and // otherwise the length of the bytes read here. - body, err := p.ReadMetadataSized(resp.Body, resp.ContentLength) + body, err := p.ReadMetadataSized(upstreamBodyReader{resp.Body}, resp.ContentLength) if err != nil { return nil, fmt.Errorf("reading response: %w", err) } diff --git a/internal/handler/metadata_stored.go b/internal/handler/metadata_stored.go index 7d43d01..8bbf3eb 100644 --- a/internal/handler/metadata_stored.go +++ b/internal/handler/metadata_stored.go @@ -200,9 +200,9 @@ func (p *Proxy) storeUpstreamMetadata(ctx context.Context, ecosystem, cacheKey, } // Go decompresses the body, since no Accept-Encoding was set, so it is // stored uncompressed. - var body io.Reader = resp.Body + var body io.Reader = upstreamBodyReader{resp.Body} if validate != nil { - staged, err := p.stageMetadata(ctx, ecosystem, cacheKey, resp.Body, validate) + staged, err := p.stageMetadata(ctx, ecosystem, cacheKey, body, validate) if err != nil { return nil, err } @@ -210,7 +210,7 @@ func (p *Proxy) storeUpstreamMetadata(ctx context.Context, ecosystem, cacheKey, body = staged } if _, err := p.storeMetadataBlob(ctx, ecosystem, cacheKey, metadataStoragePath(ecosystem, cacheKey), body, meta); err != nil { - return nil, fmt.Errorf("%w: %w", errMetadataNotStored, err) + return nil, notStored("", err) } stored, err := p.DB.GetMetadataCache(ecosystem, cacheKey) if err != nil || !usableStoredMetadata(stored) { @@ -232,7 +232,7 @@ func (p *Proxy) stageMetadata(ctx context.Context, ecosystem, cacheKey string, b } if _, _, err := p.Storage.Store(ctx, path, body); err != nil { discard() - return nil, fmt.Errorf("%w: staging: %w", errMetadataNotStored, err) + return nil, notStored("staging: ", err) } if err := readStored(func() (io.ReadCloser, error) { return p.Storage.Open(ctx, path) }, validate); err != nil { discard() @@ -246,6 +246,34 @@ func (p *Proxy) stageMetadata(ctx context.Context, ecosystem, cacheKey string, b return &stagedReader{ReadCloser: staged, discard: discard}, nil } +// upstreamBodyReader marks a failure to read a document from upstream, so +// it is told apart from a failure to store it. +type upstreamBodyReader struct{ r io.Reader } + +func (u upstreamBodyReader) Read(b []byte) (int, error) { + n, err := u.r.Read(b) + if err != nil && !errors.Is(err, io.EOF) { + err = &upstreamBodyError{err} + } + return n, err +} + +// upstreamBodyError is a failure to read a document from upstream. It is +// handled as an upstream failure: the stored copy, if any, is served. +type upstreamBodyError struct{ err error } + +func (e *upstreamBodyError) Error() string { return "reading upstream metadata: " + e.err.Error() } +func (e *upstreamBodyError) Unwrap() error { return e.err } + +// notStored returns a failure to store a document as errMetadataNotStored, +// unless reading it from upstream failed. +func notStored(what string, err error) error { + if body := (*upstreamBodyError)(nil); errors.As(err, &body) { + return body + } + return fmt.Errorf("%w: %s%w", errMetadataNotStored, what, err) +} + // stagedReader deletes the staged copy it reads when closed. type stagedReader struct { io.ReadCloser diff --git a/internal/handler/npm.go b/internal/handler/npm.go index 054400b..50757a3 100644 --- a/internal/handler/npm.go +++ b/internal/handler/npm.go @@ -422,19 +422,19 @@ func (h *NPMHandler) storedPackument(ctx context.Context, packumentURL, accept, } // readStoredPackument reads the packument stored for cacheKey with read, -// fetching it into storage first when needed. When the stored copy is gone, -// although its row says otherwise, the packument is fetched again and read -// once more. fetchErr is what storedPackument returned when the packument +// fetching it into storage first when needed. When the stored copy is gone +// or no longer has the digest its row records, the packument is fetched +// again and read once more. fetchErr is what storedPackument returned when the packument // couldn't be fetched, and err is what read returned. func (h *NPMHandler) readStoredPackument(ctx context.Context, packumentURL, accept, cacheKey string, read func(*database.MetadataCacheEntry) error) (err, fetchErr error) { entry, fetchErr := h.storedPackument(ctx, packumentURL, accept, cacheKey) if fetchErr != nil { return nil, fetchErr } - if err = read(entry); !errors.Is(err, storage.ErrNotFound) { + if err = read(entry); !errors.Is(err, storage.ErrNotFound) && !errors.Is(err, errStoredMetadataChanged) { return err, nil } - h.proxy.Logger.Warn("stored npm metadata is missing, fetching it again", "url", packumentURL) + h.proxy.Logger.Warn("stored npm metadata is missing or changed, fetching it again", "url", packumentURL, "error", err) entry, fetchErr = h.proxy.restoreStoredMetadata(ctx, "npm", cacheKey, packumentURL, accept, h.packumentStreamValidator(), entry) if errors.Is(fetchErr, errMetadataNotStored) { return nil, errInMemory diff --git a/internal/handler/npm_stream_harden_test.go b/internal/handler/npm_stream_harden_test.go new file mode 100644 index 0000000..ff8f97e --- /dev/null +++ b/internal/handler/npm_stream_harden_test.go @@ -0,0 +1,206 @@ +package handler + +import ( + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + "time" +) + +// bigMetadataFiles returns a copy of what is in storage under the metadata path +// of big, staged copies included. +func bigMetadataFiles(store *mockStorage) map[string]string { + store.mu.Lock() + defer store.mu.Unlock() + files := map[string]string{} + for path, b := range store.files { + if strings.HasPrefix(path, metadataStoragePath("npm", "big")) { + files[path] = string(b) + } + } + return files +} + +// TestNPMStreamedPackumentBlobCorrupt checks that a stored copy whose bytes +// no longer match the digest of its row, within its TTL, is fetched again +// rather than failing the request, also above metadata_max_size. +func TestNPMStreamedPackumentBlobCorrupt(t *testing.T) { + body := bigPackument(400) + u := newStreamUpstream(t, body) + proxy, store, handler := streamProxy(t, u) + proxy.MetadataMaxSize = int64(len(body) / 4) + decodeStreamed(t, serve(handler, "/big", nil)) + + path := metadataStoragePath("npm", "big") + store.mu.Lock() + store.files[path] = []byte(strings.Replace(body, "1.0.5", "1.0.X", 1)) + store.mu.Unlock() + proxy.SetMetadataRewriteCacheSize(64 << 20) + + if doc := decodeStreamed(t, serve(handler, "/big", nil)); len(doc.Versions) != 400 { + t.Errorf("served %d versions, want 400", len(doc.Versions)) + } + if got := bigMetadataFiles(store)[path]; got != body { + t.Error("the corrupt copy was not replaced") + } +} + +// TestNPMStreamedPackumentTruncated checks that a document upstream cuts +// off halfway neither replaces the stored copy, which is served instead, +// nor leaves anything behind, with and without cooldown, streamed and in +// memory. +func TestNPMStreamedPackumentTruncated(t *testing.T) { + for _, tt := range []struct { + name string + cooldown, inMemory bool + }{ + {"cooldown", true, false}, + {"no cooldown", false, false}, + {"in memory with cooldown", true, true}, + {"in memory without cooldown", false, true}, + } { + t.Run(tt.name, func(t *testing.T) { + body := bigPackument(400) + u := newStreamUpstream(t, body) + proxy, store, handler := streamProxy(t, u) + if !tt.cooldown { + proxy.Cooldown = nil + } + if tt.inMemory { + proxy.SetMetadataRewriteCacheSize(0) + } else { + proxy.MetadataMaxSize = int64(len(body) / 4) + } + want := len(decodeStreamed(t, serve(handler, "/big", nil)).Versions) + before := bigMetadataFiles(store) + + u.replace(bigPackument(401), `"v2"`) + u.truncate.Store(true) + proxy.MetadataTTL = time.Nanosecond + if !tt.inMemory { + proxy.SetMetadataRewriteCacheSize(64 << 20) + } + + if doc := decodeStreamed(t, serve(handler, "/big", nil)); len(doc.Versions) != want { + t.Errorf("served %d versions, want the stored %d", len(doc.Versions), want) + } + after := bigMetadataFiles(store) + if len(after) != len(before) { + t.Errorf("storage holds %d files, %d before", len(after), len(before)) + } + for path, b := range before { + if after[path] != b { + t.Errorf("%s was changed", path) + } + } + }) + } +} + +// TestNPMStreamedPackumentTruncatedUncached checks that a document upstream +// cuts off halfway fails the request and is not kept, streamed and in memory. +func TestNPMStreamedPackumentTruncatedUncached(t *testing.T) { + for _, inMemory := range []bool{false, true} { + t.Run(map[bool]string{false: "streamed", true: "in memory"}[inMemory], func(t *testing.T) { + body := bigPackument(400) + u := newStreamUpstream(t, body) + proxy, store, handler := streamProxy(t, u) + proxy.Cooldown = nil + if inMemory { + proxy.SetMetadataRewriteCacheSize(0) + } else { + proxy.MetadataMaxSize = int64(len(body) / 4) + } + u.truncate.Store(true) + + if rec := serve(handler, "/big", nil); rec.Code != http.StatusBadGateway { + t.Errorf("status %d, want 502", rec.Code) + } + if files := bigMetadataFiles(store); len(files) != 0 { + t.Errorf("left %d files in storage", len(files)) + } + if entry, _ := proxy.DB.GetMetadataCache("npm", "big"); entry != nil { + t.Error("left a metadata cache row") + } + }) + } +} + +// TestNPMStreamedPackumentUncachedErrors checks the answers to upstream +// failures with nothing cached. +func TestNPMStreamedPackumentUncachedErrors(t *testing.T) { + for _, tt := range []struct { + name string + set func(*streamUpstream) + want int + }{ + {"not found", func(u *streamUpstream) { u.gone.Store(true) }, http.StatusNotFound}, + {"upstream down", func(u *streamUpstream) { u.fail.Store(true) }, http.StatusBadGateway}, + } { + t.Run(tt.name, func(t *testing.T) { + u := newStreamUpstream(t, bigPackument(10)) + _, store, handler := streamProxy(t, u) + tt.set(u) + if rec := serve(handler, "/big", nil); rec.Code != tt.want { + t.Errorf("status %d, want %d", rec.Code, tt.want) + } + if files := bigMetadataFiles(store); len(files) != 0 { + t.Errorf("left %d files in storage", len(files)) + } + }) + } +} + +// TestNPMStreamedPackumentConcurrent checks that concurrent requests for an +// uncached packument share one upstream fetch and all get it. +func TestNPMStreamedPackumentConcurrent(t *testing.T) { + body := bigPackument(400) + u := newStreamUpstream(t, body) + proxy, _, handler := streamProxy(t, u) + proxy.MetadataMaxSize = int64(len(body) / 4) + const n = 16 + var wg sync.WaitGroup + recs := make([]*httptest.ResponseRecorder, n) + for i := range n { + wg.Go(func() { recs[i] = serve(handler, "/big", nil) }) + } + wg.Wait() + for _, rec := range recs { + if doc := decodeStreamed(t, rec); len(doc.Versions) != 400 { + t.Errorf("served %d versions, want 400", len(doc.Versions)) + } + } + if got := u.sent.Load(); got != 1 { + t.Errorf("upstream sent the document %d times, want once", got) + } +} + +// TestNPMStreamedPackumentClientRevalidates checks that a client sending +// the ETag it got gets a 304, and a new ETag once the document changes. +func TestNPMStreamedPackumentClientRevalidates(t *testing.T) { + body := bigPackument(400) + u := newStreamUpstream(t, body) + proxy, _, handler := streamProxy(t, u) + proxy.MetadataMaxSize = int64(len(body) / 4) + + first := serve(handler, "/big", nil) + etag := first.Header().Get("ETag") + if first.Code != http.StatusOK || etag == "" { + t.Fatalf("status %d, ETag %q", first.Code, etag) + } + if rec := serve(handler, "/big", http.Header{"If-None-Match": {etag}}); rec.Code != http.StatusNotModified { + t.Errorf("same document: status %d, want 304", rec.Code) + } + + u.replace(bigPackument(401), `"v2"`) + proxy.MetadataTTL = time.Nanosecond + rec := serve(handler, "/big", http.Header{"If-None-Match": {etag}}) + if rec.Code != http.StatusOK || rec.Header().Get("ETag") == etag { + t.Errorf("changed document: status %d, ETag %q (was %q)", rec.Code, rec.Header().Get("ETag"), etag) + } + if doc := decodeStreamed(t, rec); len(doc.Versions) != 401 { + t.Errorf("served %d versions, want 401", len(doc.Versions)) + } +} diff --git a/internal/handler/npm_stream_test.go b/internal/handler/npm_stream_test.go index d63ab9b..a8a4ea2 100644 --- a/internal/handler/npm_stream_test.go +++ b/internal/handler/npm_stream_test.go @@ -49,10 +49,13 @@ func bigPackumentAt(base string, n int) string { type streamUpstream struct { *httptest.Server - body atomic.Value // string - etag atomic.Value // string - fail atomic.Bool - gone atomic.Bool + body atomic.Value // string + etag atomic.Value // string + fail atomic.Bool + gone atomic.Bool + // truncate makes upstream close the connection halfway through the + // document it announced. + truncate atomic.Bool requests atomic.Int32 // sent counts the responses that carried the document. sent atomic.Int32 @@ -80,9 +83,15 @@ func newStreamUpstream(t *testing.T, body string) *streamUpstream { w.WriteHeader(http.StatusNotModified) case r.URL.Path == "/big": u.sent.Add(1) + body := u.body.Load().(string) w.Header().Set("Content-Type", "application/json") w.Header().Set("ETag", u.etag.Load().(string)) - _, _ = io.WriteString(w, u.body.Load().(string)) + if u.truncate.Load() { + w.Header().Set("Content-Length", fmt.Sprint(len(body))) + _, _ = io.WriteString(w, body[:len(body)/2]) + return + } + _, _ = io.WriteString(w, body) default: http.NotFound(w, r) } @@ -98,7 +107,13 @@ func streamProxy(t *testing.T, u *streamUpstream) (*Proxy, *mockStorage, http.Ha proxy.CacheMetadata = true proxy.MetadataTTL = time.Hour proxy.SetMetadataRewriteCacheSize(64 << 20) - proxy.Cooldown = &cooldown.Config{Default: "3d"} + // As the server does: a cooldown.Config parses itself on first use, + // which concurrent requests would race on. + policy, err := cooldown.NewPolicy(&cooldown.Config{Default: "3d"}, nil) + if err != nil { + t.Fatal(err) + } + proxy.Cooldown = policy return proxy, store, NewNPMHandler(proxy, "http://proxy.example", u.URL).Routes() } diff --git a/internal/handler/read_metadata_test.go b/internal/handler/read_metadata_test.go index 639405c..d87d416 100644 --- a/internal/handler/read_metadata_test.go +++ b/internal/handler/read_metadata_test.go @@ -6,6 +6,7 @@ import ( "fmt" "io" "runtime" + "strings" "testing" "testing/iotest" ) @@ -131,3 +132,18 @@ func TestReadMetadataSizedAllocatesOnce(t *testing.T) { t.Errorf("allocated %d bytes reading %d", allocated, size) } } + +// TestReadMetadataSizedTruncatedUpstream checks that an upstream body cut +// off before its Content-Length fails, while a body shorter than a size +// hint is still read. +func TestReadMetadataSizedTruncatedUpstream(t *testing.T) { + p := &Proxy{} + cut := io.MultiReader(strings.NewReader(`{"name":`), iotest.ErrReader(io.ErrUnexpectedEOF)) + if _, err := p.ReadMetadataSized(upstreamBodyReader{cut}, 100); !errors.Is(err, io.ErrUnexpectedEOF) { + t.Errorf("cut off body: error %v, want io.ErrUnexpectedEOF", err) + } + got, err := p.ReadMetadataSized(strings.NewReader(`{"name":"x"}`), 100) + if err != nil || string(got) != `{"name":"x"}` { + t.Errorf("short body = %q, %v", got, err) + } +} From 83a018bfd4adb87e3713813b80ee4d0f58ff7619 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sat, 31 Oct 2026 18:09:05 +0100 Subject: [PATCH 524/633] Test that stats versions add up with orphan publish times Covers namespaced npm and Maven PURLs, a publish time under an alias of a package row's ecosystem, and a PURL starting with the orphan prefix, and checks the per-ecosystem versions against GetCacheStats' total. --- internal/database/analytics_test.go | 50 +++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/internal/database/analytics_test.go b/internal/database/analytics_test.go index e537694..5084acd 100644 --- a/internal/database/analytics_test.go +++ b/internal/database/analytics_test.go @@ -412,3 +412,53 @@ func TestGetEcosystemStatsAttributesPublishTimesByPURL(t *testing.T) { } }) } + +// TestGetEcosystemStatsVersionsReconcile checks that the versions counted +// per ecosystem add up to the cache stats' total, with publish times stored +// under namespaced PURLs, an alias of a package row's ecosystem, and a PURL +// that can't be parsed. +func TestGetEcosystemStatsVersionsReconcile(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + seedArtifact(t, db, "npm", "lodash", "4.17.21", 1000, 2) + seedArtifact(t, db, "go", "github.com/pkg/errors", "v0.9.1", 500, 1) + published := time.Now().Add(-time.Hour) + for _, v := range []struct{ version, pkg string }{ + {"pkg:npm/%40babel/core@7.25.2", "pkg:npm/%40babel/core"}, + {"pkg:maven/org.apache.commons/commons-lang3@3.14.0", "pkg:maven/org.apache.commons/commons-lang3"}, + {"pkg:pypi/requests@2.32.3", "pkg:pypi/requests"}, + {"pkg:golang/github.com/swaggo/swag@v1.16.4", "pkg:golang/github.com/swaggo/swag"}, + {"~odd@1", "~odd"}, + } { + if err := db.SetVersionPublishedAt(v.version, v.pkg, published); err != nil { + t.Fatal(err) + } + } + + stats, err := db.GetEcosystemStats() + if err != nil { + t.Fatalf("GetEcosystemStats: %v", err) + } + versions := map[string]int64{} + var sum int64 + for _, e := range stats { + versions[e.Ecosystem] += e.Versions + sum += e.Versions + } + want := map[string]int64{"npm": 2, "go": 2, "maven": 1, "pypi": 1, unattributedEcosystem: 1} + if len(versions) != len(want) { + t.Errorf("ecosystems = %v, want %v", versions, want) + } + for ecosystem, n := range want { + if versions[ecosystem] != n { + t.Errorf("versions of %s = %d, want %d (all: %v)", ecosystem, versions[ecosystem], n, versions) + } + } + cache, err := db.GetCacheStats() + if err != nil { + t.Fatal(err) + } + if sum != cache.TotalVersions { + t.Errorf("versions per ecosystem add up to %d, cache stats count %d", sum, cache.TotalVersions) + } + }) +} From fd16ba166464632104155d2cc270a26cef7e3dd8 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sat, 31 Oct 2026 18:15:11 +0100 Subject: [PATCH 525/633] Let in-flight blobs finish when one fails errgroup.WithContext cancelled the blobs being fetched as soon as one failed. A client pull sharing one of those fetches failed with it, and the next run had to fetch them again. A failure now only stops further blobs from starting. --- internal/handler/container_mirror.go | 18 ++++++++++++--- internal/handler/container_mirror_test.go | 27 +++++++++++++++++++++++ 2 files changed, 42 insertions(+), 3 deletions(-) diff --git a/internal/handler/container_mirror.go b/internal/handler/container_mirror.go index 82a98cf..c8ca65c 100644 --- a/internal/handler/container_mirror.go +++ b/internal/handler/container_mirror.go @@ -12,6 +12,7 @@ import ( "slices" "strings" "sync" + "sync/atomic" "time" "golang.org/x/sync/errgroup" @@ -462,15 +463,26 @@ func (m *imageMirror) mirrorBlobs(ctx context.Context, manifest *ociManifest) er } // Large layers take most of the time, so they are fetched side by side. + // A failure starts no further blobs but lets those in flight finish + // rather than cancelling them: a client pull may share their fetch, and + // they would have to be fetched again next run. var mu sync.Mutex - g, gctx := errgroup.WithContext(ctx) + var failed atomic.Bool + var g errgroup.Group g.SetLimit(mirrorBlobConcurrency) for _, digest := range digests { g.Go(func() error { - size, fetched, err := m.mirrorBlob(gctx, digest) - if err != nil || !fetched { + if failed.Load() { + return nil + } + size, fetched, err := m.mirrorBlob(ctx, digest) + if err != nil { + failed.Store(true) return err } + if !fetched { + return nil + } mu.Lock() m.result.Fetched++ m.result.Bytes += size diff --git a/internal/handler/container_mirror_test.go b/internal/handler/container_mirror_test.go index d2cb54b..2131928 100644 --- a/internal/handler/container_mirror_test.go +++ b/internal/handler/container_mirror_test.go @@ -546,3 +546,30 @@ func TestMirrorImageFetchesBlobsConcurrently(t *testing.T) { t.Errorf("fetched = %d, want the manifest and its two blobs", result.Fetched) } } + +// TestMirrorImageBlobFailureKeepsOtherFetches checks that a blob failing +// doesn't cancel the blobs already being fetched: their fetch may be shared +// with a client pull, which would fail with it, and the next run would have +// to fetch them again. +func TestMirrorImageBlobFailureKeepsOtherFetches(t *testing.T) { + registry, image := newFakeImageRegistry(t) + proxy := newMirrorTestProxy(t, registry) + h := NewContainerHandlerWithRegistry(proxy, "http://proxy.example", registry.server.URL) + + // The config blob is gone upstream; the layer is still being sent when + // its 404 arrives. + for path := range registry.blobs { + if !strings.HasSuffix(path, image.singleLayer) { + delete(registry.blobs, path) + } + } + registry.onBlob = func(string) { time.Sleep(300 * time.Millisecond) } + + if _, err := h.MirrorImage(context.Background(), ImageRef{Name: "team/app", References: []string{"single"}}, nil); err == nil { + t.Fatal("MirrorImage succeeded with a blob missing upstream") + } + registry.offline.Store(true) + if response := pull(t, h, "/team/app/blobs/"+image.singleLayer, ""); response.Code != http.StatusOK { + t.Errorf("layer fetched alongside the failing blob: status %d, want it cached", response.Code) + } +} From 3a0e79d6b4e5e94bc52ccf24bda18eaa8e347159 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sat, 31 Oct 2026 18:26:17 +0100 Subject: [PATCH 526/633] Fuzz SBOM parsing and test concurrent project uploads FuzzParse checks that an uploaded body never makes the inventory panic and that every component it records has a package URL, ecosystem and version. TestSaveProjectConcurrentUploads saves one project from eight uploads at once and checks that each succeeds and the components of exactly one remain. --- internal/database/projects_test.go | 52 ++++++++++++++++++++++++++++++ internal/inventory/fuzz_test.go | 28 ++++++++++++++++ 2 files changed, 80 insertions(+) create mode 100644 internal/inventory/fuzz_test.go diff --git a/internal/database/projects_test.go b/internal/database/projects_test.go index 2a5dded..bf19d44 100644 --- a/internal/database/projects_test.go +++ b/internal/database/projects_test.go @@ -3,6 +3,7 @@ package database import ( "database/sql" "slices" + "sync" "testing" "time" ) @@ -205,3 +206,54 @@ func TestMigrateAddsProjects(t *testing.T) { } }) } + +// TestSaveProjectConcurrentUploads checks that concurrent uploads of one +// project all succeed and leave the components of exactly one of them. +func TestSaveProjectConcurrentUploads(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + const uploads = 8 + sets := make([][]ProjectComponent, uploads) + for i := range sets { + for j := 0; j <= i; j++ { + sets[i] = append(sets[i], ProjectComponent{ + PackagePURL: "pkg:npm/dep" + string(rune('a'+j)), Ecosystem: "npm", + Name: "dep" + string(rune('a'+j)), Version: "1.0." + string(rune('0'+i)), + }) + } + } + now := time.Now().UTC() + errs := make(chan error, uploads) + var wg sync.WaitGroup + for i := range uploads { + wg.Go(func() { errs <- db.SaveProject("acme/web", "", "", now, sets[i]) }) + } + wg.Wait() + close(errs) + for err := range errs { + if err != nil { + t.Errorf("SaveProject: %v", err) + } + } + + project, err := db.GetProject("acme/web") + if err != nil || project == nil { + t.Fatalf("GetProject = %v, %v", project, err) + } + components, err := db.GetProjectComponents(project.ID) + if err != nil { + t.Fatal(err) + } + if len(components) == 0 || len(components) > uploads { + t.Fatalf("%d components, want those of one upload", len(components)) + } + version := components[0].Version + for _, c := range components { + if c.Version != version { + t.Fatalf("components mix uploads: %+v", components) + } + } + if want := int(version[len(version)-1]-'0') + 1; len(components) != want { + t.Errorf("%d components of the upload with version %s, want %d", len(components), version, want) + } + }) +} diff --git a/internal/inventory/fuzz_test.go b/internal/inventory/fuzz_test.go new file mode 100644 index 0000000..f7fcadd --- /dev/null +++ b/internal/inventory/fuzz_test.go @@ -0,0 +1,28 @@ +package inventory + +import ( + "os" + "testing" +) + +// FuzzParse checks that no uploaded body makes the SBOM parsing panic. +func FuzzParse(f *testing.F) { + if data, err := os.ReadFile("testdata/syft-cyclonedx.json"); err == nil { + f.Add(data) + } + f.Add([]byte(`{"bomFormat":"CycloneDX","specVersion":"1.5","components":[{"type":"library","name":"x","version":"1","purl":"pkg:npm/x@1"}]}`)) + f.Add([]byte(`{"spdxVersion":"SPDX-2.3","SPDXID":"SPDXRef-DOCUMENT","packages":[{"SPDXID":"SPDXRef-a","name":"a","versionInfo":"1","externalRefs":[{"referenceCategory":"PACKAGE-MANAGER","referenceType":"purl","referenceLocator":"pkg:conda/a@1?channel=c"}]}]}`)) + f.Add([]byte("SPDXVersion: SPDX-2.3\nPackageName: a\nSPDXID: SPDXRef-a\nExternalRef: PACKAGE-MANAGER purl pkg:deb/debian/a@1\n")) + f.Add([]byte(`xpkg:pypi/X_Y@1`)) + f.Fuzz(func(t *testing.T, data []byte) { + components, _, err := Parse(data) + if err != nil { + return + } + for _, c := range components { + if c.PackagePURL == "" || c.Version == "" || c.Ecosystem == "" { + t.Errorf("component with an empty field: %+v", c) + } + } + }) +} From 15789d40bb47556b05c2e142c3923d5290c3544f Mon Sep 17 00:00:00 2001 From: simon chrzanowski Date: Sat, 31 Oct 2026 21:14:53 +0100 Subject: [PATCH 527/633] Update golang.org/x/net to v0.62.0 (#647) Fixes CVE-2026-78661 and CVE-2026-97045 in x/net/html, which parses upstream PyPI simple pages for the denylist. --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 2ce65e2..db6695b 100644 --- a/go.mod +++ b/go.mod @@ -31,7 +31,7 @@ require ( github.com/vmihailenco/msgpack/v5 v5.4.1 gocloud.dev v0.47.0 golang.org/x/mod v0.42.0 - golang.org/x/net v0.61.0 + golang.org/x/net v0.62.0 golang.org/x/sync v0.24.0 google.golang.org/protobuf v1.36.12 gopkg.in/yaml.v3 v3.0.1 diff --git a/go.sum b/go.sum index c322161..d1c0563 100644 --- a/go.sum +++ b/go.sum @@ -312,8 +312,8 @@ golang.org/x/crypto v0.58.0/go.mod h1:Xh+kl5A+M0gMiWDytH2Hhr5Z7WbNCpv/eB/EFgfSk+ golang.org/x/mod v0.42.0 h1:ICeL3t2L30ljSPHHxDnR4fqVcBt/zR2zJAb9juavmQg= golang.org/x/mod v0.42.0/go.mod h1:u5fzIrRInkL8MaO1cbRkUHT64v303YIT5KEqpNr7jRs= golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1Kcs5dz7/ng1VjMUvfKvpfy+jM= -golang.org/x/net v0.61.0 h1:RR+6j/BTBCrggOLugxa+66R8FvoN6MHlzDlZIbN9YFw= -golang.org/x/net v0.61.0/go.mod h1:WgDCOTH8iwtB66T4/GfI94AHo/hhdb7Hah9NDseQfiI= +golang.org/x/net v0.62.0 h1:18ELalfM5+XNRwzM2nYqXsZQ9ByS1uhj2O/U7+5jqRY= +golang.org/x/net v0.62.0/go.mod h1:WgDCOTH8iwtB66T4/GfI94AHo/hhdb7Hah9NDseQfiI= golang.org/x/oauth2 v0.37.0 h1:JUlcxA8oAtauLfiH8FX2/FkAWHAdi0QtGCGc+hofE98= golang.org/x/oauth2 v0.37.0/go.mod h1:IxwZNxUULJmpBFf9K/9NTMSIfZZuvuTy1gGxhigP/58= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= From 358b1355a9704ac61dffd3c9600a0f4f74b03f6a Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 1 Nov 2026 12:48:04 +0100 Subject: [PATCH 528/633] Fail metadata reads when upstream cuts a document off ReadMetadataSized took the transport's io.ErrUnexpectedEOF for the end of a body shorter than its size hint. A document upstream closed before its Content-Length was served with a 200 and cached, or failed the cooldown filter with a 502, instead of the cached copy being served. Upstream bodies are now read through a reader that marks their read errors, and such a body fails the read as an upstream failure. --- internal/handler/handler.go | 9 +- internal/handler/metadata_truncated_test.go | 121 ++++++++++++++++++++ internal/handler/read_metadata_test.go | 16 +++ internal/handler/upstream_body.go | 26 +++++ 4 files changed, 170 insertions(+), 2 deletions(-) create mode 100644 internal/handler/metadata_truncated_test.go create mode 100644 internal/handler/upstream_body.go diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 4b7ebe7..f3fdcbb 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -157,7 +157,9 @@ func (p *Proxy) ReadMetadata(r io.Reader) ([]byte, error) { // Reading into a buffer of that size avoids io.ReadAll growing its buffer // step by step, which peaks at more than twice the document size. size is // only a hint: a negative one, or one above the limit, reads as ReadMetadata -// does, and a body of a different length is still read correctly. +// does, and a body of a different length is still read correctly. A body +// read through upstreamBodyReader that upstream cuts off fails, even though +// it ends like a body shorter than size. func (p *Proxy) ReadMetadataSized(r io.Reader, size int64) ([]byte, error) { limit := p.MetadataMaxSize if limit <= 0 { @@ -170,7 +172,10 @@ func (p *Proxy) ReadMetadataSized(r io.Reader, size int64) ([]byte, error) { // the buffer and a longer one is noticed. buf := make([]byte, size+1) n, err := io.ReadFull(r, buf) + var cut *upstreamBodyError switch { + case errors.As(err, &cut): + return nil, err case errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF): if int64(n) < size/2 { // The hint was far off; don't keep the oversized buffer alive. @@ -2010,7 +2015,7 @@ func (p *Proxy) fetchUpstreamMetadata(ctx context.Context, upstreamURL string, e // ContentLength is -1 when the transport decompresses the body, and // otherwise the length of the bytes read here. - body, err := p.ReadMetadataSized(resp.Body, resp.ContentLength) + body, err := p.ReadMetadataSized(upstreamBodyReader{resp.Body}, resp.ContentLength) if err != nil { return nil, fmt.Errorf("reading response: %w", err) } diff --git a/internal/handler/metadata_truncated_test.go b/internal/handler/metadata_truncated_test.go new file mode 100644 index 0000000..5044374 --- /dev/null +++ b/internal/handler/metadata_truncated_test.go @@ -0,0 +1,121 @@ +package handler + +import ( + "fmt" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + + "github.com/git-pkgs/cooldown" +) + +// cutMetadataUpstream serves one npm packument and, when cut is set, closes +// the connection halfway through the document it announced. +type cutMetadataUpstream struct { + *httptest.Server + mu sync.Mutex + body string + cut bool +} + +func newCutMetadataUpstream(t *testing.T, body string) *cutMetadataUpstream { + t.Helper() + u := &cutMetadataUpstream{body: body} + u.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + u.mu.Lock() + body, cut := u.body, u.cut + u.mu.Unlock() + if r.URL.Path != "/pkg" { + http.NotFound(w, r) + return + } + w.Header().Set("Content-Type", "application/json") + w.Header().Set("Content-Length", fmt.Sprint(len(body))) + if cut { + _, _ = io.WriteString(w, body[:len(body)/2]) + return + } + _, _ = io.WriteString(w, body) + })) + t.Cleanup(u.Close) + return u +} + +func (u *cutMetadataUpstream) set(body string, cut bool) { + u.mu.Lock() + defer u.mu.Unlock() + u.body, u.cut = body, cut +} + +// truncPackument is an npm packument with n versions published long ago. +func truncPackument(n int) string { + var versions, times []string + for i := range n { + v := fmt.Sprintf("1.0.%d", i) + versions = append(versions, fmt.Sprintf(`%q:{"name":"pkg","version":%q,"dist":{"tarball":"https://registry.npmjs.org/pkg/-/pkg-%s.tgz"}}`, v, v, v)) + times = append(times, fmt.Sprintf(`%q:"2020-01-01T00:00:00Z"`, v)) + } + return `{"name":"pkg","dist-tags":{"latest":"1.0.0"},"versions":{` + strings.Join(versions, ",") + + `},"time":{` + strings.Join(times, ",") + `}}` +} + +// TestMetadataTruncatedUpstream checks that a document upstream cuts off +// before its Content-Length is neither served nor cached: the cached copy is +// served instead, with and without cooldown. +func TestMetadataTruncatedUpstream(t *testing.T) { + for _, withCooldown := range []bool{false, true} { + t.Run(map[bool]string{false: "no cooldown", true: "cooldown"}[withCooldown], func(t *testing.T) { + u := newCutMetadataUpstream(t, truncPackument(50)) + proxy, db, _, _ := setupTestProxy(t) + proxy.HTTPClient = u.Client() + proxy.CacheMetadata = true + if withCooldown { + proxy.Cooldown = &cooldown.Config{Default: "3d"} + } + handler := NewNPMHandler(proxy, "http://proxy.example", u.URL).Routes() + + first := serve(handler, "/pkg", nil) + if first.Code != http.StatusOK { + t.Fatalf("first request: status %d", first.Code) + } + before, err := db.GetMetadataCache("npm", "pkg") + if err != nil || before == nil { + t.Fatalf("no metadata row after the first request: %v", err) + } + + u.set(truncPackument(51), true) + expireMetadata(t, db, "npm", "pkg") + proxy.SetMetadataRewriteCacheSize(0) + + rec := serve(handler, "/pkg", nil) + if rec.Code != http.StatusOK || rec.Body.String() != first.Body.String() { + t.Errorf("status %d, want 200 with the cached document", rec.Code) + } + after, err := db.GetMetadataCache("npm", "pkg") + if err != nil || after == nil || after.ContentDigest != before.ContentDigest { + t.Error("the cut-off document replaced the cached copy") + } + }) + } +} + +// TestMetadataTruncatedUpstreamUncached checks that a cut-off document with +// nothing cached fails the request and is not cached. +func TestMetadataTruncatedUpstreamUncached(t *testing.T) { + u := newCutMetadataUpstream(t, truncPackument(50)) + u.set(truncPackument(50), true) + proxy, db, _, _ := setupTestProxy(t) + proxy.HTTPClient = u.Client() + proxy.CacheMetadata = true + handler := NewNPMHandler(proxy, "http://proxy.example", u.URL).Routes() + + if rec := serve(handler, "/pkg", nil); rec.Code != http.StatusBadGateway { + t.Errorf("status %d, want 502", rec.Code) + } + if entry, _ := db.GetMetadataCache("npm", "pkg"); entry != nil { + t.Error("cached the cut-off document") + } +} diff --git a/internal/handler/read_metadata_test.go b/internal/handler/read_metadata_test.go index 639405c..d87d416 100644 --- a/internal/handler/read_metadata_test.go +++ b/internal/handler/read_metadata_test.go @@ -6,6 +6,7 @@ import ( "fmt" "io" "runtime" + "strings" "testing" "testing/iotest" ) @@ -131,3 +132,18 @@ func TestReadMetadataSizedAllocatesOnce(t *testing.T) { t.Errorf("allocated %d bytes reading %d", allocated, size) } } + +// TestReadMetadataSizedTruncatedUpstream checks that an upstream body cut +// off before its Content-Length fails, while a body shorter than a size +// hint is still read. +func TestReadMetadataSizedTruncatedUpstream(t *testing.T) { + p := &Proxy{} + cut := io.MultiReader(strings.NewReader(`{"name":`), iotest.ErrReader(io.ErrUnexpectedEOF)) + if _, err := p.ReadMetadataSized(upstreamBodyReader{cut}, 100); !errors.Is(err, io.ErrUnexpectedEOF) { + t.Errorf("cut off body: error %v, want io.ErrUnexpectedEOF", err) + } + got, err := p.ReadMetadataSized(strings.NewReader(`{"name":"x"}`), 100) + if err != nil || string(got) != `{"name":"x"}` { + t.Errorf("short body = %q, %v", got, err) + } +} diff --git a/internal/handler/upstream_body.go b/internal/handler/upstream_body.go new file mode 100644 index 0000000..7db9076 --- /dev/null +++ b/internal/handler/upstream_body.go @@ -0,0 +1,26 @@ +package handler + +import ( + "errors" + "io" +) + +// upstreamBodyReader marks a failure to read a document from upstream, so a +// body upstream cuts off is told apart from one that ends, and from a +// failure to store it. +type upstreamBodyReader struct{ r io.Reader } + +func (u upstreamBodyReader) Read(b []byte) (int, error) { + n, err := u.r.Read(b) + if err != nil && !errors.Is(err, io.EOF) { + err = &upstreamBodyError{err} + } + return n, err +} + +// upstreamBodyError is a failure to read a document from upstream. It is +// handled as an upstream failure: the stored copy, if any, is served. +type upstreamBodyError struct{ err error } + +func (e *upstreamBodyError) Error() string { return "reading upstream metadata: " + e.err.Error() } +func (e *upstreamBodyError) Unwrap() error { return e.err } From 143eef7f05d6b356e604d6a88bd89c2c7554f5e6 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 1 Nov 2026 12:48:37 +0100 Subject: [PATCH 529/633] Test snapshots on cache hits and record them through proxyCachedWithEncoding TestAPKSnapshots_CacheHit covers the cache-hit branch with a metadata TTL above 0. proxyCachedWithEncoding takes optional hooks for a cache hit and a fetched body, so the APK handler no longer keeps a copy of it. --- internal/database/metadata_snapshots.go | 16 +++----- internal/handler/apk_snapshots.go | 34 ++++----------- internal/handler/apk_snapshots_test.go | 41 +++++++++++++++++++ internal/handler/conda.go | 2 +- internal/handler/handler.go | 25 +++++++++-- internal/handler/homebrew.go | 2 +- .../handler/proxy_cached_encoding_test.go | 6 +-- 7 files changed, 81 insertions(+), 45 deletions(-) diff --git a/internal/database/metadata_snapshots.go b/internal/database/metadata_snapshots.go index 4df1fa1..268baf7 100644 --- a/internal/database/metadata_snapshots.go +++ b/internal/database/metadata_snapshots.go @@ -61,17 +61,11 @@ func (db *DB) metadataSnapshotBefore(ecosystem, name string, beforeMS int64) (*M } // AddMetadataSnapshot records snap as the version of its document current -// from snap.ValidFromMS, unless the snapshot current at that time already -// has its digest, and reports whether it did. Comparing with the version -// current at that time rather than the newest one keeps a version a request -// recorded late, after a newer one, in its place in the history. On -// Postgres, replicas recording the same document take an advisory lock on it -// first, so their check and insert don't interleave: without it, two -// transactions can both read the previous digest and both insert. SQLite -// runs one write at a time, so the single statement is enough there. A -// document going back to an earlier digest is recorded again, since only the -// snapshot current at that time is compared. The casts let Postgres type the -// parameters of the INSERT ... SELECT. +// from snap.ValidFromMS, unless the snapshot current at that time already has +// its digest, and reports whether it did. On Postgres it first takes an +// advisory lock on the document, so two replicas can't both read the previous +// digest and both insert. The casts let Postgres type the parameters of the +// INSERT ... SELECT. func (db *DB) AddMetadataSnapshot(snap *MetadataSnapshot) (bool, error) { tx, err := db.Beginx() if err != nil { diff --git a/internal/handler/apk_snapshots.go b/internal/handler/apk_snapshots.go index d581980..401dc00 100644 --- a/internal/handler/apk_snapshots.go +++ b/internal/handler/apk_snapshots.go @@ -105,35 +105,17 @@ func (h *APKHandler) serveSnapshot(w http.ResponseWriter, r *http.Request, repos } } -// serveMetadataWithSnapshot is ProxyCached for an index or signature that -// keeps each version it serves as a snapshot. A version is recorded from the -// document the request serves, not by reading the cache again afterwards: a -// concurrent refresh can replace the cached copy at any time, and the version -// it replaced would then never be recorded. +// serveMetadataWithSnapshot is ProxyCached, recording each version it serves. func (h *APKHandler) serveMetadataWithSnapshot(w http.ResponseWriter, r *http.Request, upstream, cacheKey string) { - p := h.proxy ctx := context.WithoutCancel(r.Context()) - - // A cached copy within its TTL was recorded by the request that fetched - // it; this only catches one stored before snapshots were enabled, or - // whose recording failed. - if entry := p.freshMetadataEntry(apkEcosystem, cacheKey); entry != nil { - seen := time.Now() - if p.serveCachedMetadata(w, r, apkEcosystem, cacheKey) { + h.proxy.proxyCachedWithEncoding(w, r, upstream, apkEcosystem, cacheKey, "identity", &metadataHooks{ + hit: func(entry *database.MetadataCacheEntry, seen time.Time) { h.recordStoredSnapshot(ctx, cacheKey, entry, seen) - return - } - } - - body, contentType, contentEncoding, err := p.fetchOrCacheMetadata(r.Context(), apkEcosystem, cacheKey, upstream, "identity", nil, "*/*") - if err != nil { - p.writeMetadataFetchError(w, err) - return - } - // Record before responding, so the version is kept by the time a - // client has it. - h.recordServedSnapshot(ctx, cacheKey, body, contentType, contentEncoding, time.Now()) - p.writeMetadataCachedResponseWithEncoding(w, r, apkEcosystem, cacheKey, body, contentType, contentEncoding) + }, + fetched: func(body []byte, contentType, contentEncoding string) { + h.recordServedSnapshot(ctx, cacheKey, body, contentType, contentEncoding, time.Now()) + }, + }, "*/*") } // recordServedSnapshot keeps body, the document a request is serving, as the diff --git a/internal/handler/apk_snapshots_test.go b/internal/handler/apk_snapshots_test.go index b0f109d..83bb26d 100644 --- a/internal/handler/apk_snapshots_test.go +++ b/internal/handler/apk_snapshots_test.go @@ -339,3 +339,44 @@ func TestAPKSnapshots_ConcurrentReplicas(t *testing.T) { } } } + +// TestAPKSnapshots_CacheHit records an index cached before snapshots were +// enabled when a request is served from the cache within its TTL, and adds +// no row for a later hit on the same version. +func TestAPKSnapshots_CacheHit(t *testing.T) { + u := newAPKSnapshotUpstream(t) + h, db := newSnapshotAPKHandler(t, u.server.URL) + h.proxy.MetadataTTL = 5 * time.Minute + h.WithSnapshots(0) + expectAPKResponse(t, h, "/alpine"+apkSnapshotIndexPath, http.StatusOK, "index v1") + if n := snapshotCount(t, db); n != 0 { + t.Fatalf("%d snapshots recorded with snapshots off, want 0", n) + } + + h.WithSnapshots(30 * 24 * time.Hour) + // Upstream moved on; a hit still serves the cached v1. + u.index.Store("index v2") + expectAPKResponse(t, h, "/alpine"+apkSnapshotIndexPath, http.StatusOK, "index v1") + if n := snapshotCount(t, db); n != 1 { + t.Fatalf("%d snapshots recorded on a cache hit, want 1", n) + } + key := h.metadataCacheKey("alpine", u.server.URL, apkSnapshotIndexPath[1:]) + snap, err := db.LatestMetadataSnapshot(apkEcosystem, key) + if err != nil || snap == nil { + t.Fatalf("latest snapshot = %v, %v", snap, err) + } + body, err := h.proxy.Storage.Open(t.Context(), snap.StoragePath) + if err != nil { + t.Fatalf("recorded snapshot blob unreadable: %v", err) + } + got, _ := io.ReadAll(body) + _ = body.Close() + if string(got) != "index v1" { + t.Fatalf("snapshot blob = %q, want %q", got, "index v1") + } + + expectAPKResponse(t, h, "/alpine"+apkSnapshotIndexPath, http.StatusOK, "index v1") + if n := snapshotCount(t, db); n != 1 { + t.Fatalf("%d snapshots after a second hit on the same version, want 1", n) + } +} diff --git a/internal/handler/conda.go b/internal/handler/conda.go index 3cbaf93..ca8fd2f 100644 --- a/internal/handler/conda.go +++ b/internal/handler/conda.go @@ -540,7 +540,7 @@ func (h *CondaHandler) proxyCached(w http.ResponseWriter, r *http.Request) { if strings.HasSuffix(r.URL.Path, ".json") { acceptEncoding = "gzip" } - h.proxy.proxyCachedWithEncoding(w, r, h.upstreamURL+r.URL.Path, "conda", cacheKey, acceptEncoding, "*/*") + h.proxy.proxyCachedWithEncoding(w, r, h.upstreamURL+r.URL.Path, "conda", cacheKey, acceptEncoding, nil, "*/*") } // proxyUpstream forwards a request to Anaconda without caching. diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 4b7ebe7..7fef62e 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -2151,7 +2151,7 @@ func (p *Proxy) cachedMetaFor(entry *database.MetadataCacheEntry) cachedMeta { // When metadata caching is disabled, the response is streamed directly to avoid buffering // large metadata responses (e.g. npm packages with many versions) in memory. func (p *Proxy) ProxyCached(w http.ResponseWriter, r *http.Request, upstreamURL, ecosystem, cacheKey string, acceptHeaders ...string) { - p.proxyCachedWithEncoding(w, r, upstreamURL, ecosystem, cacheKey, "identity", acceptHeaders...) + p.proxyCachedWithEncoding(w, r, upstreamURL, ecosystem, cacheKey, "identity", nil, acceptHeaders...) } // proxyCachedWithEncoding is ProxyCached with an explicit upstream Accept-Encoding. @@ -2159,14 +2159,30 @@ func (p *Proxy) ProxyCached(w http.ResponseWriter, r *http.Request, upstreamURL, // compressed for large, non-hash-pinned metadata whose clients decode gzip // (Homebrew API and Conda repodata). The stored bytes and Content-Encoding are // replayed verbatim either way. -func (p *Proxy) proxyCachedWithEncoding(w http.ResponseWriter, r *http.Request, upstreamURL, ecosystem, cacheKey, acceptEncoding string, acceptHeaders ...string) { +// metadataHooks are optional callbacks for proxyCachedWithEncoding: hit runs +// after a fresh cached row is served, with that row and when it was read, and +// fetched runs with a fetched document before it is written. +type metadataHooks struct { + hit func(entry *database.MetadataCacheEntry, seen time.Time) + fetched func(body []byte, contentType, contentEncoding string) +} + +func (p *Proxy) proxyCachedWithEncoding(w http.ResponseWriter, r *http.Request, upstreamURL, ecosystem, cacheKey, acceptEncoding string, hooks *metadataHooks, acceptHeaders ...string) { if !p.CacheMetadata { // Stream directly without buffering when caching is off. p.proxyMetadataStream(w, r, upstreamURL, acceptEncoding, acceptHeaders...) return } - if p.serveCachedMetadata(w, r, ecosystem, cacheKey) { + if hooks != nil && hooks.hit != nil { + if entry := p.freshMetadataEntry(ecosystem, cacheKey); entry != nil { + seen := time.Now() + if p.serveCachedMetadata(w, r, ecosystem, cacheKey) { + hooks.hit(entry, seen) + return + } + } + } else if p.serveCachedMetadata(w, r, ecosystem, cacheKey) { return } @@ -2176,6 +2192,9 @@ func (p *Proxy) proxyCachedWithEncoding(w http.ResponseWriter, r *http.Request, return } + if hooks != nil && hooks.fetched != nil { + hooks.fetched(body, contentType, contentEncoding) + } p.writeMetadataCachedResponseWithEncoding(w, r, ecosystem, cacheKey, body, contentType, contentEncoding) } diff --git a/internal/handler/homebrew.go b/internal/handler/homebrew.go index 0af67af..4e40d02 100644 --- a/internal/handler/homebrew.go +++ b/internal/handler/homebrew.go @@ -64,7 +64,7 @@ func (h *HomebrewHandler) Routes() http.Handler { if strings.HasPrefix(requestPath, "analytics/") { acceptEncoding = "identity" } - h.proxy.proxyCachedWithEncoding(w, r, upstreamURL, homebrewMetadataEcosystem, homebrewMetadataCacheKey(requestPath, r.URL.RawQuery), acceptEncoding, "*/*") + h.proxy.proxyCachedWithEncoding(w, r, upstreamURL, homebrewMetadataEcosystem, homebrewMetadataCacheKey(requestPath, r.URL.RawQuery), acceptEncoding, nil, "*/*") }) } diff --git a/internal/handler/proxy_cached_encoding_test.go b/internal/handler/proxy_cached_encoding_test.go index 35c8793..45eaef3 100644 --- a/internal/handler/proxy_cached_encoding_test.go +++ b/internal/handler/proxy_cached_encoding_test.go @@ -56,7 +56,7 @@ func (u *gzipWhenAskedUpstream) sawAcceptEncoding() string { func serveGzip(proxy *Proxy, upstreamURL string) *httptest.ResponseRecorder { w := httptest.NewRecorder() r := httptest.NewRequest(http.MethodGet, "/index.json", nil) - proxy.proxyCachedWithEncoding(w, r, upstreamURL, "gzip-test", "index", "gzip", "*/*") + proxy.proxyCachedWithEncoding(w, r, upstreamURL, "gzip-test", "index", "gzip", nil, "*/*") return w } @@ -187,7 +187,7 @@ func TestProxyCachedWithEncoding_UpsertFailureDiscardsBlob(t *testing.T) { // Seed an identity row + plain blob, as every key has before the gzip rollout. w := httptest.NewRecorder() proxy.proxyCachedWithEncoding(w, httptest.NewRequest(http.MethodGet, "/index.json", nil), - upstream.URL+"/index.json", "gzip-test", "index", "identity", "*/*") + upstream.URL+"/index.json", "gzip-test", "index", "identity", nil, "*/*") if w.Code != http.StatusOK { t.Fatalf("seed status = %d, want 200", w.Code) } @@ -250,7 +250,7 @@ func TestProxyCachedWithEncoding_StaleFallbackRereadsRow(t *testing.T) { w := httptest.NewRecorder() proxy.proxyCachedWithEncoding(w, httptest.NewRequest(http.MethodGet, "/index.json", nil), - upstream.URL+"/index.json", "gzip-test", "index", "identity", "*/*") + upstream.URL+"/index.json", "gzip-test", "index", "identity", nil, "*/*") if w.Code != http.StatusOK { t.Fatalf("seed status = %d, want 200", w.Code) } From 6dbda21faf5092e663029d5fcb8c59ee5adb4654 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 1 Nov 2026 12:49:09 +0100 Subject: [PATCH 530/633] Move the in-memory truncation fix to its own pull request The change to ReadMetadataSized and fetchUpstreamMetadata, with its tests, is now #649. upstreamBodyReader stays for storeUpstreamMetadata, in a file of its own that #649 adds with the same content. --- internal/handler/handler.go | 9 +-- internal/handler/metadata_stored.go | 19 ------- internal/handler/npm_stream_harden_test.go | 65 +++++++--------------- internal/handler/read_metadata_test.go | 16 ------ internal/handler/upstream_body.go | 26 +++++++++ 5 files changed, 49 insertions(+), 86 deletions(-) create mode 100644 internal/handler/upstream_body.go diff --git a/internal/handler/handler.go b/internal/handler/handler.go index e0cede7..96ae655 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -157,9 +157,7 @@ func (p *Proxy) ReadMetadata(r io.Reader) ([]byte, error) { // Reading into a buffer of that size avoids io.ReadAll growing its buffer // step by step, which peaks at more than twice the document size. size is // only a hint: a negative one, or one above the limit, reads as ReadMetadata -// does, and a body of a different length is still read correctly. A body -// read through upstreamBodyReader that upstream cuts off fails, even though -// it ends like a body shorter than size. +// does, and a body of a different length is still read correctly. func (p *Proxy) ReadMetadataSized(r io.Reader, size int64) ([]byte, error) { limit := p.MetadataMaxSize if limit <= 0 { @@ -172,10 +170,7 @@ func (p *Proxy) ReadMetadataSized(r io.Reader, size int64) ([]byte, error) { // the buffer and a longer one is noticed. buf := make([]byte, size+1) n, err := io.ReadFull(r, buf) - var cut *upstreamBodyError switch { - case errors.As(err, &cut): - return nil, err case errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF): if int64(n) < size/2 { // The hint was far off; don't keep the oversized buffer alive. @@ -1994,7 +1989,7 @@ func (p *Proxy) fetchUpstreamMetadata(ctx context.Context, upstreamURL string, e // ContentLength is -1 when the transport decompresses the body, and // otherwise the length of the bytes read here. - body, err := p.ReadMetadataSized(upstreamBodyReader{resp.Body}, resp.ContentLength) + body, err := p.ReadMetadataSized(resp.Body, resp.ContentLength) if err != nil { return nil, fmt.Errorf("reading response: %w", err) } diff --git a/internal/handler/metadata_stored.go b/internal/handler/metadata_stored.go index 8bbf3eb..b3bc420 100644 --- a/internal/handler/metadata_stored.go +++ b/internal/handler/metadata_stored.go @@ -246,25 +246,6 @@ func (p *Proxy) stageMetadata(ctx context.Context, ecosystem, cacheKey string, b return &stagedReader{ReadCloser: staged, discard: discard}, nil } -// upstreamBodyReader marks a failure to read a document from upstream, so -// it is told apart from a failure to store it. -type upstreamBodyReader struct{ r io.Reader } - -func (u upstreamBodyReader) Read(b []byte) (int, error) { - n, err := u.r.Read(b) - if err != nil && !errors.Is(err, io.EOF) { - err = &upstreamBodyError{err} - } - return n, err -} - -// upstreamBodyError is a failure to read a document from upstream. It is -// handled as an upstream failure: the stored copy, if any, is served. -type upstreamBodyError struct{ err error } - -func (e *upstreamBodyError) Error() string { return "reading upstream metadata: " + e.err.Error() } -func (e *upstreamBodyError) Unwrap() error { return e.err } - // notStored returns a failure to store a document as errMetadataNotStored, // unless reading it from upstream failed. func notStored(what string, err error) error { diff --git a/internal/handler/npm_stream_harden_test.go b/internal/handler/npm_stream_harden_test.go index ff8f97e..ca7143f 100644 --- a/internal/handler/npm_stream_harden_test.go +++ b/internal/handler/npm_stream_harden_test.go @@ -49,39 +49,24 @@ func TestNPMStreamedPackumentBlobCorrupt(t *testing.T) { // TestNPMStreamedPackumentTruncated checks that a document upstream cuts // off halfway neither replaces the stored copy, which is served instead, -// nor leaves anything behind, with and without cooldown, streamed and in -// memory. +// nor leaves anything behind, with and without cooldown. func TestNPMStreamedPackumentTruncated(t *testing.T) { - for _, tt := range []struct { - name string - cooldown, inMemory bool - }{ - {"cooldown", true, false}, - {"no cooldown", false, false}, - {"in memory with cooldown", true, true}, - {"in memory without cooldown", false, true}, - } { - t.Run(tt.name, func(t *testing.T) { + for _, withCooldown := range []bool{true, false} { + t.Run(map[bool]string{true: "cooldown", false: "no cooldown"}[withCooldown], func(t *testing.T) { body := bigPackument(400) u := newStreamUpstream(t, body) proxy, store, handler := streamProxy(t, u) - if !tt.cooldown { + if !withCooldown { proxy.Cooldown = nil } - if tt.inMemory { - proxy.SetMetadataRewriteCacheSize(0) - } else { - proxy.MetadataMaxSize = int64(len(body) / 4) - } + proxy.MetadataMaxSize = int64(len(body) / 4) want := len(decodeStreamed(t, serve(handler, "/big", nil)).Versions) before := bigMetadataFiles(store) u.replace(bigPackument(401), `"v2"`) u.truncate.Store(true) proxy.MetadataTTL = time.Nanosecond - if !tt.inMemory { - proxy.SetMetadataRewriteCacheSize(64 << 20) - } + proxy.SetMetadataRewriteCacheSize(64 << 20) if doc := decodeStreamed(t, serve(handler, "/big", nil)); len(doc.Versions) != want { t.Errorf("served %d versions, want the stored %d", len(doc.Versions), want) @@ -100,31 +85,23 @@ func TestNPMStreamedPackumentTruncated(t *testing.T) { } // TestNPMStreamedPackumentTruncatedUncached checks that a document upstream -// cuts off halfway fails the request and is not kept, streamed and in memory. +// cuts off halfway fails the request and is not kept. func TestNPMStreamedPackumentTruncatedUncached(t *testing.T) { - for _, inMemory := range []bool{false, true} { - t.Run(map[bool]string{false: "streamed", true: "in memory"}[inMemory], func(t *testing.T) { - body := bigPackument(400) - u := newStreamUpstream(t, body) - proxy, store, handler := streamProxy(t, u) - proxy.Cooldown = nil - if inMemory { - proxy.SetMetadataRewriteCacheSize(0) - } else { - proxy.MetadataMaxSize = int64(len(body) / 4) - } - u.truncate.Store(true) + body := bigPackument(400) + u := newStreamUpstream(t, body) + proxy, store, handler := streamProxy(t, u) + proxy.Cooldown = nil + proxy.MetadataMaxSize = int64(len(body) / 4) + u.truncate.Store(true) - if rec := serve(handler, "/big", nil); rec.Code != http.StatusBadGateway { - t.Errorf("status %d, want 502", rec.Code) - } - if files := bigMetadataFiles(store); len(files) != 0 { - t.Errorf("left %d files in storage", len(files)) - } - if entry, _ := proxy.DB.GetMetadataCache("npm", "big"); entry != nil { - t.Error("left a metadata cache row") - } - }) + if rec := serve(handler, "/big", nil); rec.Code != http.StatusBadGateway { + t.Errorf("status %d, want 502", rec.Code) + } + if files := bigMetadataFiles(store); len(files) != 0 { + t.Errorf("left %d files in storage", len(files)) + } + if entry, _ := proxy.DB.GetMetadataCache("npm", "big"); entry != nil { + t.Error("left a metadata cache row") } } diff --git a/internal/handler/read_metadata_test.go b/internal/handler/read_metadata_test.go index d87d416..639405c 100644 --- a/internal/handler/read_metadata_test.go +++ b/internal/handler/read_metadata_test.go @@ -6,7 +6,6 @@ import ( "fmt" "io" "runtime" - "strings" "testing" "testing/iotest" ) @@ -132,18 +131,3 @@ func TestReadMetadataSizedAllocatesOnce(t *testing.T) { t.Errorf("allocated %d bytes reading %d", allocated, size) } } - -// TestReadMetadataSizedTruncatedUpstream checks that an upstream body cut -// off before its Content-Length fails, while a body shorter than a size -// hint is still read. -func TestReadMetadataSizedTruncatedUpstream(t *testing.T) { - p := &Proxy{} - cut := io.MultiReader(strings.NewReader(`{"name":`), iotest.ErrReader(io.ErrUnexpectedEOF)) - if _, err := p.ReadMetadataSized(upstreamBodyReader{cut}, 100); !errors.Is(err, io.ErrUnexpectedEOF) { - t.Errorf("cut off body: error %v, want io.ErrUnexpectedEOF", err) - } - got, err := p.ReadMetadataSized(strings.NewReader(`{"name":"x"}`), 100) - if err != nil || string(got) != `{"name":"x"}` { - t.Errorf("short body = %q, %v", got, err) - } -} diff --git a/internal/handler/upstream_body.go b/internal/handler/upstream_body.go new file mode 100644 index 0000000..7db9076 --- /dev/null +++ b/internal/handler/upstream_body.go @@ -0,0 +1,26 @@ +package handler + +import ( + "errors" + "io" +) + +// upstreamBodyReader marks a failure to read a document from upstream, so a +// body upstream cuts off is told apart from one that ends, and from a +// failure to store it. +type upstreamBodyReader struct{ r io.Reader } + +func (u upstreamBodyReader) Read(b []byte) (int, error) { + n, err := u.r.Read(b) + if err != nil && !errors.Is(err, io.EOF) { + err = &upstreamBodyError{err} + } + return n, err +} + +// upstreamBodyError is a failure to read a document from upstream. It is +// handled as an upstream failure: the stored copy, if any, is served. +type upstreamBodyError struct{ err error } + +func (e *upstreamBodyError) Error() string { return "reading upstream metadata: " + e.err.Error() } +func (e *upstreamBodyError) Unwrap() error { return e.err } From b08cebf9e4d45f723f4e39b9c5d6bf94f596427d Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 1 Nov 2026 12:53:00 +0100 Subject: [PATCH 531/633] Cap npm packuments streamed into storage metadata_max_size bounds what is read into memory, but a packument streamed from upstream into storage had no limit, and with cooldown was stored twice. It is now capped by metadata_stream_max_size, 1 GB by default. A document over the cap fails as an upstream failure: nothing is stored, and the stored copy, if any, is served instead. The metadata settings move into configureMetadata to keep serve under the maintidx limit. --- cmd/proxy/main.go | 1 + docs/configuration.md | 7 +-- internal/config/config.go | 31 ++++++++++-- internal/config/config_test.go | 23 +++++++++ internal/handler/handler.go | 23 +++++---- internal/handler/metadata_stored.go | 21 ++++++++- internal/handler/npm_stream_harden_test.go | 55 ++++++++++++++++++++++ internal/server/server.go | 14 ++++-- 8 files changed, 155 insertions(+), 20 deletions(-) diff --git a/cmd/proxy/main.go b/cmd/proxy/main.go index 824f2e2..d39125d 100644 --- a/cmd/proxy/main.go +++ b/cmd/proxy/main.go @@ -561,6 +561,7 @@ func runMirror() { proxy.CacheMetadata = true // mirror always caches metadata proxy.MetadataTTL = cfg.ParseMetadataTTL() proxy.MetadataMaxSize = cfg.ParseMetadataMaxSize() + proxy.MetadataStreamMaxSize = cfg.ParseMetadataStreamMaxSize() m := newCommandMirror(cfg, proxy, images, db, store, logger, *concurrency) switch s := source.(type) { diff --git a/docs/configuration.md b/docs/configuration.md index 93a7299..5c33a6e 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1403,13 +1403,14 @@ When upstream is unreachable and the cached entry is past its TTL, the proxy ser ### Metadata size limit -Upstream metadata responses are buffered in memory before being rewritten and served. `metadata_max_size` caps that buffer to protect against OOM from a misbehaving upstream. Some npm packages with thousands of versions (for example `renovate`) exceed the 100 MB default, so raise this if you see `metadata response exceeds size limit` in the logs. Conda `repodata.json` with cooldown enabled is filtered as it streams and isn't held in memory decompressed. With `cache_metadata` the limit applies to the compressed document as it does without cooldown; one over it is filtered as it streams from the upstream and not cached. With `cache_metadata` and the metadata rewrite cache on, an npm packument is streamed from the upstream into storage and rewritten from there in two passes, so it isn't held in memory and the limit doesn't apply to it; the rewritten copy is kept gzip-compressed. Requests for a single version (`//`) still read the packument into memory. +Upstream metadata responses are buffered in memory before being rewritten and served. `metadata_max_size` caps that buffer to protect against OOM from a misbehaving upstream. Some npm packages with thousands of versions (for example `renovate`) exceed the 100 MB default, so raise this if you see `metadata response exceeds size limit` in the logs. Conda `repodata.json` with cooldown enabled is filtered as it streams and isn't held in memory decompressed. With `cache_metadata` the limit applies to the compressed document as it does without cooldown; one over it is filtered as it streams from the upstream and not cached. With `cache_metadata` and the metadata rewrite cache on, an npm packument is streamed from the upstream into storage and rewritten from there in two passes, so it isn't held in memory and the limit doesn't apply to it; the rewritten copy is kept gzip-compressed. Requests for a single version (`//`) and tarball downloads read the version from the stored copy the same way. `metadata_stream_max_size` (1 GB by default) caps a document streamed this way; one over it isn't stored, and the stored copy, if any, is served instead. ```yaml -metadata_max_size: "100MB" # default +metadata_max_size: "100MB" # default +metadata_stream_max_size: "1GB" # default ``` -Or via environment variable: `PROXY_METADATA_MAX_SIZE=250MB`. +Or via environment variable: `PROXY_METADATA_MAX_SIZE=250MB`, and `PROXY_METADATA_STREAM_MAX_SIZE=2GB` for `metadata_stream_max_size`. ### Artifact size limit diff --git a/internal/config/config.go b/internal/config/config.go index 5e90150..14790a9 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -169,6 +169,11 @@ type Config struct { // size return ErrMetadataTooLarge. Default: "100MB". MetadataMaxSize string `json:"metadata_max_size" yaml:"metadata_max_size"` + // MetadataStreamMaxSize is the largest metadata document the proxy + // streams from upstream into storage instead of buffering it, which + // metadata_max_size doesn't cover (e.g. "1GB"). Default: "1GB". + MetadataStreamMaxSize string `json:"metadata_stream_max_size" yaml:"metadata_stream_max_size"` + // ArtifactMaxSize is the largest upstream artifact the proxy will fetch // (e.g. "2GB"). Larger artifacts are refused and not cached. Default: "" // (unlimited). "0" is also unlimited. @@ -1480,6 +1485,7 @@ func (c *Config) LoadFromEnv() { setEnvStringSlice(&c.Hosted.ComposerVendors, "PROXY_HOSTED_COMPOSER_VENDORS") setEnvString(&c.MetadataTTL, "PROXY_METADATA_TTL") setEnvString(&c.MetadataMaxSize, "PROXY_METADATA_MAX_SIZE") + setEnvString(&c.MetadataStreamMaxSize, "PROXY_METADATA_STREAM_MAX_SIZE") setEnvString(&c.ArtifactMaxSize, "PROXY_ARTIFACT_MAX_SIZE") setEnvString(&c.MetadataRewriteCacheSize, "PROXY_METADATA_REWRITE_CACHE_SIZE") setEnvString(&c.HTTPTimeout, "PROXY_HTTP_TIMEOUT") @@ -1726,6 +1732,7 @@ const ( defaultHTTPTimeout = 30 * time.Second //nolint:mnd // sensible default defaultHitFlushInterval = time.Second defaultMetadataMaxSize = 100 << 20 + defaultMetadataStreamMaxSize = 1 << 30 defaultMetadataRewriteCacheSize = 256 << 20 defaultGradleBuildCacheMaxUploadSize = 100 << 20 defaultGradleBuildCacheSweepInterval = 10 * time.Minute @@ -1779,22 +1786,25 @@ func validateRequestTimeout(field, s string) error { return nil } -func validateMetadataMaxSize(s string) error { +func validateMetadataMaxSize(name, s string) error { if s == "" { return nil } size, err := ParseSize(s) if err != nil { - return fmt.Errorf("invalid metadata_max_size: %w", err) + return fmt.Errorf("invalid %s: %w", name, err) } if size <= 0 { - return fmt.Errorf("invalid metadata_max_size %q: must be positive", s) + return fmt.Errorf("invalid %s %q: must be positive", name, s) } return nil } func (c *Config) validateDownloadSizeLimits() error { - if err := validateMetadataMaxSize(c.MetadataMaxSize); err != nil { + if err := validateMetadataMaxSize("metadata_max_size", c.MetadataMaxSize); err != nil { + return err + } + if err := validateMetadataMaxSize("metadata_stream_max_size", c.MetadataStreamMaxSize); err != nil { return err } return validateArtifactMaxSize(c.ArtifactMaxSize) @@ -1857,6 +1867,19 @@ func (c *Config) ParseMetadataMaxSize() int64 { return size } +// ParseMetadataStreamMaxSize returns the size limit for metadata streamed +// into storage in bytes. Returns 1GB if unset or invalid. +func (c *Config) ParseMetadataStreamMaxSize() int64 { + if c.MetadataStreamMaxSize == "" { + return defaultMetadataStreamMaxSize + } + size, err := ParseSize(c.MetadataStreamMaxSize) + if err != nil || size <= 0 { + return defaultMetadataStreamMaxSize + } + return size +} + // ParseArtifactMaxSize returns the artifact size limit in bytes, or 0 for // no limit. func (c *Config) ParseArtifactMaxSize() int64 { diff --git a/internal/config/config_test.go b/internal/config/config_test.go index a0132cf..00f45d9 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -785,6 +785,29 @@ func TestParseMetadataMaxSize(t *testing.T) { } } +func TestParseMetadataStreamMaxSize(t *testing.T) { + cfg := Default() + if got := cfg.ParseMetadataStreamMaxSize(); got != defaultMetadataStreamMaxSize { + t.Errorf("unset: %d, want %d", got, defaultMetadataStreamMaxSize) + } + cfg.MetadataStreamMaxSize = "2GB" + if got := cfg.ParseMetadataStreamMaxSize(); got != 2<<30 { + t.Errorf("2GB: %d", got) + } + t.Setenv("PROXY_METADATA_STREAM_MAX_SIZE", "300MB") + cfg = Default() + cfg.LoadFromEnv() + if got := cfg.ParseMetadataStreamMaxSize(); got != 300<<20 { + t.Errorf("from env: %d", got) + } + for _, bad := range []string{"0", "lots"} { + cfg.MetadataStreamMaxSize = bad + if err := cfg.Validate(); err == nil { + t.Errorf("metadata_stream_max_size %q validated", bad) + } + } +} + func TestValidateMetadataMaxSize(t *testing.T) { cfg := Default() cfg.MetadataMaxSize = "not-a-size" diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 96ae655..832ac47 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -142,6 +142,10 @@ func ifNoneMatchHits(header, etag string) bool { // defaultMetadataMaxSize is used when Proxy.MetadataMaxSize is unset. const defaultMetadataMaxSize = 100 << 20 +// defaultMetadataStreamMaxSize is used when Proxy.MetadataStreamMaxSize is +// unset. +const defaultMetadataStreamMaxSize = 1 << 30 + // ErrMetadataTooLarge is returned when upstream metadata exceeds the configured limit. var ErrMetadataTooLarge = errors.New("metadata response exceeds size limit") @@ -215,14 +219,17 @@ type Proxy struct { Cooldown CooldownPolicy // ShortestCooldown is the shortest cooldown configured anywhere. It // bounds how long a rewrite filtered by cooldown is cached. - ShortestCooldown time.Duration - Denylist *denylist.Policy - CacheMetadata bool - MetadataTTL time.Duration - MetadataMaxSize int64 - GradleReadOnly bool - HostedEnabled bool - GradleMaxUploadSize int64 + ShortestCooldown time.Duration + Denylist *denylist.Policy + CacheMetadata bool + MetadataTTL time.Duration + MetadataMaxSize int64 + // MetadataStreamMaxSize caps a metadata document streamed from upstream + // into storage, which metadata_max_size doesn't cover. Default 1 GB. + MetadataStreamMaxSize int64 + GradleReadOnly bool + HostedEnabled bool + GradleMaxUploadSize int64 // NPMFullMetadata requests full npm packuments from upstream even when // cooldown is disabled, so served metadata carries publish times. NPMFullMetadata bool diff --git a/internal/handler/metadata_stored.go b/internal/handler/metadata_stored.go index b3bc420..ffe524d 100644 --- a/internal/handler/metadata_stored.go +++ b/internal/handler/metadata_stored.go @@ -198,9 +198,13 @@ func (p *Proxy) storeUpstreamMetadata(ctx context.Context, ecosystem, cacheKey, if lm := resp.Header.Get(headerLastModified); lm != "" { meta.lastModified, _ = http.ParseTime(lm) } + limit := p.MetadataStreamMaxSize + if limit <= 0 { + limit = defaultMetadataStreamMaxSize + } // Go decompresses the body, since no Accept-Encoding was set, so it is // stored uncompressed. - var body io.Reader = upstreamBodyReader{resp.Body} + var body io.Reader = &limitedBody{r: upstreamBodyReader{resp.Body}, left: limit} if validate != nil { staged, err := p.stageMetadata(ctx, ecosystem, cacheKey, body, validate) if err != nil { @@ -246,6 +250,21 @@ func (p *Proxy) stageMetadata(ctx context.Context, ecosystem, cacheKey string, b return &stagedReader{ReadCloser: staged, discard: discard}, nil } +// limitedBody fails with ErrMetadataTooLarge, as an upstream failure, once +// r has sent more than left bytes. +type limitedBody struct { + r io.Reader + left int64 +} + +func (l *limitedBody) Read(b []byte) (int, error) { + n, err := l.r.Read(b) + if l.left -= int64(n); l.left < 0 { + return 0, &upstreamBodyError{ErrMetadataTooLarge} + } + return n, err +} + // notStored returns a failure to store a document as errMetadataNotStored, // unless reading it from upstream failed. func notStored(what string, err error) error { diff --git a/internal/handler/npm_stream_harden_test.go b/internal/handler/npm_stream_harden_test.go index ca7143f..0cebf0a 100644 --- a/internal/handler/npm_stream_harden_test.go +++ b/internal/handler/npm_stream_harden_test.go @@ -181,3 +181,58 @@ func TestNPMStreamedPackumentClientRevalidates(t *testing.T) { t.Errorf("served %d versions, want 401", len(doc.Versions)) } } + +// TestNPMStreamedPackumentOverStreamLimit checks that a document over +// MetadataStreamMaxSize is neither stored nor served: the stored copy is +// served instead, with and without cooldown, and with nothing stored the +// request fails. +func TestNPMStreamedPackumentOverStreamLimit(t *testing.T) { + for _, withCooldown := range []bool{true, false} { + t.Run(map[bool]string{true: "cooldown", false: "no cooldown"}[withCooldown], func(t *testing.T) { + body := bigPackument(400) + u := newStreamUpstream(t, body) + proxy, store, handler := streamProxy(t, u) + if !withCooldown { + proxy.Cooldown = nil + } + proxy.MetadataMaxSize = int64(len(body) / 4) + proxy.MetadataStreamMaxSize = int64(len(body) + 1000) + want := len(decodeStreamed(t, serve(handler, "/big", nil)).Versions) + before := bigMetadataFiles(store) + + u.replace(bigPackument(500), `"v2"`) + proxy.MetadataTTL = time.Nanosecond + proxy.SetMetadataRewriteCacheSize(64 << 20) + + if doc := decodeStreamed(t, serve(handler, "/big", nil)); len(doc.Versions) != want { + t.Errorf("served %d versions, want the stored %d", len(doc.Versions), want) + } + after := bigMetadataFiles(store) + if len(after) != len(before) { + t.Errorf("storage holds %d files, %d before", len(after), len(before)) + } + for path, b := range before { + if after[path] != b { + t.Errorf("%s was changed", path) + } + } + }) + } + t.Run("uncached", func(t *testing.T) { + body := bigPackument(400) + u := newStreamUpstream(t, body) + proxy, store, handler := streamProxy(t, u) + proxy.MetadataMaxSize = int64(len(body) / 4) + proxy.MetadataStreamMaxSize = int64(len(body) / 2) + + if rec := serve(handler, "/big", nil); rec.Code != http.StatusBadGateway { + t.Errorf("status %d, want 502", rec.Code) + } + if files := bigMetadataFiles(store); len(files) != 0 { + t.Errorf("left %d files in storage", len(files)) + } + if entry, _ := proxy.DB.GetMetadataCache("npm", "big"); entry != nil { + t.Error("left a metadata cache row") + } + }) +} diff --git a/internal/server/server.go b/internal/server/server.go index 2f0bd68..caebb80 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -268,10 +268,7 @@ func (s *Server) serve(listener net.Listener) error { if err != nil { return fmt.Errorf("configuring scanners: %w", err) } - proxy.CacheMetadata = s.cfg.CacheMetadata - proxy.MetadataTTL = s.cfg.ParseMetadataTTL() - proxy.MetadataMaxSize = s.cfg.ParseMetadataMaxSize() - proxy.SetMetadataRewriteCacheSize(s.cfg.ParseMetadataRewriteCacheSize()) + configureMetadata(proxy, s.cfg) proxy.GradleReadOnly = s.cfg.Gradle.BuildCache.ReadOnly proxy.HostedEnabled = s.cfg.Hosted.Enabled proxy.HostedNPMScopes = s.cfg.Hosted.NPMScopes @@ -618,6 +615,15 @@ func (s *Server) newContainerHandler(proxy *handler.Proxy) *handler.ContainerHan return containerHandler } +// configureMetadata applies the metadata cache settings to proxy. +func configureMetadata(proxy *handler.Proxy, cfg *config.Config) { + proxy.CacheMetadata = cfg.CacheMetadata + proxy.MetadataTTL = cfg.ParseMetadataTTL() + proxy.MetadataMaxSize = cfg.ParseMetadataMaxSize() + proxy.MetadataStreamMaxSize = cfg.ParseMetadataStreamMaxSize() + proxy.SetMetadataRewriteCacheSize(cfg.ParseMetadataRewriteCacheSize()) +} + func configureScanning(proxy *handler.Proxy, cfg config.ScanningConfig, baseURL string, logger *slog.Logger) (*scanner.Group, error) { scanGroup, err := scanner.NewGroup(cfg, logger) if err != nil { From 26cae44e41feb62f1bcc284fa5623de6edfb8f87 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Mon, 2 Nov 2026 11:33:11 +0100 Subject: [PATCH 532/633] Bump cooldown to v0.3.1 and set cooldown directly in stream tests v0.3.1 parses a Config once when it is first used concurrently, so the stream tests no longer need cooldown.NewPolicy. --- go.mod | 2 +- go.sum | 4 ++-- internal/handler/npm_stream_test.go | 8 +------- 3 files changed, 4 insertions(+), 10 deletions(-) diff --git a/go.mod b/go.mod index db6695b..0f44186 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/ecr v1.66.5 github.com/git-pkgs/archives v0.10.0 github.com/git-pkgs/artifacts v0.2.1 - github.com/git-pkgs/cooldown v0.3.0 + github.com/git-pkgs/cooldown v0.3.1 github.com/git-pkgs/enrichment v0.7.5 github.com/git-pkgs/gcs v0.1.0 github.com/git-pkgs/integrity v0.2.0 diff --git a/go.sum b/go.sum index d1c0563..52f4c62 100644 --- a/go.sum +++ b/go.sum @@ -121,8 +121,8 @@ github.com/git-pkgs/archives v0.10.0 h1:/qgk0MI6VK2sKOoVTd0B4zs5hwTo+St0X8cYYftz github.com/git-pkgs/archives v0.10.0/go.mod h1:GJ/h7NPTUYiPinOUxpYNsHFCUikYZcJXvugBvP1VGNc= github.com/git-pkgs/artifacts v0.2.1 h1:VwdxR4yTDaqBZ34h0slxQBVyr2Xfa+QGOKCKviMx2xk= github.com/git-pkgs/artifacts v0.2.1/go.mod h1:Otosgq52pXT5UNN7lh6s/lszpWKVDO8XrOjN1M70/IA= -github.com/git-pkgs/cooldown v0.3.0 h1:eAl2CQFwqXxluoVP9hHmropAf1VoV3gSxLMkN+NEu8A= -github.com/git-pkgs/cooldown v0.3.0/go.mod h1:FnTg8wufEALATC9Sm/9AHjLdA+u9eVn96e9BqXXjGB0= +github.com/git-pkgs/cooldown v0.3.1 h1:XDt54+ZuYoJclTLysaQ2GUZPhYYuJdjpaqeLSwjMdto= +github.com/git-pkgs/cooldown v0.3.1/go.mod h1:FnTg8wufEALATC9Sm/9AHjLdA+u9eVn96e9BqXXjGB0= github.com/git-pkgs/enrichment v0.7.5 h1:jbfFzf7+HSS7el8AuMri4vHXq2izIIZDVIU0uYI5LFM= github.com/git-pkgs/enrichment v0.7.5/go.mod h1:TbxC/nGQN96YCbdoiDs8a9z3gk2zXmLVS1989/jvnsU= github.com/git-pkgs/gcs v0.1.0 h1:E3awGtsO0xZyHT9FUfEwMHjMkRxw41Bh+c7lgTmPvBo= diff --git a/internal/handler/npm_stream_test.go b/internal/handler/npm_stream_test.go index a8a4ea2..5d2ab73 100644 --- a/internal/handler/npm_stream_test.go +++ b/internal/handler/npm_stream_test.go @@ -107,13 +107,7 @@ func streamProxy(t *testing.T, u *streamUpstream) (*Proxy, *mockStorage, http.Ha proxy.CacheMetadata = true proxy.MetadataTTL = time.Hour proxy.SetMetadataRewriteCacheSize(64 << 20) - // As the server does: a cooldown.Config parses itself on first use, - // which concurrent requests would race on. - policy, err := cooldown.NewPolicy(&cooldown.Config{Default: "3d"}, nil) - if err != nil { - t.Fatal(err) - } - proxy.Cooldown = policy + proxy.Cooldown = &cooldown.Config{Default: "3d"} return proxy, store, NewNPMHandler(proxy, "http://proxy.example", u.URL).Routes() } From f35d152be0cc34af1ade85399f936fdca164c243 Mon Sep 17 00:00:00 2001 From: Ondrej Kokes Date: Mon, 2 Nov 2026 11:37:28 +0100 Subject: [PATCH 533/633] expose commit sha in UI footer --- .dockerignore | 17 +++++++++++++++++ .github/workflows/publish.yml | 3 +++ Dockerfile | 6 +++++- 3 files changed, 25 insertions(+), 1 deletion(-) create mode 100644 .dockerignore diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..05fb224 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,17 @@ +# Version and commit come in as build args, so the build doesn't need .git. +.git +.github + +# Local build output and runtime data +/proxy +dist/ +build/ +cache/ +*.db +*.tar.gz + +# Local config and secrets +.env +.env.local +proxy.cfg +config.yaml diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 0fadcbf..cbb6e15 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -61,6 +61,9 @@ jobs: push: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} + build-args: | + VERSION=${{ steps.meta.outputs.version }} + COMMIT=${{ github.sha }} provenance: mode=max sbom: true diff --git a/Dockerfile b/Dockerfile index 1a934ed..49be462 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,7 +14,11 @@ COPY . . # Build the binary for the target platform ARG TARGETARCH -RUN CGO_ENABLED=0 GOOS=linux GOARCH=${TARGETARCH} go build -ldflags="-s -w" -o /proxy ./cmd/proxy +ARG VERSION=dev +ARG COMMIT=unknown +RUN CGO_ENABLED=0 GOOS=linux GOARCH=${TARGETARCH} go build \ + -ldflags="-s -w -X main.Version=${VERSION} -X main.Commit=${COMMIT}" \ + -o /proxy ./cmd/proxy FROM alpine:3.24.2 From dd987f2de78da42343fe7a4cd59eb51d5040b367 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Mon, 2 Nov 2026 16:56:33 +0100 Subject: [PATCH 534/633] Document that SQLite needs a local disk or block volume --- README.md | 8 +++++++- deploy/charts/proxy/values.yaml | 6 ++++++ docs/configuration.md | 2 ++ 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index a324c53..f249c85 100644 --- a/README.md +++ b/README.md @@ -104,6 +104,12 @@ using SQLite and filesystem artifact storage under `/data`. See [`deploy/charts/proxy/values.yaml`](deploy/charts/proxy/values.yaml) for ingress, external database and object-storage configuration options. +The SQLite database needs a local disk or a block-backed volume, even with one +replica; `ReadWriteOnce` alone doesn't guarantee that, since the storage class +can still be NFS. If the cluster only offers network filesystem storage, use +Postgres. Artifact storage can stay on NFS, separately from the database. See +[SQLite](docs/configuration.md#sqlite). + To trust a private CA, for example when upstreams sit behind a TLS-intercepting corporate proxy or are internal registries signed by an in-house CA, mount the certificate with `extraVolumes` and `extraVolumeMounts` and add its directory @@ -1048,7 +1054,7 @@ Run with config file: ### PostgreSQL -SQLite is the default and works well for single-node deployments. For multi-node setups or if you prefer a managed database, switch to Postgres: +SQLite is the default and works well for single-node deployments, with the database file on a local disk or block-backed volume. For multi-node setups, when only network filesystem storage such as NFS is available, or if you prefer a managed database, switch to Postgres: ```yaml database: diff --git a/deploy/charts/proxy/values.yaml b/deploy/charts/proxy/values.yaml index 5936ef3..bbb253f 100644 --- a/deploy/charts/proxy/values.yaml +++ b/deploy/charts/proxy/values.yaml @@ -79,6 +79,12 @@ extraInitContainers: [] # image: busybox:1.37 # command: ["sh", "-c", "until nc -z postgres 5432; do sleep 2; done"] +# With SQLite, the database file needs a local disk or block-backed volume, +# even with one replica. ReadWriteOnce doesn't guarantee that: pick a +# storageClass that isn't NFS or another network filesystem. If only network +# filesystem storage is available, set config.data.database to Postgres. +# Artifact storage can stay on NFS on a separate volume (extraVolumes and +# extraVolumeMounts, with config.data.storage.url pointing at it). persistence: enabled: true # Keep this aligned with config.data.storage.url and config.data.database.path diff --git a/docs/configuration.md b/docs/configuration.md index 3e94531..2089c5f 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -201,6 +201,8 @@ database: The database runs in WAL mode. Writes go through a single connection, since SQLite allows one writer at a time. Reads use a separate pool of read-only connections (one per CPU, between 2 and 8), so request lookups don't wait behind writes, stats queries or eviction scans. An in-memory database (`:memory:`) uses one connection for both. +The SQLite database needs a local disk or a block-backed volume, even with one replica. SQLite's WAL mode doesn't work on network filesystems such as NFS ([sqlite.org/wal.html](https://www.sqlite.org/wal.html)), which shows up as stalled requests and `database disk image is malformed`. A Kubernetes `ReadWriteOnce` volume doesn't guarantee suitable storage, since its storage class can still be NFS. If only network filesystem storage is available to the proxy, use `driver: postgres`. Artifact storage can stay on NFS, on a separate volume from the SQLite database. + ### PostgreSQL ```yaml From 7cb905638cbc4a92552051a4cdc61a77179eebce Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Tue, 3 Nov 2026 09:28:29 +0000 Subject: [PATCH 535/633] Keep proxyCachedWithEncoding doc on the function it describes --- internal/handler/handler.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 7fef62e..a3fc2b7 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -2154,11 +2154,6 @@ func (p *Proxy) ProxyCached(w http.ResponseWriter, r *http.Request, upstreamURL, p.proxyCachedWithEncoding(w, r, upstreamURL, ecosystem, cacheKey, "identity", nil, acceptHeaders...) } -// proxyCachedWithEncoding is ProxyCached with an explicit upstream Accept-Encoding. -// "identity" preserves signed index bytes (the default); "gzip" keeps both hops -// compressed for large, non-hash-pinned metadata whose clients decode gzip -// (Homebrew API and Conda repodata). The stored bytes and Content-Encoding are -// replayed verbatim either way. // metadataHooks are optional callbacks for proxyCachedWithEncoding: hit runs // after a fresh cached row is served, with that row and when it was read, and // fetched runs with a fetched document before it is written. @@ -2167,6 +2162,11 @@ type metadataHooks struct { fetched func(body []byte, contentType, contentEncoding string) } +// proxyCachedWithEncoding is ProxyCached with an explicit upstream Accept-Encoding. +// "identity" preserves signed index bytes (the default); "gzip" keeps both hops +// compressed for large, non-hash-pinned metadata whose clients decode gzip +// (Homebrew API and Conda repodata). The stored bytes and Content-Encoding are +// replayed verbatim either way. func (p *Proxy) proxyCachedWithEncoding(w http.ResponseWriter, r *http.Request, upstreamURL, ecosystem, cacheKey, acceptEncoding string, hooks *metadataHooks, acceptHeaders ...string) { if !p.CacheMetadata { // Stream directly without buffering when caching is off. From 683455aecb386a8a8cb54616cb7db42268e30dd3 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Tue, 3 Nov 2026 14:39:36 +0100 Subject: [PATCH 536/633] Test available_at through the API and drop FillVersionPublishedAt The API handler takes the enrichment calls it uses as an interface, so the test can answer them without reaching a registry. Remove FillVersionPublishedAt until the shared cooldown filter calls it. --- internal/database/database_test.go | 36 ------------- internal/database/queries.go | 35 ------------- internal/server/api.go | 14 +++-- internal/server/api_cooldown_test.go | 76 ++++++++++++++++++++++++++++ 4 files changed, 87 insertions(+), 74 deletions(-) create mode 100644 internal/server/api_cooldown_test.go diff --git a/internal/database/database_test.go b/internal/database/database_test.go index 6b49c32..1374dd9 100644 --- a/internal/database/database_test.go +++ b/internal/database/database_test.go @@ -1165,42 +1165,6 @@ func TestVersionPublishedAtPreserved(t *testing.T) { }) } -func TestFillVersionPublishedAt(t *testing.T) { - runWithBothDatabases(t, func(t *testing.T, db *DB) { - const pkg, held, known = "pkg:npm/leftpad", "pkg:npm/leftpad@2.0.0", "pkg:npm/leftpad@1.0.0" - stored := time.Date(2020, 1, 2, 3, 4, 5, 0, time.UTC) - filtered := time.Date(2021, 6, 7, 8, 9, 10, 0, time.UTC) - - if err := db.SetVersionPublishedAt(known, pkg, stored); err != nil { - t.Fatalf("SetVersionPublishedAt: %v", err) - } - if err := db.UpsertVersion(&Version{PURL: held, PackagePURL: pkg}); err != nil { - t.Fatalf("UpsertVersion: %v", err) - } - - for _, purl := range []string{known, held, "pkg:npm/leftpad@3.0.0"} { - if err := db.FillVersionPublishedAt(purl, pkg, filtered); err != nil { - t.Fatalf("FillVersionPublishedAt(%s): %v", purl, err) - } - } - - want := map[string]time.Time{ - known: stored, - held: filtered, - "pkg:npm/leftpad@3.0.0": filtered, - } - for purl, at := range want { - got, err := db.GetVersionByPURL(purl) - if err != nil || got == nil { - t.Fatalf("GetVersionByPURL(%s): %v", purl, err) - } - if !got.PublishedAt.Valid || !got.PublishedAt.Time.Equal(at) { - t.Errorf("%s PublishedAt = %v, want %v", purl, got.PublishedAt, at) - } - } - }) -} - func TestGetFetchedVersionPURLs(t *testing.T) { runWithBothDatabases(t, func(t *testing.T, db *DB) { const pkg = "pkg:npm/leftpad" diff --git a/internal/database/queries.go b/internal/database/queries.go index 825841e..5d8f5f3 100644 --- a/internal/database/queries.go +++ b/internal/database/queries.go @@ -224,41 +224,6 @@ func (db *DB) SetVersionPublishedAt(versionPURL, packagePURL string, publishedAt return nil } -// FillVersionPublishedAt records a version's publish time like -// SetVersionPublishedAt, but keeps a time already stored. The cooldown -// filters call it for each version they hold back, on every uncached -// metadata request, so it writes without reading first. -func (db *DB) FillVersionPublishedAt(versionPURL, packagePURL string, publishedAt time.Time) error { - now := time.Now() - var query string - - if db.dialect == DialectPostgres { - query = ` - INSERT INTO versions (purl, package_purl, published_at, created_at, updated_at) - VALUES ($1, $2, $3, $4, $5) - ON CONFLICT(purl) DO UPDATE SET - published_at = EXCLUDED.published_at, - updated_at = EXCLUDED.updated_at - WHERE versions.published_at IS NULL - ` - } else { - query = ` - INSERT INTO versions (purl, package_purl, published_at, created_at, updated_at) - VALUES (?, ?, ?, ?, ?) - ON CONFLICT(purl) DO UPDATE SET - published_at = excluded.published_at, - updated_at = excluded.updated_at - WHERE versions.published_at IS NULL - ` - } - - _, err := db.Exec(query, versionPURL, packagePURL, publishedAt, now, now) - if err != nil { - return fmt.Errorf("filling version publish time: %w", err) - } - return nil -} - // GetFetchedVersionPURLs returns the versions of a package that have an // artifact row, cached now or evicted since. Versions without one were only // seen in metadata. diff --git a/internal/server/api.go b/internal/server/api.go index b08055d..07080aa 100644 --- a/internal/server/api.go +++ b/internal/server/api.go @@ -1,6 +1,7 @@ package server import ( + "context" "encoding/json" "errors" "net/http" @@ -22,13 +23,22 @@ const ( // APIHandler provides REST endpoints for package enrichment data. type APIHandler struct { - enrichment *enrichment.Service + enrichment enricher ecosystems *shared.EcosystemsClient db DBSearcher // cooldown is nil with cooldown off. cooldown *cooldownView } +// enricher is the part of enrichment.Service the API uses. +type enricher interface { + EnrichPackage(ctx context.Context, ecosystem, name string) (*enrichment.PackageInfo, error) + EnrichFull(ctx context.Context, ecosystem, name, version string) (*enrichment.EnrichmentResult, error) + CheckVulnerabilities(ctx context.Context, ecosystem, name, version string) ([]enrichment.VulnInfo, error) + GetLatestVersion(ctx context.Context, ecosystem, name string) (string, error) + BulkEnrichPackages(ctx context.Context, packages []struct{ Ecosystem, Name string }) map[string]*enrichment.PackageInfo +} + // DBSearcher defines the interface for database search operations. type DBSearcher interface { SearchPackages(query string, ecosystem string, limit int, offset int) ([]database.SearchResult, error) @@ -50,7 +60,6 @@ func NewAPIHandler(svc *enrichment.Service, db DBSearcher) *APIHandler { return h } -// newAPIHandler returns the server's API handler, with its cooldown policy. func (s *Server) newAPIHandler(svc *enrichment.Service) *APIHandler { h := NewAPIHandler(svc, s.db) h.cooldown = s.cooldown @@ -87,7 +96,6 @@ type VersionResponse struct { IsOutdated bool `json:"is_outdated"` } -// apiTimeFormat is the format of the API's timestamps. const apiTimeFormat = "2006-01-02T15:04:05Z" // VulnResponse contains vulnerability information. diff --git a/internal/server/api_cooldown_test.go b/internal/server/api_cooldown_test.go new file mode 100644 index 0000000..4e3e3c7 --- /dev/null +++ b/internal/server/api_cooldown_test.go @@ -0,0 +1,76 @@ +package server + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/git-pkgs/proxy/internal/config" + "github.com/git-pkgs/proxy/internal/enrichment" + "github.com/go-chi/chi/v5" +) + +func TestAPIVersionAvailableAt(t *testing.T) { + now := time.Now().UTC().Truncate(time.Second) + recent, old := now.Add(-time.Hour), now.Add(-96*time.Hour) + h := &APIHandler{ + enrichment: fakeEnricher{published: map[string]time.Time{"1.0.0": old, "2.0.0": recent}}, + cooldown: testCooldownView(t, config.CooldownConfig{Default: "3d"}), + } + r := chi.NewRouter() + r.Get("/api/package/{ecosystem}/*", h.HandlePackagePath) + + tests := []struct{ version, availableAt string }{ + {"2.0.0", recent.Add(72 * time.Hour).Format(apiTimeFormat)}, + {"1.0.0", ""}, + } + for _, tt := range tests { + t.Run(tt.version, func(t *testing.T) { + w := httptest.NewRecorder() + r.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/api/package/npm/leftpad/"+tt.version, nil)) + if w.Code != http.StatusOK { + t.Fatalf("status = %d, body = %s", w.Code, w.Body) + } + var resp EnrichmentResponse + if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil { + t.Fatal(err) + } + if resp.Package == nil || resp.Package.Cooldown != "3d" { + t.Errorf("package = %+v, want cooldown 3d", resp.Package) + } + if resp.Version == nil || resp.Version.AvailableAt != tt.availableAt { + t.Errorf("version = %+v, want available_at %q", resp.Version, tt.availableAt) + } + }) + } +} + +// fakeEnricher answers version lookups with the given publish times and +// knows no packages by name alone. +type fakeEnricher struct{ published map[string]time.Time } + +func (fakeEnricher) EnrichPackage(context.Context, string, string) (*enrichment.PackageInfo, error) { + return nil, nil +} + +func (f fakeEnricher) EnrichFull(_ context.Context, ecosystem, name, version string) (*enrichment.EnrichmentResult, error) { + return &enrichment.EnrichmentResult{ + Package: &enrichment.PackageInfo{Ecosystem: ecosystem, Name: name}, + Version: &enrichment.VersionInfo{Number: version, PublishedAt: f.published[version]}, + }, nil +} + +func (fakeEnricher) CheckVulnerabilities(context.Context, string, string, string) ([]enrichment.VulnInfo, error) { + return nil, nil +} + +func (fakeEnricher) GetLatestVersion(context.Context, string, string) (string, error) { + return "", nil +} + +func (fakeEnricher) BulkEnrichPackages(context.Context, []struct{ Ecosystem, Name string }) map[string]*enrichment.PackageInfo { + return nil +} From cb330a5cbf87df0c37cd4da480926235b9bf1e73 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Tue, 3 Nov 2026 20:10:24 +0100 Subject: [PATCH 537/633] Answer 502 when a Composer download's metadata lookup fails upstream The lookup took any non-200 metadata response for a version that isn't listed, so a Packagist outage or rate limit answered the archive request with 404. Composer treats that as the version being gone. Only 404 and 410 now mean the document is missing. A 429 is passed on as 503 with Retry-After while the upstream is held, and any other status answers 502. --- internal/handler/composer.go | 16 +++-- .../handler/composer_download_lookup_test.go | 67 +++++++++++++++---- 2 files changed, 67 insertions(+), 16 deletions(-) diff --git a/internal/handler/composer.go b/internal/handler/composer.go index ccb42bb..aa79080 100644 --- a/internal/handler/composer.go +++ b/internal/handler/composer.go @@ -905,9 +905,9 @@ type composerDist struct { } // findDownloadURLFromMetadata fetches a metadata document and returns the dist -// of the given version, with an empty URL if the version is not present. An -// error is returned only on transport failure; a missing document (non-200) -// or a missing version both yield an empty URL so the caller can fall back. +// of the given version, with an empty URL if the version is not present. A +// missing document (404 or 410) or a missing version both yield an empty URL +// so the caller can fall back; any other failure is an error. func (h *ComposerHandler) findDownloadURLFromMetadata(ctx context.Context, metaURL, packageName, version string) (composerDist, error) { h.proxy.Logger.Debug("fetching upstream metadata for download lookup", "url", metaURL, "package", packageName, "version", version) @@ -926,8 +926,16 @@ func (h *ComposerHandler) findDownloadURLFromMetadata(ctx context.Context, metaU h.proxy.Logger.Debug("upstream metadata response", "url", metaURL, "status", resp.StatusCode) - if resp.StatusCode != http.StatusOK { + switch { + case resp.StatusCode == http.StatusNotFound || resp.StatusCode == http.StatusGone: return composerDist{}, nil + case resp.StatusCode == http.StatusTooManyRequests && h.proxy.RateLimits != nil: + if err := h.proxy.RateLimits.Held(metaURL); err != nil { + return composerDist{}, err + } + } + if resp.StatusCode != http.StatusOK { + return composerDist{}, fmt.Errorf("upstream metadata returned %d", resp.StatusCode) } body, err := h.proxy.ReadMetadata(resp.Body) diff --git a/internal/handler/composer_download_lookup_test.go b/internal/handler/composer_download_lookup_test.go index 5635000..53a0cfe 100644 --- a/internal/handler/composer_download_lookup_test.go +++ b/internal/handler/composer_download_lookup_test.go @@ -11,23 +11,25 @@ import ( "time" "github.com/git-pkgs/cooldown" + registryclient "github.com/git-pkgs/registries/client" "github.com/git-pkgs/registries/fetch" ) // composerLookupUpstream serves v/p's metadata listing versions and the // archives they point at, counts metadata and archive requests, and fails -// every request while down is set. A version listed in published carries that -// publish time. +// every request while down is set, with downStatus or 503. A version listed in +// published carries that publish time. type composerLookupUpstream struct { - h http.Handler - handler *ComposerHandler - proxy *Proxy - versions atomic.Value // []string - published map[string]time.Time - down atomic.Bool - metadata atomic.Int32 - archives atomic.Int32 - routeURL string + h http.Handler + handler *ComposerHandler + proxy *Proxy + versions atomic.Value // []string + published map[string]time.Time + down atomic.Bool + downStatus atomic.Int32 + metadata atomic.Int32 + archives atomic.Int32 + routeURL string } func setupComposerLookup(t *testing.T, cacheMetadata bool, versions ...string) *composerLookupUpstream { @@ -37,7 +39,12 @@ func setupComposerLookup(t *testing.T, cacheMetadata bool, versions ...string) * var srv *httptest.Server srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if u.down.Load() { - http.Error(w, "down", http.StatusServiceUnavailable) + status := int(u.downStatus.Load()) + if status == 0 { + status = http.StatusServiceUnavailable + } + w.Header().Set("Retry-After", "30") + http.Error(w, "down", status) return } switch { @@ -146,6 +153,42 @@ func TestComposerDownloadWithoutMetadataCache(t *testing.T) { } } +// TestComposerDownloadLookupUpstreamError checks that a download whose +// metadata lookup fails upstream answers 502, or 503 while upstream holds +// requests, and not 404: Composer treats 404 as the version being gone. +func TestComposerDownloadLookupUpstreamError(t *testing.T) { + for _, status := range []int{http.StatusInternalServerError, http.StatusServiceUnavailable} { + t.Run(http.StatusText(status), func(t *testing.T) { + u := setupComposerLookup(t, false, "1.0.0") + u.downStatus.Store(int32(status)) + u.down.Store(true) + if w := u.get(t, "/files/v/p/1.0.0/1.0.0.zip"); w.Code != http.StatusBadGateway { + t.Errorf("status = %d, want 502", w.Code) + } + }) + } + + t.Run("rate limited", func(t *testing.T) { + u := setupComposerLookup(t, false, "1.0.0") + gate := registryclient.NewRateLimitGate(http.DefaultTransport) + u.proxy.HTTPClient = &http.Client{Transport: gate} + u.proxy.RateLimits = gate + u.downStatus.Store(http.StatusTooManyRequests) + u.down.Store(true) + w := u.get(t, "/files/v/p/1.0.0/1.0.0.zip") + if w.Code != http.StatusServiceUnavailable || w.Header().Get("Retry-After") == "" { + t.Errorf("status = %d, Retry-After %q; want 503 with Retry-After", w.Code, w.Header().Get("Retry-After")) + } + }) + + t.Run("not listed", func(t *testing.T) { + u := setupComposerLookup(t, false, "1.0.0") + if w := u.get(t, "/files/v/p/2.0.0/2.0.0.zip"); w.Code != http.StatusNotFound { + t.Errorf("status = %d, want 404", w.Code) + } + }) +} + // storedPublishTime returns the publish time recorded for v/p at version. func (u *composerLookupUpstream) storedPublishTime(t *testing.T, version string) (time.Time, bool) { t.Helper() From 88c0ba8ab3be75703296c33d3b1917478f25c8ee Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Wed, 4 Nov 2026 13:20:29 +0100 Subject: [PATCH 538/633] Use sbom's root detection for the project's own packages Bump sbom to v0.1.8 and replace lockFileRoots with sbom.Roots. Inferred roots count as the project only when read from a lock file, so an image's packages that nothing depends on are kept. --- go.mod | 2 +- go.sum | 4 +-- internal/inventory/inventory.go | 49 +++++++++++++------------ internal/inventory/inventory_test.go | 53 ++++++++++++++++++++++++++++ 4 files changed, 82 insertions(+), 26 deletions(-) diff --git a/go.mod b/go.mod index e5cb0c8..2fe061f 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( github.com/git-pkgs/magic v0.6.0 github.com/git-pkgs/purl v0.2.1 github.com/git-pkgs/registries v0.13.0 - github.com/git-pkgs/sbom v0.1.7 + github.com/git-pkgs/sbom v0.1.8 github.com/git-pkgs/spdx v0.3.3 github.com/git-pkgs/vers v0.7.2 github.com/go-chi/chi/v5 v5.5.0 diff --git a/go.sum b/go.sum index 52f4c62..3e33929 100644 --- a/go.sum +++ b/go.sum @@ -137,8 +137,8 @@ github.com/git-pkgs/purl v0.2.1 h1:b46hRlOMbrG6aXjjFJdYixGi5WTh0TLlAnH9iNeuOS0= github.com/git-pkgs/purl v0.2.1/go.mod h1:l8rCfvt5iXVrFIcoQ8+vCLOaHhrtEW2id0LENP+IzMw= github.com/git-pkgs/registries v0.13.0 h1:5i9dHnbiXseIbXLIZ61YnuFySJ9dQX+KEu1MrmwB2BU= github.com/git-pkgs/registries v0.13.0/go.mod h1:1g/jXVi1M/3jWvW1Wom1NMnPrhglJ7UVEZ215l2Zh0M= -github.com/git-pkgs/sbom v0.1.7 h1:4eboy/Yed/SC9ZcC/xT50iywHPzWbmCU7azDb5T1RbA= -github.com/git-pkgs/sbom v0.1.7/go.mod h1:KIkZX0v31l62SeFsi2qsQXcZRVrAXvIdPQ4x4en+G2M= +github.com/git-pkgs/sbom v0.1.8 h1:B+ohNEQfk1M+ch1/hMUKAitZp792g6fUrMaO4GTSqhI= +github.com/git-pkgs/sbom v0.1.8/go.mod h1:KIkZX0v31l62SeFsi2qsQXcZRVrAXvIdPQ4x4en+G2M= github.com/git-pkgs/spdx v0.3.3 h1:xXlUoZzjDSkwxC6mWHfaW+kc8a0+LH6d+gGHuZWftqI= github.com/git-pkgs/spdx v0.3.3/go.mod h1:n9rAicgw+Wqtfursju6oKaAL6cUCTiIS926mb1KWoKU= github.com/git-pkgs/vers v0.7.2 h1:Ctex1O+Ai/FONmyDaw37EolBXg0og6UdxbCjjYbeRHs= diff --git a/internal/inventory/inventory.go b/internal/inventory/inventory.go index d115fb7..a0b6b2b 100644 --- a/internal/inventory/inventory.go +++ b/internal/inventory/inventory.go @@ -88,18 +88,18 @@ const unknownVersion = "UNKNOWN" // type the proxy doesn't serve, without a version, or conda packages without // a channel are counted in skipped. // So are the project's own packages: Go modules its binaries were built from, -// and the root of a lock file. +// and the roots sbom finds for the dependency graph. func Parse(data []byte) (components []Component, skipped int, err error) { doc, err := sbom.Parse(data) if err != nil { return nil, 0, err } ownModules := mainModules(doc.Packages) - lockRoots := lockFileRoots(doc) + roots := projectRoots(doc) seen := make(map[[2]string]bool, len(doc.Packages)) for _, pkg := range doc.Packages { c, ok := componentOf(pkg) - if !ok || lockRoots[pkg.ID] || (c.Ecosystem == "golang" && ownModules[c.Name]) { + if !ok || roots[pkg.ID] || (c.Ecosystem == "golang" && ownModules[c.Name]) { skipped++ continue } @@ -133,33 +133,36 @@ func componentOf(pkg sbom.Package) (Component, bool) { return c, true } -// lockFileRoots returns the IDs of packages read from a lock file that depend -// on others while nothing depends on them: the project's own entry, such as -// the root of package-lock.json, which carries the version from package.json. -func lockFileRoots(doc *sbom.SBOM) map[string]bool { - dependents := map[string]bool{} - dependencies := map[string]bool{} - for _, r := range doc.Relationships { - if r.Type != sbom.RelDependsOn { - continue - } - dependencies[r.SourceID] = true - dependents[r.TargetID] = true +// projectRoots returns the IDs of the roots sbom finds that are the +// project: all of them when the document declares them, otherwise only those +// read from a lock file. An image's packages that nothing depends on, such as +// busybox, are roots of the graph but not the project. +func projectRoots(doc *sbom.SBOM) map[string]bool { + ids, declared := doc.Roots() + roots := make(map[string]bool, len(ids)) + for _, id := range ids { + roots[id] = declared + } + if declared { + return roots } - roots := map[string]bool{} for _, pkg := range doc.Packages { - if pkg.ID == "" || !dependencies[pkg.ID] || dependents[pkg.ID] { - continue - } - for _, prop := range pkg.Properties { - if prop.Name == syftMetadataType && strings.Contains(prop.Value, "lock") { - roots[pkg.ID] = true - } + if _, ok := roots[pkg.ID]; ok && readFromLockFile(pkg) { + roots[pkg.ID] = true } } return roots } +func readFromLockFile(pkg sbom.Package) bool { + for _, prop := range pkg.Properties { + if prop.Name == syftMetadataType && strings.Contains(prop.Value, "lock") { + return true + } + } + return false +} + // mainModules returns the Go modules the SBOM's binaries were built from, // which are the project's own. Module paths are case-sensitive and package // URLs keep their case, so they are compared as they are. diff --git a/internal/inventory/inventory_test.go b/internal/inventory/inventory_test.go index 3fbe113..36e67ef 100644 --- a/internal/inventory/inventory_test.go +++ b/internal/inventory/inventory_test.go @@ -246,6 +246,59 @@ func TestParseSkipsTheProjectsOwnPackages(t *testing.T) { } } +func TestParseKeepsTopLevelDependenciesOfLockFiles(t *testing.T) { + // A poetry.lock doesn't list the project, so requests is a direct + // dependency, not the project's own entry. + doc := `{"bomFormat":"CycloneDX","specVersion":"1.6","components":[ + {"bom-ref":"requests","type":"library","name":"requests","version":"2.32.3","purl":"pkg:pypi/requests@2.32.3", + "properties":[{"name":"syft:package:metadataType","value":"python-poetry-lock-entry"},{"name":"syft:location:0:path","value":"/poetry.lock"}]}, + {"bom-ref":"urllib3","type":"library","name":"urllib3","version":"2.2.3","purl":"pkg:pypi/urllib3@2.2.3", + "properties":[{"name":"syft:package:metadataType","value":"python-poetry-lock-entry"},{"name":"syft:location:0:path","value":"/poetry.lock"}]} + ],"dependencies":[{"ref":"requests","dependsOn":["urllib3"]}]}` + components, skipped, err := Parse([]byte(doc)) + if err != nil { + t.Fatal(err) + } + if len(components) != 2 || skipped != 0 { + t.Errorf("components = %+v, skipped = %d; want requests and urllib3, 0 skipped", components, skipped) + } +} + +func TestParseKeepsOSPackagesNothingDependsOn(t *testing.T) { + // syft's CycloneDX of an image: metadata.component is the image, with + // no edges, and busybox is something nothing else depends on. + doc := `{"bomFormat":"CycloneDX","specVersion":"1.6", + "metadata":{"component":{"bom-ref":"image","type":"container","name":"alpine:3.20"}}, + "components":[ + {"bom-ref":"busybox","type":"library","name":"busybox","version":"1.36.1-r29","purl":"pkg:apk/alpine/busybox@1.36.1-r29?distro=alpine-3.20.3", + "properties":[{"name":"syft:package:metadataType","value":"apk-db-entry"}]}, + {"bom-ref":"musl","type":"library","name":"musl","version":"1.2.5-r0","purl":"pkg:apk/alpine/musl@1.2.5-r0?distro=alpine-3.20.3", + "properties":[{"name":"syft:package:metadataType","value":"apk-db-entry"}]} + ],"dependencies":[{"ref":"busybox","dependsOn":["musl"]}]}` + components, skipped, err := Parse([]byte(doc)) + if err != nil { + t.Fatal(err) + } + if len(components) != 2 || skipped != 0 { + t.Errorf("components = %+v, skipped = %d; want busybox and musl, 0 skipped", components, skipped) + } +} + +func TestParseSkipsTheDeclaredRoot(t *testing.T) { + doc := `{"bomFormat":"CycloneDX","specVersion":"1.6", + "metadata":{"component":{"bom-ref":"app","type":"application","name":"app","version":"1.0.0","purl":"pkg:npm/app@1.0.0"}}, + "components":[ + {"bom-ref":"lodash","type":"library","name":"lodash","version":"4.17.21","purl":"pkg:npm/lodash@4.17.21"} + ],"dependencies":[{"ref":"app","dependsOn":["lodash"]}]}` + components, _, err := Parse([]byte(doc)) + if err != nil { + t.Fatal(err) + } + if len(components) != 1 || components[0].Name != "lodash" { + t.Errorf("components = %+v, want only lodash", components) + } +} + func TestParseSkipsMixedCaseMainModule(t *testing.T) { // syft keeps the module path's case in mainModule, and so does the // package URL. From 3e2a84a88ada3196b6dedff0f3f600b27a59ecb1 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Wed, 4 Nov 2026 15:44:01 +0100 Subject: [PATCH 539/633] Mirror Alpine packages The PURL namespace names the upstream.apk repository, and the distro and arch qualifiers syft writes give the release and architecture. The package is looked up in the main and community indexes of that release and cached under the path the APK handler serves it from. --- docs/configuration.md | 2 + internal/handler/apk.go | 29 +++-- internal/mirror/apk.go | 151 +++++++++++++++++++++++++++ internal/mirror/mirror.go | 9 ++ internal/mirror/source.go | 5 +- internal/server/mirror_apk_test.go | 142 +++++++++++++++++++++++++ internal/server/mirror_fetch_test.go | 1 + internal/server/mirror_versions.go | 5 +- 8 files changed, 333 insertions(+), 11 deletions(-) create mode 100644 internal/mirror/apk.go create mode 100644 internal/server/mirror_apk_test.go diff --git a/docs/configuration.md b/docs/configuration.md index ae2ae79..c71a8ca 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1726,6 +1726,8 @@ A Swift release is mirrored from `upstream.swift` with its source archive, check A CRAN version is mirrored as its source package from `upstream.cran`, for example `pkg:cran/glue@1.8.1`, and cached under the file name, so it is served for both `/cran/src/contrib/{file}` and `/cran/src/contrib/Archive/{name}/{file}`, where CRAN moves it once a newer version is published. The current version is fetched from `src/contrib` and checked against the MD5 sum in `src/contrib/PACKAGES.gz`; older versions are fetched unchecked from `src/contrib/Archive/{name}/`, as the CRAN handler fetches them. Binary packages aren't mirrored, and neither are the `PACKAGES` indexes, so `install.packages` needs the upstream or an index the proxy already cached; installing the tarball by URL works without either. +An Alpine package is mirrored from the repository in `upstream.apk` named as the namespace, `alpine` by default, as syft writes it in image SBOMs: `pkg:apk/alpine/busybox@1.36.1-r29?arch=x86_64&distro=alpine-3.20.3`. The `distro` qualifier gives the release (`v3.20`) and `arch` the architecture; packages without either are refused. The package is looked up in the `main` and then the `community` `APKINDEX.tar.gz` of that release and cached under the path `/apk/{name}/{release}/{repo}/{arch}/{file}` serves it from. The file isn't checked by the proxy, as when serving, since the index checksum covers only the package's control section; apk checks it on install. The indexes aren't mirrored, so `apk add` needs the upstream or an index the proxy already cached. Give a version: Alpine repositories keep only the current one. + | Flag | Default | Description | |------|---------|-------------| | `--sbom` | | Path to CycloneDX or SPDX SBOM file | diff --git a/internal/handler/apk.go b/internal/handler/apk.go index 5de909a..045dbac 100644 --- a/internal/handler/apk.go +++ b/internal/handler/apk.go @@ -60,18 +60,31 @@ type APKHandler struct { // When repositories is empty, a single repository named "alpine" pointing at // the official Alpine mirror is used. func NewAPKHandler(proxy *Proxy, proxyURL string, repositories map[string]string) *APKHandler { - h := &APKHandler{ + return &APKHandler{ proxy: proxy, proxyURL: strings.TrimSuffix(proxyURL, "/"), - repositories: make(map[string]string, len(repositories)), + repositories: APKRepositories(repositories), } - for name, repositoryURL := range repositories { - h.repositories[name] = strings.TrimSuffix(repositoryURL, "/") +} + +// APKRepositories returns the upstream.apk repositories by name without +// trailing slashes, or the official Alpine mirror as "alpine" when none are +// configured. +func APKRepositories(configured map[string]string) map[string]string { + repositories := make(map[string]string, len(configured)) + for name, repositoryURL := range configured { + repositories[name] = strings.TrimSuffix(repositoryURL, "/") } - if len(h.repositories) == 0 { - h.repositories[defaultAPKRepositoryName] = defaultAPKUpstream + if len(repositories) == 0 { + repositories[defaultAPKRepositoryName] = defaultAPKUpstream } - return h + return repositories +} + +// APKCacheFilename returns the filename a package at path +// ({release}/{repo}/{arch}/{file}) of repository is cached under. +func APKCacheFilename(repository, path string) string { + return repository + "/" + path } // Routes returns the HTTP handler for APK requests. @@ -145,7 +158,7 @@ func (h *APKHandler) handlePackageDownload(w http.ResponseWriter, r *http.Reques } downloadURL := upstreamURL + "/" + path - cacheFilename := repository + "/" + path + cacheFilename := APKCacheFilename(repository, path) h.proxy.Logger.Info("apk package download", "repository", repository, "name", name, "version", version, "arch", arch) diff --git a/internal/mirror/apk.go b/internal/mirror/apk.go new file mode 100644 index 0000000..b6b4ea3 --- /dev/null +++ b/internal/mirror/apk.go @@ -0,0 +1,151 @@ +package mirror + +import ( + "context" + "errors" + "fmt" + "regexp" + "strings" + "sync" + "time" + + "github.com/git-pkgs/proxy/internal/handler" + "github.com/git-pkgs/registries" +) + +// apkIndexTTL is how long a repository's APKINDEX is reused, so mirroring +// the packages of an image reads each index once. +const apkIndexTTL = 5 * time.Minute + +var ( + // apkDistroPattern takes the release from syft's distro qualifier, such + // as alpine-3.20.3 for v3.20. + apkDistroPattern = regexp.MustCompile(`^alpine-(\d+)\.(\d+)(?:[.-]|$)`) + apkArchPattern = regexp.MustCompile(`^[a-z0-9_]+$`) + // apkComponents are the repositories of a release a package is looked + // up in, in order. + apkComponents = []string{"main", "community"} +) + +// apkIndexes keeps one registries client per repository directory, which +// holds the parsed APKINDEX. +type apkIndexes struct { + mu sync.Mutex + regs map[string]apkIndex +} + +type apkIndex struct { + reg registries.Registry + fetched time.Time +} + +// mirrorAPK mirrors an Alpine package under the path the handler caches it +// at, {repository}/v{release}/{component}/{arch}/{file}. The PURL namespace +// names the upstream.apk repository; the distro and arch qualifiers give +// the release and architecture, and the package is looked up in the main +// and community indexes of that release. +func (m *Mirror) mirrorAPK(ctx context.Context, pv PackageVersion, tracker *progressTracker) { + fail := func(err error) { m.recordFailure(tracker, pv, err) } + repository, name, ok := strings.Cut(pv.Name, "/") + if !ok { + repository, name = "alpine", pv.Name + } + repositoryURL, ok := handler.APKRepositories(m.APKRepositories)[repository] + if !ok { + fail(fmt.Errorf("unknown APK repository %q: it isn't in upstream.apk", repository)) + return + } + match := apkDistroPattern.FindStringSubmatch(pv.Qualifiers["distro"]) + if match == nil { + fail(fmt.Errorf("apk packages need a distro qualifier with the Alpine release, such as distro=alpine-3.20.3, not %q", pv.Qualifiers["distro"])) + return + } + release := "v" + match[1] + "." + match[2] + arch := pv.Qualifiers["arch"] + if !apkArchPattern.MatchString(arch) { + fail(fmt.Errorf("apk packages need an arch qualifier, such as arch=x86_64, not %q", arch)) + return + } + + for _, component := range apkComponents { + dir := release + "/" + component + "/" + arch + artifact, found, err := m.apkArtifact(ctx, repositoryURL+"/"+dir, name, pv.Version) + if err != nil { + fail(fmt.Errorf("reading %s/APKINDEX.tar.gz: %w", dir, err)) + return + } + if !found { + continue + } + cacheFilename := handler.APKCacheFilename(repository, dir+"/"+artifact.Filename) + result, err := m.proxy.GetOrFetchArtifactFromURL(ctx, pv.Ecosystem, name, pv.Version, cacheFilename, artifact.URL) + if err != nil { + fail(err) + return + } + m.recordResult(tracker, pv, result) + return + } + fail(fmt.Errorf("%s %s isn't in %s of %s for %s", name, pv.Version, strings.Join(apkComponents, " or "), release, arch)) +} + +// apkArtifact returns the file of version of name that the index in dirURL +// lists, and whether it lists one. +func (m *Mirror) apkArtifact(ctx context.Context, dirURL, name, version string) (registries.Artifact, bool, error) { + reg, err := m.apkRegistry(dirURL) + if err != nil { + return registries.Artifact{}, false, err + } + versions, err := reg.FetchVersions(ctx, name) + if isNotFound(err) { + return registries.Artifact{}, false, nil + } + if err != nil { + m.apk.forget(dirURL) + return registries.Artifact{}, false, err + } + for _, v := range versions { + if v.Number == version && len(v.Artifacts) > 0 { + return v.Artifacts[0], true, nil + } + } + return registries.Artifact{}, false, nil +} + +// apkRegistry returns the client for the repository directory at dirURL, +// reusing it, and the index it read, for apkIndexTTL. +func (m *Mirror) apkRegistry(dirURL string) (registries.Registry, error) { //nolint:ireturn // registries.New returns the interface + m.apk.mu.Lock() + defer m.apk.mu.Unlock() + if idx, ok := m.apk.regs[dirURL]; ok && time.Since(idx.fetched) < apkIndexTTL { + return idx.reg, nil + } + r, _ := m.Versions.(RegistryVersions) + reg, err := registries.New("apk", dirURL, r.client()) + if err != nil { + return nil, err + } + if m.apk.regs == nil { + m.apk.regs = map[string]apkIndex{} + } + m.apk.regs[dirURL] = apkIndex{reg: reg, fetched: time.Now()} + return reg, nil +} + +// forget drops the client for dirURL, so an index that failed to load is +// read again next time. +func (a *apkIndexes) forget(dirURL string) { + a.mu.Lock() + defer a.mu.Unlock() + delete(a.regs, dirURL) +} + +// isNotFound reports whether err means a package or index doesn't exist, +// such as a release without a community repository for an architecture. +func isNotFound(err error) bool { + if errors.Is(err, registries.ErrNotFound) { + return true + } + var httpErr *registries.HTTPError + return errors.As(err, &httpErr) && httpErr.IsNotFound() +} diff --git a/internal/mirror/mirror.go b/internal/mirror/mirror.go index ac20a4e..d5ec9a5 100644 --- a/internal/mirror/mirror.go +++ b/internal/mirror/mirror.go @@ -45,11 +45,16 @@ type Mirror struct { // uses https://pkg.julialang.org. JuliaUpstream string + // APKRepositories are the upstream.apk repositories by name. Nil uses + // the official Alpine mirror as "alpine". + APKRepositories map[string]string + // RefusePackages, when set, fails every item Images does not take with // this reason instead of fetching it, so nothing is cached. RefusePackages string cran cranIndex + apk apkIndexes } // New creates a new Mirror with the given dependencies. @@ -270,6 +275,10 @@ func (m *Mirror) mirrorOne(ctx context.Context, pv PackageVersion, tracker *prog m.mirrorCRAN(ctx, pv, tracker) return } + if pv.Ecosystem == "alpine" { + m.mirrorAPK(ctx, pv, tracker) + return + } result, err := m.proxy.GetOrFetchArtifact(ctx, pv.Ecosystem, pv.Name, pv.Version, "") if err != nil { m.recordFailure(tracker, pv, err) diff --git a/internal/mirror/source.go b/internal/mirror/source.go index ecc3a0c..1d3954c 100644 --- a/internal/mirror/source.go +++ b/internal/mirror/source.go @@ -148,8 +148,11 @@ func mirrorTarget(p *purl.PURL) (ecosystem, name string, qualifiers map[string]s name = p.Namespace + "/" + p.Name } qualifiers = fileQualifiers(ecosystem, p.Qualifiers.Map()) - if ecosystem == "julia" { + switch ecosystem { + case "julia": qualifiers = map[string]string{"uuid": p.Qualifier("uuid")} + case "alpine": + qualifiers = map[string]string{"arch": p.Qualifier("arch"), "distro": p.Qualifier("distro")} } return ecosystem, name, qualifiers } diff --git a/internal/server/mirror_apk_test.go b/internal/server/mirror_apk_test.go new file mode 100644 index 0000000..8172744 --- /dev/null +++ b/internal/server/mirror_apk_test.go @@ -0,0 +1,142 @@ +package server + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + + "github.com/git-pkgs/proxy/internal/config" + "github.com/git-pkgs/proxy/internal/handler" +) + +const ( + apkBusyboxPath = "/v3.20/main/x86_64/busybox-1.36.1-r29.apk" + apkHtopPath = "/v3.20/community/x86_64/htop-3.3.0-r0.apk" + apkMainIndex = "/v3.20/main/x86_64/APKINDEX.tar.gz" + apkCommIndex = "/v3.20/community/x86_64/APKINDEX.tar.gz" +) + +// apkUpstream serves an Alpine mirror with busybox in v3.20/main and htop in +// v3.20/community, for x86_64. It counts requests by path. +type apkUpstream struct { + *httptest.Server + mu sync.Mutex + files map[string]string + hits map[string]int +} + +func apkIndexArchive(t *testing.T, index string) string { + t.Helper() + var buf bytes.Buffer + zw := gzip.NewWriter(&buf) + tw := tar.NewWriter(zw) + if err := tw.WriteHeader(&tar.Header{Name: "APKINDEX", Mode: 0o644, Size: int64(len(index))}); err != nil { + t.Fatal(err) + } + _, _ = io.WriteString(tw, index) + _ = tw.Close() + _ = zw.Close() + return buf.String() +} + +func newAPKUpstream(t *testing.T) *apkUpstream { + t.Helper() + u := &apkUpstream{ + files: map[string]string{ + apkMainIndex: apkIndexArchive(t, "P:busybox\nV:1.36.1-r29\nA:x86_64\nS:18\n\nP:musl\nV:1.2.5-r0\nA:x86_64\nS:15\n\n"), + apkCommIndex: apkIndexArchive(t, "P:htop\nV:3.3.0-r0\nA:x86_64\nS:15\n\n"), + apkBusyboxPath: "busybox 1.36.1-r29", + apkHtopPath: "htop 3.3.0-r0 x86", + "/v3.20/main/x86_64/musl-1.2.5-r0.apk": "musl 1.2.5-r0", + }, + hits: map[string]int{}, + } + u.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + u.mu.Lock() + body, ok := u.files[r.URL.Path] + u.hits[r.URL.Path]++ + u.mu.Unlock() + if !ok { + http.NotFound(w, r) + return + } + _, _ = io.WriteString(w, body) + })) + t.Cleanup(u.Close) + return u +} + +func (u *apkUpstream) hitsFor(path string) int { + u.mu.Lock() + defer u.mu.Unlock() + return u.hits[path] +} + +func apkMirrorConfig(upstream *apkUpstream) *config.Config { + cfg := config.Default() + cfg.Upstream.APK = map[string]string{"alpine": upstream.URL} + return cfg +} + +func getAPK(proxy *handler.Proxy, upstream *apkUpstream, path string) *httptest.ResponseRecorder { + rec := httptest.NewRecorder() + h := handler.NewAPKHandler(proxy, "http://proxy.test", map[string]string{"alpine": upstream.URL}) + h.Routes().ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/alpine"+path, nil)) + return rec +} + +func TestMirrorAPKPackagesAreServedFromCache(t *testing.T) { + upstream := newAPKUpstream(t) + + progress, proxy := mirrorWithConfig(t, apkMirrorConfig(upstream), + "pkg:apk/alpine/busybox@1.36.1-r29?arch=x86_64&distro=alpine-3.20.3", + "pkg:apk/alpine/htop@3.3.0-r0?arch=x86_64&distro=alpine-3.20.3", + "pkg:apk/alpine/musl@1.2.5-r0?arch=x86_64&distro=alpine-3.20.3") + + if progress.Completed != 3 || progress.Failed != 0 { + t.Fatalf("completed = %d, failed = %d, errors = %v", progress.Completed, progress.Failed, progress.Errors) + } + for _, path := range []string{apkBusyboxPath, apkHtopPath, apkMainIndex, apkCommIndex} { + if got := upstream.hitsFor(path); got != 1 { + t.Errorf("%s fetched %d times, want 1", path, got) + } + } + + // apk gets the packages from the cache once the upstream is down. + upstream.Close() + for path, want := range map[string]string{ + apkBusyboxPath: "busybox 1.36.1-r29", + apkHtopPath: "htop 3.3.0-r0 x86", + } { + if rec := getAPK(proxy, upstream, path); rec.Code != http.StatusOK || rec.Body.String() != want { + t.Errorf("%s from cache: status = %d, body = %q, want 200 and %q", path, rec.Code, rec.Body, want) + } + } +} + +func TestMirrorAPKRefusesIncompletePURLs(t *testing.T) { + upstream := newAPKUpstream(t) + + tests := map[string]string{ + "pkg:apk/alpine/busybox@1.36.1-r29?arch=x86_64": "distro", + "pkg:apk/alpine/busybox@1.36.1-r29?distro=alpine-3.20.3": "arch", + "pkg:apk/alpine/busybox@1.36.1-r29?arch=x86_64&distro=debian-12": "distro", + "pkg:apk/other/busybox@1.36.1-r29?arch=x86_64&distro=alpine-3.20.3": "upstream.apk", + "pkg:apk/alpine/busybox@9.9.9-r0?arch=x86_64&distro=alpine-3.20.3": "9.9.9-r0", + "pkg:apk/alpine/busybox@1.36.1-r29?arch=../../etc&distro=alpine-3.20.3": "arch", + } + for p, want := range tests { + t.Run(p, func(t *testing.T) { + progress, _ := mirrorWithConfig(t, apkMirrorConfig(upstream), p) + if progress.Failed != 1 || len(progress.Errors) != 1 || !strings.Contains(progress.Errors[0].Error, want) { + t.Errorf("failed = %d, errors = %v; want one error mentioning %q", progress.Failed, progress.Errors, want) + } + }) + } +} diff --git a/internal/server/mirror_fetch_test.go b/internal/server/mirror_fetch_test.go index dba27d3..3e2cebb 100644 --- a/internal/server/mirror_fetch_test.go +++ b/internal/server/mirror_fetch_test.go @@ -64,6 +64,7 @@ func mirrorWithVersions(t *testing.T, cfg *config.Config, versions mirror.Versio m.Charts = handler.NewHelmHandler(proxy, cfg.BaseURL, cfg.Upstream.Helm) m.Versions = versions m.JuliaUpstream = cfg.Upstream.Julia + m.APKRepositories = cfg.Upstream.APK if versions == nil { m.Versions = NewMirrorVersions(cfg.Upstream, proxy.HTTPClient) } diff --git a/internal/server/mirror_versions.go b/internal/server/mirror_versions.go index 27b3d51..6d4599e 100644 --- a/internal/server/mirror_versions.go +++ b/internal/server/mirror_versions.go @@ -37,11 +37,12 @@ func (s *Server) newMirror(proxy *handler.Proxy, workers int) *mirror.Mirror { } // UseMirrorUpstreams points m at the configured upstreams: it lists -// versions from them, through client, and fetches Julia packages from -// upstream.julia. +// versions from them, through client, fetches Julia packages from +// upstream.julia and Alpine packages from upstream.apk. func UseMirrorUpstreams(m *mirror.Mirror, upstream config.UpstreamConfig, client *http.Client) { m.Versions = NewMirrorVersions(upstream, client) m.JuliaUpstream = upstream.Julia + m.APKRepositories = upstream.APK } // mirrorVersionsURL returns the registry base URL to list a package's From adcda6ca8baa5140eee575fc4ace5655dfd9f98b Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Wed, 4 Nov 2026 15:54:10 +0100 Subject: [PATCH 540/633] Read APKINDEX.tar.gz also where Packages.adb is served beside it --- internal/mirror/apk.go | 4 +++- internal/server/mirror_apk_test.go | 14 ++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/internal/mirror/apk.go b/internal/mirror/apk.go index b6b4ea3..7b80261 100644 --- a/internal/mirror/apk.go +++ b/internal/mirror/apk.go @@ -121,7 +121,9 @@ func (m *Mirror) apkRegistry(dirURL string) (registries.Registry, error) { //nol return idx.reg, nil } r, _ := m.Versions.(RegistryVersions) - reg, err := registries.New("apk", dirURL, r.client()) + // Naming the v2 index reads it also where Packages.adb is served beside + // it, which the client would try first and refuse. + reg, err := registries.New("apk", dirURL+"/APKINDEX.tar.gz", r.client()) if err != nil { return nil, err } diff --git a/internal/server/mirror_apk_test.go b/internal/server/mirror_apk_test.go index 8172744..c82602d 100644 --- a/internal/server/mirror_apk_test.go +++ b/internal/server/mirror_apk_test.go @@ -120,6 +120,20 @@ func TestMirrorAPKPackagesAreServedFromCache(t *testing.T) { } } +func TestMirrorAPKReadsV2IndexBesideV3(t *testing.T) { + // A repository moving to apk v3 serves Packages.adb next to + // APKINDEX.tar.gz; the mirror reads the v2 index. + upstream := newAPKUpstream(t) + upstream.files["/v3.20/main/x86_64/Packages.adb"] = "ADB.v3-index" + + progress, _ := mirrorWithConfig(t, apkMirrorConfig(upstream), + "pkg:apk/alpine/busybox@1.36.1-r29?arch=x86_64&distro=alpine-3.20.3") + + if progress.Completed != 1 || progress.Failed != 0 { + t.Fatalf("completed = %d, failed = %d, errors = %v", progress.Completed, progress.Failed, progress.Errors) + } +} + func TestMirrorAPKRefusesIncompletePURLs(t *testing.T) { upstream := newAPKUpstream(t) From 6d3410df41f583816513b6bbdcc668b343810786 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Wed, 4 Nov 2026 16:05:16 +0100 Subject: [PATCH 541/633] Use sbom's ProjectRoots for the project's own packages Drop projectRoots and readFromLockFile; sbom now decides which inferred roots are the project. sbom is on the merge commit of git-pkgs/sbom#27 until it is tagged. --- go.mod | 2 +- go.sum | 4 +-- internal/inventory/inventory.go | 38 +++------------------------- internal/inventory/inventory_test.go | 4 +-- 4 files changed, 9 insertions(+), 39 deletions(-) diff --git a/go.mod b/go.mod index 2fe061f..b6182d2 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( github.com/git-pkgs/magic v0.6.0 github.com/git-pkgs/purl v0.2.1 github.com/git-pkgs/registries v0.13.0 - github.com/git-pkgs/sbom v0.1.8 + github.com/git-pkgs/sbom v0.1.9-0.20261104140234-f3632df60df9 github.com/git-pkgs/spdx v0.3.3 github.com/git-pkgs/vers v0.7.2 github.com/go-chi/chi/v5 v5.5.0 diff --git a/go.sum b/go.sum index 3e33929..6c0e3d1 100644 --- a/go.sum +++ b/go.sum @@ -137,8 +137,8 @@ github.com/git-pkgs/purl v0.2.1 h1:b46hRlOMbrG6aXjjFJdYixGi5WTh0TLlAnH9iNeuOS0= github.com/git-pkgs/purl v0.2.1/go.mod h1:l8rCfvt5iXVrFIcoQ8+vCLOaHhrtEW2id0LENP+IzMw= github.com/git-pkgs/registries v0.13.0 h1:5i9dHnbiXseIbXLIZ61YnuFySJ9dQX+KEu1MrmwB2BU= github.com/git-pkgs/registries v0.13.0/go.mod h1:1g/jXVi1M/3jWvW1Wom1NMnPrhglJ7UVEZ215l2Zh0M= -github.com/git-pkgs/sbom v0.1.8 h1:B+ohNEQfk1M+ch1/hMUKAitZp792g6fUrMaO4GTSqhI= -github.com/git-pkgs/sbom v0.1.8/go.mod h1:KIkZX0v31l62SeFsi2qsQXcZRVrAXvIdPQ4x4en+G2M= +github.com/git-pkgs/sbom v0.1.9-0.20261104140234-f3632df60df9 h1:mghNOVmaxRvz8I0TsY+augx8GcVuvqbtD5fjMrBXGa8= +github.com/git-pkgs/sbom v0.1.9-0.20261104140234-f3632df60df9/go.mod h1:KIkZX0v31l62SeFsi2qsQXcZRVrAXvIdPQ4x4en+G2M= github.com/git-pkgs/spdx v0.3.3 h1:xXlUoZzjDSkwxC6mWHfaW+kc8a0+LH6d+gGHuZWftqI= github.com/git-pkgs/spdx v0.3.3/go.mod h1:n9rAicgw+Wqtfursju6oKaAL6cUCTiIS926mb1KWoKU= github.com/git-pkgs/vers v0.7.2 h1:Ctex1O+Ai/FONmyDaw37EolBXg0og6UdxbCjjYbeRHs= diff --git a/internal/inventory/inventory.go b/internal/inventory/inventory.go index a0b6b2b..c48f387 100644 --- a/internal/inventory/inventory.go +++ b/internal/inventory/inventory.go @@ -76,9 +76,6 @@ func FromPURL(s string) (Component, error) { // naming the module the binary was built from. const syftMainModule = "syft:metadata:mainModule" -// syftMetadataType is the property naming where syft read a package from. -const syftMetadataType = "syft:package:metadataType" - // unknownVersion is what syft gives a package without a resolved version, // such as the project's own module in go.mod or the root of a lock file. const unknownVersion = "UNKNOWN" @@ -95,7 +92,10 @@ func Parse(data []byte) (components []Component, skipped int, err error) { return nil, 0, err } ownModules := mainModules(doc.Packages) - roots := projectRoots(doc) + roots := map[string]bool{} + for _, id := range doc.ProjectRoots() { + roots[id] = true + } seen := make(map[[2]string]bool, len(doc.Packages)) for _, pkg := range doc.Packages { c, ok := componentOf(pkg) @@ -133,36 +133,6 @@ func componentOf(pkg sbom.Package) (Component, bool) { return c, true } -// projectRoots returns the IDs of the roots sbom finds that are the -// project: all of them when the document declares them, otherwise only those -// read from a lock file. An image's packages that nothing depends on, such as -// busybox, are roots of the graph but not the project. -func projectRoots(doc *sbom.SBOM) map[string]bool { - ids, declared := doc.Roots() - roots := make(map[string]bool, len(ids)) - for _, id := range ids { - roots[id] = declared - } - if declared { - return roots - } - for _, pkg := range doc.Packages { - if _, ok := roots[pkg.ID]; ok && readFromLockFile(pkg) { - roots[pkg.ID] = true - } - } - return roots -} - -func readFromLockFile(pkg sbom.Package) bool { - for _, prop := range pkg.Properties { - if prop.Name == syftMetadataType && strings.Contains(prop.Value, "lock") { - return true - } - } - return false -} - // mainModules returns the Go modules the SBOM's binaries were built from, // which are the project's own. Module paths are case-sensitive and package // URLs keep their case, so they are compared as they are. diff --git a/internal/inventory/inventory_test.go b/internal/inventory/inventory_test.go index 36e67ef..89b070b 100644 --- a/internal/inventory/inventory_test.go +++ b/internal/inventory/inventory_test.go @@ -225,10 +225,10 @@ func TestParseSkipsTheProjectsOwnPackages(t *testing.T) { {"bom-ref":"cobra","type":"library","name":"github.com/spf13/cobra","version":"v1.8.0","purl":"pkg:golang/github.com/spf13/cobra@v1.8.0", "properties":[{"name":"syft:metadata:mainModule","value":"github.com/acme/app"}]}, {"bom-ref":"root","type":"library","name":"app-frontend","version":"2.1.0","purl":"pkg:npm/app-frontend@2.1.0", - "properties":[{"name":"syft:package:metadataType","value":"javascript-npm-package-lock-entry"}]}, + "properties":[{"name":"syft:package:metadataType","value":"javascript-npm-package-lock-entry"},{"name":"syft:location:0:path","value":"/package-lock.json"}]}, {"bom-ref":"nover","type":"library","name":"app-tools","purl":"pkg:npm/app-tools"}, {"bom-ref":"jsdom","type":"library","name":"jsdom","version":"30.1.3","purl":"pkg:npm/jsdom@30.1.3", - "properties":[{"name":"syft:package:metadataType","value":"javascript-npm-package-lock-entry"}]} + "properties":[{"name":"syft:package:metadataType","value":"javascript-npm-package-lock-entry"},{"name":"syft:location:0:path","value":"/package-lock.json"}]} ],"dependencies":[{"ref":"root","dependsOn":["jsdom"]}]}` components, skipped, err := Parse([]byte(doc)) if err != nil { From 4e2539633180d43765a088c9775f37c9e5aa33b0 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Wed, 4 Nov 2026 16:09:00 +0100 Subject: [PATCH 542/633] Refuse distro qualifiers that aren't an Alpine release --- docs/configuration.md | 2 +- internal/mirror/apk.go | 5 +++-- internal/server/mirror_apk_test.go | 14 ++++++++------ 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index c71a8ca..35e2e48 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1726,7 +1726,7 @@ A Swift release is mirrored from `upstream.swift` with its source archive, check A CRAN version is mirrored as its source package from `upstream.cran`, for example `pkg:cran/glue@1.8.1`, and cached under the file name, so it is served for both `/cran/src/contrib/{file}` and `/cran/src/contrib/Archive/{name}/{file}`, where CRAN moves it once a newer version is published. The current version is fetched from `src/contrib` and checked against the MD5 sum in `src/contrib/PACKAGES.gz`; older versions are fetched unchecked from `src/contrib/Archive/{name}/`, as the CRAN handler fetches them. Binary packages aren't mirrored, and neither are the `PACKAGES` indexes, so `install.packages` needs the upstream or an index the proxy already cached; installing the tarball by URL works without either. -An Alpine package is mirrored from the repository in `upstream.apk` named as the namespace, `alpine` by default, as syft writes it in image SBOMs: `pkg:apk/alpine/busybox@1.36.1-r29?arch=x86_64&distro=alpine-3.20.3`. The `distro` qualifier gives the release (`v3.20`) and `arch` the architecture; packages without either are refused. The package is looked up in the `main` and then the `community` `APKINDEX.tar.gz` of that release and cached under the path `/apk/{name}/{release}/{repo}/{arch}/{file}` serves it from. The file isn't checked by the proxy, as when serving, since the index checksum covers only the package's control section; apk checks it on install. The indexes aren't mirrored, so `apk add` needs the upstream or an index the proxy already cached. Give a version: Alpine repositories keep only the current one. +An Alpine package is mirrored from the repository in `upstream.apk` named as the namespace, `alpine` by default, as syft writes it in image SBOMs: `pkg:apk/alpine/busybox@1.36.1-r29?arch=x86_64&distro=alpine-3.20.3`. The `distro` qualifier gives the release (`v3.20`) and `arch` the architecture; packages without either, and edge packages, are refused. The package is looked up in the `main` and then the `community` `APKINDEX.tar.gz` of that release and cached under the path `/apk/{name}/{release}/{repo}/{arch}/{file}` serves it from. The file isn't checked by the proxy, as when serving, since the index checksum covers only the package's control section; apk checks it on install. The indexes aren't mirrored, so `apk add` needs the upstream or an index the proxy already cached. Give a version: Alpine repositories keep only the current one. | Flag | Default | Description | |------|---------|-------------| diff --git a/internal/mirror/apk.go b/internal/mirror/apk.go index 7b80261..3e08572 100644 --- a/internal/mirror/apk.go +++ b/internal/mirror/apk.go @@ -19,8 +19,9 @@ const apkIndexTTL = 5 * time.Minute var ( // apkDistroPattern takes the release from syft's distro qualifier, such - // as alpine-3.20.3 for v3.20. - apkDistroPattern = regexp.MustCompile(`^alpine-(\d+)\.(\d+)(?:[.-]|$)`) + // as alpine-3.20.3 for v3.20. Edge, as alpine-3.21.0_alpha20240807, + // isn't a release and doesn't match. + apkDistroPattern = regexp.MustCompile(`^alpine-(\d+)\.(\d+)(?:\.\d+)?$`) apkArchPattern = regexp.MustCompile(`^[a-z0-9_]+$`) // apkComponents are the repositories of a release a package is looked // up in, in order. diff --git a/internal/server/mirror_apk_test.go b/internal/server/mirror_apk_test.go index c82602d..d5a8c0d 100644 --- a/internal/server/mirror_apk_test.go +++ b/internal/server/mirror_apk_test.go @@ -138,12 +138,14 @@ func TestMirrorAPKRefusesIncompletePURLs(t *testing.T) { upstream := newAPKUpstream(t) tests := map[string]string{ - "pkg:apk/alpine/busybox@1.36.1-r29?arch=x86_64": "distro", - "pkg:apk/alpine/busybox@1.36.1-r29?distro=alpine-3.20.3": "arch", - "pkg:apk/alpine/busybox@1.36.1-r29?arch=x86_64&distro=debian-12": "distro", - "pkg:apk/other/busybox@1.36.1-r29?arch=x86_64&distro=alpine-3.20.3": "upstream.apk", - "pkg:apk/alpine/busybox@9.9.9-r0?arch=x86_64&distro=alpine-3.20.3": "9.9.9-r0", - "pkg:apk/alpine/busybox@1.36.1-r29?arch=../../etc&distro=alpine-3.20.3": "arch", + "pkg:apk/alpine/busybox@1.36.1-r29?arch=x86_64": "distro", + "pkg:apk/alpine/busybox@1.36.1-r29?distro=alpine-3.20.3": "arch", + "pkg:apk/alpine/busybox@1.36.1-r29?arch=x86_64&distro=alpine-3.20-evil": "distro", + "pkg:apk/alpine/busybox@1.36.1-r29?arch=x86_64&distro=alpine-3.21.0_alpha20240807": "distro", + "pkg:apk/alpine/busybox@1.36.1-r29?arch=x86_64&distro=debian-12": "distro", + "pkg:apk/other/busybox@1.36.1-r29?arch=x86_64&distro=alpine-3.20.3": "upstream.apk", + "pkg:apk/alpine/busybox@9.9.9-r0?arch=x86_64&distro=alpine-3.20.3": "9.9.9-r0", + "pkg:apk/alpine/busybox@1.36.1-r29?arch=../../etc&distro=alpine-3.20.3": "arch", } for p, want := range tests { t.Run(p, func(t *testing.T) { From 7095ab67ef105c5466f6431a162ba2839eec8a70 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Wed, 4 Nov 2026 16:16:35 +0100 Subject: [PATCH 543/633] Keep the Projects link on an empty projects page ProjectsListData.Projects hid Layout.Projects, so the header left the link out when no project had been uploaded. --- internal/server/projects_ui.go | 8 +++++--- internal/server/projects_ui_test.go | 7 +++++++ internal/server/templates/pages/projects_list.html | 6 +++--- 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/internal/server/projects_ui.go b/internal/server/projects_ui.go index 5bafb39..c7c2c05 100644 --- a/internal/server/projects_ui.go +++ b/internal/server/projects_ui.go @@ -13,7 +13,9 @@ import ( // ProjectsListData contains data for rendering the projects list page. type ProjectsListData struct { Layout - Projects []ProjectRow + // Rows isn't called Projects, which would hide Layout.Projects from + // the header. + Rows []ProjectRow } // ProjectRow is a project in the UI. @@ -79,9 +81,9 @@ func (s *Server) handleProjectsList(w http.ResponseWriter, r *http.Request) { http.Error(w, "failed to list projects", http.StatusInternalServerError) return } - data := ProjectsListData{Layout: s.layoutFor(r), Projects: make([]ProjectRow, len(projects))} + data := ProjectsListData{Layout: s.layoutFor(r), Rows: make([]ProjectRow, len(projects))} for i, p := range projects { - data.Projects[i] = projectRow(p) + data.Rows[i] = projectRow(p) } if err := s.templates.Render(w, "projects_list", data); err != nil { s.logger.Error("failed to render projects list", "error", err) diff --git a/internal/server/projects_ui_test.go b/internal/server/projects_ui_test.go index 5be2224..bd868c9 100644 --- a/internal/server/projects_ui_test.go +++ b/internal/server/projects_ui_test.go @@ -116,3 +116,10 @@ func TestProjectPageFailsWhenComponentsCannotBeRead(t *testing.T) { t.Errorf("status = %d, want 500 rather than an empty project", rec.Code) } } + +func TestProjectsPageLinksProjectsWhenEmpty(t *testing.T) { + ts := newProjectsTestServer(t) + + page := ts.page(t, "/ui/projects") + assertContains(t, page, `href="/ui/projects"`, "0 projects") +} diff --git a/internal/server/templates/pages/projects_list.html b/internal/server/templates/pages/projects_list.html index f5ce62d..d7256da 100644 --- a/internal/server/templates/pages/projects_list.html +++ b/internal/server/templates/pages/projects_list.html @@ -3,13 +3,13 @@ {{define "content"}}

Projects

-

{{len .Projects}} projects, as of their latest SBOM upload

+

{{len .Rows}} projects, as of their latest SBOM upload

-{{if .Projects}} +{{if .Rows}}
- {{range .Projects}} + {{range .Rows}}
{{.Name}} From e4e14696d0f7fdd870c1b8482eac298d2325ff73 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Wed, 4 Nov 2026 16:20:30 +0100 Subject: [PATCH 544/633] Say which edge packages the APK mirror can't recognise --- docs/configuration.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/configuration.md b/docs/configuration.md index 35e2e48..2f4e31e 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1726,7 +1726,7 @@ A Swift release is mirrored from `upstream.swift` with its source archive, check A CRAN version is mirrored as its source package from `upstream.cran`, for example `pkg:cran/glue@1.8.1`, and cached under the file name, so it is served for both `/cran/src/contrib/{file}` and `/cran/src/contrib/Archive/{name}/{file}`, where CRAN moves it once a newer version is published. The current version is fetched from `src/contrib` and checked against the MD5 sum in `src/contrib/PACKAGES.gz`; older versions are fetched unchecked from `src/contrib/Archive/{name}/`, as the CRAN handler fetches them. Binary packages aren't mirrored, and neither are the `PACKAGES` indexes, so `install.packages` needs the upstream or an index the proxy already cached; installing the tarball by URL works without either. -An Alpine package is mirrored from the repository in `upstream.apk` named as the namespace, `alpine` by default, as syft writes it in image SBOMs: `pkg:apk/alpine/busybox@1.36.1-r29?arch=x86_64&distro=alpine-3.20.3`. The `distro` qualifier gives the release (`v3.20`) and `arch` the architecture; packages without either, and edge packages, are refused. The package is looked up in the `main` and then the `community` `APKINDEX.tar.gz` of that release and cached under the path `/apk/{name}/{release}/{repo}/{arch}/{file}` serves it from. The file isn't checked by the proxy, as when serving, since the index checksum covers only the package's control section; apk checks it on install. The indexes aren't mirrored, so `apk add` needs the upstream or an index the proxy already cached. Give a version: Alpine repositories keep only the current one. +An Alpine package is mirrored from the repository in `upstream.apk` named as the namespace, `alpine` by default, as syft writes it in image SBOMs: `pkg:apk/alpine/busybox@1.36.1-r29?arch=x86_64&distro=alpine-3.20.3`. The `distro` qualifier gives the release (`v3.20`) and `arch` the architecture; packages without either are refused. Edge isn't mirrored: a `distro` with a pre-release suffix, such as `alpine-3.21.0_alpha20240807`, is refused, but an edge system that reports a plain release number can't be told apart and is mirrored from that release. The package is looked up in the `main` and then the `community` `APKINDEX.tar.gz` of that release and cached under the path `/apk/{name}/{release}/{repo}/{arch}/{file}` serves it from. The file isn't checked by the proxy, as when serving, since the index checksum covers only the package's control section; apk checks it on install. The indexes aren't mirrored, so `apk add` needs the upstream or an index the proxy already cached. Give a version: Alpine repositories keep only the current one. | Flag | Default | Description | |------|---------|-------------| From c3f0c23dc3624eac050f88f7ee6659df30ea96e5 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Wed, 4 Nov 2026 16:31:50 +0100 Subject: [PATCH 545/633] Name the repository segment of the APK path in the mirror docs --- docs/configuration.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/configuration.md b/docs/configuration.md index 2f4e31e..73a0b92 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1726,7 +1726,7 @@ A Swift release is mirrored from `upstream.swift` with its source archive, check A CRAN version is mirrored as its source package from `upstream.cran`, for example `pkg:cran/glue@1.8.1`, and cached under the file name, so it is served for both `/cran/src/contrib/{file}` and `/cran/src/contrib/Archive/{name}/{file}`, where CRAN moves it once a newer version is published. The current version is fetched from `src/contrib` and checked against the MD5 sum in `src/contrib/PACKAGES.gz`; older versions are fetched unchecked from `src/contrib/Archive/{name}/`, as the CRAN handler fetches them. Binary packages aren't mirrored, and neither are the `PACKAGES` indexes, so `install.packages` needs the upstream or an index the proxy already cached; installing the tarball by URL works without either. -An Alpine package is mirrored from the repository in `upstream.apk` named as the namespace, `alpine` by default, as syft writes it in image SBOMs: `pkg:apk/alpine/busybox@1.36.1-r29?arch=x86_64&distro=alpine-3.20.3`. The `distro` qualifier gives the release (`v3.20`) and `arch` the architecture; packages without either are refused. Edge isn't mirrored: a `distro` with a pre-release suffix, such as `alpine-3.21.0_alpha20240807`, is refused, but an edge system that reports a plain release number can't be told apart and is mirrored from that release. The package is looked up in the `main` and then the `community` `APKINDEX.tar.gz` of that release and cached under the path `/apk/{name}/{release}/{repo}/{arch}/{file}` serves it from. The file isn't checked by the proxy, as when serving, since the index checksum covers only the package's control section; apk checks it on install. The indexes aren't mirrored, so `apk add` needs the upstream or an index the proxy already cached. Give a version: Alpine repositories keep only the current one. +An Alpine package is mirrored from the repository in `upstream.apk` named as the namespace, `alpine` by default, as syft writes it in image SBOMs: `pkg:apk/alpine/busybox@1.36.1-r29?arch=x86_64&distro=alpine-3.20.3`. The `distro` qualifier gives the release (`v3.20`) and `arch` the architecture; packages without either are refused. Edge isn't mirrored: a `distro` with a pre-release suffix, such as `alpine-3.21.0_alpha20240807`, is refused, but an edge system that reports a plain release number can't be told apart and is mirrored from that release. The package is looked up in the `main` and then the `community` `APKINDEX.tar.gz` of that release and cached under the path `/apk/{repository}/{release}/{repo}/{arch}/{file}` serves it from. The file isn't checked by the proxy, as when serving, since the index checksum covers only the package's control section; apk checks it on install. The indexes aren't mirrored, so `apk add` needs the upstream or an index the proxy already cached. Give a version: Alpine repositories keep only the current one. | Flag | Default | Description | |------|---------|-------------| From 60872fe990f32788cb7c79c7eac5fc27b0c8d2d2 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Thu, 5 Nov 2026 09:34:02 +0100 Subject: [PATCH 546/633] Use sbom v0.1.9 --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index b6182d2..76a1fad 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( github.com/git-pkgs/magic v0.6.0 github.com/git-pkgs/purl v0.2.1 github.com/git-pkgs/registries v0.13.0 - github.com/git-pkgs/sbom v0.1.9-0.20261104140234-f3632df60df9 + github.com/git-pkgs/sbom v0.1.9 github.com/git-pkgs/spdx v0.3.3 github.com/git-pkgs/vers v0.7.2 github.com/go-chi/chi/v5 v5.5.0 diff --git a/go.sum b/go.sum index 6c0e3d1..c6b5f02 100644 --- a/go.sum +++ b/go.sum @@ -137,8 +137,8 @@ github.com/git-pkgs/purl v0.2.1 h1:b46hRlOMbrG6aXjjFJdYixGi5WTh0TLlAnH9iNeuOS0= github.com/git-pkgs/purl v0.2.1/go.mod h1:l8rCfvt5iXVrFIcoQ8+vCLOaHhrtEW2id0LENP+IzMw= github.com/git-pkgs/registries v0.13.0 h1:5i9dHnbiXseIbXLIZ61YnuFySJ9dQX+KEu1MrmwB2BU= github.com/git-pkgs/registries v0.13.0/go.mod h1:1g/jXVi1M/3jWvW1Wom1NMnPrhglJ7UVEZ215l2Zh0M= -github.com/git-pkgs/sbom v0.1.9-0.20261104140234-f3632df60df9 h1:mghNOVmaxRvz8I0TsY+augx8GcVuvqbtD5fjMrBXGa8= -github.com/git-pkgs/sbom v0.1.9-0.20261104140234-f3632df60df9/go.mod h1:KIkZX0v31l62SeFsi2qsQXcZRVrAXvIdPQ4x4en+G2M= +github.com/git-pkgs/sbom v0.1.9 h1:sfF5cJAwmgrm2N4HTXQWjiBe8tdu8xwhjwfAOyDNMfc= +github.com/git-pkgs/sbom v0.1.9/go.mod h1:KIkZX0v31l62SeFsi2qsQXcZRVrAXvIdPQ4x4en+G2M= github.com/git-pkgs/spdx v0.3.3 h1:xXlUoZzjDSkwxC6mWHfaW+kc8a0+LH6d+gGHuZWftqI= github.com/git-pkgs/spdx v0.3.3/go.mod h1:n9rAicgw+Wqtfursju6oKaAL6cUCTiIS926mb1KWoKU= github.com/git-pkgs/vers v0.7.2 h1:Ctex1O+Ai/FONmyDaw37EolBXg0og6UdxbCjjYbeRHs= From c20f64ebaa1151e4b65f31452c6d660f054dad4a Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Thu, 5 Nov 2026 10:48:44 +0100 Subject: [PATCH 547/633] Keep the apk namespace for the mirror's repository lookup The inventory drops the namespace from apk names so they match the handler's, but the apk mirror picks the upstream.apk repository from it. --- internal/mirror/apk.go | 6 +++--- internal/mirror/source.go | 11 +++++++++-- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/internal/mirror/apk.go b/internal/mirror/apk.go index 3e08572..9746f20 100644 --- a/internal/mirror/apk.go +++ b/internal/mirror/apk.go @@ -47,9 +47,9 @@ type apkIndex struct { // and community indexes of that release. func (m *Mirror) mirrorAPK(ctx context.Context, pv PackageVersion, tracker *progressTracker) { fail := func(err error) { m.recordFailure(tracker, pv, err) } - repository, name, ok := strings.Cut(pv.Name, "/") - if !ok { - repository, name = "alpine", pv.Name + repository, name := pv.Namespace, pv.Name + if repository == "" { + repository = "alpine" } repositoryURL, ok := handler.APKRepositories(m.APKRepositories)[repository] if !ok { diff --git a/internal/mirror/source.go b/internal/mirror/source.go index 1e326ee..2c41705 100644 --- a/internal/mirror/source.go +++ b/internal/mirror/source.go @@ -16,6 +16,9 @@ type PackageVersion struct { Ecosystem string Name string Version string + // Namespace is the PURL namespace of apk packages, which Name leaves + // out. It picks the upstream.apk repository. + Namespace string // Qualifiers are the PURL qualifiers. Images use repository_url, tag and // arch; PyPI uses file_name, and conda subdir, build and type, to select // which files of the version are mirrored. Julia uses uuid to identify @@ -61,11 +64,15 @@ func (s *PURLSource) Enumerate(ctx context.Context, fn func(PackageVersion) erro } ecosystem, name, qualifiers := mirrorTarget(p) + var namespace string + if ecosystem == "alpine" { + namespace = p.Namespace + } // Images are never expanded to all tags, and conda packages without // a channel aren't listed; both fail when they are mirrored. if p.Version != "" || isImageEcosystem(ecosystem) || (ecosystem == "conda" && !strings.Contains(name, "/")) { - pv := PackageVersion{Ecosystem: ecosystem, Name: name, Version: p.Version, Qualifiers: qualifiers} + pv := PackageVersion{Ecosystem: ecosystem, Name: name, Version: p.Version, Namespace: namespace, Qualifiers: qualifiers} if isImageEcosystem(ecosystem) { pv.Qualifiers = p.Qualifiers.Map() } @@ -84,7 +91,7 @@ func (s *PURLSource) Enumerate(ctx context.Context, fn func(PackageVersion) erro return fmt.Errorf("fetching versions for %s/%s: %w", ecosystem, name, err) } for _, v := range listed { - pv := PackageVersion{Ecosystem: ecosystem, Name: name, Version: v, Qualifiers: qualifiers} + pv := PackageVersion{Ecosystem: ecosystem, Name: name, Version: v, Namespace: namespace, Qualifiers: qualifiers} if takesPlatforms(ecosystem) { pv.Platforms = s.Platforms } From 4d707933512e4e4c70adde33e5927e29381efea7 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Thu, 5 Nov 2026 11:56:42 +0100 Subject: [PATCH 548/633] Name the apk repository in mirror errors --- internal/mirror/apk.go | 4 ++-- internal/server/mirror_apk_test.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/mirror/apk.go b/internal/mirror/apk.go index 9746f20..c4f6868 100644 --- a/internal/mirror/apk.go +++ b/internal/mirror/apk.go @@ -72,7 +72,7 @@ func (m *Mirror) mirrorAPK(ctx context.Context, pv PackageVersion, tracker *prog dir := release + "/" + component + "/" + arch artifact, found, err := m.apkArtifact(ctx, repositoryURL+"/"+dir, name, pv.Version) if err != nil { - fail(fmt.Errorf("reading %s/APKINDEX.tar.gz: %w", dir, err)) + fail(fmt.Errorf("reading %s/APKINDEX.tar.gz: %w", repositoryURL+"/"+dir, err)) return } if !found { @@ -87,7 +87,7 @@ func (m *Mirror) mirrorAPK(ctx context.Context, pv PackageVersion, tracker *prog m.recordResult(tracker, pv, result) return } - fail(fmt.Errorf("%s %s isn't in %s of %s for %s", name, pv.Version, strings.Join(apkComponents, " or "), release, arch)) + fail(fmt.Errorf("%s %s isn't in %s of %s %s for %s", name, pv.Version, strings.Join(apkComponents, " or "), repository, release, arch)) } // apkArtifact returns the file of version of name that the index in dirURL diff --git a/internal/server/mirror_apk_test.go b/internal/server/mirror_apk_test.go index d5a8c0d..0c46ba1 100644 --- a/internal/server/mirror_apk_test.go +++ b/internal/server/mirror_apk_test.go @@ -144,7 +144,7 @@ func TestMirrorAPKRefusesIncompletePURLs(t *testing.T) { "pkg:apk/alpine/busybox@1.36.1-r29?arch=x86_64&distro=alpine-3.21.0_alpha20240807": "distro", "pkg:apk/alpine/busybox@1.36.1-r29?arch=x86_64&distro=debian-12": "distro", "pkg:apk/other/busybox@1.36.1-r29?arch=x86_64&distro=alpine-3.20.3": "upstream.apk", - "pkg:apk/alpine/busybox@9.9.9-r0?arch=x86_64&distro=alpine-3.20.3": "9.9.9-r0", + "pkg:apk/alpine/busybox@9.9.9-r0?arch=x86_64&distro=alpine-3.20.3": "isn't in main or community of alpine v3.20", "pkg:apk/alpine/busybox@1.36.1-r29?arch=../../etc&distro=alpine-3.20.3": "arch", } for p, want := range tests { From eb34a1ac6f0049d6b0f126234afdbe900c48a8c3 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Thu, 5 Nov 2026 12:33:46 +0100 Subject: [PATCH 549/633] Name the apk repository, not its URL, in index errors --- internal/mirror/apk.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/mirror/apk.go b/internal/mirror/apk.go index c4f6868..30a9771 100644 --- a/internal/mirror/apk.go +++ b/internal/mirror/apk.go @@ -72,7 +72,7 @@ func (m *Mirror) mirrorAPK(ctx context.Context, pv PackageVersion, tracker *prog dir := release + "/" + component + "/" + arch artifact, found, err := m.apkArtifact(ctx, repositoryURL+"/"+dir, name, pv.Version) if err != nil { - fail(fmt.Errorf("reading %s/APKINDEX.tar.gz: %w", repositoryURL+"/"+dir, err)) + fail(fmt.Errorf("reading %s %s/APKINDEX.tar.gz: %w", repository, dir, err)) return } if !found { From 903fd94df90930fc69c0b768e3143d28e4f02310 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Thu, 5 Nov 2026 12:34:32 +0100 Subject: [PATCH 550/633] Test that apk index errors leave out the URL --- internal/server/mirror_apk_test.go | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/internal/server/mirror_apk_test.go b/internal/server/mirror_apk_test.go index 0c46ba1..ff74df9 100644 --- a/internal/server/mirror_apk_test.go +++ b/internal/server/mirror_apk_test.go @@ -156,3 +156,19 @@ func TestMirrorAPKRefusesIncompletePURLs(t *testing.T) { }) } } + +func TestMirrorAPKIndexErrorsLeaveOutTheURL(t *testing.T) { + upstream := newAPKUpstream(t) + upstream.files[apkMainIndex] = "not an index" + + progress, _ := mirrorWithConfig(t, apkMirrorConfig(upstream), + "pkg:apk/alpine/busybox@1.36.1-r29?arch=x86_64&distro=alpine-3.20.3") + + if progress.Failed != 1 || len(progress.Errors) != 1 { + t.Fatalf("failed = %d, errors = %v; want one error", progress.Failed, progress.Errors) + } + msg := progress.Errors[0].Error + if !strings.Contains(msg, "alpine v3.20/main/x86_64") || strings.Contains(msg, upstream.URL) { + t.Errorf("error = %q; want the repository and directory without %s", msg, upstream.URL) + } +} From 308cc133abff7fbb750ba7cebc932b0a04860adf Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Thu, 5 Nov 2026 15:09:16 +0000 Subject: [PATCH 551/633] Remove connection details from stats response --- README.md | 4 +--- docs/swagger/docs.go | 6 ------ docs/swagger/swagger.json | 6 ------ internal/server/server.go | 4 ---- internal/server/server_test.go | 26 -------------------------- internal/server/stats_test.go | 2 +- 6 files changed, 2 insertions(+), 46 deletions(-) diff --git a/README.md b/README.md index 2a08eaa..e47216e 100644 --- a/README.md +++ b/README.md @@ -1515,9 +1515,7 @@ Response: { "cached_artifacts": 142, "total_size_bytes": 523456789, - "total_size": "499.2 MB", - "storage_url": "file:///path/to/cache/artifacts", - "database_path": "./cache/proxy.db" + "total_size": "499.2 MB" } ``` diff --git a/docs/swagger/docs.go b/docs/swagger/docs.go index 3ad61f7..fe8723d 100644 --- a/docs/swagger/docs.go +++ b/docs/swagger/docs.go @@ -1461,9 +1461,6 @@ const docTemplate = `{ "cached_artifacts": { "type": "integer" }, - "database_path": { - "type": "string" - }, "downloaded": { "type": "string" }, @@ -1484,9 +1481,6 @@ const docTemplate = `{ "description": "StatsUnavailable distinguishes a proxy that has served nothing from one\nwhose aggregation failed with no snapshot to fall back on. Without it\nboth report zeros and an empty array.", "type": "boolean" }, - "storage_url": { - "type": "string" - }, "total_size": { "type": "string" }, diff --git a/docs/swagger/swagger.json b/docs/swagger/swagger.json index d0f27c3..496aecd 100644 --- a/docs/swagger/swagger.json +++ b/docs/swagger/swagger.json @@ -1454,9 +1454,6 @@ "cached_artifacts": { "type": "integer" }, - "database_path": { - "type": "string" - }, "downloaded": { "type": "string" }, @@ -1477,9 +1474,6 @@ "description": "StatsUnavailable distinguishes a proxy that has served nothing from one\nwhose aggregation failed with no snapshot to fall back on. Without it\nboth report zeros and an empty array.", "type": "boolean" }, - "storage_url": { - "type": "string" - }, "total_size": { "type": "string" }, diff --git a/internal/server/server.go b/internal/server/server.go index 55fe597..8599c48 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -1387,8 +1387,6 @@ type StatsResponse struct { CachedArtifacts int64 `json:"cached_artifacts"` TotalSize int64 `json:"total_size_bytes"` TotalSizeHuman string `json:"total_size"` - StorageURL string `json:"storage_url"` - DatabasePath string `json:"database_path"` // DownloadedBytes is the accumulated download volume across every // ecosystem: cache hits multiplied by the artifact size they served. DownloadedBytes int64 `json:"downloaded_bytes"` @@ -1452,8 +1450,6 @@ func (s *Server) handleStats(w http.ResponseWriter, r *http.Request) { TotalSizeHuman: formatSize(size), Ecosystems: make([]EcosystemStatsEntry, 0, len(ecosystems)), StatsUnavailable: statsErr != nil && len(ecosystems) == 0, - StorageURL: s.storage.URL(), - DatabasePath: s.cfg.Database.String(), } for _, e := range ecosystems { diff --git a/internal/server/server_test.go b/internal/server/server_test.go index e2d2264..5fa49e9 100644 --- a/internal/server/server_test.go +++ b/internal/server/server_test.go @@ -534,10 +534,6 @@ func TestStatsEndpoint(t *testing.T) { if stats.CachedArtifacts != 0 { t.Errorf("expected 0 cached artifacts, got %d", stats.CachedArtifacts) } - - if !strings.HasPrefix(stats.StorageURL, "file://") { - t.Errorf("expected storage_url to start with file://, got %q", stats.StorageURL) - } } func TestDashboard(t *testing.T) { @@ -1676,28 +1672,6 @@ func TestNewServer_InvalidAccessLogFailsBeforeDatabaseInit(t *testing.T) { } } -func TestStatsEndpoint_StorageURL(t *testing.T) { - ts := newTestServer(t) - defer ts.close() - - req := httptest.NewRequest("GET", "/stats", nil) - w := httptest.NewRecorder() - ts.handler.ServeHTTP(w, req) - - if w.Code != http.StatusOK { - t.Fatalf("expected status 200, got %d", w.Code) - } - - // Verify the JSON response uses storage_url (not storage_path) - body := w.Body.String() - if !strings.Contains(body, `"storage_url"`) { - t.Errorf("expected JSON key storage_url in response, got: %s", body) - } - if strings.Contains(body, `"storage_path"`) { - t.Errorf("unexpected JSON key storage_path in response (should be storage_url)") - } -} - // TestUIRefusesFraming checks that other sites cannot frame UI pages, where // a framed Delete button could be clicked by a signed-in user. func TestUIRefusesFraming(t *testing.T) { diff --git a/internal/server/stats_test.go b/internal/server/stats_test.go index 467f3c1..e777758 100644 --- a/internal/server/stats_test.go +++ b/internal/server/stats_test.go @@ -108,7 +108,7 @@ func TestStatsFlagsUnavailableFigures(t *testing.T) { if len(stats.Ecosystems) != 0 { t.Errorf("ecosystems = %v, want empty", stats.Ecosystems) } - if stats.StorageURL == "" { + if stats.TotalSizeHuman == "" { t.Error("the rest of the response must still be served") } } From 17355d2227f0361ed399dd11316b13b4045dd777 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 5 Nov 2026 15:13:42 +0000 Subject: [PATCH 552/633] Bump alpine from 3.24.2 to 3.24.3 Bumps alpine from 3.24.2 to 3.24.3. --- updated-dependencies: - dependency-name: alpine dependency-version: 3.24.3 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 6a56b03..d434e58 100644 --- a/Dockerfile +++ b/Dockerfile @@ -20,7 +20,7 @@ RUN CGO_ENABLED=0 GOOS=linux GOARCH=${TARGETARCH} go build \ -ldflags="-s -w -X main.Version=${VERSION} -X main.Commit=${COMMIT}" \ -o /proxy ./cmd/proxy -FROM alpine:3.24.2 +FROM alpine:3.24.3 RUN apk add --no-cache ca-certificates git From 12c5cc9e4ca1ebee793c6c26d798882c7279387f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 5 Nov 2026 15:14:00 +0000 Subject: [PATCH 553/633] Bump docker/login-action from 4.6.0 to 4.7.0 Bumps [docker/login-action](https://github.com/docker/login-action) from 4.6.0 to 4.7.0. - [Release notes](https://github.com/docker/login-action/releases) - [Commits](https://github.com/docker/login-action/compare/dbcb813823bdd20940b903addbd779551569679f...81c6f93d4bc991e510b342a00b419b3bb4efc5c6) --- updated-dependencies: - dependency-name: docker/login-action dependency-version: 4.7.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index cbb6e15..e325a8a 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -38,7 +38,7 @@ jobs: uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1 - name: Log in to the Container registry - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f + uses: docker/login-action@81c6f93d4bc991e510b342a00b419b3bb4efc5c6 with: registry: ghcr.io username: ${{ github.actor }} From d5dbd7a95a23c7a788210e6d8648576457091332 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 5 Nov 2026 15:14:03 +0000 Subject: [PATCH 554/633] Bump zizmorcore/zizmor-action from 0.7.1 to 0.8.0 Bumps [zizmorcore/zizmor-action](https://github.com/zizmorcore/zizmor-action) from 0.7.1 to 0.8.0. - [Release notes](https://github.com/zizmorcore/zizmor-action/releases) - [Commits](https://github.com/zizmorcore/zizmor-action/compare/85b2b9fb8e3325e7a23d7d0a9d7ba5ae0f999e3e...53290badfa369cb0bf7ad16342cb62c2fa47ac7b) --- updated-dependencies: - dependency-name: zizmorcore/zizmor-action dependency-version: 0.8.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/zizmor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/zizmor.yml b/.github/workflows/zizmor.yml index 02a0722..00041a6 100644 --- a/.github/workflows/zizmor.yml +++ b/.github/workflows/zizmor.yml @@ -34,4 +34,4 @@ jobs: persist-credentials: false - name: Run zizmor - uses: zizmorcore/zizmor-action@85b2b9fb8e3325e7a23d7d0a9d7ba5ae0f999e3e # v0.7.1 + uses: zizmorcore/zizmor-action@53290badfa369cb0bf7ad16342cb62c2fa47ac7b # v0.8.0 From 06dd126211658fda388c4b4a64da11912e0921a4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 5 Nov 2026 15:15:18 +0000 Subject: [PATCH 555/633] Bump github.com/aws/aws-sdk-go-v2/config from 1.33.9 to 1.33.12 Bumps [github.com/aws/aws-sdk-go-v2/config](https://github.com/aws/aws-sdk-go-v2) from 1.33.9 to 1.33.12. - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/config/v1.33.9...config/v1.33.12) --- updated-dependencies: - dependency-name: github.com/aws/aws-sdk-go-v2/config dependency-version: 1.33.12 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 28 ++++++++++++++-------------- go.sum | 56 ++++++++++++++++++++++++++++---------------------------- 2 files changed, 42 insertions(+), 42 deletions(-) diff --git a/go.mod b/go.mod index 76a1fad..ad2e259 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ toolchain go1.27.3 require ( github.com/BurntSushi/toml v1.6.0 github.com/Masterminds/semver/v3 v3.5.0 - github.com/aws/aws-sdk-go-v2/config v1.33.9 + github.com/aws/aws-sdk-go-v2/config v1.33.12 github.com/aws/aws-sdk-go-v2/service/ecr v1.66.5 github.com/git-pkgs/archives v0.10.0 github.com/git-pkgs/artifacts v0.2.1 @@ -52,24 +52,24 @@ require ( github.com/PuerkitoBio/purell v1.1.1 // indirect github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect github.com/apapsch/go-jsonmerge/v2 v2.0.0 // indirect - github.com/aws/aws-sdk-go-v2 v1.47.3 // indirect + github.com/aws/aws-sdk-go-v2 v1.47.5 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.20 // indirect - github.com/aws/aws-sdk-go-v2/credentials v1.20.9 // indirect - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.3 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.20.12 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.5 // indirect github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.4.3 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.6 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.6 // indirect - github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.6 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.21 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.8 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.8 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.8 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.23 // indirect github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.2 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.6 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.8 // indirect github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.2 // indirect github.com/aws/aws-sdk-go-v2/service/s3 v1.111.0 // indirect - github.com/aws/aws-sdk-go-v2/service/signin v1.10.4 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.38.4 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.4 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.51.4 // indirect - github.com/aws/smithy-go v1.28.5 // indirect + github.com/aws/aws-sdk-go-v2/service/signin v1.12.0 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.39.0 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.44.0 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.51.6 // indirect + github.com/aws/smithy-go v1.28.7 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cenk/backoff v2.2.1+incompatible // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect diff --git a/go.sum b/go.sum index c6b5f02..448e33b 100644 --- a/go.sum +++ b/go.sum @@ -52,46 +52,46 @@ github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdko github.com/RaveNoX/go-jsoncommentstrip v1.0.0/go.mod h1:78ihd09MekBnJnxpICcwzCMzGrKSKYe4AqU6PDYYpjk= github.com/apapsch/go-jsonmerge/v2 v2.0.0 h1:axGnT1gRIfimI7gJifB699GoE/oq+F2MU7Dml6nw9rQ= github.com/apapsch/go-jsonmerge/v2 v2.0.0/go.mod h1:lvDnEdqiQrp0O42VQGgmlKpxL1AP2+08jFMw88y4klk= -github.com/aws/aws-sdk-go-v2 v1.47.3 h1:HRoT/MkNi3bjO5xF4dCzLzvVmkvRGHj3/cLINDnJejk= -github.com/aws/aws-sdk-go-v2 v1.47.3/go.mod h1:xSl+OSI5ZUw6bSMupWc218zE5Pl+65MCCu3ySWFY99A= +github.com/aws/aws-sdk-go-v2 v1.47.5 h1:5yA+A+BiXbDq1ZLbYwPViwhEGDMdR0MEhL8Jh2LnLS4= +github.com/aws/aws-sdk-go-v2 v1.47.5/go.mod h1:iNIkogrRvFwm+1OQIUOP2lPBtlXZboOIUWiCa+pHEMw= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.20 h1:GPRlPwz40I2B2VrBEASOA3Bi77NyeqejNLkifosX0rs= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.20/go.mod h1:g7PNzKcsOKWb4fkSRBA7BZVAS6Y8IcxzN+nRohhQ1Q8= -github.com/aws/aws-sdk-go-v2/config v1.33.9 h1:Hv6bcQUJVnmuWaKjqD6Ot8sE75NrWsWV20oGT5aWle0= -github.com/aws/aws-sdk-go-v2/config v1.33.9/go.mod h1:ivowmrZNfBQLcg4QkaItgi5wae4Bzot9RzIdbY1hfTg= -github.com/aws/aws-sdk-go-v2/credentials v1.20.9 h1:SNO55PnWhaDafsOcv16lCA0kdoFMl/YlegWoVsU02s0= -github.com/aws/aws-sdk-go-v2/credentials v1.20.9/go.mod h1:+2reOTRtdD3VSBqpRDNwrZIYy8yMCYa7oK3WKTJ/I1I= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.3 h1:gr3b87OLB29jymWPzp/CAyXwQRJVE+sC4Wm0qBIKHsk= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.3/go.mod h1:BRjf4L/Yx2hY54o+ZibPbLJyMMeSwqYQXwcJfqW66a8= +github.com/aws/aws-sdk-go-v2/config v1.33.12 h1:IV1YARaXahwCqUIyP8IlWsP85Lil+5ozTozDQXPumRk= +github.com/aws/aws-sdk-go-v2/config v1.33.12/go.mod h1:h51R06bzMVwNBRZOX5ylHhNDnkB/azZeq/LvxM6q2pQ= +github.com/aws/aws-sdk-go-v2/credentials v1.20.12 h1:6P7Q7yH/GIwHg6MLKv3ySsBqaCoE1z+C0OTzYMenhEY= +github.com/aws/aws-sdk-go-v2/credentials v1.20.12/go.mod h1:Kw/SaoW/wc7IqJErqXJIzKo8b7Aelxfbuc6XlbrRnsc= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.5 h1:iXjy2SFhGYNB1jWZBhQqq8HbhsbPF23w+/zbMvPXsTg= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.5/go.mod h1:k0RUiBnyXU0AXRJ9tJmY2jQke6XUIX2PNVnWtvgaPtc= github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.4.3 h1:vsFkmFrfzODkxVbyuPLscv5j5lC8KbG48LZwV+p56Y0= github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.4.3/go.mod h1:pjQYVtvVBEtlsf2p6nXMsjPuDaWs9GM44FMdHbmnmYU= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.6 h1:CqL8NN3p4I0JTwwV5MEoatIl4Y2amZ2X9dRD+0hbRL0= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.6/go.mod h1:exdJ32hMqXl0Wyo6NwqEC5yIhtua3+PDaKE8P29m4qE= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.6 h1:FpBq24TG6WVSObTNhQOv7Q0088rounUUlwn3zlNItkM= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.6/go.mod h1:ONzdPVYC95oBOu0VWNEsKEntcoemP3xVKjT+qWml1s4= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.6 h1:Qmn60RWuk9+bd2ddvJJqwf3Cpac3VvGMOdbGemtUhX4= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.6/go.mod h1:h/YJhe4AVyQxEUMluph10iqV7mjajH7A9L+BHF5ppis= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.8 h1:gzIWc3DTPfPbWKrQnmL6BV6WD7NPD1htxzgdk/bQ/Z4= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.8/go.mod h1:vvUA5cDAZWGo4R14FZPZfXfJCmgsafOWEq78Kg0D+ug= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.8 h1:PWv+vG8aCLS8qd+HPdYhK7Vees90aKHWFFQ1AGYD/8E= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.8/go.mod h1:xxPZEpflYHJA12366ZGyq66z9dHPCoeCIu99Jck+MPw= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.8 h1:WOp5De9iWdEHDPh2Kb0tBP3vGBm/dVAuRGh4a4tqduo= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.8/go.mod h1:z8W8YgfhbWl62XqcTtK+CxPrE+iHGRVUOtrrbIOp7ZY= github.com/aws/aws-sdk-go-v2/service/ecr v1.66.5 h1:163OpmhcdzTzgbXbLV+u2v1LY/QmGT148/eOwkczKCY= github.com/aws/aws-sdk-go-v2/service/ecr v1.66.5/go.mod h1:uDza1Eipo/Olii3CCgLglHmAni24PScSAhpfIKbUl0c= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.21 h1:e6YW5uME30TkrElR+4/xcY7NB6CCj2dcv7vtmGXhfhQ= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.21/go.mod h1:obpHAMP693+f3/A8PkLhIqfK1C+NWWn8zXzqD3OH520= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.23 h1:MCIR6iZaCKO4UE0hUvoWWLI6JVjppopV64JfAJaMjng= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.23/go.mod h1:pphWFS+z0MmIPivj8c/6bATd1Z+Av7Ust75DPdEFsPA= github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.2 h1:CO485zrLOZBeq8mrdsudc8cjyhvC4LY4JFt8ZfHvXZ0= github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.2/go.mod h1:cBAjVUKBr+5wnx1K3XWS8u5AZsqpA2S/0z7YtI6BGFc= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.6 h1:LHHdobjo5le1sjJdolitV7uVvYg3wkaeM1Ju2fDgeZQ= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.6/go.mod h1:CBGB0evAiaMD3Lk4wPO7LQCcCZLuhzL5Qf/wfEUHQ5w= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.8 h1:+PMpWlJYOn2tm7sm6OZ/ZLd4jStl17T7wjNapd1Rczs= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.8/go.mod h1:cnG3xZLGQL4c/bt8pjdzlFtZsW7epBBuNS04a5xMjdo= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.2 h1:Enx5JcIy/o90kio1LFZ/HguxIsos3clbGuSpGdyshX8= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.2/go.mod h1:Ldmll8/2VqaVerWWDf8IGG+6pAvZY04gs3tevq3nIFs= github.com/aws/aws-sdk-go-v2/service/s3 v1.111.0 h1:4PokON/ISKuD6Zl062n4WfQNnEFBMHvfeJy73Jp7K+g= github.com/aws/aws-sdk-go-v2/service/s3 v1.111.0/go.mod h1:6ZFrmv/qIAYFmG+2WhA7mFU8+edldRlISwT+tNnyLuA= -github.com/aws/aws-sdk-go-v2/service/signin v1.10.4 h1:tvpDjuUvgKURf3Yk4hOaGbZ4QdY+070OWcA5d6LUv2A= -github.com/aws/aws-sdk-go-v2/service/signin v1.10.4/go.mod h1:3ibVKg5Ng82Zby8++jELTD2nHxiPk2DJRLEeW/LKp8M= -github.com/aws/aws-sdk-go-v2/service/sso v1.38.4 h1:ChCtagixSFOxltv0o6E3NT8F7ksvZtZGPwUBynO5Nmk= -github.com/aws/aws-sdk-go-v2/service/sso v1.38.4/go.mod h1:+FEO9jrVMpc9oN3CdAxJbtlQEu3z8UddDPSedpeM3PY= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.4 h1:j6c+HA4AoS0EBEjArPjavFEBrucJEryfrQU0TffmACk= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.4/go.mod h1:RhghaXUk9q66VtOQUX1FdPddQuHnqQZzbB0fEYXmPRA= -github.com/aws/aws-sdk-go-v2/service/sts v1.51.4 h1:OKEfbf2tcfsF2pBk/S9QqNpo6alaAtiBs4hysCsBjPw= -github.com/aws/aws-sdk-go-v2/service/sts v1.51.4/go.mod h1:mCd6qGo9b8hTtfjgsmNkFlzwjyce4owd7HDVFnCLXhM= -github.com/aws/smithy-go v1.28.5 h1:OqbqKtzY9NUnr2yAe9+hiTdFb20tNaK/lI0PMOGXX9k= -github.com/aws/smithy-go v1.28.5/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= +github.com/aws/aws-sdk-go-v2/service/signin v1.12.0 h1:3vu/eUqh0rjh7Zy4N4YKcpZX4Aud+zinB3VsLhhlQFw= +github.com/aws/aws-sdk-go-v2/service/signin v1.12.0/go.mod h1:wxAk0D2oVBSjuXEIzvcKxTZfM90zticdAbh/VD42L0Y= +github.com/aws/aws-sdk-go-v2/service/sso v1.39.0 h1:kIEpD5dUry7VET3Ybl7zj7PPHRB4lb52G9iTz+a1Tqk= +github.com/aws/aws-sdk-go-v2/service/sso v1.39.0/go.mod h1:UGsEYgWz4Gi+46vDQ5zkFLqUgJziyOE2SXjZjgmfmHc= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.44.0 h1:WfFev+HgTodggcNon5HieiLxxwWwb8Br8R2+nDkixvk= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.44.0/go.mod h1:gJSdIY6YUzDvqQLcMtPVZgBRHrFwzHryvpE8Igzbxps= +github.com/aws/aws-sdk-go-v2/service/sts v1.51.6 h1:BVjKrtRUtSYKztBCCfE1oAB+d3ViKcW2cKZfserOqMQ= +github.com/aws/aws-sdk-go-v2/service/sts v1.51.6/go.mod h1:eSukFLeLkPMm+1OxDR+F0joauZdyBkF47Ow7MqjZiJw= +github.com/aws/smithy-go v1.28.7 h1:h/n1SHrwVtaDYLz8mR1/Kq4Xh4RaFbFW7bOYckg9mfE= +github.com/aws/smithy-go v1.28.7/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bmatcuk/doublestar v1.1.1/go.mod h1:UD6OnuiIn0yFxxA2le/rnRU1G4RaI4UvFv1sNto9p6w= From 0f26841f5661fb39a138022458053665a59bec38 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Thu, 5 Nov 2026 17:39:19 +0100 Subject: [PATCH 556/633] Keep blobs of scheduled images from retention A mirror run that finds a blob cached records it in mirrored_at, which retention counts as use. LRU eviction keeps using last_accessed_at, so #508's decision not to count mirror runs as cache hits stands. Fixes #648. --- docs/configuration.md | 2 +- internal/database/retention.go | 35 +++++++++++++++------ internal/database/retention_test.go | 20 ++++++------ internal/database/schema.go | 33 +++++++++++++++++--- internal/database/types.go | 3 ++ internal/handler/container_mirror.go | 5 +++ internal/handler/container_mirror_test.go | 37 +++++++++++++++++++++++ internal/handler/handler.go | 10 ++++++ 8 files changed, 121 insertions(+), 24 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index bf34b23..a15978c 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -111,7 +111,7 @@ storage: sweep_interval: "10m" ``` -Each cached file ages on its own, counted from the last time the proxy served it, or from when it was fetched if it has not been served since. A newer release of the same package does not shorten the life of older versions. Durations take a `d` suffix for days, and `"0"` or an empty value means never evict by age. With no retention configured, artifacts stay until `max_size` evicts them. `sweep_interval` must be at least `1m`. +Each cached file ages on its own, counted from the last time the proxy served it, or from when it was fetched if it has not been served since. The blobs of an image that `mirror.schedules` keeps current also count from the last run that found them cached, so the schedule keeps them; for `max_size` eviction such a run doesn't count as use. A newer release of the same package does not shorten the life of older versions. Durations take a `d` suffix for days, and `"0"` or an empty value means never evict by age. With no retention configured, artifacts stay until `max_size` evicts them. `sweep_interval` must be at least `1m`. A package rule wins over an ecosystem rule, which wins over `default`. Package rules are keyed by package PURL without a version, written in one of the forms the table below lists for the ecosystem. The ecosystems and package rules can only name ecosystems that support retention in the running version; anything else, including a typo, stops the proxy at startup with an error naming the setting. `default` applies only to supported ecosystems, except `generic`, which requires an explicit rule. Support is added one ecosystem at a time, so a later release can make an existing `default` cover more ecosystems. On startup the proxy logs which ecosystems retention supports (`covered`) and which ones `default` does not reach (`default_not_applied`), including `generic`. diff --git a/internal/database/retention.go b/internal/database/retention.go index 90fffd2..0036362 100644 --- a/internal/database/retention.go +++ b/internal/database/retention.go @@ -17,28 +17,33 @@ type RetentionCandidate struct { Size sql.NullInt64 `db:"size"` FetchedAt sql.NullTime `db:"fetched_at"` LastAccessedAt sql.NullTime `db:"last_accessed_at"` + MirroredAt sql.NullTime `db:"mirrored_at"` Ecosystem string `db:"ecosystem"` PackagePURL string `db:"package_purl"` } -// ExpiredBefore reports whether the artifact was neither fetched nor -// accessed at or after cutoff. Both are checked: clearing a record keeps its -// last access time, so a record fetched again after an eviction has a fresh -// fetch time next to an old access time. +// ExpiredBefore reports whether the artifact was neither fetched, accessed +// nor found by a mirror run at or after cutoff. Fetch and access are both +// checked: clearing a record keeps its last access time, so a record fetched +// again after an eviction has a fresh fetch time next to an old access time. func (c RetentionCandidate) ExpiredBefore(cutoff time.Time) bool { if !c.FetchedAt.Valid || !c.FetchedAt.Time.Before(cutoff) { return false } + if c.MirroredAt.Valid && !c.MirroredAt.Time.Before(cutoff) { + return false + } return !c.LastAccessedAt.Valid || c.LastAccessedAt.Time.Before(cutoff) } -// expiredCondition selects cached artifacts fetched before the cutoff and not -// accessed since. It takes the cutoff twice. +// expiredCondition selects cached artifacts fetched before the cutoff and +// neither accessed nor mirrored since. It takes the cutoff three times. const expiredCondition = ` storage_path IS NOT NULL AND source <> 'local' AND fetched_at < ? - AND (last_accessed_at IS NULL OR last_accessed_at < ?)` + AND (last_accessed_at IS NULL OR last_accessed_at < ?) + AND (mirrored_at IS NULL OR mirrored_at < ?)` // MaxArtifactID returns the highest artifact id, or zero for an empty table. func (db *DB) MaxArtifactID() (int64, error) { @@ -56,7 +61,7 @@ func (db *DB) GetRetentionCandidates(fromID, toID int64, cutoff time.Time) ([]Re var candidates []RetentionCandidate query := db.Rebind(` SELECT a.id, a.version_purl, a.filename, a.storage_path, a.size, - a.fetched_at, a.last_accessed_at, p.ecosystem, p.purl AS package_purl + a.fetched_at, a.last_accessed_at, a.mirrored_at, p.ecosystem, p.purl AS package_purl FROM ( SELECT * FROM artifacts WHERE id > ? AND id <= ? AND` + expiredCondition + ` @@ -65,7 +70,7 @@ func (db *DB) GetRetentionCandidates(fromID, toID int64, cutoff time.Time) ([]Re JOIN packages p ON p.purl = v.package_purl ORDER BY a.id `) - if err := db.Select(&candidates, query, fromID, toID, cutoff.UTC(), cutoff.UTC()); err != nil { + if err := db.Select(&candidates, query, fromID, toID, cutoff.UTC(), cutoff.UTC(), cutoff.UTC()); err != nil { return nil, err } return candidates, nil @@ -89,7 +94,7 @@ func (db *DB) ClearExpiredArtifact(versionPURL, filename, storagePath string, cu SET storage_path = NULL, content_hash = NULL, size = NULL, content_type = NULL, fetched_at = NULL, updated_at = ? WHERE version_purl = ? AND filename = ? AND storage_path = ? AND` + expiredCondition) - res, err := tx.Exec(query, time.Now(), versionPURL, filename, storagePath, cutoff.UTC(), cutoff.UTC()) + res, err := tx.Exec(query, time.Now(), versionPURL, filename, storagePath, cutoff.UTC(), cutoff.UTC(), cutoff.UTC()) if err != nil { return false, err } @@ -103,6 +108,16 @@ func (db *DB) ClearExpiredArtifact(versionPURL, filename, storagePath string, cu return true, tx.Commit() } +// MarkArtifactMirrored records that a mirror run found the artifact cached, +// which keeps it from retention without counting as a cache hit. +func (db *DB) MarkArtifactMirrored(versionPURL, filename string) error { + _, err := db.Exec(db.Rebind(` + UPDATE artifacts SET mirrored_at = ? + WHERE version_purl = ? AND filename = ? AND storage_path IS NOT NULL + `), time.Now().UTC(), versionPURL, filename) + return err +} + // GetArtifactEcosystem returns the packages.ecosystem value of the package an // artifact version belongs to, or "" when it has no package record. func (db *DB) GetArtifactEcosystem(versionPURL string) (string, error) { diff --git a/internal/database/retention_test.go b/internal/database/retention_test.go index bd9ad28..2382011 100644 --- a/internal/database/retention_test.go +++ b/internal/database/retention_test.go @@ -341,18 +341,20 @@ func TestRetentionCandidateExpiredBefore(t *testing.T) { recent := sql.NullTime{Time: now, Valid: true} tests := []struct { - name string - fetched, accessed sql.NullTime - want bool + name string + fetched, accessed, mirrored sql.NullTime + want bool }{ - {"fetched and accessed before", old, old, true}, - {"never served", old, sql.NullTime{}, true}, - {"accessed since", old, recent, false}, - {"fetched again since", recent, old, false}, - {"not cached", sql.NullTime{}, old, false}, + {"fetched and accessed before", old, old, sql.NullTime{}, true}, + {"never served", old, sql.NullTime{}, sql.NullTime{}, true}, + {"accessed since", old, recent, sql.NullTime{}, false}, + {"fetched again since", recent, old, sql.NullTime{}, false}, + {"not cached", sql.NullTime{}, old, sql.NullTime{}, false}, + {"mirrored since", old, old, recent, false}, + {"mirrored before", old, old, old, true}, } for _, tt := range tests { - c := RetentionCandidate{FetchedAt: tt.fetched, LastAccessedAt: tt.accessed} + c := RetentionCandidate{FetchedAt: tt.fetched, LastAccessedAt: tt.accessed, MirroredAt: tt.mirrored} if got := c.ExpiredBefore(cutoff); got != tt.want { t.Errorf("%s: ExpiredBefore() = %v, want %v", tt.name, got, tt.want) } diff --git a/internal/database/schema.go b/internal/database/schema.go index efc2b2d..6a9adaa 100644 --- a/internal/database/schema.go +++ b/internal/database/schema.go @@ -75,7 +75,8 @@ CREATE TABLE IF NOT EXISTS artifacts ( created_at DATETIME, updated_at DATETIME, source TEXT NOT NULL DEFAULT 'upstream', - manifest TEXT + manifest TEXT, + mirrored_at DATETIME ); CREATE UNIQUE INDEX IF NOT EXISTS idx_artifacts_version_filename ON artifacts(version_purl, filename); CREATE INDEX IF NOT EXISTS idx_artifacts_storage_path ON artifacts(storage_path); @@ -252,7 +253,8 @@ CREATE TABLE IF NOT EXISTS artifacts ( created_at TIMESTAMP, updated_at TIMESTAMP, source TEXT NOT NULL DEFAULT 'upstream', - manifest TEXT + manifest TEXT, + mirrored_at TIMESTAMP ); CREATE UNIQUE INDEX IF NOT EXISTS idx_artifacts_version_filename ON artifacts(version_purl, filename); CREATE INDEX IF NOT EXISTS idx_artifacts_storage_path ON artifacts(storage_path); @@ -388,7 +390,8 @@ CREATE TABLE IF NOT EXISTS artifacts ( created_at DATETIME, updated_at DATETIME, source TEXT NOT NULL DEFAULT 'upstream', - manifest TEXT + manifest TEXT, + mirrored_at DATETIME ); CREATE UNIQUE INDEX IF NOT EXISTS idx_artifacts_version_filename ON artifacts(version_purl, filename); CREATE INDEX IF NOT EXISTS idx_artifacts_storage_path ON artifacts(storage_path); @@ -413,7 +416,8 @@ CREATE TABLE IF NOT EXISTS artifacts ( created_at TIMESTAMP, updated_at TIMESTAMP, source TEXT NOT NULL DEFAULT 'upstream', - manifest TEXT + manifest TEXT, + mirrored_at TIMESTAMP ); CREATE UNIQUE INDEX IF NOT EXISTS idx_artifacts_version_filename ON artifacts(version_purl, filename); CREATE INDEX IF NOT EXISTS idx_artifacts_storage_path ON artifacts(storage_path); @@ -538,6 +542,7 @@ var migrations = []migration{ {"019_add_artifact_filename_index", migrateAddArtifactFilenameIndex}, {"020_add_metadata_snapshots", migrateAddMetadataSnapshots}, {"021_add_projects", migrateAddProjects}, + {"022_add_artifact_mirrored_at", migrateAddArtifactMirroredAt}, } // isTableNotFound returns true if the error indicates a missing table. @@ -1080,6 +1085,26 @@ func migrateAddArtifactSource(db *DB) error { return nil } +// migrateAddArtifactMirroredAt records when a mirror run last found an +// artifact cached, which retention counts as use and LRU eviction doesn't. +func migrateAddArtifactMirroredAt(db *DB) error { + hasCol, err := db.HasColumn("artifacts", "mirrored_at") + if err != nil { + return fmt.Errorf("checking column mirrored_at: %w", err) + } + if hasCol { + return nil + } + ts := sqliteDatetime + if db.dialect == DialectPostgres { + ts = postgresTimestamp + } + if _, err := db.Exec(fmt.Sprintf("ALTER TABLE artifacts ADD COLUMN mirrored_at %s", ts)); err != nil { + return fmt.Errorf("adding column mirrored_at to artifacts: %w", err) + } + return nil +} + // migrateAddArtifactManifest stores the manifest read from a pushed archive, // which the metadata of its ecosystem lists the version with. func migrateAddArtifactManifest(db *DB) error { diff --git a/internal/database/types.go b/internal/database/types.go index fb49a1c..c2675d1 100644 --- a/internal/database/types.go +++ b/internal/database/types.go @@ -151,6 +151,9 @@ type Artifact struct { UpdatedAt time.Time `db:"updated_at" json:"updated_at"` Source string `db:"source" json:"source"` Manifest sql.NullString `db:"manifest" json:"manifest,omitempty"` + // MirroredAt is when a mirror run last found the artifact cached. It + // keeps the artifact from retention but not from LRU eviction. + MirroredAt sql.NullTime `db:"mirrored_at" json:"-"` } func (a *Artifact) sourceOrUpstream() string { diff --git a/internal/handler/container_mirror.go b/internal/handler/container_mirror.go index c8ca65c..a973afd 100644 --- a/internal/handler/container_mirror.go +++ b/internal/handler/container_mirror.go @@ -502,6 +502,11 @@ func (m *imageMirror) mirrorBlob(ctx context.Context, digest string) (int64, boo return 0, false, fmt.Errorf("blob %s: %w", digest, err) } if cached { + // Not a cache hit, so LRU eviction still sees the image as unused, + // but retention keeps it while the schedule does. + if err := proxy.MarkMirrored("oci", m.cacheName, digest, digest); err != nil { + return 0, false, fmt.Errorf("blob %s: %w", digest, err) + } return 0, false, nil } result, err := proxy.GetOrFetchArtifactFromURLWithDigest(ctx, "oci", m.cacheName, digest, digest, diff --git a/internal/handler/container_mirror_test.go b/internal/handler/container_mirror_test.go index 2131928..e6be5a5 100644 --- a/internal/handler/container_mirror_test.go +++ b/internal/handler/container_mirror_test.go @@ -488,6 +488,43 @@ func TestMirrorImageCountsNoCacheHits(t *testing.T) { } } +// A scheduled image that no client pulls keeps its blobs: a mirror run that +// finds them cached holds them back from retention, though not from LRU +// eviction, which TestMirrorImageCountsNoCacheHits covers. +func TestMirrorImageKeepsCachedBlobsFromRetention(t *testing.T) { + registry, _ := newFakeImageRegistry(t) + proxy := newMirrorTestProxy(t, registry) + h := NewContainerHandlerWithRegistry(proxy, "http://proxy.example", registry.server.URL) + ref := ImageRef{Name: "team/app", References: []string{"multi"}} + + if _, err := h.MirrorImage(context.Background(), ref, nil); err != nil { + t.Fatalf("first MirrorImage: %v", err) + } + old := time.Now().Add(-30 * 24 * time.Hour).UTC() + if _, err := proxy.DB.Exec(proxy.DB.Rebind("UPDATE artifacts SET fetched_at = ?, last_accessed_at = ?"), old, old); err != nil { + t.Fatal(err) + } + cutoff := time.Now().Add(-14 * 24 * time.Hour) + maxID, err := proxy.DB.MaxArtifactID() + if err != nil { + t.Fatal(err) + } + if candidates, err := proxy.DB.GetRetentionCandidates(0, maxID, cutoff); err != nil || len(candidates) == 0 { + t.Fatalf("before the second run: %d candidates, err %v; want the aged blobs", len(candidates), err) + } + + if _, err := h.MirrorImage(context.Background(), ref, nil); err != nil { + t.Fatalf("second MirrorImage: %v", err) + } + candidates, err := proxy.DB.GetRetentionCandidates(0, maxID, cutoff) + if err != nil { + t.Fatal(err) + } + if len(candidates) != 0 { + t.Errorf("after the second run %d blobs are retention candidates, want none", len(candidates)) + } +} + func TestMirrorImageRefetchesBlobMissingFromStorage(t *testing.T) { registry, image := newFakeImageRegistry(t) proxy := newMirrorTestProxy(t, registry) diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 196f648..dae3bb7 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -451,6 +451,16 @@ func (p *Proxy) HasCachedArtifact(ctx context.Context, ecosystem, name, version, return p.Storage.Exists(ctx, artifact.StoragePath) } +// MarkMirrored records that a mirror run found an artifact cached, which +// keeps it from retention without counting as a cache hit. +func (p *Proxy) MarkMirrored(ecosystem, name, version, filename string) error { + _, versionPURL, err := packagePURLStrings(ecosystem, name, version) + if err != nil { + return err + } + return p.DB.MarkArtifactMirrored(versionPURL, filename) +} + // ClearCachedArtifact clears an artifact cache record after an external // integrity check fails, and queues its stored bytes for deletion. func (p *Proxy) ClearCachedArtifact(ecosystem, name, version, filename string) error { From 550a908d888ce2bb8df714b9f12111d3e2d72517 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 5 Nov 2026 18:23:50 +0000 Subject: [PATCH 557/633] Bump github.com/aws/aws-sdk-go-v2/service/ecr from 1.66.5 to 1.67.1 Bumps [github.com/aws/aws-sdk-go-v2/service/ecr](https://github.com/aws/aws-sdk-go-v2) from 1.66.5 to 1.67.1. - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/service/ecr/v1.66.5...service/s3/v1.67.1) --- updated-dependencies: - dependency-name: github.com/aws/aws-sdk-go-v2/service/ecr dependency-version: 1.67.1 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index ad2e259..2f4a0ce 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( github.com/BurntSushi/toml v1.6.0 github.com/Masterminds/semver/v3 v3.5.0 github.com/aws/aws-sdk-go-v2/config v1.33.12 - github.com/aws/aws-sdk-go-v2/service/ecr v1.66.5 + github.com/aws/aws-sdk-go-v2/service/ecr v1.67.1 github.com/git-pkgs/archives v0.10.0 github.com/git-pkgs/artifacts v0.2.1 github.com/git-pkgs/cooldown v0.3.1 diff --git a/go.sum b/go.sum index 448e33b..cb9226b 100644 --- a/go.sum +++ b/go.sum @@ -70,8 +70,8 @@ github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.8 h1:PWv+vG8aCLS8qd+HPdY github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.8/go.mod h1:xxPZEpflYHJA12366ZGyq66z9dHPCoeCIu99Jck+MPw= github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.8 h1:WOp5De9iWdEHDPh2Kb0tBP3vGBm/dVAuRGh4a4tqduo= github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.8/go.mod h1:z8W8YgfhbWl62XqcTtK+CxPrE+iHGRVUOtrrbIOp7ZY= -github.com/aws/aws-sdk-go-v2/service/ecr v1.66.5 h1:163OpmhcdzTzgbXbLV+u2v1LY/QmGT148/eOwkczKCY= -github.com/aws/aws-sdk-go-v2/service/ecr v1.66.5/go.mod h1:uDza1Eipo/Olii3CCgLglHmAni24PScSAhpfIKbUl0c= +github.com/aws/aws-sdk-go-v2/service/ecr v1.67.1 h1:EJlz0wSlZDKaCN2FHwg9RBR+C+gsuVdYblJnEnrq3M4= +github.com/aws/aws-sdk-go-v2/service/ecr v1.67.1/go.mod h1:6rVJFO9ZO+4j/QL5/FbtAsbb8xHC85j79wXA9ndOCTY= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.23 h1:MCIR6iZaCKO4UE0hUvoWWLI6JVjppopV64JfAJaMjng= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.23/go.mod h1:pphWFS+z0MmIPivj8c/6bATd1Z+Av7Ust75DPdEFsPA= github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.2 h1:CO485zrLOZBeq8mrdsudc8cjyhvC4LY4JFt8ZfHvXZ0= From 3456b394beb793db9a5926e26d565c8ccbc7eb52 Mon Sep 17 00:00:00 2001 From: Christian Heim Date: Sun, 18 Oct 2026 21:30:21 +0200 Subject: [PATCH 558/633] Cargo, gem, hex, pub, conda and helm now check cooldown on artifact downloads too, so a version inside the window is withheld even when it was cached before. Publish times are stored in UTC. Fixes #521. --- internal/database/database_test.go | 41 +++ internal/database/queries.go | 4 +- internal/handler/cargo.go | 28 ++ internal/handler/conda.go | 44 +++ internal/handler/download_cooldown.go | 82 +++++ internal/handler/download_cooldown_test.go | 347 +++++++++++++++++++++ internal/handler/filename_download.go | 11 + internal/handler/gem.go | 30 ++ internal/handler/helm.go | 54 ++++ internal/handler/hex.go | 25 ++ internal/handler/pub.go | 34 ++ 11 files changed, 698 insertions(+), 2 deletions(-) create mode 100644 internal/handler/download_cooldown.go create mode 100644 internal/handler/download_cooldown_test.go diff --git a/internal/database/database_test.go b/internal/database/database_test.go index 9ac4140..91734e1 100644 --- a/internal/database/database_test.go +++ b/internal/database/database_test.go @@ -1204,3 +1204,44 @@ func TestGetFetchedVersionPURLs(t *testing.T) { } }) } + +// Publish times are written in UTC whatever zone the caller's time is in, so +// a TIMESTAMP column without a zone reads back the same instant. +func TestVersionPublishedAtIsUTC(t *testing.T) { + tokyo, err := time.LoadLocation("Asia/Tokyo") + if err != nil { + t.Skipf("time zone data unavailable: %v", err) + } + prev := time.Local + time.Local = tokyo + t.Cleanup(func() { time.Local = prev }) + + runWithBothDatabases(t, func(t *testing.T, db *DB) { + written := time.Date(2026, 10, 10, 21, 0, 1, 0, tokyo) + want := written.UTC() + check := func(purl string) { + t.Helper() + got, err := db.GetVersionByPURL(purl) + if err != nil || got == nil || !got.PublishedAt.Valid { + t.Fatalf("GetVersionByPURL(%s) = %+v, %v", purl, got, err) + } + if !got.PublishedAt.Time.Equal(want) || got.PublishedAt.Time.Location() != time.UTC { + t.Errorf("%s PublishedAt read back as %v, want %v", purl, got.PublishedAt.Time, want) + } + } + + if err := db.SetVersionPublishedAt("pkg:cargo/demo@1.0.0", "pkg:cargo/demo", written); err != nil { + t.Fatal(err) + } + check("pkg:cargo/demo@1.0.0") + + if err := db.UpsertVersion(&Version{ + PURL: "pkg:cargo/demo@2.0.0", + PackagePURL: "pkg:cargo/demo", + PublishedAt: sql.NullTime{Time: written, Valid: true}, + }); err != nil { + t.Fatal(err) + } + check("pkg:cargo/demo@2.0.0") + }) +} diff --git a/internal/database/queries.go b/internal/database/queries.go index 23d55df..86f31ef 100644 --- a/internal/database/queries.go +++ b/internal/database/queries.go @@ -185,7 +185,7 @@ func (db *DB) upsertVersion(exec execer, v *Version) error { _, err := exec.Exec(query, v.PURL, v.PackagePURL, v.License, v.Integrity, - v.PublishedAt, v.Yanked, v.EnrichedAt, now, now, + inUTC(v.PublishedAt), v.Yanked, v.EnrichedAt, now, now, ) if err != nil { return fmt.Errorf("upserting version: %w", err) @@ -218,7 +218,7 @@ func (db *DB) SetVersionPublishedAt(versionPURL, packagePURL string, publishedAt ` } - _, err := db.Exec(query, versionPURL, packagePURL, publishedAt, now, now) + _, err := db.Exec(query, versionPURL, packagePURL, publishedAt.UTC(), now, now) if err != nil { return fmt.Errorf("setting version publish time: %w", err) } diff --git a/internal/handler/cargo.go b/internal/handler/cargo.go index 0c26551..43bcbc4 100644 --- a/internal/handler/cargo.go +++ b/internal/handler/cargo.go @@ -2,6 +2,7 @@ package handler import ( "bufio" + "bytes" "encoding/json" "errors" "fmt" @@ -186,6 +187,26 @@ func (h *CargoHandler) applyCooldownFiltering(downstreamResponse http.ResponseWr } } +// publishTime reads a crate version's pubtime from the sparse index, sharing +// the index cache with handleIndex. Entries published before crates.io +// recorded pubtime have none. +func (h *CargoHandler) publishTime(r *http.Request, name, version string) (time.Time, bool, error) { + upstreamURL := fmt.Sprintf("%s/%s", h.indexURL, h.buildIndexPath(name)) + body, _, err := h.proxy.FetchOrCacheMetadata(r.Context(), "cargo", name, upstreamURL, "text/plain") + if err != nil { + return time.Time{}, false, err + } + for line := range bytes.SplitSeq(body, []byte("\n")) { + if vers, _, _ := lookupJSONString(line, "vers"); vers != version { + continue + } + pubtime, _, _ := lookupJSONString(line, "pubtime") + published, err := time.Parse(time.RFC3339, pubtime) + return published, err == nil, nil + } + return time.Time{}, false, nil +} + // buildIndexPath builds the sparse index path for a crate name. func (h *CargoHandler) buildIndexPath(name string) string { name = strings.ToLower(name) @@ -217,6 +238,13 @@ func (h *CargoHandler) handleDownload(w http.ResponseWriter, r *http.Request) { h.proxy.Logger.Debug("cargo download request", "crate", name, "version", version, "filename", filename) + if h.proxy.withholdInCooldown(w, downloadVersion{ + ecosystem: "cargo", policyName: name, storedName: name, version: version, + lookup: func() (time.Time, bool, error) { return h.publishTime(r, name, version) }, + }) { + return + } + downloadURL := fmt.Sprintf( "%s/%s/%s", h.downloadURL, diff --git a/internal/handler/conda.go b/internal/handler/conda.go index 8581c88..22ec603 100644 --- a/internal/handler/conda.go +++ b/internal/handler/conda.go @@ -139,6 +139,13 @@ func (h *CondaHandler) handleDownload(w http.ResponseWriter, r *http.Request) { h.proxy.Logger.Debug("conda download request", "channel", channel, "arch", arch, "name", name, "version", version) + if h.proxy.withholdInCooldown(w, downloadVersion{ + ecosystem: "conda", policyName: name, storedName: packageName, version: version, + lookup: func() (time.Time, bool, error) { return h.publishTime(r, channel, arch, filename) }, + }) { + return + } + upstreamURL := h.upstreamURL + r.URL.Path // The same filename can hold different bytes in each subdir, so the // subdir is part of the cache key. @@ -531,6 +538,43 @@ func (h *CondaHandler) metadataCacheKey(urlPath string) string { return hashedCacheKey(h.upstreamURL, urlPath) } +// publishTime reads a package file's timestamp from the subdir's +// repodata.json. It shares proxyCached's cache entry and encoding, so the +// repodata a client just fetched is not downloaded again, and reads the one +// entry in place rather than decoding the whole index. +func (h *CondaHandler) publishTime(r *http.Request, channel, arch, filename string) (time.Time, bool, error) { + repodataPath := "/" + channel + "/" + arch + "/repodata.json" + cacheKey := h.metadataCacheKey(repodataPath) + body, _, encoding, err := h.proxy.fetchOrCacheMetadata(r.Context(), "conda", cacheKey, h.upstreamURL+repodataPath, "gzip", nil, "*/*") + if err != nil { + return time.Time{}, false, err + } + if strings.EqualFold(strings.TrimSpace(encoding), "gzip") { + decoded, err := gzip.NewReader(bytes.NewReader(body)) + if err != nil { + return time.Time{}, false, err + } + body, err = h.proxy.ReadMetadata(decoded) + if err != nil { + return time.Time{}, false, err + } + } + + key := "packages" + if strings.HasSuffix(filename, ".conda") { + key = "packages.conda" + } + raw, err := lookupJSON(body, key, filename, "timestamp") + if err != nil || len(raw) == 0 { + return time.Time{}, false, err + } + var ts float64 + if err := json.Unmarshal(raw, &ts); err != nil || ts == 0 { + return time.Time{}, false, nil + } + return time.Unix(int64(ts)/condaTimestampDivisor, 0), true, nil +} + // proxyCached forwards a metadata request with caching. func (h *CondaHandler) proxyCached(w http.ResponseWriter, r *http.Request) { cacheKey := h.metadataCacheKey(r.URL.Path) diff --git a/internal/handler/download_cooldown.go b/internal/handler/download_cooldown.go new file mode 100644 index 0000000..94554d3 --- /dev/null +++ b/internal/handler/download_cooldown.go @@ -0,0 +1,82 @@ +package handler + +import ( + "net/http" + "time" +) + +// publishTimeLookup reads a version's publish time from upstream metadata. ok +// is false when the metadata has no usable time for the version. +type publishTimeLookup func() (published time.Time, ok bool, err error) + +// downloadVersion names the version an artifact download is for, so its +// cooldown can be checked before the artifact is served. +type downloadVersion struct { + ecosystem string + // policyName is the package name the cooldown policy is keyed by, + // canonicalPackagePURL(ecosystem, policyName). + policyName string + // storedName and version name the versions row the publish time is kept + // in, the same names the artifact is cached under. An empty storedName + // keeps no time, and every check asks lookup. + storedName string + version string + lookup publishTimeLookup +} + +// withholdInCooldown reports whether a download is inside the cooldown +// window, and if so answers it with a 404. Filtering metadata is not enough on +// its own: lockfiles record artifact URLs, so clients reach the download path +// without reading the metadata first, and a cached artifact would be served +// even after the policy was raised. +// +// A publish time never changes, so the stored versions row is read first and +// lookup is only asked for a version the proxy has no time for yet; the time +// it finds is stored. A version with no usable publish time is allowed +// through, matching how the metadata filters treat it. +func (p *Proxy) withholdInCooldown(w http.ResponseWriter, d downloadVersion) bool { + if !p.versionInCooldown(d) { + return false + } + p.Logger.Info("cooldown: withholding "+d.ecosystem+" download", + "name", d.policyName, "version", d.version) + http.Error(w, ErrVersionNotFound.Error(), http.StatusNotFound) + return true +} + +func (p *Proxy) versionInCooldown(d downloadVersion) bool { + if p.Cooldown == nil || !p.Cooldown.Enabled() { + return false + } + policyKey := canonicalPackagePURL(d.ecosystem, d.policyName) + if p.Cooldown.For(d.ecosystem, policyKey) <= 0 { + return false + } + + var versionPURL string + if d.storedName != "" { + versionPURL = canonicalVersionPURL(d.ecosystem, d.storedName, d.version) + if ver, err := p.DB.GetVersionByPURL(versionPURL); err == nil && ver != nil && ver.PublishedAt.Valid { + return !p.Cooldown.IsAllowed(d.ecosystem, policyKey, ver.PublishedAt.Time) + } + } + + published, ok, err := d.lookup() + if err != nil { + p.Logger.Warn("cooldown: could not read publish time for download check", + "ecosystem", d.ecosystem, "name", d.policyName, "version", d.version, "error", err) + return false + } + if !ok { + return false + } + + if versionPURL != "" { + if err := p.DB.SetVersionPublishedAt(versionPURL, canonicalPackagePURL(d.ecosystem, d.storedName), published); err != nil { + p.Logger.Warn("cooldown: could not store publish time", + "ecosystem", d.ecosystem, "name", d.policyName, "version", d.version, "error", err) + } + } + + return !p.Cooldown.IsAllowed(d.ecosystem, policyKey, published) +} diff --git a/internal/handler/download_cooldown_test.go b/internal/handler/download_cooldown_test.go new file mode 100644 index 0000000..9b791be --- /dev/null +++ b/internal/handler/download_cooldown_test.go @@ -0,0 +1,347 @@ +package handler + +import ( + "compress/gzip" + "encoding/json" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/git-pkgs/cooldown" +) + +// Versions every download cooldown fixture lists: one published before the +// window, one inside it, and one without a publish time. +const ( + cooldownOldVersion = "1.0.0" + cooldownRecentVersion = "2.0.0" + cooldownUntimed = "3.0.0" +) + +// downloadCooldownCase drives one ecosystem's artifact endpoint. The package +// is always named demo, so the policy key is pkg:{ecosystem}/demo. +type downloadCooldownCase struct { + ecosystem string + // storedName is the name the artifact is cached under; empty when the + // ecosystem keeps no publish time in the versions table. + storedName string + // artifactVersion maps a release to the version the artifact is cached + // under, and filename and path to its file and download path. + artifactVersion func(version string) string + filename func(version string) string + path func(version string) string + // metadata serves the upstream document holding the publish times; a + // zero time is listed without one. + metadata func(w http.ResponseWriter, r *http.Request, times map[string]time.Time) + handler func(proxy *Proxy, upstreamURL string) http.Handler +} + +func sameVersion(version string) string { return version } + +func chartContent(version string) string { return "chart " + version } + +func downloadCooldownCases() []downloadCooldownCase { + return []downloadCooldownCase{ + { + ecosystem: "cargo", + storedName: "demo", + artifactVersion: sameVersion, + filename: func(v string) string { return "demo-" + v + ".crate" }, + path: func(v string) string { return "/crates/demo/" + v + "/download" }, + metadata: cargoCooldownMetadata, + handler: func(p *Proxy, u string) http.Handler { return NewCargoHandler(p, "http://proxy", u, u).Routes() }, + }, + { + ecosystem: "gem", + storedName: "demo", + // Platform gems carry the platform in the filename and the + // cached version, but not in the versions API number. + artifactVersion: func(v string) string { return v + "-x86_64-linux" }, + filename: func(v string) string { return "demo-" + v + "-x86_64-linux.gem" }, + path: func(v string) string { return "/gems/demo-" + v + "-x86_64-linux.gem" }, + metadata: gemCooldownMetadata, + handler: func(p *Proxy, u string) http.Handler { return NewGemHandlerWithUpstream(p, "http://proxy", u).Routes() }, + }, + { + ecosystem: "hex", + storedName: "demo", + artifactVersion: sameVersion, + filename: func(v string) string { return "demo-" + v + ".tar" }, + path: func(v string) string { return "/tarballs/demo-" + v + ".tar" }, + metadata: hexCooldownMetadata, + handler: func(p *Proxy, u string) http.Handler { + return NewHexHandlerWithUpstreams(p, "http://proxy", u, u).Routes() + }, + }, + { + ecosystem: "pub", + storedName: "demo", + artifactVersion: sameVersion, + filename: func(v string) string { return "demo-" + v + ".tar.gz" }, + path: func(v string) string { return "/packages/demo/versions/" + v + ".tar.gz" }, + metadata: pubCooldownMetadata, + handler: func(p *Proxy, u string) http.Handler { return NewPubHandlerWithUpstream(p, "http://proxy", u).Routes() }, + }, + { + // Conda caches under {channel}/{name}, but the policy is keyed by + // the name alone, as the repodata filter keys it. + ecosystem: "conda", + storedName: "conda-forge/demo", + artifactVersion: sameVersion, + filename: func(v string) string { return "noarch/demo-" + v + "-py_0.conda" }, + path: func(v string) string { return "/conda-forge/noarch/demo-" + v + "-py_0.conda" }, + metadata: condaCooldownMetadata, + handler: func(p *Proxy, u string) http.Handler { + return NewCondaHandlerWithUpstream(p, "http://proxy", u).Routes() + }, + }, + { + // Helm caches charts under the repository and digest, so the + // chart name and time come from the index on every check. + ecosystem: "helm", + artifactVersion: func(v string) string { return sha256Hex(chartContent(v)) }, + filename: func(v string) string { return "demo-" + v + ".tgz" }, + path: func(v string) string { return "/stable/charts/" + sha256Hex(chartContent(v)) + "/demo-" + v + ".tgz" }, + metadata: helmCooldownMetadata, + handler: func(p *Proxy, u string) http.Handler { + return NewHelmHandler(p, "http://proxy", map[string]string{"stable": u + "/charts"}).Routes() + }, + }, + } +} + +func cargoCooldownMetadata(w http.ResponseWriter, r *http.Request, times map[string]time.Time) { + if r.URL.Path != "/de/mo/demo" { + http.NotFound(w, r) + return + } + for version, published := range times { + line := map[string]string{"name": "demo", "vers": version} + if !published.IsZero() { + line["pubtime"] = published.Format(time.RFC3339) + } + encoded, _ := json.Marshal(line) + _, _ = fmt.Fprintf(w, "%s\n", encoded) + } +} + +func gemCooldownMetadata(w http.ResponseWriter, r *http.Request, times map[string]time.Time) { + if r.URL.Path != "/api/v1/versions/demo.json" { + http.NotFound(w, r) + return + } + var versions []gemVersion + for version, published := range times { + v := gemVersion{Number: version, Platform: "x86_64-linux"} + if !published.IsZero() { + v.CreatedAt = published.Format(time.RFC3339) + } + versions = append(versions, v) + } + _ = json.NewEncoder(w).Encode(versions) +} + +func hexCooldownMetadata(w http.ResponseWriter, r *http.Request, times map[string]time.Time) { + if r.URL.Path != "/api/packages/demo" { + http.NotFound(w, r) + return + } + var pkg hexPackageAPI + for version, published := range times { + release := hexRelease{Version: version} + if !published.IsZero() { + release.InsertedAt = published.Format(time.RFC3339Nano) + } + pkg.Releases = append(pkg.Releases, release) + } + _ = json.NewEncoder(w).Encode(pkg) +} + +func pubCooldownMetadata(w http.ResponseWriter, r *http.Request, times map[string]time.Time) { + if r.URL.Path != "/api/packages/demo" { + http.NotFound(w, r) + return + } + var versions []map[string]string + for version, published := range times { + v := map[string]string{"version": version} + if !published.IsZero() { + v["published"] = published.Format(time.RFC3339) + } + versions = append(versions, v) + } + _ = json.NewEncoder(w).Encode(map[string]any{"name": "demo", "versions": versions}) +} + +func condaCooldownMetadata(w http.ResponseWriter, r *http.Request, times map[string]time.Time) { + if r.URL.Path != "/conda-forge/noarch/repodata.json" { + http.NotFound(w, r) + return + } + packages := map[string]map[string]any{} + for version, published := range times { + entry := map[string]any{"name": "demo", "version": version} + if !published.IsZero() { + entry["timestamp"] = published.UnixMilli() + } + packages["demo-"+version+"-py_0.conda"] = entry + } + // The download check asks for gzip, as proxyCached does. + w.Header().Set(headerContentEncoding, "gzip") + gz := gzip.NewWriter(w) + _ = json.NewEncoder(gz).Encode(map[string]any{"packages.conda": packages}) + _ = gz.Close() +} + +func helmCooldownMetadata(w http.ResponseWriter, r *http.Request, times map[string]time.Time) { + if r.URL.Path != "/charts/index.yaml" { + http.NotFound(w, r) + return + } + _, _ = fmt.Fprintln(w, "apiVersion: v1\nentries:\n demo:") + for version, published := range times { + _, _ = fmt.Fprintf(w, " - digest: %s\n urls: [demo-%s.tgz]\n version: %s\n", + sha256Hex(chartContent(version)), version, version) + if !published.IsZero() { + _, _ = fmt.Fprintf(w, " created: %s\n", published.Format(time.RFC3339)) + } + } +} + +// seedCachedArtifact caches the release's artifact the way an earlier, +// cooldown-free download would have. +func (c downloadCooldownCase) seedCachedArtifact(t *testing.T, proxy *Proxy, version string) { + t.Helper() + db, store := proxy.DB, proxy.Storage.(*mockStorage) + name := c.storedName + content := "artifact " + version + if c.ecosystem == "helm" { + name, content = "stable", chartContent(version) + } + seedPackage(t, db, store, c.ecosystem, name, c.artifactVersion(version), c.filename(version), content) +} + +// TestDownloadCooldownWithholdsCachedArtifacts requests each ecosystem's +// artifact endpoint for a version that was cached before the cooldown was +// raised. A version inside the window must be withheld, from the cache too; +// one past the window, or without a known publish time, is served. +func TestDownloadCooldownWithholdsCachedArtifacts(t *testing.T) { + now := time.Now().Truncate(time.Second) + times := map[string]time.Time{ + cooldownOldVersion: now.Add(-30 * 24 * time.Hour), + cooldownRecentVersion: now.Add(-time.Hour), + cooldownUntimed: {}, + } + statuses := map[string]int{ + cooldownOldVersion: http.StatusOK, + cooldownRecentVersion: http.StatusNotFound, + cooldownUntimed: http.StatusOK, + } + + for _, c := range downloadCooldownCases() { + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + c.metadata(w, r, times) + })) + t.Cleanup(upstream.Close) + + for _, version := range []string{cooldownOldVersion, cooldownRecentVersion, cooldownUntimed} { + t.Run(c.ecosystem+"/"+version, func(t *testing.T) { + proxy, db, _, fetcher := setupTestProxy(t) + proxy.HTTPClient = upstream.Client() + fetcher.fetchErr = errors.New("artifact must be served from the cache") + c.seedCachedArtifact(t, proxy, version) + proxy.Cooldown = &cooldown.Config{Packages: map[string]string{ + canonicalPackagePURL(c.ecosystem, "demo"): "7d", + }} + + w := httptest.NewRecorder() + c.handler(proxy, upstream.URL).ServeHTTP(w, httptest.NewRequest(http.MethodGet, c.path(version), nil)) + + if w.Code != statuses[version] { + t.Fatalf("status = %d, want %d (body %q)", w.Code, statuses[version], w.Body.String()) + } + if w.Code == http.StatusOK && w.Body.String() == "" { + t.Error("served an empty body") + } + if c.storedName == "" || times[version].IsZero() { + return + } + ver, err := db.GetVersionByPURL(canonicalVersionPURL(c.ecosystem, c.storedName, c.artifactVersion(version))) + if err != nil || ver == nil || !ver.PublishedAt.Valid { + t.Fatalf("publish time not stored: %+v, %v", ver, err) + } + if got, want := ver.PublishedAt.Time, times[version]; !got.Equal(want) { + t.Errorf("stored publish time = %v, want %v", got, want) + } + }) + } + } +} + +// TestDownloadCooldownUsesStoredPublishTime checks that a stored publish +// time decides the download without asking the upstream, which here fails. +func TestDownloadCooldownUsesStoredPublishTime(t *testing.T) { + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + http.Error(w, "unavailable", http.StatusServiceUnavailable) + })) + t.Cleanup(upstream.Close) + + for _, c := range downloadCooldownCases() { + if c.storedName == "" { + continue + } + t.Run(c.ecosystem, func(t *testing.T) { + proxy, db, _, fetcher := setupTestProxy(t) + proxy.HTTPClient = upstream.Client() + fetcher.fetchErr = errors.New("artifact must be served from the cache") + c.seedCachedArtifact(t, proxy, cooldownRecentVersion) + versionPURL := canonicalVersionPURL(c.ecosystem, c.storedName, c.artifactVersion(cooldownRecentVersion)) + packagePURL := canonicalPackagePURL(c.ecosystem, c.storedName) + if err := db.SetVersionPublishedAt(versionPURL, packagePURL, time.Now().Add(-time.Hour)); err != nil { + t.Fatalf("seeding publish time: %v", err) + } + proxy.Cooldown = &cooldown.Config{Default: "7d"} + + w := httptest.NewRecorder() + c.handler(proxy, upstream.URL).ServeHTTP(w, httptest.NewRequest(http.MethodGet, c.path(cooldownRecentVersion), nil)) + if w.Code != http.StatusNotFound { + t.Fatalf("status = %d, want 404 (body %q)", w.Code, w.Body.String()) + } + }) + } +} + +// TestDownloadCooldownOtherPackagesSkipLookup checks that a download outside +// every cooldown rule is served without reading upstream metadata. Helm +// needs the index to learn the chart name, so it is left out. +func TestDownloadCooldownOtherPackagesSkipLookup(t *testing.T) { + for _, c := range downloadCooldownCases() { + if c.storedName == "" { + continue + } + t.Run(c.ecosystem, func(t *testing.T) { + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + t.Errorf("unexpected upstream request %s", r.URL.Path) + http.Error(w, "unexpected", http.StatusInternalServerError) + })) + t.Cleanup(upstream.Close) + proxy, _, _, fetcher := setupTestProxy(t) + proxy.HTTPClient = upstream.Client() + fetcher.fetchErr = errors.New("artifact must be served from the cache") + c.seedCachedArtifact(t, proxy, cooldownRecentVersion) + proxy.Cooldown = &cooldown.Config{Packages: map[string]string{ + canonicalPackagePURL(c.ecosystem, "other"): "7d", + }} + + w := httptest.NewRecorder() + c.handler(proxy, upstream.URL).ServeHTTP(w, httptest.NewRequest(http.MethodGet, c.path(cooldownRecentVersion), nil)) + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want 200 (body %q)", w.Code, strings.TrimSpace(w.Body.String())) + } + }) + } +} diff --git a/internal/handler/filename_download.go b/internal/handler/filename_download.go index 153c0e6..dbc0e64 100644 --- a/internal/handler/filename_download.go +++ b/internal/handler/filename_download.go @@ -3,6 +3,7 @@ package handler import ( "net/http" "strings" + "time" ) type filenameDownload struct { @@ -12,6 +13,9 @@ type filenameDownload struct { parseErr string fetchErr string parse func(string) (name, version string) + // publishTime reads a version's publish time from upstream metadata for + // the download-time cooldown check. + publishTime func(r *http.Request, name, version string) (time.Time, bool, error) } func (p *Proxy) handleFilenameDownload(w http.ResponseWriter, r *http.Request, d filenameDownload) { @@ -30,6 +34,13 @@ func (p *Proxy) handleFilenameDownload(w http.ResponseWriter, r *http.Request, d p.Logger.Debug(d.ecosystem+" download request", "name", name, "version", version, "filename", filename) + if p.withholdInCooldown(w, downloadVersion{ + ecosystem: d.ecosystem, policyName: name, storedName: name, version: version, + lookup: func() (time.Time, bool, error) { return d.publishTime(r, name, version) }, + }) { + return + } + downloadURL := d.upstreamURL + r.URL.Path result, err := p.GetOrFetchArtifactFromURL( r.Context(), d.ecosystem, name, version, filename, downloadURL, diff --git a/internal/handler/gem.go b/internal/handler/gem.go index 6e57559..009e0f5 100644 --- a/internal/handler/gem.go +++ b/internal/handler/gem.go @@ -5,6 +5,7 @@ import ( "encoding/json" "fmt" "net/http" + "net/url" "regexp" "strings" "time" @@ -82,6 +83,7 @@ func (h *GemHandler) handleDownload(w http.ResponseWriter, r *http.Request) { parseErr: "could not parse gem filename", fetchErr: "failed to fetch gem", parse: h.parseGemFilename, + publishTime: h.publishTime, }) } @@ -304,6 +306,34 @@ func parseGemVersions(body []byte) (publishTimes, error) { return times, nil } +// publishTime reads a gem version's created_at from the versions API. The +// download check caches the response, unlike fetchFilteredVersions, so each +// download does not cost an upstream request. version carries the platform +// the way gem filenames do, as in 1.0.0-x86_64-linux. +func (h *GemHandler) publishTime(r *http.Request, name, version string) (time.Time, bool, error) { + versionsURL := fmt.Sprintf("%s/api/v1/versions/%s.json", h.upstreamURL, url.PathEscape(name)) + body, _, err := h.proxy.FetchOrCacheMetadata(r.Context(), "gem", "api/v1/versions/"+name, versionsURL) + if err != nil { + return time.Time{}, false, err + } + var versions []gemVersion + if err := json.Unmarshal(body, &versions); err != nil { + return time.Time{}, false, err + } + for _, v := range versions { + number := v.Number + if v.Platform != "" && v.Platform != "ruby" { + number += "-" + v.Platform + } + if number != version { + continue + } + createdAt, err := time.Parse(time.RFC3339, v.CreatedAt) + return createdAt, err == nil, nil + } + return time.Time{}, false, nil +} + // proxyCached forwards a metadata request with caching. func (h *GemHandler) proxyCached(w http.ResponseWriter, r *http.Request) { upstreamURL := h.upstreamURL + r.URL.Path diff --git a/internal/handler/helm.go b/internal/handler/helm.go index fdc000e..9fb947f 100644 --- a/internal/handler/helm.go +++ b/internal/handler/helm.go @@ -127,6 +127,12 @@ func (h *HelmHandler) handleChart(w http.ResponseWriter, r *http.Request) { return } if cached != nil { + if h.withholdCachedChart(w, r, repository, upstreamURL, digest) { + if cached.Reader != nil { + _ = cached.Reader.Close() + } + return + } h.serveChart(w, r, repository, digest, filename, cached) return } @@ -376,6 +382,54 @@ func (h *HelmHandler) buildChartLookup(upstreamURL string, body []byte) (*helmCh return lookup, nil } +// withholdCachedChart checks a cached chart against the cooldown before it is +// served. An uncached chart is checked by findChartDownload, but a cached one +// may have been fetched under an earlier policy. Charts are cached under the +// repository and digest rather than the chart name the policy is keyed by, so +// the name and publish time come from the index on every check. +func (h *HelmHandler) withholdCachedChart(w http.ResponseWriter, r *http.Request, repository, upstreamURL, digest string) bool { + if h.proxy.Cooldown == nil || !h.proxy.Cooldown.Enabled() { + return false + } + index := h.fetchIndex(r, repository, upstreamURL) + if index.err != nil { + h.proxy.Logger.Warn("cooldown: could not fetch Helm index for download check", + "repository", repository, "digest", digest, "error", index.err) + return false + } + chartName, created, ok := h.findChartRelease(upstreamURL, index.body, digest) + if !ok { + return false + } + return h.proxy.withholdInCooldown(w, downloadVersion{ + ecosystem: helmMetadataEcosystem, policyName: chartName, version: digest, + lookup: func() (time.Time, bool, error) { return created, !created.IsZero(), nil }, + }) +} + +// findChartRelease returns the name and created time of the index release +// with the given digest. +func (h *HelmHandler) findChartRelease(upstreamURL string, body []byte, digest string) (string, time.Time, bool) { + _, entries, err := parseHelmIndex(body) + if err != nil { + return "", time.Time{}, false + } + for i := 0; i < len(entries.Content); i += 2 { + chartName := entries.Content[i].Value + releases := entries.Content[i+1] + if releases.Kind != yaml.SequenceNode { + continue + } + for _, release := range releases.Content { + chart, err := h.parseChartRelease(chartName, upstreamURL, release) + if err == nil && chart.digest == digest { + return chartName, chart.created, true + } + } + } + return "", time.Time{}, false +} + func (h *HelmHandler) chartOnCooldown(chartName string, created time.Time) bool { return !created.IsZero() && h.proxy.Cooldown != nil && h.proxy.Cooldown.Enabled() && !h.proxy.cooldown().IsAllowed(helmMetadataEcosystem, canonicalPackagePURL(helmMetadataEcosystem, chartName), created) diff --git a/internal/handler/hex.go b/internal/handler/hex.go index f2a93ad..6836dad 100644 --- a/internal/handler/hex.go +++ b/internal/handler/hex.go @@ -6,6 +6,7 @@ import ( "encoding/json" "fmt" "net/http" + "net/url" "strings" "time" @@ -78,6 +79,7 @@ func (h *HexHandler) handleDownload(w http.ResponseWriter, r *http.Request) { parseErr: "could not parse tarball filename", fetchErr: "failed to fetch package", parse: h.parseTarballFilename, + publishTime: h.publishTime, }) } @@ -241,6 +243,29 @@ func parseHexReleases(body []byte) (publishTimes, error) { return times, nil } +// publishTime reads a release's inserted_at from the Hex HTTP API. The +// download check caches the response, unlike fetchFilteredVersions, so each +// download does not cost an upstream request. +func (h *HexHandler) publishTime(r *http.Request, name, version string) (time.Time, bool, error) { + apiURL := fmt.Sprintf("%s/api/packages/%s", h.apiURL, url.PathEscape(name)) + body, _, err := h.proxy.FetchOrCacheMetadata(r.Context(), "hex", "api/packages/"+name, apiURL) + if err != nil { + return time.Time{}, false, err + } + var pkg hexPackageAPI + if err := json.Unmarshal(body, &pkg); err != nil { + return time.Time{}, false, err + } + for _, release := range pkg.Releases { + if release.Version != version { + continue + } + insertedAt, err := time.Parse(time.RFC3339Nano, release.InsertedAt) + return insertedAt, err == nil, nil + } + return time.Time{}, false, nil +} + // signedPayload decompresses a gzipped Signed message, with the metadata // limit applied to the decompressed size, and returns its payload. func (h *HexHandler) signedPayload(gzippedData []byte) ([]byte, error) { diff --git a/internal/handler/pub.go b/internal/handler/pub.go index 24fdd02..22540f1 100644 --- a/internal/handler/pub.go +++ b/internal/handler/pub.go @@ -94,6 +94,13 @@ func (h *PubHandler) handleDownload(w http.ResponseWriter, r *http.Request) { h.proxy.Logger.Debug("pub download request", "name", name, "version", version) + if h.proxy.withholdInCooldown(w, downloadVersion{ + ecosystem: "pub", policyName: name, storedName: name, version: version, + lookup: func() (time.Time, bool, error) { return h.publishTime(r, name, version) }, + }) { + return + } + downloadURL := fmt.Sprintf("%s/packages/%s/versions/%s%s", h.upstreamURL, url.PathEscape(name), url.PathEscape(version), pubArchiveExt) h.discardVersionPage(name, version, filename) @@ -134,6 +141,33 @@ func (h *PubHandler) discardVersionPage(name, version, filename string) { } } +// publishTime reads a version's published time from the package metadata, +// sharing the metadata cache with handlePackageMetadata. +func (h *PubHandler) publishTime(r *http.Request, name, version string) (time.Time, bool, error) { + upstreamURL := fmt.Sprintf("%s/api/packages/%s", h.upstreamURL, name) + body, _, err := h.proxy.FetchOrCacheMetadata(r.Context(), "pub", name, upstreamURL) + if err != nil { + return time.Time{}, false, err + } + var metadata struct { + Versions []struct { + Version string `json:"version"` + Published string `json:"published"` + } `json:"versions"` + } + if err := json.Unmarshal(body, &metadata); err != nil { + return time.Time{}, false, err + } + for _, v := range metadata.Versions { + if v.Version != version { + continue + } + published, err := time.Parse(time.RFC3339, v.Published) + return published, err == nil, nil + } + return time.Time{}, false, nil +} + // handlePackageMetadata proxies package metadata and rewrites archive URLs. func (h *PubHandler) handlePackageMetadata(w http.ResponseWriter, r *http.Request) { name := r.PathValue("name") From 89c60abd1a51a551aceef46d030f55f0f0b94cb6 Mon Sep 17 00:00:00 2001 From: Christian Heim Date: Sun, 18 Oct 2026 21:43:57 +0200 Subject: [PATCH 559/633] Conda now checks each package file against its own timestamp, because builds of one version differ. Conda repodata and helm indexes are parsed once per cached document instead of on every download. --- internal/handler/conda.go | 55 +++++++---- internal/handler/download_cooldown.go | 62 ++++++++++++ internal/handler/download_cooldown_test.go | 108 +++++++++++++++++---- internal/handler/handler.go | 4 + internal/handler/helm.go | 62 +++++++----- 5 files changed, 232 insertions(+), 59 deletions(-) diff --git a/internal/handler/conda.go b/internal/handler/conda.go index 22ec603..d7cf196 100644 --- a/internal/handler/conda.go +++ b/internal/handler/conda.go @@ -139,8 +139,10 @@ func (h *CondaHandler) handleDownload(w http.ResponseWriter, r *http.Request) { h.proxy.Logger.Debug("conda download request", "channel", channel, "arch", arch, "name", name, "version", version) + // Builds and subdirs of one version each have their own timestamp, so + // the check is per file and keeps no time in the shared versions row. if h.proxy.withholdInCooldown(w, downloadVersion{ - ecosystem: "conda", policyName: name, storedName: packageName, version: version, + ecosystem: "conda", policyName: name, version: version, lookup: func() (time.Time, bool, error) { return h.publishTime(r, channel, arch, filename) }, }) { return @@ -539,40 +541,57 @@ func (h *CondaHandler) metadataCacheKey(urlPath string) string { } // publishTime reads a package file's timestamp from the subdir's -// repodata.json. It shares proxyCached's cache entry and encoding, so the -// repodata a client just fetched is not downloaded again, and reads the one -// entry in place rather than decoding the whole index. +// repodata.json, fetched with the cache key and encoding proxyCached uses. +// The repodata is indexed once per document (see publishIndex), since a +// subdir's index runs to tens of megabytes and an install downloads many +// files from it. func (h *CondaHandler) publishTime(r *http.Request, channel, arch, filename string) (time.Time, bool, error) { repodataPath := "/" + channel + "/" + arch + "/repodata.json" cacheKey := h.metadataCacheKey(repodataPath) - body, _, encoding, err := h.proxy.fetchOrCacheMetadata(r.Context(), "conda", cacheKey, h.upstreamURL+repodataPath, "gzip", nil, "*/*") + res := h.proxy.lookupMetadata(r.Context(), "conda", cacheKey, h.upstreamURL+repodataPath, "*/*", "gzip", nil) + if res.err != nil { + return time.Time{}, false, res.err + } + index, err := publishIndex(h.proxy, "conda\x00"+cacheKey, res, h.indexRepodata) if err != nil { return time.Time{}, false, err } + published, ok := index[filename] + return published, ok, nil +} + +// indexRepodata maps each package file in a repodata.json to its timestamp, +// leaving out files without one. +func (h *CondaHandler) indexRepodata(body []byte, encoding string) (map[string]time.Time, error) { if strings.EqualFold(strings.TrimSpace(encoding), "gzip") { decoded, err := gzip.NewReader(bytes.NewReader(body)) if err != nil { - return time.Time{}, false, err + return nil, err } body, err = h.proxy.ReadMetadata(decoded) if err != nil { - return time.Time{}, false, err + return nil, err } } - - key := "packages" - if strings.HasSuffix(filename, ".conda") { - key = "packages.conda" + type entry struct { + Timestamp float64 `json:"timestamp"` } - raw, err := lookupJSON(body, key, filename, "timestamp") - if err != nil || len(raw) == 0 { - return time.Time{}, false, err + var repodata struct { + Packages map[string]entry `json:"packages"` + PackagesConda map[string]entry `json:"packages.conda"` } - var ts float64 - if err := json.Unmarshal(raw, &ts); err != nil || ts == 0 { - return time.Time{}, false, nil + if err := json.Unmarshal(body, &repodata); err != nil { + return nil, err } - return time.Unix(int64(ts)/condaTimestampDivisor, 0), true, nil + index := make(map[string]time.Time, len(repodata.Packages)+len(repodata.PackagesConda)) + for _, files := range []map[string]entry{repodata.Packages, repodata.PackagesConda} { + for filename, e := range files { + if e.Timestamp != 0 { + index[filename] = time.Unix(int64(e.Timestamp)/condaTimestampDivisor, 0) + } + } + } + return index, nil } // proxyCached forwards a metadata request with caching. diff --git a/internal/handler/download_cooldown.go b/internal/handler/download_cooldown.go index 94554d3..5aec959 100644 --- a/internal/handler/download_cooldown.go +++ b/internal/handler/download_cooldown.go @@ -2,7 +2,10 @@ package handler import ( "net/http" + "sync" "time" + + "golang.org/x/sync/singleflight" ) // publishTimeLookup reads a version's publish time from upstream metadata. ok @@ -80,3 +83,62 @@ func (p *Proxy) versionInCooldown(d downloadVersion) bool { return !p.Cooldown.IsAllowed(d.ecosystem, policyKey, published) } + +// maxPublishIndexes bounds how many documents publishIndexCache keeps an +// index for. +const maxPublishIndexes = 64 + +// publishIndexCache keeps, for each large metadata document, the publish +// times a download check reads from it, so a document is decompressed and +// parsed once per version from upstream rather than on every download. +// Concurrent checks for a document not indexed yet share one parse. +type publishIndexCache struct { + mu sync.Mutex + entries map[string]publishIndexEntry + group singleflight.Group +} + +type publishIndexEntry struct { + digest string + index any +} + +// publishIndex returns build's index of res, reusing the one built for the +// same document under key. res.digest identifies the document; without one, +// as when the metadata cache is off, the index is built for this call alone. +func publishIndex[V any](p *Proxy, key string, res metadataResult, build func([]byte, string) (map[string]V, error)) (map[string]V, error) { + c := &p.publishIndexes + if res.digest == "" { + return build(res.body, res.contentEncoding) + } + c.mu.Lock() + if e, ok := c.entries[key]; ok && e.digest == res.digest { + c.mu.Unlock() + return e.index.(map[string]V), nil + } + c.mu.Unlock() + + v, err, _ := c.group.Do(key+"\x00"+res.digest, func() (any, error) { + index, err := build(res.body, res.contentEncoding) + if err != nil { + return nil, err + } + c.mu.Lock() + defer c.mu.Unlock() + if c.entries == nil { + c.entries = make(map[string]publishIndexEntry) + } + if _, ok := c.entries[key]; !ok && len(c.entries) >= maxPublishIndexes { + for evict := range c.entries { + delete(c.entries, evict) + break + } + } + c.entries[key] = publishIndexEntry{digest: res.digest, index: index} + return index, nil + }) + if err != nil { + return nil, err + } + return v.(map[string]V), nil +} diff --git a/internal/handler/download_cooldown_test.go b/internal/handler/download_cooldown_test.go index 9b791be..c504edb 100644 --- a/internal/handler/download_cooldown_test.go +++ b/internal/handler/download_cooldown_test.go @@ -26,9 +26,10 @@ const ( // is always named demo, so the policy key is pkg:{ecosystem}/demo. type downloadCooldownCase struct { ecosystem string - // storedName is the name the artifact is cached under; empty when the - // ecosystem keeps no publish time in the versions table. - storedName string + // cachedName is the name the artifact is cached under, and keepsTime + // whether the publish time is kept in its versions row. + cachedName string + keepsTime bool // artifactVersion maps a release to the version the artifact is cached // under, and filename and path to its file and download path. artifactVersion func(version string) string @@ -48,7 +49,8 @@ func downloadCooldownCases() []downloadCooldownCase { return []downloadCooldownCase{ { ecosystem: "cargo", - storedName: "demo", + cachedName: "demo", + keepsTime: true, artifactVersion: sameVersion, filename: func(v string) string { return "demo-" + v + ".crate" }, path: func(v string) string { return "/crates/demo/" + v + "/download" }, @@ -57,7 +59,8 @@ func downloadCooldownCases() []downloadCooldownCase { }, { ecosystem: "gem", - storedName: "demo", + cachedName: "demo", + keepsTime: true, // Platform gems carry the platform in the filename and the // cached version, but not in the versions API number. artifactVersion: func(v string) string { return v + "-x86_64-linux" }, @@ -68,7 +71,8 @@ func downloadCooldownCases() []downloadCooldownCase { }, { ecosystem: "hex", - storedName: "demo", + cachedName: "demo", + keepsTime: true, artifactVersion: sameVersion, filename: func(v string) string { return "demo-" + v + ".tar" }, path: func(v string) string { return "/tarballs/demo-" + v + ".tar" }, @@ -79,7 +83,8 @@ func downloadCooldownCases() []downloadCooldownCase { }, { ecosystem: "pub", - storedName: "demo", + cachedName: "demo", + keepsTime: true, artifactVersion: sameVersion, filename: func(v string) string { return "demo-" + v + ".tar.gz" }, path: func(v string) string { return "/packages/demo/versions/" + v + ".tar.gz" }, @@ -88,9 +93,10 @@ func downloadCooldownCases() []downloadCooldownCase { }, { // Conda caches under {channel}/{name}, but the policy is keyed by - // the name alone, as the repodata filter keys it. + // the name alone, as the repodata filter keys it. Each build has + // its own timestamp, so none is kept in the version's row. ecosystem: "conda", - storedName: "conda-forge/demo", + cachedName: "conda-forge/demo", artifactVersion: sameVersion, filename: func(v string) string { return "noarch/demo-" + v + "-py_0.conda" }, path: func(v string) string { return "/conda-forge/noarch/demo-" + v + "-py_0.conda" }, @@ -102,6 +108,7 @@ func downloadCooldownCases() []downloadCooldownCase { { // Helm caches charts under the repository and digest, so the // chart name and time come from the index on every check. + cachedName: "stable", ecosystem: "helm", artifactVersion: func(v string) string { return sha256Hex(chartContent(v)) }, filename: func(v string) string { return "demo-" + v + ".tgz" }, @@ -217,12 +224,11 @@ func helmCooldownMetadata(w http.ResponseWriter, r *http.Request, times map[stri func (c downloadCooldownCase) seedCachedArtifact(t *testing.T, proxy *Proxy, version string) { t.Helper() db, store := proxy.DB, proxy.Storage.(*mockStorage) - name := c.storedName content := "artifact " + version if c.ecosystem == "helm" { - name, content = "stable", chartContent(version) + content = chartContent(version) } - seedPackage(t, db, store, c.ecosystem, name, c.artifactVersion(version), c.filename(version), content) + seedPackage(t, db, store, c.ecosystem, c.cachedName, c.artifactVersion(version), c.filename(version), content) } // TestDownloadCooldownWithholdsCachedArtifacts requests each ecosystem's @@ -267,10 +273,10 @@ func TestDownloadCooldownWithholdsCachedArtifacts(t *testing.T) { if w.Code == http.StatusOK && w.Body.String() == "" { t.Error("served an empty body") } - if c.storedName == "" || times[version].IsZero() { + if !c.keepsTime || times[version].IsZero() { return } - ver, err := db.GetVersionByPURL(canonicalVersionPURL(c.ecosystem, c.storedName, c.artifactVersion(version))) + ver, err := db.GetVersionByPURL(canonicalVersionPURL(c.ecosystem, c.cachedName, c.artifactVersion(version))) if err != nil || ver == nil || !ver.PublishedAt.Valid { t.Fatalf("publish time not stored: %+v, %v", ver, err) } @@ -291,7 +297,7 @@ func TestDownloadCooldownUsesStoredPublishTime(t *testing.T) { t.Cleanup(upstream.Close) for _, c := range downloadCooldownCases() { - if c.storedName == "" { + if !c.keepsTime { continue } t.Run(c.ecosystem, func(t *testing.T) { @@ -299,8 +305,8 @@ func TestDownloadCooldownUsesStoredPublishTime(t *testing.T) { proxy.HTTPClient = upstream.Client() fetcher.fetchErr = errors.New("artifact must be served from the cache") c.seedCachedArtifact(t, proxy, cooldownRecentVersion) - versionPURL := canonicalVersionPURL(c.ecosystem, c.storedName, c.artifactVersion(cooldownRecentVersion)) - packagePURL := canonicalPackagePURL(c.ecosystem, c.storedName) + versionPURL := canonicalVersionPURL(c.ecosystem, c.cachedName, c.artifactVersion(cooldownRecentVersion)) + packagePURL := canonicalPackagePURL(c.ecosystem, c.cachedName) if err := db.SetVersionPublishedAt(versionPURL, packagePURL, time.Now().Add(-time.Hour)); err != nil { t.Fatalf("seeding publish time: %v", err) } @@ -320,7 +326,7 @@ func TestDownloadCooldownUsesStoredPublishTime(t *testing.T) { // needs the index to learn the chart name, so it is left out. func TestDownloadCooldownOtherPackagesSkipLookup(t *testing.T) { for _, c := range downloadCooldownCases() { - if c.storedName == "" { + if c.ecosystem == "helm" { continue } t.Run(c.ecosystem, func(t *testing.T) { @@ -345,3 +351,69 @@ func TestDownloadCooldownOtherPackagesSkipLookup(t *testing.T) { }) } } + +// TestCondaDownloadCooldownPerBuild checks that builds of one version are +// judged by their own timestamps, whichever is requested first: a rebuild +// inside the window is withheld and an old build stays available. +func TestCondaDownloadCooldownPerBuild(t *testing.T) { + now := time.Now() + repodata := map[string]any{"packages.conda": map[string]any{ + "demo-1.0.0-py_0.conda": map[string]any{"name": "demo", "version": "1.0.0", "timestamp": now.Add(-400 * 24 * time.Hour).UnixMilli()}, + "demo-1.0.0-py_1.conda": map[string]any{"name": "demo", "version": "1.0.0", "timestamp": now.Add(-time.Hour).UnixMilli()}, + }} + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _ = json.NewEncoder(w).Encode(repodata) + })) + t.Cleanup(upstream.Close) + + builds := map[string]int{"py_0": http.StatusOK, "py_1": http.StatusNotFound} + for _, order := range [][]string{{"py_0", "py_1", "py_0"}, {"py_1", "py_0", "py_1"}} { + t.Run(strings.Join(order, ","), func(t *testing.T) { + proxy, db, store, fetcher := setupTestProxy(t) + proxy.HTTPClient = upstream.Client() + fetcher.fetchErr = errors.New("artifact must be served from the cache") + for build := range builds { + filename := "noarch/demo-1.0.0-" + build + ".conda" + seedPackage(t, db, store, "conda", "conda-forge/demo", "1.0.0", filename, "artifact "+build) + } + proxy.Cooldown = &cooldown.Config{Default: "7d"} + h := NewCondaHandlerWithUpstream(proxy, "http://proxy", upstream.URL).Routes() + + for _, build := range order { + w := httptest.NewRecorder() + h.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/conda-forge/noarch/demo-1.0.0-"+build+".conda", nil)) + if w.Code != builds[build] { + t.Fatalf("%s status = %d, want %d", build, w.Code, builds[build]) + } + } + }) + } +} + +// TestPublishIndexBuildsOncePerDocument checks that an index is reused for +// the same document, rebuilt for a new one, and not kept without a digest. +func TestPublishIndexBuildsOncePerDocument(t *testing.T) { + proxy, _, _, _ := setupTestProxy(t) + builds := 0 + build := func(body []byte, _ string) (map[string]bool, error) { + builds++ + return map[string]bool{string(body): true}, nil + } + steps := []struct { + res metadataResult + want string + wantBuilds int + }{ + {metadataResult{body: []byte("a"), digest: "d1"}, "a", 1}, + {metadataResult{body: []byte("ignored"), digest: "d1"}, "a", 1}, + {metadataResult{body: []byte("b"), digest: "d2"}, "b", 2}, + {metadataResult{body: []byte("c")}, "c", 3}, + {metadataResult{body: []byte("c")}, "c", 4}, + } + for i, step := range steps { + got, err := publishIndex(proxy, "key", step.res, build) + if err != nil || !got[step.want] || builds != step.wantBuilds { + t.Fatalf("step %d: publishIndex = %v, %v after %d builds, want %q after %d", i, got, err, builds, step.want, step.wantBuilds) + } + } +} diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 196f648..125b117 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -292,6 +292,10 @@ type Proxy struct { // Nil leaves every request to rewrite its own copy. rewrites *rewriteCache + // publishIndexes keeps publish times read out of large metadata + // documents for the download cooldown check; see publishIndex. + publishIndexes publishIndexCache + // now is the clock rewritten metadata is dated by, and started the time // the process started; see metadataDate. now func() time.Time diff --git a/internal/handler/helm.go b/internal/handler/helm.go index 9fb947f..cbbf8b0 100644 --- a/internal/handler/helm.go +++ b/internal/handler/helm.go @@ -120,6 +120,10 @@ func (h *HelmHandler) handleChart(w http.ResponseWriter, r *http.Request) { return } + if h.withholdChart(w, r, repository, upstreamURL, digest) { + return + } + cached, err := h.proxy.GetCachedArtifact(r.Context(), helmMetadataEcosystem, repository, digest, filename) if err != nil { h.proxy.Logger.Error("failed to check Helm chart cache", "error", err) @@ -127,12 +131,6 @@ func (h *HelmHandler) handleChart(w http.ResponseWriter, r *http.Request) { return } if cached != nil { - if h.withholdCachedChart(w, r, repository, upstreamURL, digest) { - if cached.Reader != nil { - _ = cached.Reader.Close() - } - return - } h.serveChart(w, r, repository, digest, filename, cached) return } @@ -382,38 +380,55 @@ func (h *HelmHandler) buildChartLookup(upstreamURL string, body []byte) (*helmCh return lookup, nil } -// withholdCachedChart checks a cached chart against the cooldown before it is -// served. An uncached chart is checked by findChartDownload, but a cached one +// withholdChart checks a chart against the cooldown before it is served. +// findChartDownload already leaves out charts in cooldown, but a cached chart // may have been fetched under an earlier policy. Charts are cached under the // repository and digest rather than the chart name the policy is keyed by, so -// the name and publish time come from the index on every check. -func (h *HelmHandler) withholdCachedChart(w http.ResponseWriter, r *http.Request, repository, upstreamURL, digest string) bool { +// the name and publish time come from the index, indexed once per document +// (see publishIndex). +func (h *HelmHandler) withholdChart(w http.ResponseWriter, r *http.Request, repository, upstreamURL, digest string) bool { if h.proxy.Cooldown == nil || !h.proxy.Cooldown.Enabled() { return false } - index := h.fetchIndex(r, repository, upstreamURL) - if index.err != nil { + res := h.fetchIndex(r, repository, upstreamURL) + if res.err != nil { h.proxy.Logger.Warn("cooldown: could not fetch Helm index for download check", - "repository", repository, "digest", digest, "error", index.err) + "repository", repository, "digest", digest, "error", res.err) return false } - chartName, created, ok := h.findChartRelease(upstreamURL, index.body, digest) + index, err := publishIndex(h.proxy, "helm\x00"+h.indexCacheKey(repository, upstreamURL), res, + func(body []byte, _ string) (map[string]helmChartTime, error) { + return h.indexChartTimes(upstreamURL, body) + }) + if err != nil { + h.proxy.Logger.Warn("cooldown: could not read Helm index for download check", + "repository", repository, "digest", digest, "error", err) + return false + } + chart, ok := index[digest] if !ok { return false } return h.proxy.withholdInCooldown(w, downloadVersion{ - ecosystem: helmMetadataEcosystem, policyName: chartName, version: digest, - lookup: func() (time.Time, bool, error) { return created, !created.IsZero(), nil }, + ecosystem: helmMetadataEcosystem, policyName: chart.name, version: digest, + lookup: func() (time.Time, bool, error) { return chart.created, !chart.created.IsZero(), nil }, }) } -// findChartRelease returns the name and created time of the index release -// with the given digest. -func (h *HelmHandler) findChartRelease(upstreamURL string, body []byte, digest string) (string, time.Time, bool) { +// helmChartTime is the chart a release digest belongs to and when it was +// created. +type helmChartTime struct { + name string + created time.Time +} + +// indexChartTimes maps each release digest in an index to its chart. +func (h *HelmHandler) indexChartTimes(upstreamURL string, body []byte) (map[string]helmChartTime, error) { _, entries, err := parseHelmIndex(body) if err != nil { - return "", time.Time{}, false + return nil, err } + index := make(map[string]helmChartTime) for i := 0; i < len(entries.Content); i += 2 { chartName := entries.Content[i].Value releases := entries.Content[i+1] @@ -422,12 +437,13 @@ func (h *HelmHandler) findChartRelease(upstreamURL string, body []byte, digest s } for _, release := range releases.Content { chart, err := h.parseChartRelease(chartName, upstreamURL, release) - if err == nil && chart.digest == digest { - return chartName, chart.created, true + if err != nil { + continue } + index[chart.digest] = helmChartTime{name: chartName, created: chart.created} } } - return "", time.Time{}, false + return index, nil } func (h *HelmHandler) chartOnCooldown(chartName string, created time.Time) bool { From 5db16cbb991c25714673aebd9fd860ca451da9dc Mon Sep 17 00:00:00 2001 From: Christian Heim Date: Mon, 26 Oct 2026 07:28:44 +0100 Subject: [PATCH 560/633] Conda reads repodata as a stream for the download check, under the same bound as the repodata filter, so a repodata.json over metadata_max_size no longer lets a package in cooldown through. If even the compressed copy is too large to cache, the check streams it from upstream. --- internal/handler/conda.go | 93 ++++++++++++++++------ internal/handler/download_cooldown_test.go | 48 +++++++++++ 2 files changed, 115 insertions(+), 26 deletions(-) diff --git a/internal/handler/conda.go b/internal/handler/conda.go index d7cf196..5d79b3f 100644 --- a/internal/handler/conda.go +++ b/internal/handler/conda.go @@ -543,16 +543,23 @@ func (h *CondaHandler) metadataCacheKey(urlPath string) string { // publishTime reads a package file's timestamp from the subdir's // repodata.json, fetched with the cache key and encoding proxyCached uses. // The repodata is indexed once per document (see publishIndex), since a -// subdir's index runs to tens of megabytes and an install downloads many -// files from it. +// subdir's index runs to hundreds of megabytes and an install downloads many +// files from it. A compressed copy over metadata_max_size can't be cached, +// so it is streamed from upstream for each check, as handleRepodata does. func (h *CondaHandler) publishTime(r *http.Request, channel, arch, filename string) (time.Time, bool, error) { repodataPath := "/" + channel + "/" + arch + "/repodata.json" cacheKey := h.metadataCacheKey(repodataPath) res := h.proxy.lookupMetadata(r.Context(), "conda", cacheKey, h.upstreamURL+repodataPath, "*/*", "gzip", nil) - if res.err != nil { - return time.Time{}, false, res.err + var index map[string]time.Time + var err error + switch { + case errors.Is(res.err, ErrMetadataTooLarge): + index, err = h.streamRepodataIndex(r, h.upstreamURL+repodataPath) + case res.err != nil: + err = res.err + default: + index, err = publishIndex(h.proxy, "conda\x00"+cacheKey, res, h.indexRepodata) } - index, err := publishIndex(h.proxy, "conda\x00"+cacheKey, res, h.indexRepodata) if err != nil { return time.Time{}, false, err } @@ -560,36 +567,70 @@ func (h *CondaHandler) publishTime(r *http.Request, channel, arch, filename stri return published, ok, nil } -// indexRepodata maps each package file in a repodata.json to its timestamp, -// leaving out files without one. +// indexRepodata indexes a cached repodata.json body. func (h *CondaHandler) indexRepodata(body []byte, encoding string) (map[string]time.Time, error) { + var reader io.Reader = bytes.NewReader(body) if strings.EqualFold(strings.TrimSpace(encoding), "gzip") { - decoded, err := gzip.NewReader(bytes.NewReader(body)) - if err != nil { - return nil, err - } - body, err = h.proxy.ReadMetadata(decoded) + decoded, err := gzip.NewReader(reader) if err != nil { return nil, err } + reader = decoded } - type entry struct { - Timestamp float64 `json:"timestamp"` - } - var repodata struct { - Packages map[string]entry `json:"packages"` - PackagesConda map[string]entry `json:"packages.conda"` - } - if err := json.Unmarshal(body, &repodata); err != nil { + return readRepodataIndex(reader) +} + +// streamRepodataIndex indexes repodata.json as it streams from upstream. +func (h *CondaHandler) streamRepodataIndex(r *http.Request, upstreamURL string) (map[string]time.Time, error) { + req, err := http.NewRequestWithContext(r.Context(), http.MethodGet, upstreamURL, nil) + if err != nil { return nil, err } - index := make(map[string]time.Time, len(repodata.Packages)+len(repodata.PackagesConda)) - for _, files := range []map[string]entry{repodata.Packages, repodata.PackagesConda} { - for filename, e := range files { - if e.Timestamp != 0 { - index[filename] = time.Unix(int64(e.Timestamp)/condaTimestampDivisor, 0) - } + req.Header.Set(headerAcceptEncoding, "gzip") + resp, err := h.proxy.HTTPClient.Do(req) + if err != nil { + return nil, err + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("repodata returned %d", resp.StatusCode) + } + var reader io.Reader = resp.Body + if strings.EqualFold(strings.TrimSpace(resp.Header.Get(headerContentEncoding)), "gzip") { + decoded, err := gzip.NewReader(resp.Body) + if err != nil { + return nil, err } + defer func() { _ = decoded.Close() }() + reader = decoded + } + return readRepodataIndex(reader) +} + +// readRepodataIndex maps each package file in a repodata.json to its +// timestamp, leaving out files without one. It reads the document as it +// streams, under the same bound as filterRepodata, so the decompressed size +// is not limited by metadata_max_size. +func readRepodataIndex(r io.Reader) (map[string]time.Time, error) { + index := make(map[string]time.Time) + err := jsonstream.CopyObject(json.NewDecoder(io.LimitReader(r, maxRepodataSize)), io.Discard, func(m *jsonstream.Member) error { + if m.Key != "packages" && m.Key != "packages.conda" { + return m.Copy() + } + return m.Object(func(entry *jsonstream.Member) error { + raw, err := entry.Raw() + if err != nil { + return err + } + var e repodataEntry + if json.Unmarshal(raw, &e) == nil && e.Timestamp != 0 { + index[entry.Key] = time.Unix(int64(e.Timestamp)/condaTimestampDivisor, 0) + } + return nil + }) + }) + if err != nil { + return nil, err } return index, nil } diff --git a/internal/handler/download_cooldown_test.go b/internal/handler/download_cooldown_test.go index c504edb..1dd44e0 100644 --- a/internal/handler/download_cooldown_test.go +++ b/internal/handler/download_cooldown_test.go @@ -417,3 +417,51 @@ func TestPublishIndexBuildsOncePerDocument(t *testing.T) { } } } + +// TestCondaDownloadCooldownPastMetadataLimit checks that a cached package is +// withheld when its repodata is over metadata_max_size, as the repodata +// endpoint hides it: once only decompressed, and once already compressed. +func TestCondaDownloadCooldownPastMetadataLimit(t *testing.T) { + plain := largeRepodata(500) + compressed := gzipPayload(t, plain) + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set(headerContentEncoding, "gzip") + _, _ = w.Write(compressed) + })) + t.Cleanup(upstream.Close) + + for _, tt := range []struct { + name string + limit int64 + }{ + {"decompressed over the limit", int64(len(compressed)) + 1}, + {"compressed over the limit", int64(len(compressed)) - 1}, + } { + t.Run(tt.name, func(t *testing.T) { + proxy, db, store, fetcher := setupTestProxy(t) + proxy.HTTPClient = upstream.Client() + proxy.CacheMetadata = true + proxy.MetadataTTL = time.Hour + proxy.MetadataMaxSize = tt.limit + fetcher.fetchErr = errors.New("artifact must be served from the cache") + for _, file := range []string{"pkg-0-old.conda", "pkg-1-new.conda"} { + seedPackage(t, db, store, "conda", "conda-forge/pkg", strings.Split(file, "-")[1], "linux-64/"+file, "artifact "+file) + } + proxy.Cooldown = &cooldown.Config{Default: "3d"} + h := NewCondaHandlerWithUpstream(proxy, "http://proxy", upstream.URL).Routes() + + w := httptest.NewRecorder() + h.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/conda-forge/linux-64/repodata.json", nil)) + if w.Code != http.StatusOK || strings.Contains(w.Body.String(), "pkg-1-new.conda") { + t.Fatalf("repodata status = %d, want 200 without pkg-1-new.conda", w.Code) + } + for file, want := range map[string]int{"pkg-0-old.conda": http.StatusOK, "pkg-1-new.conda": http.StatusNotFound} { + w := httptest.NewRecorder() + h.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/conda-forge/linux-64/"+file, nil)) + if w.Code != want { + t.Errorf("%s status = %d, want %d (body %q)", file, w.Code, want, w.Body.String()) + } + } + }) + } +} From 3ab7dcc1dc0fdcd821f08cc9eb23d020274a8625 Mon Sep 17 00:00:00 2001 From: Christian Heim Date: Mon, 26 Oct 2026 08:55:25 +0100 Subject: [PATCH 561/633] The UTC test compares the zone offset, since Postgres reads TIMESTAMP back as a fixed +0000 zone rather than time.UTC. --- internal/database/database_test.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/internal/database/database_test.go b/internal/database/database_test.go index 91734e1..500ad0d 100644 --- a/internal/database/database_test.go +++ b/internal/database/database_test.go @@ -1225,7 +1225,9 @@ func TestVersionPublishedAtIsUTC(t *testing.T) { if err != nil || got == nil || !got.PublishedAt.Valid { t.Fatalf("GetVersionByPURL(%s) = %+v, %v", purl, got, err) } - if !got.PublishedAt.Time.Equal(want) || got.PublishedAt.Time.Location() != time.UTC { + // Postgres reads TIMESTAMP back as a fixed +0000 zone rather + // than time.UTC, so compare the offset. + if _, offset := got.PublishedAt.Time.Zone(); !got.PublishedAt.Time.Equal(want) || offset != 0 { t.Errorf("%s PublishedAt read back as %v, want %v", purl, got.PublishedAt.Time, want) } } From b9dcbd27fc065e6e82e8b930b36a243170d9befb Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Thu, 5 Nov 2026 18:41:03 +0000 Subject: [PATCH 562/633] Handle cooldown lookup failures and reuse download indexes --- docs/configuration.md | 4 + internal/handler/cargo.go | 16 ++- internal/handler/conda.go | 39 +++--- internal/handler/cooldown_times.go | 15 +++ internal/handler/download_cooldown.go | 98 ++++++++------- .../handler/download_cooldown_cache_test.go | 26 ++++ .../handler/download_cooldown_errors_test.go | 101 +++++++++++++++ internal/handler/download_cooldown_test.go | 11 +- internal/handler/filename_download.go | 4 +- internal/handler/gem.go | 27 +--- internal/handler/helm.go | 117 ++++++++---------- internal/handler/hex.go | 22 +--- internal/handler/pub.go | 6 +- 13 files changed, 294 insertions(+), 192 deletions(-) create mode 100644 internal/handler/download_cooldown_cache_test.go create mode 100644 internal/handler/download_cooldown_errors_test.go diff --git a/docs/configuration.md b/docs/configuration.md index bf34b23..38765e0 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1201,6 +1201,10 @@ Maven caveats: Helm uses each release's `created` time from the repository's `index.yaml`. Releases in cooldown are left out of the index the proxy serves, and their archives aren't served; a release without `created` is let through. Charts pulled over OCI have no publish time and aren't held back. +Cargo, RubyGems, Hex, pub.dev, Conda and Helm check artifact downloads against cooldown, including files cached before the policy changed. When a publish-time lookup fails, an already-cached file is still served; an uncached download fails with 502, or 503 with `Retry-After` when the rate-limit gate holds the request. Conda checks timestamps per file because builds of one version can have different publish times. + +Conda repodata and Helm indexes are parsed once per cached document for download checks. With `cache_metadata` off, the parsed index is kept for one minute, including the fetch, so an install can reuse it across files. Conda streams repodata when its compressed copy exceeds `metadata_max_size`, under the same decompressed size bound as the metadata filter. + RubyGems, Hex and PyPI keep their publish times outside the metadata clients read: the RubyGems versions API, the Hex HTTP API (`upstream.hex_api`) and the PyPI JSON API. The proxy fetches them alongside each compact index, package or simple page it filters, and for PyPI downloads. Cooldown fails closed: when it applies to a package and those times can't be fetched or parsed, or the metadata can't be rewritten, the request fails with 502, or with 503 and `Retry-After` while the upstream is rate limiting the proxy, instead of serving the metadata unfiltered. The same goes for npm, Composer and pub.dev metadata that can't be rewritten, and for npm tarball downloads whose publish time can't be read. A PyPI file or npm tarball that is already cached is still served when its publish time can't be read; only a download that would have to be fetched fails. A PyPI release's time is the upload time of its newest file. The proxy stores it when a download looks it up, and serves files of that release that are already cached against the stored time without reading the JSON API again. A file that isn't cached yet is always looked up, since it may have been added to the release since, and the newer time is stored. Packages exempted with a duration of `0` are served as before and don't need the times. A PyPI package the JSON API returns 404 for has no times to go by and is served unfiltered. For RubyGems and Hex, both documents go through the metadata cache. With [`cache_metadata`](#metadata-caching) they are served from it within `metadata_ttl` without asking the upstream, revalidated with their ETags after it, and filtered from the last cached copies, marked stale, while the upstream is down; without it they are still fetched once for concurrent requests. Each document has its own TTL, so the compact index or Hex package can be newer than the cached times and list a version they don't have. The times are then fetched again before filtering, and if that fails, such versions are left out until the times can be read. The filtered index or package carries an ETag of its own, so Bundler and Mix revalidate it with `304 Not Modified`. Publish times are compared with the current time on each request, and the filtered result is kept until the first version it held back leaves cooldown (see [rewritten metadata cache](#rewritten-metadata-cache)); a newly published version shows up within `metadata_ttl`, as with other cached metadata. Upstream errors are answered as for other cached metadata (404, or 502) rather than relayed. diff --git a/internal/handler/cargo.go b/internal/handler/cargo.go index 43bcbc4..fa93a74 100644 --- a/internal/handler/cargo.go +++ b/internal/handler/cargo.go @@ -197,11 +197,17 @@ func (h *CargoHandler) publishTime(r *http.Request, name, version string) (time. return time.Time{}, false, err } for line := range bytes.SplitSeq(body, []byte("\n")) { - if vers, _, _ := lookupJSONString(line, "vers"); vers != version { + if len(bytes.TrimSpace(line)) == 0 { continue } - pubtime, _, _ := lookupJSONString(line, "pubtime") - published, err := time.Parse(time.RFC3339, pubtime) + var entry crateIndexEntry + if err := json.Unmarshal(line, &entry); err != nil { + return time.Time{}, false, err + } + if entry.Version != version { + continue + } + published, err := time.Parse(time.RFC3339, entry.PublishTime) return published, err == nil, nil } return time.Time{}, false, nil @@ -238,8 +244,8 @@ func (h *CargoHandler) handleDownload(w http.ResponseWriter, r *http.Request) { h.proxy.Logger.Debug("cargo download request", "crate", name, "version", version, "filename", filename) - if h.proxy.withholdInCooldown(w, downloadVersion{ - ecosystem: "cargo", policyName: name, storedName: name, version: version, + if h.proxy.withholdInCooldown(w, r, downloadVersion{ + ecosystem: "cargo", policyName: name, storedName: name, version: version, cacheName: name, filename: filename, lookup: func() (time.Time, bool, error) { return h.publishTime(r, name, version) }, }) { return diff --git a/internal/handler/conda.go b/internal/handler/conda.go index 5d79b3f..de6ae8a 100644 --- a/internal/handler/conda.go +++ b/internal/handler/conda.go @@ -141,18 +141,16 @@ func (h *CondaHandler) handleDownload(w http.ResponseWriter, r *http.Request) { // Builds and subdirs of one version each have their own timestamp, so // the check is per file and keeps no time in the shared versions row. - if h.proxy.withholdInCooldown(w, downloadVersion{ - ecosystem: "conda", policyName: name, version: version, + // Subdirs distinguish cached files with the same filename. + cacheFilename := arch + "/" + filename + if h.proxy.withholdInCooldown(w, r, downloadVersion{ + ecosystem: "conda", policyName: name, version: version, cacheName: packageName, filename: cacheFilename, lookup: func() (time.Time, bool, error) { return h.publishTime(r, channel, arch, filename) }, }) { return } upstreamURL := h.upstreamURL + r.URL.Path - // The same filename can hold different bytes in each subdir, so the - // subdir is part of the cache key. - cacheFilename := arch + "/" + filename - result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "conda", packageName, version, cacheFilename, upstreamURL) if err != nil { h.proxy.serveArtifactError(w, err, "failed to fetch package") @@ -540,26 +538,21 @@ func (h *CondaHandler) metadataCacheKey(urlPath string) string { return hashedCacheKey(h.upstreamURL, urlPath) } -// publishTime reads a package file's timestamp from the subdir's -// repodata.json, fetched with the cache key and encoding proxyCached uses. -// The repodata is indexed once per document (see publishIndex), since a -// subdir's index runs to hundreds of megabytes and an install downloads many -// files from it. A compressed copy over metadata_max_size can't be cached, -// so it is streamed from upstream for each check, as handleRepodata does. +// publishTime streams compressed repodata when it exceeds metadata_max_size. func (h *CondaHandler) publishTime(r *http.Request, channel, arch, filename string) (time.Time, bool, error) { repodataPath := "/" + channel + "/" + arch + "/repodata.json" cacheKey := h.metadataCacheKey(repodataPath) - res := h.proxy.lookupMetadata(r.Context(), "conda", cacheKey, h.upstreamURL+repodataPath, "*/*", "gzip", nil) - var index map[string]time.Time - var err error - switch { - case errors.Is(res.err, ErrMetadataTooLarge): - index, err = h.streamRepodataIndex(r, h.upstreamURL+repodataPath) - case res.err != nil: - err = res.err - default: - index, err = publishIndex(h.proxy, "conda\x00"+cacheKey, res, h.indexRepodata) - } + index, err := publishIndex(h.proxy, "conda\x00"+cacheKey, func() metadataResult { + return h.proxy.lookupMetadata(r.Context(), "conda", cacheKey, h.upstreamURL+repodataPath, "*/*", "gzip", nil) + }, func(res metadataResult) (map[string]time.Time, error) { + if errors.Is(res.err, ErrMetadataTooLarge) { + return h.streamRepodataIndex(r, h.upstreamURL+repodataPath) + } + if res.err != nil { + return nil, res.err + } + return h.indexRepodata(res.body, res.contentEncoding) + }) if err != nil { return time.Time{}, false, err } diff --git a/internal/handler/cooldown_times.go b/internal/handler/cooldown_times.go index 5d95d16..b7a0516 100644 --- a/internal/handler/cooldown_times.go +++ b/internal/handler/cooldown_times.go @@ -16,6 +16,21 @@ import ( // through. type publishTimes map[string]time.Time +func (p *Proxy) lookupPublishTime(ctx context.Context, t timedMetadata, version string) (time.Time, bool, error) { + var times publishTimes + validate := func(body []byte) error { + var err error + times, err = t.parseTimes(body) + return err + } + res := p.lookupMetadata(ctx, t.ecosystem, t.timesKey, t.timesURL, t.timesAccept, "", validate) + if res.err != nil { + return time.Time{}, false, res.err + } + published, ok := times[version] + return published, ok && !published.IsZero(), nil +} + // timedMetadata describes metadata whose publish times live in a second // document, as with the RubyGems compact index and the versions API, or a // Hex package and the Hex HTTP API. Both are read through the metadata cache. diff --git a/internal/handler/download_cooldown.go b/internal/handler/download_cooldown.go index 5aec959..47d8b70 100644 --- a/internal/handler/download_cooldown.go +++ b/internal/handler/download_cooldown.go @@ -24,21 +24,22 @@ type downloadVersion struct { // keeps no time, and every check asks lookup. storedName string version string + cacheName string + filename string lookup publishTimeLookup } // withholdInCooldown reports whether a download is inside the cooldown -// window, and if so answers it with a 404. Filtering metadata is not enough on -// its own: lockfiles record artifact URLs, so clients reach the download path -// without reading the metadata first, and a cached artifact would be served -// even after the policy was raised. -// -// A publish time never changes, so the stored versions row is read first and -// lookup is only asked for a version the proxy has no time for yet; the time -// it finds is stored. A version with no usable publish time is allowed -// through, matching how the metadata filters treat it. -func (p *Proxy) withholdInCooldown(w http.ResponseWriter, d downloadVersion) bool { - if !p.versionInCooldown(d) { +// window, or a lookup failure prevents an uncached download. +func (p *Proxy) withholdInCooldown(w http.ResponseWriter, r *http.Request, d downloadVersion) bool { + withheld, err := p.versionInCooldown(d) + if err != nil { + if !p.serveCachedDespiteCooldownError(w, r, d.ecosystem, d.cacheName, d.version, d.filename, err) { + p.writeCooldownLookupError(w, d.ecosystem, d.policyName, err) + } + return true + } + if !withheld { return false } p.Logger.Info("cooldown: withholding "+d.ecosystem+" download", @@ -47,31 +48,26 @@ func (p *Proxy) withholdInCooldown(w http.ResponseWriter, d downloadVersion) boo return true } -func (p *Proxy) versionInCooldown(d downloadVersion) bool { - if p.Cooldown == nil || !p.Cooldown.Enabled() { - return false - } +func (p *Proxy) versionInCooldown(d downloadVersion) (bool, error) { policyKey := canonicalPackagePURL(d.ecosystem, d.policyName) - if p.Cooldown.For(d.ecosystem, policyKey) <= 0 { - return false + if !p.cooldownApplies(d.ecosystem, d.policyName) { + return false, nil } var versionPURL string if d.storedName != "" { versionPURL = canonicalVersionPURL(d.ecosystem, d.storedName, d.version) if ver, err := p.DB.GetVersionByPURL(versionPURL); err == nil && ver != nil && ver.PublishedAt.Valid { - return !p.Cooldown.IsAllowed(d.ecosystem, policyKey, ver.PublishedAt.Time) + return !p.cooldown().IsAllowed(d.ecosystem, policyKey, ver.PublishedAt.Time), nil } } published, ok, err := d.lookup() if err != nil { - p.Logger.Warn("cooldown: could not read publish time for download check", - "ecosystem", d.ecosystem, "name", d.policyName, "version", d.version, "error", err) - return false + return false, err } if !ok { - return false + return false, nil } if versionPURL != "" { @@ -81,7 +77,7 @@ func (p *Proxy) versionInCooldown(d downloadVersion) bool { } } - return !p.Cooldown.IsAllowed(d.ecosystem, policyKey, published) + return !p.cooldown().IsAllowed(d.ecosystem, policyKey, published), nil } // maxPublishIndexes bounds how many documents publishIndexCache keeps an @@ -99,27 +95,40 @@ type publishIndexCache struct { } type publishIndexEntry struct { - digest string - index any + digest string + index any + expires time.Time } -// publishIndex returns build's index of res, reusing the one built for the -// same document under key. res.digest identifies the document; without one, -// as when the metadata cache is off, the index is built for this call alone. -func publishIndex[V any](p *Proxy, key string, res metadataResult, build func([]byte, string) (map[string]V, error)) (map[string]V, error) { +// publishIndex reuses parsed documents by digest, or for a minute without one. +func publishIndex[V any](p *Proxy, key string, load func() metadataResult, build func(metadataResult) (V, error)) (V, error) { //nolint:ireturn // Supports timestamp maps and Helm lookups. c := &p.publishIndexes - if res.digest == "" { - return build(res.body, res.contentEncoding) + fresh := func() (V, bool) { + c.mu.Lock() + defer c.mu.Unlock() + if e, ok := c.entries[key]; ok && !e.expires.IsZero() && p.clock().Before(e.expires) { + return e.index.(V), true + } + var zero V + return zero, false } - c.mu.Lock() - if e, ok := c.entries[key]; ok && e.digest == res.digest { - c.mu.Unlock() - return e.index.(map[string]V), nil + if index, ok := fresh(); ok { + return index, nil } - c.mu.Unlock() - - v, err, _ := c.group.Do(key+"\x00"+res.digest, func() (any, error) { - index, err := build(res.body, res.contentEncoding) + v, err, _ := c.group.Do(key, func() (any, error) { + if index, ok := fresh(); ok { + return index, nil + } + res := load() + if res.err == nil && res.digest != "" { + c.mu.Lock() + entry, ok := c.entries[key] + c.mu.Unlock() + if ok && entry.digest == res.digest { + return entry.index, nil + } + } + index, err := build(res) if err != nil { return nil, err } @@ -134,11 +143,16 @@ func publishIndex[V any](p *Proxy, key string, res metadataResult, build func([] break } } - c.entries[key] = publishIndexEntry{digest: res.digest, index: index} + entry := publishIndexEntry{digest: res.digest, index: index} + if res.digest == "" || !p.CacheMetadata { + entry.expires = p.clock().Add(time.Minute) + } + c.entries[key] = entry return index, nil }) if err != nil { - return nil, err + var zero V + return zero, err } - return v.(map[string]V), nil + return v.(V), nil } diff --git a/internal/handler/download_cooldown_cache_test.go b/internal/handler/download_cooldown_cache_test.go new file mode 100644 index 0000000..be488de --- /dev/null +++ b/internal/handler/download_cooldown_cache_test.go @@ -0,0 +1,26 @@ +package handler + +import ( + "net/http" + "testing" +) + +func TestGemDownloadSharesPublishTimeMetadata(t *testing.T) { + u, proxy, h := gemCacheSetup(t) + wantStatus(t, serveGet(t, h, gemInfoPath), http.StatusOK, "") + seedPackage(t, proxy.DB, proxy.Storage.(*mockStorage), "gem", "testgem", "1.0.0", "testgem-1.0.0.gem", "gem bytes") + wantStatus(t, serveGet(t, h, "/gems/testgem-1.0.0.gem"), http.StatusOK, "") + if n, _ := u.counts(gemVersionsPath); n != 1 { + t.Fatalf("versions API fetched %d times, want 1", n) + } +} + +func TestHexDownloadSharesPublishTimeMetadata(t *testing.T) { + u, proxy, h, _ := hexCacheSetup(t, 0) + wantStatus(t, serveGet(t, h, hexPackagePath), http.StatusOK, "") + seedPackage(t, proxy.DB, proxy.Storage.(*mockStorage), "hex", "testpkg", "1.0.0", "testpkg-1.0.0.tar", "hex bytes") + wantStatus(t, serveGet(t, h, "/tarballs/testpkg-1.0.0.tar"), http.StatusOK, "") + if n, _ := u.counts(hexAPIPath); n != 1 { + t.Fatalf("Hex API fetched %d times, want 1", n) + } +} diff --git a/internal/handler/download_cooldown_errors_test.go b/internal/handler/download_cooldown_errors_test.go new file mode 100644 index 0000000..b214db2 --- /dev/null +++ b/internal/handler/download_cooldown_errors_test.go @@ -0,0 +1,101 @@ +package handler + +import ( + "fmt" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + "time" + + "github.com/git-pkgs/cooldown" + registryclient "github.com/git-pkgs/registries/client" +) + +func TestDownloadCooldownLookupFailures(t *testing.T) { + for _, c := range downloadCooldownCases() { + for _, failure := range []string{"upstream error", "invalid metadata", "rate limited"} { + for _, cached := range []bool{false, true} { + t.Run(fmt.Sprintf("%s/%s/cached=%t", c.ecosystem, failure, cached), func(t *testing.T) { + assertDownloadLookupFailure(t, c, failure, cached) + }) + } + } + } +} + +func assertDownloadLookupFailure(t *testing.T, c downloadCooldownCase, failure string, cached bool) { + t.Helper() + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + if failure == "upstream error" { + w.WriteHeader(http.StatusInternalServerError) + } + _, _ = w.Write([]byte("invalid metadata")) + })) + t.Cleanup(upstream.Close) + proxy, _, _, fetcher := setupTestProxy(t) + proxy.Cooldown = &cooldown.Config{Default: "7d"} + proxy.HTTPClient = upstream.Client() + if failure == "rate limited" { + proxy.HTTPClient = &http.Client{Transport: roundTripperFunc(func(_ *http.Request) (*http.Response, error) { + return nil, ®istryclient.HeldError{Host: "upstream.example", Until: time.Now().Add(time.Minute), RetryAfter: time.Minute} + })} + } + if cached { + c.seedCachedArtifact(t, proxy, cooldownRecentVersion) + } + w := serveGet(t, c.handler(proxy, upstream.URL), c.path(cooldownRecentVersion)) + want := http.StatusBadGateway + if failure == "rate limited" { + want = http.StatusServiceUnavailable + } + if cached { + want = http.StatusOK + if w.Body.Len() == 0 { + t.Fatal("cached artifact body is empty") + } + } + wantStatus(t, w, want, "") + if fetcher.fetchCalled { + t.Fatal("lookup failure fetched an unchecked artifact") + } +} + +func TestDownloadPublishIndexesHaveShortTTLWithoutMetadataCache(t *testing.T) { + for _, c := range downloadCooldownCases() { + if c.ecosystem != "conda" && c.ecosystem != "helm" { + continue + } + t.Run(c.ecosystem, func(t *testing.T) { + var calls atomic.Int64 + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls.Add(1) + c.metadata(w, r, map[string]time.Time{ + cooldownOldVersion: time.Now().Add(-30 * 24 * time.Hour), + cooldownRecentVersion: time.Now().Add(-time.Hour), + }) + })) + t.Cleanup(upstream.Close) + proxy, _, _, _ := setupTestProxy(t) + proxy.HTTPClient = upstream.Client() + proxy.Cooldown = &cooldown.Config{Default: "7d"} + clock := &testClock{t: time.Now()} + proxy.now = clock.now + c.seedCachedArtifact(t, proxy, cooldownOldVersion) + c.seedCachedArtifact(t, proxy, cooldownRecentVersion) + h := c.handler(proxy, upstream.URL) + for range 3 { + wantStatus(t, serveGet(t, h, c.path(cooldownOldVersion)), http.StatusOK, "") + wantStatus(t, serveGet(t, h, c.path(cooldownRecentVersion)), http.StatusNotFound, "") + } + if n := calls.Load(); n != 1 { + t.Fatalf("index downloads within TTL = %d, want 1", n) + } + clock.advance(time.Minute + time.Second) + wantStatus(t, serveGet(t, h, c.path(cooldownRecentVersion)), http.StatusNotFound, "") + if n := calls.Load(); n != 2 { + t.Errorf("index downloads after expiry = %d, want 2", n) + } + }) + } +} diff --git a/internal/handler/download_cooldown_test.go b/internal/handler/download_cooldown_test.go index 1dd44e0..a3c89ae 100644 --- a/internal/handler/download_cooldown_test.go +++ b/internal/handler/download_cooldown_test.go @@ -391,13 +391,14 @@ func TestCondaDownloadCooldownPerBuild(t *testing.T) { } // TestPublishIndexBuildsOncePerDocument checks that an index is reused for -// the same document, rebuilt for a new one, and not kept without a digest. +// the same document, rebuilt for a new one, and kept briefly without a digest. func TestPublishIndexBuildsOncePerDocument(t *testing.T) { proxy, _, _, _ := setupTestProxy(t) + proxy.CacheMetadata = true builds := 0 - build := func(body []byte, _ string) (map[string]bool, error) { + build := func(res metadataResult) (map[string]bool, error) { builds++ - return map[string]bool{string(body): true}, nil + return map[string]bool{string(res.body): true}, nil } steps := []struct { res metadataResult @@ -408,10 +409,10 @@ func TestPublishIndexBuildsOncePerDocument(t *testing.T) { {metadataResult{body: []byte("ignored"), digest: "d1"}, "a", 1}, {metadataResult{body: []byte("b"), digest: "d2"}, "b", 2}, {metadataResult{body: []byte("c")}, "c", 3}, - {metadataResult{body: []byte("c")}, "c", 4}, + {metadataResult{body: []byte("c")}, "c", 3}, } for i, step := range steps { - got, err := publishIndex(proxy, "key", step.res, build) + got, err := publishIndex(proxy, "key", func() metadataResult { return step.res }, build) if err != nil || !got[step.want] || builds != step.wantBuilds { t.Fatalf("step %d: publishIndex = %v, %v after %d builds, want %q after %d", i, got, err, builds, step.want, step.wantBuilds) } diff --git a/internal/handler/filename_download.go b/internal/handler/filename_download.go index dbc0e64..373c8f4 100644 --- a/internal/handler/filename_download.go +++ b/internal/handler/filename_download.go @@ -34,8 +34,8 @@ func (p *Proxy) handleFilenameDownload(w http.ResponseWriter, r *http.Request, d p.Logger.Debug(d.ecosystem+" download request", "name", name, "version", version, "filename", filename) - if p.withholdInCooldown(w, downloadVersion{ - ecosystem: d.ecosystem, policyName: name, storedName: name, version: version, + if p.withholdInCooldown(w, r, downloadVersion{ + ecosystem: d.ecosystem, policyName: name, storedName: name, version: version, cacheName: name, filename: filename, lookup: func() (time.Time, bool, error) { return d.publishTime(r, name, version) }, }) { return diff --git a/internal/handler/gem.go b/internal/handler/gem.go index 009e0f5..e9ee55a 100644 --- a/internal/handler/gem.go +++ b/internal/handler/gem.go @@ -5,7 +5,6 @@ import ( "encoding/json" "fmt" "net/http" - "net/url" "regexp" "strings" "time" @@ -306,32 +305,8 @@ func parseGemVersions(body []byte) (publishTimes, error) { return times, nil } -// publishTime reads a gem version's created_at from the versions API. The -// download check caches the response, unlike fetchFilteredVersions, so each -// download does not cost an upstream request. version carries the platform -// the way gem filenames do, as in 1.0.0-x86_64-linux. func (h *GemHandler) publishTime(r *http.Request, name, version string) (time.Time, bool, error) { - versionsURL := fmt.Sprintf("%s/api/v1/versions/%s.json", h.upstreamURL, url.PathEscape(name)) - body, _, err := h.proxy.FetchOrCacheMetadata(r.Context(), "gem", "api/v1/versions/"+name, versionsURL) - if err != nil { - return time.Time{}, false, err - } - var versions []gemVersion - if err := json.Unmarshal(body, &versions); err != nil { - return time.Time{}, false, err - } - for _, v := range versions { - number := v.Number - if v.Platform != "" && v.Platform != "ruby" { - number += "-" + v.Platform - } - if number != version { - continue - } - createdAt, err := time.Parse(time.RFC3339, v.CreatedAt) - return createdAt, err == nil, nil - } - return time.Time{}, false, nil + return h.proxy.lookupPublishTime(r.Context(), h.timedIndex(name), version) } // proxyCached forwards a metadata request with caching. diff --git a/internal/handler/helm.go b/internal/handler/helm.go index cbbf8b0..661ca98 100644 --- a/internal/handler/helm.go +++ b/internal/handler/helm.go @@ -120,8 +120,22 @@ func (h *HelmHandler) handleChart(w http.ResponseWriter, r *http.Request) { return } - if h.withholdChart(w, r, repository, upstreamURL, digest) { - return + var charts *helmChartLookup + if h.proxy.cooldownEnabled() { + var err error + charts, err = h.downloadChartLookup(r, repository, upstreamURL) + if err != nil { + cached, cacheErr := h.proxy.GetCachedArtifact(r.Context(), helmMetadataEcosystem, repository, digest, filename) + if cacheErr == nil && cached != nil { + h.serveChart(w, r, repository, digest, filename, cached) + } else { + h.proxy.writeCooldownLookupError(w, helmMetadataEcosystem, repository, err) + } + return + } + if h.withholdChart(w, charts, digest, filename) { + return + } } cached, err := h.proxy.GetCachedArtifact(r.Context(), helmMetadataEcosystem, repository, digest, filename) @@ -135,13 +149,17 @@ func (h *HelmHandler) handleChart(w http.ResponseWriter, r *http.Request) { return } - index := h.fetchIndex(r.Context(), repository, upstreamURL) - if index.err != nil { - h.serveIndexError(w, index.err) - return + var downloadURL string + if charts == nil { + index := h.fetchIndex(r.Context(), repository, upstreamURL) + if index.err != nil { + h.serveIndexError(w, index.err) + return + } + downloadURL, err = h.findChartDownload(repository, upstreamURL, index, digest, filename) + } else { + downloadURL, err = h.findChartURL(charts, digest, filename) } - - downloadURL, err := h.findChartDownload(repository, upstreamURL, index, digest, filename) if err != nil { if errors.Is(err, errHelmChartNotFound) { http.NotFound(w, r) @@ -269,6 +287,10 @@ func (h *HelmHandler) findChartDownload(repository, upstreamURL string, index me if err != nil { return "", err } + return h.findChartURL(charts, digest, filename) +} + +func (h *HelmHandler) findChartURL(charts *helmChartLookup, digest, filename string) (string, error) { for _, chart := range charts.byDigest[digest] { if h.chartOnCooldown(chart.name, chart.created) { continue @@ -380,70 +402,35 @@ func (h *HelmHandler) buildChartLookup(upstreamURL string, body []byte) (*helmCh return lookup, nil } -// withholdChart checks a chart against the cooldown before it is served. -// findChartDownload already leaves out charts in cooldown, but a cached chart -// may have been fetched under an earlier policy. Charts are cached under the -// repository and digest rather than the chart name the policy is keyed by, so -// the name and publish time come from the index, indexed once per document -// (see publishIndex). -func (h *HelmHandler) withholdChart(w http.ResponseWriter, r *http.Request, repository, upstreamURL, digest string) bool { - if h.proxy.Cooldown == nil || !h.proxy.Cooldown.Enabled() { - return false - } - res := h.fetchIndex(r, repository, upstreamURL) - if res.err != nil { - h.proxy.Logger.Warn("cooldown: could not fetch Helm index for download check", - "repository", repository, "digest", digest, "error", res.err) - return false - } - index, err := publishIndex(h.proxy, "helm\x00"+h.indexCacheKey(repository, upstreamURL), res, - func(body []byte, _ string) (map[string]helmChartTime, error) { - return h.indexChartTimes(upstreamURL, body) - }) - if err != nil { - h.proxy.Logger.Warn("cooldown: could not read Helm index for download check", - "repository", repository, "digest", digest, "error", err) - return false - } - chart, ok := index[digest] - if !ok { - return false - } - return h.proxy.withholdInCooldown(w, downloadVersion{ - ecosystem: helmMetadataEcosystem, policyName: chart.name, version: digest, - lookup: func() (time.Time, bool, error) { return chart.created, !chart.created.IsZero(), nil }, +func (h *HelmHandler) downloadChartLookup(r *http.Request, repository, upstreamURL string) (*helmChartLookup, error) { + key := "helm\x00" + h.indexCacheKey(repository, upstreamURL) + return publishIndex(h.proxy, key, func() metadataResult { + return h.fetchIndex(r.Context(), repository, upstreamURL) + }, func(res metadataResult) (*helmChartLookup, error) { + if res.err != nil { + return nil, res.err + } + return h.chartLookup(repository, upstreamURL, res) }) } -// helmChartTime is the chart a release digest belongs to and when it was -// created. -type helmChartTime struct { - name string - created time.Time -} - -// indexChartTimes maps each release digest in an index to its chart. -func (h *HelmHandler) indexChartTimes(upstreamURL string, body []byte) (map[string]helmChartTime, error) { - _, entries, err := parseHelmIndex(body) - if err != nil { - return nil, err - } - index := make(map[string]helmChartTime) - for i := 0; i < len(entries.Content); i += 2 { - chartName := entries.Content[i].Value - releases := entries.Content[i+1] - if releases.Kind != yaml.SequenceNode { - continue - } - for _, release := range releases.Content { - chart, err := h.parseChartRelease(chartName, upstreamURL, release) - if err != nil { +func (h *HelmHandler) withholdChart(w http.ResponseWriter, charts *helmChartLookup, digest, filename string) bool { + withheld := false + for _, chart := range charts.byDigest[digest] { + for _, download := range chart.downloads { + if download.filename != filename { continue } - index[chart.digest] = helmChartTime{name: chartName, created: chart.created} + if !h.chartOnCooldown(chart.name, chart.created) { + return false + } + withheld = true } } - return index, nil + if withheld { + http.Error(w, ErrVersionNotFound.Error(), http.StatusNotFound) + } + return withheld } func (h *HelmHandler) chartOnCooldown(chartName string, created time.Time) bool { diff --git a/internal/handler/hex.go b/internal/handler/hex.go index 6836dad..ceaf587 100644 --- a/internal/handler/hex.go +++ b/internal/handler/hex.go @@ -6,7 +6,6 @@ import ( "encoding/json" "fmt" "net/http" - "net/url" "strings" "time" @@ -243,27 +242,8 @@ func parseHexReleases(body []byte) (publishTimes, error) { return times, nil } -// publishTime reads a release's inserted_at from the Hex HTTP API. The -// download check caches the response, unlike fetchFilteredVersions, so each -// download does not cost an upstream request. func (h *HexHandler) publishTime(r *http.Request, name, version string) (time.Time, bool, error) { - apiURL := fmt.Sprintf("%s/api/packages/%s", h.apiURL, url.PathEscape(name)) - body, _, err := h.proxy.FetchOrCacheMetadata(r.Context(), "hex", "api/packages/"+name, apiURL) - if err != nil { - return time.Time{}, false, err - } - var pkg hexPackageAPI - if err := json.Unmarshal(body, &pkg); err != nil { - return time.Time{}, false, err - } - for _, release := range pkg.Releases { - if release.Version != version { - continue - } - insertedAt, err := time.Parse(time.RFC3339Nano, release.InsertedAt) - return insertedAt, err == nil, nil - } - return time.Time{}, false, nil + return h.proxy.lookupPublishTime(r.Context(), h.timedPackage(name), version) } // signedPayload decompresses a gzipped Signed message, with the metadata diff --git a/internal/handler/pub.go b/internal/handler/pub.go index 22540f1..001b0c6 100644 --- a/internal/handler/pub.go +++ b/internal/handler/pub.go @@ -94,8 +94,9 @@ func (h *PubHandler) handleDownload(w http.ResponseWriter, r *http.Request) { h.proxy.Logger.Debug("pub download request", "name", name, "version", version) - if h.proxy.withholdInCooldown(w, downloadVersion{ - ecosystem: "pub", policyName: name, storedName: name, version: version, + h.discardVersionPage(name, version, filename) + if h.proxy.withholdInCooldown(w, r, downloadVersion{ + ecosystem: "pub", policyName: name, storedName: name, version: version, cacheName: name, filename: filename, lookup: func() (time.Time, bool, error) { return h.publishTime(r, name, version) }, }) { return @@ -103,7 +104,6 @@ func (h *PubHandler) handleDownload(w http.ResponseWriter, r *http.Request) { downloadURL := fmt.Sprintf("%s/packages/%s/versions/%s%s", h.upstreamURL, url.PathEscape(name), url.PathEscape(version), pubArchiveExt) - h.discardVersionPage(name, version, filename) result, err := h.proxy.GetOrFetchArtifactFromURL( r.Context(), "pub", name, version, filename, downloadURL, ) From 61ac273862069436dac9412c366aa6dff9cec895 Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Thu, 5 Nov 2026 20:00:19 +0000 Subject: [PATCH 563/633] Check primary metadata before allowing untimed downloads --- internal/handler/cargo.go | 5 +- internal/handler/conda.go | 5 +- internal/handler/cooldown_times.go | 28 +++--- internal/handler/download_cooldown.go | 9 +- .../handler/download_cooldown_cache_test.go | 89 +++++++++++++++++++ internal/handler/download_cooldown_test.go | 15 ++++ internal/handler/filename_download.go | 4 +- internal/handler/gem.go | 4 +- internal/handler/hex.go | 4 +- internal/handler/pub.go | 5 +- 10 files changed, 144 insertions(+), 24 deletions(-) diff --git a/internal/handler/cargo.go b/internal/handler/cargo.go index fa93a74..aca7416 100644 --- a/internal/handler/cargo.go +++ b/internal/handler/cargo.go @@ -246,7 +246,10 @@ func (h *CargoHandler) handleDownload(w http.ResponseWriter, r *http.Request) { if h.proxy.withholdInCooldown(w, r, downloadVersion{ ecosystem: "cargo", policyName: name, storedName: name, version: version, cacheName: name, filename: filename, - lookup: func() (time.Time, bool, error) { return h.publishTime(r, name, version) }, + lookup: func() (time.Time, bool, bool, error) { + published, ok, err := h.publishTime(r, name, version) + return published, ok, false, err + }, }) { return } diff --git a/internal/handler/conda.go b/internal/handler/conda.go index de6ae8a..1c14980 100644 --- a/internal/handler/conda.go +++ b/internal/handler/conda.go @@ -145,7 +145,10 @@ func (h *CondaHandler) handleDownload(w http.ResponseWriter, r *http.Request) { cacheFilename := arch + "/" + filename if h.proxy.withholdInCooldown(w, r, downloadVersion{ ecosystem: "conda", policyName: name, version: version, cacheName: packageName, filename: cacheFilename, - lookup: func() (time.Time, bool, error) { return h.publishTime(r, channel, arch, filename) }, + lookup: func() (time.Time, bool, bool, error) { + published, ok, err := h.publishTime(r, channel, arch, filename) + return published, ok, false, err + }, }) { return } diff --git a/internal/handler/cooldown_times.go b/internal/handler/cooldown_times.go index b7a0516..b765f9a 100644 --- a/internal/handler/cooldown_times.go +++ b/internal/handler/cooldown_times.go @@ -7,6 +7,7 @@ import ( "crypto/sha256" "encoding/hex" "fmt" + "slices" "strings" "time" ) @@ -16,19 +17,24 @@ import ( // through. type publishTimes map[string]time.Time -func (p *Proxy) lookupPublishTime(ctx context.Context, t timedMetadata, version string) (time.Time, bool, error) { - var times publishTimes - validate := func(body []byte) error { +func (p *Proxy) lookupPublishTime(ctx context.Context, t timedMetadata, version string, listed func(metadataResult) ([]string, error)) (time.Time, bool, bool, error) { + var versions []string + docs, err := p.fetchTimedMetadata(ctx, t, func(doc metadataResult) ([]string, error) { var err error - times, err = t.parseTimes(body) - return err + versions, err = listed(doc) + return versions, err + }) + switch { + case docs.doc.err != nil: + return time.Time{}, false, false, docs.doc.err + case docs.times.err != nil: + return time.Time{}, false, false, docs.times.err + case err != nil: + return time.Time{}, false, false, err } - res := p.lookupMetadata(ctx, t.ecosystem, t.timesKey, t.timesURL, t.timesAccept, "", validate) - if res.err != nil { - return time.Time{}, false, res.err - } - published, ok := times[version] - return published, ok && !published.IsZero(), nil + published, ok := docs.published[version] + withholdUnknown := !ok && docs.withholdUnknown && slices.Contains(versions, version) + return published, ok && !published.IsZero(), withholdUnknown, nil } // timedMetadata describes metadata whose publish times live in a second diff --git a/internal/handler/download_cooldown.go b/internal/handler/download_cooldown.go index 47d8b70..c491ef3 100644 --- a/internal/handler/download_cooldown.go +++ b/internal/handler/download_cooldown.go @@ -9,8 +9,9 @@ import ( ) // publishTimeLookup reads a version's publish time from upstream metadata. ok -// is false when the metadata has no usable time for the version. -type publishTimeLookup func() (published time.Time, ok bool, err error) +// is false when the metadata has no usable time for the version. withholdUnknown +// marks a listed version missing from stale publish-time metadata. +type publishTimeLookup func() (published time.Time, ok, withholdUnknown bool, err error) // downloadVersion names the version an artifact download is for, so its // cooldown can be checked before the artifact is served. @@ -62,12 +63,12 @@ func (p *Proxy) versionInCooldown(d downloadVersion) (bool, error) { } } - published, ok, err := d.lookup() + published, ok, withholdUnknown, err := d.lookup() if err != nil { return false, err } if !ok { - return false, nil + return withholdUnknown, nil } if versionPURL != "" { diff --git a/internal/handler/download_cooldown_cache_test.go b/internal/handler/download_cooldown_cache_test.go index be488de..d51b862 100644 --- a/internal/handler/download_cooldown_cache_test.go +++ b/internal/handler/download_cooldown_cache_test.go @@ -1,8 +1,12 @@ package handler import ( + "errors" + "fmt" "net/http" + "net/http/httptest" "testing" + "time" ) func TestGemDownloadSharesPublishTimeMetadata(t *testing.T) { @@ -15,6 +19,91 @@ func TestGemDownloadSharesPublishTimeMetadata(t *testing.T) { } } +type downloadPublishTimeCase struct { + name string + down bool + absent bool + untimed bool + refresh bool + wantStatus int +} + +func TestDownloadCooldownRefreshesMissingPublishTimes(t *testing.T) { + for _, c := range downloadCooldownCases() { + if c.ecosystem != "gem" && c.ecosystem != "hex" { + continue + } + for _, scenario := range []downloadPublishTimeCase{ + {name: "refreshed times", refresh: true, wantStatus: http.StatusNotFound}, + {name: "stale times and listed version", down: true, wantStatus: http.StatusNotFound}, + {name: "stale times and absent version", down: true, absent: true, wantStatus: http.StatusOK}, + {name: "fresh times without version", wantStatus: http.StatusOK}, + {name: "stale times with untimed version", down: true, untimed: true, wantStatus: http.StatusOK}, + } { + for _, cached := range []bool{false, true} { + if !cached && scenario.wantStatus == http.StatusOK { + continue + } + t.Run(fmt.Sprintf("%s/%s/cached=%t", c.ecosystem, scenario.name, cached), func(t *testing.T) { + assertDownloadPublishTime(t, c, scenario, cached) + }) + } + } + } +} + +func assertDownloadPublishTime(t *testing.T, c downloadCooldownCase, scenario downloadPublishTimeCase, cached bool) { + t.Helper() + u := newCooldownCountingUpstream(t) + proxy := cooldownCacheProxy(t, u) + fetcher := proxy.Fetcher.(*mockFetcher) + fetcher.fetchErr = errors.New("unchecked artifact download") + h := c.handler(proxy, u.URL) + docPath, timesPath := "/info/demo", "/api/v1/versions/demo.json" + if c.ecosystem == "hex" { + docPath, timesPath = "/packages/demo", "/api/packages/demo" + } + oldTimes := map[string]time.Time{cooldownOldVersion: time.Now().Add(-30 * 24 * time.Hour)} + if scenario.untimed { + oldTimes[cooldownRecentVersion] = time.Time{} + } + setDownloadMetadata(t, u, c, docPath, oldTimes) + setDownloadMetadata(t, u, c, timesPath, oldTimes) + wantStatus(t, serveGet(t, h, docPath), http.StatusOK, "") + + newTimes := map[string]time.Time{ + cooldownOldVersion: oldTimes[cooldownOldVersion], + cooldownUntimed: time.Now().Add(-time.Minute), + } + if !scenario.absent { + newTimes[cooldownRecentVersion] = time.Now().Add(-time.Hour) + } + setDownloadMetadata(t, u, c, docPath, newTimes) + expireCachedMetadata(t, proxy, c.ecosystem, docPath[1:]) + if scenario.refresh { + setDownloadMetadata(t, u, c, timesPath, newTimes) + } + u.setDown(timesPath, scenario.down) + if cached { + c.seedCachedArtifact(t, proxy, cooldownRecentVersion) + } + wantStatus(t, serveGet(t, h, c.path(cooldownRecentVersion)), scenario.wantStatus, "") + if fetcher.fetchCalled { + t.Error("fetched an unchecked artifact") + } + if n, _ := u.counts(timesPath); n != 2 { + t.Errorf("publish times fetched %d times, want 2", n) + } +} + +func setDownloadMetadata(t *testing.T, u *cooldownCountingUpstream, c downloadCooldownCase, path string, times map[string]time.Time) { + t.Helper() + w := httptest.NewRecorder() + c.metadata(w, httptest.NewRequest(http.MethodGet, path, nil), times) + wantStatus(t, w, http.StatusOK, "") + u.set(path, w.Body.Bytes()) +} + func TestHexDownloadSharesPublishTimeMetadata(t *testing.T) { u, proxy, h, _ := hexCacheSetup(t, 0) wantStatus(t, serveGet(t, h, hexPackagePath), http.StatusOK, "") diff --git a/internal/handler/download_cooldown_test.go b/internal/handler/download_cooldown_test.go index a3c89ae..1fd915f 100644 --- a/internal/handler/download_cooldown_test.go +++ b/internal/handler/download_cooldown_test.go @@ -137,6 +137,13 @@ func cargoCooldownMetadata(w http.ResponseWriter, r *http.Request, times map[str } func gemCooldownMetadata(w http.ResponseWriter, r *http.Request, times map[string]time.Time) { + if r.URL.Path == "/info/demo" { + _, _ = fmt.Fprintln(w, "---") + for version := range times { + _, _ = fmt.Fprintf(w, "%s-x86_64-linux |checksum:a\n", version) + } + return + } if r.URL.Path != "/api/v1/versions/demo.json" { http.NotFound(w, r) return @@ -153,6 +160,14 @@ func gemCooldownMetadata(w http.ResponseWriter, r *http.Request, times map[strin } func hexCooldownMetadata(w http.ResponseWriter, r *http.Request, times map[string]time.Time) { + if r.URL.Path == "/packages/demo" { + var versions []string + for version := range times { + versions = append(versions, version) + } + _, _ = w.Write(buildHexSigned(buildHexPackage("demo", versions))) + return + } if r.URL.Path != "/api/packages/demo" { http.NotFound(w, r) return diff --git a/internal/handler/filename_download.go b/internal/handler/filename_download.go index 373c8f4..c37d07d 100644 --- a/internal/handler/filename_download.go +++ b/internal/handler/filename_download.go @@ -15,7 +15,7 @@ type filenameDownload struct { parse func(string) (name, version string) // publishTime reads a version's publish time from upstream metadata for // the download-time cooldown check. - publishTime func(r *http.Request, name, version string) (time.Time, bool, error) + publishTime func(r *http.Request, name, version string) (time.Time, bool, bool, error) } func (p *Proxy) handleFilenameDownload(w http.ResponseWriter, r *http.Request, d filenameDownload) { @@ -36,7 +36,7 @@ func (p *Proxy) handleFilenameDownload(w http.ResponseWriter, r *http.Request, d if p.withholdInCooldown(w, r, downloadVersion{ ecosystem: d.ecosystem, policyName: name, storedName: name, version: version, cacheName: name, filename: filename, - lookup: func() (time.Time, bool, error) { return d.publishTime(r, name, version) }, + lookup: func() (time.Time, bool, bool, error) { return d.publishTime(r, name, version) }, }) { return } diff --git a/internal/handler/gem.go b/internal/handler/gem.go index e9ee55a..fa7c24b 100644 --- a/internal/handler/gem.go +++ b/internal/handler/gem.go @@ -305,8 +305,8 @@ func parseGemVersions(body []byte) (publishTimes, error) { return times, nil } -func (h *GemHandler) publishTime(r *http.Request, name, version string) (time.Time, bool, error) { - return h.proxy.lookupPublishTime(r.Context(), h.timedIndex(name), version) +func (h *GemHandler) publishTime(r *http.Request, name, version string) (time.Time, bool, bool, error) { + return h.proxy.lookupPublishTime(r.Context(), h.timedIndex(name), version, h.indexVersions) } // proxyCached forwards a metadata request with caching. diff --git a/internal/handler/hex.go b/internal/handler/hex.go index ceaf587..d394c4d 100644 --- a/internal/handler/hex.go +++ b/internal/handler/hex.go @@ -242,8 +242,8 @@ func parseHexReleases(body []byte) (publishTimes, error) { return times, nil } -func (h *HexHandler) publishTime(r *http.Request, name, version string) (time.Time, bool, error) { - return h.proxy.lookupPublishTime(r.Context(), h.timedPackage(name), version) +func (h *HexHandler) publishTime(r *http.Request, name, version string) (time.Time, bool, bool, error) { + return h.proxy.lookupPublishTime(r.Context(), h.timedPackage(name), version, h.packageVersions) } // signedPayload decompresses a gzipped Signed message, with the metadata diff --git a/internal/handler/pub.go b/internal/handler/pub.go index 001b0c6..31e00d8 100644 --- a/internal/handler/pub.go +++ b/internal/handler/pub.go @@ -97,7 +97,10 @@ func (h *PubHandler) handleDownload(w http.ResponseWriter, r *http.Request) { h.discardVersionPage(name, version, filename) if h.proxy.withholdInCooldown(w, r, downloadVersion{ ecosystem: "pub", policyName: name, storedName: name, version: version, cacheName: name, filename: filename, - lookup: func() (time.Time, bool, error) { return h.publishTime(r, name, version) }, + lookup: func() (time.Time, bool, bool, error) { + published, ok, err := h.publishTime(r, name, version) + return published, ok, false, err + }, }) { return } From 79be33049a05b9f964c203326d9dfdc27a07090e Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Thu, 5 Nov 2026 18:03:21 +0000 Subject: [PATCH 564/633] Add cooldown profiles selected by registry URL --- README.md | 2 + config.example.yaml | 7 + docs/configuration.md | 31 +++ internal/config/config.go | 7 + internal/config/cooldown_profiles.go | 64 ++++++ internal/config/cooldown_profiles_test.go | 81 +++++++ internal/handler/handler.go | 61 +++--- internal/handler/profile.go | 30 +++ internal/handler/profile_test.go | 113 ++++++++++ internal/server/cooldown_profiles.go | 26 +++ internal/server/cooldown_profiles_test.go | 253 ++++++++++++++++++++++ internal/server/middleware.go | 4 + internal/server/server.go | 67 +++--- 13 files changed, 688 insertions(+), 58 deletions(-) create mode 100644 internal/config/cooldown_profiles.go create mode 100644 internal/config/cooldown_profiles_test.go create mode 100644 internal/handler/profile.go create mode 100644 internal/handler/profile_test.go create mode 100644 internal/server/cooldown_profiles.go create mode 100644 internal/server/cooldown_profiles_test.go diff --git a/README.md b/README.md index 2a08eaa..556e676 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,8 @@ A 3-day cooldown means that when `lodash` publishes version `4.18.0`, your build Resolution order: package override, then ecosystem override, then global default. This lets you set a conservative default and carve out exceptions for packages where you need faster updates. See [docs/configuration.md](docs/configuration.md) for the full config reference. +One instance can offer several cooldown policies through [named profiles](docs/configuration.md#cooldown-profiles). Configure `cooldown.profiles` and point clients at paths such as `/cooldown/3d/npm/` or `/cooldown/7d/cargo/`. Profiles share the package cache; existing URLs keep using the global policy. + ## Artifact Scanning Cooldown only looks at a version's publish timestamp — it never inspects the actual bytes. Artifact scanning closes that gap: when enabled, every artifact is staged into storage and scanned by one or more external services (trivy, ClamAV, Wiz, or anything else that speaks a small HTTP/JSON contract) before it's committed to the cache and served to clients. diff --git a/config.example.yaml b/config.example.yaml index e43e434..c8ab11b 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -482,6 +482,13 @@ health: # to spot malicious releases before they're pulled into projects. # Supported durations: "7d" (days), "48h" (hours), "30m" (minutes), "0" (disabled) cooldown: + # Independent policies selected through /cooldown/{name}/{registry}/. + # profiles: + # 3d: + # default: "3d" + # 7d: + # default: "7d" + # Global default cooldown for all ecosystems # default: "3d" diff --git a/docs/configuration.md b/docs/configuration.md index 38765e0..63cc175 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1148,6 +1148,37 @@ Uploads are off by default, so a client that can reach the proxy cannot overwrit `max_age` and `max_size` are independent and can be combined. When both are set, age-based eviction runs first, then size-based eviction trims remaining entries oldest-first. +## Cooldown profiles + +Set `cooldown.profiles` to offer several policies from one instance. Clients select a profile through their package manager's registry URL: + +```yaml +cooldown: + default: "0" + profiles: + 3d: + default: "3d" + 7d: + default: "7d" + packages: + "pkg:npm/@example/tool": "0" +``` + +For the `3d` profile, use these paths after your proxy's base URL: + +| Client | Path | +|--------|------| +| npm registry | `/cooldown/3d/npm/` | +| Go `GOPROXY` | `/cooldown/3d/go` | +| Cargo registry | `sparse+https://your-proxy.example/cooldown/3d/cargo/` | +| Bundler RubyGems mirror | `/cooldown/3d/gem` | + +The prefix works with all package protocol routes. Cooldown has an effect only for [registries with cooldown support](../README.md#supported-registries). Each profile defines its own `default`, `ecosystems`, `packages` and `package_patterns`; it does not inherit the global cooldown overrides. Existing unprefixed URLs, the UI, APIs and background mirror jobs use the global policy. + +Profile names contain lowercase letters, digits, hyphens and underscores, and start with a letter or digit. Profiles must be configured before startup; an unknown profile returns 404. Nested profiles are rejected. Profile paths expose package protocols only, so `/cooldown/3d/ui/` and `/cooldown/3d/api/` return 404. + +Database records, artifacts and upstream metadata are shared. Generated download URLs retain the profile prefix, and filtered metadata and its validators are separated by profile. Changing a registry URL does not rewrite artifact URLs already recorded in a lockfile; regenerate those URLs if the selected policy must cover the whole installation. + ## Cooldown The cooldown feature hides package versions published too recently, giving the community time to spot malicious releases before they reach your projects. When a version is within its cooldown period, it's stripped from metadata responses so package managers won't install it. diff --git a/internal/config/config.go b/internal/config/config.go index 68c578f..3b5c6cb 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -237,6 +237,9 @@ type Config struct { // CooldownConfig configures version cooldown periods. // Versions published more recently than the cooldown are hidden from metadata responses. type CooldownConfig struct { + // Profiles are independent policies exposed at /cooldown/{name}/. + Profiles map[string]CooldownConfig `json:"profiles,omitempty" yaml:"profiles,omitempty"` + // Default is the global default cooldown (e.g., "3d", "48h", "0" to disable). Default string `json:"default" yaml:"default"` @@ -1665,6 +1668,10 @@ func (c *Config) validateCacheArtifacts() error { } func (c *Config) validateComponents() error { + if err := c.Cooldown.ValidateProfiles(); err != nil { + return err + } + if err := validateMetadataRewriteCacheSize(c.MetadataRewriteCacheSize); err != nil { return err } diff --git a/internal/config/cooldown_profiles.go b/internal/config/cooldown_profiles.go new file mode 100644 index 0000000..225dd20 --- /dev/null +++ b/internal/config/cooldown_profiles.go @@ -0,0 +1,64 @@ +package config + +import ( + "fmt" + "maps" + "regexp" + "slices" + + "github.com/git-pkgs/cooldown" +) + +var cooldownProfileName = regexp.MustCompile(`^[a-z0-9][a-z0-9_-]*$`) + +// Policy builds the policy for this URL's cooldown settings. +func (c *CooldownConfig) Policy() (*cooldown.Policy, error) { + return cooldown.NewPolicy(&cooldown.Config{ + Default: c.Default, Ecosystems: c.Ecosystems, Packages: c.NormalizedPackages(), + }, c.PackagePatterns) +} + +// ValidateProfiles checks names and policies before they become routes. +func (c *CooldownConfig) ValidateProfiles() error { + for _, name := range slices.Sorted(maps.Keys(c.Profiles)) { + profile := c.Profiles[name] + if !cooldownProfileName.MatchString(name) { + return fmt.Errorf("invalid cooldown profile name %q: use lowercase letters, digits, hyphens or underscores", name) + } + if len(profile.Profiles) > 0 { + return fmt.Errorf("cooldown.profiles.%s cannot contain profiles", name) + } + if err := profile.validateProfileDurations(); err != nil { + return fmt.Errorf("cooldown.profiles.%s: %w", name, err) + } + if _, err := profile.Policy(); err != nil { + return fmt.Errorf("cooldown.profiles.%s: %w", name, err) + } + } + return nil +} + +func (c *CooldownConfig) validateProfileDurations() error { + values := map[string]string{"default": c.Default} + for group, overrides := range map[string]map[string]string{ + "ecosystems": c.Ecosystems, "packages": c.Packages, "package_patterns": c.PackagePatterns, + } { + for key, value := range overrides { + values[group+"."+key] = value + } + } + for _, key := range slices.Sorted(maps.Keys(values)) { + value := values[key] + if key == "default" && value == "" { + continue + } + duration, err := cooldown.ParseDuration(value) + if err != nil { + return fmt.Errorf("invalid %s: %w", key, err) + } + if duration < 0 { + return fmt.Errorf("invalid %s: duration must be non-negative", key) + } + } + return nil +} diff --git a/internal/config/cooldown_profiles_test.go b/internal/config/cooldown_profiles_test.go new file mode 100644 index 0000000..6f10362 --- /dev/null +++ b/internal/config/cooldown_profiles_test.go @@ -0,0 +1,81 @@ +package config + +import ( + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +func TestLoadCooldownProfiles(t *testing.T) { + filename := filepath.Join(t.TempDir(), "config.yaml") + data := `cooldown: + default: 7d + ecosystems: + npm: "0" + profiles: + 3d: + default: 3d + packages: + "pkg:pypi/Django": "0" + strict: + default: 14d + package_patterns: + "pkg:npm/@example/*": 1d +` + if err := os.WriteFile(filename, []byte(data), 0o600); err != nil { + t.Fatal(err) + } + cfg, err := Load(filename) + if err != nil { + t.Fatal(err) + } + if err := cfg.Validate(); err != nil { + t.Fatal(err) + } + profile := cfg.Cooldown.Profiles["3d"] + policy, err := profile.Policy() + if err != nil { + t.Fatal(err) + } + if got := policy.For("npm", "pkg:npm/demo"); got != 72*time.Hour { + t.Fatalf("profile inherited global ecosystem override: %v", got) + } + if got := policy.For("pypi", "pkg:pypi/django"); got != 0 { + t.Fatalf("canonical package override = %v", got) + } + strict := cfg.Cooldown.Profiles["strict"] + policy, err = strict.Policy() + if err != nil { + t.Fatal(err) + } + if got := policy.For("npm", "pkg:npm/%40example/tool"); got != 24*time.Hour { + t.Fatalf("profile pattern = %v", got) + } +} + +func TestLoadRejectsInvalidCooldownProfiles(t *testing.T) { + for _, tc := range []struct{ name, profile, want string }{ + {"path", `"a/b": {default: 3d}`, "profile name"}, + {"uppercase", `Strict: {default: 3d}`, "profile name"}, + {"empty", `"": {default: 3d}`, "profile name"}, + {"duration", `strict: {default: nope}`, "cooldown.profiles.strict"}, + {"pattern", `strict: {package_patterns: {"pkg:npm/[": 3d}}`, "cooldown.profiles.strict"}, + {"nested", `strict: {profiles: {nested: {default: 3d}}}`, "cannot contain profiles"}, + } { + t.Run(tc.name, func(t *testing.T) { + filename := filepath.Join(t.TempDir(), "config.yaml") + if err := os.WriteFile(filename, []byte("cooldown:\n profiles:\n "+tc.profile+"\n"), 0o600); err != nil { + t.Fatal(err) + } + cfg, err := Load(filename) + if err == nil { + err = cfg.Validate() + } + if err == nil || !strings.Contains(err.Error(), tc.want) { + t.Fatalf("Load error = %v, want %q", err, tc.want) + } + }) + } +} diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 125b117..d188594 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -304,6 +304,7 @@ type Proxy struct { // cooldownWatch holds the rewrites recording which versions cooldown // holds back; see cooldown_expiry.go. cooldownWatch cooldownWatch + shared *Proxy } // cooldownEcosystems are the ecosystems whose handlers apply cooldown. Others @@ -946,18 +947,19 @@ type inflightFetch struct { // proxy_cache_misses_total and upstream fetch observations is what coalescing // saved. func (p *Proxy) coalesceFetch(ctx context.Context, key string, recheck func() (artifacts.Artifact, string, bool), commit func(context.Context) (artifacts.Artifact, string, error)) (*CacheResult, error) { - p.fetchMu.Lock() - if p.inFlight == nil { - p.inFlight = make(map[string]*inflightFetch) + shared := p.sharedProxy() + shared.fetchMu.Lock() + if shared.inFlight == nil { + shared.inFlight = make(map[string]*inflightFetch) } - f, joined := p.inFlight[key] + f, joined := shared.inFlight[key] if joined { f.waiters++ } else { f = &inflightFetch{done: make(chan struct{})} - p.inFlight[key] = f + shared.inFlight[key] = f } - p.fetchMu.Unlock() + shared.fetchMu.Unlock() if !joined { if streamsCacheMiss(ctx) { @@ -985,28 +987,30 @@ func (p *Proxy) coalesceFetch(ctx context.Context, key string, recheck func() (a // leaveFetch withdraws a waiter that gave up. Once the fetch has finished the // waiter was counted as holding any unrecorded object, so it releases it. func (p *Proxy) leaveFetch(f *inflightFetch) { - p.fetchMu.Lock() + shared := p.sharedProxy() + shared.fetchMu.Lock() select { case <-f.done: - p.fetchMu.Unlock() + shared.fetchMu.Unlock() if f.unrecorded != nil { f.unrecorded.release() } default: f.waiters-- - p.fetchMu.Unlock() + shared.fetchMu.Unlock() } } // runSharedFetch performs the fetch that joined callers are waiting on. It is // never abandoned early, and always releases the key and wakes the waiters. func (p *Proxy) runSharedFetch(ctx context.Context, key string, f *inflightFetch, recheck func() (artifacts.Artifact, string, bool), commit func(context.Context) (artifacts.Artifact, string, error)) (*CacheResult, error) { + shared := p.sharedProxy() // Set before running so a panicking commit leaves waiters with an error // rather than a zero-valued artifact. f.err = errSharedFetchAbandoned defer func() { - p.fetchMu.Lock() - delete(p.inFlight, key) + shared.fetchMu.Lock() + delete(shared.inFlight, key) if f.unrecorded != nil { // Hand each waiter a hold, then drop the one kept while the // waiters were still unknown. @@ -1014,7 +1018,7 @@ func (p *Proxy) runSharedFetch(ctx context.Context, key string, f *inflightFetch defer f.unrecorded.release() } close(f.done) - p.fetchMu.Unlock() + shared.fetchMu.Unlock() }() // A caller checks the cache before reaching here, so a fetch that finished @@ -1162,12 +1166,13 @@ func upstreamFetchDetached(ctx context.Context) bool { const maxDetachedFetches = 64 func (p *Proxy) acquireDetachSlot() bool { - p.fetchMu.Lock() - if p.detachSlots == nil { - p.detachSlots = make(chan struct{}, maxDetachedFetches) + shared := p.sharedProxy() + shared.fetchMu.Lock() + if shared.detachSlots == nil { + shared.detachSlots = make(chan struct{}, maxDetachedFetches) } - slots := p.detachSlots - p.fetchMu.Unlock() + slots := shared.detachSlots + shared.fetchMu.Unlock() select { case slots <- struct{}{}: @@ -1178,7 +1183,8 @@ func (p *Proxy) acquireDetachSlot() bool { } func (p *Proxy) releaseDetachSlot() { - <-p.detachSlots + shared := p.sharedProxy() + <-shared.detachSlots } // fetchedArtifact is what a shared fetch hands its callers: metadata and a @@ -1877,18 +1883,19 @@ func metadataCoalesceKey(ecosystem, cacheKey, upstreamURL, accept, acceptEncodin // fail when the first of them disconnects. It stays bounded by the HTTP // client's timeout. Waiters still leave when their own context ends. func (p *Proxy) coalesceMetadata(ctx context.Context, key string, fetch func(context.Context) metadataResult) (metadataResult, bool) { - p.metaMu.Lock() - if p.inFlightMeta == nil { - p.inFlightMeta = make(map[string]*inflightMetadata) + shared := p.sharedProxy() + shared.metaMu.Lock() + if shared.inFlightMeta == nil { + shared.inFlightMeta = make(map[string]*inflightMetadata) } - f, joined := p.inFlightMeta[key] + f, joined := shared.inFlightMeta[key] if joined { f.waiters++ } else { f = &inflightMetadata{done: make(chan struct{})} - p.inFlightMeta[key] = f + shared.inFlightMeta[key] = f } - p.metaMu.Unlock() + shared.metaMu.Unlock() if joined { select { @@ -1903,9 +1910,9 @@ func (p *Proxy) coalesceMetadata(ctx context.Context, key string, fetch func(con // rather than an empty body. f.res = metadataResult{err: errSharedFetchAbandoned} defer func() { - p.metaMu.Lock() - delete(p.inFlightMeta, key) - p.metaMu.Unlock() + shared.metaMu.Lock() + delete(shared.inFlightMeta, key) + shared.metaMu.Unlock() close(f.done) }() f.res = fetch(context.WithoutCancel(ctx)) diff --git a/internal/handler/profile.go b/internal/handler/profile.go new file mode 100644 index 0000000..233e7bb --- /dev/null +++ b/internal/handler/profile.go @@ -0,0 +1,30 @@ +package handler + +import "time" + +// WithCooldown creates a policy view sharing storage, fetches and rewrite space. +// Configure the original proxy before constructing its views. +func (p *Proxy) WithCooldown(policy CooldownPolicy, shortest time.Duration) *Proxy { + return &Proxy{ + DB: p.DB, Storage: p.Storage, Fetcher: p.Fetcher, Resolver: p.Resolver, + RouteFor: p.RouteFor, Logger: p.Logger, + Cooldown: policy, ShortestCooldown: shortest, Denylist: p.Denylist, + CacheMetadata: p.CacheMetadata, MetadataTTL: p.MetadataTTL, + MetadataMaxSize: p.MetadataMaxSize, MetadataStreamMaxSize: p.MetadataStreamMaxSize, + GradleReadOnly: p.GradleReadOnly, HostedEnabled: p.HostedEnabled, + GradleMaxUploadSize: p.GradleMaxUploadSize, NPMFullMetadata: p.NPMFullMetadata, + DirectServe: p.DirectServe, DirectServeTTL: p.DirectServeTTL, + DirectServeBaseURL: p.DirectServeBaseURL, HTTPClient: p.HTTPClient, + RateLimits: p.RateLimits, HostedNPMScopes: p.HostedNPMScopes, + HostedComposerVendors: p.HostedComposerVendors, StreamArtifacts: p.StreamArtifacts, + Scanners: p.Scanners, ScanSigningKey: p.ScanSigningKey, ScanFetchBaseURL: p.ScanFetchBaseURL, + rewrites: p.rewrites, now: p.now, started: p.started, shared: p.sharedProxy(), + } +} + +func (p *Proxy) sharedProxy() *Proxy { + if p.shared != nil { + return p.shared + } + return p +} diff --git a/internal/handler/profile_test.go b/internal/handler/profile_test.go new file mode 100644 index 0000000..6377062 --- /dev/null +++ b/internal/handler/profile_test.go @@ -0,0 +1,113 @@ +package handler + +import ( + "context" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + "time" + + "github.com/git-pkgs/cooldown" +) + +func profilePolicy(t *testing.T, duration string) *cooldown.Policy { + t.Helper() + policy, err := cooldown.NewPolicy(&cooldown.Config{Default: duration}, nil) + if err != nil { + t.Fatal(err) + } + return policy +} + +func TestProfilesCoalesceUncachedMetadata(t *testing.T) { + u := newMetadataUpstream(t, http.StatusOK) + p := metadataTestProxy(t, u) + views := []*Proxy{p.WithCooldown(profilePolicy(t, "3d"), 72*time.Hour), p.WithCooldown(profilePolicy(t, "7d"), 168*time.Hour)} + first := startMetadataCalls(views[0], context.Background(), 1, u.URL, contentTypeJSON) + <-u.entered + second := startMetadataCalls(views[1], context.Background(), 1, u.URL, contentTypeJSON) + key := metadataCoalesceKey("npm", "left-pad", u.URL, contentTypeJSON, "", false) + var release sync.Once + t.Cleanup(func() { release.Do(func() { close(u.release) }) }) + waitForMetadataWaiters(t, p, key, 1) + release.Do(func() { close(u.release) }) + for _, calls := range [][]chan metadataCall{first, second} { + result := <-calls[0] + if result.err != nil || !strings.Contains(string(result.body), "accept") { + t.Fatalf("metadata = %s, error %v", result.body, result.err) + } + } + if n := u.calls.Load(); n != 1 { + t.Fatalf("upstream requests = %d, want 1", n) + } +} + +func TestProfilesCoalesceArtifactRequests(t *testing.T) { + proxy, db, _, _ := setupTestProxy(t) + if err := db.SetVersionPublishedAt("pkg:cargo/demo@1.0.0", "pkg:cargo/demo", time.Now().Add(-10*24*time.Hour)); err != nil { + t.Fatal(err) + } + fetcher := &countingFetcher{content: "crate bytes", delay: fetchHoldTime} + proxy.Fetcher = fetcher + views := []*Proxy{proxy.WithCooldown(profilePolicy(t, "3d"), 72*time.Hour), proxy.WithCooldown(profilePolicy(t, "7d"), 168*time.Hour)} + var wg sync.WaitGroup + start := make(chan struct{}) + for range 4 { + for _, view := range views { + wg.Go(func() { + <-start + h := NewCargoHandler(view, "http://proxy.test", "https://index.crates.io", "https://static.crates.io/crates") + w := httptest.NewRecorder() + h.Routes().ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/crates/demo/1.0.0/download", nil)) + if w.Code != http.StatusOK || w.Body.String() != "crate bytes" { + t.Errorf("artifact response %d: %s", w.Code, w.Body) + } + }) + } + } + close(start) + wg.Wait() + if n := fetcher.calls.Load(); n != 1 { + t.Fatalf("upstream fetches = %d, want 1", n) + } +} + +type profileClockPolicy struct { + *cooldown.Policy + now func() time.Time +} + +func (p profileClockPolicy) IsAllowed(ecosystem, packagePURL string, publishedAt time.Time) bool { + return p.Evaluate(ecosystem, packagePURL, publishedAt, p.now()).Allowed +} + +func TestProfileExpiryReleasesVersionsIndependently(t *testing.T) { + u, proxy, _ := gemCacheSetup(t) + clock := &testClock{t: time.Now()} + proxy.now = clock.now + proxy.SetMetadataRewriteCacheSize(1 << 20) + view := proxy.WithCooldown(profileClockPolicy{profilePolicy(t, "3d"), clock.now}, 72*time.Hour) + h := NewGemHandlerWithUpstream(view, "http://proxy.test/cooldown/3d", u.URL).Routes() + strictView := proxy.WithCooldown(profileClockPolicy{profilePolicy(t, "7d"), clock.now}, 168*time.Hour) + strict := NewGemHandlerWithUpstream(strictView, "http://proxy.test/cooldown/7d", u.URL).Routes() + first := getWith(t, h, gemInfoPath, nil) + if first.Code != http.StatusOK || first.Body.String() != gemFiltered { + t.Fatalf("initial response %d: %s", first.Code, first.Body) + } + if first := getWith(t, strict, gemInfoPath, nil); first.Code != http.StatusOK || first.Body.String() != gemFiltered { + t.Fatalf("initial strict response %d: %s", first.Code, first.Body) + } + clock.advance(4 * 24 * time.Hour) + second := getWith(t, h, gemInfoPath, nil) + if second.Code != http.StatusOK || !strings.Contains(second.Body.String(), "2.0.0 |") { + t.Fatalf("version did not leave cooldown: %d %s", second.Code, second.Body) + } + if second := getWith(t, strict, gemInfoPath, nil); second.Code != http.StatusOK || second.Body.String() != gemFiltered { + t.Fatalf("strict profile released version early: %d %s", second.Code, second.Body) + } + if n, _ := u.counts(gemInfoPath); n != 1 { + t.Errorf("expiry refetched raw metadata %d times", n) + } +} diff --git a/internal/server/cooldown_profiles.go b/internal/server/cooldown_profiles.go new file mode 100644 index 0000000..e818a01 --- /dev/null +++ b/internal/server/cooldown_profiles.go @@ -0,0 +1,26 @@ +package server + +import ( + "net/http" + "strings" + + "github.com/git-pkgs/proxy/internal/handler" + "github.com/go-chi/chi/v5" +) + +func (s *Server) mountCooldownProfiles(r chi.Router, proxy *handler.Proxy) error { + if err := s.cfg.Cooldown.ValidateProfiles(); err != nil { + return err + } + for name, cfg := range s.cfg.Cooldown.Profiles { + policy, err := cfg.Policy() + if err != nil { + return err + } + prefix := "/cooldown/" + name + profile := chi.NewRouter() + s.mountProtocolHandlersAt(profile, proxy.WithCooldown(policy, cfg.Shortest()), strings.TrimRight(s.cfg.BaseURL, "/")+prefix) + r.Mount(prefix, http.StripPrefix(prefix, profile)) + } + return nil +} diff --git a/internal/server/cooldown_profiles_test.go b/internal/server/cooldown_profiles_test.go new file mode 100644 index 0000000..ad92434 --- /dev/null +++ b/internal/server/cooldown_profiles_test.go @@ -0,0 +1,253 @@ +package server + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "log/slog" + "net" + "net/http" + "net/http/httptest" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/git-pkgs/proxy/internal/config" +) + +func TestCooldownProfilesThroughServerStart(t *testing.T) { + if os.Getenv("PROXY_TEST_PROFILES") != "1" { + cmd := exec.Command(os.Args[0], "-test.run=^TestCooldownProfilesThroughServerStart$") + cmd.Env = append(os.Environ(), "PROXY_TEST_PROFILES=1") + if output, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("profile server: %v\n%s", err, output) + } + return + } + upstream, counts := profileTestUpstream(t) + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = listener.Close() }) + cfg := config.Default() + cfg.BaseURL = "http://" + listener.Addr().String() + cfg.Database.Path = filepath.Join(t.TempDir(), "cache.db") + cfg.Storage.URL = "file://" + t.TempDir() + cfg.CacheMetadata = true + cfg.MetadataTTL = "1h" + cfg.Upstream.AllowLoopback = true + cfg.Upstream.NPM = upstream.URL + "/npm" + cfg.Upstream.Cargo = upstream.URL + "/cargo" + cfg.Upstream.CargoDownload = upstream.URL + "/crates" + cfg.Upstream.Gem = upstream.URL + "/gem" + cfg.Upstream.Go = upstream.URL + "/go" + cfg.Cooldown = config.CooldownConfig{Profiles: map[string]config.CooldownConfig{ + "3d": {Default: "3d"}, "7d": {Default: "7d"}, + }} + if err := cfg.Validate(); err != nil { + t.Fatal(err) + } + s, err := New(cfg, slog.New(slog.NewTextHandler(io.Discard, nil)), BuildInfo{}) + if err != nil { + t.Fatal(err) + } + done := make(chan error, 1) + go func() { done <- s.Start(listener) }() + client := &http.Client{Timeout: 5 * time.Second} + defer func() { + client.CloseIdleConnections() + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if err := s.Shutdown(ctx); err != nil { + t.Error(err) + } + if err := <-done; !errors.Is(err, http.ErrServerClosed) { + t.Error(err) + } + }() + get := func(path string, headers map[string]string) (*http.Response, string) { + t.Helper() + req, err := http.NewRequest(http.MethodGet, cfg.BaseURL+path, nil) + if err != nil { + t.Fatal(err) + } + for name, value := range headers { + req.Header.Set(name, value) + } + resp, err := client.Do(req) + if err != nil { + t.Fatal(err) + } + defer func() { _ = resp.Body.Close() }() + body, err := io.ReadAll(resp.Body) + if err != nil { + t.Fatal(err) + } + return resp, string(body) + } + assertProfileVersionLists(t, get) + assertProfileRouting(t, cfg.BaseURL, get, counts) + assertConcurrentProfileRequests(t, client, cfg.BaseURL) +} + +type profileGetter func(string, map[string]string) (*http.Response, string) + +func assertProfileVersionLists(t *testing.T, get profileGetter) { + t.Helper() + for _, registry := range []struct{ path, old, middle, newest string }{ + {"npm/demo", `"1.0.0"`, `"2.0.0"`, `"3.0.0"`}, + {"cargo/de/mo/demo", `"vers":"1.0.0"`, `"vers":"2.0.0"`, `"vers":"3.0.0"`}, + {"gem/info/demo", "1.0.0 |", "2.0.0 |", "3.0.0 |"}, + {"go/example.com/demo/@v/list", "v1.0.0", "v2.0.0", "v3.0.0"}, + } { + t.Run(registry.path, func(t *testing.T) { + for _, profile := range []struct { + prefix string + middle, newest bool + }{ + {"/cooldown/3d/", true, false}, {"/cooldown/7d/", false, false}, {"/", true, true}, + } { + resp, body := get(profile.prefix+registry.path, nil) + if resp.StatusCode != http.StatusOK || !strings.Contains(body, registry.old) || + strings.Contains(body, registry.middle) != profile.middle || strings.Contains(body, registry.newest) != profile.newest { + t.Fatalf("%s%s: status %d, body %s", profile.prefix, registry.path, resp.StatusCode, body) + } + } + }) + } +} + +func assertProfileRouting(t *testing.T, baseURL string, get profileGetter, counts func(string) int) { + t.Helper() + for _, profile := range []string{"3d", "7d"} { + _, body := get("/cooldown/"+profile+"/cargo/config.json", nil) + if !strings.Contains(body, baseURL+"/cooldown/"+profile+"/cargo/crates/") { + t.Fatalf("Cargo download URL lost profile: %s", body) + } + _, body = get("/cooldown/"+profile+"/npm/demo", nil) + if !strings.Contains(body, baseURL+"/cooldown/"+profile+"/npm/demo/-/") { + t.Fatalf("npm download URL lost profile: %s", body) + } + } + loose, _ := get("/cooldown/3d/npm/demo", nil) + strict, _ := get("/cooldown/7d/npm/demo", map[string]string{"If-None-Match": loose.Header.Get("ETag")}) + if strict.StatusCode != http.StatusOK || strict.Header.Get("ETag") == loose.Header.Get("ETag") { + t.Fatalf("cross-profile validator returned status %d", strict.StatusCode) + } + resp, _ := get("/cooldown/7d/npm/demo", map[string]string{"If-None-Match": strict.Header.Get("ETag")}) + if resp.StatusCode != http.StatusNotModified { + t.Fatalf("same-profile revalidation returned %d", resp.StatusCode) + } + for _, tc := range []struct { + path string + status int + }{ + {"/cooldown/3d/npm/demo/-/demo-2.0.0.tgz", http.StatusOK}, + {"/cooldown/7d/npm/demo/-/demo-2.0.0.tgz", http.StatusNotFound}, + {"/cooldown/3d/npm/demo/-/demo-2.0.0.tgz", http.StatusOK}, + {"/cooldown/7d/go/example.com/demo/@v/v2.0.0.mod", http.StatusForbidden}, + {"/cooldown/unknown/npm/demo", http.StatusNotFound}, + {"/cooldown/3d/ui/", http.StatusNotFound}, + {"/cooldown/3d/api/packages", http.StatusNotFound}, + {"/cooldown/3d/metrics", http.StatusNotFound}, + } { + resp, _ := get(tc.path, nil) + if resp.StatusCode != tc.status { + t.Errorf("%s: status %d, want %d", tc.path, resp.StatusCode, tc.status) + } + } + if got := counts("/npm/demo/-/demo-2.0.0.tgz"); got != 1 { + t.Errorf("artifact upstream fetches = %d, want 1", got) + } +} + +func assertConcurrentProfileRequests(t *testing.T, client *http.Client, baseURL string) { + t.Helper() + var wg sync.WaitGroup + for range 8 { + for _, profile := range []string{"3d", "7d"} { + wg.Go(func() { + resp, err := client.Get(baseURL + "/cooldown/" + profile + "/npm/demo") + if err != nil { + t.Error(err) + return + } + defer func() { _ = resp.Body.Close() }() + var doc struct{ Versions map[string]json.RawMessage } + if err := json.NewDecoder(resp.Body).Decode(&doc); err != nil { + t.Error(err) + return + } + _, middle := doc.Versions["2.0.0"] + if middle != (profile == "3d") { + t.Errorf("%s served another profile's versions", profile) + } + }) + } + } + wg.Wait() +} + +func profileTestUpstream(t *testing.T) (*httptest.Server, func(string) int) { + t.Helper() + var mu sync.Mutex + counts := make(map[string]int) + ages := []time.Duration{10 * 24 * time.Hour, 5 * 24 * time.Hour, 24 * time.Hour} + var upstream *httptest.Server + upstream = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + mu.Lock() + counts[r.URL.Path]++ + mu.Unlock() + w.Header().Set("Content-Type", "application/json") + switch r.URL.Path { + case "/npm/demo": + versions := map[string]any{} + times := map[string]string{} + for i, age := range ages { + version := fmt.Sprintf("%d.0.0", i+1) + times[version] = time.Now().Add(-age).UTC().Format(time.RFC3339) + versions[version] = map[string]any{"name": "demo", "version": version, "dist": map[string]string{ + "tarball": upstream.URL + "/npm/demo/-/demo-" + version + ".tgz", + }} + } + _ = json.NewEncoder(w).Encode(map[string]any{"name": "demo", "versions": versions, "time": times, "dist-tags": map[string]string{"latest": "3.0.0"}}) + case "/npm/demo/-/demo-2.0.0.tgz": + _, _ = io.WriteString(w, "cached package bytes") + case "/cargo/de/mo/demo": + for i, age := range ages { + _, _ = fmt.Fprintf(w, "{\"name\":\"demo\",\"vers\":\"%d.0.0\",\"deps\":[],\"features\":{},\"yanked\":false,\"pubtime\":%q}\n", i+1, time.Now().Add(-age).UTC().Format(time.RFC3339)) + } + case "/gem/info/demo": + _, _ = io.WriteString(w, "---\n1.0.0 |checksum:a\n2.0.0 |checksum:b\n3.0.0 |checksum:c\n") + case "/gem/api/v1/versions/demo.json": + versions := make([]map[string]string, 0, len(ages)) + for i, age := range ages { + versions = append(versions, map[string]string{"number": fmt.Sprintf("%d.0.0", i+1), "created_at": time.Now().Add(-age).UTC().Format(time.RFC3339)}) + } + _ = json.NewEncoder(w).Encode(versions) + case "/go/example.com/demo/@v/list": + _, _ = io.WriteString(w, "v1.0.0\nv2.0.0\nv3.0.0\n") + default: + for i, age := range ages { + if r.URL.Path == fmt.Sprintf("/go/example.com/demo/@v/v%d.0.0.info", i+1) { + _ = json.NewEncoder(w).Encode(map[string]string{"Version": fmt.Sprintf("v%d.0.0", i+1), "Time": time.Now().Add(-age).UTC().Format(time.RFC3339)}) + return + } + } + http.NotFound(w, r) + } + })) + t.Cleanup(upstream.Close) + return upstream, func(path string) int { + mu.Lock() + defer mu.Unlock() + return counts[path] + } +} diff --git a/internal/server/middleware.go b/internal/server/middleware.go index 7d2bd98..aa4da11 100644 --- a/internal/server/middleware.go +++ b/internal/server/middleware.go @@ -121,6 +121,10 @@ func (s *Server) LoggerMiddleware(next http.Handler) http.Handler { // "other" along with the UI, health and metrics paths, pooling a real // ecosystem's traffic with traffic that belongs to no ecosystem at all. func requestEcosystem(path string) string { + if rest, ok := strings.CutPrefix(path, "/cooldown/"); ok { + _, rest, _ = strings.Cut(rest, "/") + path = "/" + rest + } segment, _, _ := strings.Cut(strings.TrimPrefix(path, "/"), "/") switch segment { case "npm", "cargo", "hex", "pub", "pypi", "maven", "gradle", "nuget", diff --git a/internal/server/server.go b/internal/server/server.go index 55fe597..aa12c40 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -77,7 +77,6 @@ import ( "time" "github.com/git-pkgs/archives" - "github.com/git-pkgs/cooldown" shared "github.com/git-pkgs/enrichment" swaggerdoc "github.com/git-pkgs/proxy/docs/swagger" "github.com/git-pkgs/proxy/internal/accesslog" @@ -286,16 +285,11 @@ func (s *Server) serve(listener net.Listener) error { baseFetcher := fetch.NewFetcher(fetch.WithHTTPClient(artifactClient), fetch.WithMaxArtifactSize(s.cfg.ParseArtifactMaxSize())) fetcher := fetch.NewCircuitBreakerFetcher(baseFetcher) s.breakers = newBreakerMonitor(fetcher, s.logger) - cd := &cooldown.Config{ - Default: s.cfg.Cooldown.Default, - Ecosystems: s.cfg.Cooldown.Ecosystems, - Packages: s.cfg.Cooldown.NormalizedPackages(), - } proxy := handler.NewProxy(s.db, s.storage, fetcher, nil, s.logger) configureMirrorResolvers(proxy, s.cfg.Upstream) proxy.HTTPClient = metadataClient proxy.RateLimits = s.rateLimits - cooldownPolicy, err := cooldown.NewPolicy(cd, s.cfg.Cooldown.PackagePatterns) + cooldownPolicy, err := s.cfg.Cooldown.Policy() if err != nil { return fmt.Errorf("configuring cooldown policy: %w", err) } @@ -348,6 +342,9 @@ func (s *Server) serve(listener net.Listener) error { // Mount protocol handlers s.mountProtocolHandlers(r, proxy) + if err := s.mountCooldownProfiles(r, proxy); err != nil { + return err + } // Health, stats, and metrics endpoints r.Get("/health", s.handleHealth) @@ -462,71 +459,75 @@ func (s *Server) serve(listener net.Listener) error { // mountProtocolHandlers constructs every ecosystem handler and mounts it on // r under its protocol prefix. func (s *Server) mountProtocolHandlers(r chi.Router, proxy *handler.Proxy) { - npmHandler := handler.NewNPMHandler(proxy, s.cfg.BaseURL, s.cfg.Upstream.NPM). + s.mountProtocolHandlersAt(r, proxy, s.cfg.BaseURL) +} + +func (s *Server) mountProtocolHandlersAt(r chi.Router, proxy *handler.Proxy, baseURL string) { + npmHandler := handler.NewNPMHandler(proxy, baseURL, s.cfg.Upstream.NPM). WithPackageRoutes(s.cfg.Upstream.NPMRoutes) cargoHandler := handler.NewCargoHandler( proxy, - s.cfg.BaseURL, + baseURL, s.cfg.Upstream.Cargo, s.cfg.Upstream.CargoDownload, ) - gemHandler := handler.NewGemHandlerWithUpstream(proxy, s.cfg.BaseURL, s.cfg.Upstream.Gem) - goHandler := handler.NewGoHandlerWithUpstream(proxy, s.cfg.BaseURL, s.cfg.Upstream.Go) + gemHandler := handler.NewGemHandlerWithUpstream(proxy, baseURL, s.cfg.Upstream.Gem) + goHandler := handler.NewGoHandlerWithUpstream(proxy, baseURL, s.cfg.Upstream.Go) goHandler.SetSumDBs(s.cfg.Upstream.GoSumDBs, s.cfg.Upstream.GoSumDBsOffline) goHandler.SetVulnDB(s.cfg.Upstream.GoVulnDB) hexHandler := handler.NewHexHandlerWithUpstreams( proxy, - s.cfg.BaseURL, + baseURL, s.cfg.Upstream.Hex, s.cfg.Upstream.HexAPI, ) - pubHandler := handler.NewPubHandlerWithUpstream(proxy, s.cfg.BaseURL, s.cfg.Upstream.Pub) + pubHandler := handler.NewPubHandlerWithUpstream(proxy, baseURL, s.cfg.Upstream.Pub) pypiHandler := handler.NewPyPIHandlerWithUpstreams( proxy, - s.cfg.BaseURL, + baseURL, s.cfg.Upstream.PyPI, s.cfg.Upstream.PyPIDownload, ) mavenHandler := handler.NewMavenHandler( proxy, - s.cfg.BaseURL, + baseURL, s.cfg.Upstream.Maven, s.cfg.Upstream.GradlePluginPortal, ).WithRepositories(s.cfg.Upstream.MavenRepositories) gradleHandler := handler.NewGradleBuildCacheHandler(proxy) nugetHandler := handler.NewNuGetHandlerWithUpstreams( proxy, - s.cfg.BaseURL, + baseURL, s.cfg.Upstream.NuGet, s.cfg.Upstream.NuGetSearch, ) composerHandler := handler.NewComposerHandlerWithUpstreams( proxy, - s.cfg.BaseURL, + baseURL, s.cfg.Upstream.Composer, s.cfg.Upstream.ComposerRepository, ).WithPackageRoutes(s.cfg.Upstream.ComposerRoutes) - conanHandler := handler.NewConanHandlerWithUpstream(proxy, s.cfg.BaseURL, s.cfg.Upstream.Conan) - chefHandler := handler.NewChefHandler(proxy, s.cfg.BaseURL, s.cfg.Upstream.Chef) - condaHandler := handler.NewCondaHandlerWithUpstream(proxy, s.cfg.BaseURL, s.cfg.Upstream.Conda) - cranHandler := handler.NewCRANHandlerWithUpstream(proxy, s.cfg.BaseURL, s.cfg.Upstream.CRAN) + conanHandler := handler.NewConanHandlerWithUpstream(proxy, baseURL, s.cfg.Upstream.Conan) + chefHandler := handler.NewChefHandler(proxy, baseURL, s.cfg.Upstream.Chef) + condaHandler := handler.NewCondaHandlerWithUpstream(proxy, baseURL, s.cfg.Upstream.Conda) + cranHandler := handler.NewCRANHandlerWithUpstream(proxy, baseURL, s.cfg.Upstream.CRAN) juliaHandler := handler.NewJuliaHandlerWithUpstream(proxy, s.cfg.Upstream.Julia) - swiftHandler := handler.NewSwiftHandler(proxy, s.cfg.BaseURL, s.cfg.Upstream.Swift) + swiftHandler := handler.NewSwiftHandler(proxy, baseURL, s.cfg.Upstream.Swift) homebrewHandler := handler.NewHomebrewHandler(proxy, s.cfg.Upstream.HomebrewAPI) - containerHandler := s.newContainerHandler(proxy) + containerHandler := s.newContainerHandlerAt(proxy, baseURL) helmHandler := handler.NewHelmHandlerWithOCIRegistries( - proxy, s.cfg.BaseURL, s.cfg.Upstream.Helm, s.cfg.Upstream.OCIDefault, s.cfg.Upstream.OCI) - apkHandler := handler.NewAPKHandler(proxy, s.cfg.BaseURL, s.cfg.Upstream.APK). + proxy, baseURL, s.cfg.Upstream.Helm, s.cfg.Upstream.OCIDefault, s.cfg.Upstream.OCI) + apkHandler := handler.NewAPKHandler(proxy, baseURL, s.cfg.Upstream.APK). WithSnapshots(s.cfg.ParseAPKSnapshots()) - archHandler := handler.NewArchHandler(proxy, s.cfg.BaseURL, s.cfg.Upstream.Arch) - freebsdHandler := handler.NewFreeBSDHandler(proxy, s.cfg.BaseURL, s.cfg.Upstream.FreeBSD) + archHandler := handler.NewArchHandler(proxy, baseURL, s.cfg.Upstream.Arch) + freebsdHandler := handler.NewFreeBSDHandler(proxy, baseURL, s.cfg.Upstream.FreeBSD) debianHandler := handler.NewDebianHandler( proxy, - s.cfg.BaseURL, + baseURL, s.cfg.Upstream.Debian, s.cfg.Upstream.DebianRepositories, ) - rpmHandler := handler.NewRPMHandlerWithUpstream(proxy, s.cfg.BaseURL, s.cfg.Upstream.RPM) + rpmHandler := handler.NewRPMHandlerWithUpstream(proxy, baseURL, s.cfg.Upstream.RPM) genericHandler := handler.NewGenericHandler(proxy, s.cfg.Upstream.Generic, s.cfg.Upstream.GenericArtifacts). WithPaths(s.cfg.Upstream.GenericPaths) @@ -558,7 +559,7 @@ func (s *Server) mountProtocolHandlers(r chi.Router, proxy *handler.Proxy) { r.Mount("/rpm", http.StripPrefix("/rpm", rpmHandler.Routes())) r.Mount("/generic", http.StripPrefix("/generic", genericHandler.Routes())) if s.git != nil { - gitHandler := handler.NewGitHandler(proxy, s.git, s.cfg.Upstream).WithBaseURL(s.cfg.BaseURL) + gitHandler := handler.NewGitHandler(proxy, s.git, s.cfg.Upstream).WithBaseURL(baseURL) r.Mount("/git", http.StripPrefix("/git", gitHandler.Routes())) } } @@ -629,9 +630,13 @@ func NewMirrorProxy(cfg *config.Config, db *database.DB, store storage.Storage, // newContainerHandler builds the OCI handler with the configured default // registry, named upstreams, mirrored registries and Homebrew routes. func (s *Server) newContainerHandler(proxy *handler.Proxy) *handler.ContainerHandler { + return s.newContainerHandlerAt(proxy, s.cfg.BaseURL) +} + +func (s *Server) newContainerHandlerAt(proxy *handler.Proxy, baseURL string) *handler.ContainerHandler { containerHandler := handler.NewContainerHandlerWithRegistry( proxy, - s.cfg.BaseURL, + baseURL, s.cfg.Upstream.OCIDefault, s.cfg.Upstream.OCI, ) From 03be03b8fef9f5da0dc7bf886e70d245e593c570 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Fri, 6 Nov 2026 07:48:51 +0100 Subject: [PATCH 565/633] Mirror pkg:git commits into the git cache proxy mirror, mirror API jobs and SBOMs take pkg:git/{host}/{path}@{commit} and cache the repository and commit in the cache /git serves, so a later clone is a cache hit, also with the upstream down. Part of #403. --- README.md | 3 + cmd/proxy/main.go | 23 +++++ cmd/proxy/mirror_git_test.go | 132 ++++++++++++++++++++++++++++ docs/architecture.md | 2 +- docs/configuration.md | 14 +++ internal/gitcache/mirror.go | 88 +++++++++++++++++++ internal/gitcache/mirror_test.go | 108 +++++++++++++++++++++++ internal/handler/git_mirror.go | 57 ++++++++++++ internal/handler/git_mirror_test.go | 73 +++++++++++++++ internal/handler/git_test.go | 9 +- internal/mirror/git.go | 49 +++++++++++ internal/mirror/git_test.go | 92 +++++++++++++++++++ internal/mirror/mirror.go | 8 ++ internal/mirror/source.go | 5 +- internal/server/git_test.go | 123 +++++++++++++++++++++----- internal/server/mirror_versions.go | 3 + internal/server/server.go | 15 ++++ 17 files changed, 779 insertions(+), 25 deletions(-) create mode 100644 cmd/proxy/mirror_git_test.go create mode 100644 internal/gitcache/mirror.go create mode 100644 internal/gitcache/mirror_test.go create mode 100644 internal/handler/git_mirror.go create mode 100644 internal/handler/git_mirror_test.go create mode 100644 internal/mirror/git.go create mode 100644 internal/mirror/git_test.go diff --git a/README.md b/README.md index 2a08eaa..2eea25d 100644 --- a/README.md +++ b/README.md @@ -1194,6 +1194,9 @@ proxy mirror --concurrency 8 pkg:npm/lodash@4.17.21 # Mirror a container image for one platform proxy mirror --platform linux/amd64 pkg:docker/library/alpine@3.20 + +# Mirror a commit of an upstream.git repository into the git cache +proxy mirror pkg:git/github.com/git-pkgs/purl@0123456789abcdef0123456789abcdef01234567 ``` The mirror command accepts the same storage and database flags as `serve`. Already-cached artifacts are skipped. If any package version fails to mirror, the command lists the errors and exits with status 1, so a CI step that warms the cache fails on a partial mirror. diff --git a/cmd/proxy/main.go b/cmd/proxy/main.go index 70ae8e6..f9ab849 100644 --- a/cmd/proxy/main.go +++ b/cmd/proxy/main.go @@ -595,7 +595,9 @@ func runMirror() { return } + closeGit := useCommandGit(cfg, proxy, m, logger) progress, err := m.Run(ctx, source) + closeGit() if err != nil { fmt.Fprintf(os.Stderr, "error: %v\n", err) os.Exit(1) @@ -634,6 +636,27 @@ func newCommandMirror(cfg *config.Config, proxy *handler.Proxy, images *handler. return m } +// useCommandGit opens the git cache for the pkg:git items of proxy mirror, +// when upstream.git is configured, and returns the function that closes it. +// Where it can't be opened, such as while the running server holds +// git.cache_dir, those items fail with the reason and the rest are mirrored. +func useCommandGit(cfg *config.Config, proxy *handler.Proxy, m *mirror.Mirror, logger *slog.Logger) func() { + git, closeGit, err := server.NewMirrorGit(cfg, proxy, logger) + if err != nil { + m.Git = mirror.UnavailableGit{Reason: fmt.Sprintf("this command can't open the git cache (%v); while the server runs, use its mirror API", err)} + return func() {} + } + if git == nil { + return func() {} + } + m.Git = git + return func() { + if err := closeGit(); err != nil { + logger.Warn("closing the git cache", "error", err) + } + } +} + // mirrorExitCode returns the exit status for a finished mirror run: 1 when any // package version failed to mirror, 0 otherwise. func mirrorExitCode(progress *mirror.Progress) int { diff --git a/cmd/proxy/mirror_git_test.go b/cmd/proxy/mirror_git_test.go new file mode 100644 index 0000000..2789536 --- /dev/null +++ b/cmd/proxy/mirror_git_test.go @@ -0,0 +1,132 @@ +package main + +import ( + "net/http" + "net/http/cgi" + "net/http/httptest" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/git-pkgs/proxy/internal/gitcache" +) + +// gitMirrorUpstream serves team/repo, with one commit on main, from git +// http-backend, and returns the server and the commit. +func gitMirrorUpstream(t *testing.T) (*httptest.Server, string) { + t.Helper() + git := func(dir string, args ...string) string { + t.Helper() + cmd := exec.Command("git", args...) + cmd.Dir = dir + cmd.Env = []string{ + "PATH=" + os.Getenv("PATH"), "HOME=" + t.TempDir(), "GIT_CONFIG_NOSYSTEM=1", + "GIT_AUTHOR_NAME=test", "GIT_AUTHOR_EMAIL=test@example.com", + "GIT_COMMITTER_NAME=test", "GIT_COMMITTER_EMAIL=test@example.com", + } + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("git %s: %v\n%s", strings.Join(args, " "), err, out) + } + return strings.TrimSpace(string(out)) + } + backend := "" + for _, name := range []string{"git-http-backend", "git-http-backend.exe"} { + if path := filepath.Join(git("", "--exec-path"), name); fileExistsForTest(path) { + backend = path + } + } + if backend == "" { + t.Skip("git http-backend is not installed") + } + root := t.TempDir() + bare := filepath.Join(root, "team", "repo.git") + git("", "init", "--quiet", "--bare", "--initial-branch=main", bare) + work := t.TempDir() + git("", "clone", "--quiet", bare, work) + git(work, "commit", "--quiet", "--allow-empty", "-m", "first") + git(work, "push", "--quiet", "origin", "HEAD:main") + head := git(work, "rev-parse", "HEAD") + + server := httptest.NewServer(&cgi.Handler{ + Path: backend, + Env: []string{"PATH=" + os.Getenv("PATH"), "GIT_PROJECT_ROOT=" + root, "GIT_HTTP_EXPORT_ALL=1"}, + }) + t.Cleanup(server.Close) + return server, head +} + +func fileExistsForTest(path string) bool { + _, err := os.Stat(path) + return err == nil +} + +// writeGitMirrorConfig writes a config with upstream.git "up" at +// upstreamURL and the git cache, storage and database under dir. +func writeGitMirrorConfig(t *testing.T, dir, upstreamURL string) string { + t.Helper() + cfg := "storage:\n" + + " url: file://" + filepath.Join(dir, "artifacts") + "\n" + + "database:\n" + + " path: " + filepath.Join(dir, "proxy.db") + "\n" + + "upstream:\n" + + " allow_loopback: true\n" + + " git:\n" + + " up: " + upstreamURL + "\n" + + "git:\n" + + " cache_dir: " + filepath.Join(dir, "git") + "\n" + path := filepath.Join(dir, "config.yaml") + if err := os.WriteFile(path, []byte(cfg), 0o600); err != nil { + t.Fatalf("writing config: %v", err) + } + return path +} + +func TestMirrorCommandCachesGitCommit(t *testing.T) { + upstream, head := gitMirrorUpstream(t) + dir := t.TempDir() + configPath := writeGitMirrorConfig(t, dir, upstream.URL) + purl := "pkg:git/" + strings.TrimPrefix(upstream.URL, "http://") + "/team/repo@" + head + + output := runMirrorProcess(t, 0, "-config", configPath, purl) + if !strings.Contains(output, "1 downloaded") { + t.Errorf("output:\n%s", output) + } + cached, _ := filepath.Glob(filepath.Join(dir, "git", "up-*", "team", "repo.git")) + if len(cached) != 1 { + t.Errorf("cached repositories %v, want team/repo.git", cached) + } + + output = runMirrorProcess(t, 0, "-config", configPath, purl) + if !strings.Contains(output, "1 skipped") { + t.Errorf("second run output:\n%s", output) + } +} + +// TestMirrorCommandGitCacheInUse checks pkg:git items fail, pointing at the +// mirror API, while the running server holds git.cache_dir. +func TestMirrorCommandGitCacheInUse(t *testing.T) { + upstream, head := gitMirrorUpstream(t) + dir := t.TempDir() + configPath := writeGitMirrorConfig(t, dir, upstream.URL) + cache, err := gitcache.New(gitcache.Options{ + CacheDir: filepath.Join(dir, "git"), + RefreshWait: time.Second, + Client: &http.Client{}, + }) + if err != nil { + t.Fatal(err) + } + defer func() { _ = cache.Close() }() + purl := "pkg:git/" + strings.TrimPrefix(upstream.URL, "http://") + "/team/repo@" + head + + output := runMirrorProcess(t, 1, "-config", configPath, purl) + for _, want := range []string{"in use by another process", "mirror API", "1 failed"} { + if !strings.Contains(output, want) { + t.Errorf("output missing %q:\n%s", want, output) + } + } +} diff --git a/docs/architecture.md b/docs/architecture.md index d8b294c..655b309 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -363,7 +363,7 @@ Package metadata enrichment, built on [github.com/git-pkgs/enrichment](https://g ### `internal/mirror` -Selective package mirroring for pre-populating the proxy cache. Supports multiple input sources: individual PURLs (versioned or unversioned), CycloneDX/SPDX SBOM files, and full registry enumeration. Uses a bounded worker pool backed by `errgroup` to download artifacts in parallel, reusing `handler.Proxy.GetOrFetchArtifact()` for the actual fetch-and-cache work. Container images (`pkg:docker`, `pkg:oci`) go through `handler.ContainerHandler.MirrorImage()` instead, which walks index, platform manifests and blobs and stores them under the keys a pull reads. +Selective package mirroring for pre-populating the proxy cache. Supports multiple input sources: individual PURLs (versioned or unversioned), CycloneDX/SPDX SBOM files, and full registry enumeration. Uses a bounded worker pool backed by `errgroup` to download artifacts in parallel, reusing `handler.Proxy.GetOrFetchArtifact()` for the actual fetch-and-cache work. Container images (`pkg:docker`, `pkg:oci`) go through `handler.ContainerHandler.MirrorImage()` instead, which walks index, platform manifests and blobs and stores them under the keys a pull reads. Git repositories (`pkg:git`) go through `handler.GitHandler.Mirror()`, which caches the repository and commit in the git cache that `/git` serves. The package also provides a `MetadataCache` for storing raw upstream metadata blobs so the proxy can serve metadata responses offline. The `JobStore` manages async mirror jobs exposed via the `/api/mirror` endpoints. diff --git a/docs/configuration.md b/docs/configuration.md index 38765e0..b2b830f 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1877,6 +1877,20 @@ proxy mirror --platform linux/arm64 pkg:brew/hello@2.12.3 pkg:brew/openssl%403 - Dependencies aren't mirrored with a formula; list them too. brew also needs the JSON API, which the proxy keeps with `cache_metadata`. - Casks, source archives and taps other than homebrew/core aren't mirrored. +### Git repositories + +`pkg:git` PURLs mirror a repository of an [`upstream.git`](#git-repositories) host into the git cache, so a clone through `/git` is served from it, also when the upstream is unreachable. The PURL names the repository by host and path, and the version is a full commit ID: + +```bash +proxy mirror pkg:git/github.com/git-pkgs/purl@0123456789abcdef0123456789abcdef01234567 +proxy mirror pkg:git/github.com/git-pkgs/purl +``` + +- The repository is served by the `upstream.git` host whose URL its host and path start with, the longest one if several do, and has to match that host's `repos`. Others fail. +- The repository's branches and tags are cached, as a clone through the proxy caches them. A commit on none of them is fetched by ID and kept, like one a client fetches by ID, for seven days or until a branch or tag includes it. Where `upstream.auth` has credentials for the host, commits aren't fetched by ID, and one on no branch or tag fails. +- Without a version the repository's branches and tags are mirrored. A tag or branch name as the version fails; give the commit ID. +- The git cache is used by one process. While the server runs, the command can't open `git.cache_dir`, so `pkg:git` items fail and the rest are mirrored; use the mirror API instead. + ### Scheduled refresh Tags such as `alpine:3.20` move. The server can re-mirror them on its own, so the cache follows the tag even when nobody pulls it or the proxy is offline later: diff --git a/internal/gitcache/mirror.go b/internal/gitcache/mirror.go new file mode 100644 index 0000000..854ea08 --- /dev/null +++ b/internal/gitcache/mirror.go @@ -0,0 +1,88 @@ +package gitcache + +import ( + "context" + "errors" + "fmt" +) + +// MirrorResult is what Mirror did. +type MirrorResult struct { + // Cached reports that the repository, and the commit if one was given, + // were cached already. + Cached bool + // Bytes is how much the cached repository grew. + Bytes int64 +} + +// Mirror brings the repository cached under key up to date with repoURL, +// cloning it if needed, and waits for that to finish. With commit, an object +// ID, it also makes sure the commit is cached. One on no branch or tag is +// fetched by ID when fetchWants allows it (see OpenOptions.FetchWants), and +// is kept as long as a commit a client fetched by ID. +func (c *Cache) Mirror(ctx context.Context, key, repoURL, commit string, fetchWants bool) (MirrorResult, error) { + if commit != "" && !isObjectID(commit) { + return MirrorResult{}, fmt.Errorf("%q is not a full commit ID", commit) + } + lease, err := c.Open(ctx, key, repoURL, OpenOptions{FetchWants: fetchWants}) + if err != nil { + return MirrorResult{}, err + } + defer lease.Release() + rp := lease.repo + cached := lease.hit + before := int64(0) + var refreshErr error + // Open cloned a repository that wasn't cached; one that was is + // refreshed here, waiting for the fetch a client would not wait for. + if cached { + before = rp.size.Load() + refresh := c.startRefresh(rp, repoURL) + select { + case <-refresh.done: + refreshErr = refresh.err + case <-ctx.Done(): + return MirrorResult{}, ctx.Err() + } + } + + if commit != "" { + fetched, err := c.ensureCommit(ctx, rp, repoURL, commit, fetchWants) + if err != nil { + return MirrorResult{}, err + } + cached = cached && !fetched + } + // A refresh that failed leaves the commit cached all the same. + if refreshErr != nil && commit == "" { + return MirrorResult{}, refreshErr + } + return MirrorResult{Cached: cached, Bytes: max(rp.size.Load()-before, 0)}, nil +} + +// ensureCommit makes sure the repository has commit, fetching it by ID when +// it doesn't and fetchWants allows it, and reports whether it was fetched. +func (c *Cache) ensureCommit(ctx context.Context, rp *repo, repoURL, commit string, fetchWants bool) (bool, error) { + missing, err := c.missingObjects(ctx, rp.dir, []string{commit}) + if err != nil || len(missing) == 0 { + return false, err + } + if !fetchWants { + return false, fmt.Errorf("commit %s: %w", commit, ErrNotFound) + } + if err := c.fetchMissingWants(ctx, rp, repoURL, missing); err != nil { + if ctx.Err() != nil || errors.Is(err, ErrTooLarge) || errors.Is(err, ErrCacheFull) || errors.Is(err, errUpstreamUnanswered) { + return false, err + } + // The upstream refuses a commit it doesn't have. + return false, fmt.Errorf("commit %s: %w: %w", commit, ErrNotFound, err) + } + // fetchMissingWants skips a commit that failed moments ago. + if missing, err = c.missingObjects(ctx, rp.dir, missing); err != nil { + return false, err + } + if len(missing) > 0 { + return false, fmt.Errorf("commit %s: %w", commit, ErrNotFound) + } + return true, nil +} diff --git a/internal/gitcache/mirror_test.go b/internal/gitcache/mirror_test.go new file mode 100644 index 0000000..a1fd07e --- /dev/null +++ b/internal/gitcache/mirror_test.go @@ -0,0 +1,108 @@ +package gitcache + +import ( + "context" + "errors" + "strings" + "testing" +) + +func TestMirrorClonesRepository(t *testing.T) { + upstream := newFakeUpstream(t) + upstream.newRepo("repo") + tc := newTestCache(t, upstream.server.URL, nil) + + got, err := tc.Mirror(context.Background(), "up/repo.git", upstream.server.URL+"/repo", "", true) + if err != nil { + t.Fatalf("Mirror: %v", err) + } + if got.Cached || got.Bytes <= 0 { + t.Errorf("first mirror = %+v, want fetched bytes", got) + } + if !strings.Contains(tc.cachedRefs(t), "refs/tags/v1") { + t.Errorf("cached refs %q lack the tag", tc.cachedRefs(t)) + } + + got, err = tc.Mirror(context.Background(), "up/repo.git", upstream.server.URL+"/repo", "", true) + if err != nil || !got.Cached { + t.Errorf("second mirror = %+v, %v, want cached", got, err) + } +} + +func TestMirrorFetchesCommitOnNoBranch(t *testing.T) { + upstream := newFakeUpstream(t) + work := upstream.newRepo("repo") + runGit(t, upstream.bare(), "config", "uploadpack.allowAnySHA1InWant", "true") + sha := upstream.pushPullRequest(work, 1, nil) + tc := newTestCache(t, upstream.server.URL, nil) + + got, err := tc.Mirror(context.Background(), "up/repo.git", upstream.server.URL+"/repo", sha, true) + if err != nil { + t.Fatalf("Mirror: %v", err) + } + if got.Cached { + t.Errorf("mirror = %+v, want fetched", got) + } + if refs := tc.wantedRefs(t); len(refs) != 1 { + t.Errorf("wanted refs %v, want one", refs) + } + + sessions := upstream.gitSessions.Load() + got, err = tc.Mirror(context.Background(), "up/repo.git", upstream.server.URL+"/repo", sha, true) + if err != nil || !got.Cached { + t.Errorf("second mirror = %+v, %v, want cached", got, err) + } + if n := upstream.gitSessions.Load() - sessions; n != 0 { + t.Errorf("second mirror fetched %d times, want none", n) + } +} + +func TestMirrorFindsCommitOnABranch(t *testing.T) { + upstream := newFakeUpstream(t) + work := upstream.newRepo("repo") + sha := upstream.commit(work, "second") + runGit(t, work, "push", "--quiet", "origin", "main") + tc := newTestCache(t, upstream.server.URL, nil) + + got, err := tc.Mirror(context.Background(), "up/repo.git", upstream.server.URL+"/repo", sha, false) + if err != nil || got.Cached { + t.Errorf("mirror = %+v, %v, want fetched", got, err) + } +} + +func TestMirrorDoesNotFetchByIDWhenTurnedOff(t *testing.T) { + upstream := newFakeUpstream(t) + work := upstream.newRepo("repo") + runGit(t, upstream.bare(), "config", "uploadpack.allowAnySHA1InWant", "true") + sha := upstream.pushPullRequest(work, 1, nil) + tc := newTestCache(t, upstream.server.URL, nil) + + _, err := tc.Mirror(context.Background(), "up/repo.git", upstream.server.URL+"/repo", sha, false) + if !errors.Is(err, ErrNotFound) { + t.Errorf("mirror of a commit on no branch with fetching by ID off = %v, want ErrNotFound", err) + } + if refs := tc.wantedRefs(t); len(refs) != 0 { + t.Errorf("wanted refs %v, want none", refs) + } +} + +func TestMirrorReportsMissingCommit(t *testing.T) { + upstream := newFakeUpstream(t) + upstream.newRepo("repo") + tc := newTestCache(t, upstream.server.URL, nil) + + _, err := tc.Mirror(context.Background(), "up/repo.git", upstream.server.URL+"/repo", strings.Repeat("ab", 20), true) + if !errors.Is(err, ErrNotFound) { + t.Errorf("mirror of a commit the upstream lacks = %v, want ErrNotFound", err) + } +} + +func TestMirrorRefusesInvalidCommit(t *testing.T) { + upstream := newFakeUpstream(t) + upstream.newRepo("repo") + tc := newTestCache(t, upstream.server.URL, nil) + + if _, err := tc.Mirror(context.Background(), "up/repo.git", upstream.server.URL+"/repo", "main", true); err == nil { + t.Error("mirrored a branch name as a commit") + } +} diff --git a/internal/handler/git_mirror.go b/internal/handler/git_mirror.go new file mode 100644 index 0000000..eca76c7 --- /dev/null +++ b/internal/handler/git_mirror.go @@ -0,0 +1,57 @@ +package handler + +import ( + "context" + "fmt" + "net/url" + "strings" + + "github.com/git-pkgs/proxy/internal/gitcache" +) + +// Mirror caches the repository at location, a host and path such as +// github.com/acme/widgets as a pkg:git PURL names it, from the upstream.git +// host serving it, and with commit, a full commit ID, that commit too. It +// is the copy clones through /git are then served. +func (h *GitHandler) Mirror(ctx context.Context, location, commit string) (gitcache.MirrorResult, error) { + repo, err := h.mirrorRepository(location) + if err != nil { + return gitcache.MirrorResult{}, err + } + return h.cache.Mirror(ctx, repo.key, repo.url, commit, repo.upstream.fetchWants) +} + +// mirrorRepository resolves location to a repository of the upstream.git +// host whose URL it starts with, the longest such URL if several do. +func (h *GitHandler) mirrorRepository(location string) (gitRepository, error) { + var best, bestPrefix, rest string + for name, upstream := range h.upstreams { + prefix := gitLocationPrefix(upstream.URL) + path, ok := strings.CutPrefix(location, prefix) + if prefix == "" || !ok || path == "" { + continue + } + if len(prefix) > len(bestPrefix) || (len(prefix) == len(bestPrefix) && name < best) { + best, bestPrefix, rest = name, prefix, path + } + } + if best == "" { + return gitRepository{}, fmt.Errorf("no upstream.git host serves %s", location) + } + repo, status, msg := h.repository(best + "/" + rest) + if status != 0 { + return gitRepository{}, fmt.Errorf("%s: %s", location, msg) + } + return repo, nil +} + +// gitLocationPrefix is how a pkg:git PURL names the repositories under an +// upstream URL: its host and path, ending in a slash. Empty for a URL +// without a host. +func gitLocationPrefix(upstreamURL string) string { + u, err := url.Parse(upstreamURL) + if err != nil || u.Host == "" { + return "" + } + return strings.ToLower(u.Host) + strings.TrimSuffix(u.Path, "/") + "/" +} diff --git a/internal/handler/git_mirror_test.go b/internal/handler/git_mirror_test.go new file mode 100644 index 0000000..4ebcf98 --- /dev/null +++ b/internal/handler/git_mirror_test.go @@ -0,0 +1,73 @@ +package handler + +import ( + "context" + "strings" + "testing" + + "github.com/git-pkgs/proxy/internal/config" +) + +func TestGitMirrorLocation(t *testing.T) { + h := NewGitHandler(nil, nil, config.UpstreamConfig{Git: map[string]config.GitUpstream{ + "github": {URL: "https://github.com"}, + "wikimedia": {URL: "https://gitlab.wikimedia.org", Repos: []string{"mediawiki/**"}}, + "sub": {URL: "https://git.example/r/"}, + "nested": {URL: "https://git.example/r/deep"}, + }}) + tests := []struct { + location string + ok bool + key, repoURL string + }{ + {"github.com/acme/widgets", true, h.upstreams["github"].dir + "/acme/widgets.git", "https://github.com/acme/widgets"}, + {"github.com/acme/widgets.git", true, h.upstreams["github"].dir + "/acme/widgets.git", "https://github.com/acme/widgets.git"}, + {"gitlab.wikimedia.org/mediawiki/skins/Vector", true, h.upstreams["wikimedia"].dir + "/mediawiki/skins/Vector.git", "https://gitlab.wikimedia.org/mediawiki/skins/Vector"}, + {"gitlab.wikimedia.org/other/repo", false, "", ""}, + {"git.example/r/a", true, h.upstreams["sub"].dir + "/a.git", "https://git.example/r/a"}, + // The longest matching upstream URL wins. + {"git.example/r/deep/a", true, h.upstreams["nested"].dir + "/a.git", "https://git.example/r/deep/a"}, + {"git.example/other", false, "", ""}, + {"git.example/r", false, "", ""}, + {"github.com.evil/acme/widgets", false, "", ""}, + {"github.com/acme/../widgets", false, "", ""}, + {"codeberg.org/forgejo/forgejo", false, "", ""}, + } + for _, tt := range tests { + t.Run(tt.location, func(t *testing.T) { + repo, err := h.mirrorRepository(tt.location) + if (err == nil) != tt.ok { + t.Fatalf("error %v, want ok %v", err, tt.ok) + } + if tt.ok && (repo.key != tt.key || repo.url != tt.repoURL) { + t.Errorf("key %q, URL %q; want %q, %q", repo.key, repo.url, tt.key, tt.repoURL) + } + }) + } +} + +func TestGitMirrorCachesCommitForClones(t *testing.T) { + g := newGitTestSetup(t, 0, 0) + sha := runGitCommand(t, g.work, "rev-parse", "HEAD") + location := strings.TrimPrefix(g.upstreamURL, "http://") + "/repo" + + got, err := g.handler.Mirror(context.Background(), location, sha) + if err != nil { + t.Fatalf("Mirror: %v", err) + } + if got.Cached { + t.Errorf("first mirror = %+v, want fetched", got) + } + got, err = g.handler.Mirror(context.Background(), location, sha) + if err != nil || !got.Cached { + t.Errorf("second mirror = %+v, %v, want cached", got, err) + } + + // With the upstream gone, a clone through the proxy gets the mirrored copy. + g.upstream.Close() + clone := t.TempDir() + runGitCommand(t, "", "clone", "--quiet", g.url(), clone) + if head := runGitCommand(t, clone, "rev-parse", "HEAD"); head != sha { + t.Errorf("cloned HEAD %s, want %s", head, sha) + } +} diff --git a/internal/handler/git_test.go b/internal/handler/git_test.go index 0aeff4b..cf553d6 100644 --- a/internal/handler/git_test.go +++ b/internal/handler/git_test.go @@ -131,7 +131,12 @@ func runGitCommand(t *testing.T, dir string, args ...string) string { type gitTestSetup struct { work string server *httptest.Server - logs *syncLog + // upstream serves the repository, at upstreamURL/repo, and handler is + // the GitHandler under test. + upstream *httptest.Server + upstreamURL string + handler *GitHandler + logs *syncLog // slow delays every upstream request by half a second. slow atomic.Bool // active counts the requests the handler is serving. @@ -172,6 +177,7 @@ func newGitTestSetup(t *testing.T, readTimeout, writeStall time.Duration) *gitTe backend.ServeHTTP(w, r) })) t.Cleanup(upstream.Close) + g.upstream, g.upstreamURL = upstream, upstream.URL logger := slog.New(slog.NewTextHandler(g.logs, nil)) cache, err := gitcache.New(gitcache.Options{ @@ -190,6 +196,7 @@ func newGitTestSetup(t *testing.T, readTimeout, writeStall time.Duration) *gitTe if writeStall > 0 { h.writeStall = writeStall } + g.handler = h routes := http.StripPrefix("/git", h.Routes()) g.server = httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { g.active.Add(1) diff --git a/internal/mirror/git.go b/internal/mirror/git.go new file mode 100644 index 0000000..a4908dc --- /dev/null +++ b/internal/mirror/git.go @@ -0,0 +1,49 @@ +package mirror + +import ( + "context" + "errors" + + "github.com/git-pkgs/proxy/internal/gitcache" +) + +// GitMirrorer caches Git repositories for pkg:git items. location is the +// repository's host and path, such as codeberg.org/forgejo/forgejo, and +// commit a full commit ID, or empty to cache the repository's branches and +// tags only. +type GitMirrorer interface { + Mirror(ctx context.Context, location, commit string) (gitcache.MirrorResult, error) +} + +// UnavailableGit fails every pkg:git item with Reason. +type UnavailableGit struct { + Reason string +} + +func (u UnavailableGit) Mirror(context.Context, string, string) (gitcache.MirrorResult, error) { + return gitcache.MirrorResult{}, errors.New(u.Reason) +} + +var errNoGitUpstream = errors.New("git repositories are mirrored from upstream.git, and none is configured") + +// mirrorGit caches a pkg:git repository, and the commit its version names, +// in the cache /git serves clones from. +func (m *Mirror) mirrorGit(ctx context.Context, pv PackageVersion, tracker *progressTracker) { + if m.Git == nil { + m.recordFailure(tracker, pv, errNoGitUpstream) + return + } + result, err := m.Git.Mirror(ctx, pv.Name, pv.Version) + if err != nil { + m.recordFailure(tracker, pv, err) + return + } + if result.Cached { + tracker.skipped.Add(1) + m.logger.Debug("already cached", "ecosystem", pv.Ecosystem, "name", pv.Name, "version", pv.Version) + return + } + tracker.completed.Add(1) + tracker.bytes.Add(result.Bytes) + m.logger.Info("mirrored", "ecosystem", pv.Ecosystem, "name", pv.Name, "version", pv.Version, "size", result.Bytes) +} diff --git a/internal/mirror/git_test.go b/internal/mirror/git_test.go new file mode 100644 index 0000000..aa3aed4 --- /dev/null +++ b/internal/mirror/git_test.go @@ -0,0 +1,92 @@ +package mirror + +import ( + "context" + "errors" + "sync" + "testing" + + "github.com/git-pkgs/proxy/internal/gitcache" +) + +type gitCall struct{ location, commit string } + +type fakeGit struct { + mu sync.Mutex + calls []gitCall + result gitcache.MirrorResult + err error +} + +func (f *fakeGit) Mirror(_ context.Context, location, commit string) (gitcache.MirrorResult, error) { + f.mu.Lock() + defer f.mu.Unlock() + f.calls = append(f.calls, gitCall{location, commit}) + return f.result, f.err +} + +const gitTestCommit = "a72d2c07cfca03b55371089de6aa230d8c951fa0" + +func TestMirrorRunCountsGitRepositories(t *testing.T) { + tests := []struct { + name string + result gitcache.MirrorResult + err error + wantCompleted, wantSkipped, wantFailed int64 + }{ + {name: "fetched", result: gitcache.MirrorResult{Bytes: 42}, wantCompleted: 1}, + {name: "cached", result: gitcache.MirrorResult{Cached: true}, wantSkipped: 1}, + {name: "failed", err: errors.New("boom"), wantFailed: 1}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + m := setupTestMirror(t, 1) + git := &fakeGit{result: tt.result, err: tt.err} + m.Git = git + progress, err := m.Run(context.Background(), &PURLSource{PURLs: []string{"pkg:git/codeberg.org/forgejo/forgejo@" + gitTestCommit}}) + if err != nil { + t.Fatalf("Run: %v", err) + } + if progress.Completed != tt.wantCompleted || progress.Skipped != tt.wantSkipped || progress.Failed != tt.wantFailed { + t.Errorf("progress = %+v", progress) + } + if progress.Bytes != tt.result.Bytes { + t.Errorf("bytes = %d, want %d", progress.Bytes, tt.result.Bytes) + } + want := gitCall{"codeberg.org/forgejo/forgejo", gitTestCommit} + if len(git.calls) != 1 || git.calls[0] != want { + t.Errorf("calls = %+v, want %+v", git.calls, want) + } + }) + } +} + +// TestMirrorRunGitRepositoryWithoutCommit checks a pkg:git PURL without a +// version mirrors the repository instead of listing versions. +func TestMirrorRunGitRepositoryWithoutCommit(t *testing.T) { + m := setupTestMirror(t, 1) + git := &fakeGit{} + m.Git = git + versions := &fakeVersions{err: errors.New("listed versions")} + progress, err := m.Run(context.Background(), &PURLSource{PURLs: []string{"pkg:git/codeberg.org/forgejo/forgejo"}, Versions: versions}) + if err != nil { + t.Fatalf("Run: %v", err) + } + if len(versions.asked) != 0 { + t.Errorf("listed versions of %v", versions.asked) + } + if progress.Failed != 0 || len(git.calls) != 1 || git.calls[0] != (gitCall{"codeberg.org/forgejo/forgejo", ""}) { + t.Errorf("progress = %+v, calls = %+v", progress, git.calls) + } +} + +func TestMirrorRunGitWithoutUpstream(t *testing.T) { + m := setupTestMirror(t, 1) + progress, err := m.Run(context.Background(), &PURLSource{PURLs: []string{"pkg:git/codeberg.org/forgejo/forgejo@" + gitTestCommit}}) + if err != nil { + t.Fatalf("Run: %v", err) + } + if progress.Failed != 1 { + t.Errorf("progress = %+v, want one failure", progress) + } +} diff --git a/internal/mirror/mirror.go b/internal/mirror/mirror.go index d5ec9a5..c08b1fb 100644 --- a/internal/mirror/mirror.go +++ b/internal/mirror/mirror.go @@ -32,6 +32,10 @@ type Mirror struct { // Charts mirrors helm charts. Without it they fail. Charts ChartMirrorer + // Git mirrors pkg:git repositories into the upstream.git cache. Without + // it they fail. + Git GitMirrorer + // Bottles resolves Homebrew formula versions to the images their // bottles are published under, which Images mirrors. Without it brew // PURLs fail. @@ -279,6 +283,10 @@ func (m *Mirror) mirrorOne(ctx context.Context, pv PackageVersion, tracker *prog m.mirrorAPK(ctx, pv, tracker) return } + if pv.Ecosystem == "git" { + m.mirrorGit(ctx, pv, tracker) + return + } result, err := m.proxy.GetOrFetchArtifact(ctx, pv.Ecosystem, pv.Name, pv.Version, "") if err != nil { m.recordFailure(tracker, pv, err) diff --git a/internal/mirror/source.go b/internal/mirror/source.go index 2c41705..c0441c4 100644 --- a/internal/mirror/source.go +++ b/internal/mirror/source.go @@ -70,8 +70,9 @@ func (s *PURLSource) Enumerate(ctx context.Context, fn func(PackageVersion) erro } // Images are never expanded to all tags, and conda packages without - // a channel aren't listed; both fail when they are mirrored. - if p.Version != "" || isImageEcosystem(ecosystem) || (ecosystem == "conda" && !strings.Contains(name, "/")) { + // a channel aren't listed; both fail when they are mirrored. A Git + // repository without a commit is mirrored as a whole. + if p.Version != "" || isImageEcosystem(ecosystem) || ecosystem == "git" || (ecosystem == "conda" && !strings.Contains(name, "/")) { pv := PackageVersion{Ecosystem: ecosystem, Name: name, Version: p.Version, Namespace: namespace, Qualifiers: qualifiers} if isImageEcosystem(ecosystem) { pv.Qualifiers = p.Qualifiers.Map() diff --git a/internal/server/git_test.go b/internal/server/git_test.go index d52f1a9..e59f30d 100644 --- a/internal/server/git_test.go +++ b/internal/server/git_test.go @@ -3,6 +3,7 @@ package server import ( "bytes" "context" + "encoding/json" "errors" "log/slog" "net" @@ -18,6 +19,7 @@ import ( "time" "github.com/git-pkgs/proxy/internal/config" + "github.com/git-pkgs/proxy/internal/mirror" ) func gitTestEnv(home string) []string { @@ -75,11 +77,11 @@ func (b *syncBuffer) String() string { return b.buf.String() } -// TestGitRouteServesClonesWithUpstreamCredentials clones through the whole -// server, with credentials from upstream.auth that only the proxy holds. -func TestGitRouteServesClonesWithUpstreamCredentials(t *testing.T) { - root := t.TempDir() - bare := filepath.Join(root, "team", "repo.git") +// newGitTestRepo creates the bare repository {root}/{path}.git with one +// commit on main and returns the commit. +func newGitTestRepo(t *testing.T, root, path string) string { + t.Helper() + bare := filepath.Join(root, filepath.FromSlash(path)+".git") gitCommand(t, "", "init", "--quiet", "--bare", "--initial-branch=main", bare) work := t.TempDir() gitCommand(t, "", "clone", "--quiet", bare, work) @@ -89,21 +91,22 @@ func TestGitRouteServesClonesWithUpstreamCredentials(t *testing.T) { gitCommand(t, work, "add", "file.txt") gitCommand(t, work, "commit", "--quiet", "-m", "first") gitCommand(t, work, "push", "--quiet", "origin", "HEAD:main") - head := gitCommand(t, work, "rev-parse", "HEAD") + return gitCommand(t, work, "rev-parse", "HEAD") +} - backend := &cgi.Handler{ +// gitBackend serves the repositories under root with git http-backend. +func gitBackend(t *testing.T, root string) http.Handler { + t.Helper() + return &cgi.Handler{ Path: gitHTTPBackend(t), Env: append(gitTestEnv(t.TempDir()), "GIT_PROJECT_ROOT="+root, "GIT_HTTP_EXPORT_ALL=1"), } - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.Header.Get("Authorization") != "Bearer git-secret" { - http.Error(w, "unauthorized", http.StatusUnauthorized) - return - } - backend.ServeHTTP(w, r) - })) - defer upstream.Close() +} +// startGitTestServer starts the whole server with the git cache, after +// configure has set up cfg, and returns cfg and the server's log. +func startGitTestServer(t *testing.T, configure func(*config.Config)) (*config.Config, *syncBuffer) { + t.Helper() listener, err := net.Listen("tcp", "127.0.0.1:0") if err != nil { t.Fatal(err) @@ -115,21 +118,20 @@ func TestGitRouteServesClonesWithUpstreamCredentials(t *testing.T) { cfg.Database.Path = filepath.Join(tempDir, "proxy.db") cfg.Storage.URL = "file://" + filepath.Join(tempDir, "artifacts") cfg.Upstream.AllowLoopback = true - cfg.Upstream.Git = map[string]config.GitUpstream{"private": {URL: upstream.URL, Repos: []string{"team/*"}}} - cfg.Upstream.Auth = map[string]config.AuthConfig{upstream.URL: {Type: "bearer", Token: "git-secret"}} cfg.Git.CacheDir = filepath.Join(tempDir, "git") + configure(cfg) if err := cfg.Validate(); err != nil { t.Fatalf("validating config: %v", err) } - var logs syncBuffer - proxyServer, err := New(cfg, slog.New(slog.NewTextHandler(&logs, nil)), BuildInfo{Version: "test", Commit: "test"}) + logs := &syncBuffer{} + proxyServer, err := New(cfg, slog.New(slog.NewTextHandler(logs, nil)), BuildInfo{Version: "test", Commit: "test"}) if err != nil { t.Fatalf("creating server: %v", err) } startErr := make(chan error, 1) go func() { startErr <- proxyServer.Start(listener) }() - defer func() { + t.Cleanup(func() { ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() if err := proxyServer.Shutdown(ctx); err != nil { @@ -138,7 +140,29 @@ func TestGitRouteServesClonesWithUpstreamCredentials(t *testing.T) { if err := <-startErr; !errors.Is(err, http.ErrServerClosed) { t.Errorf("Start() error = %v, want %v", err, http.ErrServerClosed) } - }() + }) + return cfg, logs +} + +// TestGitRouteServesClonesWithUpstreamCredentials clones through the whole +// server, with credentials from upstream.auth that only the proxy holds. +func TestGitRouteServesClonesWithUpstreamCredentials(t *testing.T) { + root := t.TempDir() + head := newGitTestRepo(t, root, "team/repo") + backend := gitBackend(t, root) + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("Authorization") != "Bearer git-secret" { + http.Error(w, "unauthorized", http.StatusUnauthorized) + return + } + backend.ServeHTTP(w, r) + })) + defer upstream.Close() + + cfg, logs := startGitTestServer(t, func(cfg *config.Config) { + cfg.Upstream.Git = map[string]config.GitUpstream{"private": {URL: upstream.URL, Repos: []string{"team/*"}}} + cfg.Upstream.Auth = map[string]config.AuthConfig{upstream.URL: {Type: "bearer", Token: "git-secret"}} + }) clone := filepath.Join(t.TempDir(), "clone") gitCommand(t, "", "clone", "--quiet", cfg.BaseURL+"/git/private/team/repo.git", clone) @@ -158,3 +182,60 @@ func TestGitRouteServesClonesWithUpstreamCredentials(t *testing.T) { t.Errorf("repository outside repos: status %d, want 404", resp.StatusCode) } } + +// TestMirrorAPICachesGitCommit mirrors a pkg:git commit through the mirror +// API, then clones it through /git while the upstream is down. +func TestMirrorAPICachesGitCommit(t *testing.T) { + root := t.TempDir() + head := newGitTestRepo(t, root, "team/repo") + upstream := httptest.NewServer(gitBackend(t, root)) + defer upstream.Close() + cfg, _ := startGitTestServer(t, func(cfg *config.Config) { + cfg.MirrorAPI = true + cfg.Upstream.Git = map[string]config.GitUpstream{"up": {URL: upstream.URL}} + }) + + location := strings.TrimPrefix(upstream.URL, "http://") + "/team/repo" + body, err := json.Marshal(map[string][]string{"purls": {"pkg:git/" + location + "@" + head}}) + if err != nil { + t.Fatal(err) + } + resp, err := http.Post(cfg.BaseURL+"/api/mirror", "application/json", bytes.NewReader(body)) + if err != nil { + t.Fatal(err) + } + var created struct{ ID string } + err = json.NewDecoder(resp.Body).Decode(&created) + _ = resp.Body.Close() + if err != nil || resp.StatusCode != http.StatusAccepted { + t.Fatalf("creating job: status %d, %v", resp.StatusCode, err) + } + + var job mirror.Job + deadline := time.Now().Add(30 * time.Second) + for job.State != mirror.JobStateComplete && job.State != mirror.JobStateFailed && job.State != mirror.JobStateCanceled { + if time.Now().After(deadline) { + t.Fatalf("job did not finish: %+v", job) + } + time.Sleep(50 * time.Millisecond) + resp, err := http.Get(cfg.BaseURL + "/api/mirror/" + created.ID) + if err != nil { + t.Fatal(err) + } + err = json.NewDecoder(resp.Body).Decode(&job) + _ = resp.Body.Close() + if err != nil { + t.Fatal(err) + } + } + if job.State != mirror.JobStateComplete || job.Progress.Completed != 1 { + t.Fatalf("job = %+v, want one repository mirrored", job) + } + + upstream.Close() + clone := filepath.Join(t.TempDir(), "clone") + gitCommand(t, "", "clone", "--quiet", cfg.BaseURL+"/git/up/team/repo.git", clone) + if got := gitCommand(t, clone, "rev-parse", "HEAD"); got != head { + t.Errorf("HEAD = %s, want %s", got, head) + } +} diff --git a/internal/server/mirror_versions.go b/internal/server/mirror_versions.go index 6d4599e..81491d9 100644 --- a/internal/server/mirror_versions.go +++ b/internal/server/mirror_versions.go @@ -32,6 +32,9 @@ func (s *Server) newMirror(proxy *handler.Proxy, workers int) *mirror.Mirror { m.Images = s.newContainerHandler(proxy) m.Bottles = handler.NewHomebrewHandler(proxy, s.cfg.Upstream.HomebrewAPI) m.Charts = handler.NewHelmHandler(proxy, s.cfg.BaseURL, s.cfg.Upstream.Helm) + if s.git != nil { + m.Git = handler.NewGitHandler(proxy, s.git, s.cfg.Upstream) + } UseMirrorUpstreams(m, s.cfg.Upstream, proxy.HTTPClient) return m } diff --git a/internal/server/server.go b/internal/server/server.go index 55fe597..86b4a4c 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -626,6 +626,21 @@ func NewMirrorProxy(cfg *config.Config, db *database.DB, store storage.Storage, return proxy, s.newContainerHandler(proxy) } +// NewMirrorGit opens the git cache for proxy mirror, with the upstream client +// serve uses, and returns the handler that mirrors pkg:git items into it and +// the function that closes it. Both are nil without upstream.git. +func NewMirrorGit(cfg *config.Config, proxy *handler.Proxy, logger *slog.Logger) (*handler.GitHandler, func() error, error) { + if len(cfg.Upstream.Git) == 0 { + return nil, nil, nil + } + s := &Server{cfg: cfg, logger: logger, ecr: newECRTokens(logger)} + _, artifactClient := s.newUpstreamClients(0) + if err := s.configureGitCache(artifactClient); err != nil { + return nil, nil, err + } + return handler.NewGitHandler(proxy, s.git, cfg.Upstream), s.git.Close, nil +} + // newContainerHandler builds the OCI handler with the configured default // registry, named upstreams, mirrored registries and Homebrew routes. func (s *Server) newContainerHandler(proxy *handler.Proxy) *handler.ContainerHandler { From 0ed91b5872ef9c1cfa6785b0bde4783a02b4de5d Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Fri, 6 Nov 2026 09:20:43 +0000 Subject: [PATCH 566/633] Redact storage passwords in startup logs --- internal/server/server.go | 6 ++- internal/server/server_test.go | 89 ++++++++++++++++++++++++++++++++++ 2 files changed, 94 insertions(+), 1 deletion(-) diff --git a/internal/server/server.go b/internal/server/server.go index 8599c48..b6c67e4 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -436,11 +436,15 @@ func (s *Server) serve(listener net.Listener) error { IdleTimeout: serverIdleTimeout, } + storageURL := "[redacted]" + if u, err := url.Parse(s.storage.URL()); err == nil { + storageURL = u.Redacted() + } s.logger.Info("starting server", "listen", s.cfg.Listen, "base_url", s.cfg.BaseURL, "ui_url", s.cfg.UIBaseURL, - "storage", s.storage.URL(), + "storage", storageURL, "database", s.cfg.Database.String()) for _, warning := range s.cfg.EnvWarnings() { s.logger.Warn(warning) diff --git a/internal/server/server_test.go b/internal/server/server_test.go index 5fa49e9..02d781d 100644 --- a/internal/server/server_test.go +++ b/internal/server/server_test.go @@ -150,6 +150,95 @@ func (ts *testServer) close() { _ = os.RemoveAll(ts.tempDir) } +type storageWithURL struct { + storage.Storage + url string +} + +func (s storageWithURL) URL() string { return s.url } + +func TestStartRedactsStoragePassword(t *testing.T) { + for _, tc := range []struct { + name string + url string + want string + }{ + {"password", "s3://user:secret@bucket?region=us-east-1", "s3://user:xxxxx@bucket?region=us-east-1"}, + {"escaped password", "s3://user:secret%40word@bucket", "s3://user:xxxxx@bucket"}, + {"no password", "s3://bucket?region=us-east-1", "s3://bucket?region=us-east-1"}, + {"invalid URL", "s3://user:secret%zz@bucket", "[redacted]"}, + } { + t.Run(tc.name, func(t *testing.T) { + output := startStorageLog(t, tc.url) + if strings.Contains(output, "secret") { + t.Errorf("startup log contains storage password: %s", output) + } + decoder := json.NewDecoder(strings.NewReader(output)) + for decoder.More() { + var entry struct { + Message string `json:"msg"` + Storage string `json:"storage"` + } + if err := decoder.Decode(&entry); err != nil { + t.Fatal(err) + } + if entry.Message == "starting server" { + if entry.Storage != tc.want { + t.Errorf("logged storage = %q, want %q", entry.Storage, tc.want) + } + t.Logf("logged storage = %q", entry.Storage) + return + } + } + t.Fatal("startup log missing") + }) + } +} + +func startStorageLog(t *testing.T, storageURL string) string { + t.Helper() + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = listener.Close() }) + + cfg := config.Default() + cfg.Listen = listener.Addr().String() + cfg.BaseURL = "http://" + cfg.Listen + cfg.Database.Path = filepath.Join(t.TempDir(), "proxy.db") + cfg.Storage.URL = "file://" + filepath.Join(t.TempDir(), "artifacts") + var logs syncBuffer + srv, err := New(cfg, slog.New(slog.NewJSONHandler(&logs, nil)), BuildInfo{}) + if err != nil { + t.Fatal(err) + } + srv.storage = storageWithURL{Storage: srv.storage, url: storageURL} + done := make(chan error, 1) + go func() { done <- srv.Start(listener) }() + t.Cleanup(func() { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if err := srv.Shutdown(ctx); err != nil { + t.Errorf("shutting down server: %v", err) + } + if err := <-done; !errors.Is(err, http.ErrServerClosed) { + t.Errorf("Start() = %v, want %v", err, http.ErrServerClosed) + } + }) + + client := &http.Client{Timeout: 5 * time.Second} + resp, err := client.Get(cfg.BaseURL + "/health") + if err != nil { + t.Fatal(err) + } + _ = resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Fatalf("health status = %d, want 200", resp.StatusCode) + } + return logs.String() +} + func TestUpstreamSafeHTTPOptions(t *testing.T) { opts := upstreamSafeHTTPOptions(config.UpstreamConfig{ AllowPrivateHosts: []string{"registry.internal"}, From c2dd16903a9c078dc2f09ea15208916234f30288 Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Fri, 6 Nov 2026 09:38:07 +0000 Subject: [PATCH 567/633] Share publish indexes across cooldown profiles --- internal/handler/download_cooldown.go | 2 +- internal/handler/profile_test.go | 121 ++++++++++++++++++++++++++ 2 files changed, 122 insertions(+), 1 deletion(-) diff --git a/internal/handler/download_cooldown.go b/internal/handler/download_cooldown.go index c491ef3..4ede9a8 100644 --- a/internal/handler/download_cooldown.go +++ b/internal/handler/download_cooldown.go @@ -103,7 +103,7 @@ type publishIndexEntry struct { // publishIndex reuses parsed documents by digest, or for a minute without one. func publishIndex[V any](p *Proxy, key string, load func() metadataResult, build func(metadataResult) (V, error)) (V, error) { //nolint:ireturn // Supports timestamp maps and Helm lookups. - c := &p.publishIndexes + c := &p.sharedProxy().publishIndexes fresh := func() (V, bool) { c.mu.Lock() defer c.mu.Unlock() diff --git a/internal/handler/profile_test.go b/internal/handler/profile_test.go index 6377062..5caf7b8 100644 --- a/internal/handler/profile_test.go +++ b/internal/handler/profile_test.go @@ -2,16 +2,137 @@ package handler import ( "context" + "errors" "net/http" "net/http/httptest" + "reflect" "strings" "sync" + "sync/atomic" "testing" "time" "github.com/git-pkgs/cooldown" + "github.com/git-pkgs/proxy/internal/denylist" + "github.com/git-pkgs/proxy/internal/scanner" + registryclient "github.com/git-pkgs/registries/client" ) +func TestWithCooldownCopiesProxyFields(t *testing.T) { + p, _, _, _ := setupTestProxy(t) + p.RouteFor = func(string, string) *ArtifactRoute { return nil } + p.Cooldown = profilePolicy(t, "1d") + p.ShortestCooldown = 24 * time.Hour + p.Denylist = &denylist.Policy{} + p.CacheMetadata = true + p.MetadataTTL = time.Hour + p.MetadataMaxSize = 1024 + p.MetadataStreamMaxSize = 2048 + p.GradleReadOnly = true + p.HostedEnabled = true + p.GradleMaxUploadSize = 4096 + p.NPMFullMetadata = true + p.DirectServe = true + p.DirectServeTTL = time.Minute + p.DirectServeBaseURL = "https://storage.example.com" + p.RateLimits = registryclient.NewRateLimitGate(http.DefaultTransport) + p.HostedNPMScopes = []string{"@example"} + p.HostedComposerVendors = []string{"example"} + p.StreamArtifacts = true + p.Scanners = &scanner.Group{} + p.ScanSigningKey = []byte("signing-key") + p.ScanFetchBaseURL = "http://scanner.example.com" + p.SetMetadataRewriteCacheSize(1024) + p.inFlight = map[string]*inflightFetch{"artifact": {}} + p.detachSlots = make(chan struct{}, 1) + p.inFlightMeta = map[string]*inflightMetadata{"metadata": {}} + p.publishIndexes.entries = map[string]publishIndexEntry{"document": {}} + p.cooldownWatch.active = map[*cooldownRecorder]struct{}{{}: {}} + + policy := profilePolicy(t, "3d") + view := p.WithCooldown(policy, 72*time.Hour) + if view.Cooldown != policy || view.ShortestCooldown != 72*time.Hour || view.shared != p { + t.Fatal("view did not select its policy and shared proxy") + } + original, copied := reflect.ValueOf(p).Elem(), reflect.ValueOf(view).Elem() + for i := range original.NumField() { + name := original.Type().Field(i).Name + before, after := original.Field(i), copied.Field(i) + switch name { + case "Cooldown", "ShortestCooldown", "shared": + continue + case "fetchMu", "inFlight", "detachSlots", "metaMu", "inFlightMeta", "publishIndexes", "cooldownWatch": + if !after.IsZero() { + t.Errorf("view field %s is not empty", name) + } + continue + } + if before.IsZero() { + t.Errorf("populate Proxy.%s to check whether WithCooldown copies it", name) + continue + } + switch before.Kind() { + case reflect.Func, reflect.Slice: + if before.Pointer() != after.Pointer() { + t.Errorf("WithCooldown did not copy %s", name) + } + if before.Kind() == reflect.Slice && (before.Len() != after.Len() || before.Cap() != after.Cap()) { + t.Errorf("WithCooldown changed slice bounds for %s", name) + } + default: + if !before.Equal(after) { + t.Errorf("WithCooldown did not copy %s", name) + } + } + } +} + +func TestProfilesShareDownloadPublishIndexes(t *testing.T) { + for _, c := range downloadCooldownCases() { + if c.ecosystem != "conda" && c.ecosystem != "helm" { + continue + } + t.Run(c.ecosystem, func(t *testing.T) { + var calls atomic.Int64 + times := map[string]time.Time{cooldownRecentVersion: time.Now().Add(-4 * 24 * time.Hour)} + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls.Add(1) + c.metadata(w, r, times) + })) + t.Cleanup(upstream.Close) + p, _, _, fetcher := setupTestProxy(t) + p.HTTPClient = upstream.Client() + p.Cooldown = profilePolicy(t, "1d") + fetcher.fetchErr = errors.New("artifact must be served from the cache") + c.seedCachedArtifact(t, p, cooldownRecentVersion) + permissive := p.WithCooldown(profilePolicy(t, "3d"), 72*time.Hour) + strict := p.WithCooldown(profilePolicy(t, "7d"), 168*time.Hour) + nested := permissive.WithCooldown(profilePolicy(t, "5d"), 120*time.Hour) + for _, request := range []struct { + proxy *Proxy + want int + }{ + {permissive, http.StatusOK}, + {strict, http.StatusNotFound}, + {p, http.StatusOK}, + {nested, http.StatusNotFound}, + } { + w := serveGet(t, c.handler(request.proxy, upstream.URL), c.path(cooldownRecentVersion)) + wantStatus(t, w, request.want, "") + } + if n := calls.Load(); n != 1 { + t.Errorf("index downloads across profiles = %d, want 1", n) + } + if n := len(p.publishIndexes.entries); n != 1 { + t.Errorf("shared publish indexes = %d, want 1", n) + } + if fetcher.fetchCalled { + t.Error("cached artifact was fetched again") + } + }) + } +} + func profilePolicy(t *testing.T, duration string) *cooldown.Policy { t.Helper() policy, err := cooldown.NewPolicy(&cooldown.Config{Default: duration}, nil) From abe94017a1bfb6e1547e21b1cf43424db023c057 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Fri, 6 Nov 2026 11:36:42 +0100 Subject: [PATCH 568/633] Renew wanted refs of mirrored commits and count refreshes that fetched Mirroring a commit on no branch or tag again writes a fresh wanted ref, so its 7 days count from the last mirror run. A cached repository whose refresh fetched new commits is no longer reported as cached. --- docs/configuration.md | 2 +- internal/gitcache/mirror.go | 53 +++++++++++++++------ internal/gitcache/mirror_test.go | 79 ++++++++++++++++++++++++++++++++ internal/gitcache/wants.go | 29 ++++++++---- internal/handler/git_mirror.go | 4 +- 5 files changed, 140 insertions(+), 27 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index b2b830f..540078c 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1887,7 +1887,7 @@ proxy mirror pkg:git/github.com/git-pkgs/purl ``` - The repository is served by the `upstream.git` host whose URL its host and path start with, the longest one if several do, and has to match that host's `repos`. Others fail. -- The repository's branches and tags are cached, as a clone through the proxy caches them. A commit on none of them is fetched by ID and kept, like one a client fetches by ID, for seven days or until a branch or tag includes it. Where `upstream.auth` has credentials for the host, commits aren't fetched by ID, and one on no branch or tag fails. +- The repository's branches and tags are cached, as a clone through the proxy caches them. A commit on none of them is fetched by ID and kept, like one a client fetches by ID, until a branch or tag includes it, or for seven days from the last mirror run that named it. Where `upstream.auth` has credentials for the host, commits aren't fetched by ID, and one on no branch or tag fails. - Without a version the repository's branches and tags are mirrored. A tag or branch name as the version fails; give the commit ID. - The git cache is used by one process. While the server runs, the command can't open `git.cache_dir`, so `pkg:git` items fail and the rest are mirrored; use the mirror API instead. diff --git a/internal/gitcache/mirror.go b/internal/gitcache/mirror.go index 854ea08..62ae932 100644 --- a/internal/gitcache/mirror.go +++ b/internal/gitcache/mirror.go @@ -4,6 +4,8 @@ import ( "context" "errors" "fmt" + "strings" + "time" ) // MirrorResult is what Mirror did. @@ -47,42 +49,65 @@ func (c *Cache) Mirror(ctx context.Context, key, repoURL, commit string, fetchWa } if commit != "" { - fetched, err := c.ensureCommit(ctx, rp, repoURL, commit, fetchWants) - if err != nil { + if err := c.ensureCommit(ctx, rp, repoURL, commit, fetchWants); err != nil { return MirrorResult{}, err } - cached = cached && !fetched } // A refresh that failed leaves the commit cached all the same. if refreshErr != nil && commit == "" { return MirrorResult{}, refreshErr } - return MirrorResult{Cached: cached, Bytes: max(rp.size.Load()-before, 0)}, nil + grown := max(rp.size.Load()-before, 0) + return MirrorResult{Cached: cached && grown == 0, Bytes: grown}, nil } // ensureCommit makes sure the repository has commit, fetching it by ID when -// it doesn't and fetchWants allows it, and reports whether it was fetched. -func (c *Cache) ensureCommit(ctx context.Context, rp *repo, repoURL, commit string, fetchWants bool) (bool, error) { +// it doesn't and fetchWants allows it. A commit on no branch or tag that is +// cached already gets a new wanted ref, so it is kept for wantedMaxAge from +// this run on. +func (c *Cache) ensureCommit(ctx context.Context, rp *repo, repoURL, commit string, fetchWants bool) error { missing, err := c.missingObjects(ctx, rp.dir, []string{commit}) - if err != nil || len(missing) == 0 { - return false, err + if err != nil { + return err + } + if len(missing) == 0 { + return c.renewWanted(ctx, rp, commit) } if !fetchWants { - return false, fmt.Errorf("commit %s: %w", commit, ErrNotFound) + return fmt.Errorf("commit %s: %w", commit, ErrNotFound) } if err := c.fetchMissingWants(ctx, rp, repoURL, missing); err != nil { if ctx.Err() != nil || errors.Is(err, ErrTooLarge) || errors.Is(err, ErrCacheFull) || errors.Is(err, errUpstreamUnanswered) { - return false, err + return err } // The upstream refuses a commit it doesn't have. - return false, fmt.Errorf("commit %s: %w: %w", commit, ErrNotFound, err) + return fmt.Errorf("commit %s: %w: %w", commit, ErrNotFound, err) } // fetchMissingWants skips a commit that failed moments ago. if missing, err = c.missingObjects(ctx, rp.dir, missing); err != nil { - return false, err + return err } if len(missing) > 0 { - return false, fmt.Errorf("commit %s: %w", commit, ErrNotFound) + return fmt.Errorf("commit %s: %w", commit, ErrNotFound) } - return true, nil + return nil +} + +// renewWanted writes a new wanted ref for commit when no branch or tag +// includes it. +func (c *Cache) renewWanted(ctx context.Context, rp *repo, commit string) error { + unlock, err := rp.lockWrites(ctx) + if err != nil { + return err + } + defer unlock() + unmerged, err := c.unmergedCommits(ctx, rp.dir, []string{commit}) + if err != nil || !unmerged[commit] { + return err + } + cmd := c.command(ctx, nil, "-C", rp.dir, "update-ref", wantedRef(commit, time.Now()), commit) + if out, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("renewing wanted ref: %w: %s", err, strings.TrimSpace(string(out))) + } + return nil } diff --git a/internal/gitcache/mirror_test.go b/internal/gitcache/mirror_test.go index a1fd07e..f87b322 100644 --- a/internal/gitcache/mirror_test.go +++ b/internal/gitcache/mirror_test.go @@ -3,8 +3,12 @@ package gitcache import ( "context" "errors" + "path/filepath" + "slices" + "strconv" "strings" "testing" + "time" ) func TestMirrorClonesRepository(t *testing.T) { @@ -106,3 +110,78 @@ func TestMirrorRefusesInvalidCommit(t *testing.T) { t.Error("mirrored a branch name as a commit") } } + +func TestMirrorCountsRefreshThatFetched(t *testing.T) { + upstream := newFakeUpstream(t) + work := upstream.newRepo("repo") + tc := newTestCache(t, upstream.server.URL, nil) + if _, err := tc.Mirror(context.Background(), "up/repo.git", upstream.server.URL+"/repo", "", true); err != nil { + t.Fatalf("Mirror: %v", err) + } + upstream.commit(work, "second") + runGit(t, work, "push", "--quiet", "origin", "main") + + got, err := tc.Mirror(context.Background(), "up/repo.git", upstream.server.URL+"/repo", "", true) + if err != nil { + t.Fatalf("Mirror: %v", err) + } + if got.Cached || got.Bytes <= 0 { + t.Errorf("mirror after a push = %+v, want fetched bytes", got) + } +} + +// TestMirrorRenewsWantedCommit checks mirroring a commit on no branch again +// makes its 7 days count from that run, not from the first fetch. +func TestMirrorRenewsWantedCommit(t *testing.T) { + upstream := newFakeUpstream(t) + work := upstream.newRepo("repo") + runGit(t, upstream.bare(), "config", "uploadpack.allowAnySHA1InWant", "true") + sha := upstream.pushPullRequest(work, 1, nil) + tc := newTestCache(t, upstream.server.URL, nil) + if _, err := tc.Mirror(context.Background(), "up/repo.git", upstream.server.URL+"/repo", sha, true); err != nil { + t.Fatalf("Mirror: %v", err) + } + cached := filepath.Join(tc.dir, "up", "repo.git") + for _, ref := range tc.wantedRefs(t) { + runGit(t, cached, "update-ref", "-d", ref) + } + old := wantedRef(sha, time.Now().Add(-wantedMaxAge+time.Hour)) + runGit(t, cached, "update-ref", old, sha) + + start := time.Now().Add(-time.Second) + got, err := tc.Mirror(context.Background(), "up/repo.git", upstream.server.URL+"/repo", sha, true) + if err != nil || !got.Cached { + t.Fatalf("second mirror = %+v, %v, want cached", got, err) + } + if !slices.ContainsFunc(tc.wantedRefs(t), func(ref string) bool { + return ref != old && wantedRefTime(t, ref, sha).After(start) + }) { + t.Errorf("wanted refs %v, want one written by the second mirror", tc.wantedRefs(t)) + } +} + +func TestMirrorDoesNotPinCommitOnABranch(t *testing.T) { + upstream := newFakeUpstream(t) + work := upstream.newRepo("repo") + sha := upstream.commit(work, "second") + runGit(t, work, "push", "--quiet", "origin", "main") + tc := newTestCache(t, upstream.server.URL, nil) + for range 2 { + if _, err := tc.Mirror(context.Background(), "up/repo.git", upstream.server.URL+"/repo", sha, true); err != nil { + t.Fatalf("Mirror: %v", err) + } + } + if refs := tc.wantedRefs(t); len(refs) != 0 { + t.Errorf("wanted refs %v, want none for a commit on a branch", refs) + } +} + +func wantedRefTime(t *testing.T, ref, oid string) time.Time { + t.Helper() + created, got, _ := strings.Cut(strings.TrimPrefix(ref, wantedPrefix), "/") + unix, err := strconv.ParseInt(created, 10, 64) + if err != nil || got != oid { + return time.Time{} + } + return time.Unix(unix, 0) +} diff --git a/internal/gitcache/wants.go b/internal/gitcache/wants.go index c715f2f..8755ba1 100644 --- a/internal/gitcache/wants.go +++ b/internal/gitcache/wants.go @@ -161,19 +161,10 @@ func (c *Cache) pruneWanted(ctx context.Context, dir string, wanted []string, fe } var mergedErr error if fetched && len(keep) > 0 { - // rev-list prints what the wanted commits, read from stdin, reach - // and no branch or tag does, so a wanted commit it leaves out is - // merged. - cmd := c.command(ctx, nil, "-C", dir, "rev-list", "--stdin", "--not", "--branches", "--tags") - cmd.Stdin = strings.NewReader(strings.Join(slices.Sorted(maps.Keys(keep)), "\n") + "\n") - out, err := cmd.Output() + unmerged, err := c.unmergedCommits(ctx, dir, slices.Sorted(maps.Keys(keep))) if err != nil { mergedErr = fmt.Errorf("listing unmerged wanted commits: %w", err) } else { - unmerged := make(map[string]bool) - for line := range strings.Lines(string(out)) { - unmerged[strings.TrimSpace(line)] = true - } for oid, ref := range keep { if !unmerged[oid] { drop = append(drop, ref) @@ -196,6 +187,24 @@ func (c *Cache) pruneWanted(ctx context.Context, dir string, wanted []string, fe return mergedErr } +// unmergedCommits returns which of commits no cached branch or tag +// includes. +func (c *Cache) unmergedCommits(ctx context.Context, dir string, commits []string) (map[string]bool, error) { + // rev-list prints what the commits, read from stdin, reach and no + // branch or tag does, so a commit it leaves out is merged. + cmd := c.command(ctx, nil, "-C", dir, "rev-list", "--stdin", "--not", "--branches", "--tags") + cmd.Stdin = strings.NewReader(strings.Join(commits, "\n") + "\n") + out, err := cmd.Output() + if err != nil { + return nil, err + } + unmerged := make(map[string]bool) + for line := range strings.Lines(string(out)) { + unmerged[strings.TrimSpace(line)] = true + } + return unmerged, nil +} + // withoutFailedWants drops the object IDs a recent fetch for the // repository could not get. func (c *Cache) withoutFailedWants(key string, oids []string) []string { diff --git a/internal/handler/git_mirror.go b/internal/handler/git_mirror.go index eca76c7..709df31 100644 --- a/internal/handler/git_mirror.go +++ b/internal/handler/git_mirror.go @@ -11,8 +11,8 @@ import ( // Mirror caches the repository at location, a host and path such as // github.com/acme/widgets as a pkg:git PURL names it, from the upstream.git -// host serving it, and with commit, a full commit ID, that commit too. It -// is the copy clones through /git are then served. +// host serving it, and with commit, a full commit ID, that commit too. +// Clones through /git are then served from that copy. func (h *GitHandler) Mirror(ctx context.Context, location, commit string) (gitcache.MirrorResult, error) { repo, err := h.mirrorRepository(location) if err != nil { From 90ba3f9fd116129348eae34941be9450cb4da85c Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Fri, 6 Nov 2026 11:44:22 +0100 Subject: [PATCH 569/633] Say which commits fail to mirror without fetching by ID --- docs/configuration.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/configuration.md b/docs/configuration.md index 540078c..2b119c2 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1887,7 +1887,7 @@ proxy mirror pkg:git/github.com/git-pkgs/purl ``` - The repository is served by the `upstream.git` host whose URL its host and path start with, the longest one if several do, and has to match that host's `repos`. Others fail. -- The repository's branches and tags are cached, as a clone through the proxy caches them. A commit on none of them is fetched by ID and kept, like one a client fetches by ID, until a branch or tag includes it, or for seven days from the last mirror run that named it. Where `upstream.auth` has credentials for the host, commits aren't fetched by ID, and one on no branch or tag fails. +- The repository's branches and tags are cached, as a clone through the proxy caches them. A commit on none of them is fetched by ID and kept, like one a client fetches by ID, until a branch or tag includes it, or for seven days from the last mirror run that named it. Where `upstream.auth` has credentials for the host, commits aren't fetched by ID, and one on no branch or tag that isn't cached yet fails. - Without a version the repository's branches and tags are mirrored. A tag or branch name as the version fails; give the commit ID. - The git cache is used by one process. While the server runs, the command can't open `git.cache_dir`, so `pkg:git` items fail and the rest are mirrored; use the mirror API instead. From c037fb9e8b94cb0c5580c6903b72050b2afff39a Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Fri, 6 Nov 2026 12:04:14 +0100 Subject: [PATCH 570/633] Note that git.max_size can evict a mirrored repository sooner --- docs/configuration.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/configuration.md b/docs/configuration.md index 2b119c2..d447c0c 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1887,7 +1887,7 @@ proxy mirror pkg:git/github.com/git-pkgs/purl ``` - The repository is served by the `upstream.git` host whose URL its host and path start with, the longest one if several do, and has to match that host's `repos`. Others fail. -- The repository's branches and tags are cached, as a clone through the proxy caches them. A commit on none of them is fetched by ID and kept, like one a client fetches by ID, until a branch or tag includes it, or for seven days from the last mirror run that named it. Where `upstream.auth` has credentials for the host, commits aren't fetched by ID, and one on no branch or tag that isn't cached yet fails. +- The repository's branches and tags are cached, as a clone through the proxy caches them. A commit on none of them is fetched by ID and kept, like one a client fetches by ID, until a branch or tag includes it, or for seven days from the last mirror run that named it, as long as `git.max_size` doesn't evict the repository. Where `upstream.auth` has credentials for the host, commits aren't fetched by ID, and one on no branch or tag that isn't cached yet fails. - Without a version the repository's branches and tags are mirrored. A tag or branch name as the version fails; give the commit ID. - The git cache is used by one process. While the server runs, the command can't open `git.cache_dir`, so `pkg:git` items fail and the rest are mirrored; use the mirror API instead. From b0566aa4568277323a8c84dc7d6b9618398edf49 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Fri, 6 Nov 2026 13:00:31 +0100 Subject: [PATCH 571/633] Fail a mirror whose refresh is past max_repo_size The repository is dropped once released, so a cached commit in it would not be served for long. --- internal/gitcache/mirror.go | 5 +++++ internal/gitcache/mirror_test.go | 27 +++++++++++++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/internal/gitcache/mirror.go b/internal/gitcache/mirror.go index 62ae932..b5bde40 100644 --- a/internal/gitcache/mirror.go +++ b/internal/gitcache/mirror.go @@ -48,6 +48,11 @@ func (c *Cache) Mirror(ctx context.Context, key, repoURL, commit string, fetchWa } } + // A refresh past max_repo_size gets the repository dropped once it is + // released, commit and all. + if errors.Is(refreshErr, ErrTooLarge) { + return MirrorResult{}, refreshErr + } if commit != "" { if err := c.ensureCommit(ctx, rp, repoURL, commit, fetchWants); err != nil { return MirrorResult{}, err diff --git a/internal/gitcache/mirror_test.go b/internal/gitcache/mirror_test.go index f87b322..86f652f 100644 --- a/internal/gitcache/mirror_test.go +++ b/internal/gitcache/mirror_test.go @@ -3,6 +3,7 @@ package gitcache import ( "context" "errors" + "os" "path/filepath" "slices" "strconv" @@ -185,3 +186,29 @@ func wantedRefTime(t *testing.T, ref, oid string) time.Time { } return time.Unix(unix, 0) } + +// TestMirrorReportsRefreshTooLarge checks a commit that is cached still +// fails to mirror when the refresh doesn't fit in max_repo_size, since the +// repository is dropped once released. +func TestMirrorReportsRefreshTooLarge(t *testing.T) { + upstream := newFakeUpstream(t) + work := upstream.newRepo("repo") + runGit(t, upstream.bare(), "config", "uploadpack.allowAnySHA1InWant", "true") + pinned := upstream.pushPullRequest(work, 1, randomBytes(100<<10)) + base := repoSize(t, upstream, "repo") + tc := newTestCache(t, upstream.server.URL, func(o *Options) { o.MaxRepoSize = base + 150<<10 }) + if _, err := tc.Mirror(context.Background(), "up/repo.git", upstream.server.URL+"/repo", pinned, true); err != nil { + t.Fatalf("Mirror: %v", err) + } + if err := os.WriteFile(filepath.Join(work, "grown.bin"), randomBytes(80<<10), 0o600); err != nil { + t.Fatal(err) + } + runGit(t, work, "add", "grown.bin") + upstream.commit(work, "grown") + runGit(t, work, "push", "--quiet", "origin", "main") + + _, err := tc.Mirror(context.Background(), "up/repo.git", upstream.server.URL+"/repo", pinned, true) + if !errors.Is(err, ErrTooLarge) { + t.Errorf("mirror with a refresh past max_repo_size = %v, want ErrTooLarge", err) + } +} From 292ecc148843699bc0424486404e0956ed51cc64 Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Fri, 6 Nov 2026 22:36:47 +0000 Subject: [PATCH 572/633] Correct package manager configuration examples --- README.md | 34 ++++++++++++++++++---------------- docs/configuration.md | 2 +- internal/server/dashboard.go | 27 ++++++++++----------------- 3 files changed, 29 insertions(+), 34 deletions(-) diff --git a/README.md b/README.md index be4f0ee..a43ae27 100644 --- a/README.md +++ b/README.md @@ -77,7 +77,7 @@ The proxy never uploads artifact bytes to a scanner. Each scanner is notified wi Cooldown requires publish timestamps in metadata. Registries without a "Yes" in the cooldown column either don't expose timestamps or haven't been wired up yet. -\* Hex cooldown requires disabling registry signature verification (`HEX_NO_VERIFY_REPO_ORIGIN=1`) since the proxy re-encodes the protobuf payload. +\* Hex cooldown requires disabling registry signature verification (`HEX_UNSAFE_REGISTRY=1`) since the proxy re-encodes the protobuf payload. Origin verification can stay enabled. † Go cooldown uses the time in each version's `.info`, which is the commit time of the tagged revision, not when the tag was pushed. @@ -288,10 +288,10 @@ The equivalent environment variables are `PROXY_UPSTREAM_HOMEBREW_API` and `PROX ### Hex (Elixir) -Configure in `~/.hex/hex.config`: +Configure the mirror: -```erlang -{default_url, <<"http://localhost:8080/hex">>}. +```bash +mix hex.config mirror_url http://localhost:8080/hex ``` Or set the environment variable: @@ -300,6 +300,14 @@ Or set the environment variable: export HEX_MIRROR=http://localhost:8080/hex ``` +When the proxy applies a cooldown, also set: + +```bash +export HEX_UNSAFE_REGISTRY=1 +``` + +Filtering releases removes the registry signature. The proxy preserves the package name and repository fields, so origin verification can stay enabled. Without a cooldown, leave signature verification enabled too. + ### pub.dev (Dart/Flutter) Set the PUB_HOSTED_URL environment variable: @@ -359,17 +367,15 @@ Configure in `settings.gradle.kts`: ```kotlin buildCache { - local { - enabled = false - } remote { url = uri("http://localhost:8080/gradle/") - isPush = System.getenv("CI") != null } } ``` -The proxy rejects uploads unless `gradle.build_cache.read_only` is set to `false`, see [docs/configuration.md](docs/configuration.md#gradle-build-cache). +Enable build caching with `org.gradle.caching=true` in `gradle.properties`, or pass `--build-cache` when running Gradle. The local cache stays enabled and remote uploads are disabled by default. + +To upload from CI, set `gradle.build_cache.read_only` to `false` on the proxy and add `isPush = System.getenv("CI") != null` inside the client's `remote` block. See [docs/configuration.md](docs/configuration.md#gradle-build-cache). ### NuGet @@ -459,8 +465,6 @@ Configure in `~/.condarc`: channels: - http://localhost:8080/conda/main - http://localhost:8080/conda/conda-forge -default_channels: - - http://localhost:8080/conda/main ``` Or set via command: @@ -716,16 +720,14 @@ indexes unchanged. To keep APT from selecting a denied version, configure ### RPM / Yum / DNF -Configure yum/dnf to use the proxy in `/etc/yum.repos.d/proxy.repo`: +In the `[fedora]` section of your existing repository file under `/etc/yum.repos.d/`, replace the `metalink` or `mirrorlist` entry with: ```ini -[proxy-fedora] -name=Fedora via Proxy baseurl=http://localhost:8080/rpm/releases/$releasever/Everything/$basearch/os/ -enabled=1 -gpgcheck=0 ``` +Keep the existing `gpgkey` and signature settings, including `gpgcheck=1`. The proxy serves RPM packages unchanged, so their signatures remain valid. + Then: ```bash diff --git a/docs/configuration.md b/docs/configuration.md index 63cc175..accaefb 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1244,7 +1244,7 @@ With cooldown on, the proxy asks npm for full packuments, since the abbreviated Conda cooldown covers `repodata.json`, `current_repodata.json`, `repodata_from_packages.json` and sharded repodata (CEP 16). Other forms of the repodata, such as `repodata.json.zst`, `repodata.json.bz2` and `repodata.jlap` patches, would list versions still in cooldown, so with cooldown on they are answered 404 and conda and mamba fall back to `repodata.json`. With `cache_metadata`, `repodata.json` is read through the metadata cache as it is without cooldown: within `metadata_ttl` the stored copy is filtered as it is read from storage, after it the upstream is asked with its ETag, and while the upstream is down the stale copy is filtered. Once a document has been filtered, the proxy remembers the ETag of the result until the first entry it left out leaves cooldown, so a client revalidating with `If-None-Match` gets `304 Not Modified` without the document being filtered or sent again. The first response for each document has no ETag, as it is sent while it is filtered. Without `cache_metadata`, `repodata.json` is fetched and filtered on every request. Clients cache shards by the hash in the shard index, so with cooldown on the proxy serves a rewritten index whose shard hashes change every hour and filters each shard as it is fetched. Versions leaving cooldown show up within the hour, and clients fetch the shards they need again once an hour. If a channel's shard index points packages or shards somewhere the proxy doesn't route, the proxy answers 404 for it and clients fall back to `repodata.json`. -Note: Hex cooldown requires disabling registry signature verification since the proxy re-encodes the protobuf payload without the original signature. Set `HEX_NO_VERIFY_REPO_ORIGIN=1` or configure your repo with `no_verify: true`. +Hex cooldown requires `HEX_UNSAFE_REGISTRY=1` because filtering releases re-encodes the protobuf payload without the original signature. The proxy preserves the package name and repository fields, so origin verification can stay enabled. `HEX_NO_VERIFY_REPO_ORIGIN` controls origin verification and does not disable signature verification. Without a cooldown, leave signature verification enabled too. ## Version denylist diff --git a/internal/server/dashboard.go b/internal/server/dashboard.go index 217f018..f566d55 100644 --- a/internal/server/dashboard.go +++ b/internal/server/dashboard.go @@ -324,11 +324,11 @@ bundle config mirror.https://rubygems.org ` + baseURL + `/gem`), Language: "Elixir", Endpoint: "/hex/", Instructions: template.HTML(`

Configure Hex to use the proxy:

-
# In ~/.hex/hex.config
-{default_url, <<"` + baseURL + `/hex">>}.
+
mix hex.config mirror_url ` + baseURL + `/hex
 
 # Or via environment variable
-export HEX_MIRROR=` + baseURL + `/hex
`),
+export HEX_MIRROR=` + baseURL + `/hex
+

When the proxy applies a cooldown, also set HEX_UNSAFE_REGISTRY=1 to disable registry signature verification. Origin verification can stay enabled.

`), }, { ID: "pub", @@ -378,10 +378,10 @@ index-url = ` + baseURL + `/pypi/simple/`), buildCache { remote<HttpBuildCache> { url = uri("` + baseURL + `/gradle/") - isPush = System.getenv("CI") != null } } -

Uploads need gradle.build_cache.read_only: false on the proxy.

`), +

Enable build caching with org.gradle.caching=true in gradle.properties, or pass --build-cache when running Gradle. The local cache stays enabled and remote uploads are disabled by default.

+

To upload from CI, set gradle.build_cache.read_only: false on the proxy and add isPush = System.getenv("CI") != null inside the client's remote<HttpBuildCache> block.

`), }, { ID: "nuget", @@ -448,8 +448,6 @@ conan remote disable conancenter`), channels: - ` + baseURL + `/conda/main - ` + baseURL + `/conda/conda-forge -default_channels: - - ` + baseURL + `/conda/main # Or via command conda config --add channels ` + baseURL + `/conda/main`), @@ -523,17 +521,12 @@ sudo apt update`), { ID: "rpm", Name: "RPM/Yum", - Language: "Fedora/RHEL", + Language: "Fedora", Endpoint: "/rpm/", - Instructions: template.HTML(`

Configure yum/dnf to use the proxy:

-
# In /etc/yum.repos.d/proxy.repo
-[proxy-fedora]
-name=Fedora via Proxy
-baseurl=` + baseURL + `/rpm/releases/$releasever/Everything/$basearch/os/
-enabled=1
-gpgcheck=0
-
-# Then run:
+			Instructions: template.HTML(`
In the [fedora] section of your existing repository file under /etc/yum.repos.d/, replace the metalink or mirrorlist entry with:

+
baseurl=` + baseURL + `/rpm/releases/$releasever/Everything/$basearch/os/

+
Keep the existing gpgkey and signature settings, including gpgcheck=1. The proxy serves RPM packages unchanged.

+
# Then run:
 sudo dnf clean all
 sudo dnf update
`),
 		},

From 611feb003adb611813a1944f73764f28641ba4f5 Mon Sep 17 00:00:00 2001
From: Milas Bowman 
Date: Wed, 4 Nov 2026 18:29:37 -0500
Subject: [PATCH 573/633] fix(debian): route packages by filename rather than
 `/pool` prefix

---
 docs/architecture.md            |   9 +++
 docs/configuration.md           |   4 ++
 internal/handler/debian.go      |  28 +++++++-
 internal/handler/debian_test.go | 112 ++++++++++++++++++++++++++++++++
 4 files changed, 150 insertions(+), 3 deletions(-)

diff --git a/docs/architecture.md b/docs/architecture.md
index d8b294c..cbd6dab 100644
--- a/docs/architecture.md
+++ b/docs/architecture.md
@@ -304,6 +304,11 @@ HTTP protocol handlers for each registry type.
 - `handleIndex()` - Proxy sparse index
 - `handleDownload()` - Serve cached crate
 
+**DebianHandler:**
+- Serves `/debian` and `/debian/{name}` mirrors of APT archives (`upstream.debian_repositories`)
+- Routes `.deb` packages to the artifact cache, source-package files directly to upstream, `dists/…` to the metadata cache, and proxies all other paths
+- Keys the artifact cache on name, version, and architecture, scoped by repository for named archives
+
 **SwiftHandler:**
 - Proxies the Swift Package Registry v1 read endpoints
 - Rewrites release URLs and caches source archives
@@ -429,6 +434,10 @@ An object a record stops pointing at, because a refetch replaced it or its entry
 - Set `cache_metadata: true` or use the mirror command to enable metadata caching for offline use via the `metadata_cache` table
 - OCI manifests and tag lists are exceptions: they are cached automatically so previously fetched images remain pullable and tag resolution works when the registry or token service is unavailable
 
+**Why are Debian source-package files streamed instead of cached?**
+- Source-package files (`.dsc`, source tarballs, `.buildinfo`, `.changes`) can be re-uploaded or re-signed in place under the same path, whereas the artifact cache keys entries by version and filename without an expected digest from the `Sources` index
+- Streaming them avoids serving stale bytes when an upstream file is replaced or caching them under the `dists/` metadata TTL in flat repositories
+
 **Why stream artifacts?**
 - Memory efficient - don't load large files into RAM
 - Better latency - start sending while still receiving
diff --git a/docs/configuration.md b/docs/configuration.md
index 63cc175..05914eb 100644
--- a/docs/configuration.md
+++ b/docs/configuration.md
@@ -805,6 +805,10 @@ and `dists` are refused at config load for that reason, and other root paths
 an archive may serve (`indices`, `project`, `doc`, `tools`) are not, so avoid
 those names unless the shadowing is intended.
 
+Binary packages (`.deb`) are cached; source-package files (`.dsc`, source
+tarballs, `.buildinfo`, `.changes`) are streamed directly from upstream because
+their contents can be replaced in place under the same path.
+
 This field has no environment variable, as with the other named upstream maps.
 `PROXY_UPSTREAM_DEBIAN` still sets `upstream.debian`.
 
diff --git a/internal/handler/debian.go b/internal/handler/debian.go
index b2419f6..87dbd65 100644
--- a/internal/handler/debian.go
+++ b/internal/handler/debian.go
@@ -108,11 +108,13 @@ func (h *DebianHandler) Routes() http.Handler {
 			}
 		}
 
-		// Route based on path type
+		// Route by filename rather than path prefix because flat repositories
+		// place pools under dists//pool/....
 		switch {
-		case strings.HasPrefix(rest, "pool/"):
-			// Package downloads - cache these
+		case isDebPackageFile(rest):
 			h.handlePackageDownload(w, r, repository, upstreamURL, rest)
+		case isSourcePackageFile(rest):
+			h.proxyFile(w, r, upstreamURL, rest)
 		case strings.HasPrefix(rest, "dists/"):
 			// Repository metadata - served through the metadata cache
 			h.handleMetadata(w, r, repository, upstreamURL, rest)
@@ -123,6 +125,26 @@ func (h *DebianHandler) Routes() http.Handler {
 	}))
 }
 
+func isDebPackageFile(path string) bool {
+	return strings.HasSuffix(path, ".deb")
+}
+
+// isSourcePackageFile reports whether path is a Debian source-package artifact.
+// Source files can be re-uploaded or re-signed in place under the same filename,
+// so they must be streamed rather than cached without digest verification.
+func isSourcePackageFile(path string) bool {
+	switch {
+	case strings.HasSuffix(path, ".dsc"),
+		strings.HasSuffix(path, ".buildinfo"),
+		strings.HasSuffix(path, ".changes"):
+		return true
+	}
+	// Match Debian source tarballs without matching suite tarballs (e.g. installer netboot.tar.gz).
+	return strings.Contains(path, ".orig.tar.") ||
+		strings.Contains(path, ".orig-") ||
+		strings.Contains(path, ".debian.tar.")
+}
+
 // repositoryNotFound answers a request addressed to a repository that is not
 // configured, naming the ones that are so a misspelling is self-evident.
 func (h *DebianHandler) repositoryNotFound(w http.ResponseWriter, name string) {
diff --git a/internal/handler/debian_test.go b/internal/handler/debian_test.go
index f31202f..848f9e2 100644
--- a/internal/handler/debian_test.go
+++ b/internal/handler/debian_test.go
@@ -251,6 +251,45 @@ func TestDebianHandler_PackageCacheScopedPerRepository(t *testing.T) {
 	}
 }
 
+// TestDebianHandler_FlatArchivePackageIsCached verifies that .deb packages
+// served under dists//pool/ (as used by flat repositories like Docker's)
+// are cached as artifacts rather than treated as uncached/TTL metadata.
+func TestDebianHandler_FlatArchivePackageIsCached(t *testing.T) {
+	const (
+		flatPath = "/docker/dists/trixie/pool/stable/amd64/docker-ce_5.0.1-1~debian.13~trixie_amd64.deb"
+		pkgBody  = "docker-ce package bytes"
+	)
+
+	var upstreamHits int
+
+	archive := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		upstreamHits++
+		_, _ = fmt.Fprint(w, pkgBody)
+	}))
+	defer archive.Close()
+
+	proxy, _, _, _ := setupTestProxy(t)
+	fetcher := fetch.NewFetcher(fetch.WithHTTPClient(archive.Client()), fetch.WithMaxRetries(0))
+	proxy.Fetcher = fetcher
+	t.Cleanup(func() { _ = fetcher.Close() })
+
+	h := NewDebianHandler(proxy, "http://proxy.example", "https://archive.test", map[string]string{
+		"docker": archive.URL,
+	})
+
+	for attempt := 1; attempt <= 2; attempt++ {
+		w := serveDebianRequest(h, flatPath)
+		if w.Code != http.StatusOK || w.Body.String() != pkgBody {
+			t.Fatalf("request %d: status = %d, body = %q, want 200 %q",
+				attempt, w.Code, w.Body.String(), pkgBody)
+		}
+	}
+
+	if upstreamHits != 1 {
+		t.Errorf("upstream fetched %d times for two requests, want 1: the package is not cached", upstreamHits)
+	}
+}
+
 func serveDebianRequest(h *DebianHandler, target string) *httptest.ResponseRecorder {
 	w := httptest.NewRecorder()
 	h.Routes().ServeHTTP(w, httptest.NewRequest(http.MethodGet, target, nil))
@@ -285,3 +324,76 @@ func TestDebRetentionKey(t *testing.T) {
 		t.Error("debRetentionKey(nil) accepted")
 	}
 }
+
+func TestDebianHandler_ClassifiesFilesByExtension(t *testing.T) {
+	tests := []struct {
+		path   string
+		pkg    bool
+		source bool
+		why    string
+	}{
+		{"pool/main/h/hello/hello_2.10-3_amd64.deb", true, false, "canonical pool"},
+		{"dists/trixie/pool/stable/amd64/docker-ce_5.0.1-1~debian.13~trixie_amd64.deb", true, false, "flat archive"},
+		{"pool/main/h/hello/hello_2.10-3.udeb", false, false, "udeb not matched by debPackagePattern"},
+		{"pool/main/h/hello/hello_2.10-3_amd64.ddeb", false, false, "ddeb not matched by debPackagePattern"},
+		{"pool/main/h/hello/hello_2.10-3.dsc", false, true, "source control file"},
+		{"pool/main/h/hello/hello_2.10.orig.tar.xz", false, true, "source tarball"},
+		{"pool/main/h/hello/hello_2.10.orig-tex.tar.gz", false, true, "component tarball"},
+		{"pool/main/h/hello/hello_2.10-3.debian.tar.xz", false, true, "debian tarball"},
+		{"pool/main/h/hello/hello_2.10-3_amd64.buildinfo", false, true, "buildinfo"},
+		{"pool/main/h/hello/hello_2.10-3_amd64.changes", false, true, "changes"},
+		{"dists/trixie/pool/stable/amd64/docker-ce_5.0.1-1.dsc", false, true, "flat archive source file"},
+		{"dists/trixie/InRelease", false, false, "metadata"},
+		{"dists/trixie/main/binary-amd64/Packages.gz", false, false, "metadata"},
+		{"dists/trixie/main/installer-amd64/current/images/netboot/netboot.tar.gz", false, false, "suite tarball"},
+	}
+	for _, tt := range tests {
+		if got := isDebPackageFile(tt.path); got != tt.pkg {
+			t.Errorf("isDebPackageFile(%q) = %v, want %v (%s)", tt.path, got, tt.pkg, tt.why)
+		}
+		if got := isSourcePackageFile(tt.path); got != tt.source {
+			t.Errorf("isSourcePackageFile(%q) = %v, want %v (%s)", tt.path, got, tt.source, tt.why)
+		}
+	}
+}
+
+func TestDebianHandler_SourcePackageFilesAreNotCached(t *testing.T) {
+	const dscBody = "hello source control bytes"
+
+	var upstreamHits int
+
+	archive := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		upstreamHits++
+		_, _ = fmt.Fprint(w, dscBody)
+	}))
+	defer archive.Close()
+
+	proxy, _, store, _ := setupTestProxy(t)
+	proxy.HTTPClient = archive.Client()
+	proxy.CacheMetadata = true
+	proxy.MetadataTTL = time.Hour
+
+	h := NewDebianHandler(proxy, "http://proxy.example", archive.URL, map[string]string{
+		"docker": archive.URL,
+	})
+
+	for _, target := range []string{
+		"/pool/main/h/hello/hello_2.10-3.dsc",
+		"/docker/dists/trixie/pool/stable/amd64/docker-ce_5.0.1-1.dsc",
+	} {
+		for attempt := 1; attempt <= 2; attempt++ {
+			w := serveDebianRequest(h, target)
+			if w.Code != http.StatusOK || w.Body.String() != dscBody {
+				t.Fatalf("%s request %d: status = %d, body = %q, want 200 %q",
+					target, attempt, w.Code, w.Body.String(), dscBody)
+			}
+		}
+	}
+
+	if upstreamHits != 4 {
+		t.Errorf("upstream fetched %d times for four requests, want 4: source-package files must not be cached", upstreamHits)
+	}
+	if store.stores != 0 {
+		t.Errorf("storage wrote %d object(s), want 0: a source-package file must not be stored", store.stores)
+	}
+}

From f153a83b3a286b9e3c9f9c04559bd3b674763389 Mon Sep 17 00:00:00 2001
From: Andrew Nesbitt 
Date: Sat, 7 Nov 2026 09:46:54 +0000
Subject: [PATCH 574/633] Cover Fedora updates and Conda defaults in setup docs

---
 README.md                    | 11 +++++++++++
 internal/server/dashboard.go | 36 +++++++++++++++++++++++-------------
 2 files changed, 34 insertions(+), 13 deletions(-)

diff --git a/README.md b/README.md
index a43ae27..7739645 100644
--- a/README.md
+++ b/README.md
@@ -465,12 +465,17 @@ Configure in `~/.condarc`:
 channels:
   - http://localhost:8080/conda/main
   - http://localhost:8080/conda/conda-forge
+default_channels:
+  - http://localhost:8080/conda/main
 ```
 
+`default_channels` also routes requests for `defaults` through the proxy, including `-c defaults` and `environment.yml` files that name it.
+
 Or set via command:
 
 ```bash
 conda config --add channels http://localhost:8080/conda/main
+conda config --add default_channels http://localhost:8080/conda/main
 ```
 
 pixi and other rattler-based clients use sharded repodata ([CEP 16](https://github.com/conda/ceps/blob/main/cep-0016.md)) where a channel offers it, as conda-forge does. Point them at the same channel URLs, for example `pixi init -c http://localhost:8080/conda/conda-forge`. The shard index and shards are cached as metadata when `cache_metadata` is on.
@@ -726,6 +731,12 @@ In the `[fedora]` section of your existing repository file under `/etc/yum.repos
 baseurl=http://localhost:8080/rpm/releases/$releasever/Everything/$basearch/os/
 ```
 
+In the `[updates]` section of `fedora-updates.repo`, replace its `metalink` or `mirrorlist` entry with:
+
+```ini
+baseurl=http://localhost:8080/rpm/updates/$releasever/Everything/$basearch/
+```
+
 Keep the existing `gpgkey` and signature settings, including `gpgcheck=1`. The proxy serves RPM packages unchanged, so their signatures remain valid.
 
 Then:
diff --git a/internal/server/dashboard.go b/internal/server/dashboard.go
index f566d55..604a4bf 100644
--- a/internal/server/dashboard.go
+++ b/internal/server/dashboard.go
@@ -448,9 +448,13 @@ conan remote disable conancenter
`), channels: - ` + baseURL + `/conda/main - ` + baseURL + `/conda/conda-forge +default_channels: + - ` + baseURL + `/conda/main # Or via command -conda config --add channels ` + baseURL + `/conda/main`), +conda config --add channels ` + baseURL + `/conda/main +conda config --add default_channels ` + baseURL + `/conda/main +

default_channels also routes requests for defaults through the proxy, including -c defaults and environment.yml files that name it.

`), }, { ID: "cran", @@ -518,18 +522,7 @@ deb ` + baseURL + `/debian stable main contrib # Then run: sudo apt update`), }, - { - ID: "rpm", - Name: "RPM/Yum", - Language: "Fedora", - Endpoint: "/rpm/", - Instructions: template.HTML(`

In the [fedora] section of your existing repository file under /etc/yum.repos.d/, replace the metalink or mirrorlist entry with:

-
baseurl=` + baseURL + `/rpm/releases/$releasever/Everything/$basearch/os/
-

Keep the existing gpgkey and signature settings, including gpgcheck=1. The proxy serves RPM packages unchanged.

-
# Then run:
-sudo dnf clean all
-sudo dnf update
`), - }, + rpmRegistryConfig(baseURL), { ID: "apk", Name: "Alpine APK", @@ -559,6 +552,23 @@ pacman -Syu`), } } +func rpmRegistryConfig(baseURL string) RegistryConfig { + return RegistryConfig{ + ID: "rpm", + Name: "RPM/Yum", + Language: "Fedora", + Endpoint: "/rpm/", + Instructions: template.HTML(`

In the [fedora] section of your existing repository file under /etc/yum.repos.d/, replace the metalink or mirrorlist entry with:

+
baseurl=` + baseURL + `/rpm/releases/$releasever/Everything/$basearch/os/
+

In the [updates] section of fedora-updates.repo, replace its metalink or mirrorlist entry with:

+
baseurl=` + baseURL + `/rpm/updates/$releasever/Everything/$basearch/
+

Keep the existing gpgkey and signature settings, including gpgcheck=1. The proxy serves RPM packages unchanged.

+
# Then run:
+sudo dnf clean all
+sudo dnf update
`), + } +} + // freebsdRegistryConfig describes the FreeBSD pkg endpoint for the dashboard. func freebsdRegistryConfig(baseURL string) RegistryConfig { return RegistryConfig{ From e9aa078a8572160557a4d4cfb09aa9546db952bf Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sat, 7 Nov 2026 17:29:03 +0100 Subject: [PATCH 575/633] Mirror pkg:git tags and the submodules a commit pins A pkg:git version can now be a tag as well as a full commit ID. With a version, the submodules the commit records are mirrored at the commits it pins, from the upstream.git hosts serving them; one no host serves is skipped with a warning. --- docs/architecture.md | 2 +- docs/configuration.md | 6 +- internal/gitcache/mirror.go | 129 +++++++++++++++++++++++++--- internal/gitcache/mirror_test.go | 79 +++++++++++++++++ internal/handler/git_mirror.go | 100 ++++++++++++++++++++- internal/handler/git_mirror_test.go | 81 +++++++++++++++++ internal/handler/git_test.go | 6 +- internal/mirror/git.go | 10 +-- 8 files changed, 387 insertions(+), 26 deletions(-) diff --git a/docs/architecture.md b/docs/architecture.md index 655b309..80120ee 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -363,7 +363,7 @@ Package metadata enrichment, built on [github.com/git-pkgs/enrichment](https://g ### `internal/mirror` -Selective package mirroring for pre-populating the proxy cache. Supports multiple input sources: individual PURLs (versioned or unversioned), CycloneDX/SPDX SBOM files, and full registry enumeration. Uses a bounded worker pool backed by `errgroup` to download artifacts in parallel, reusing `handler.Proxy.GetOrFetchArtifact()` for the actual fetch-and-cache work. Container images (`pkg:docker`, `pkg:oci`) go through `handler.ContainerHandler.MirrorImage()` instead, which walks index, platform manifests and blobs and stores them under the keys a pull reads. Git repositories (`pkg:git`) go through `handler.GitHandler.Mirror()`, which caches the repository and commit in the git cache that `/git` serves. +Selective package mirroring for pre-populating the proxy cache. Supports multiple input sources: individual PURLs (versioned or unversioned), CycloneDX/SPDX SBOM files, and full registry enumeration. Uses a bounded worker pool backed by `errgroup` to download artifacts in parallel, reusing `handler.Proxy.GetOrFetchArtifact()` for the actual fetch-and-cache work. Container images (`pkg:docker`, `pkg:oci`) go through `handler.ContainerHandler.MirrorImage()` instead, which walks index, platform manifests and blobs and stores them under the keys a pull reads. Git repositories (`pkg:git`) go through `handler.GitHandler.Mirror()`, which caches the repository and the commit or tag, with its submodules, in the git cache that `/git` serves. The package also provides a `MetadataCache` for storing raw upstream metadata blobs so the proxy can serve metadata responses offline. The `JobStore` manages async mirror jobs exposed via the `/api/mirror` endpoints. diff --git a/docs/configuration.md b/docs/configuration.md index 584896b..1c01589 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1910,16 +1910,18 @@ proxy mirror --platform linux/arm64 pkg:brew/hello@2.12.3 pkg:brew/openssl%403 ### Git repositories -`pkg:git` PURLs mirror a repository of an [`upstream.git`](#git-repositories) host into the git cache, so a clone through `/git` is served from it, also when the upstream is unreachable. The PURL names the repository by host and path, and the version is a full commit ID: +`pkg:git` PURLs mirror a repository of an [`upstream.git`](#git-repositories) host into the git cache, so a clone through `/git` is served from it, also when the upstream is unreachable. The PURL names the repository by host and path, and the version is a full commit ID or a tag: ```bash proxy mirror pkg:git/github.com/git-pkgs/purl@0123456789abcdef0123456789abcdef01234567 +proxy mirror pkg:git/github.com/git-pkgs/purl@v0.2.1 proxy mirror pkg:git/github.com/git-pkgs/purl ``` - The repository is served by the `upstream.git` host whose URL its host and path start with, the longest one if several do, and has to match that host's `repos`. Others fail. - The repository's branches and tags are cached, as a clone through the proxy caches them. A commit on none of them is fetched by ID and kept, like one a client fetches by ID, until a branch or tag includes it, or for seven days from the last mirror run that named it, as long as `git.max_size` doesn't evict the repository. Where `upstream.auth` has credentials for the host, commits aren't fetched by ID, and one on no branch or tag that isn't cached yet fails. -- Without a version the repository's branches and tags are mirrored. A tag or branch name as the version fails; give the commit ID. +- Without a version the repository's branches and tags are mirrored. A tag as the version has to be one of them, so a tag the upstream doesn't have fails. A branch name as the version fails too; give the commit ID. +- With a version, the submodules that commit records are mirrored at the commits it pins, with their submodules in turn. A relative submodule URL is resolved against the repository's, and an `ssh://`, `git://` or `git@host:path` URL names the repository by its host and path. A submodule no `upstream.git` host serves is skipped with a warning in the log. - The git cache is used by one process. While the server runs, the command can't open `git.cache_dir`, so `pkg:git` items fail and the rest are mirrored; use the mirror API instead. ### Scheduled refresh diff --git a/internal/gitcache/mirror.go b/internal/gitcache/mirror.go index b5bde40..047934f 100644 --- a/internal/gitcache/mirror.go +++ b/internal/gitcache/mirror.go @@ -4,27 +4,44 @@ import ( "context" "errors" "fmt" + "os/exec" + "slices" "strings" "time" ) // MirrorResult is what Mirror did. type MirrorResult struct { - // Cached reports that the repository, and the commit if one was given, - // were cached already. + // Cached reports that the repository, and the commit if a version was + // given, were cached already. Cached bool // Bytes is how much the cached repository grew. Bytes int64 + // Commit is the commit the version names, empty without a version. + Commit string + // Submodules are the submodules Commit records, sorted by path. + Submodules []Submodule +} + +// Submodule is a submodule a commit records: its path, the URL .gitmodules +// gives for it, which may be relative to the repository's, and the commit +// the gitlink pins. +type Submodule struct { + Path, URL, Commit string } // Mirror brings the repository cached under key up to date with repoURL, -// cloning it if needed, and waits for that to finish. With commit, an object -// ID, it also makes sure the commit is cached. One on no branch or tag is -// fetched by ID when fetchWants allows it (see OpenOptions.FetchWants), and -// is kept as long as a commit a client fetched by ID. -func (c *Cache) Mirror(ctx context.Context, key, repoURL, commit string, fetchWants bool) (MirrorResult, error) { - if commit != "" && !isObjectID(commit) { - return MirrorResult{}, fmt.Errorf("%q is not a full commit ID", commit) +// cloning it if needed, and waits for that to finish. version, if given, is +// a full commit ID or a tag. A commit is made sure to be cached: one on no +// branch or tag is fetched by ID when fetchWants allows it (see +// OpenOptions.FetchWants), and is kept as long as a commit a client fetched +// by ID. A tag is cached with the repository's tags. +func (c *Cache) Mirror(ctx context.Context, key, repoURL, version string, fetchWants bool) (MirrorResult, error) { + tag := version != "" && !isObjectID(version) + if tag { + if err := c.checkTagName(ctx, version); err != nil { + return MirrorResult{}, err + } } lease, err := c.Open(ctx, key, repoURL, OpenOptions{FetchWants: fetchWants}) if err != nil { @@ -53,17 +70,105 @@ func (c *Cache) Mirror(ctx context.Context, key, repoURL, commit string, fetchWa if errors.Is(refreshErr, ErrTooLarge) { return MirrorResult{}, refreshErr } - if commit != "" { + commit := version + switch { + case tag: + if commit, err = c.resolveTag(ctx, rp.dir, version); err != nil { + if refreshErr != nil { + // The tag may be one the failed refresh would have brought. + return MirrorResult{}, refreshErr + } + return MirrorResult{}, err + } + case version != "": if err := c.ensureCommit(ctx, rp, repoURL, commit, fetchWants); err != nil { return MirrorResult{}, err } } // A refresh that failed leaves the commit cached all the same. - if refreshErr != nil && commit == "" { + if refreshErr != nil && version == "" { return MirrorResult{}, refreshErr } + var submodules []Submodule + if commit != "" { + if submodules, err = c.submodules(ctx, rp.dir, commit); err != nil { + return MirrorResult{}, err + } + } grown := max(rp.size.Load()-before, 0) - return MirrorResult{Cached: cached && grown == 0, Bytes: grown}, nil + return MirrorResult{Cached: cached && grown == 0, Bytes: grown, Commit: commit, Submodules: submodules}, nil +} + +// checkTagName refuses a version that is no valid tag name, such as one +// with .. or a revision suffix like ^{tree}. +func (c *Cache) checkTagName(ctx context.Context, name string) error { + if err := c.command(ctx, nil, "check-ref-format", "refs/tags/"+name).Run(); err != nil { + return fmt.Errorf("%q is neither a full commit ID nor a tag name", name) + } + return nil +} + +// resolveTag returns the commit tag points at. +func (c *Cache) resolveTag(ctx context.Context, dir, tag string) (string, error) { + out, err := c.command(ctx, nil, "-C", dir, "rev-parse", "--verify", "--quiet", "--end-of-options", "refs/tags/"+tag+"^{commit}").Output() + var exit *exec.ExitError + if errors.As(err, &exit) && exit.ExitCode() == 1 { + return "", fmt.Errorf("tag %s: %w", tag, ErrNotFound) + } + if err != nil { + return "", fmt.Errorf("resolving tag %s: %w", tag, err) + } + return strings.TrimSpace(string(out)), nil +} + +// submodules lists the submodules commit records: its gitlinks that +// .gitmodules names a URL for. +func (c *Cache) submodules(ctx context.Context, dir, commit string) ([]Submodule, error) { + if c.command(ctx, nil, "-C", dir, "cat-file", "-e", commit+":.gitmodules").Run() != nil { + return nil, nil + } + out, err := c.run(ctx, dir, "config", "-z", "--blob", commit+":.gitmodules", "--get-regexp", `^submodule\..*\.(path|url)$`) + if err != nil { + return nil, fmt.Errorf("reading .gitmodules: %w", err) + } + paths, urls := map[string]string{}, map[string]string{} + for entry := range strings.SplitSeq(out, "\x00") { + key, value, ok := strings.Cut(entry, "\n") + name, field, dotted := cutLast(strings.TrimPrefix(key, "submodule."), ".") + if !ok || !dotted { + continue + } + if field == "path" { + paths[value] = name + } else { + urls[name] = value + } + } + tree, err := c.run(ctx, dir, "ls-tree", "-r", "-z", "--full-tree", commit) + if err != nil { + return nil, err + } + var submodules []Submodule + for entry := range strings.SplitSeq(tree, "\x00") { + meta, path, ok := strings.Cut(entry, "\t") + fields := strings.Fields(meta) + if !ok || len(fields) != 3 || fields[0] != "160000" { + continue + } + if url := urls[paths[path]]; url != "" && isObjectID(fields[2]) { + submodules = append(submodules, Submodule{Path: path, URL: url, Commit: fields[2]}) + } + } + slices.SortFunc(submodules, func(a, b Submodule) int { return strings.Compare(a.Path, b.Path) }) + return submodules, nil +} + +func cutLast(s, sep string) (before, after string, found bool) { + i := strings.LastIndex(s, sep) + if i < 0 { + return s, "", false + } + return s[:i], s[i+len(sep):], true } // ensureCommit makes sure the repository has commit, fetching it by ID when diff --git a/internal/gitcache/mirror_test.go b/internal/gitcache/mirror_test.go index 86f652f..499acfb 100644 --- a/internal/gitcache/mirror_test.go +++ b/internal/gitcache/mirror_test.go @@ -212,3 +212,82 @@ func TestMirrorReportsRefreshTooLarge(t *testing.T) { t.Errorf("mirror with a refresh past max_repo_size = %v, want ErrTooLarge", err) } } + +func TestMirrorResolvesTag(t *testing.T) { + upstream := newFakeUpstream(t) + work := upstream.newRepo("repo") + tagged := runGit(t, work, "rev-parse", "v1^{commit}") + tc := newTestCache(t, upstream.server.URL, nil) + + got, err := tc.Mirror(context.Background(), "up/repo.git", upstream.server.URL+"/repo", "v1", true) + if err != nil { + t.Fatalf("Mirror: %v", err) + } + if got.Commit != tagged { + t.Errorf("commit %q, want %s the tag points at", got.Commit, tagged) + } +} + +func TestMirrorReportsMissingTag(t *testing.T) { + upstream := newFakeUpstream(t) + upstream.newRepo("repo") + tc := newTestCache(t, upstream.server.URL, nil) + + for _, version := range []string{"v9", "main"} { + _, err := tc.Mirror(context.Background(), "up/repo.git", upstream.server.URL+"/repo", version, true) + if !errors.Is(err, ErrNotFound) { + t.Errorf("mirror at %q = %v, want ErrNotFound", version, err) + } + } +} + +func TestMirrorRefusesInvalidTag(t *testing.T) { + upstream := newFakeUpstream(t) + upstream.newRepo("repo") + tc := newTestCache(t, upstream.server.URL, nil) + + for _, version := range []string{"a..b", "v1^{tree}", "v1~1", "v 1", "v1:x"} { + _, err := tc.Mirror(context.Background(), "up/repo.git", upstream.server.URL+"/repo", version, true) + if err == nil || errors.Is(err, ErrNotFound) { + t.Errorf("mirror at %q = %v, want an invalid version error", version, err) + } + } +} + +func TestMirrorListsSubmodules(t *testing.T) { + upstream := newFakeUpstream(t) + libWork := upstream.newRepo("lib") + libCommit := runGit(t, libWork, "rev-parse", "HEAD") + work := upstream.newRepo("repo") + gitmodules := "[submodule \"vendor/lib\"]\n\tpath = vendor/lib\n\turl = ../lib.git\n" + + "[submodule \"other\"]\n\tpath = third_party/other\n\turl = https://example.com/other.git\n" + if err := os.WriteFile(filepath.Join(work, ".gitmodules"), []byte(gitmodules), 0o600); err != nil { + t.Fatal(err) + } + otherCommit := strings.Repeat("ab", 20) + runGit(t, work, "add", ".gitmodules") + runGit(t, work, "update-index", "--add", "--cacheinfo", "160000,"+libCommit+",vendor/lib") + runGit(t, work, "update-index", "--add", "--cacheinfo", "160000,"+otherCommit+",third_party/other") + runGit(t, work, "commit", "--quiet", "-m", "submodules") + runGit(t, work, "push", "--quiet", "origin", "main") + commit := runGit(t, work, "rev-parse", "HEAD") + tc := newTestCache(t, upstream.server.URL, nil) + + got, err := tc.Mirror(context.Background(), "up/repo.git", upstream.server.URL+"/repo", commit, true) + if err != nil { + t.Fatalf("Mirror: %v", err) + } + want := []Submodule{ + {Path: "third_party/other", URL: "https://example.com/other.git", Commit: otherCommit}, + {Path: "vendor/lib", URL: "../lib.git", Commit: libCommit}, + } + if !slices.Equal(got.Submodules, want) { + t.Errorf("submodules %+v, want %+v", got.Submodules, want) + } + + // Without a version, nothing names a commit to read submodules from. + got, err = tc.Mirror(context.Background(), "up/repo.git", upstream.server.URL+"/repo", "", true) + if err != nil || len(got.Submodules) != 0 { + t.Errorf("unversioned mirror = %+v, %v, want no submodules", got, err) + } +} diff --git a/internal/handler/git_mirror.go b/internal/handler/git_mirror.go index 709df31..38be233 100644 --- a/internal/handler/git_mirror.go +++ b/internal/handler/git_mirror.go @@ -3,22 +3,114 @@ package handler import ( "context" "fmt" + "log/slog" "net/url" + "path" "strings" "github.com/git-pkgs/proxy/internal/gitcache" ) +// maxSubmoduleDepth bounds how deeply Mirror follows submodules of +// submodules. +const maxSubmoduleDepth = 8 + // Mirror caches the repository at location, a host and path such as // github.com/acme/widgets as a pkg:git PURL names it, from the upstream.git -// host serving it, and with commit, a full commit ID, that commit too. -// Clones through /git are then served from that copy. -func (h *GitHandler) Mirror(ctx context.Context, location, commit string) (gitcache.MirrorResult, error) { +// host serving it. With version, a full commit ID or a tag, it caches that +// commit too, and the commits its submodules pin, from the upstream.git +// hosts serving them. A submodule no upstream.git host serves is skipped. +// Clones through /git are then served from those copies. +func (h *GitHandler) Mirror(ctx context.Context, location, version string) (gitcache.MirrorResult, error) { + return h.mirror(ctx, location, version, map[string]bool{}, 0) +} + +func (h *GitHandler) mirror(ctx context.Context, location, version string, seen map[string]bool, depth int) (gitcache.MirrorResult, error) { repo, err := h.mirrorRepository(location) if err != nil { return gitcache.MirrorResult{}, err } - return h.cache.Mirror(ctx, repo.key, repo.url, commit, repo.upstream.fetchWants) + result, err := h.cache.Mirror(ctx, repo.key, repo.url, version, repo.upstream.fetchWants) + if err != nil { + return result, err + } + for _, sub := range result.Submodules { + subLocation, err := gitSubmoduleLocation(repo.url, sub.URL) + if err == nil { + _, err = h.mirrorRepository(subLocation) + } + if err != nil { + h.logger().Warn("git mirror: skipping submodule", "repository", location, "submodule", sub.Path, "url", sub.URL, "error", err) + continue + } + if seen[subLocation+"@"+sub.Commit] { + continue + } + seen[subLocation+"@"+sub.Commit] = true + if depth >= maxSubmoduleDepth { + return result, fmt.Errorf("submodule %s: nested more than %d deep", sub.Path, maxSubmoduleDepth) + } + subResult, err := h.mirror(ctx, subLocation, sub.Commit, seen, depth+1) + if err != nil { + return result, fmt.Errorf("submodule %s: %w", sub.Path, err) + } + result.Cached = result.Cached && subResult.Cached + result.Bytes += subResult.Bytes + } + return result, nil +} + +func (h *GitHandler) logger() *slog.Logger { + if h.proxy != nil && h.proxy.Logger != nil { + return h.proxy.Logger + } + return slog.Default() +} + +// gitSubmoduleLocation turns the URL .gitmodules gives a submodule into a +// location as Mirror takes it. A relative URL is resolved against the +// superproject's repoURL, as git does; ssh, git and scp-like URLs name the +// host and path the repository has there. +func gitSubmoduleLocation(repoURL, submoduleURL string) (string, error) { + var host, repoPath string + switch { + case strings.HasPrefix(submoduleURL, "./"), strings.HasPrefix(submoduleURL, "../"): + u, err := url.Parse(repoURL) + if err != nil { + return "", err + } + host, repoPath = u.Host, path.Join(u.Path, submoduleURL) + case strings.Contains(submoduleURL, "://"): + u, err := url.Parse(submoduleURL) + if err != nil { + return "", err + } + switch u.Scheme { + case "http", "https": + host = u.Host + case "ssh", "git", "git+ssh", "ssh+git": + host = u.Hostname() + default: + return "", fmt.Errorf("unsupported submodule URL %q", submoduleURL) + } + repoPath = u.Path + default: + // scp-like: [user@]host:path + hostPart, p, ok := strings.Cut(submoduleURL, ":") + if !ok || strings.Contains(hostPart, "/") { + return "", fmt.Errorf("unsupported submodule URL %q", submoduleURL) + } + _, host, _ = strings.Cut(hostPart, "@") + if host == "" { + host = hostPart + } + repoPath = p + } + repoPath = strings.Trim(path.Clean("/"+repoPath), "/") + if host == "" || repoPath == "" { + return "", fmt.Errorf("unsupported submodule URL %q", submoduleURL) + } + return strings.ToLower(host) + "/" + repoPath, nil } // mirrorRepository resolves location to a repository of the upstream.git diff --git a/internal/handler/git_mirror_test.go b/internal/handler/git_mirror_test.go index 4ebcf98..227bd71 100644 --- a/internal/handler/git_mirror_test.go +++ b/internal/handler/git_mirror_test.go @@ -2,6 +2,8 @@ package handler import ( "context" + "os" + "path/filepath" "strings" "testing" @@ -71,3 +73,82 @@ func TestGitMirrorCachesCommitForClones(t *testing.T) { t.Errorf("cloned HEAD %s, want %s", head, sha) } } + +func TestGitSubmoduleLocation(t *testing.T) { + tests := []struct { + repoURL, submoduleURL, want string + }{ + {"https://github.com/acme/widgets", "../lib.git", "github.com/acme/lib.git"}, + {"https://github.com/acme/widgets.git", "../../other/lib", "github.com/other/lib"}, + {"https://github.com/acme/widgets", "./vendored", "github.com/acme/widgets/vendored"}, + {"http://127.0.0.1:8080/team/repo", "../lib.git", "127.0.0.1:8080/team/lib.git"}, + {"https://github.com/acme/widgets", "https://GitLab.com/group/lib.git/", "gitlab.com/group/lib.git"}, + {"https://github.com/acme/widgets", "ssh://git@github.com:22/acme/lib.git", "github.com/acme/lib.git"}, + {"https://github.com/acme/widgets", "git://git.example/lib", "git.example/lib"}, + {"https://github.com/acme/widgets", "git@github.com:acme/lib.git", "github.com/acme/lib.git"}, + {"https://github.com/acme/widgets", "/srv/git/lib.git", ""}, + {"https://github.com/acme/widgets", "file:///srv/git/lib.git", ""}, + {"https://github.com/acme/widgets", "", ""}, + } + for _, tt := range tests { + got, err := gitSubmoduleLocation(tt.repoURL, tt.submoduleURL) + if tt.want == "" { + if err == nil { + t.Errorf("location of %q = %q, want an error", tt.submoduleURL, got) + } + continue + } + if err != nil || got != tt.want { + t.Errorf("location of %q from %s = %q, %v; want %q", tt.submoduleURL, tt.repoURL, got, err, tt.want) + } + } +} + +// TestGitMirrorCachesSubmodules mirrors a commit with a submodule served by +// the same upstream and one on a host no upstream.git serves, which is +// skipped. +func TestGitMirrorCachesSubmodules(t *testing.T) { + g := newGitTestSetup(t, 0, 0) + libBare := filepath.Join(g.root, "lib.git") + runGitCommand(t, "", "init", "--quiet", "--bare", "--initial-branch=main", libBare) + libWork := t.TempDir() + runGitCommand(t, "", "clone", "--quiet", libBare, libWork) + runGitCommand(t, libWork, "commit", "--quiet", "--allow-empty", "-m", "lib") + runGitCommand(t, libWork, "push", "--quiet", "origin", "HEAD:main") + libCommit := runGitCommand(t, libWork, "rev-parse", "HEAD") + + gitmodules := "[submodule \"lib\"]\n\tpath = lib\n\turl = ../lib.git\n" + + "[submodule \"elsewhere\"]\n\tpath = elsewhere\n\turl = https://elsewhere.example/x.git\n" + if err := os.WriteFile(filepath.Join(g.work, ".gitmodules"), []byte(gitmodules), 0o600); err != nil { + t.Fatal(err) + } + runGitCommand(t, g.work, "add", ".gitmodules") + runGitCommand(t, g.work, "update-index", "--add", "--cacheinfo", "160000,"+libCommit+",lib") + runGitCommand(t, g.work, "update-index", "--add", "--cacheinfo", "160000,"+strings.Repeat("ab", 20)+",elsewhere") + runGitCommand(t, g.work, "commit", "--quiet", "-m", "submodules") + runGitCommand(t, g.work, "push", "--quiet", "origin", "HEAD:main") + sha := runGitCommand(t, g.work, "rev-parse", "HEAD") + location := strings.TrimPrefix(g.upstreamURL, "http://") + "/repo" + + got, err := g.handler.Mirror(context.Background(), location, sha) + if err != nil { + t.Fatalf("Mirror: %v", err) + } + if got.Cached { + t.Errorf("first mirror = %+v, want fetched", got) + } + if got, err = g.handler.Mirror(context.Background(), location, sha); err != nil || !got.Cached { + t.Errorf("second mirror = %+v, %v, want cached", got, err) + } + if !strings.Contains(g.logs.String(), "elsewhere.example") { + t.Errorf("log doesn't name the skipped submodule:\n%s", g.logs.String()) + } + + // With the upstream gone, the submodule's commit is served from the cache. + g.upstream.Close() + clone := t.TempDir() + runGitCommand(t, "", "clone", "--quiet", g.server.URL+"/git/up/lib", clone) + if head := runGitCommand(t, clone, "rev-parse", "HEAD"); head != libCommit { + t.Errorf("cloned lib HEAD %s, want %s", head, libCommit) + } +} diff --git a/internal/handler/git_test.go b/internal/handler/git_test.go index cf553d6..aeec2bc 100644 --- a/internal/handler/git_test.go +++ b/internal/handler/git_test.go @@ -129,7 +129,9 @@ func runGitCommand(t *testing.T, dir string, args ...string) string { // gitTestSetup is a GitHandler in front of a cache of one upstream // repository, "repo", served by git http-backend. type gitTestSetup struct { - work string + work string + // root holds the upstream's bare repositories. + root string server *httptest.Server // upstream serves the repository, at upstreamURL/repo, and handler is // the GitHandler under test. @@ -161,7 +163,7 @@ func newGitTestSetup(t *testing.T, readTimeout, writeStall time.Duration) *gitTe bare := filepath.Join(root, "repo.git") runGitCommand(t, "", "init", "--quiet", "--bare", "--initial-branch=main", bare) runGitCommand(t, bare, "config", "uploadpack.allowAnySHA1InWant", "true") - g := &gitTestSetup{work: t.TempDir(), logs: &syncLog{}} + g := &gitTestSetup{work: t.TempDir(), root: root, logs: &syncLog{}} runGitCommand(t, "", "clone", "--quiet", bare, g.work) runGitCommand(t, g.work, "commit", "--quiet", "--allow-empty", "-m", "first") runGitCommand(t, g.work, "push", "--quiet", "origin", "HEAD:main") diff --git a/internal/mirror/git.go b/internal/mirror/git.go index a4908dc..e9c04ff 100644 --- a/internal/mirror/git.go +++ b/internal/mirror/git.go @@ -9,10 +9,10 @@ import ( // GitMirrorer caches Git repositories for pkg:git items. location is the // repository's host and path, such as codeberg.org/forgejo/forgejo, and -// commit a full commit ID, or empty to cache the repository's branches and -// tags only. +// version a full commit ID or a tag, or empty to cache the repository's +// branches and tags only. type GitMirrorer interface { - Mirror(ctx context.Context, location, commit string) (gitcache.MirrorResult, error) + Mirror(ctx context.Context, location, version string) (gitcache.MirrorResult, error) } // UnavailableGit fails every pkg:git item with Reason. @@ -26,8 +26,8 @@ func (u UnavailableGit) Mirror(context.Context, string, string) (gitcache.Mirror var errNoGitUpstream = errors.New("git repositories are mirrored from upstream.git, and none is configured") -// mirrorGit caches a pkg:git repository, and the commit its version names, -// in the cache /git serves clones from. +// mirrorGit caches a pkg:git repository, and the commit its version names +// with the submodules it pins, in the cache /git serves clones from. func (m *Mirror) mirrorGit(ctx context.Context, pv PackageVersion, tracker *progressTracker) { if m.Git == nil { m.recordFailure(tracker, pv, errNoGitUpstream) From 7f2ac614c421a5b21e6eba71e82fb570e47c9dca Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sat, 7 Nov 2026 17:51:13 +0100 Subject: [PATCH 576/633] Mirror commits whose .gitmodules lists no submodule git config --get-regexp exits 1 when nothing matches, which failed the mirror of a commit with an empty .gitmodules. The submodules are now read from --list. --- internal/gitcache/mirror.go | 13 ++++++++----- internal/gitcache/mirror_test.go | 24 ++++++++++++++++++++++++ 2 files changed, 32 insertions(+), 5 deletions(-) diff --git a/internal/gitcache/mirror.go b/internal/gitcache/mirror.go index 047934f..5aaa6fb 100644 --- a/internal/gitcache/mirror.go +++ b/internal/gitcache/mirror.go @@ -127,20 +127,23 @@ func (c *Cache) submodules(ctx context.Context, dir, commit string) ([]Submodule if c.command(ctx, nil, "-C", dir, "cat-file", "-e", commit+":.gitmodules").Run() != nil { return nil, nil } - out, err := c.run(ctx, dir, "config", "-z", "--blob", commit+":.gitmodules", "--get-regexp", `^submodule\..*\.(path|url)$`) + // --list rather than --get-regexp, which fails when nothing matches. + out, err := c.run(ctx, dir, "config", "-z", "--blob", commit+":.gitmodules", "--list") if err != nil { return nil, fmt.Errorf("reading .gitmodules: %w", err) } paths, urls := map[string]string{}, map[string]string{} for entry := range strings.SplitSeq(out, "\x00") { key, value, ok := strings.Cut(entry, "\n") - name, field, dotted := cutLast(strings.TrimPrefix(key, "submodule."), ".") - if !ok || !dotted { + section, ok2 := strings.CutPrefix(key, "submodule.") + name, field, dotted := cutLast(section, ".") + if !ok || !ok2 || !dotted { continue } - if field == "path" { + switch field { + case "path": paths[value] = name - } else { + case "url": urls[name] = value } } diff --git a/internal/gitcache/mirror_test.go b/internal/gitcache/mirror_test.go index 499acfb..1f85a6c 100644 --- a/internal/gitcache/mirror_test.go +++ b/internal/gitcache/mirror_test.go @@ -291,3 +291,27 @@ func TestMirrorListsSubmodules(t *testing.T) { t.Errorf("unversioned mirror = %+v, %v, want no submodules", got, err) } } + +// TestMirrorWithEmptyGitmodules checks a .gitmodules with no path or url, +// for which git config --get-regexp finds nothing, lists no submodules. +func TestMirrorWithEmptyGitmodules(t *testing.T) { + upstream := newFakeUpstream(t) + work := upstream.newRepo("repo") + for i, content := range []string{"", "[submodule \"x\"]\n\tbranch = main\n"} { + if err := os.WriteFile(filepath.Join(work, ".gitmodules"), []byte(content), 0o600); err != nil { + t.Fatal(err) + } + runGit(t, work, "add", ".gitmodules") + runGit(t, work, "commit", "--quiet", "--allow-empty", "-m", "gitmodules "+strconv.Itoa(i)) + } + runGit(t, work, "push", "--quiet", "origin", "main") + tc := newTestCache(t, upstream.server.URL, nil) + + for _, rev := range []string{"HEAD~1", "HEAD"} { + commit := runGit(t, work, "rev-parse", rev) + got, err := tc.Mirror(context.Background(), "up/repo.git", upstream.server.URL+"/repo", commit, true) + if err != nil || len(got.Submodules) != 0 { + t.Errorf("mirror of %s = %+v, %v, want no submodules", rev, got, err) + } + } +} From d58b07b2a71ea119d6e2f257a0b053a3aa10446a Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sat, 7 Nov 2026 18:02:59 +0100 Subject: [PATCH 577/633] Skip discarded npm packument values with jsontext ScanPackument and LookupVersion read with a jsontext.Decoder, skipping values they don't keep with SkipValue and reading the one LookupVersion returns with ReadValue, instead of decoding each into a json.RawMessage. Repeated names and invalid UTF-8 are still accepted as encoding/json accepts them. Go 1.27 is now required, for encoding/json/jsontext. Benchmarks for both use the 5,000-version packument the handler benchmarks build, now shared in internal/metadata/npm/npmbench. --- go.mod | 4 +- internal/handler/npm_rewrite_bench_test.go | 39 +----- internal/metadata/npm/npmbench/npmbench.go | 42 ++++++ internal/metadata/npm/stream.go | 146 ++++++++++++--------- internal/metadata/npm/stream_bench_test.go | 31 +++++ internal/metadata/npm/stream_test.go | 29 ++++ 6 files changed, 189 insertions(+), 102 deletions(-) create mode 100644 internal/metadata/npm/npmbench/npmbench.go create mode 100644 internal/metadata/npm/stream_bench_test.go diff --git a/go.mod b/go.mod index 2f4a0ce..9b77ac2 100644 --- a/go.mod +++ b/go.mod @@ -1,8 +1,6 @@ module github.com/git-pkgs/proxy -go 1.26.7 - -toolchain go1.27.3 +go 1.27.3 require ( github.com/BurntSushi/toml v1.6.0 diff --git a/internal/handler/npm_rewrite_bench_test.go b/internal/handler/npm_rewrite_bench_test.go index 86552db..cf8b895 100644 --- a/internal/handler/npm_rewrite_bench_test.go +++ b/internal/handler/npm_rewrite_bench_test.go @@ -1,44 +1,13 @@ package handler import ( - "fmt" - "strings" "testing" + + "github.com/git-pkgs/proxy/internal/metadata/npm/npmbench" ) -// benchPackument builds a full packument shaped like a large real one: -// thousands of versions, each with dependencies, scripts and a readme. -func benchPackument(versions int) []byte { - var b strings.Builder - readme := strings.Repeat("Some documentation text. ", 160) - b.WriteString(`{"_id":"big","name":"big","dist-tags":{"latest":"1.0.` + fmt.Sprint(versions-1) + `"},"versions":{`) - for i := range versions { - if i > 0 { - b.WriteByte(',') - } - v := fmt.Sprintf("1.0.%d", i) - fmt.Fprintf(&b, `%q:{"name":"big","version":%q,"description":"A big package","main":"index.js",`+ - `"scripts":{"test":"node test.js","build":"tsc -p ."},"dependencies":{`, v, v) - for d := range 25 { - if d > 0 { - b.WriteByte(',') - } - fmt.Fprintf(&b, `"dep-%d":"^%d.0.0"`, d, d) - } - fmt.Fprintf(&b, `},"readme":%q,"dist":{"shasum":"0123456789abcdef0123456789abcdef01234567",`+ - `"integrity":"sha512-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==",`+ - `"tarball":"https://registry.npmjs.org/big/-/big-%s.tgz"}}`, readme, v) - } - b.WriteString(`},"time":{"created":"2015-01-01T00:00:00.000Z"`) - for i := range versions { - fmt.Fprintf(&b, `,"1.0.%d":"2016-01-01T00:00:00.000Z"`, i) - } - b.WriteString(`}}`) - return []byte(b.String()) -} - func BenchmarkNPMRewriteMetadata(b *testing.B) { - body := benchPackument(5000) + body := npmbench.Packument(5000) h := &NPMHandler{proxy: testProxy(), proxyURL: "http://proxy.example"} b.SetBytes(int64(len(body))) b.ReportAllocs() @@ -50,7 +19,7 @@ func BenchmarkNPMRewriteMetadata(b *testing.B) { } func BenchmarkNPMVersionTarball(b *testing.B) { - body := benchPackument(5000) + body := npmbench.Packument(5000) b.SetBytes(int64(len(body))) b.ReportAllocs() for b.Loop() { diff --git a/internal/metadata/npm/npmbench/npmbench.go b/internal/metadata/npm/npmbench/npmbench.go new file mode 100644 index 0000000..cc4ad57 --- /dev/null +++ b/internal/metadata/npm/npmbench/npmbench.go @@ -0,0 +1,42 @@ +// Package npmbench builds npm packuments for benchmarks. +package npmbench + +import ( + "fmt" + "strings" +) + +// readmeSentences makes each version's readme about 4KB. +const readmeSentences = 160 + +// Packument builds a full packument shaped like a large real one: versions +// 1.0.0 to 1.0.{versions-1}, each with dependencies, scripts and a readme, +// and a time entry for each. +func Packument(versions int) []byte { + var b strings.Builder + readme := strings.Repeat("Some documentation text. ", readmeSentences) + b.WriteString(`{"_id":"big","name":"big","dist-tags":{"latest":"1.0.` + fmt.Sprint(versions-1) + `"},"versions":{`) + for i := range versions { + if i > 0 { + b.WriteByte(',') + } + v := fmt.Sprintf("1.0.%d", i) + fmt.Fprintf(&b, `%q:{"name":"big","version":%q,"description":"A big package","main":"index.js",`+ + `"scripts":{"test":"node test.js","build":"tsc -p ."},"dependencies":{`, v, v) + for d := range 25 { + if d > 0 { + b.WriteByte(',') + } + fmt.Fprintf(&b, `"dep-%d":"^%d.0.0"`, d, d) + } + fmt.Fprintf(&b, `},"readme":%q,"dist":{"shasum":"0123456789abcdef0123456789abcdef01234567",`+ + `"integrity":"sha512-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==",`+ + `"tarball":"https://registry.npmjs.org/big/-/big-%s.tgz"}}`, readme, v) + } + b.WriteString(`},"time":{"created":"2015-01-01T00:00:00.000Z"`) + for i := range versions { + fmt.Fprintf(&b, `,"1.0.%d":"2016-01-01T00:00:00.000Z"`, i) + } + b.WriteString(`}}`) + return []byte(b.String()) +} diff --git a/internal/metadata/npm/stream.go b/internal/metadata/npm/stream.go index 3e86faa..b00f6a2 100644 --- a/internal/metadata/npm/stream.go +++ b/internal/metadata/npm/stream.go @@ -3,6 +3,7 @@ package npm import ( "bytes" "encoding/json" + "encoding/json/jsontext" "errors" "fmt" "io" @@ -44,11 +45,11 @@ func (s *Scan) Abbreviated() bool { // time and dropped. func ScanPackument(r io.Reader) (*Scan, error) { s := &Scan{occurrences: map[string]int{}} - dec := json.NewDecoder(r) + dec := newDecoder(r) if err := expectDelim(dec, '{'); err != nil { return nil, err } - for dec.More() { + for dec.PeekKind() != '}' { key, err := objectKey(dec) if err != nil { return nil, err @@ -60,16 +61,16 @@ func ScanPackument(r io.Reader) (*Scan, error) { case "time": s.occurrences[key]++ s.time = nil - err = dec.Decode(&s.time) + err = decodeValue(dec, &s.time) case "dist-tags": s.occurrences[key]++ s.distTags = nil - err = dec.Decode(&s.distTags) + err = decodeValue(dec, &s.distTags) default: if key == "modified" { s.hasModified = true } - err = skipValue(dec) + err = dec.SkipValue() } if err != nil { return nil, err @@ -78,13 +79,7 @@ func ScanPackument(r io.Reader) (*Scan, error) { if err := expectDelim(dec, '}'); err != nil { return nil, err } - if _, err := dec.Token(); err != io.EOF { - if err == nil { - err = errors.New("unexpected data after the top-level object") - } - return nil, err - } - return s, nil + return s, expectEnd(dec) } // RewriteStream is Rewrite for a packument too large to hold in memory. scan @@ -230,62 +225,82 @@ func (s *Scan) objectValue(metadata map[string]any, name string, value any) map[ return object } -func expectDelim(dec *json.Decoder, want json.Delim) error { - tok, err := dec.Token() +// newDecoder reads JSON as encoding/json does: of repeated names the last +// counts, and invalid UTF-8 in strings is replaced rather than refused. +func newDecoder(r io.Reader) *jsontext.Decoder { + return jsontext.NewDecoder(r, jsontext.AllowDuplicateNames(true), jsontext.AllowInvalidUTF8(true)) +} + +func expectDelim(dec *jsontext.Decoder, want jsontext.Kind) error { + tok, err := dec.ReadToken() if err != nil { return err } - if tok != want { - return fmt.Errorf("npm metadata: expected %v, got %v", want, tok) + if tok.Kind() != want { + return fmt.Errorf("npm metadata: expected %v, got %v", want, tok.Kind()) } return nil } -func objectKey(dec *json.Decoder) (string, error) { - tok, err := dec.Token() +// expectEnd checks nothing follows the top-level object. +func expectEnd(dec *jsontext.Decoder) error { + _, err := dec.ReadToken() + switch err { + case io.EOF: + return nil + case nil: + return errors.New("unexpected data after the top-level object") + } + return err +} + +func objectKey(dec *jsontext.Decoder) (string, error) { + tok, err := dec.ReadToken() if err != nil { return "", err } - key, ok := tok.(string) - if !ok { - return "", fmt.Errorf("npm metadata: object key is %T, not a string", tok) + if tok.Kind() != '"' { + return "", fmt.Errorf("npm metadata: object key is %v, not a string", tok.Kind()) } - return key, nil + return tok.String(), nil } -// objectKeys reads a value and returns its keys when it is an object. -func objectKeys(dec *json.Decoder) ([]string, bool, error) { - tok, err := dec.Token() +// decodeValue reads the next value into v, as encoding/json decodes it. +func decodeValue(dec *jsontext.Decoder, v any) error { + raw, err := dec.ReadValue() if err != nil { + return err + } + return json.Unmarshal(raw, v) +} + +// objectKeys reads a value and returns its keys when it is an object. The +// members' values are skipped without being kept. +func objectKeys(dec *jsontext.Decoder) ([]string, bool, error) { + kind := dec.PeekKind() + if kind != '{' && kind != '[' { + return nil, false, dec.SkipValue() + } + if _, err := dec.ReadToken(); err != nil { return nil, false, err } - delim, ok := tok.(json.Delim) - if !ok { - return nil, false, nil // a scalar, read whole by Token - } var keys []string - for dec.More() { - if delim == '{' { + for next := dec.PeekKind(); next != '}' && next != ']'; next = dec.PeekKind() { + if kind == '{' { key, err := objectKey(dec) if err != nil { return nil, false, err } keys = append(keys, key) } - if err := skipValue(dec); err != nil { + if err := dec.SkipValue(); err != nil { return nil, false, err } } - if _, err := dec.Token(); err != nil { + if _, err := dec.ReadToken(); err != nil { return nil, false, err } - return keys, delim == '{', nil -} - -// skipValue reads the next value without keeping it. -func skipValue(dec *json.Decoder) error { - var raw json.RawMessage - return dec.Decode(&raw) + return keys, kind == '{', nil } // VersionStream is Version for a packument too large to hold in memory. scan @@ -337,11 +352,11 @@ type VersionEntry struct { func LookupVersion(r io.Reader, version string) (VersionEntry, error) { var e VersionEntry var hasTime, hasModified bool - dec := json.NewDecoder(r) + dec := newDecoder(r) if err := expectDelim(dec, '{'); err != nil { return e, err } - for dec.More() { + for dec.PeekKind() != '}' { key, err := objectKey(dec) if err != nil { return e, err @@ -359,7 +374,7 @@ func LookupVersion(r io.Reader, version string) (VersionEntry, error) { } default: hasModified = hasModified || key == "modified" - err = skipValue(dec) + err = dec.SkipValue() } if err != nil { return e, err @@ -368,10 +383,7 @@ func LookupVersion(r io.Reader, version string) (VersionEntry, error) { if err := expectDelim(dec, '}'); err != nil { return e, err } - if _, err := dec.Token(); err != io.EOF { - if err == nil { - err = errors.New("unexpected data after the top-level object") - } + if err := expectEnd(dec); err != nil { return e, err } e.Abbreviated = hasModified && !hasTime @@ -379,33 +391,39 @@ func LookupVersion(r io.Reader, version string) (VersionEntry, error) { } // objectMember reads a value and returns its member key when it is an object -// that has one, the last if repeated, or nil. -func objectMember(dec *json.Decoder, key string) (json.RawMessage, error) { - tok, err := dec.Token() - if err != nil { +// that has one, the last if repeated, or nil. Other members are skipped +// without being kept. +func objectMember(dec *jsontext.Decoder, key string) (json.RawMessage, error) { + kind := dec.PeekKind() + if kind != '{' && kind != '[' { + return nil, dec.SkipValue() + } + if _, err := dec.ReadToken(); err != nil { return nil, err } - delim, ok := tok.(json.Delim) - if !ok { - return nil, nil - } var found json.RawMessage - for dec.More() { - var k string - if delim == '{' { + for next := dec.PeekKind(); next != '}' && next != ']'; next = dec.PeekKind() { + k := "" + if kind == '{' { + var err error if k, err = objectKey(dec); err != nil { return nil, err } } - var raw json.RawMessage - if err := dec.Decode(&raw); err != nil { + if kind == '{' && k == key { + raw, err := dec.ReadValue() + if err != nil { + return nil, err + } + // ReadValue's bytes are only valid until the next read. + found = bytes.Clone(raw) + continue + } + if err := dec.SkipValue(); err != nil { return nil, err } - if delim == '{' && k == key { - found = raw - } } - if _, err := dec.Token(); err != nil { + if _, err := dec.ReadToken(); err != nil { return nil, err } return found, nil diff --git a/internal/metadata/npm/stream_bench_test.go b/internal/metadata/npm/stream_bench_test.go new file mode 100644 index 0000000..827cf94 --- /dev/null +++ b/internal/metadata/npm/stream_bench_test.go @@ -0,0 +1,31 @@ +package npm + +import ( + "bytes" + "testing" + + "github.com/git-pkgs/proxy/internal/metadata/npm/npmbench" +) + +func BenchmarkScanPackument(b *testing.B) { + body := npmbench.Packument(5000) + b.SetBytes(int64(len(body))) + b.ReportAllocs() + for b.Loop() { + if _, err := ScanPackument(bytes.NewReader(body)); err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkLookupVersion(b *testing.B) { + body := npmbench.Packument(5000) + b.SetBytes(int64(len(body))) + b.ReportAllocs() + for b.Loop() { + entry, err := LookupVersion(bytes.NewReader(body), "1.0.2500") + if err != nil || entry.Entry == nil || entry.Published == "" { + b.Fatalf("LookupVersion = %+v, %v", entry, err) + } + } +} diff --git a/internal/metadata/npm/stream_test.go b/internal/metadata/npm/stream_test.go index 0c8a4ce..0bd93ae 100644 --- a/internal/metadata/npm/stream_test.go +++ b/internal/metadata/npm/stream_test.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "fmt" + "reflect" "slices" "strings" "testing" @@ -223,3 +224,31 @@ func TestLookupVersion(t *testing.T) { t.Error("expected an error for truncated JSON") } } + +// TestStreamReadsJSONAsEncodingJSON checks what the streaming readers accept +// matches what Rewrite's encoding/json does: names repeated at any depth, +// invalid UTF-8 in strings, and a version entry's bytes kept as they are. +func TestStreamReadsJSONAsEncodingJSON(t *testing.T) { + entry := "{ \"name\" : \"demo\",\n \"x\":1, \"x\":2, \"note\":\"caf\xe9\" }" + doc := "{\"readme\":\"bad \xff byte\",\"meta\":{\"k\":1,\"k\":2},\"dist-tags\":{\"latest\":\"1.0.0\",\"latest\":\"1.0.0\"}," + + "\"versions\":{\"1.0.0\":" + entry + "},\"time\":{\"1.0.0\":\"2020-01-01T00:00:00Z\"}}\n" + if !json.Valid([]byte(strings.ToValidUTF8(doc, "?"))) { + t.Fatal("test document isn't JSON") + } + + scan, err := ScanPackument(strings.NewReader(doc)) + if err != nil { + t.Fatalf("ScanPackument: %v", err) + } + if !slices.Equal(scan.versions, []string{"1.0.0"}) || !reflect.DeepEqual(scan.distTags, map[string]any{"latest": "1.0.0"}) { + t.Errorf("scan versions %v, dist-tags %v", scan.versions, scan.distTags) + } + + got, err := LookupVersion(strings.NewReader(doc), "1.0.0") + if err != nil { + t.Fatalf("LookupVersion: %v", err) + } + if string(got.Entry) != entry || got.Published != "2020-01-01T00:00:00Z" { + t.Errorf("entry %q, published %q; want the entry's bytes %q", got.Entry, got.Published, entry) + } +} From 7d84bf768211382ba40995ee0bbd25ab4c0288df Mon Sep 17 00:00:00 2001 From: Milas Bowman Date: Sat, 7 Nov 2026 15:28:26 -0500 Subject: [PATCH 578/633] chore: remove deb source pkg logic / fix root-level pkg caching --- docs/architecture.md | 8 +-- docs/configuration.md | 4 -- internal/handler/debian.go | 29 +-------- internal/handler/debian_test.go | 101 +++++--------------------------- 4 files changed, 19 insertions(+), 123 deletions(-) diff --git a/docs/architecture.md b/docs/architecture.md index cbd6dab..19d9c84 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -306,8 +306,8 @@ HTTP protocol handlers for each registry type. **DebianHandler:** - Serves `/debian` and `/debian/{name}` mirrors of APT archives (`upstream.debian_repositories`) -- Routes `.deb` packages to the artifact cache, source-package files directly to upstream, `dists/…` to the metadata cache, and proxies all other paths -- Keys the artifact cache on name, version, and architecture, scoped by repository for named archives +- Routes `.deb` packages to the artifact cache, `dists/…` to the metadata cache, and proxies all other paths +- Keys the artifact cache on name, version, and filename, scoped by repository for named archives **SwiftHandler:** - Proxies the Swift Package Registry v1 read endpoints @@ -434,10 +434,6 @@ An object a record stops pointing at, because a refetch replaced it or its entry - Set `cache_metadata: true` or use the mirror command to enable metadata caching for offline use via the `metadata_cache` table - OCI manifests and tag lists are exceptions: they are cached automatically so previously fetched images remain pullable and tag resolution works when the registry or token service is unavailable -**Why are Debian source-package files streamed instead of cached?** -- Source-package files (`.dsc`, source tarballs, `.buildinfo`, `.changes`) can be re-uploaded or re-signed in place under the same path, whereas the artifact cache keys entries by version and filename without an expected digest from the `Sources` index -- Streaming them avoids serving stale bytes when an upstream file is replaced or caching them under the `dists/` metadata TTL in flat repositories - **Why stream artifacts?** - Memory efficient - don't load large files into RAM - Better latency - start sending while still receiving diff --git a/docs/configuration.md b/docs/configuration.md index 05914eb..63cc175 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -805,10 +805,6 @@ and `dists` are refused at config load for that reason, and other root paths an archive may serve (`indices`, `project`, `doc`, `tools`) are not, so avoid those names unless the shadowing is intended. -Binary packages (`.deb`) are cached; source-package files (`.dsc`, source -tarballs, `.buildinfo`, `.changes`) are streamed directly from upstream because -their contents can be replaced in place under the same path. - This field has no environment variable, as with the other named upstream maps. `PROXY_UPSTREAM_DEBIAN` still sets `upstream.debian`. diff --git a/internal/handler/debian.go b/internal/handler/debian.go index 87dbd65..565f181 100644 --- a/internal/handler/debian.go +++ b/internal/handler/debian.go @@ -108,13 +108,11 @@ func (h *DebianHandler) Routes() http.Handler { } } - // Route by filename rather than path prefix because flat repositories - // place pools under dists//pool/.... + // Route by filename rather than path prefix because repositories such as + // Docker's nest the pool under dists//. switch { case isDebPackageFile(rest): h.handlePackageDownload(w, r, repository, upstreamURL, rest) - case isSourcePackageFile(rest): - h.proxyFile(w, r, upstreamURL, rest) case strings.HasPrefix(rest, "dists/"): // Repository metadata - served through the metadata cache h.handleMetadata(w, r, repository, upstreamURL, rest) @@ -129,22 +127,6 @@ func isDebPackageFile(path string) bool { return strings.HasSuffix(path, ".deb") } -// isSourcePackageFile reports whether path is a Debian source-package artifact. -// Source files can be re-uploaded or re-signed in place under the same filename, -// so they must be streamed rather than cached without digest verification. -func isSourcePackageFile(path string) bool { - switch { - case strings.HasSuffix(path, ".dsc"), - strings.HasSuffix(path, ".buildinfo"), - strings.HasSuffix(path, ".changes"): - return true - } - // Match Debian source tarballs without matching suite tarballs (e.g. installer netboot.tar.gz). - return strings.Contains(path, ".orig.tar.") || - strings.Contains(path, ".orig-") || - strings.Contains(path, ".debian.tar.") -} - // repositoryNotFound answers a request addressed to a repository that is not // configured, naming the ones that are so a misspelling is self-evident. func (h *DebianHandler) repositoryNotFound(w http.ResponseWriter, name string) { @@ -248,12 +230,7 @@ var debPackagePattern = regexp.MustCompile(`^(.+)_([^_]+)_([^_]+)\.deb$`) // parsePoolPath extracts package info from a pool path. // Example: pool/main/n/nginx/nginx_1.18.0-6_amd64.deb func (h *DebianHandler) parsePoolPath(path string) (name, version, arch string) { - // Get the filename - idx := strings.LastIndex(path, "/") - if idx < 0 { - return "", "", "" - } - filename := path[idx+1:] + filename := path[strings.LastIndex(path, "/")+1:] // Parse the filename matches := debPackagePattern.FindStringSubmatch(filename) diff --git a/internal/handler/debian_test.go b/internal/handler/debian_test.go index 848f9e2..54a72b6 100644 --- a/internal/handler/debian_test.go +++ b/internal/handler/debian_test.go @@ -25,6 +25,7 @@ func TestDebianHandler_parsePoolPath(t *testing.T) { "nmap", "7.91+dfsg1+really7.80+dfsg1-2ubuntu0.1", "amd64", }, {"pool/main/o/openssl/openssl_3.0.2-0ubuntu1.15~build1_amd64.deb", "openssl", "3.0.2-0ubuntu1.15~build1", "amd64"}, + {"nginx_1.18.0-6_amd64.deb", "nginx", "1.18.0-6", "amd64"}, {"invalid/path", "", "", ""}, {"pool/main/n/nginx/nginx.deb", "", "", ""}, }) @@ -251,14 +252,8 @@ func TestDebianHandler_PackageCacheScopedPerRepository(t *testing.T) { } } -// TestDebianHandler_FlatArchivePackageIsCached verifies that .deb packages -// served under dists//pool/ (as used by flat repositories like Docker's) -// are cached as artifacts rather than treated as uncached/TTL metadata. func TestDebianHandler_FlatArchivePackageIsCached(t *testing.T) { - const ( - flatPath = "/docker/dists/trixie/pool/stable/amd64/docker-ce_5.0.1-1~debian.13~trixie_amd64.deb" - pkgBody = "docker-ce package bytes" - ) + const pkgBody = "docker-ce package bytes" var upstreamHits int @@ -277,16 +272,21 @@ func TestDebianHandler_FlatArchivePackageIsCached(t *testing.T) { "docker": archive.URL, }) - for attempt := 1; attempt <= 2; attempt++ { - w := serveDebianRequest(h, flatPath) - if w.Code != http.StatusOK || w.Body.String() != pkgBody { - t.Fatalf("request %d: status = %d, body = %q, want 200 %q", - attempt, w.Code, w.Body.String(), pkgBody) + for _, path := range []string{ + "/docker/dists/trixie/pool/stable/amd64/docker-ce_5.0.1-1~debian.13~trixie_amd64.deb", + "/docker/foo_1.0_amd64.deb", + } { + for attempt := 1; attempt <= 2; attempt++ { + w := serveDebianRequest(h, path) + if w.Code != http.StatusOK || w.Body.String() != pkgBody { + t.Fatalf("%s request %d: status = %d, body = %q, want 200 %q", + path, attempt, w.Code, w.Body.String(), pkgBody) + } } } - if upstreamHits != 1 { - t.Errorf("upstream fetched %d times for two requests, want 1: the package is not cached", upstreamHits) + if upstreamHits != 2 { + t.Errorf("upstream fetched %d times for four requests, want 2: the package is not cached", upstreamHits) } } @@ -324,76 +324,3 @@ func TestDebRetentionKey(t *testing.T) { t.Error("debRetentionKey(nil) accepted") } } - -func TestDebianHandler_ClassifiesFilesByExtension(t *testing.T) { - tests := []struct { - path string - pkg bool - source bool - why string - }{ - {"pool/main/h/hello/hello_2.10-3_amd64.deb", true, false, "canonical pool"}, - {"dists/trixie/pool/stable/amd64/docker-ce_5.0.1-1~debian.13~trixie_amd64.deb", true, false, "flat archive"}, - {"pool/main/h/hello/hello_2.10-3.udeb", false, false, "udeb not matched by debPackagePattern"}, - {"pool/main/h/hello/hello_2.10-3_amd64.ddeb", false, false, "ddeb not matched by debPackagePattern"}, - {"pool/main/h/hello/hello_2.10-3.dsc", false, true, "source control file"}, - {"pool/main/h/hello/hello_2.10.orig.tar.xz", false, true, "source tarball"}, - {"pool/main/h/hello/hello_2.10.orig-tex.tar.gz", false, true, "component tarball"}, - {"pool/main/h/hello/hello_2.10-3.debian.tar.xz", false, true, "debian tarball"}, - {"pool/main/h/hello/hello_2.10-3_amd64.buildinfo", false, true, "buildinfo"}, - {"pool/main/h/hello/hello_2.10-3_amd64.changes", false, true, "changes"}, - {"dists/trixie/pool/stable/amd64/docker-ce_5.0.1-1.dsc", false, true, "flat archive source file"}, - {"dists/trixie/InRelease", false, false, "metadata"}, - {"dists/trixie/main/binary-amd64/Packages.gz", false, false, "metadata"}, - {"dists/trixie/main/installer-amd64/current/images/netboot/netboot.tar.gz", false, false, "suite tarball"}, - } - for _, tt := range tests { - if got := isDebPackageFile(tt.path); got != tt.pkg { - t.Errorf("isDebPackageFile(%q) = %v, want %v (%s)", tt.path, got, tt.pkg, tt.why) - } - if got := isSourcePackageFile(tt.path); got != tt.source { - t.Errorf("isSourcePackageFile(%q) = %v, want %v (%s)", tt.path, got, tt.source, tt.why) - } - } -} - -func TestDebianHandler_SourcePackageFilesAreNotCached(t *testing.T) { - const dscBody = "hello source control bytes" - - var upstreamHits int - - archive := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - upstreamHits++ - _, _ = fmt.Fprint(w, dscBody) - })) - defer archive.Close() - - proxy, _, store, _ := setupTestProxy(t) - proxy.HTTPClient = archive.Client() - proxy.CacheMetadata = true - proxy.MetadataTTL = time.Hour - - h := NewDebianHandler(proxy, "http://proxy.example", archive.URL, map[string]string{ - "docker": archive.URL, - }) - - for _, target := range []string{ - "/pool/main/h/hello/hello_2.10-3.dsc", - "/docker/dists/trixie/pool/stable/amd64/docker-ce_5.0.1-1.dsc", - } { - for attempt := 1; attempt <= 2; attempt++ { - w := serveDebianRequest(h, target) - if w.Code != http.StatusOK || w.Body.String() != dscBody { - t.Fatalf("%s request %d: status = %d, body = %q, want 200 %q", - target, attempt, w.Code, w.Body.String(), dscBody) - } - } - } - - if upstreamHits != 4 { - t.Errorf("upstream fetched %d times for four requests, want 4: source-package files must not be cached", upstreamHits) - } - if store.stores != 0 { - t.Errorf("storage wrote %d object(s), want 0: a source-package file must not be stored", store.stores) - } -} From 70d9db29065099cade5ed82f149233d501009342 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 8 Nov 2026 07:55:24 +0100 Subject: [PATCH 579/633] Add CRAN routes for macOS arch layout and Meta files R 4.3 and later on macOS fetch binaries from a two-segment platform path such as bin/macosx/big-sur-arm64/contrib/4.4/. The router only matched a single platform segment, so both the PACKAGES indexes and the .tgz downloads returned 404. Add routes for that layout and include the arch in the stored version so arm64 and x86_64 builds stay apart. src/contrib/Meta/archive.rds and current.rds, which remotes and pak use to install archived versions, had no route either. Serve .rds files under Meta as cached metadata. --- README.md | 2 + internal/handler/cran.go | 23 ++++++++++++ internal/handler/cran_test.go | 71 +++++++++++++++++++++++++++++++++++ 3 files changed, 96 insertions(+) diff --git a/README.md b/README.md index a43c417..dff6964 100644 --- a/README.md +++ b/README.md @@ -498,6 +498,8 @@ local({ }) ``` +Source packages, Windows and macOS binaries (including the `bin/macosx/big-sur-arm64/` layout used by R 4.3 and later) are cached as artifacts. `PACKAGES` indexes and the `src/contrib/Meta/*.rds` files that `remotes::install_version` and pak read are cached as metadata. + ### Julia Set the Pkg server before starting Julia: diff --git a/internal/handler/cran.go b/internal/handler/cran.go index 7cba9f4..10f1750 100644 --- a/internal/handler/cran.go +++ b/internal/handler/cran.go @@ -51,21 +51,41 @@ func (h *CRANHandler) Routes() http.Handler { mux.HandleFunc("GET /src/contrib/PACKAGES.gz", h.proxyCached) mux.HandleFunc("GET /src/contrib/PACKAGES.rds", h.proxyCached) + // Repository metadata such as archive.rds and current.rds, used by + // remotes::install_version and pak to find archived versions. + mux.HandleFunc("GET /src/contrib/Meta/{filename}", h.handleMeta) + // Binary package indexes mux.HandleFunc("GET /bin/{platform}/contrib/{rversion}/PACKAGES", h.proxyCached) mux.HandleFunc("GET /bin/{platform}/contrib/{rversion}/PACKAGES.gz", h.proxyCached) mux.HandleFunc("GET /bin/{platform}/contrib/{rversion}/PACKAGES.rds", h.proxyCached) + // macOS binaries for R >= 4.3 use two platform segments, e.g. + // bin/macosx/big-sur-arm64/contrib/4.4/. + mux.HandleFunc("GET /bin/{platform}/{arch}/contrib/{rversion}/PACKAGES", h.proxyCached) + mux.HandleFunc("GET /bin/{platform}/{arch}/contrib/{rversion}/PACKAGES.gz", h.proxyCached) + mux.HandleFunc("GET /bin/{platform}/{arch}/contrib/{rversion}/PACKAGES.rds", h.proxyCached) + // Source package downloads mux.HandleFunc("GET /src/contrib/{filename}", h.handleSourceDownload) mux.HandleFunc("GET /src/contrib/Archive/{name}/{filename}", h.handleSourceDownload) // Binary package downloads mux.HandleFunc("GET /bin/{platform}/contrib/{rversion}/{filename}", h.handleBinaryDownload) + mux.HandleFunc("GET /bin/{platform}/{arch}/contrib/{rversion}/{filename}", h.handleBinaryDownload) return rejectQueryChars(mux) } +// handleMeta serves the .rds files under src/contrib/Meta as cached metadata. +func (h *CRANHandler) handleMeta(w http.ResponseWriter, r *http.Request) { + if !strings.HasSuffix(r.PathValue("filename"), ".rds") { + http.NotFound(w, r) + return + } + h.proxyCached(w, r) +} + // handleSourceDownload serves a source package, fetching and caching from upstream. func (h *CRANHandler) handleSourceDownload(w http.ResponseWriter, r *http.Request) { filename := r.PathValue("filename") @@ -99,6 +119,9 @@ func (h *CRANHandler) handleSourceDownload(w http.ResponseWriter, r *http.Reques // handleBinaryDownload serves a binary package, fetching and caching from upstream. func (h *CRANHandler) handleBinaryDownload(w http.ResponseWriter, r *http.Request) { platform := r.PathValue("platform") + if arch := r.PathValue("arch"); arch != "" { + platform += "-" + arch + } rversion := r.PathValue("rversion") filename := r.PathValue("filename") diff --git a/internal/handler/cran_test.go b/internal/handler/cran_test.go index 6a9c151..6c156c5 100644 --- a/internal/handler/cran_test.go +++ b/internal/handler/cran_test.go @@ -1,8 +1,14 @@ package handler import ( + "io" "log/slog" + "net/http" + "net/http/httptest" + "strings" "testing" + + "github.com/git-pkgs/registries/fetch" ) func TestCRANParseSourceFilename(t *testing.T) { @@ -50,3 +56,68 @@ func TestCRANParseBinaryFilename(t *testing.T) { } } } + +func TestCRANHandler_MacOSTwoSegmentIndexesAndMeta(t *testing.T) { + for _, path := range []string{ + "/bin/macosx/big-sur-arm64/contrib/4.4/PACKAGES", + "/bin/macosx/big-sur-x86_64/contrib/4.4/PACKAGES.gz", + "/bin/macosx/sonoma-arm64/contrib/4.5/PACKAGES.rds", + "/src/contrib/Meta/archive.rds", + "/src/contrib/Meta/current.rds", + } { + t.Run(path, func(t *testing.T) { + assertUpstreamProxied(t, "index", path, + func(proxy *Proxy, upstreamURL string) http.Handler { + h := &CRANHandler{proxy: proxy, upstreamURL: upstreamURL, proxyURL: "http://localhost"} + return h.Routes() + }, + ) + }) + } +} + +func TestCRANHandler_MacOSTwoSegmentBinaryDownload(t *testing.T) { + proxy, _, _, fetcher := setupTestProxy(t) + fetcher.artifact = &fetch.Artifact{ + Body: io.NopCloser(strings.NewReader("fetched binary")), + ContentType: "application/x-gzip", + } + + h := NewCRANHandler(proxy, "http://localhost") + h.upstreamURL = "https://cran.r-project.org" + srv := httptest.NewServer(h.Routes()) + defer srv.Close() + + resp, err := http.Get(srv.URL + "/bin/macosx/big-sur-arm64/contrib/4.4/dplyr_1.1.4.tgz") + if err != nil { + t.Fatalf("request failed: %v", err) + } + _ = resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + t.Errorf("status = %d, want %d", resp.StatusCode, http.StatusOK) + } + want := "https://cran.r-project.org/bin/macosx/big-sur-arm64/contrib/4.4/dplyr_1.1.4.tgz" + if fetcher.fetchedURL != want { + t.Errorf("upstream URL = %q, want %q", fetcher.fetchedURL, want) + } +} + +func TestCRANHandler_MacOSTwoSegmentBinaryCacheIdentity(t *testing.T) { + proxy, db, store, _ := setupTestProxy(t) + seedPackageWithPURL(t, db, store, "cran", "dplyr", "1.1.4_macosx-big-sur-arm64_4.4", "dplyr_1.1.4.tgz", "arm64 binary") + + h := NewCRANHandler(proxy, "http://localhost") + srv := httptest.NewServer(h.Routes()) + defer srv.Close() + + resp, err := http.Get(srv.URL + "/bin/macosx/big-sur-arm64/contrib/4.4/dplyr_1.1.4.tgz") + if err != nil { + t.Fatalf("request failed: %v", err) + } + defer func() { _ = resp.Body.Close() }() + body, _ := io.ReadAll(resp.Body) + if string(body) != "arm64 binary" { + t.Errorf("body = %q, want %q", body, "arm64 binary") + } +} From cbbeed423b741d7eb64e51cdd79be82d5f5f862c Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 8 Nov 2026 07:56:52 +0100 Subject: [PATCH 580/633] Cache RPM packages under their full path RPM packages were cached under their bare filename. With upstream.rpm pointing at a mirror root that serves several distributions, Rocky and Alma both publish bash-5.1.8-9.el9.x86_64.rpm with different bytes, and whichever was fetched first was served for both, so dnf failed the checksum or signature check. Use the path under the upstream as the cache filename, as the APK and Arch handlers do. Packages cached before this change are no longer found and are fetched again once; retention still matches them by package name. --- docs/configuration.md | 2 +- internal/handler/rpm.go | 9 +++-- internal/handler/rpm_test.go | 50 +++++++++++++++++++++++++++ internal/server/retention_rpm_test.go | 4 +-- 4 files changed, 59 insertions(+), 6 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 584896b..9046e28 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1820,7 +1820,7 @@ curl -X DELETE http://localhost:8080/api/cache/maven/com.google.guava:guava/32.1 curl -X DELETE 'http://localhost:8080/api/cache/maven/com.google.guava:guava/32.1.3-jre?filename=guava-32.1.3-jre.jar' ``` -The name is the package name the UI shows, for example `group:artifact` for Maven. The response lists the deleted files. The next request for a deleted file fetches it from the upstream again. The stored bytes are removed after the same delay as a replaced artifact, so downloads already in progress can finish. +The name is the package name the UI shows, for example `group:artifact` for Maven. An RPM package's `filename` is its path under `upstream.rpm` (`updates/40/Everything/x86_64/Packages/c/curl-8.6.0-10.fc40.x86_64.rpm`), since repositories under one upstream can ship different files under one name. The response lists the deleted files. The next request for a deleted file fetches it from the upstream again. The stored bytes are removed after the same delay as a replaced artifact, so downloads already in progress can finish. Pushed archives are not touched: a whole-version delete skips them, and naming one answers 409. Delete those through the [hosted API](#hosted-packages). diff --git a/internal/handler/rpm.go b/internal/handler/rpm.go index 95163b9..49773a7 100644 --- a/internal/handler/rpm.go +++ b/internal/handler/rpm.go @@ -93,6 +93,10 @@ func (h *RPMHandler) Routes() http.Handler { // Path format varies by repo structure, e.g.: // - releases/39/Everything/x86_64/os/Packages/n/nginx-1.24.0-1.fc39.x86_64.rpm // - updates/39/Everything/x86_64/Packages/n/nginx-1.24.0-2.fc39.x86_64.rpm +// +// The cache key is the full path, since repositories under one upstream +// (Rocky and Alma on one mirror, say) can ship different bytes under one +// filename. func (h *RPMHandler) handlePackageDownload(w http.ResponseWriter, r *http.Request, path string) { // Parse the path to extract package info name, version, arch := h.parseRPMPath(path) @@ -102,14 +106,13 @@ func (h *RPMHandler) handlePackageDownload(w http.ResponseWriter, r *http.Reques return } - filename := path[strings.LastIndex(path, "/")+1:] downloadURL := fmt.Sprintf("%s/%s", h.upstreamURL, path) h.proxy.Logger.Info("rpm package download", - "name", name, "version", version, "arch", arch, "filename", filename) + "name", name, "version", version, "arch", arch, "path", path) result, err := h.proxy.GetOrFetchArtifactFromURL( - r.Context(), "rpm", name, version, filename, downloadURL) + r.Context(), "rpm", name, version, path, downloadURL) if err != nil { h.proxy.serveArtifactError(w, err, "failed to fetch package") return diff --git a/internal/handler/rpm_test.go b/internal/handler/rpm_test.go index 3526dbd..87275b2 100644 --- a/internal/handler/rpm_test.go +++ b/internal/handler/rpm_test.go @@ -1,9 +1,14 @@ package handler import ( + "io" + "net/http" + "net/http/httptest" + "sync/atomic" "testing" "github.com/git-pkgs/purl" + "github.com/git-pkgs/registries/fetch" ) func TestRPMHandler_parseRPMPath(t *testing.T) { @@ -52,3 +57,48 @@ func TestRPMRetentionKey(t *testing.T) { t.Error("rpmRetentionKey(nil) accepted") } } + +// TestRPMHandler_PackageDownloadCachesPerRepository checks that two +// repositories under one upstream that ship the same filename with different +// bytes are cached apart, as on a mirror root serving several distributions. +func TestRPMHandler_PackageDownloadCachesPerRepository(t *testing.T) { + packages := map[string]string{ + "/rocky/9/BaseOS/x86_64/os/Packages/b/bash-5.1.8-9.el9.x86_64.rpm": "rocky bash", + "/almalinux/9/BaseOS/x86_64/os/Packages/b/bash-5.1.8-9.el9.x86_64.rpm": "alma bash", + } + + var available atomic.Bool + available.Store(true) + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + data, ok := packages[r.URL.Path] + if !available.Load() || !ok { + http.NotFound(w, r) + return + } + _, _ = w.Write([]byte(data)) + })) + defer upstream.Close() + + proxy, _, _, _ := setupTestProxy(t) + fetcher := fetch.NewFetcher(fetch.WithHTTPClient(upstream.Client()), fetch.WithMaxRetries(0)) + proxy.Fetcher = fetcher + t.Cleanup(func() { _ = fetcher.Close() }) + srv := httptest.NewServer(NewRPMHandlerWithUpstream(proxy, testProxyURL, upstream.URL).Routes()) + defer srv.Close() + + for round := range 2 { + for path, want := range packages { + resp, err := http.Get(srv.URL + path) + if err != nil { + t.Fatal(err) + } + body, _ := io.ReadAll(resp.Body) + _ = resp.Body.Close() + if resp.StatusCode != http.StatusOK || string(body) != want { + t.Errorf("round %d %s: %d %q, want 200 %q", round, path, resp.StatusCode, body, want) + } + } + // The second round must come from the cache. + available.Store(false) + } +} diff --git a/internal/server/retention_rpm_test.go b/internal/server/retention_rpm_test.go index 8c0536b..bc55614 100644 --- a/internal/server/retention_rpm_test.go +++ b/internal/server/retention_rpm_test.go @@ -20,8 +20,8 @@ import ( const ( rpmCurlPath = "/updates/40/Everything/x86_64/Packages/c/curl-8.6.0-10.fc40.x86_64.rpm" rpmNMPath = "/updates/40/Everything/x86_64/Packages/n/NetworkManager-1.46.2-1.fc40.x86_64.rpm" - rpmCurlFile = "curl-8.6.0-10.fc40.x86_64.rpm" - rpmNMFile = "NetworkManager-1.46.2-1.fc40.x86_64.rpm" + rpmCurlFile = "updates/40/Everything/x86_64/Packages/c/curl-8.6.0-10.fc40.x86_64.rpm" + rpmNMFile = "updates/40/Everything/x86_64/Packages/n/NetworkManager-1.46.2-1.fc40.x86_64.rpm" ) // rpmRetentionUpstream serves package files the way a Fedora mirror does, From b9f047f8dc7f21e1d3ff64ef91508dea4462ba32 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 8 Nov 2026 07:56:56 +0100 Subject: [PATCH 581/633] Parse boolean environment variables strictly Every boolean PROXY_* variable was true only for exactly "true" or "1". Anything else, including "True", "TRUE" or a typo, silently meant false, so PROXY_SCANNING_ENABLED=True turned scanning off and validation still passed. Parse them with strconv.ParseBool like PROXY_GRADLE_BUILD_CACHE_READ_ONLY already does. An unrecognised value now keeps the current setting and is logged as a warning at startup. --- docs/configuration.md | 2 ++ internal/config/config.go | 55 ++++++++++++++++++---------------- internal/config/config_test.go | 34 +++++++++++++++++++++ 3 files changed, 66 insertions(+), 25 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 584896b..40fb55d 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -2,6 +2,8 @@ The proxy can be configured via command line flags, environment variables, or a configuration file. Command line flags take precedence over environment variables, which take precedence over the configuration file. +Boolean environment variables accept `true`, `True`, `TRUE`, `t`, `T` or `1`, and `false`, `False`, `FALSE`, `f`, `F` or `0`. Any other value, such as `yes` or a typo, is ignored: the setting keeps its value from the configuration file or default, and the proxy logs a warning at startup. + ## Configuration File Create a YAML or JSON file and pass it with `-config`: diff --git a/internal/config/config.go b/internal/config/config.go index 3b5c6cb..5c74d3c 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -1390,11 +1390,20 @@ func setEnvString(dst *string, key string) { } } -// setEnvBool is setEnvString for boolean fields, parsed via envBool. -func setEnvBool(dst *bool, key string) { - if v := os.Getenv(key); v != "" { - *dst = envBool(v) +// setEnvBool is setEnvString for boolean fields. It accepts what +// strconv.ParseBool does; any other value leaves *dst unchanged and is +// reported via EnvWarnings. +func (c *Config) setEnvBool(dst *bool, key string) { + v := os.Getenv(key) + if v == "" { + return } + b, err := strconv.ParseBool(strings.TrimSpace(v)) + if err != nil { + c.envWarnings = append(c.envWarnings, fmt.Sprintf("ignoring %s=%q: not a boolean, keeping %t", key, v, *dst)) + return + } + *dst = b } // setGradleReadOnlyFromEnv parses strictly: an unrecognised value keeps @@ -1454,16 +1463,16 @@ func (c *Config) LoadFromEnv() { setEnvString(&c.Listen, "PROXY_LISTEN") setEnvString(&c.BaseURL, "PROXY_BASE_URL") setEnvString(&c.UIBaseURL, "PROXY_UI_URL") - setEnvBool(&c.TrustForwardedFor, "PROXY_TRUST_FORWARDED_FOR") - setEnvBool(&c.UIRequestSources, "PROXY_UI_REQUEST_SOURCES") - setEnvBool(&c.UIFailureDetails, "PROXY_UI_FAILURE_DETAILS") + c.setEnvBool(&c.TrustForwardedFor, "PROXY_TRUST_FORWARDED_FOR") + c.setEnvBool(&c.UIRequestSources, "PROXY_UI_REQUEST_SOURCES") + c.setEnvBool(&c.UIFailureDetails, "PROXY_UI_FAILURE_DETAILS") setEnvString(&c.Storage.URL, "PROXY_STORAGE_URL") setEnvString(&c.Storage.Path, "PROXY_STORAGE_PATH") setEnvString(&c.Storage.MaxSize, "PROXY_STORAGE_MAX_SIZE") - setEnvBool(&c.Storage.DirectServe, "PROXY_STORAGE_DIRECT_SERVE") + c.setEnvBool(&c.Storage.DirectServe, "PROXY_STORAGE_DIRECT_SERVE") setEnvString(&c.Storage.DirectServeTTL, "PROXY_STORAGE_DIRECT_SERVE_TTL") setEnvString(&c.Storage.DirectServeBaseURL, "PROXY_STORAGE_DIRECT_SERVE_BASE_URL") - setEnvBool(&c.Storage.CacheArtifacts, "PROXY_STORAGE_CACHE_ARTIFACTS") + c.setEnvBool(&c.Storage.CacheArtifacts, "PROXY_STORAGE_CACHE_ARTIFACTS") setEnvString(&c.Storage.Retention.Default, "PROXY_STORAGE_RETENTION_DEFAULT") setEnvString(&c.Storage.Retention.SweepInterval, "PROXY_STORAGE_RETENTION_SWEEP_INTERVAL") setEnvString(&c.Database.Driver, "PROXY_DATABASE_DRIVER") @@ -1474,11 +1483,11 @@ func (c *Config) LoadFromEnv() { setEnvString(&c.Log.Format, "PROXY_LOG_FORMAT") setEnvString(&c.AccessLog.Path, "PROXY_ACCESS_LOG_PATH") setEnvStringSlice(&c.Upstream.AllowPrivateHosts, "PROXY_UPSTREAM_ALLOW_PRIVATE_HOSTS") - setEnvBool(&c.Upstream.AllowLoopback, "PROXY_UPSTREAM_ALLOW_LOOPBACK") + c.setEnvBool(&c.Upstream.AllowLoopback, "PROXY_UPSTREAM_ALLOW_LOOPBACK") setEnvString(&c.Upstream.NPM, "PROXY_UPSTREAM_NPM") - setEnvBool(&c.Upstream.NPMFullMetadata, "PROXY_UPSTREAM_NPM_FULL_METADATA") - setEnvBool(&c.Upstream.NPMChangeFeed, "PROXY_UPSTREAM_NPM_CHANGE_FEED") - setEnvBool(&c.Upstream.GoSumDBsOffline, "PROXY_UPSTREAM_GO_SUMDBS_OFFLINE") + c.setEnvBool(&c.Upstream.NPMFullMetadata, "PROXY_UPSTREAM_NPM_FULL_METADATA") + c.setEnvBool(&c.Upstream.NPMChangeFeed, "PROXY_UPSTREAM_NPM_CHANGE_FEED") + c.setEnvBool(&c.Upstream.GoSumDBsOffline, "PROXY_UPSTREAM_GO_SUMDBS_OFFLINE") setEnvString(&c.Upstream.Cargo, "PROXY_UPSTREAM_CARGO") setEnvString(&c.Upstream.CargoDownload, "PROXY_UPSTREAM_CARGO_DOWNLOAD") setEnvString(&c.Upstream.Gem, "PROXY_UPSTREAM_GEM") @@ -1495,7 +1504,7 @@ func (c *Config) LoadFromEnv() { setEnvString(&c.Upstream.NuGetSearch, "PROXY_UPSTREAM_NUGET_SEARCH") setEnvString(&c.Upstream.Composer, "PROXY_UPSTREAM_COMPOSER") setEnvString(&c.Upstream.ComposerRepository, "PROXY_UPSTREAM_COMPOSER_REPOSITORY") - setEnvBool(&c.Upstream.ComposerChangeFeed, "PROXY_UPSTREAM_COMPOSER_CHANGE_FEED") + c.setEnvBool(&c.Upstream.ComposerChangeFeed, "PROXY_UPSTREAM_COMPOSER_CHANGE_FEED") setEnvString(&c.Upstream.Conan, "PROXY_UPSTREAM_CONAN") setEnvString(&c.Upstream.Chef, "PROXY_UPSTREAM_CHEF") setEnvString(&c.Upstream.Conda, "PROXY_UPSTREAM_CONDA") @@ -1509,20 +1518,20 @@ func (c *Config) LoadFromEnv() { setEnvString(&c.Upstream.HomebrewAPI, "PROXY_UPSTREAM_HOMEBREW_API") setEnvString(&c.Upstream.HomebrewArtifact, "PROXY_UPSTREAM_HOMEBREW_ARTIFACT") setEnvString(&c.Cooldown.Default, "PROXY_COOLDOWN_DEFAULT") - setEnvBool(&c.Scanning.Enabled, "PROXY_SCANNING_ENABLED") - setEnvBool(&c.Scanning.FailOpen, "PROXY_SCANNING_FAIL_OPEN") + c.setEnvBool(&c.Scanning.Enabled, "PROXY_SCANNING_ENABLED") + c.setEnvBool(&c.Scanning.FailOpen, "PROXY_SCANNING_FAIL_OPEN") setEnvString(&c.Scanning.Timeout, "PROXY_SCANNING_TIMEOUT") setEnvString(&c.Scanning.SigningKey, "PROXY_SCANNING_SIGNING_KEY") setEnvString(&c.Scanning.FetchBaseURL, "PROXY_SCANNING_FETCH_BASE_URL") - setEnvBool(&c.CacheMetadata, "PROXY_CACHE_METADATA") - setEnvBool(&c.MirrorAPI, "PROXY_MIRROR_API") - setEnvBool(&c.CacheDelete, "PROXY_CACHE_DELETE") - setEnvBool(&c.Hosted.Enabled, "PROXY_HOSTED_ENABLED") + c.setEnvBool(&c.CacheMetadata, "PROXY_CACHE_METADATA") + c.setEnvBool(&c.MirrorAPI, "PROXY_MIRROR_API") + c.setEnvBool(&c.CacheDelete, "PROXY_CACHE_DELETE") + c.setEnvBool(&c.Hosted.Enabled, "PROXY_HOSTED_ENABLED") setEnvString(&c.Hosted.Token, "PROXY_HOSTED_TOKEN") setEnvString(&c.Hosted.MaxUploadSize, "PROXY_HOSTED_MAX_UPLOAD_SIZE") setEnvStringSlice(&c.Hosted.NPMScopes, "PROXY_HOSTED_NPM_SCOPES") setEnvStringSlice(&c.Hosted.ComposerVendors, "PROXY_HOSTED_COMPOSER_VENDORS") - setEnvBool(&c.Projects.Enabled, "PROXY_PROJECTS_ENABLED") + c.setEnvBool(&c.Projects.Enabled, "PROXY_PROJECTS_ENABLED") setEnvString(&c.Projects.Token, "PROXY_PROJECTS_TOKEN") setEnvString(&c.Projects.MaxAge, "PROXY_PROJECTS_MAX_AGE") setEnvString(&c.MetadataTTL, "PROXY_METADATA_TTL") @@ -2176,7 +2185,3 @@ func (a *AuthConfig) Header() (name, value string) { func expandEnv(s string) string { return os.Expand(s, os.Getenv) } - -func envBool(v string) bool { - return v == "true" || v == "1" -} diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 91e9d84..ac6dfff 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -2169,3 +2169,37 @@ func TestCooldownShortest(t *testing.T) { } } } + +func TestLoadBoolsFromEnvParseStrictly(t *testing.T) { + tests := []struct { + value string + start bool + want bool + warnings int + }{ + {"true", false, true, 0}, + {"True", false, true, 0}, + {" TRUE ", false, true, 0}, + {"1", false, true, 0}, + {"false", true, false, 0}, + {"FALSE", true, false, 0}, + {"0", true, false, 0}, + {"yes", true, true, 1}, + {"on", false, false, 1}, + {"ture", true, true, 1}, + } + for _, tt := range tests { + t.Run(tt.value, func(t *testing.T) { + t.Setenv("PROXY_SCANNING_ENABLED", tt.value) + cfg := Default() + cfg.Scanning.Enabled = tt.start + cfg.LoadFromEnv() + if cfg.Scanning.Enabled != tt.want { + t.Errorf("Scanning.Enabled = %v, want %v", cfg.Scanning.Enabled, tt.want) + } + if got := len(cfg.EnvWarnings()); got != tt.warnings { + t.Errorf("len(EnvWarnings()) = %d, want %d: %v", got, tt.warnings, cfg.EnvWarnings()) + } + }) + } +} From 0b399c248dfdb257bc9d55fd7e8f6c59c22f8729 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 8 Nov 2026 07:57:22 +0100 Subject: [PATCH 582/633] Accept sha512 OCI blob digests and check HEAD digests The blob route and blobPathPattern only matched sha256 digests, so a sha512 blob got 404 although the fetch path can verify sha512. The image mirror rejected sha512 digests in index entries, configs and layers for the same reason. Accept sha512 in both, with exact digest lengths. A HEAD for an uncached manifest by digest passed on whatever digest the upstream named. GET checks the body against the requested digest; HEAD now answers DIGEST_INVALID when the upstream names a different digest of the same algorithm. --- internal/handler/container.go | 4 +- internal/handler/container_manifest_fetch.go | 13 ++++ internal/handler/container_mirror.go | 2 +- internal/handler/container_mirror_test.go | 34 ++++++++ internal/handler/container_test.go | 82 +++++++++++++++++--- 5 files changed, 123 insertions(+), 12 deletions(-) diff --git a/internal/handler/container.go b/internal/handler/container.go index d9360fb..40f5d0f 100644 --- a/internal/handler/container.go +++ b/internal/handler/container.go @@ -353,7 +353,7 @@ func (h *ContainerHandler) Routes() http.Handler { case path == "" || path == "/": // Version check: GET /v2/ h.handleVersionCheck(w, r) - case strings.HasSuffix(path, "/blobs/"+r.URL.Query().Get("digest")) || strings.Contains(path, "/blobs/sha256:"): + case strings.HasSuffix(path, "/blobs/"+r.URL.Query().Get("digest")) || strings.Contains(path, "/blobs/sha256:") || strings.Contains(path, "/blobs/sha512:"): // Blob download: GET /v2/{name}/blobs/{digest} h.handleBlobDownload(w, r, path) case manifestPathPattern.MatchString(path): @@ -692,7 +692,7 @@ func (h *ContainerHandler) containerError(w http.ResponseWriter, status int, cod } // blobPathPattern matches blob paths: {name}/blobs/{digest} -var blobPathPattern = regexp.MustCompile(`^(.+)/blobs/(sha256:[a-f0-9]+)$`) +var blobPathPattern = regexp.MustCompile(`^(.+)/blobs/(sha256:[a-f0-9]{64}|sha512:[a-f0-9]{128})$`) // parseBlobPath extracts repository name and digest from a blob path. func (h *ContainerHandler) parseBlobPath(path string) (name, digest string) { diff --git a/internal/handler/container_manifest_fetch.go b/internal/handler/container_manifest_fetch.go index 4b5ef15..7568671 100644 --- a/internal/handler/container_manifest_fetch.go +++ b/internal/handler/container_manifest_fetch.go @@ -210,6 +210,11 @@ func (h *ContainerHandler) headManifest(ctx context.Context, req manifestRequest digest := resp.Header.Get("Docker-Content-Digest") switch { case resp.StatusCode == http.StatusOK && manifestDigestReferencePattern.MatchString(digest): + if req.immutable() && digestMismatch(req.reference, digest) { + h.proxy.Logger.Error("upstream manifest HEAD names another digest", + "name", req.name, "reference", req.reference, "upstream", digest) + return manifestOutcome{digestInvalid: true} + } if !req.immutable() { if outcome, ok := h.tagFromCachedDigest(ctx, req, cached, fromFallback, resp, digest); ok { return outcome @@ -234,6 +239,14 @@ func (h *ContainerHandler) headManifest(ctx context.Context, req manifestRequest return manifestOutcome{relay: h.captureResponse(resp)} } +// digestMismatch reports whether two digests of the same algorithm differ. +// Digests of different algorithms can't be compared without the body. +func digestMismatch(requested, upstream string) bool { + reqAlg, reqHex, _ := strings.Cut(requested, ":") + upAlg, upHex, _ := strings.Cut(upstream, ":") + return reqAlg == upAlg && !strings.EqualFold(reqHex, upHex) +} + // tagFromCachedDigest answers a tag from the cache once a HEAD named the // digest it points to: the cached tag entry if the tag did not move, or the // manifest stored under the new digest. It reports false when neither is diff --git a/internal/handler/container_mirror.go b/internal/handler/container_mirror.go index a973afd..e17311c 100644 --- a/internal/handler/container_mirror.go +++ b/internal/handler/container_mirror.go @@ -36,7 +36,7 @@ var ( // Grammar from the OCI distribution spec. mirrorNamePattern = regexp.MustCompile(`^[a-z0-9]+((\.|_|__|-+)[a-z0-9]+)*(/[a-z0-9]+((\.|_|__|-+)[a-z0-9]+)*)*$`) mirrorTagPattern = regexp.MustCompile(`^[a-zA-Z0-9_][a-zA-Z0-9._-]{0,127}$`) - mirrorDigestPattern = regexp.MustCompile(`^sha256:[a-f0-9]{64}$`) + mirrorDigestPattern = regexp.MustCompile(`^(sha256:[a-f0-9]{64}|sha512:[a-f0-9]{128})$`) ) // dockerHubMirrorAliases are registry hosts that name Docker Hub in image diff --git a/internal/handler/container_mirror_test.go b/internal/handler/container_mirror_test.go index e6be5a5..f3a15ee 100644 --- a/internal/handler/container_mirror_test.go +++ b/internal/handler/container_mirror_test.go @@ -2,7 +2,9 @@ package handler import ( "context" + "crypto/sha512" "database/sql" + "encoding/hex" "encoding/json" "io" "net/http" @@ -610,3 +612,35 @@ func TestMirrorImageBlobFailureKeepsOtherFetches(t *testing.T) { t.Errorf("layer fetched alongside the failing blob: status %d, want it cached", response.Code) } } + +func TestMirrorImageAcceptsSHA512Blobs(t *testing.T) { + registry, image := newFakeImageRegistry(t) + const layer = "sha512 layer" + sum := sha512.Sum512([]byte(layer)) + layerDigest := "sha512:" + hex.EncodeToString(sum[:]) + body, err := json.Marshal(map[string]any{ + "schemaVersion": 2, "mediaType": ociManifestMediaType, + "config": map[string]any{"mediaType": "application/vnd.oci.image.config.v1+json", "digest": image.singleLayer}, + "layers": []map[string]any{{"mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": layerDigest}}, + }) + if err != nil { + t.Fatal(err) + } + registry.mu.Lock() + registry.manifests["/v2/team/app/manifests/sha512"] = fakeManifest{contentType: ociManifestMediaType, body: string(body)} + registry.blobs["/v2/team/app/blobs/"+layerDigest] = layer + registry.mu.Unlock() + + proxy := newMirrorTestProxy(t, registry) + h := NewContainerHandlerWithRegistry(proxy, "http://proxy.example", registry.server.URL) + if _, err := h.MirrorImage(context.Background(), ImageRef{Name: "team/app", References: []string{"sha512"}}, nil); err != nil { + t.Fatalf("MirrorImage: %v", err) + } + cached, err := proxy.GetCachedArtifact(t.Context(), "oci", "team/app", layerDigest, layerDigest) + if err != nil || cached == nil { + t.Fatalf("sha512 layer not cached: %v, %v", cached, err) + } + if cached.Reader != nil { + _ = cached.Reader.Close() + } +} diff --git a/internal/handler/container_test.go b/internal/handler/container_test.go index d546725..fd20e38 100644 --- a/internal/handler/container_test.go +++ b/internal/handler/container_test.go @@ -18,6 +18,11 @@ import ( "github.com/git-pkgs/registries/fetch" ) +var ( + testSHA256Digest = "sha256:" + strings.Repeat("ab", 32) + testSHA512Digest = "sha512:" + strings.Repeat("cd", 64) +) + func TestContainerHandler_parseBlobPath(t *testing.T) { h := &ContainerHandler{} @@ -27,19 +32,29 @@ func TestContainerHandler_parseBlobPath(t *testing.T) { wantDigest string }{ { - path: "library/nginx/blobs/sha256:abc123def456", + path: "library/nginx/blobs/" + testSHA256Digest, wantName: "library/nginx", - wantDigest: "sha256:abc123def456", + wantDigest: testSHA256Digest, }, { - path: "myorg/myrepo/blobs/sha256:0123456789abcdef", - wantName: "myorg/myrepo", - wantDigest: "sha256:0123456789abcdef", - }, - { - path: "deep/nested/repo/name/blobs/sha256:fedcba9876543210", + path: "deep/nested/repo/name/blobs/" + testSHA256Digest, wantName: "deep/nested/repo/name", - wantDigest: "sha256:fedcba9876543210", + wantDigest: testSHA256Digest, + }, + { + path: "myorg/myrepo/blobs/" + testSHA512Digest, + wantName: "myorg/myrepo", + wantDigest: testSHA512Digest, + }, + { + path: "library/nginx/blobs/sha256:abc123def456", + wantName: "", + wantDigest: "", + }, + { + path: "library/nginx/blobs/sha512:" + strings.Repeat("a", 64), + wantName: "", + wantDigest: "", }, { path: "invalid/path", @@ -1476,3 +1491,52 @@ func TestContainerHandler_ManifestHonoursSeparateAcceptHeaders(t *testing.T) { t.Errorf("offline body = %q, want the cached manifest", w.Body.String()) } } + +func TestContainerHandler_BlobSHA512(t *testing.T) { + proxy, _, _, fetcher := setupTestProxy(t) + const blob = "sha512 layer" + fetcher.artifact = &fetch.Artifact{ + Body: io.NopCloser(strings.NewReader(blob)), + ContentType: "application/octet-stream", + } + sum := sha512.Sum512([]byte(blob)) + digest := "sha512:" + hex.EncodeToString(sum[:]) + h := &ContainerHandler{proxy: proxy, registryURL: "https://registry.example.test"} + + w := httptest.NewRecorder() + h.Routes().ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/library/app/blobs/"+digest, nil)) + + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want %d; body: %s", w.Code, http.StatusOK, w.Body.String()) + } + if w.Body.String() != blob { + t.Errorf("body = %q, want %q", w.Body.String(), blob) + } + if got := w.Header().Get("Docker-Content-Digest"); got != digest { + t.Errorf("Docker-Content-Digest = %q, want %q", got, digest) + } +} + +func TestContainerHandler_HeadManifestByDigestChecksUpstreamDigest(t *testing.T) { + requested := sha256Digest([]byte("requested")) + other := sha256Digest([]byte("other")) + registry := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", ociManifestMediaType) + w.Header().Set("Docker-Content-Digest", other) + w.Header().Set("Content-Length", "5") + })) + defer registry.Close() + proxy, _, _, _ := setupTestProxy(t) + proxy.HTTPClient = registry.Client() + h := &ContainerHandler{proxy: proxy, registryURL: registry.URL} + + w := httptest.NewRecorder() + h.Routes().ServeHTTP(w, httptest.NewRequest(http.MethodHead, "/library/app/manifests/"+requested, nil)) + + if w.Code != http.StatusBadGateway { + t.Fatalf("status = %d, want %d", w.Code, http.StatusBadGateway) + } + if got := w.Header().Get("Docker-Content-Digest"); got == other { + t.Errorf("Docker-Content-Digest = %q, the upstream's other digest", got) + } +} From 6398ed63f6f1a03a884ad9a0d300a6546d51025c Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 8 Nov 2026 07:57:30 +0100 Subject: [PATCH 583/633] Validate and normalise scanner ecosystem names scanning.scanners[].ecosystems was never checked, and the scanner group compared it exactly against the keys handlers pass. Natural names such as go, docker and rubygems passed validation but never matched the handler keys golang, oci and gem, so a block-mode scanner restricted to them silently never ran. Map common aliases to the handler keys, reject unknown names at startup, and normalise the names again when building the group. --- docs/configuration.md | 2 +- internal/config/config.go | 36 ++++++++++++++++++++++++++++++++++ internal/config/config_test.go | 25 +++++++++++++++++++++++ internal/scanner/group.go | 2 +- internal/scanner/group_test.go | 18 +++++++++++++++++ 5 files changed, 81 insertions(+), 2 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 40fb55d..2ea49c6 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1400,7 +1400,7 @@ scanning: | `scanning.scanners[].name` | - | Identifies this scanner in logs and metrics | | `scanning.scanners[].url` | - | Endpoint the proxy POSTs scan notifications to | | `scanning.scanners[].mode` | - | `block` (default) or `monitor` | -| `scanning.scanners[].ecosystems` | - | Restricts this scanner to specific ecosystems (e.g. `npm`, `pypi`). Empty means all ecosystems | +| `scanning.scanners[].ecosystems` | - | Restricts this scanner to specific ecosystems (e.g. `npm`, `pypi`). Empty means all ecosystems. Valid names: `alpine`, `arch`, `cargo`, `chef`, `composer`, `conan`, `conda`, `cran`, `deb`, `freebsd`, `gem`, `generic`, `golang`, `helm`, `hex`, `julia`, `maven`, `npm`, `nuget`, `oci`, `pub`, `pypi`, `rpm`, `swift`. The aliases `go`, `docker`, `rubygems`, `packagist`, `debian` and `apk` are accepted too; any other name stops the proxy at startup | | `scanning.scanners[].headers` | - | Extra HTTP headers sent with every scan request (e.g. for authenticating to the scanner service). Values support `${VAR_NAME}` expansion | ### How caching defers to a scan verdict diff --git a/internal/config/config.go b/internal/config/config.go index 5c74d3c..7db98c5 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -70,6 +70,7 @@ import ( "path" "path/filepath" "regexp" + "slices" "sort" "strconv" "strings" @@ -436,9 +437,44 @@ func (s *ScannerConfig) Validate() error { default: return fmt.Errorf("invalid mode %q (must be block or monitor)", s.Mode) } + for i, eco := range s.Ecosystems { + key := NormalizeScannerEcosystem(eco) + if !slices.Contains(scannerEcosystems, key) { + return fmt.Errorf("unknown ecosystem %q (valid: %s)", eco, strings.Join(scannerEcosystems, ", ")) + } + s.Ecosystems[i] = key + } return nil } +// scannerEcosystems are the ecosystem keys handlers pass to scanners: the +// retention keys plus the handlers retention does not cover. +var scannerEcosystems = func() []string { + keys := append(slices.Clone(retention.KnownKeys), "arch", "chef") + slices.Sort(keys) + return keys +}() + +// scannerEcosystemAliases maps common ecosystem names to handler keys. +var scannerEcosystemAliases = map[string]string{ + "apk": "alpine", + "debian": "deb", + "docker": "oci", + "go": "golang", + "packagist": "composer", + "rubygems": "gem", +} + +// NormalizeScannerEcosystem returns the handler key for a +// scanning.scanners[].ecosystems entry, resolving case and common aliases. +func NormalizeScannerEcosystem(eco string) string { + eco = strings.ToLower(strings.TrimSpace(eco)) + if key, ok := scannerEcosystemAliases[eco]; ok { + return key + } + return eco +} + // StorageConfig configures artifact storage. type StorageConfig struct { // URL is the storage backend URL. diff --git a/internal/config/config_test.go b/internal/config/config_test.go index ac6dfff..e91a4f2 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -3,6 +3,7 @@ package config import ( "os" "path/filepath" + "slices" "strings" "testing" "time" @@ -2203,3 +2204,27 @@ func TestLoadBoolsFromEnvParseStrictly(t *testing.T) { }) } } + +func TestScannerEcosystems(t *testing.T) { + cfg := ScanningConfig{ + Enabled: true, + SigningKey: "s3cret", + Scanners: []ScannerConfig{{ + Name: "clamav", URL: "http://scanner.invalid/scan", + Ecosystems: []string{"go", "Docker", "rubygems", "npm", "arch"}, + }}, + } + if err := cfg.Validate(); err != nil { + t.Fatalf("Validate() error: %v", err) + } + want := []string{"golang", "oci", "gem", "npm", "arch"} + if got := cfg.Scanners[0].Ecosystems; !slices.Equal(got, want) { + t.Errorf("Ecosystems = %v, want %v", got, want) + } + + cfg.Scanners[0].Ecosystems = []string{"npm", "pypy"} + err := cfg.Validate() + if err == nil || !strings.Contains(err.Error(), "pypy") { + t.Errorf("Validate() error = %v, want unknown ecosystem pypy", err) + } +} diff --git a/internal/scanner/group.go b/internal/scanner/group.go index cf5930f..3066219 100644 --- a/internal/scanner/group.go +++ b/internal/scanner/group.go @@ -67,7 +67,7 @@ func NewGroup(cfg config.ScanningConfig, logger *slog.Logger) (*Group, error) { if len(sc.Ecosystems) > 0 { ecosystems = make(map[string]struct{}, len(sc.Ecosystems)) for _, eco := range sc.Ecosystems { - ecosystems[eco] = struct{}{} + ecosystems[config.NormalizeScannerEcosystem(eco)] = struct{}{} } } diff --git a/internal/scanner/group_test.go b/internal/scanner/group_test.go index cbbaa01..d80166b 100644 --- a/internal/scanner/group_test.go +++ b/internal/scanner/group_test.go @@ -294,3 +294,21 @@ func TestGroupBlocks(t *testing.T) { t.Error("nil group blocks") } } + +func TestGroup_NormalisesEcosystemAliases(t *testing.T) { + g, err := NewGroup(config.ScanningConfig{ + Enabled: true, + SigningKey: "test-signing-key", + Scanners: []config.ScannerConfig{ + {Name: "clamav", URL: "http://clamav.invalid", Ecosystems: []string{"go", "docker", "rubygems"}}, + }, + }, discardLogger()) + if err != nil { + t.Fatalf("NewGroup() error: %v", err) + } + for _, ecosystem := range []string{"golang", "oci", "gem"} { + if !g.Blocks(ecosystem) { + t.Errorf("Blocks(%s) = false, want true", ecosystem) + } + } +} From 2a6c8091032ebb3eb2f30a209884d4a673416990 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 8 Nov 2026 07:58:49 +0100 Subject: [PATCH 584/633] Reject negative and out-of-range sizes and TTLs ParseSize accepted "-5GB", "NaNGB", "InfGB" and "1e30GB", the last two clamping to the largest int64. A negative or NaN storage.max_size passed validation, and eviction treats a limit of zero or less as unlimited. Reject negative, NaN, infinite and overflowing sizes in ParseSize, and reject negative metadata_ttl and storage.direct_serve_ttl at startup. --- docs/configuration.md | 2 +- internal/config/config.go | 19 ++++++++++++++----- internal/config/config_test.go | 27 +++++++++++++++++++++++++++ 3 files changed, 42 insertions(+), 6 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 2ea49c6..8baac0f 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -67,7 +67,7 @@ storage: |--------|-------------|------|-------------| | `storage.url` | `PROXY_STORAGE_URL` | `-storage-url` | Storage URL (file:// or s3://) | | `storage.path` | `PROXY_STORAGE_PATH` | `-storage-path` | Local path (deprecated, use url) | -| `storage.max_size` | `PROXY_STORAGE_MAX_SIZE` | - | Max cache size (e.g., "10GB") | +| `storage.max_size` | `PROXY_STORAGE_MAX_SIZE` | - | Max cache size (e.g., "10GB"). Negative or out-of-range sizes are rejected at startup | | `storage.cache_artifacts` | `PROXY_STORAGE_CACHE_ARTIFACTS` | - | Store fetched artifacts (default: true); `false` streams them from upstream | | `storage.retention.default` | `PROXY_STORAGE_RETENTION_DEFAULT` | - | Evict artifacts not served for this long (e.g. "30d"); see [Retention](#retention) | | `storage.retention.sweep_interval` | `PROXY_STORAGE_RETENTION_SWEEP_INTERVAL` | - | How often expired artifacts are looked for (default: "10m", at least "1m") | diff --git a/internal/config/config.go b/internal/config/config.go index 7db98c5..d9d9b29 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -64,6 +64,7 @@ import ( "encoding/base64" "encoding/json" "fmt" + "math" "net" "net/url" "os" @@ -1655,8 +1656,8 @@ func (c *Config) Validate() error { // Validate direct serve TTL if specified if c.Storage.DirectServeTTL != "" { - if _, err := time.ParseDuration(c.Storage.DirectServeTTL); err != nil { - return fmt.Errorf("invalid storage.direct_serve_ttl %q: %w", c.Storage.DirectServeTTL, err) + if _, err := parseNonNegativeDuration("storage.direct_serve_ttl", c.Storage.DirectServeTTL); err != nil { + return err } } @@ -1673,8 +1674,8 @@ func (c *Config) Validate() error { // Validate metadata TTL if specified if c.MetadataTTL != "" && c.MetadataTTL != "0" { - if _, err := time.ParseDuration(c.MetadataTTL); err != nil { - return fmt.Errorf("invalid metadata_ttl %q: %w", c.MetadataTTL, err) + if _, err := parseNonNegativeDuration("metadata_ttl", c.MetadataTTL); err != nil { + return err } } @@ -2172,7 +2173,12 @@ func ParseSize(s string) (int64, error) { if err != nil { return 0, fmt.Errorf("invalid number %q", numStr) } - return int64(num * float64(s2.mult)), nil + // The comparisons also reject NaN. + size := num * float64(s2.mult) + if !(size >= 0 && size < math.MaxInt64) { + return 0, fmt.Errorf("invalid size %q: must be between 0 and %d bytes", s, int64(math.MaxInt64)) + } + return int64(size), nil } } @@ -2181,6 +2187,9 @@ func ParseSize(s string) (int64, error) { if err != nil { return 0, fmt.Errorf("invalid size %q", s) } + if num < 0 { + return 0, fmt.Errorf("invalid size %q: must not be negative", s) + } return num, nil } diff --git a/internal/config/config_test.go b/internal/config/config_test.go index e91a4f2..2139b0e 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -2228,3 +2228,30 @@ func TestScannerEcosystems(t *testing.T) { t.Errorf("Validate() error = %v, want unknown ecosystem pypy", err) } } + +func TestParseSizeRejectsOutOfRange(t *testing.T) { + for _, s := range []string{"-5GB", "-1", "-0.5K", "NaNGB", "InfGB", "+InfB", "1e30GB", "9999999999TB"} { + if got, err := ParseSize(s); err == nil { + t.Errorf("ParseSize(%q) = %d, want error", s, got) + } + } + cfg := Default() + cfg.Storage.MaxSize = "-5GB" + if err := cfg.Validate(); err == nil { + t.Error("Validate() accepted storage.max_size -5GB") + } +} + +func TestValidateRejectsNegativeTTLs(t *testing.T) { + cfg := Default() + cfg.MetadataTTL = "-5m" + if err := cfg.Validate(); err == nil { + t.Error("Validate() accepted metadata_ttl -5m") + } + + cfg = Default() + cfg.Storage.DirectServeTTL = "-5m" + if err := cfg.Validate(); err == nil { + t.Error("Validate() accepted storage.direct_serve_ttl -5m") + } +} From d073fdcd52fcf0e988f78941650e37578d8791a4 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 8 Nov 2026 08:00:14 +0100 Subject: [PATCH 585/633] Close npm, PyPI and pub cooldown gaps in metadata and downloads npm: a tarball request for a version the cached packument did not list yet (cached within metadata_ttl before the release) found no publish time and was let through cooldown, and fetched without integrity. When cooldown applies and the cached packument lacks the version, fetch it again and decide on what upstream lists now; if it can't be fetched, fail as when a publish time can't be read. npm: cooldown only moved dist-tags.latest. Other tags kept naming versions in cooldown, while the denylist removes such tags. Cooldown now removes them too, and drops latest when no version is left, in both the in-memory and streamed rewrites. PyPI: the simple JSON rewrite removed filtered files but left their versions in the PEP 700 versions list. Remove them there too. pub: latest was never rewritten, so its archive_url pointed at pub.dev, it kept naming a version in cooldown when no version was left, and the replacement was the last list entry, which may be a prerelease. Point latest's archive_url at the proxy, replace a held back latest with the newest stable version left (or the newest prerelease), and drop it when nothing is left. --- docs/configuration.md | 6 ++- internal/handler/npm.go | 73 +++++++++++++++++++------- internal/handler/npm_attestations.go | 2 +- internal/handler/npm_tarball_test.go | 57 ++++++++++++++++++++ internal/handler/pub.go | 50 ++++++++++++------ internal/handler/pub_test.go | 55 +++++++++++++++++++ internal/handler/pypi.go | 13 +++++ internal/handler/pypi_test.go | 26 +++++++++ internal/metadata/npm/metadata.go | 22 ++++++-- internal/metadata/npm/metadata_test.go | 36 +++++++++++++ 10 files changed, 297 insertions(+), 43 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 584896b..93de410 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1240,7 +1240,9 @@ RubyGems, Hex and PyPI keep their publish times outside the metadata clients rea For RubyGems and Hex, both documents go through the metadata cache. With [`cache_metadata`](#metadata-caching) they are served from it within `metadata_ttl` without asking the upstream, revalidated with their ETags after it, and filtered from the last cached copies, marked stale, while the upstream is down; without it they are still fetched once for concurrent requests. Each document has its own TTL, so the compact index or Hex package can be newer than the cached times and list a version they don't have. The times are then fetched again before filtering, and if that fails, such versions are left out until the times can be read. The filtered index or package carries an ETag of its own, so Bundler and Mix revalidate it with `304 Not Modified`. Publish times are compared with the current time on each request, and the filtered result is kept until the first version it held back leaves cooldown (see [rewritten metadata cache](#rewritten-metadata-cache)); a newly published version shows up within `metadata_ttl`, as with other cached metadata. Upstream errors are answered as for other cached metadata (404, or 502) rather than relayed. -With cooldown on, the proxy asks npm for full packuments, since the abbreviated ones it uses otherwise carry no publish times. Abbreviated packuments cached before cooldown was turned on are fetched again in full rather than served, including as a stale copy when the upstream is down. When `latest` is in cooldown, the packument tags the newest version out of cooldown instead, and `npm dist-tag ls` shows the same tags. +With cooldown on, the proxy asks npm for full packuments, since the abbreviated ones it uses otherwise carry no publish times. Abbreviated packuments cached before cooldown was turned on are fetched again in full rather than served, including as a stale copy when the upstream is down. When `latest` is in cooldown, the packument tags the newest version out of cooldown instead; other tags that name a version in cooldown are removed, as for the denylist, and `npm dist-tag ls` shows the same tags. A tarball request for a version the cached packument doesn't list yet fetches the packument again, within `metadata_ttl` or not, and is decided on what the upstream lists now; if the packument can't be fetched, the download is not let through, as when a publish time can't be read. + +In pub.dev package metadata, when `latest` is in cooldown it names the newest stable version out of cooldown instead, or the newest prerelease when no stable version is left, and it is left out when every version is in cooldown. Conda cooldown covers `repodata.json`, `current_repodata.json`, `repodata_from_packages.json` and sharded repodata (CEP 16). Other forms of the repodata, such as `repodata.json.zst`, `repodata.json.bz2` and `repodata.jlap` patches, would list versions still in cooldown, so with cooldown on they are answered 404 and conda and mamba fall back to `repodata.json`. With `cache_metadata`, `repodata.json` is read through the metadata cache as it is without cooldown: within `metadata_ttl` the stored copy is filtered as it is read from storage, after it the upstream is asked with its ETag, and while the upstream is down the stale copy is filtered. Once a document has been filtered, the proxy remembers the ETag of the result until the first entry it left out leaves cooldown, so a client revalidating with `If-None-Match` gets `304 Not Modified` without the document being filtered or sent again. The first response for each document has no ETag, as it is sent while it is filtered. Without `cache_metadata`, `repodata.json` is fetched and filtered on every request. Clients cache shards by the hash in the shard index, so with cooldown on the proxy serves a rewritten index whose shard hashes change every hour and filters each shard as it is fetched. Versions leaving cooldown show up within the hour, and clients fetch the shards they need again once an hour. If a channel's shard index points packages or shards somewhere the proxy doesn't route, the proxy answers 404 for it and clients fall back to `repodata.json`. @@ -1296,7 +1298,7 @@ Names and versions are compared the way each ecosystem treats them: channel only. Denied versions are removed from npm packuments, Cargo sparse indexes, and PyPI -simple HTML/JSON and release metadata, including responses served from cached +simple HTML/JSON (files and the JSON `versions` list) and release metadata, including responses served from cached metadata during an upstream outage. No publication timestamp or enabled cooldown is required. npm tags pointing at denied versions are removed; `latest` is moved to the highest remaining stable version when available. Other tags are not diff --git a/internal/handler/npm.go b/internal/handler/npm.go index 50757a3..0e9f32a 100644 --- a/internal/handler/npm.go +++ b/internal/handler/npm.go @@ -344,7 +344,7 @@ func (h *NPMHandler) serveStreamedPackument(w http.ResponseWriter, r *http.Reque fullRequired := h.packumentValidator() != nil options := h.npmRewriteOptions(packageName, nil) var rewritten rewrittenMetadata - err, fetchErr := h.readStoredPackument(r.Context(), upstreamURL, accept, metadataCacheKey, func(entry *database.MetadataCacheEntry) (err error) { + err, fetchErr := h.readStoredPackument(r.Context(), upstreamURL, accept, metadataCacheKey, false, func(entry *database.MetadataCacheEntry) (err error) { rewritten, err = h.proxy.streamedRewrite(r.Context(), "npm", h.proxyURL, metadataCacheKey, entry, func(open func() (io.ReadCloser, error), out io.Writer) error { var scan *npm.Scan err := readStored(open, func(in io.Reader) (err error) { @@ -413,7 +413,18 @@ var errInMemory = errors.New("npm packument needs the in-memory path") // fetching it into storage first when needed. It returns errInMemory when // metadata isn't cached in storage, and ErrUpstreamNotFound and upstream // errors as they come. -func (h *NPMHandler) storedPackument(ctx context.Context, packumentURL, accept, cacheKey string) (*database.MetadataCacheEntry, error) { +// With refresh, a stored copy within its TTL is fetched again rather than +// served. +func (h *NPMHandler) storedPackument(ctx context.Context, packumentURL, accept, cacheKey string, refresh bool) (*database.MetadataCacheEntry, error) { + if refresh && h.proxy.DB != nil { + if row, _ := h.proxy.DB.GetMetadataCache("npm", cacheKey); usableStoredMetadata(row) { + entry, err := h.proxy.restoreStoredMetadata(ctx, "npm", cacheKey, packumentURL, accept, h.packumentStreamValidator(), row) + if errors.Is(err, errMetadataNotStored) { + return nil, errInMemory + } + return entry, err + } + } entry, err := h.proxy.storedMetadataDocument(ctx, "npm", cacheKey, packumentURL, accept, h.packumentStreamValidator()) if errors.Is(err, errMetadataNotStored) { return nil, errInMemory @@ -426,8 +437,8 @@ func (h *NPMHandler) storedPackument(ctx context.Context, packumentURL, accept, // or no longer has the digest its row records, the packument is fetched // again and read once more. fetchErr is what storedPackument returned when the packument // couldn't be fetched, and err is what read returned. -func (h *NPMHandler) readStoredPackument(ctx context.Context, packumentURL, accept, cacheKey string, read func(*database.MetadataCacheEntry) error) (err, fetchErr error) { - entry, fetchErr := h.storedPackument(ctx, packumentURL, accept, cacheKey) +func (h *NPMHandler) readStoredPackument(ctx context.Context, packumentURL, accept, cacheKey string, refresh bool, read func(*database.MetadataCacheEntry) error) (err, fetchErr error) { + entry, fetchErr := h.storedPackument(ctx, packumentURL, accept, cacheKey, refresh) if fetchErr != nil { return nil, fetchErr } @@ -475,7 +486,7 @@ func (h *NPMHandler) streamedVersionMetadata(ctx context.Context, packageName, s fullRequired := h.packumentValidator() != nil options := h.npmRewriteOptions(packageName, nil) var m rewrittenMetadata - err, fetchErr := h.readStoredPackument(ctx, packumentURL, accept, cacheKey, func(entry *database.MetadataCacheEntry) (err error) { + err, fetchErr := h.readStoredPackument(ctx, packumentURL, accept, cacheKey, false, func(entry *database.MetadataCacheEntry) (err error) { m, err = h.proxy.streamedRewrite(ctx, "npm", h.proxyURL, metadataCacheKey, entry, func(open func() (io.ReadCloser, error), out io.Writer) error { var scan *npm.Scan err := readStored(open, func(in io.Reader) (err error) { @@ -776,7 +787,7 @@ func (h *NPMHandler) handleDownload(w http.ResponseWriter, r *http.Request) { } registryURL, cacheKey := h.sourceFor(packageName) metadata := sync.OnceValues(func() (npmVersionData, error) { - return h.versionData(r.Context(), registryURL+"/"+url.PathEscape(packageName), cacheKey, version) + return h.versionData(r.Context(), packageName, registryURL+"/"+url.PathEscape(packageName), cacheKey, version) }) inCooldown, err := h.versionInCooldown(packageName, version, metadata) if err != nil { @@ -883,10 +894,27 @@ type npmVersionData struct { // versionData reads what a tarball download needs about version from the // packument: from storage in one pass when metadata is cached there, and // otherwise from the packument read into memory. -func (h *NPMHandler) versionData(ctx context.Context, packumentURL, cacheKey, version string) (npmVersionData, error) { +// +// A packument cached within its TTL can predate the version. Under cooldown a +// missing entry would let the version through with no publish time, so the +// packument is fetched again and the version decided on what upstream lists +// now, as the Composer handler does. +func (h *NPMHandler) versionData(ctx context.Context, packageName, packumentURL, cacheKey, version string) (npmVersionData, error) { + data, fromCache, err := h.readVersionData(ctx, packumentURL, cacheKey, version, false) + if err == nil && data.entry == nil && fromCache && h.proxy.cooldownApplies("npm", packageName) { + h.proxy.Logger.Info("cooldown: cached npm metadata lacks version, fetching it again", + "package", packageName, "version", version) + data, _, err = h.readVersionData(ctx, packumentURL, cacheKey, version, true) + } + return data, err +} + +// readVersionData implements versionData. fromCache reports whether the +// packument may have been a cached copy; refresh fetches it from upstream. +func (h *NPMHandler) readVersionData(ctx context.Context, packumentURL, cacheKey, version string, refresh bool) (data npmVersionData, fromCache bool, err error) { accept := h.packumentAccept() var found npm.VersionEntry - err, fetchErr := h.readStoredPackument(ctx, packumentURL, accept, cacheKey, func(entry *database.MetadataCacheEntry) error { + err, fetchErr := h.readStoredPackument(ctx, packumentURL, accept, cacheKey, refresh, func(entry *database.MetadataCacheEntry) error { return readStored(func() (io.ReadCloser, error) { return h.proxy.openStoredMetadata(ctx, entry) }, func(in io.Reader) (err error) { found, err = npm.LookupVersion(in, version) return err @@ -894,23 +922,32 @@ func (h *NPMHandler) versionData(ctx context.Context, packumentURL, cacheKey, ve }) switch { case fetchErr == nil && err == nil && (!found.Abbreviated || h.packumentValidator() == nil): - return npmVersionData{entry: found.Entry, published: found.Published}, nil + return npmVersionData{entry: found.Entry, published: found.Published}, !refresh, nil case fetchErr == nil && err != nil: h.proxy.Logger.Warn("failed to read stored npm metadata, reading it in memory", "url", packumentURL, "error", err) case fetchErr != nil && !errors.Is(fetchErr, errInMemory): - return npmVersionData{}, fetchErr + return npmVersionData{}, false, fetchErr } - body, _, _, err := h.proxy.fetchOrCacheMetadata(ctx, "npm", cacheKey, packumentURL, "", h.packumentValidator(), accept) - if err != nil { - return npmVersionData{}, err + var res metadataResult + if refresh { + res = h.proxy.refreshMetadata(ctx, "npm", cacheKey, packumentURL, accept, "", h.packumentValidator()) + } else { + res = h.proxy.fetchMetadataDocument(ctx, "npm", cacheKey, packumentURL, accept, h.packumentValidator()) } - entry, _ := jsonscan.Lookup(body, "versions", version) - published, _, err := jsonscan.LookupString(body, "time", version) - if err != nil { - return npmVersionData{}, fmt.Errorf("parsing npm metadata: %w", err) + if res.err != nil { + return npmVersionData{}, false, res.err } - return npmVersionData{entry: entry, published: published}, nil + // The cached copy is what lacked the version, so it can't decide it. + if refresh && res.stale { + return npmVersionData{}, false, errors.New("npm metadata could not be fetched again") + } + entry, _ := jsonscan.Lookup(res.body, "versions", version) + published, _, err := jsonscan.LookupString(res.body, "time", version) + if err != nil { + return npmVersionData{}, false, fmt.Errorf("parsing npm metadata: %w", err) + } + return npmVersionData{entry: entry, published: published}, res.fromCache, nil } func validateTarballURL(raw, registryURL string) (string, error) { diff --git a/internal/handler/npm_attestations.go b/internal/handler/npm_attestations.go index 02c5751..cc84003 100644 --- a/internal/handler/npm_attestations.go +++ b/internal/handler/npm_attestations.go @@ -41,7 +41,7 @@ func (h *NPMHandler) handleAttestations(w http.ResponseWriter, r *http.Request) registryURL, cacheKey := h.sourceFor(packageName) metadata := sync.OnceValues(func() (npmVersionData, error) { - return h.versionData(r.Context(), registryURL+"/"+url.PathEscape(packageName), cacheKey, version) + return h.versionData(r.Context(), packageName, registryURL+"/"+url.PathEscape(packageName), cacheKey, version) }) inCooldown, err := h.versionInCooldown(packageName, version, metadata) diff --git a/internal/handler/npm_tarball_test.go b/internal/handler/npm_tarball_test.go index fe1508d..d844dd5 100644 --- a/internal/handler/npm_tarball_test.go +++ b/internal/handler/npm_tarball_test.go @@ -201,3 +201,60 @@ func TestNPMTarballRejectsUnsafeMetadataURL(t *testing.T) { }) } } + +// TestNPMTarballCooldownRefetchesStalePackument checks that a tarball for a +// version the cached packument does not list yet is decided on the packument +// upstream serves now, not let through for want of a publish time. +func TestNPMTarballCooldownRefetchesStalePackument(t *testing.T) { + for _, tc := range []struct { + name string + published time.Time + down bool + want int + }{ + {"recent", time.Now().Add(-time.Hour), false, http.StatusNotFound}, + // Without upstream the version can't be checked, so it isn't served. + {"upstream down", time.Now().Add(-time.Hour), true, http.StatusBadGateway}, + // The refetched entry's integrity is checked, so a tarball that + // doesn't match it is refused rather than served unverified. + {"old", time.Now().Add(-30 * 24 * time.Hour), false, http.StatusBadGateway}, + } { + t.Run(tc.name, func(t *testing.T) { + var released atomic.Bool + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + old := time.Now().Add(-60 * 24 * time.Hour).Format(time.RFC3339) + if released.Load() && tc.down { + w.WriteHeader(http.StatusServiceUnavailable) + return + } + if !released.Load() { + _, _ = fmt.Fprintf(w, `{"name":"widget","time":{"1.0.0":%q},"versions":{"1.0.0":{"dist":{}}}}`, old) + return + } + _, _ = fmt.Fprintf(w, `{"name":"widget","time":{"1.0.0":%q,"2.0.0":%q},"versions":{"1.0.0":{"dist":{}},"2.0.0":{"dist":{"integrity":%q}}}}`, + old, tc.published.Format(time.RFC3339), sha512SRI("package")) + })) + t.Cleanup(upstream.Close) + p, _, _, fetcher := setupTestProxy(t) + p.CacheMetadata = true + p.MetadataTTL = time.Hour + p.Cooldown = &cooldown.Config{Default: "7d"} + p.HTTPClient = upstream.Client() + fetcher.artifact = &fetch.Artifact{Body: io.NopCloser(strings.NewReader("tampered"))} + h := NewNPMHandler(p, "http://proxy.test", upstream.URL) + + metadata := httptest.NewRecorder() + h.Routes().ServeHTTP(metadata, httptest.NewRequest(http.MethodGet, "/widget", nil)) + if metadata.Code != http.StatusOK { + t.Fatalf("metadata status = %d: %s", metadata.Code, metadata.Body.String()) + } + released.Store(true) + + response := httptest.NewRecorder() + h.Routes().ServeHTTP(response, httptest.NewRequest(http.MethodGet, "/widget/-/widget-2.0.0.tgz", nil)) + if response.Code != tc.want { + t.Fatalf("status = %d, want %d: %s", response.Code, tc.want, response.Body.String()) + } + }) + } +} diff --git a/internal/handler/pub.go b/internal/handler/pub.go index 31e00d8..5a6d426 100644 --- a/internal/handler/pub.go +++ b/internal/handler/pub.go @@ -10,6 +10,7 @@ import ( "strings" "time" + "github.com/Masterminds/semver/v3" "github.com/git-pkgs/proxy/internal/database" "github.com/git-pkgs/proxy/internal/retention" ) @@ -293,32 +294,47 @@ func (h *PubHandler) shouldFilterVersion(packagePURL, name, version string, vmap return false } -// updateLatestVersion updates the latest field if the current latest version -// was removed by cooldown filtering. +// updateLatestVersion points latest at the kept entry for its version, so +// its archive_url is the proxy's too. When cooldown removed that version, +// latest becomes the newest stable version left, or the newest version left +// if all are prereleases, and is dropped when none is left. func (h *PubHandler) updateLatestVersion(metadata map[string]any, filtered []any) { - if h.proxy.Cooldown == nil || !h.proxy.Cooldown.Enabled() { - return - } - latest, ok := metadata["latest"].(map[string]any) if !ok { return } + latestVer, _ := latest["version"].(string) - latestVer, ok := latest["version"].(string) - if !ok { - return - } - + var newest, newestStable *semver.Version + var newestEntry, newestStableEntry any for _, vdata := range filtered { - if vmap, ok := vdata.(map[string]any); ok { - if vmap["version"] == latestVer { - return - } + vmap, ok := vdata.(map[string]any) + if !ok { + continue + } + version, _ := vmap["version"].(string) + if version == latestVer { + latest["archive_url"] = vmap["archive_url"] + return + } + parsed, err := semver.NewVersion(version) + if err != nil { + continue + } + if newest == nil || parsed.GreaterThan(newest) { + newest, newestEntry = parsed, vdata + } + if parsed.Prerelease() == "" && (newestStable == nil || parsed.GreaterThan(newestStable)) { + newestStable, newestStableEntry = parsed, vdata } } - if len(filtered) > 0 { - metadata["latest"] = filtered[len(filtered)-1] + switch { + case newestStableEntry != nil: + metadata["latest"] = newestStableEntry + case newestEntry != nil: + metadata["latest"] = newestEntry + default: + delete(metadata, "latest") } } diff --git a/internal/handler/pub_test.go b/internal/handler/pub_test.go index 29526e0..5311e25 100644 --- a/internal/handler/pub_test.go +++ b/internal/handler/pub_test.go @@ -206,3 +206,58 @@ func TestPubDownloadServesCachedArchive(t *testing.T) { t.Errorf("body = %q, want %q", got, "cached archive") } } + +// TestPubRewriteMetadataLatest checks that latest points at the proxy and +// never names a version cooldown holds back, and that a held back latest is +// replaced with the newest stable version left. +func TestPubRewriteMetadataLatest(t *testing.T) { + old := time.Now().Add(-10 * 24 * time.Hour).Format(time.RFC3339) + recent := time.Now().Add(-time.Hour).Format(time.RFC3339) + entry := func(version, published string) string { + return `{"version":"` + version + `","published":"` + published + `","archive_url":"https://pub.dev/packages/demo/versions/` + version + `.tar.gz"}` + } + for _, tc := range []struct { + name, latest, versions string + cooldown bool + want, wantURL string + }{ + {"kept", entry("1.0.0", old), entry("1.0.0", old), false, "1.0.0", + "http://localhost:8080/pub/packages/demo/versions/1.0.0.tar.gz"}, + {"stable over prerelease", entry("2.0.0", recent), + entry("1.0.0", old) + "," + entry("1.1.0", old) + "," + entry("2.0.0-dev.1", old) + "," + entry("2.0.0", recent), true, "1.1.0", + "http://localhost:8080/pub/packages/demo/versions/1.1.0.tar.gz"}, + {"only prereleases", entry("2.0.0", recent), entry("2.0.0-dev.1", old) + "," + entry("2.0.0", recent), true, "2.0.0-dev.1", + "http://localhost:8080/pub/packages/demo/versions/2.0.0-dev.1.tar.gz"}, + {"all held back", entry("2.0.0", recent), entry("2.0.0", recent), true, "", ""}, + } { + t.Run(tc.name, func(t *testing.T) { + proxy := &Proxy{Logger: slog.Default()} + if tc.cooldown { + proxy.Cooldown = &cooldown.Config{Default: "3d"} + } + h := &PubHandler{proxy: proxy, proxyURL: "http://localhost:8080"} + output, err := h.rewriteMetadata("demo", []byte(`{"name":"demo","latest":`+tc.latest+`,"versions":[`+tc.versions+`]}`)) + if err != nil { + t.Fatal(err) + } + var result struct { + Latest *struct { + Version string `json:"version"` + ArchiveURL string `json:"archive_url"` + } `json:"latest"` + } + if err := json.Unmarshal(output, &result); err != nil { + t.Fatal(err) + } + if tc.want == "" { + if result.Latest != nil { + t.Errorf("latest = %+v, want none", *result.Latest) + } + return + } + if result.Latest == nil || result.Latest.Version != tc.want || result.Latest.ArchiveURL != tc.wantURL { + t.Errorf("latest = %+v, want %s at %s", result.Latest, tc.want, tc.wantURL) + } + }) + } +} diff --git a/internal/handler/pypi.go b/internal/handler/pypi.go index ebb3082..ac4315f 100644 --- a/internal/handler/pypi.go +++ b/internal/handler/pypi.go @@ -467,6 +467,19 @@ func (h *PyPIHandler) rewriteSimpleJSON(body []byte, filteredVersions map[string } metadata["files"] = rewrittenFiles + + // The versions list (PEP 700) loses the filtered versions too, whose + // files were all left out above. + if versions, ok := metadata["versions"].([]any); ok && len(filteredVersions) > 0 { + keptVersions := make([]any, 0, len(versions)) + for _, v := range versions { + if version, ok := v.(string); ok && filteredVersions[version] { + continue + } + keptVersions = append(keptVersions, v) + } + metadata["versions"] = keptVersions + } return json.Marshal(metadata) } diff --git a/internal/handler/pypi_test.go b/internal/handler/pypi_test.go index dd59974..f38492d 100644 --- a/internal/handler/pypi_test.go +++ b/internal/handler/pypi_test.go @@ -276,6 +276,32 @@ func TestPyPISimpleJSONCooldown(t *testing.T) { } } +// TestPyPISimpleJSONFiltersVersions checks that the PEP 700 versions list +// loses a version whose files were all filtered out, and keeps one that +// still has files. +func TestPyPISimpleJSONFiltersVersions(t *testing.T) { + h := &PyPIHandler{proxy: &Proxy{Logger: slog.Default()}, proxyURL: "http://proxy.test"} + body := `{"meta":{"api-version":"1.1"},"name":"ruff","versions":["1.0.0","2.0.0","3.0.0"],"files":[` + + `{"filename":"ruff-1.0.0.tar.gz","url":"https://files.pythonhosted.org/packages/ab/ruff-1.0.0.tar.gz"},` + + `{"filename":"ruff-2.0.0.tar.gz","url":"https://files.pythonhosted.org/packages/cd/ruff-2.0.0.tar.gz"},` + + `{"filename":"ruff-2.0.0-py3-none-any.whl","url":"https://files.pythonhosted.org/packages/ef/ruff-2.0.0-py3-none-any.whl"}` + + `]}` + out, err := h.rewriteSimpleJSON([]byte(body), map[string]bool{"2.0.0": true, "3.0.0": true}) + if err != nil { + t.Fatal(err) + } + var result struct { + Versions []string `json:"versions"` + } + if err := json.Unmarshal(out, &result); err != nil { + t.Fatal(err) + } + // 3.0.0 has no files to filter, but it is still held back. + if strings.Join(result.Versions, ",") != "1.0.0" { + t.Errorf("versions = %v, want [1.0.0]", result.Versions) + } +} + func TestPyPIParseFilename(t *testing.T) { h := &PyPIHandler{proxy: &Proxy{Logger: slog.Default()}} diff --git a/internal/metadata/npm/metadata.go b/internal/metadata/npm/metadata.go index 3a434c5..7fc4a40 100644 --- a/internal/metadata/npm/metadata.go +++ b/internal/metadata/npm/metadata.go @@ -402,7 +402,7 @@ func (h *transformer) tarballsRewritten(count int) { } // applyCooldownFiltering removes versions that are too recently published, -// and updates dist-tags.latest if the current latest was filtered out. +// and updates the dist-tags that named them. func (h *transformer) applyCooldownFiltering(metadata map[string]any, versions map[string]any, packageName string) { if h.Cooldown == nil || !h.Cooldown.Enabled() { return @@ -415,6 +415,7 @@ func (h *transformer) applyCooldownFiltering(metadata map[string]any, versions m packagePURL := packageurl.MakeString("npm", packageName, "") + withheld := map[string]bool{} for version := range versions { publishedStr, ok := timeMap[version].(string) if !ok { @@ -430,20 +431,29 @@ func (h *transformer) applyCooldownFiltering(metadata map[string]any, versions m } delete(versions, version) delete(timeMap, version) + withheld[version] = true } } - h.updateDistTagsLatest(metadata, versions, timeMap) + h.updateDistTags(metadata, versions, timeMap, withheld) } -// updateDistTagsLatest updates the dist-tags.latest field if the current latest -// version was removed by cooldown filtering. -func (h *transformer) updateDistTagsLatest(metadata, versions, timeMap map[string]any) { +// updateDistTags moves dist-tags.latest to the newest remaining version if +// the current latest was removed by cooldown filtering, and, as the denylist +// does, drops other tags that name a withheld version. +func (h *transformer) updateDistTags(metadata, versions, timeMap map[string]any, withheld map[string]bool) { distTags, ok := metadata["dist-tags"].(map[string]any) if !ok { return } + for tag, value := range distTags { + version, _ := value.(string) + if tag != "latest" && withheld[version] { + delete(distTags, tag) + } + } + latest, ok := distTags["latest"].(string) if !ok { return @@ -455,6 +465,8 @@ func (h *transformer) updateDistTagsLatest(metadata, versions, timeMap map[strin if newLatest := h.findNewestVersion(versions, timeMap); newLatest != "" { distTags["latest"] = newLatest + } else if withheld[latest] { + delete(distTags, "latest") } } diff --git a/internal/metadata/npm/metadata_test.go b/internal/metadata/npm/metadata_test.go index 28e9669..2748ed3 100644 --- a/internal/metadata/npm/metadata_test.go +++ b/internal/metadata/npm/metadata_test.go @@ -194,3 +194,39 @@ func TestNPMRewriteAttestationsURL(t *testing.T) { } } } + +// TestNPMRewriteCooldownDropsTags checks that cooldown, like the denylist, +// drops a tag other than latest that names a version it withholds, in both +// the in-memory and the streamed rewrite. +func TestNPMRewriteCooldownDropsTags(t *testing.T) { + now := time.Date(2026, 10, 18, 12, 0, 0, 0, time.UTC) + ts := func(age time.Duration) string { return now.Add(-age).UTC().Format(time.RFC3339) } + options := Options{ProxyURL: "http://proxy", EvaluatedAt: now, Cooldown: &cooldown.Config{Default: "3d"}} + input := `{"name":"demo",` + + `"dist-tags":{"latest":"3.0.0","next":"3.0.0","legacy":"1.0.0"},` + + `"versions":{"1.0.0":{},"2.0.0":{},"3.0.0":{}},` + + `"time":{"1.0.0":"` + ts(900*time.Hour) + `","2.0.0":"` + ts(800*time.Hour) + `","3.0.0":"` + ts(time.Hour) + `"}}` + + rewrite := map[string]func() ([]byte, []string, error){ + "memory": func() ([]byte, []string, error) { return Rewrite("demo", []byte(input), options) }, + "stream": func() ([]byte, []string, error) { return rewriteStream("demo", []byte(input), options) }, + } + for name, run := range rewrite { + t.Run(name, func(t *testing.T) { + out, _, err := run() + if err != nil { + t.Fatal(err) + } + var doc struct { + DistTags map[string]string `json:"dist-tags"` + } + if err := json.Unmarshal(out, &doc); err != nil { + t.Fatalf("output is not JSON: %v\n%s", err, out) + } + want := map[string]string{"latest": "2.0.0", "legacy": "1.0.0"} + if fmt.Sprint(doc.DistTags) != fmt.Sprint(want) { + t.Errorf("dist-tags = %v, want %v", doc.DistTags, want) + } + }) + } +} From 3860a6cae5a6831641bc02b36ecd5efce70418a4 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 8 Nov 2026 08:00:50 +0100 Subject: [PATCH 586/633] Don't cache Maven paths with dotted group directories parsePath joins the group directories with dots, and the cache filename is the file's base name, so org.foo/bar/1.0/bar-1.0.jar and org/foo/bar/1.0/bar-1.0.jar shared one cache entry. Only upstreams that serve content at such directories, such as raw or hosted repositories, are affected. The Maven layout splits a group at its dots, so a group directory never contains a dot or is empty. Treat such paths as outside the layout and proxy them uncached, which keeps every existing cache entry valid. --- docs/configuration.md | 4 ++++ internal/handler/maven.go | 9 +++++++++ internal/handler/maven_repositories_test.go | 19 +++++++++++++++++++ internal/handler/maven_test.go | 5 +++++ 4 files changed, 37 insertions(+) diff --git a/docs/configuration.md b/docs/configuration.md index 9046e28..59a33b0 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1004,6 +1004,10 @@ one doesn't fall back to the Gradle Plugin Portal. Files are cached per repository, so the same coordinates in two repositories never share a cache entry. +In every repository, a path whose group directories contain a dot or are empty +(`org.foo/bar/1.0/bar-1.0.jar`) is not in the Maven layout and is proxied +without caching, since its group would read the same as `org/foo`'s. + `upstream.maven_repositories` is additive. `/maven/…` keeps serving `upstream.maven` with unchanged cache identities. A repository name is matched before the default repository, so it hides every group ID whose first segment diff --git a/internal/handler/maven.go b/internal/handler/maven.go index 6c6a647..ca81b74 100644 --- a/internal/handler/maven.go +++ b/internal/handler/maven.go @@ -266,6 +266,7 @@ func (h *MavenHandler) handleDownload(w http.ResponseWriter, r *http.Request, so // parsePath extracts Maven coordinates from a URL path. // e.g., "com/google/guava/guava/32.1.3-jre/guava-32.1.3-jre.jar" // -> ("com.google.guava", "guava", "32.1.3-jre", "guava-32.1.3-jre.jar") +// A path that isn't in the Maven layout returns empty strings. func (h *MavenHandler) parsePath(urlPath string) (group, artifact, version, filename string) { parts := strings.Split(urlPath, "/") if len(parts) < minMavenParts { @@ -276,6 +277,14 @@ func (h *MavenHandler) parsePath(urlPath string) (group, artifact, version, file version = parts[len(parts)-2] artifact = parts[len(parts)-3] groupParts := parts[:len(parts)-3] + // The Maven layout splits a group at its dots, so a group directory + // never holds a dot or is empty. Such a path would spell the same group + // as another and share its cache entries, so it is not parsed. + for _, part := range groupParts { + if part == "" || strings.Contains(part, ".") { + return "", "", "", "" + } + } group = strings.Join(groupParts, ".") return group, artifact, version, filename diff --git a/internal/handler/maven_repositories_test.go b/internal/handler/maven_repositories_test.go index 8272f37..15db066 100644 --- a/internal/handler/maven_repositories_test.go +++ b/internal/handler/maven_repositories_test.go @@ -137,3 +137,22 @@ func TestMavenRepositoryRepointedServesTheNewUpstream(t *testing.T) { } } } + +// TestMavenDottedGroupDirectoryNotCachedAsGroup checks that a path whose +// group directories contain dots is not cached under the group the dots +// spell, where it would share an entry with the standard layout's path. +func TestMavenDottedGroupDirectoryNotCachedAsGroup(t *testing.T) { + const standard = "/org/foo/bar/1.0/bar-1.0.jar" + const dotted = "/org.foo/bar/1.0/bar-1.0.jar" + central, _ := mavenRepoUpstream(t, "central", standard, dotted) + portal, _ := mavenRepoUpstream(t, "portal") + grails, _ := mavenRepoUpstream(t, "grails") + h := newMavenRepositoriesHandler(t, central, portal, grails) + + for _, path := range []string{standard, dotted, standard} { + w := getMaven(h, path) + if want := "central " + path; w.Code != http.StatusOK || w.Body.String() != want { + t.Errorf("GET %s = %d %q, want 200 %q", path, w.Code, w.Body.String(), want) + } + } +} diff --git a/internal/handler/maven_test.go b/internal/handler/maven_test.go index 9d34b09..3d70a32 100644 --- a/internal/handler/maven_test.go +++ b/internal/handler/maven_test.go @@ -31,6 +31,11 @@ func TestMavenParsePath(t *testing.T) { "short/path", "", "", "", "", }, + // Group directories never hold dots or are empty in the Maven + // layout; these would spell the same group as org/foo. + {"org.foo/bar/1.0/bar-1.0.jar", "", "", "", ""}, + {"org/.foo/bar/1.0/bar-1.0.jar", "", "", "", ""}, + {"org//foo/bar/1.0/bar-1.0.jar", "", "", "", ""}, } for _, tt := range tests { From 81c6ea26c2350bab1c6c610e419e866e76733022 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 8 Nov 2026 08:00:50 +0100 Subject: [PATCH 587/633] Treat git keys differing only in case as one repository On a case-insensitive file system, as macOS and Windows use by default, acme/WIDGETS and acme/widgets name the same cache directory but were tracked as two repositories. Users of one spelling didn't keep the other from being evicted, so eviction could move the directory away while it was being read, and each entry had its own write lock, so two refreshes could fetch into the directory at once. Check at startup whether the cache directory ignores case and, if so, track repositories under their lowercased key. The directory and the upstream URL keep the spelling requested. Case-sensitive file systems, and so existing caches on Linux, are unchanged. --- docs/configuration.md | 6 ++++++ internal/gitcache/cache.go | 35 +++++++++++++++++++++++++++++---- internal/gitcache/cache_test.go | 27 +++++++++++++++++++++++++ 3 files changed, 64 insertions(+), 4 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 59a33b0..30c9a0c 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -946,6 +946,12 @@ those yourself. The proxy locks `cache_dir` while it runs, through a `.gitcache-lock` file in it, so a second process given the same directory, such as another replica on a shared volume, fails to start; give each its own. +Each repository's directory is named by its path as requested. Where +`cache_dir` ignores case in file names, as on macOS and Windows by default, +paths that differ only in case (`acme/widgets` and `acme/WIDGETS`) share one +directory and are treated as one repository. Use a case-sensitive file system +for an upstream whose repositories differ only in case. + Git LFS objects are cached in the artifact cache, so they use the configured storage backend and retention rather than `cache_dir`. git-lfs finds the LFS API under the rewritten remote, at `/git/{name}/{repo}.git/info/lfs`, with no diff --git a/internal/gitcache/cache.go b/internal/gitcache/cache.go index eb1578f..6bc3c20 100644 --- a/internal/gitcache/cache.go +++ b/internal/gitcache/cache.go @@ -95,6 +95,9 @@ type Cache struct { refCheckTimeout time.Duration lowSpeedTime int maxRequestBody int64 + // foldCase is set where the cache directory ignores case, so keys that + // differ only in case, which name one directory, are one repository. + foldCase bool ctx context.Context cancel context.CancelFunc @@ -214,6 +217,7 @@ func New(opts Options) (*Cache, error) { tooLarge: make(map[string]time.Time), failedWants: make(map[string]time.Time), freed: make(chan struct{}), + foldCase: caseInsensitive(dir), } if opts.refCheckTimeout > 0 { c.refCheckTimeout = opts.refCheckTimeout @@ -284,7 +288,7 @@ func (c *Cache) load() error { return err } rel, _ := filepath.Rel(c.dir, path) - rp := newRepo(filepath.ToSlash(rel), path) + rp := newRepo(c.repoKey(filepath.ToSlash(rel)), path) rp.size.Store(dirSize(path)) rp.lastChecked.Store(stampTime(path, checkedStamp)) rp.lastUsed.Store(stampTime(path, usedStamp)) @@ -352,15 +356,24 @@ func (c *Cache) removeRepointed() { func (c *Cache) acquire(key string) *repo { c.mu.Lock() defer c.mu.Unlock() - rp := c.repos[key] + rp := c.repos[c.repoKey(key)] if rp == nil { - rp = newRepo(key, filepath.Join(c.dir, filepath.FromSlash(key))) - c.repos[key] = rp + rp = newRepo(c.repoKey(key), filepath.Join(c.dir, filepath.FromSlash(key))) + c.repos[rp.key] = rp } rp.users++ return rp } +// repoKey is the key a repository is tracked under: key itself, or key +// lowercased where the cache directory ignores case. +func (c *Cache) repoKey(key string) string { + if c.foldCase { + return strings.ToLower(key) + } + return key +} + func (c *Cache) release(rp *repo) { now := time.Now() rp.lastUsed.Store(now.UnixNano()) @@ -511,3 +524,17 @@ func touch(dir, name string, t time.Time) { } } } + +// caseInsensitive reports whether the file system holding dir ignores case +// in names, as macOS and Windows do by default. +func caseInsensitive(dir string) bool { + f, err := os.CreateTemp(dir, ".gitcache-case-") + if err != nil { + return false + } + name := f.Name() + _ = f.Close() + defer func() { _ = os.Remove(name) }() + _, err = os.Stat(filepath.Join(dir, strings.ToUpper(filepath.Base(name)))) + return err == nil +} diff --git a/internal/gitcache/cache_test.go b/internal/gitcache/cache_test.go index 4d43c5d..690b11e 100644 --- a/internal/gitcache/cache_test.go +++ b/internal/gitcache/cache_test.go @@ -786,3 +786,30 @@ func TestCloseTwice(t *testing.T) { t.Errorf("closing twice: %v", err) } } + +// TestCaseOnlyDifferentKeysShareOneRepository checks that, where the file +// system ignores case, keys that differ only in case are one repository, so +// a user of one spelling keeps the directory from being evicted under it. +func TestCaseOnlyDifferentKeysShareOneRepository(t *testing.T) { + if !caseInsensitive(t.TempDir()) { + t.Skip("file system is case-sensitive") + } + upstream := newFakeUpstream(t) + upstream.newRepo("repo") + tc := newTestCache(t, upstream.server.URL, nil) + tc.clone(t, "repo") + waitIdle(t, tc.Cache) + + rp := tc.acquire("up/REPO.git") + defer tc.release(rp) + if other := tc.repo("up/repo.git"); other != nil && other != rp { + t.Error("up/REPO.git and up/repo.git are separate repositories") + } + tc.mu.Lock() + tc.opts.MaxSize = 1 + tc.mu.Unlock() + tc.evict() + if !rp.exists() { + t.Error("repository in use as up/REPO.git was evicted") + } +} From 1fbcdc95c8fbd4dc3c929de0765e115c766c0f38 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 8 Nov 2026 07:58:55 +0100 Subject: [PATCH 588/633] Rewrite project links in the PyPI root simple index pypi.org's HTML root index links each project as /simple//. The proxy relayed it unchanged, but it is mounted at /pypi, so following a link went to /simple// on the proxy and got 404. Rewrite those hrefs to the proxy's /pypi/simple/ while streaming the response, since the index is tens of megabytes. The JSON index has no links and is left alone. --- internal/handler/pypi.go | 70 +++++++++++++++++++++++++++++++++++ internal/handler/pypi_test.go | 50 +++++++++++++++++++++++++ 2 files changed, 120 insertions(+) diff --git a/internal/handler/pypi.go b/internal/handler/pypi.go index ebb3082..7dd3f12 100644 --- a/internal/handler/pypi.go +++ b/internal/handler/pypi.go @@ -7,6 +7,7 @@ import ( "encoding/json" "errors" "fmt" + "io" "maps" "mime" "net/http" @@ -1022,8 +1023,77 @@ func (h *PyPIHandler) proxySimple(w http.ResponseWriter, r *http.Request, path s } defer func() { _ = resp.Body.Close() }() + // The HTML root index links projects as /simple//, which must + // point at the proxy's /pypi mount instead. It is large, so it is + // rewritten as it streams. + if resp.StatusCode == http.StatusOK && resp.Header.Get("Content-Encoding") == "" && + !strings.Contains(resp.Header.Get(headerContentType), "json") { + resp.Body = &replacingReader{ + src: resp.Body, + from: []byte(`href="/simple/`), + to: []byte(`href="` + h.proxyURL + `/pypi/simple/`), + } + resp.Header.Del(headerContentLength) + resp.ContentLength = -1 + } + h.proxy.relayResponse(w, r, resp, func(dst, src http.Header) { copyRelayHeaders(dst, src) ensureVaryAccept(dst) }) } + +const replacingReaderBufferSize = 32 << 10 + +// replacingReader replaces every occurrence of from with to in src as it is +// read. +type replacingReader struct { + src io.ReadCloser + from, to []byte + in []byte // read from src, not yet scanned + out []byte // scanned, not yet returned + buf []byte + err error +} + +func (r *replacingReader) Read(p []byte) (int, error) { + for len(r.out) == 0 { + if r.err != nil { + return 0, r.err + } + if r.buf == nil { + r.buf = make([]byte, replacingReaderBufferSize) + } + n, err := r.src.Read(r.buf) + r.in = append(r.in, r.buf[:n]...) + r.err = err + // Keep a tail that could be the start of a match split across reads. + keep := len(r.from) - 1 + if err != nil { + keep = 0 + } + scan := max(len(r.in)-keep, 0) + var out []byte + done := 0 + for { + i := bytes.Index(r.in[done:], r.from) + if i < 0 || done+i >= scan { + break + } + out = append(out, r.in[done:done+i]...) + out = append(out, r.to...) + done += i + len(r.from) + } + if done < scan { + out = append(out, r.in[done:scan]...) + done = scan + } + r.out = out + r.in = append([]byte(nil), r.in[done:]...) + } + n := copy(p, r.out) + r.out = r.out[n:] + return n, nil +} + +func (r *replacingReader) Close() error { return r.src.Close() } diff --git a/internal/handler/pypi_test.go b/internal/handler/pypi_test.go index dd59974..73309cb 100644 --- a/internal/handler/pypi_test.go +++ b/internal/handler/pypi_test.go @@ -2,13 +2,16 @@ package handler import ( "encoding/json" + "fmt" "io" "log/slog" "net/http" "net/http/httptest" + "strconv" "strings" "sync/atomic" "testing" + "testing/iotest" "time" "github.com/git-pkgs/cooldown" @@ -611,3 +614,50 @@ func TestPyPIDownloadCooldownMetadataCache(t *testing.T) { t.Errorf("upstream metadata JSON requests = %d, want 1 (repeated downloads should reuse the cached metadata)", got) } } + +func TestPyPISimpleIndexRewritesProjectLinks(t *testing.T) { + // Long enough that links cross the rewriter's read boundaries. + var upstream, want strings.Builder + upstream.WriteString(``) + want.WriteString(``) + for i := range 3000 { + fmt.Fprintf(&upstream, `pkg-%d`+"\n", i, i) + fmt.Fprintf(&want, `pkg-%d`+"\n", i, i) + } + upstream.WriteString(``) + want.WriteString(``) + + h, _ := setupPyPIHandler(t, func(r *http.Request) (*http.Response, error) { + resp := pypiHTTPResponse(r, "text/html", upstream.String()) + resp.Header.Set("Content-Length", strconv.Itoa(upstream.Len())) + return resp, nil + }) + + req := httptest.NewRequest(http.MethodGet, "/simple/", nil) + w := httptest.NewRecorder() + h.Routes().ServeHTTP(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want 200", w.Code) + } + if got := w.Body.String(); got != want.String() { + t.Errorf("body not rewritten; starts %q", got[:min(len(got), 120)]) + } + if got := w.Header().Get("Content-Length"); got != "" && got != strconv.Itoa(want.Len()) { + t.Errorf("Content-Length = %q, want %d or unset", got, want.Len()) + } +} + +func TestReplacingReaderSplitReads(t *testing.T) { + const in = `abxhref="/simplehref="/simple/` + const want = `abxhref="/simplehref="P/` + r := &replacingReader{ + src: io.NopCloser(iotest.OneByteReader(strings.NewReader(in))), + from: []byte(`href="/simple/`), + to: []byte(`href="P/`), + } + got, err := io.ReadAll(r) + if err != nil || string(got) != want { + t.Errorf("got %q, %v; want %q", got, err, want) + } +} From 20cbeeff22f25a0645ab3023e41114ce75ff8e12 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 8 Nov 2026 07:59:45 +0100 Subject: [PATCH 589/633] Disable install notifications for pushed Composer versions Composer reports installs to the repository's notify-batch URL unless a version has its own notification-url, and the proxy's packages.json sets notify-batch to the upstream. Routed packages get an empty notification-url for this reason, but pushed and hosted-only versions did not, so installs of private packages were reported to packagist.org. Give pushed versions the same empty notification-url. --- docs/configuration.md | 1 + internal/handler/composer_hosted.go | 5 +++++ internal/handler/composer_hosted_test.go | 19 +++++++++++++++++++ 3 files changed, 25 insertions(+) diff --git a/docs/configuration.md b/docs/configuration.md index 584896b..3e7abc7 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1643,6 +1643,7 @@ Pushed versions are listed in the package metadata clients resolve against, with - Pushed versions are added next to the upstream ones and are never held back by cooldown. npm `dist-tags` stay as the upstream has them, also in `npm dist-tag ls`. - A package the upstream doesn't know is served from its pushed versions alone. Its npm `latest` tag is the highest pushed version. - Pushed Composer dev versions (`dev-*`, `*-dev`) are listed in the `~dev` metadata file. +- Pushed Composer versions carry an empty `notification-url`, as routed packages do, so Composer does not report their installs to the default upstream's `notify-batch` endpoint. - npm entries carry the archive's sha512 as `dist.integrity`. Composer entries carry its sha256 as `dist.reference` and leave `shasum` empty. Metadata of packages with pushed versions is rewritten on every request instead of being cached, so a push or delete shows up on the next request. diff --git a/internal/handler/composer_hosted.go b/internal/handler/composer_hosted.go index e94d11c..3c42a6a 100644 --- a/internal/handler/composer_hosted.go +++ b/internal/handler/composer_hosted.go @@ -43,6 +43,11 @@ func (h *ComposerHandler) composerLocalEntries(packageName string, dev bool) ([] manifest["name"] = packageName manifest["version"] = lv.Version manifest["time"] = lv.PushedAt.Format(time.RFC3339) + // As for routed packages, keep Composer from reporting installs of + // a private version to the default repository's notify-batch. + if _, ok := manifest["notification-url"]; !ok { + manifest["notification-url"] = "" + } manifest["dist"] = map[string]string{ "type": "zip", "url": h.localDistURL(packageName, lv.Version, lv.Filename), diff --git a/internal/handler/composer_hosted_test.go b/internal/handler/composer_hosted_test.go index 539e999..6c439dc 100644 --- a/internal/handler/composer_hosted_test.go +++ b/internal/handler/composer_hosted_test.go @@ -243,3 +243,22 @@ func TestComposerDownloadOfPushedArchiveWhileStreamingArtifacts(t *testing.T) { t.Error("a pushed archive was fetched upstream while streaming artifacts") } } + +func TestComposerPushedVersionsDisableDefaultNotification(t *testing.T) { + f := newComposerHostedFixture(t, map[string]string{"/p2/monolog/monolog.json": monologMetadata}) + f.proxy.HostedComposerVendors = []string{"acme"} + f.push(t, "monolog/monolog", "3.6.0", `{"name":"monolog/monolog"}`) + f.push(t, "acme/private", "1.0.0", `{"name":"acme/private"}`) + f.push(t, "other/private", "1.0.0", `{"name":"other/private"}`) + + for _, tc := range []struct{ path, name, version string }{ + {"/p2/monolog/monolog.json", "monolog/monolog", "3.6.0"}, + {"/p2/acme/private.json", "acme/private", "1.0.0"}, + {"/p2/other/private.json", "other/private", "1.0.0"}, + } { + v := f.versions(t, tc.path, tc.name)[tc.version] + if got, ok := v["notification-url"]; !ok || got != "" { + t.Errorf("%s %s notification-url = %v (set %v), want empty", tc.name, tc.version, got, ok) + } + } +} From 12666071ce2d61ed656560781980b52198522cde Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 8 Nov 2026 08:00:50 +0100 Subject: [PATCH 590/633] Make package search case-insensitive on Postgres too SearchPackages and CountSearchResults matched names with p.name LIKE '%query%'. LIKE ignores ASCII case on SQLite but not on Postgres, so the same search found different packages depending on the database. The % and _ in a query were also taken as wildcards. Compare LOWER(name) with LOWER(pattern) and escape the wildcards. --- internal/database/queries.go | 23 ++++++++--- .../database/queries_packages_list_test.go | 39 +++++++++++++++++++ 2 files changed, 56 insertions(+), 6 deletions(-) diff --git a/internal/database/queries.go b/internal/database/queries.go index 86f31ef..7176152 100644 --- a/internal/database/queries.go +++ b/internal/database/queries.go @@ -849,7 +849,7 @@ func (db *DB) SearchPackages(query string, ecosystem string, limit int, offset i } var results []SearchResult - searchPattern := "%" + query + "%" + searchPattern := nameSearchPattern(query) var sqlQuery string var args []any @@ -864,7 +864,7 @@ func (db *DB) SearchPackages(query string, ecosystem string, limit int, offset i FROM packages p LEFT JOIN versions v ON v.package_purl = p.purl LEFT JOIN artifacts a ON a.version_purl = v.purl - WHERE p.name LIKE ? AND ` + condition + ` AND a.storage_path IS NOT NULL + WHERE ` + nameSearchCondition + ` AND ` + condition + ` AND a.storage_path IS NOT NULL GROUP BY p.purl, p.ecosystem, p.name, p.latest_version, p.license ORDER BY hits DESC LIMIT ? OFFSET ? @@ -879,7 +879,7 @@ func (db *DB) SearchPackages(query string, ecosystem string, limit int, offset i FROM packages p LEFT JOIN versions v ON v.package_purl = p.purl LEFT JOIN artifacts a ON a.version_purl = v.purl - WHERE p.name LIKE ? AND a.storage_path IS NOT NULL + WHERE ` + nameSearchCondition + ` AND a.storage_path IS NOT NULL GROUP BY p.purl, p.ecosystem, p.name, p.latest_version, p.license ORDER BY hits DESC LIMIT ? OFFSET ? @@ -894,6 +894,17 @@ func (db *DB) SearchPackages(query string, ecosystem string, limit int, offset i return results, nil } +// nameSearchCondition matches package names containing a search pattern +// from nameSearchPattern, ignoring case on SQLite and Postgres alike. +const nameSearchCondition = `LOWER(p.name) LIKE LOWER(?) ESCAPE '!'` + +// nameSearchPattern is the LIKE pattern for names containing query, with +// the LIKE wildcards in query matched literally. +func nameSearchPattern(query string) string { + escaped := strings.NewReplacer("!", "!!", "%", "!%", "_", "!_").Replace(query) + return "%" + escaped + "%" +} + func (db *DB) CountSearchResults(query string, ecosystem string) (int64, error) { hasArtifacts, err := db.HasTable("artifacts") if err != nil { @@ -903,7 +914,7 @@ func (db *DB) CountSearchResults(query string, ecosystem string) (int64, error) return 0, nil } - searchPattern := "%" + query + "%" + searchPattern := nameSearchPattern(query) var sqlQuery string var args []any @@ -914,7 +925,7 @@ func (db *DB) CountSearchResults(query string, ecosystem string) (int64, error) FROM packages p LEFT JOIN versions v ON v.package_purl = p.purl LEFT JOIN artifacts a ON a.version_purl = v.purl - WHERE p.name LIKE ? AND ` + condition + ` AND a.storage_path IS NOT NULL + WHERE ` + nameSearchCondition + ` AND ` + condition + ` AND a.storage_path IS NOT NULL `) args = append([]any{searchPattern}, ecosystemArgs...) } else { @@ -923,7 +934,7 @@ func (db *DB) CountSearchResults(query string, ecosystem string) (int64, error) FROM packages p LEFT JOIN versions v ON v.package_purl = p.purl LEFT JOIN artifacts a ON a.version_purl = v.purl - WHERE p.name LIKE ? AND a.storage_path IS NOT NULL + WHERE ` + nameSearchCondition + ` AND a.storage_path IS NOT NULL `) args = []any{searchPattern} } diff --git a/internal/database/queries_packages_list_test.go b/internal/database/queries_packages_list_test.go index 325fee7..704b388 100644 --- a/internal/database/queries_packages_list_test.go +++ b/internal/database/queries_packages_list_test.go @@ -3,6 +3,7 @@ package database import ( "database/sql" "maps" + "slices" "testing" "time" ) @@ -250,3 +251,41 @@ func TestCachedPackageFiltersMatchEcosystemAliases(t *testing.T) { } }) } + +func TestSearchPackagesIgnoresCaseAndEscapesWildcards(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + seedArtifact(t, db, "pypi", "Django", "5.0.0", 100, 1) + seedArtifact(t, db, "npm", "my_pkg", "1.0.0", 100, 1) + seedArtifact(t, db, "npm", "myxpkg", "1.0.0", 100, 1) + seedArtifact(t, db, "npm", "100%-done", "1.0.0", 100, 1) + seedArtifact(t, db, "npm", "1000-done", "1.0.0", 100, 1) + seedArtifact(t, db, "npm", `back\slash`, "1.0.0", 100, 1) + + for _, tc := range []struct { + query string + want []string + }{ + {"django", []string{"Django"}}, + {"DJANGO", []string{"Django"}}, + {"my_pkg", []string{"my_pkg"}}, + {"100%", []string{"100%-done"}}, + {`k\s`, []string{`back\slash`}}, + } { + found, err := db.SearchPackages(tc.query, "", 50, 0) + if err != nil { + t.Fatalf("SearchPackages(%q): %v", tc.query, err) + } + var names []string + for _, p := range found { + names = append(names, p.Name) + } + if !slices.Equal(names, tc.want) { + t.Errorf("SearchPackages(%q) = %v, want %v", tc.query, names, tc.want) + } + count, err := db.CountSearchResults(tc.query, "") + if err != nil || count != int64(len(tc.want)) { + t.Errorf("CountSearchResults(%q) = %d, %v, want %d", tc.query, count, err, len(tc.want)) + } + } + }) +} From b3728b1e1645ddf0d71fc3962b8744f1edd5e0a6 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 8 Nov 2026 07:58:23 +0100 Subject: [PATCH 591/633] Check the cached copy before serving upload-pack requests Only GET info/refs checked the upstream. A POST git-upload-pack was served from the cached copy unchecked, so a client posting ls-refs and fetch directly (protocol v2 allows it) kept getting a repository that was deleted or made private upstream, past stale_max_age, indefinitely. Remember when a refresh finds a repository missing or refused upstream, and refuse upload-pack requests for it until a refresh succeeds. Refresh first when the copy was last checked more than a minute past stale_max_age ago; the minute keeps a normal clone (info/refs, then upload-pack) at one upstream check even with stale_max_age 0. Mirror refreshes on its own and skips the check. --- docs/configuration.md | 9 +++- internal/gitcache/cache.go | 7 ++- internal/gitcache/cache_test.go | 89 +++++++++++++++++++++++++++++++++ internal/gitcache/mirror.go | 2 +- internal/gitcache/refresh.go | 6 +++ internal/gitcache/serve.go | 41 +++++++++++++-- 6 files changed, 144 insertions(+), 10 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 584896b..d384bac 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -899,8 +899,13 @@ commits, the request waits for their fetch, so a clone right after a push gets them; if the fetch fails or stalls, the cached copy is served. A `stale_max_age` of `0` never serves a copy the upstream has not just confirmed. A repository the upstream reports missing (404), refuses (401, 403) -or redirects is not served from the cache. Redirects are never followed, so -point the upstream at a repository's current location. +or redirects is not served from the cache, also to requests that skip the ref +listing and go straight to `git-upload-pack`, until a later check succeeds. +Redirects are never followed, so point the upstream at a repository's current +location. A `git-upload-pack` request does not check the upstream itself +unless the copy was last checked more than a minute past `stale_max_age` ago, +so the requests of a clone that follow its ref listing check nothing again; +an older copy is checked first, as for a ref listing. The proxy runs `git` 2.31 or newer, which must be installed (the Docker image includes it). The `git` processes never reach the network themselves: their fetches go diff --git a/internal/gitcache/cache.go b/internal/gitcache/cache.go index eb1578f..7901380 100644 --- a/internal/gitcache/cache.go +++ b/internal/gitcache/cache.go @@ -121,14 +121,17 @@ type Cache struct { failedWants map[string]time.Time } -// repo is a cached repository, or one being cloned. users, refreshing and -// the map entry itself are guarded by Cache.mu. +// repo is a cached repository, or one being cloned. users, refreshing, +// refused and the map entry itself are guarded by Cache.mu. type repo struct { key string dir string users int refreshing *refreshCall + // refused is the error of the last refresh when the upstream reported + // the repository missing or refused it, until a refresh succeeds. + refused error // writes serializes git fetches into the repository. writes chan struct{} diff --git a/internal/gitcache/cache_test.go b/internal/gitcache/cache_test.go index 4d43c5d..5f9d475 100644 --- a/internal/gitcache/cache_test.go +++ b/internal/gitcache/cache_test.go @@ -786,3 +786,92 @@ func TestCloseTwice(t *testing.T) { t.Errorf("closing twice: %v", err) } } + +// lsRefs sends a protocol v2 ls-refs request for "repo" straight to +// git-upload-pack, as a client may without asking for info/refs first, and +// returns the status. +func (tc *testCache) lsRefs(t *testing.T) int { + t.Helper() + body := pktLine("command=ls-refs\n") + pktDelim + pktLine("peel\n") + pktFlush + req, err := http.NewRequestWithContext(t.Context(), http.MethodPost, tc.url("repo")+"/git-upload-pack", strings.NewReader(body)) + if err != nil { + t.Fatal(err) + } + req.Header.Set("Content-Type", requestType) + req.Header.Set("Git-Protocol", "version=2") + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + _ = resp.Body.Close() + return resp.StatusCode +} + +// TestUploadPackChecksCopyPastStaleMaxAge sends upload-pack requests without +// info/refs: a copy not checked within stale_max_age is checked first. +func TestUploadPackChecksCopyPastStaleMaxAge(t *testing.T) { + upstream := newFakeUpstream(t) + upstream.newRepo("repo") + tc := newTestCache(t, upstream.server.URL, nil) + tc.clone(t, "repo") + if status := tc.lsRefs(t); status != http.StatusOK { + t.Fatalf("ls-refs on a fresh copy: status %d", status) + } + + waitIdle(t, tc.Cache) + rp := tc.acquire("up/repo.git") + rp.lastChecked.Store(time.Now().Add(-2 * time.Hour).UnixNano()) + tc.release(rp) + upstream.status.Store(http.StatusNotFound) + if status := tc.lsRefs(t); status == http.StatusOK { + t.Errorf("served a copy older than stale_max_age to ls-refs") + } +} + +// TestUploadPackRefusedAfterUpstreamRefusal has the upstream refuse a +// refresh: upload-pack requests are refused too until a check succeeds. +func TestUploadPackRefusedAfterUpstreamRefusal(t *testing.T) { + upstream := newFakeUpstream(t) + upstream.newRepo("repo") + tc := newTestCache(t, upstream.server.URL, nil) + tc.clone(t, "repo") + + upstream.status.Store(http.StatusForbidden) + if out, err := tryGit(t, "", "ls-remote", tc.url("repo")); err == nil { + t.Fatalf("served the cached copy after 403:\n%s", out) + } + if status := tc.lsRefs(t); status == http.StatusOK { + t.Errorf("ls-refs served after the upstream refused the repository") + } + + upstream.status.Store(0) + tc.clone(t, "repo") + if status := tc.lsRefs(t); status != http.StatusOK { + t.Errorf("ls-refs after a successful check: status %d", status) + } +} + +// TestCloneChecksUpstreamOnce clones with a stale_max_age of 0: the +// upload-pack requests after info/refs do not check the upstream again. +func TestCloneChecksUpstreamOnce(t *testing.T) { + upstream := newFakeUpstream(t) + upstream.newRepo("repo") + tc := newTestCache(t, upstream.server.URL, func(o *Options) { o.StaleMaxAge = 0 }) + tc.clone(t, "repo") + waitIdle(t, tc.Cache) + + checks := func() int32 { return upstream.requests.Load() - upstream.gitRequests.Load() } + before := checks() + resp, err := http.Get(tc.url("repo") + "/info/refs?service=git-upload-pack") + if err != nil { + t.Fatal(err) + } + _ = resp.Body.Close() + perCheck := checks() - before + + before = checks() + tc.clone(t, "repo") + if got := checks() - before; got != perCheck { + t.Errorf("clone made %d upstream check requests, want %d", got, perCheck) + } +} diff --git a/internal/gitcache/mirror.go b/internal/gitcache/mirror.go index b5bde40..ad1584f 100644 --- a/internal/gitcache/mirror.go +++ b/internal/gitcache/mirror.go @@ -26,7 +26,7 @@ func (c *Cache) Mirror(ctx context.Context, key, repoURL, commit string, fetchWa if commit != "" && !isObjectID(commit) { return MirrorResult{}, fmt.Errorf("%q is not a full commit ID", commit) } - lease, err := c.Open(ctx, key, repoURL, OpenOptions{FetchWants: fetchWants}) + lease, err := c.Open(ctx, key, repoURL, OpenOptions{FetchWants: fetchWants, NoCheck: true}) if err != nil { return MirrorResult{}, err } diff --git a/internal/gitcache/refresh.go b/internal/gitcache/refresh.go index 448b3d9..352eb2b 100644 --- a/internal/gitcache/refresh.go +++ b/internal/gitcache/refresh.go @@ -157,6 +157,12 @@ func (c *Cache) startRefresh(rp *repo, repoURL string) *refreshCall { } c.mu.Lock() rp.refreshing = nil + switch { + case call.err == nil: + rp.refused = nil + case !servesStale(call.err): + rp.refused = call.err + } c.mu.Unlock() c.release(rp) close(call.done) diff --git a/internal/gitcache/serve.go b/internal/gitcache/serve.go index bb5cfb0..68c891a 100644 --- a/internal/gitcache/serve.go +++ b/internal/gitcache/serve.go @@ -13,6 +13,7 @@ import ( "slices" "strings" "sync/atomic" + "time" ) // gitProtocolPattern is what a Git-Protocol header may hold, such as @@ -42,6 +43,11 @@ type OpenOptions struct { // commits the client may not read through this repository, such as // GitHub forks with credentials configured. FetchWants bool + // NoCheck leaves a cached repository as it is, for a caller that + // refreshes it itself. Otherwise a cached repository is refused when the + // last refresh found it gone or refused upstream, and refreshed first + // when it was not checked within StaleMaxAge. + NoCheck bool } // Open returns a lease on the repository cached under key, a relative path @@ -53,15 +59,40 @@ func (c *Cache) Open(ctx context.Context, key, repoURL string, opts OpenOptions) } rp := c.acquire(key) hit := rp.exists() - if opts.Refresh || !hit { - if err := c.ensureFresh(ctx, rp, repoURL); err != nil { - c.release(rp) - return nil, err - } + var err error + switch { + case opts.Refresh || !hit: + err = c.ensureFresh(ctx, rp, repoURL) + case !opts.NoCheck: + err = c.checkServable(ctx, rp, repoURL) + } + if err != nil { + c.release(rp) + return nil, err } return &Lease{cache: c, repo: rp, repoURL: repoURL, hit: hit, fetchWants: opts.FetchWants}, nil } +// uploadPackGrace is how long past StaleMaxAge a copy is still served +// without a check to the upload-pack requests that follow a client's +// info/refs, so a clone checks the upstream once. +const uploadPackGrace = time.Minute + +// checkServable refuses a cached repository the last refresh found gone or +// refused upstream, and refreshes one not checked recently first. +func (c *Cache) checkServable(ctx context.Context, rp *repo, repoURL string) error { + c.mu.Lock() + err := rp.refused + c.mu.Unlock() + if err != nil { + return err + } + if time.Since(time.Unix(0, rp.lastChecked.Load())) > c.opts.StaleMaxAge+uploadPackGrace { + return c.ensureFresh(ctx, rp, repoURL) + } + return nil +} + // Hit reports whether the repository was already cached when it was opened. func (l *Lease) Hit() bool { return l.hit From 3fdc53d39ac2c47bfbe73963c64f9354d75ed7b8 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 8 Nov 2026 08:02:30 +0100 Subject: [PATCH 592/633] Stop relaying for a fetch once it has ended exec.CommandContext kills only the git parent. Its git-remote-http helper kept its relay connection and went on downloading for up to http.lowSpeedTime after a client disconnect, Close or fetchTimeout. Each read it relayed called cover, which reserved the relayed total again on a reservation already settled or released, so c.reserved grew for good and could evict repositories, and the helpers outlived the fetch cap. Tie each relay route to the fetch's context and cancel it when the fetch returns, so the upstream request ends and the helper's connection is cut. Make cover refuse a reservation that has been given back. --- internal/gitcache/budget.go | 13 +++++ internal/gitcache/budget_test.go | 93 ++++++++++++++++++++++++++++++++ internal/gitcache/refresh.go | 2 +- internal/gitcache/relay.go | 27 +++++++--- internal/gitcache/relay_test.go | 4 +- 5 files changed, 128 insertions(+), 11 deletions(-) diff --git a/internal/gitcache/budget.go b/internal/gitcache/budget.go index 9d4135e..2b0c228 100644 --- a/internal/gitcache/budget.go +++ b/internal/gitcache/budget.go @@ -2,6 +2,7 @@ package gitcache import ( "context" + "errors" "sync/atomic" "time" ) @@ -29,8 +30,15 @@ type reservation struct { // cut, guarded by c.mu, marks a fetch stopped because the cache could // not make room for it. cut bool + // ended, guarded by c.mu, marks a reservation given back. A read + // relayed after that, by a git helper the fetch left behind, reserves + // nothing. + ended bool } +// errReservationEnded is returned by cover once the fetch has ended. +var errReservationEnded = errors.New("fetch has ended") + func (c *Cache) newReservation() *reservation { return &reservation{c: c} } @@ -48,6 +56,10 @@ func (r *reservation) cover(ctx context.Context, relayed int64) error { var timeout <-chan time.Time for { c.mu.Lock() + if r.ended { + c.mu.Unlock() + return errReservationEnded + } if r.cut { c.mu.Unlock() return ErrCacheFull @@ -113,6 +125,7 @@ func (r *reservation) release() { func (r *reservation) releaseLocked() { c := r.c + r.ended = true n := r.reserved.Swap(0) if r.cut { r.cut = false diff --git a/internal/gitcache/budget_test.go b/internal/gitcache/budget_test.go index 506ef2f..8ef58fa 100644 --- a/internal/gitcache/budget_test.go +++ b/internal/gitcache/budget_test.go @@ -1,10 +1,15 @@ package gitcache import ( + "bytes" "context" "crypto/rand" "errors" + "io" "net/http" + "net/http/httptest" + "net/http/httputil" + "net/url" "os" "path/filepath" "sync" @@ -216,3 +221,91 @@ func TestConcurrentClonesStayWithinMaxSize(t *testing.T) { t.Errorf("left %d entries in the temporary directory", len(entries)) } } + +// TestCoverAfterReleaseReservesNothing has a relayed read arrive after the +// fetch gave back its reservation, as one from an orphaned git helper would. +func TestCoverAfterReleaseReservesNothing(t *testing.T) { + tc := newTestCache(t, "", func(o *Options) { o.MaxSize = 100 }) + res := tc.newReservation() + if err := res.cover(t.Context(), 10); err != nil { + t.Fatal(err) + } + res.release() + ctx, cancel := context.WithCancel(t.Context()) + cancel() + _ = res.cover(ctx, 20) + if reserved, _ := tc.budget(); reserved != 0 { + t.Errorf("reserved %d after release", reserved) + } +} + +// TestCancelledFetchStopsRelaying cancels a fetch while the upstream is +// midway through sending the pack: the upstream connection is closed, and +// nothing stays reserved when the rest arrives. +func TestCancelledFetchStopsRelaying(t *testing.T) { + upstream := newFakeUpstream(t) + upstream.newRepo("repo") + upstreamURL, err := url.Parse(upstream.server.URL) + if err != nil { + t.Fatal(err) + } + arrived := make(chan struct{}) + release := make(chan struct{}) + result := make(chan string, 1) + forward := httputil.NewSingleHostReverseProxy(upstreamURL) + front := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, _ := io.ReadAll(r.Body) + r.Body = io.NopCloser(bytes.NewReader(body)) + if r.Method != http.MethodPost || !bytes.Contains(body, []byte("command=fetch")) { + forward.ServeHTTP(w, r) + return + } + rec := httptest.NewRecorder() + forward.ServeHTTP(rec, r.Clone(context.Background())) + pack := rec.Body.Bytes() + w.Header().Set("Content-Type", rec.Header().Get("Content-Type")) + _, _ = w.Write(pack[:len(pack)/2]) + w.(http.Flusher).Flush() + close(arrived) + select { + case <-r.Context().Done(): + result <- "closed" + case <-release: + _, _ = w.Write(pack[len(pack)/2:]) + w.(http.Flusher).Flush() + result <- "sent" + } + })) + defer front.Close() + defer close(release) + tc := newTestCache(t, front.URL, func(o *Options) { o.MaxSize = 1 << 30 }) + + dir := t.TempDir() + if _, err := tc.run(t.Context(), dir, "init", "--quiet", "--bare"); err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(t.Context()) + res := tc.newReservation() + done := make(chan error, 1) + go func() { done <- tc.fetch(ctx, dir, front.URL+"/repo", 0, res, nil, branchesAndTags...) }() + <-arrived + waitFor(t, func() bool { reserved, _ := tc.budget(); return reserved > 0 }) + cancel() + if err := <-done; err == nil { + t.Error("cancelled fetch succeeded") + } + res.release() + + select { + case <-result: + case <-time.After(10 * time.Second): + t.Error("upstream connection still open after the fetch was cancelled") + release <- struct{}{} + <-result + } + // Give a relay still running the time to read what the upstream sent. + time.Sleep(200 * time.Millisecond) + if reserved, _ := tc.budget(); reserved != 0 { + t.Errorf("reserved %d after the fetch ended", reserved) + } +} diff --git a/internal/gitcache/refresh.go b/internal/gitcache/refresh.go index 352eb2b..d2f8ee4 100644 --- a/internal/gitcache/refresh.go +++ b/internal/gitcache/refresh.go @@ -305,7 +305,7 @@ func (c *Cache) fetch(ctx context.Context, dir, repoURL string, limit int64, res return ctx.Err() } - relayURL, route, unregister := c.relay.register(repoURL, limit, res) + relayURL, route, unregister := c.relay.register(ctx, repoURL, limit, res) defer unregister() args := append([]string{"fetch", "--quiet", "--no-write-fetch-head"}, flags...) _, err := c.run(ctx, dir, append(append(args, relayURL), refspecs...)...) diff --git a/internal/gitcache/relay.go b/internal/gitcache/relay.go index 28f87f9..ed42001 100644 --- a/internal/gitcache/relay.go +++ b/internal/gitcache/relay.go @@ -39,6 +39,9 @@ type relay struct { type relayRoute struct { repoURL string + // ctx ends when the fetch does, which stops its relayed requests, so a + // git helper left behind by a killed git does not keep downloading. + ctx context.Context // limit caps the response bytes the route relays, 0 for no cap, and // budget, when set, holds space within MaxSize for them. exceeded and // full record which of them stopped the route. @@ -79,13 +82,15 @@ func (rl *relay) close() error { return rl.server.Close() } -// register adds a route to repoURL and returns the URL git fetches from and -// a function that removes the route. -func (rl *relay) register(repoURL string, limit int64, budget *reservation) (string, *relayRoute, func()) { +// register adds a route to repoURL for a fetch running with ctx and returns +// the URL git fetches from and a function that removes the route and stops +// its requests. +func (rl *relay) register(ctx context.Context, repoURL string, limit int64, budget *reservation) (string, *relayRoute, func()) { var b [relayIDBytes]byte _, _ = rand.Read(b[:]) id := hex.EncodeToString(b[:]) - route := &relayRoute{repoURL: repoURL, limit: limit, budget: budget} + ctx, cancel := context.WithCancel(ctx) + route := &relayRoute{repoURL: repoURL, ctx: ctx, limit: limit, budget: budget} rl.mu.Lock() rl.routes[id] = route @@ -95,6 +100,7 @@ func (rl *relay) register(repoURL string, limit int64, budget *reservation) (str rl.mu.Lock() delete(rl.routes, id) rl.mu.Unlock() + cancel() } } @@ -119,7 +125,10 @@ func (rl *relay) ServeHTTP(w http.ResponseWriter, r *http.Request) { return } - out, err := http.NewRequestWithContext(r.Context(), r.Method, target, r.Body) + ctx, cancel := context.WithCancel(r.Context()) + defer cancel() + defer context.AfterFunc(route.ctx, cancel)() + out, err := http.NewRequestWithContext(ctx, r.Method, target, r.Body) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return @@ -133,7 +142,9 @@ func (rl *relay) ServeHTTP(w http.ResponseWriter, r *http.Request) { resp, err := rl.client.Do(out) if err != nil { - rl.logger.Warn("git relay: upstream request failed", "upstream", route.repoURL, "error", err) + if ctx.Err() == nil { + rl.logger.Warn("git relay: upstream request failed", "upstream", route.repoURL, "error", err) + } http.Error(w, "upstream request failed", http.StatusBadGateway) return } @@ -151,12 +162,12 @@ func (rl *relay) ServeHTTP(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", ct) } w.WriteHeader(resp.StatusCode) - if _, err := io.Copy(w, &routeReader{ctx: r.Context(), r: resp.Body, route: route}); err != nil { + if _, err := io.Copy(w, &routeReader{ctx: ctx, r: resp.Body, route: route}); err != nil { if r.Context().Err() != nil { // git went away; there is no one to tell. return } - if !route.exceeded.Load() && !route.full.Load() { + if ctx.Err() == nil && !route.exceeded.Load() && !route.full.Load() { rl.logger.Warn("git relay: copying upstream response", "upstream", route.repoURL, "error", err) } // Cut the connection so git fails rather than taking a response diff --git a/internal/gitcache/relay_test.go b/internal/gitcache/relay_test.go index ea1d063..309e727 100644 --- a/internal/gitcache/relay_test.go +++ b/internal/gitcache/relay_test.go @@ -28,7 +28,7 @@ func TestRelayOnlyForwardsFetchRequests(t *testing.T) { })) defer upstream.Close() rl := newTestRelay(t) - base, _, unregister := rl.register(upstream.URL+"/repo", 0, nil) + base, _, unregister := rl.register(t.Context(), upstream.URL+"/repo", 0, nil) refused := []struct{ method, url string }{ {http.MethodGet, rl.base + "/unknown/info/refs?service=git-upload-pack"}, @@ -79,7 +79,7 @@ func TestRelayStopsAtLimit(t *testing.T) { t.Fatal(err) } t.Cleanup(func() { _ = rl.close() }) - base, route, unregister := rl.register(upstream.URL+"/repo", limit, nil) + base, route, unregister := rl.register(t.Context(), upstream.URL+"/repo", limit, nil) defer unregister() resp, err := http.Post(base+"/git-upload-pack", requestType, strings.NewReader(pktFlush)) From 4f7a8b1b089fc5abd5ff4b03ea63d2bac8378ce9 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 8 Nov 2026 08:01:11 +0100 Subject: [PATCH 593/633] Read Conda timestamps in seconds or milliseconds as conda does The Conda handler divided every repodata timestamp by 1000, assuming milliseconds. Some entries (a few in conda-forge noarch) use seconds, so their publish time came out in 1970 and they were never held back by cooldown. Read a timestamp as milliseconds only when it is above 253402300799, as conda does, in one helper used by the repodata filter, the shard filter and the download index. --- docs/configuration.md | 2 +- internal/handler/conda.go | 22 ++++++++++++++----- internal/handler/conda_test.go | 40 ++++++++++++++++++++++++++++++++++ 3 files changed, 57 insertions(+), 7 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 93de410..e880230 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1232,7 +1232,7 @@ Maven caveats: Helm uses each release's `created` time from the repository's `index.yaml`. Releases in cooldown are left out of the index the proxy serves, and their archives aren't served; a release without `created` is let through. Charts pulled over OCI have no publish time and aren't held back. -Cargo, RubyGems, Hex, pub.dev, Conda and Helm check artifact downloads against cooldown, including files cached before the policy changed. When a publish-time lookup fails, an already-cached file is still served; an uncached download fails with 502, or 503 with `Retry-After` when the rate-limit gate holds the request. Conda checks timestamps per file because builds of one version can have different publish times. +Cargo, RubyGems, Hex, pub.dev, Conda and Helm check artifact downloads against cooldown, including files cached before the policy changed. When a publish-time lookup fails, an already-cached file is still served; an uncached download fails with 502, or 503 with `Retry-After` when the rate-limit gate holds the request. Conda checks timestamps per file because builds of one version can have different publish times. As in conda, a repodata `timestamp` is read as milliseconds when it is above 253402300799 (the end of year 9999 in seconds) and as seconds otherwise, since some older entries use seconds. Conda repodata and Helm indexes are parsed once per cached document for download checks. With `cache_metadata` off, the parsed index is kept for one minute, including the fetch, so an install can reuse it across files. Conda streams repodata when its compressed copy exceeds `metadata_max_size`, under the same decompressed size bound as the metadata filter. diff --git a/internal/handler/conda.go b/internal/handler/conda.go index 1c14980..f593820 100644 --- a/internal/handler/conda.go +++ b/internal/handler/conda.go @@ -432,8 +432,18 @@ func (c *countingWriter) Write(p []byte) (int, error) { return n, err } -// condaTimestampDivisor converts Conda's millisecond timestamps to seconds. -const condaTimestampDivisor = 1000 +// condaMaxSecondsTimestamp is the last second of year 9999. Like conda, a +// larger repodata timestamp is read as milliseconds and a smaller one as +// seconds, which some older entries use. +const condaMaxSecondsTimestamp = 253402300799 + +// condaTime returns the time a repodata timestamp names. +func condaTime(timestamp float64) time.Time { + if timestamp > condaMaxSecondsTimestamp { + return time.UnixMilli(int64(timestamp)).Truncate(time.Second) + } + return time.Unix(int64(timestamp), 0) +} const ( repodataWriteBuffer = 64 << 10 @@ -503,7 +513,7 @@ func (f *repodataFilter) inCooldown(filename string, raw []byte) bool { if entry.Name == "" || entry.Timestamp == 0 { return false } - publishedAt := time.Unix(int64(entry.Timestamp)/condaTimestampDivisor, 0) + publishedAt := condaTime(entry.Timestamp) decision := f.h.proxy.cooldown().Evaluate("conda", canonicalPackagePURL("conda", entry.Name), publishedAt, f.now) if decision.Allowed { return false @@ -520,14 +530,14 @@ func (f *repodataFilter) inCooldown(filename string, raw []byte) bool { return true } -// recordInCooldown reports whether a package record with a millisecond +// recordInCooldown reports whether a package record with a repodata // timestamp was published too recently. Records without a name or // timestamp are kept. func (h *CondaHandler) recordInCooldown(filename, name, version string, timestamp float64) bool { if name == "" || timestamp == 0 { return false } - publishedAt := time.Unix(int64(timestamp)/condaTimestampDivisor, 0) + publishedAt := condaTime(timestamp) if h.proxy.cooldown().IsAllowed("conda", canonicalPackagePURL("conda", name), publishedAt) { return false } @@ -620,7 +630,7 @@ func readRepodataIndex(r io.Reader) (map[string]time.Time, error) { } var e repodataEntry if json.Unmarshal(raw, &e) == nil && e.Timestamp != 0 { - index[entry.Key] = time.Unix(int64(e.Timestamp)/condaTimestampDivisor, 0) + index[entry.Key] = condaTime(e.Timestamp) } return nil }) diff --git a/internal/handler/conda_test.go b/internal/handler/conda_test.go index 0c4c52a..63e518e 100644 --- a/internal/handler/conda_test.go +++ b/internal/handler/conda_test.go @@ -3,9 +3,13 @@ package handler import ( "bytes" "encoding/json" + "fmt" "log/slog" + "maps" "net/http" "net/http/httptest" + "slices" + "strings" "testing" "time" @@ -133,6 +137,42 @@ func TestCondaCooldownFiltering(t *testing.T) { } } +// TestCondaCooldownSecondTimestamps checks that a timestamp in seconds, +// which some repodata entries carry, is read as seconds, as conda does, in +// both the filter and the download index. +func TestCondaCooldownSecondTimestamps(t *testing.T) { + recent := time.Now().Add(-time.Hour) + body := fmt.Sprintf(`{"packages":{"seconds-1.0-0.tar.bz2":{"name":"seconds","version":"1.0","timestamp":%d},`+ + `"millis-1.0-0.tar.bz2":{"name":"millis","version":"1.0","timestamp":%d}}}`, recent.Unix(), recent.UnixMilli()) + + proxy := testProxy() + proxy.Cooldown = &cooldown.Config{Default: "3d"} + h := &CondaHandler{proxy: proxy, proxyURL: "http://localhost:8080"} + filtered, err := filterRepodataBytes(h, []byte(body)) + if err != nil { + t.Fatal(err) + } + var result struct { + Packages map[string]json.RawMessage `json:"packages"` + } + if err := json.Unmarshal(filtered, &result); err != nil { + t.Fatal(err) + } + if len(result.Packages) != 0 { + t.Errorf("packages = %v, want both left out", slices.Sorted(maps.Keys(result.Packages))) + } + + index, err := readRepodataIndex(strings.NewReader(body)) + if err != nil { + t.Fatal(err) + } + for filename, published := range index { + if published.Unix() != recent.Unix() { + t.Errorf("%s published at %v, want %v", filename, published, recent) + } + } +} + func TestCondaCooldownFilteringWithPackageOverride(t *testing.T) { now := time.Now() recentTimestamp := float64(now.Add(-2 * time.Hour).UnixMilli()) From a288afab3283c61277841ae8ce39abe305eef39e Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 8 Nov 2026 08:04:09 +0100 Subject: [PATCH 594/633] Match denylist entries for distro packages as handlers name them The denylist kept the PURL namespace, but the Debian and RPM handlers build pkg:deb/{name} and pkg:rpm/{name} without a vendor, while Alpine and Arch build pkg:apk/alpine/{name} and pkg:alpm/arch/{name} with one. So standard entries such as pkg:deb/debian/curl@... or pkg:rpm/fedora/curl@..., and pkg:apk/busybox@... or pkg:alpm/glibc@..., never matched. Drop the namespace of distro types on both sides, as retention and the inventory already do, using packageurl's list of those types. Debian and RPM handlers read versions from filenames, which leave out the epoch, so an entry such as pkg:deb/foo@1:2.3-1 never matched foo_2.3-1_amd64.deb. An entry with an epoch now also covers the version without it. --- docs/configuration.md | 7 ++++++ internal/denylist/denylist.go | 16 +++++++++++++- internal/denylist/denylist_test.go | 35 ++++++++++++++++++++++++++++++ internal/packageurl/handler.go | 6 +++++ 4 files changed, 63 insertions(+), 1 deletion(-) diff --git a/docs/configuration.md b/docs/configuration.md index e880230..5acbab9 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1270,6 +1270,13 @@ Names and versions are compared the way each ecosystem treats them: - PyPI names ignore case, and runs of `-`, `_` and `.` are equivalent (PEP 503), so `pkg:pypi/zope.interface@6.0` also covers `zope_interface-6.0-*.whl`. - NuGet ids and versions ignore case. +- Debian, RPM, Alpine, Arch and FreeBSD entries ignore the distribution + namespace: `pkg:deb/debian/curl@8.5.0-2` and `pkg:deb/curl@8.5.0-2` are the + same entry. +- Debian and RPM filenames leave out the epoch, so an entry with one also + covers the version without it: `pkg:deb/foo@1:2.3-1` denies + `foo_2.3-1_amd64.deb`. An entry without an epoch only matches a version + without one. - A RubyGems entry also covers that version's platform gems: `pkg:gem/nokogiri@1.16.0` denies `nokogiri-1.16.0-x86_64-linux.gem`. - Composer versions ignore a leading `v`, which Packagist keeps from the tag: diff --git a/internal/denylist/denylist.go b/internal/denylist/denylist.go index 76f2de5..eea8ee3 100644 --- a/internal/denylist/denylist.go +++ b/internal/denylist/denylist.go @@ -40,6 +40,13 @@ func New(packages []string) (*Policy, error) { p.packages[key] = make(map[string]bool) } p.packages[key][id.version] = true + // Debian and RPM filenames leave the epoch out, so an entry with + // one also matches the version without it. + if pkg.Type == "deb" || pkg.Type == "rpm" { + if _, version, ok := strings.Cut(id.version, ":"); ok && version != "" { + p.packages[key][version] = true + } + } } return p, nil } @@ -128,6 +135,9 @@ func genericSwiftRegistry(pkg *purl.PURL) (scope, name string, ok bool) { // identify compares PURLs in purl's comparison form. The generic // swift-registry spelling is the proxy's own, so it is mapped here first. +// The namespace of a distro type names the distribution, which handlers +// include for some types and not others, so it is dropped as retention and +// the inventory do. func identify(pkg *purl.PURL) identity { if scope, name, ok := genericSwiftRegistry(pkg); ok { pkg = purl.New("swift", scope, name, pkg.Version, nil) @@ -138,6 +148,10 @@ func identify(pkg *purl.PURL) identity { // An entry without a channel matches every channel. channel = c.Qualifier("channel") } - key := purl.New(c.Type, c.Namespace, c.Name, "", nil).String() + namespace := c.Namespace + if packageurl.DistroType(c.Type) { + namespace = "" + } + key := purl.New(c.Type, namespace, c.Name, "", nil).String() return identity{key: packageurl.PolicyPackageKey(key), channel: channel, version: c.Version} } diff --git a/internal/denylist/denylist_test.go b/internal/denylist/denylist_test.go index a110608..03edc99 100644 --- a/internal/denylist/denylist_test.go +++ b/internal/denylist/denylist_test.go @@ -90,3 +90,38 @@ func TestEcosystemSpellings(t *testing.T) { t.Error("pypi versions not found under another spelling") } } + +// TestDistroNamespaces checks that the distribution namespace of a distro +// package is ignored on both sides, as handlers build some of these PURLs +// without one and some with one. +func TestDistroNamespaces(t *testing.T) { + p, err := New([]string{"pkg:deb/debian/curl@8.5.0-2", "pkg:rpm/fedora/curl@8.6.0-1.fc40", "pkg:apk/busybox@1.36.1-r15", "pkg:alpm/glibc@2.39-1"}) + if err != nil { + t.Fatal(err) + } + for _, value := range []string{"pkg:deb/curl@8.5.0-2", "pkg:rpm/curl@8.6.0-1.fc40", "pkg:apk/alpine/busybox@1.36.1-r15", "pkg:alpm/arch/glibc@2.39-1"} { + if !p.Denied(value) { + t.Errorf("not denied: %s", value) + } + } + if p.Denied("pkg:deb/curl@8.5.0-3") { + t.Error("other version denied") + } +} + +// TestDistroEpochs checks that a deb or rpm entry with an epoch matches the +// version as it appears in a filename, which leaves the epoch out. +func TestDistroEpochs(t *testing.T) { + p, err := New([]string{"pkg:deb/foo@1:2.3-1", "pkg:rpm/bar@2:1.0-1.fc40"}) + if err != nil { + t.Fatal(err) + } + for _, value := range []string{"pkg:deb/foo@2.3-1", "pkg:deb/foo@1:2.3-1", "pkg:rpm/bar@1.0-1.fc40"} { + if !p.Denied(value) { + t.Errorf("not denied: %s", value) + } + } + if p.Denied("pkg:deb/foo@2.3-2") { + t.Error("other version denied") + } +} diff --git a/internal/packageurl/handler.go b/internal/packageurl/handler.go index ee3005e..5edcf80 100644 --- a/internal/packageurl/handler.go +++ b/internal/packageurl/handler.go @@ -8,6 +8,12 @@ import ( // ("alpine", "debian"), not part of the package name the handlers store. var distroTypes = map[string]bool{"alpm": true, "apk": true, "bsd": true, "deb": true, "rpm": true} +// DistroType reports whether a PURL type's namespace names the distribution, +// so that handlers ignore it. +func DistroType(purlType string) bool { + return distroTypes[purlType] +} + // HandlerEcosystem returns the ecosystem name the proxy's handlers store // packages of a PURL type under. purl names gem and composer after their // registries, rubygems and packagist; the handlers use the type. From 5c95bd10532dfa178695320694755caf7db4d520 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 8 Nov 2026 08:50:55 +0100 Subject: [PATCH 595/633] Accept git as a scanner ecosystem Git LFS objects are scanned under the git key, which the new ecosystem check rejected, so a scanner restricted to ecosystems: [git] would have stopped the proxy from starting. --- docs/configuration.md | 2 +- internal/config/config.go | 5 +++-- internal/config/config_test.go | 4 ++-- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 8baac0f..ba9bbd3 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1400,7 +1400,7 @@ scanning: | `scanning.scanners[].name` | - | Identifies this scanner in logs and metrics | | `scanning.scanners[].url` | - | Endpoint the proxy POSTs scan notifications to | | `scanning.scanners[].mode` | - | `block` (default) or `monitor` | -| `scanning.scanners[].ecosystems` | - | Restricts this scanner to specific ecosystems (e.g. `npm`, `pypi`). Empty means all ecosystems. Valid names: `alpine`, `arch`, `cargo`, `chef`, `composer`, `conan`, `conda`, `cran`, `deb`, `freebsd`, `gem`, `generic`, `golang`, `helm`, `hex`, `julia`, `maven`, `npm`, `nuget`, `oci`, `pub`, `pypi`, `rpm`, `swift`. The aliases `go`, `docker`, `rubygems`, `packagist`, `debian` and `apk` are accepted too; any other name stops the proxy at startup | +| `scanning.scanners[].ecosystems` | - | Restricts this scanner to specific ecosystems (e.g. `npm`, `pypi`). Empty means all ecosystems. Valid names: `alpine`, `arch`, `cargo`, `chef`, `composer`, `conan`, `conda`, `cran`, `deb`, `freebsd`, `gem`, `generic`, `git` (Git LFS objects), `golang`, `helm`, `hex`, `julia`, `maven`, `npm`, `nuget`, `oci`, `pub`, `pypi`, `rpm`, `swift`. The aliases `go`, `docker`, `rubygems`, `packagist`, `debian` and `apk` are accepted too; any other name stops the proxy at startup | | `scanning.scanners[].headers` | - | Extra HTTP headers sent with every scan request (e.g. for authenticating to the scanner service). Values support `${VAR_NAME}` expansion | ### How caching defers to a scan verdict diff --git a/internal/config/config.go b/internal/config/config.go index d9d9b29..c1a2b08 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -449,9 +449,10 @@ func (s *ScannerConfig) Validate() error { } // scannerEcosystems are the ecosystem keys handlers pass to scanners: the -// retention keys plus the handlers retention does not cover. +// retention keys plus the handlers retention does not cover, and git for +// Git LFS objects. var scannerEcosystems = func() []string { - keys := append(slices.Clone(retention.KnownKeys), "arch", "chef") + keys := append(slices.Clone(retention.KnownKeys), "arch", "chef", "git") slices.Sort(keys) return keys }() diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 2139b0e..eb14e6b 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -2211,13 +2211,13 @@ func TestScannerEcosystems(t *testing.T) { SigningKey: "s3cret", Scanners: []ScannerConfig{{ Name: "clamav", URL: "http://scanner.invalid/scan", - Ecosystems: []string{"go", "Docker", "rubygems", "npm", "arch"}, + Ecosystems: []string{"go", "Docker", "rubygems", "npm", "arch", "git"}, }}, } if err := cfg.Validate(); err != nil { t.Fatalf("Validate() error: %v", err) } - want := []string{"golang", "oci", "gem", "npm", "arch"} + want := []string{"golang", "oci", "gem", "npm", "arch", "git"} if got := cfg.Scanners[0].Ecosystems; !slices.Equal(got, want) { t.Errorf("Ecosystems = %v, want %v", got, want) } From 61947c9c8ccdf33aae67e22cc964cc8e258c5ac7 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 8 Nov 2026 08:51:39 +0100 Subject: [PATCH 596/633] Treat a null Composer notification-url as absent Composer reads "notification-url": null as missing and reports installs to notify-batch, so a pushed manifest with null now gets an empty one too. The docs say a pushed manifest's own notification-url is kept. --- docs/configuration.md | 2 +- internal/handler/composer_hosted.go | 3 ++- internal/handler/composer_hosted_test.go | 3 +++ 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 3e7abc7..8603e1c 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1643,7 +1643,7 @@ Pushed versions are listed in the package metadata clients resolve against, with - Pushed versions are added next to the upstream ones and are never held back by cooldown. npm `dist-tags` stay as the upstream has them, also in `npm dist-tag ls`. - A package the upstream doesn't know is served from its pushed versions alone. Its npm `latest` tag is the highest pushed version. - Pushed Composer dev versions (`dev-*`, `*-dev`) are listed in the `~dev` metadata file. -- Pushed Composer versions carry an empty `notification-url`, as routed packages do, so Composer does not report their installs to the default upstream's `notify-batch` endpoint. +- Pushed Composer versions carry an empty `notification-url`, as routed packages do, so Composer does not report their installs to the default upstream's `notify-batch` endpoint. A pushed `composer.json` that sets its own `notification-url` keeps it, and Composer then reports installs there. - npm entries carry the archive's sha512 as `dist.integrity`. Composer entries carry its sha256 as `dist.reference` and leave `shasum` empty. Metadata of packages with pushed versions is rewritten on every request instead of being cached, so a push or delete shows up on the next request. diff --git a/internal/handler/composer_hosted.go b/internal/handler/composer_hosted.go index 3c42a6a..3cab559 100644 --- a/internal/handler/composer_hosted.go +++ b/internal/handler/composer_hosted.go @@ -45,7 +45,8 @@ func (h *ComposerHandler) composerLocalEntries(packageName string, dev bool) ([] manifest["time"] = lv.PushedAt.Format(time.RFC3339) // As for routed packages, keep Composer from reporting installs of // a private version to the default repository's notify-batch. - if _, ok := manifest["notification-url"]; !ok { + // Composer reads a null notification-url as absent too. + if url, ok := manifest["notification-url"]; !ok || url == nil { manifest["notification-url"] = "" } manifest["dist"] = map[string]string{ diff --git a/internal/handler/composer_hosted_test.go b/internal/handler/composer_hosted_test.go index 6c439dc..5b8a687 100644 --- a/internal/handler/composer_hosted_test.go +++ b/internal/handler/composer_hosted_test.go @@ -250,11 +250,14 @@ func TestComposerPushedVersionsDisableDefaultNotification(t *testing.T) { f.push(t, "monolog/monolog", "3.6.0", `{"name":"monolog/monolog"}`) f.push(t, "acme/private", "1.0.0", `{"name":"acme/private"}`) f.push(t, "other/private", "1.0.0", `{"name":"other/private"}`) + // Composer reads a null notification-url as absent. + f.push(t, "acme/nulled", "1.0.0", `{"name":"acme/nulled","notification-url":null}`) for _, tc := range []struct{ path, name, version string }{ {"/p2/monolog/monolog.json", "monolog/monolog", "3.6.0"}, {"/p2/acme/private.json", "acme/private", "1.0.0"}, {"/p2/other/private.json", "other/private", "1.0.0"}, + {"/p2/acme/nulled.json", "acme/nulled", "1.0.0"}, } { v := f.versions(t, tc.path, tc.name)[tc.version] if got, ok := v["notification-url"]; !ok || got != "" { From 8f281f1dfa6335f080361b3c00f219cf30875ef3 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 8 Nov 2026 08:55:02 +0100 Subject: [PATCH 597/633] Refuse info/refs while an upstream refusal stands A check that ran past refresh_wait or failed served the cached copy, also right after the upstream had refused the repository. With protocol v0 the refs are in the info/refs response itself, so they were advertised. The stored refusal now refuses those too, until a check succeeds. --- internal/gitcache/cache_test.go | 22 ++++++++++++++++++++++ internal/gitcache/serve.go | 19 +++++++++++++++---- 2 files changed, 37 insertions(+), 4 deletions(-) diff --git a/internal/gitcache/cache_test.go b/internal/gitcache/cache_test.go index 5f9d475..fc5abef 100644 --- a/internal/gitcache/cache_test.go +++ b/internal/gitcache/cache_test.go @@ -875,3 +875,25 @@ func TestCloneChecksUpstreamOnce(t *testing.T) { t.Errorf("clone made %d upstream check requests, want %d", got, perCheck) } } + +// TestInfoRefsRefusedWhileRefusalStands checks that once the upstream +// refused a repository, a later check that runs past RefreshWait, which +// would otherwise serve the cached copy, doesn't advertise its refs. +func TestInfoRefsRefusedWhileRefusalStands(t *testing.T) { + upstream := newFakeUpstream(t) + upstream.newRepo("repo") + tc := newTestCache(t, upstream.server.URL, func(o *Options) { o.RefreshWait = 100 * time.Millisecond }) + tc.clone(t, "repo") + + upstream.status.Store(http.StatusForbidden) + if out, err := tryGit(t, "", "ls-remote", tc.url("repo")); err == nil { + t.Fatalf("served the cached copy after 403:\n%s", out) + } + upstream.status.Store(0) + release := upstream.holdRequests() + defer release() + // Protocol v0 advertises the refs in the info/refs response itself. + if out, err := tryGit(t, "", "-c", "protocol.version=0", "ls-remote", tc.url("repo")); err == nil { + t.Errorf("advertised refs while the refusal stands:\n%s", out) + } +} diff --git a/internal/gitcache/serve.go b/internal/gitcache/serve.go index 68c891a..29f54f8 100644 --- a/internal/gitcache/serve.go +++ b/internal/gitcache/serve.go @@ -63,6 +63,12 @@ func (c *Cache) Open(ctx context.Context, key, repoURL string, opts OpenOptions) switch { case opts.Refresh || !hit: err = c.ensureFresh(ctx, rp, repoURL) + if err == nil && hit && !opts.NoCheck { + // A refresh still running or failed serves the cached copy, + // but not one the upstream refused since and no check has + // cleared. + err = c.refusal(rp) + } case !opts.NoCheck: err = c.checkServable(ctx, rp, repoURL) } @@ -81,10 +87,7 @@ const uploadPackGrace = time.Minute // checkServable refuses a cached repository the last refresh found gone or // refused upstream, and refreshes one not checked recently first. func (c *Cache) checkServable(ctx context.Context, rp *repo, repoURL string) error { - c.mu.Lock() - err := rp.refused - c.mu.Unlock() - if err != nil { + if err := c.refusal(rp); err != nil { return err } if time.Since(time.Unix(0, rp.lastChecked.Load())) > c.opts.StaleMaxAge+uploadPackGrace { @@ -93,6 +96,14 @@ func (c *Cache) checkServable(ctx context.Context, rp *repo, repoURL string) err return nil } +// refusal returns the error of the last refresh that found the repository +// gone or refused upstream, until a refresh succeeds. +func (c *Cache) refusal(rp *repo) error { + c.mu.Lock() + defer c.mu.Unlock() + return rp.refused +} + // Hit reports whether the repository was already cached when it was opened. func (l *Lease) Hit() bool { return l.hit From fc0e94b080fe4b6f2bfec42800f7b0eb9995f159 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 8 Nov 2026 08:56:20 +0100 Subject: [PATCH 598/633] Remove empty directories below a fetch directory on delete APK and Arch packages, and now RPM ones, have filenames with directories in them, which put the object below its fetch directory. Delete only removed the object's own directory when it was the fetch directory, so those directories stayed behind. It now removes them up to and including the fetch directory while they are empty. --- internal/storage/blob.go | 23 +++++++++++++++++++--- internal/storage/blob_test.go | 36 +++++++++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+), 3 deletions(-) diff --git a/internal/storage/blob.go b/internal/storage/blob.go index 81af78f..a999ff5 100644 --- a/internal/storage/blob.go +++ b/internal/storage/blob.go @@ -291,13 +291,30 @@ func (b *Blob) Delete(ctx context.Context, path string) error { return fmt.Errorf("deleting object: %w", err) } if p := b.localPath(path); p != "" { - if dir := filepath.Dir(p); dir != b.fileRoot && isFetchDir(filepath.Base(dir)) { - _ = os.Remove(dir) // fails, harmlessly, while the directory holds anything - } + b.removeEmptyFetchDirs(filepath.Dir(p)) } return nil } +// removeEmptyFetchDirs removes dir and its parents while they are empty, up +// to and including the fetch directory they are in, for an object whose +// filename has directories of its own. Outside a fetch directory, as in the +// layout before them, nothing is removed. +func (b *Blob) removeEmptyFetchDirs(dir string) { + var dirs []string + for d := dir; d != b.fileRoot && d != filepath.Dir(d); d = filepath.Dir(d) { + dirs = append(dirs, d) + if isFetchDir(filepath.Base(d)) { + for _, d := range dirs { + if os.Remove(d) != nil { + return // fails, harmlessly, while the directory holds anything + } + } + return + } + } +} + func (b *Blob) SignedURL(ctx context.Context, path string, expiry time.Duration) (string, error) { url, err := b.bucket.SignedURL(ctx, path, &blob.SignedURLOptions{ Method: http.MethodGet, diff --git a/internal/storage/blob_test.go b/internal/storage/blob_test.go index 1a764de..faaace8 100644 --- a/internal/storage/blob_test.go +++ b/internal/storage/blob_test.go @@ -180,6 +180,42 @@ func TestBlobDeleteRemovesEmptyFetchDirectory(t *testing.T) { assertReadsBack(t, b, kept, "content") } +// A filename with directories in it, as APK, Arch and RPM packages have, puts +// the object below its fetch directory. Delete removes those directories +// too once empty, up to and including the fetch directory. +func TestBlobDeleteRemovesNestedFetchDirectories(t *testing.T) { + dir := t.TempDir() + b := openFileBlob(t, dir) + ctx := context.Background() + const ( + emptied = "rpm/bash/5.1/0123456789abcdef/rocky/9/BaseOS/bash.rpm" + shared = "rpm/bash/5.1/fedcba9876543210/alma/9/BaseOS/bash.rpm" + kept = "rpm/bash/5.1/fedcba9876543210/alma/9/AppStream/bash.rpm" + ) + for _, key := range []string{emptied, shared, kept} { + if _, _, err := b.Store(ctx, key, strings.NewReader("content")); err != nil { + t.Fatalf("Store(%q): %v", key, err) + } + } + for _, key := range []string{emptied, shared} { + if err := b.Delete(ctx, key); err != nil { + t.Fatalf("Delete(%q): %v", key, err) + } + } + + version := filepath.Join(dir, "rpm", "bash", "5.1") + if _, err := os.Stat(filepath.Join(version, "0123456789abcdef")); !os.IsNotExist(err) { + t.Errorf("emptied fetch directory left behind (stat err %v)", err) + } + if _, err := os.Stat(filepath.Join(version, "fedcba9876543210", "alma", "9", "BaseOS")); !os.IsNotExist(err) { + t.Errorf("emptied directory below the fetch directory left behind (stat err %v)", err) + } + if _, err := os.Stat(version); err != nil { + t.Errorf("version directory removed: %v", err) + } + assertReadsBack(t, b, kept, "content") +} + func TestBlobDeleteNotFound(t *testing.T) { b := createTestBlob(t) ctx := context.Background() From 3b4ddb43d8a0c162d74673495e37041f70e49849 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 8 Nov 2026 10:04:58 +0100 Subject: [PATCH 599/633] Revert "Remove empty directories below a fetch directory on delete" This reverts commit fc0e94b. The cleanup stopped at the first segment that looked like a fetch id, but named Maven repositories put a hex upstream hash in the filename, so it stopped early. For objects in the old layout, with no fetch directory, the walk could remove version or package directories under a concurrent fetch. Leftover empty directories will be handled separately. --- internal/storage/blob.go | 23 +++------------------- internal/storage/blob_test.go | 36 ----------------------------------- 2 files changed, 3 insertions(+), 56 deletions(-) diff --git a/internal/storage/blob.go b/internal/storage/blob.go index a999ff5..81af78f 100644 --- a/internal/storage/blob.go +++ b/internal/storage/blob.go @@ -291,28 +291,11 @@ func (b *Blob) Delete(ctx context.Context, path string) error { return fmt.Errorf("deleting object: %w", err) } if p := b.localPath(path); p != "" { - b.removeEmptyFetchDirs(filepath.Dir(p)) - } - return nil -} - -// removeEmptyFetchDirs removes dir and its parents while they are empty, up -// to and including the fetch directory they are in, for an object whose -// filename has directories of its own. Outside a fetch directory, as in the -// layout before them, nothing is removed. -func (b *Blob) removeEmptyFetchDirs(dir string) { - var dirs []string - for d := dir; d != b.fileRoot && d != filepath.Dir(d); d = filepath.Dir(d) { - dirs = append(dirs, d) - if isFetchDir(filepath.Base(d)) { - for _, d := range dirs { - if os.Remove(d) != nil { - return // fails, harmlessly, while the directory holds anything - } - } - return + if dir := filepath.Dir(p); dir != b.fileRoot && isFetchDir(filepath.Base(dir)) { + _ = os.Remove(dir) // fails, harmlessly, while the directory holds anything } } + return nil } func (b *Blob) SignedURL(ctx context.Context, path string, expiry time.Duration) (string, error) { diff --git a/internal/storage/blob_test.go b/internal/storage/blob_test.go index faaace8..1a764de 100644 --- a/internal/storage/blob_test.go +++ b/internal/storage/blob_test.go @@ -180,42 +180,6 @@ func TestBlobDeleteRemovesEmptyFetchDirectory(t *testing.T) { assertReadsBack(t, b, kept, "content") } -// A filename with directories in it, as APK, Arch and RPM packages have, puts -// the object below its fetch directory. Delete removes those directories -// too once empty, up to and including the fetch directory. -func TestBlobDeleteRemovesNestedFetchDirectories(t *testing.T) { - dir := t.TempDir() - b := openFileBlob(t, dir) - ctx := context.Background() - const ( - emptied = "rpm/bash/5.1/0123456789abcdef/rocky/9/BaseOS/bash.rpm" - shared = "rpm/bash/5.1/fedcba9876543210/alma/9/BaseOS/bash.rpm" - kept = "rpm/bash/5.1/fedcba9876543210/alma/9/AppStream/bash.rpm" - ) - for _, key := range []string{emptied, shared, kept} { - if _, _, err := b.Store(ctx, key, strings.NewReader("content")); err != nil { - t.Fatalf("Store(%q): %v", key, err) - } - } - for _, key := range []string{emptied, shared} { - if err := b.Delete(ctx, key); err != nil { - t.Fatalf("Delete(%q): %v", key, err) - } - } - - version := filepath.Join(dir, "rpm", "bash", "5.1") - if _, err := os.Stat(filepath.Join(version, "0123456789abcdef")); !os.IsNotExist(err) { - t.Errorf("emptied fetch directory left behind (stat err %v)", err) - } - if _, err := os.Stat(filepath.Join(version, "fedcba9876543210", "alma", "9", "BaseOS")); !os.IsNotExist(err) { - t.Errorf("emptied directory below the fetch directory left behind (stat err %v)", err) - } - if _, err := os.Stat(version); err != nil { - t.Errorf("version directory removed: %v", err) - } - assertReadsBack(t, b, kept, "content") -} - func TestBlobDeleteNotFound(t *testing.T) { b := createTestBlob(t) ctx := context.Background() From de280a1694f04b7527da5da1f35a9738a7070366 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 8 Nov 2026 10:05:23 +0100 Subject: [PATCH 600/633] Don't cache Maven paths with a colon in the group or artifact The cache keys Maven artifacts as group:artifact, so org:foo/bar and org/foo:bar both came out as org:foo:bar and shared one entry. Such paths are now proxied without caching, like dotted group directories. --- internal/handler/maven.go | 8 ++++++-- internal/handler/maven_test.go | 4 ++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/internal/handler/maven.go b/internal/handler/maven.go index ca81b74..0fd81ec 100644 --- a/internal/handler/maven.go +++ b/internal/handler/maven.go @@ -279,12 +279,16 @@ func (h *MavenHandler) parsePath(urlPath string) (group, artifact, version, file groupParts := parts[:len(parts)-3] // The Maven layout splits a group at its dots, so a group directory // never holds a dot or is empty. Such a path would spell the same group - // as another and share its cache entries, so it is not parsed. + // as another and share its cache entries, so it is not parsed. Nor is a + // colon in the group or artifact, which are cached as group:artifact. for _, part := range groupParts { - if part == "" || strings.Contains(part, ".") { + if part == "" || strings.ContainsAny(part, ".:") { return "", "", "", "" } } + if strings.Contains(artifact, ":") { + return "", "", "", "" + } group = strings.Join(groupParts, ".") return group, artifact, version, filename diff --git a/internal/handler/maven_test.go b/internal/handler/maven_test.go index 3d70a32..14cb49f 100644 --- a/internal/handler/maven_test.go +++ b/internal/handler/maven_test.go @@ -36,6 +36,10 @@ func TestMavenParsePath(t *testing.T) { {"org.foo/bar/1.0/bar-1.0.jar", "", "", "", ""}, {"org/.foo/bar/1.0/bar-1.0.jar", "", "", "", ""}, {"org//foo/bar/1.0/bar-1.0.jar", "", "", "", ""}, + // Cache identity is group:artifact, so a colon in either would + // let org:foo/bar and org/foo:bar share one entry. + {"org:foo/bar/1.0/bar-1.0.jar", "", "", "", ""}, + {"org/foo:bar/1.0/bar-1.0.jar", "", "", "", ""}, } for _, tt := range tests { From 3bb7f7c30d3e647d4fc049882ed3809b54c94dea Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 8 Nov 2026 10:05:43 +0100 Subject: [PATCH 601/633] Keep _ as a wildcard in package search Escaping _ in the search pattern stopped typing_extensions from finding typing-extensions, which worked before because _ matched any single character. Escape only % and the escape character. --- internal/database/queries.go | 7 ++++--- internal/database/queries_packages_list_test.go | 7 +++++-- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/internal/database/queries.go b/internal/database/queries.go index 7176152..d7eaa6e 100644 --- a/internal/database/queries.go +++ b/internal/database/queries.go @@ -898,10 +898,11 @@ func (db *DB) SearchPackages(query string, ecosystem string, limit int, offset i // from nameSearchPattern, ignoring case on SQLite and Postgres alike. const nameSearchCondition = `LOWER(p.name) LIKE LOWER(?) ESCAPE '!'` -// nameSearchPattern is the LIKE pattern for names containing query, with -// the LIKE wildcards in query matched literally. +// nameSearchPattern is the LIKE pattern for names containing query. A % +// in query is matched literally, but _ stays a single-character wildcard +// so typing_extensions still finds typing-extensions. func nameSearchPattern(query string) string { - escaped := strings.NewReplacer("!", "!!", "%", "!%", "_", "!_").Replace(query) + escaped := strings.NewReplacer("!", "!!", "%", "!%").Replace(query) return "%" + escaped + "%" } diff --git a/internal/database/queries_packages_list_test.go b/internal/database/queries_packages_list_test.go index 704b388..5eaa402 100644 --- a/internal/database/queries_packages_list_test.go +++ b/internal/database/queries_packages_list_test.go @@ -252,11 +252,11 @@ func TestCachedPackageFiltersMatchEcosystemAliases(t *testing.T) { }) } -func TestSearchPackagesIgnoresCaseAndEscapesWildcards(t *testing.T) { +func TestSearchPackagesIgnoresCaseAndEscapesPercent(t *testing.T) { runWithBothDatabases(t, func(t *testing.T, db *DB) { seedArtifact(t, db, "pypi", "Django", "5.0.0", 100, 1) + seedArtifact(t, db, "pypi", "typing-extensions", "4.12.0", 100, 1) seedArtifact(t, db, "npm", "my_pkg", "1.0.0", 100, 1) - seedArtifact(t, db, "npm", "myxpkg", "1.0.0", 100, 1) seedArtifact(t, db, "npm", "100%-done", "1.0.0", 100, 1) seedArtifact(t, db, "npm", "1000-done", "1.0.0", 100, 1) seedArtifact(t, db, "npm", `back\slash`, "1.0.0", 100, 1) @@ -268,6 +268,9 @@ func TestSearchPackagesIgnoresCaseAndEscapesWildcards(t *testing.T) { {"django", []string{"Django"}}, {"DJANGO", []string{"Django"}}, {"my_pkg", []string{"my_pkg"}}, + // _ stays a single-character wildcard, so it matches - too. + {"typing_extensions", []string{"typing-extensions"}}, + {"typing-extensions", []string{"typing-extensions"}}, {"100%", []string{"100%-done"}}, {`k\s`, []string{`back\slash`}}, } { From ae8b2c6d2126145d3930efdf3609414a1960d5ff Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 8 Nov 2026 10:05:53 +0100 Subject: [PATCH 602/633] Accept yes, no, on and off in boolean environment variables Before the strict parser, PROXY_TRUST_FORWARDED_FOR=no meant false. With only strconv.ParseBool's forms accepted, no was ignored and a setting enabled in the config file stayed on. Accept yes/no/on/off in any case for every boolean variable, the Gradle read-only one included. --- docs/configuration.md | 2 +- internal/config/config.go | 27 ++++++++++++++++++++------- internal/config/config_test.go | 22 ++++++++++++++++++---- 3 files changed, 39 insertions(+), 12 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index ba9bbd3..8bbf5e1 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -2,7 +2,7 @@ The proxy can be configured via command line flags, environment variables, or a configuration file. Command line flags take precedence over environment variables, which take precedence over the configuration file. -Boolean environment variables accept `true`, `True`, `TRUE`, `t`, `T` or `1`, and `false`, `False`, `FALSE`, `f`, `F` or `0`. Any other value, such as `yes` or a typo, is ignored: the setting keeps its value from the configuration file or default, and the proxy logs a warning at startup. +Boolean environment variables accept `true`, `t`, `1`, `yes` or `on`, and `false`, `f`, `0`, `no` or `off`, in any case. Any other value, such as a typo, is ignored: the setting keeps its value from the configuration file or default, and the proxy logs a warning at startup. ## Configuration File diff --git a/internal/config/config.go b/internal/config/config.go index c1a2b08..a9fc9e2 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -1428,16 +1428,29 @@ func setEnvString(dst *string, key string) { } } -// setEnvBool is setEnvString for boolean fields. It accepts what -// strconv.ParseBool does; any other value leaves *dst unchanged and is -// reported via EnvWarnings. +// parseEnvBool accepts what strconv.ParseBool does plus yes/no/on/off, +// ignoring case and surrounding space. +func parseEnvBool(v string) (bool, bool) { + v = strings.ToLower(strings.TrimSpace(v)) + switch v { + case "yes", "on": + return true, true + case "no", "off": + return false, true + } + b, err := strconv.ParseBool(v) + return b, err == nil +} + +// setEnvBool is setEnvString for boolean fields, parsed via parseEnvBool. +// Any other value leaves *dst unchanged and is reported via EnvWarnings. func (c *Config) setEnvBool(dst *bool, key string) { v := os.Getenv(key) if v == "" { return } - b, err := strconv.ParseBool(strings.TrimSpace(v)) - if err != nil { + b, ok := parseEnvBool(v) + if !ok { c.envWarnings = append(c.envWarnings, fmt.Sprintf("ignoring %s=%q: not a boolean, keeping %t", key, v, *dst)) return } @@ -1452,8 +1465,8 @@ func (c *Config) setGradleReadOnlyFromEnv() { if v == "" { return } - readOnly, err := strconv.ParseBool(strings.TrimSpace(v)) - if err != nil { + readOnly, ok := parseEnvBool(v) + if !ok { c.Gradle.BuildCache.ReadOnly = true c.envWarnings = append(c.envWarnings, fmt.Sprintf("ignoring %s=%q: not a boolean, keeping the Gradle build cache read-only", key, v)) return diff --git a/internal/config/config_test.go b/internal/config/config_test.go index eb14e6b..d89e300 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -467,8 +467,9 @@ func TestLoadGradleBuildCacheReadOnlyFromEnv(t *testing.T) { {" FALSE ", true, false, 0}, {"True", false, true, 0}, {"1", false, true, 0}, - {"yes", false, true, 1}, - {"off", false, true, 1}, + {"yes", false, true, 0}, + {"off", true, false, 0}, + {"maybe", false, true, 1}, {"", false, false, 0}, } for _, tt := range tests { @@ -2172,6 +2173,16 @@ func TestCooldownShortest(t *testing.T) { } func TestLoadBoolsFromEnvParseStrictly(t *testing.T) { + t.Run("no turns off a setting from the config file", func(t *testing.T) { + t.Setenv("PROXY_TRUST_FORWARDED_FOR", "no") + cfg := Default() + cfg.TrustForwardedFor = true + cfg.LoadFromEnv() + if cfg.TrustForwardedFor { + t.Error("TrustForwardedFor = true, want false") + } + }) + tests := []struct { value string start bool @@ -2185,8 +2196,11 @@ func TestLoadBoolsFromEnvParseStrictly(t *testing.T) { {"false", true, false, 0}, {"FALSE", true, false, 0}, {"0", true, false, 0}, - {"yes", true, true, 1}, - {"on", false, false, 1}, + {"yes", false, true, 0}, + {" On ", false, true, 0}, + {"no", true, false, 0}, + {"OFF", true, false, 0}, + {"maybe", false, false, 1}, {"ture", true, true, 1}, } for _, tt := range tests { From 8233ce33399f042bf24d4a87ef1d95c5ce69b3f9 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 8 Nov 2026 10:05:58 +0100 Subject: [PATCH 603/633] Clean RPM package paths before caching The cache key for an RPM package is its request path, so a path with an empty or "." segment made a second entry for the same upstream file. The path is now cleaned before it is used as the key and the upstream URL. --- internal/handler/rpm.go | 9 +++++++- internal/handler/rpm_test.go | 42 ++++++++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 1 deletion(-) diff --git a/internal/handler/rpm.go b/internal/handler/rpm.go index 49773a7..62b2798 100644 --- a/internal/handler/rpm.go +++ b/internal/handler/rpm.go @@ -3,6 +3,7 @@ package handler import ( "fmt" "net/http" + pathpkg "path" "regexp" "strings" @@ -78,7 +79,7 @@ func (h *RPMHandler) Routes() http.Handler { switch { case strings.HasSuffix(path, ".rpm"): // Package downloads - cache these - h.handlePackageDownload(w, r, path) + h.handlePackageDownload(w, r, cleanRPMPath(path)) case strings.Contains(path, "/repodata/"): // Repository metadata h.handleMetadata(w, r, path) @@ -89,6 +90,12 @@ func (h *RPMHandler) Routes() http.Handler { })) } +// cleanRPMPath drops empty and "." segments from a relative path, so each +// package has one cache key. ".." segments are rejected before this. +func cleanRPMPath(p string) string { + return strings.TrimPrefix(pathpkg.Clean("/"+p), "/") +} + // handlePackageDownload fetches and caches .rpm packages. // Path format varies by repo structure, e.g.: // - releases/39/Everything/x86_64/os/Packages/n/nginx-1.24.0-1.fc39.x86_64.rpm diff --git a/internal/handler/rpm_test.go b/internal/handler/rpm_test.go index 87275b2..d9561e3 100644 --- a/internal/handler/rpm_test.go +++ b/internal/handler/rpm_test.go @@ -102,3 +102,45 @@ func TestRPMHandler_PackageDownloadCachesPerRepository(t *testing.T) { available.Store(false) } } + +// TestRPMHandler_PackageDownloadCleansPath checks that spellings of one +// path with empty or "." segments share one cache entry. +func TestRPMHandler_PackageDownloadCleansPath(t *testing.T) { + const want = "bash" + var hits atomic.Int32 + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/rocky/9/BaseOS/x86_64/os/Packages/b/bash-5.1.8-9.el9.x86_64.rpm" { + http.NotFound(w, r) + return + } + hits.Add(1) + _, _ = w.Write([]byte(want)) + })) + defer upstream.Close() + + proxy, _, _, _ := setupTestProxy(t) + fetcher := fetch.NewFetcher(fetch.WithHTTPClient(upstream.Client()), fetch.WithMaxRetries(0)) + proxy.Fetcher = fetcher + t.Cleanup(func() { _ = fetcher.Close() }) + srv := httptest.NewServer(NewRPMHandlerWithUpstream(proxy, testProxyURL, upstream.URL).Routes()) + defer srv.Close() + + for _, path := range []string{ + "/rocky/9/BaseOS/x86_64/os/Packages/b/bash-5.1.8-9.el9.x86_64.rpm", + "/rocky//9/BaseOS/x86_64/os/Packages/b/bash-5.1.8-9.el9.x86_64.rpm", + "/rocky/./9/BaseOS/x86_64/os/Packages/b/bash-5.1.8-9.el9.x86_64.rpm", + } { + resp, err := http.Get(srv.URL + path) + if err != nil { + t.Fatal(err) + } + body, _ := io.ReadAll(resp.Body) + _ = resp.Body.Close() + if resp.StatusCode != http.StatusOK || string(body) != want { + t.Errorf("%s: %d %q, want 200 %q", path, resp.StatusCode, body, want) + } + } + if n := hits.Load(); n != 1 { + t.Errorf("upstream fetched %d times, want 1", n) + } +} From 5c798069a18632652ca7e4d25f36112a67397bcd Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 8 Nov 2026 10:06:20 +0100 Subject: [PATCH 604/633] Treat a null notification-url as absent for routed packages Composer falls back to the default repository's notify-batch when a version's notification-url is null, which would report installs of a routed private package to Packagist. Replace null with an empty string as is already done for pushed versions. Also drop the pushed-version check's local url variable, which shadowed net/url, and test that a pushed manifest's own URL is kept. --- internal/handler/composer.go | 3 ++- internal/handler/composer_hosted.go | 2 +- internal/handler/composer_hosted_test.go | 17 ++++++++++------- internal/handler/composer_routes_test.go | 7 ++++++- 4 files changed, 19 insertions(+), 10 deletions(-) diff --git a/internal/handler/composer.go b/internal/handler/composer.go index 17a4123..6b82cff 100644 --- a/internal/handler/composer.go +++ b/internal/handler/composer.go @@ -431,8 +431,9 @@ func composerPublishTime(published []byte) (time.Time, bool) { // the package name to it. Composer only falls back to notify-batch when a // version has no notification-url of its own, and skips an empty one. Setting // it on the expanded fields keeps it set for the versions that inherit them. +// Composer reads a null notification-url as absent too. func disableDefaultNotification(fields *composerFields) { - if fields.get("notification-url") == nil { + if v := fields.get("notification-url"); v == nil || string(v) == "null" { fields.set("notification-url", []byte(`"notification-url"`), []byte(`""`)) } } diff --git a/internal/handler/composer_hosted.go b/internal/handler/composer_hosted.go index 3cab559..be8a1d6 100644 --- a/internal/handler/composer_hosted.go +++ b/internal/handler/composer_hosted.go @@ -46,7 +46,7 @@ func (h *ComposerHandler) composerLocalEntries(packageName string, dev bool) ([] // As for routed packages, keep Composer from reporting installs of // a private version to the default repository's notify-batch. // Composer reads a null notification-url as absent too. - if url, ok := manifest["notification-url"]; !ok || url == nil { + if manifest["notification-url"] == nil { manifest["notification-url"] = "" } manifest["dist"] = map[string]string{ diff --git a/internal/handler/composer_hosted_test.go b/internal/handler/composer_hosted_test.go index 5b8a687..dd792ca 100644 --- a/internal/handler/composer_hosted_test.go +++ b/internal/handler/composer_hosted_test.go @@ -252,16 +252,19 @@ func TestComposerPushedVersionsDisableDefaultNotification(t *testing.T) { f.push(t, "other/private", "1.0.0", `{"name":"other/private"}`) // Composer reads a null notification-url as absent. f.push(t, "acme/nulled", "1.0.0", `{"name":"acme/nulled","notification-url":null}`) + f.push(t, "acme/own", "1.0.0", `{"name":"acme/own","notification-url":"https://acme.example.com/notify"}`) - for _, tc := range []struct{ path, name, version string }{ - {"/p2/monolog/monolog.json", "monolog/monolog", "3.6.0"}, - {"/p2/acme/private.json", "acme/private", "1.0.0"}, - {"/p2/other/private.json", "other/private", "1.0.0"}, - {"/p2/acme/nulled.json", "acme/nulled", "1.0.0"}, + for _, tc := range []struct{ path, name, version, want string }{ + {"/p2/monolog/monolog.json", "monolog/monolog", "3.6.0", ""}, + {"/p2/acme/private.json", "acme/private", "1.0.0", ""}, + {"/p2/other/private.json", "other/private", "1.0.0", ""}, + {"/p2/acme/nulled.json", "acme/nulled", "1.0.0", ""}, + // A manifest's own URL is kept. + {"/p2/acme/own.json", "acme/own", "1.0.0", "https://acme.example.com/notify"}, } { v := f.versions(t, tc.path, tc.name)[tc.version] - if got, ok := v["notification-url"]; !ok || got != "" { - t.Errorf("%s %s notification-url = %v (set %v), want empty", tc.name, tc.version, got, ok) + if got, ok := v["notification-url"]; !ok || got != tc.want { + t.Errorf("%s %s notification-url = %v (set %v), want %q", tc.name, tc.version, got, ok, tc.want) } } } diff --git a/internal/handler/composer_routes_test.go b/internal/handler/composer_routes_test.go index 5a65440..0ebf5ac 100644 --- a/internal/handler/composer_routes_test.go +++ b/internal/handler/composer_routes_test.go @@ -235,13 +235,16 @@ func TestComposerRoutedDownloadIgnoresDefaultRepositoryArtifact(t *testing.T) { // TestComposerRoutedMinifiedMetadataDisablesNotification checks that every // version of a routed package, as Composer expands minified metadata, has an // empty notification-url: versions that inherit their fields, versions after a -// ~dev reset, and versions after one that set its own URL and then unset it. +// ~dev reset, versions after one that set its own URL and then unset it, and +// versions with a null URL. func TestComposerRoutedMinifiedMetadataDisablesNotification(t *testing.T) { minified := `{"minified":"composer/2.0","packages":{"example/library":[` + `{"name":"example/library","version":"1.0.0","dist":{"url":"https://private.example.com/a/1.0.0.zip","type":"zip"}},` + `{"version":"1.1.0","dist":{"url":"https://private.example.com/a/1.1.0.zip","type":"zip"}},` + `{"version":"1.2.0","notification-url":"https://private.example.com/notify","dist":{"url":"https://private.example.com/a/1.2.0.zip","type":"zip"}},` + `{"version":"1.3.0","notification-url":"__unset","dist":{"url":"https://private.example.com/a/1.3.0.zip","type":"zip"}},` + + `{"version":"1.4.0","notification-url":null,"dist":{"url":"https://private.example.com/a/1.4.0.zip","type":"zip"}},` + + `{"version":"1.5.0","dist":{"url":"https://private.example.com/a/1.5.0.zip","type":"zip"}},` + `"~dev",` + `{"name":"example/library","version":"dev-main","dist":{"url":"https://private.example.com/a/main.zip","type":"zip"}}]}}` public := newComposerUpstream(t, map[string]string{}, http.StatusOK) @@ -258,6 +261,8 @@ func TestComposerRoutedMinifiedMetadataDisablesNotification(t *testing.T) { "1.1.0": "", "1.2.0": "https://private.example.com/notify", // the route's own URL is kept "1.3.0": "", + "1.4.0": "", // Composer reads null as absent + "1.5.0": "", "dev-main": "", } versions := composerVersions(t, w.Body.Bytes(), "example/library") From 767e401a3cf9deae3a43653f0e7ae4d283a10033 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 8 Nov 2026 10:06:50 +0100 Subject: [PATCH 605/633] Check the upstream again on upload-pack while refused An upload-pack request returned a stored refusal without checking the upstream, so a client that never asks for info/refs, such as a protocol v2 client sending ls-refs and fetch directly, stayed refused after a passing 403 or 404. Refresh first while a refusal stands, as for a copy past stale_max_age, and refuse only if the refusal remains. --- docs/configuration.md | 7 ++++--- internal/gitcache/cache_test.go | 25 +++++++++++++++++++++++++ internal/gitcache/serve.go | 15 ++++++++------- 3 files changed, 37 insertions(+), 10 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index d384bac..37d39ae 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -903,9 +903,10 @@ or redirects is not served from the cache, also to requests that skip the ref listing and go straight to `git-upload-pack`, until a later check succeeds. Redirects are never followed, so point the upstream at a repository's current location. A `git-upload-pack` request does not check the upstream itself -unless the copy was last checked more than a minute past `stale_max_age` ago, -so the requests of a clone that follow its ref listing check nothing again; -an older copy is checked first, as for a ref listing. +unless the upstream refused the repository at the last check or the copy was +last checked more than a minute past `stale_max_age` ago, so the requests of a +clone that follow its ref listing check nothing again; otherwise the +repository is checked first, as for a ref listing. The proxy runs `git` 2.31 or newer, which must be installed (the Docker image includes it). The `git` processes never reach the network themselves: their fetches go diff --git a/internal/gitcache/cache_test.go b/internal/gitcache/cache_test.go index fc5abef..baf646a 100644 --- a/internal/gitcache/cache_test.go +++ b/internal/gitcache/cache_test.go @@ -897,3 +897,28 @@ func TestInfoRefsRefusedWhileRefusalStands(t *testing.T) { t.Errorf("advertised refs while the refusal stands:\n%s", out) } } + +// refuse has the upstream refuse "repo" with a 403 and checks that the +// cached copy is no longer served. +func (tc *testCache) refuse(t *testing.T, upstream *fakeUpstream) { + t.Helper() + upstream.status.Store(http.StatusForbidden) + if out, err := tryGit(t, "", "ls-remote", tc.url("repo")); err == nil { + t.Fatalf("served the cached copy after 403:\n%s", out) + } + upstream.status.Store(0) +} + +// TestUploadPackClearsRefusal has a client that only sends upload-pack +// requests: once the upstream allows the repository again, it is served. +func TestUploadPackClearsRefusal(t *testing.T) { + upstream := newFakeUpstream(t) + upstream.newRepo("repo") + tc := newTestCache(t, upstream.server.URL, nil) + tc.clone(t, "repo") + tc.refuse(t, upstream) + + if status := tc.lsRefs(t); status != http.StatusOK { + t.Errorf("ls-refs after the upstream allowed the repository again: status %d", status) + } +} diff --git a/internal/gitcache/serve.go b/internal/gitcache/serve.go index 29f54f8..5a9f1e0 100644 --- a/internal/gitcache/serve.go +++ b/internal/gitcache/serve.go @@ -84,16 +84,17 @@ func (c *Cache) Open(ctx context.Context, key, repoURL string, opts OpenOptions) // info/refs, so a clone checks the upstream once. const uploadPackGrace = time.Minute -// checkServable refuses a cached repository the last refresh found gone or -// refused upstream, and refreshes one not checked recently first. +// checkServable refreshes a cached repository first when the last refresh +// found it gone or refused upstream, or it was not checked recently, and +// refuses it while that refusal stands. func (c *Cache) checkServable(ctx context.Context, rp *repo, repoURL string) error { - if err := c.refusal(rp); err != nil { + if c.refusal(rp) == nil && time.Since(time.Unix(0, rp.lastChecked.Load())) <= c.opts.StaleMaxAge+uploadPackGrace { + return nil + } + if err := c.ensureFresh(ctx, rp, repoURL); err != nil { return err } - if time.Since(time.Unix(0, rp.lastChecked.Load())) > c.opts.StaleMaxAge+uploadPackGrace { - return c.ensureFresh(ctx, rp, repoURL) - } - return nil + return c.refusal(rp) } // refusal returns the error of the last refresh that found the repository From 39d6b8c7b0320f790ccfd022d5282e23558088b6 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 8 Nov 2026 10:07:24 +0100 Subject: [PATCH 606/633] Rewrite PyPI root index links under the upstream's path An upstream such as Nexus served from /repository/pypi links projects as /repository/pypi/simple//, which the rewrite missed. Match links under the configured upstream's path, in single-quoted hrefs as well as double-quoted ones, and drop the upstream's ETag, which names the body before the rewrite. --- internal/handler/pypi.go | 64 +++++++++++++++++++--------- internal/handler/pypi_test.go | 79 +++++++++++++++++++++-------------- 2 files changed, 92 insertions(+), 51 deletions(-) diff --git a/internal/handler/pypi.go b/internal/handler/pypi.go index 7dd3f12..50f9964 100644 --- a/internal/handler/pypi.go +++ b/internal/handler/pypi.go @@ -11,6 +11,7 @@ import ( "maps" "mime" "net/http" + "net/url" "regexp" "slices" "strconv" @@ -1023,17 +1024,25 @@ func (h *PyPIHandler) proxySimple(w http.ResponseWriter, r *http.Request, path s } defer func() { _ = resp.Body.Close() }() - // The HTML root index links projects as /simple//, which must - // point at the proxy's /pypi mount instead. It is large, so it is - // rewritten as it streams. + // The HTML root index links projects by absolute path, such as + // /simple//, which must point at the proxy's /pypi mount instead. + // It is large, so it is rewritten as it streams. if resp.StatusCode == http.StatusOK && resp.Header.Get("Content-Encoding") == "" && !strings.Contains(resp.Header.Get(headerContentType), "json") { + prefix := "/simple/" + if u, err := url.Parse(h.upstreamURL); err == nil { + prefix = strings.TrimSuffix(u.Path, "/") + prefix + } + to := h.proxyURL + "/pypi/simple/" resp.Body = &replacingReader{ - src: resp.Body, - from: []byte(`href="/simple/`), - to: []byte(`href="` + h.proxyURL + `/pypi/simple/`), + src: resp.Body, + replacements: []replacement{ + {from: []byte(`href="` + prefix), to: []byte(`href="` + to)}, + {from: []byte(`href='` + prefix), to: []byte(`href='` + to)}, + }, } resp.Header.Del(headerContentLength) + resp.Header.Del("ETag") // it names the upstream body resp.ContentLength = -1 } @@ -1045,15 +1054,28 @@ func (h *PyPIHandler) proxySimple(w http.ResponseWriter, r *http.Request, path s const replacingReaderBufferSize = 32 << 10 -// replacingReader replaces every occurrence of from with to in src as it is -// read. +// replacingReader makes each replacement in src as it is read. type replacingReader struct { - src io.ReadCloser - from, to []byte - in []byte // read from src, not yet scanned - out []byte // scanned, not yet returned - buf []byte - err error + src io.ReadCloser + replacements []replacement + in []byte // read from src, not yet scanned + out []byte // scanned, not yet returned + buf []byte + err error +} + +type replacement struct{ from, to []byte } + +// next finds the earliest replacement in b, returning its index or -1. +func (r *replacingReader) next(b []byte) (int, *replacement) { + at, found := -1, (*replacement)(nil) + for i := range r.replacements { + rep := &r.replacements[i] + if j := bytes.Index(b, rep.from); j >= 0 && (at < 0 || j < at) { + at, found = j, rep + } + } + return at, found } func (r *replacingReader) Read(p []byte) (int, error) { @@ -1068,21 +1090,23 @@ func (r *replacingReader) Read(p []byte) (int, error) { r.in = append(r.in, r.buf[:n]...) r.err = err // Keep a tail that could be the start of a match split across reads. - keep := len(r.from) - 1 - if err != nil { - keep = 0 + keep := 0 + if err == nil { + for _, rep := range r.replacements { + keep = max(keep, len(rep.from)-1) + } } scan := max(len(r.in)-keep, 0) var out []byte done := 0 for { - i := bytes.Index(r.in[done:], r.from) + i, rep := r.next(r.in[done:]) if i < 0 || done+i >= scan { break } out = append(out, r.in[done:done+i]...) - out = append(out, r.to...) - done += i + len(r.from) + out = append(out, rep.to...) + done += i + len(rep.from) } if done < scan { out = append(out, r.in[done:scan]...) diff --git a/internal/handler/pypi_test.go b/internal/handler/pypi_test.go index 73309cb..c34679d 100644 --- a/internal/handler/pypi_test.go +++ b/internal/handler/pypi_test.go @@ -616,45 +616,62 @@ func TestPyPIDownloadCooldownMetadataCache(t *testing.T) { } func TestPyPISimpleIndexRewritesProjectLinks(t *testing.T) { - // Long enough that links cross the rewriter's read boundaries. - var upstream, want strings.Builder - upstream.WriteString(``) - want.WriteString(``) - for i := range 3000 { - fmt.Fprintf(&upstream, `pkg-%d`+"\n", i, i) - fmt.Fprintf(&want, `pkg-%d`+"\n", i, i) - } - upstream.WriteString(``) - want.WriteString(``) + for _, tc := range []struct{ name, upstream, prefix string }{ + {"root", "https://pypi.test", "/simple/"}, + {"subpath", "https://nexus.test/repository/pypi", "/repository/pypi/simple/"}, + } { + t.Run(tc.name, func(t *testing.T) { + // Long enough that links cross the rewriter's read boundaries. + var upstream, want strings.Builder + upstream.WriteString(``) + want.WriteString(``) + for i := range 3000 { + fmt.Fprintf(&upstream, `pkg-%d`+"\n", tc.prefix, i, i) + fmt.Fprintf(&want, `pkg-%d`+"\n", i, i) + } + fmt.Fprintf(&upstream, `q`, tc.prefix) + want.WriteString(`q`) + upstream.WriteString(``) + want.WriteString(``) - h, _ := setupPyPIHandler(t, func(r *http.Request) (*http.Response, error) { - resp := pypiHTTPResponse(r, "text/html", upstream.String()) - resp.Header.Set("Content-Length", strconv.Itoa(upstream.Len())) - return resp, nil - }) + h, _ := setupPyPIHandler(t, func(r *http.Request) (*http.Response, error) { + resp := pypiHTTPResponse(r, "text/html", upstream.String()) + resp.Header.Set("Content-Length", strconv.Itoa(upstream.Len())) + resp.Header.Set("ETag", `"upstream"`) + return resp, nil + }) + h.upstreamURL = tc.upstream - req := httptest.NewRequest(http.MethodGet, "/simple/", nil) - w := httptest.NewRecorder() - h.Routes().ServeHTTP(w, req) + req := httptest.NewRequest(http.MethodGet, "/simple/", nil) + w := httptest.NewRecorder() + h.Routes().ServeHTTP(w, req) - if w.Code != http.StatusOK { - t.Fatalf("status = %d, want 200", w.Code) - } - if got := w.Body.String(); got != want.String() { - t.Errorf("body not rewritten; starts %q", got[:min(len(got), 120)]) - } - if got := w.Header().Get("Content-Length"); got != "" && got != strconv.Itoa(want.Len()) { - t.Errorf("Content-Length = %q, want %d or unset", got, want.Len()) + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want 200", w.Code) + } + if got := w.Body.String(); got != want.String() { + t.Errorf("body not rewritten; starts %q", got[:min(len(got), 120)]) + } + if got := w.Header().Get("Content-Length"); got != "" && got != strconv.Itoa(want.Len()) { + t.Errorf("Content-Length = %q, want %d or unset", got, want.Len()) + } + // The upstream's ETag names the body before the rewrite. + if got := w.Header().Get("ETag"); got != "" { + t.Errorf("ETag = %q, want none on a rewritten body", got) + } + }) } } func TestReplacingReaderSplitReads(t *testing.T) { - const in = `abxhref="/simplehref="/simple/` - const want = `abxhref="/simplehref="P/` + const in = `abxhref="/simplehref="/simple/href='/simple/y/'` + const want = `abxhref="/simplehref="P/href='Q/y/'` r := &replacingReader{ - src: io.NopCloser(iotest.OneByteReader(strings.NewReader(in))), - from: []byte(`href="/simple/`), - to: []byte(`href="P/`), + src: io.NopCloser(iotest.OneByteReader(strings.NewReader(in))), + replacements: []replacement{ + {from: []byte(`href="/simple/`), to: []byte(`href="P/`)}, + {from: []byte(`href='/simple/`), to: []byte(`href='Q/`)}, + }, } got, err := io.ReadAll(r) if err != nil || string(got) != want { From 6381daa4edaf8dc5968be076a76ac0a84a441ae8 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 8 Nov 2026 10:07:53 +0100 Subject: [PATCH 607/633] Test git key case folding on every file system The existing test only runs where the file system ignores case, so it was skipped on Linux, and it passed if the second spelling had no entry at all. Add a test that sets foldCase directly and checks both spellings share one entry, require the entry in the existing test, and work out the repository key once in acquire. --- internal/gitcache/cache.go | 7 ++++--- internal/gitcache/cache_test.go | 23 ++++++++++++++++++++++- 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/internal/gitcache/cache.go b/internal/gitcache/cache.go index 6bc3c20..6680057 100644 --- a/internal/gitcache/cache.go +++ b/internal/gitcache/cache.go @@ -356,10 +356,11 @@ func (c *Cache) removeRepointed() { func (c *Cache) acquire(key string) *repo { c.mu.Lock() defer c.mu.Unlock() - rp := c.repos[c.repoKey(key)] + rk := c.repoKey(key) + rp := c.repos[rk] if rp == nil { - rp = newRepo(c.repoKey(key), filepath.Join(c.dir, filepath.FromSlash(key))) - c.repos[rp.key] = rp + rp = newRepo(rk, filepath.Join(c.dir, filepath.FromSlash(key))) + c.repos[rk] = rp } rp.users++ return rp diff --git a/internal/gitcache/cache_test.go b/internal/gitcache/cache_test.go index 690b11e..6bef422 100644 --- a/internal/gitcache/cache_test.go +++ b/internal/gitcache/cache_test.go @@ -802,7 +802,7 @@ func TestCaseOnlyDifferentKeysShareOneRepository(t *testing.T) { rp := tc.acquire("up/REPO.git") defer tc.release(rp) - if other := tc.repo("up/repo.git"); other != nil && other != rp { + if other := tc.repo("up/repo.git"); other != rp { t.Error("up/REPO.git and up/repo.git are separate repositories") } tc.mu.Lock() @@ -813,3 +813,24 @@ func TestCaseOnlyDifferentKeysShareOneRepository(t *testing.T) { t.Error("repository in use as up/REPO.git was evicted") } } + +// TestFoldCaseKeysShareOneEntry checks key folding on any file system by +// setting foldCase directly. +func TestFoldCaseKeysShareOneEntry(t *testing.T) { + tc := newTestCache(t, newFakeUpstream(t).server.URL, nil) + tc.foldCase = true + + upper := tc.acquire("up/REPO.git") + defer tc.release(upper) + lower := tc.acquire("up/repo.git") + defer tc.release(lower) + if upper != lower { + t.Error("up/REPO.git and up/repo.git are separate repositories") + } + tc.mu.Lock() + n := len(tc.repos) + tc.mu.Unlock() + if n != 1 { + t.Errorf("%d repository entries, want 1", n) + } +} From e2ef2a8083d9dd4e2891154a77c308b6da066c74 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 8 Nov 2026 10:06:12 +0100 Subject: [PATCH 608/633] Print ignored environment values from proxy mirror server.Start logs the warnings LoadFromEnv collects, but mirror never calls it, so an unparseable boolean there was dropped without a word. Print them to stderr before validating the config. stats and hosted don't load the config file and aren't affected. --- cmd/proxy/main.go | 9 +++++++++ cmd/proxy/main_test.go | 21 +++++++++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/cmd/proxy/main.go b/cmd/proxy/main.go index f9ab849..7c51732 100644 --- a/cmd/proxy/main.go +++ b/cmd/proxy/main.go @@ -495,6 +495,7 @@ func runMirror() { os.Exit(1) } cfg.LoadFromEnv() + printEnvWarnings(cfg) if *storageURL != "" { cfg.Storage.URL = *storageURL @@ -825,6 +826,14 @@ func formatSize(bytes int64) string { } } +// printEnvWarnings reports environment values LoadFromEnv ignored. serve +// logs them from server.Start; mirror never gets there. +func printEnvWarnings(cfg *config.Config) { + for _, warning := range cfg.EnvWarnings() { + fmt.Fprintf(os.Stderr, "warning: %s\n", warning) + } +} + func loadConfig(path string) (*config.Config, error) { if path != "" { return config.Load(path) diff --git a/cmd/proxy/main_test.go b/cmd/proxy/main_test.go index 3f17c27..d5a020d 100644 --- a/cmd/proxy/main_test.go +++ b/cmd/proxy/main_test.go @@ -57,3 +57,24 @@ func TestServeHelpProcess(*testing.T) { os.Args = []string{"proxy", "serve", "-help"} main() } + +func TestMirrorLogsEnvWarnings(t *testing.T) { + cmd := exec.Command(os.Args[0], "-test.run=^TestMirrorEnvWarningsProcess$") + // The negative size stops mirror before it opens any storage. + cmd.Env = append(os.Environ(), + "PROXY_TEST_MIRROR_ENV_WARNINGS=1", + "PROXY_CACHE_METADATA=maybe", + "PROXY_STORAGE_MAX_SIZE=-1") + output, _ := cmd.CombinedOutput() + if !strings.Contains(string(output), "PROXY_CACHE_METADATA") { + t.Errorf("mirror did not warn about PROXY_CACHE_METADATA:\n%s", output) + } +} + +func TestMirrorEnvWarningsProcess(*testing.T) { + if os.Getenv("PROXY_TEST_MIRROR_ENV_WARNINGS") != "1" { + return + } + os.Args = []string{"proxy", "mirror", "pkg:npm/lodash@4.17.21"} + main() +} From 7714869af16fa4edbae6a43ff30b2d3fb09b3703 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 8 Nov 2026 10:06:28 +0100 Subject: [PATCH 609/633] Reject sizes smaller than one byte ParseSize truncated fractional sizes such as 0.5B to 0, and a storage.max_size of 0 means unlimited. Reject a non-zero size that would round down to 0 bytes. --- internal/config/config.go | 4 ++++ internal/config/config_test.go | 4 +++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/internal/config/config.go b/internal/config/config.go index a9fc9e2..7b20795 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -2192,6 +2192,10 @@ func ParseSize(s string) (int64, error) { if !(size >= 0 && size < math.MaxInt64) { return 0, fmt.Errorf("invalid size %q: must be between 0 and %d bytes", s, int64(math.MaxInt64)) } + // max_size 0 means unlimited, so don't let a tiny size become 0. + if num != 0 && int64(size) == 0 { + return 0, fmt.Errorf("invalid size %q: less than one byte", s) + } return int64(size), nil } } diff --git a/internal/config/config_test.go b/internal/config/config_test.go index d89e300..ff8fc50 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -297,6 +297,8 @@ func TestParseSize(t *testing.T) { {"1TB", 1024 * 1024 * 1024 * 1024, false}, {"invalid", 0, true}, {"10XB", 0, true}, + {"0GB", 0, false}, + {"0.0MB", 0, false}, } for _, tt := range tests { @@ -2244,7 +2246,7 @@ func TestScannerEcosystems(t *testing.T) { } func TestParseSizeRejectsOutOfRange(t *testing.T) { - for _, s := range []string{"-5GB", "-1", "-0.5K", "NaNGB", "InfGB", "+InfB", "1e30GB", "9999999999TB"} { + for _, s := range []string{"-5GB", "-1", "-0.5K", "NaNGB", "InfGB", "+InfB", "1e30GB", "9999999999TB", "0.5B", "0.000000000001GB"} { if got, err := ParseSize(s); err == nil { t.Errorf("ParseSize(%q) = %d, want error", s, got) } From 7de58c523f21307ab829d6b2cd0606ecb0d9af3d Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 8 Nov 2026 10:08:40 +0100 Subject: [PATCH 610/633] Report errors reading submodules of a mirrored commit Looking for .gitmodules took any error from git cat-file as the file not being there, so a mirror whose context was cancelled reported success with no submodules. Use rev-parse --verify --quiet, whose exit status 1 means the path is missing, and return other errors. A .gitmodules git config can't parse failed the mirror of a commit that is cached all the same. Log a warning and skip its submodules instead, as an unserved submodule host is skipped. --- docs/configuration.md | 2 +- internal/gitcache/helpers_test.go | 3 +- internal/gitcache/mirror.go | 22 +++++++++++--- internal/gitcache/mirror_test.go | 50 +++++++++++++++++++++++++++++++ 4 files changed, 71 insertions(+), 6 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 1c01589..a1ced4b 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1921,7 +1921,7 @@ proxy mirror pkg:git/github.com/git-pkgs/purl - The repository is served by the `upstream.git` host whose URL its host and path start with, the longest one if several do, and has to match that host's `repos`. Others fail. - The repository's branches and tags are cached, as a clone through the proxy caches them. A commit on none of them is fetched by ID and kept, like one a client fetches by ID, until a branch or tag includes it, or for seven days from the last mirror run that named it, as long as `git.max_size` doesn't evict the repository. Where `upstream.auth` has credentials for the host, commits aren't fetched by ID, and one on no branch or tag that isn't cached yet fails. - Without a version the repository's branches and tags are mirrored. A tag as the version has to be one of them, so a tag the upstream doesn't have fails. A branch name as the version fails too; give the commit ID. -- With a version, the submodules that commit records are mirrored at the commits it pins, with their submodules in turn. A relative submodule URL is resolved against the repository's, and an `ssh://`, `git://` or `git@host:path` URL names the repository by its host and path. A submodule no `upstream.git` host serves is skipped with a warning in the log. +- With a version, the submodules that commit records are mirrored at the commits it pins, with their submodules in turn. A relative submodule URL is resolved against the repository's, and an `ssh://`, `git://` or `git@host:path` URL names the repository by its host and path. A submodule no `upstream.git` host serves is skipped with a warning in the log, as are all of a commit's submodules when git can't read its `.gitmodules`. - The git cache is used by one process. While the server runs, the command can't open `git.cache_dir`, so `pkg:git` items fail and the rest are mirrored; use the mirror API instead. ### Scheduled refresh diff --git a/internal/gitcache/helpers_test.go b/internal/gitcache/helpers_test.go index 55b7333..1f18c5b 100644 --- a/internal/gitcache/helpers_test.go +++ b/internal/gitcache/helpers_test.go @@ -302,6 +302,7 @@ func (u *fakeUpstream) bare() string { type testCache struct { *Cache server *httptest.Server + logs *lockedBuffer } func newTestCache(t *testing.T, upstreamURL string, configure func(*Options)) *testCache { @@ -333,7 +334,7 @@ func newTestCache(t *testing.T, upstreamURL string, configure func(*Options)) *t server.Close() _ = c.Close() }) - return &testCache{Cache: c, server: server} + return &testCache{Cache: c, server: server, logs: logs} } // testFront serves /git/up/{repo}/info/refs and git-upload-pack from c, as diff --git a/internal/gitcache/mirror.go b/internal/gitcache/mirror.go index 5aaa6fb..d6e07f9 100644 --- a/internal/gitcache/mirror.go +++ b/internal/gitcache/mirror.go @@ -91,7 +91,7 @@ func (c *Cache) Mirror(ctx context.Context, key, repoURL, version string, fetchW } var submodules []Submodule if commit != "" { - if submodules, err = c.submodules(ctx, rp.dir, commit); err != nil { + if submodules, err = c.submodules(ctx, key, rp.dir, commit); err != nil { return MirrorResult{}, err } } @@ -123,14 +123,28 @@ func (c *Cache) resolveTag(ctx context.Context, dir, tag string) (string, error) // submodules lists the submodules commit records: its gitlinks that // .gitmodules names a URL for. -func (c *Cache) submodules(ctx context.Context, dir, commit string) ([]Submodule, error) { - if c.command(ctx, nil, "-C", dir, "cat-file", "-e", commit+":.gitmodules").Run() != nil { +func (c *Cache) submodules(ctx context.Context, key, dir, commit string) ([]Submodule, error) { + err := c.command(ctx, nil, "-C", dir, "rev-parse", "--verify", "--quiet", "--end-of-options", commit+":.gitmodules").Run() + var exit *exec.ExitError + if errors.As(err, &exit) && exit.ExitCode() == 1 && ctx.Err() == nil { return nil, nil } + if err != nil { + if ctx.Err() != nil { + return nil, ctx.Err() + } + return nil, fmt.Errorf("looking for .gitmodules: %w", err) + } // --list rather than --get-regexp, which fails when nothing matches. out, err := c.run(ctx, dir, "config", "-z", "--blob", commit+":.gitmodules", "--list") if err != nil { - return nil, fmt.Errorf("reading .gitmodules: %w", err) + if ctx.Err() != nil { + return nil, ctx.Err() + } + // The commit is cached; a .gitmodules git can't read only loses + // its submodules. + c.logger.Warn("git cache: cannot read .gitmodules, skipping submodules", "repository", key, "commit", commit, "error", err) + return nil, nil } paths, urls := map[string]string{}, map[string]string{} for entry := range strings.SplitSeq(out, "\x00") { diff --git a/internal/gitcache/mirror_test.go b/internal/gitcache/mirror_test.go index 1f85a6c..e57e64f 100644 --- a/internal/gitcache/mirror_test.go +++ b/internal/gitcache/mirror_test.go @@ -315,3 +315,53 @@ func TestMirrorWithEmptyGitmodules(t *testing.T) { } } } + +// TestSubmodulesReportsCancel checks a cancelled context is reported, not +// read as a commit without .gitmodules. +func TestSubmodulesReportsCancel(t *testing.T) { + upstream := newFakeUpstream(t) + work := upstream.newRepo("repo") + commit := runGit(t, work, "rev-parse", "HEAD") + ctx, cancel := context.WithCancel(context.Background()) + cancel() + tc := newTestCache(t, upstream.server.URL, nil) + + if _, err := tc.submodules(ctx, "up/repo.git", work, commit); !errors.Is(err, context.Canceled) { + t.Errorf("submodules with a cancelled context = %v, want context.Canceled", err) + } +} + +// TestMirrorSkipsBrokenGitmodules checks a .gitmodules git can't read, one +// with a syntax error or committed as a symlink, lists no submodules and +// logs a warning instead of failing the mirror. +func TestMirrorSkipsBrokenGitmodules(t *testing.T) { + upstream := newFakeUpstream(t) + work := upstream.newRepo("repo") + gitmodules := filepath.Join(work, ".gitmodules") + if err := os.WriteFile(gitmodules, []byte("[submodule \"x\"\n\tpath = x\n"), 0o600); err != nil { + t.Fatal(err) + } + runGit(t, work, "add", ".gitmodules") + runGit(t, work, "commit", "--quiet", "-m", "broken gitmodules") + if err := os.Remove(gitmodules); err != nil { + t.Fatal(err) + } + if err := os.Symlink("../outside/.gitmodules", gitmodules); err != nil { + t.Fatal(err) + } + runGit(t, work, "add", ".gitmodules") + runGit(t, work, "commit", "--quiet", "-m", "symlinked gitmodules") + runGit(t, work, "push", "--quiet", "origin", "main") + tc := newTestCache(t, upstream.server.URL, nil) + + for _, rev := range []string{"HEAD~1", "HEAD"} { + commit := runGit(t, work, "rev-parse", rev) + got, err := tc.Mirror(context.Background(), "up/repo.git", upstream.server.URL+"/repo", commit, true) + if err != nil || len(got.Submodules) != 0 { + t.Errorf("mirror of %s = %+v, %v, want no submodules", rev, got, err) + } + if !strings.Contains(tc.logs.String(), "commit="+commit) { + t.Errorf("no warning naming %s in the log", commit) + } + } +} From f4d074d7c471d45131cdbbc05668d5a18c16b5a0 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 8 Nov 2026 10:08:52 +0100 Subject: [PATCH 611/633] Normalise ecosystems the scanner group is asked about applicable compared the caller's ecosystem raw against the normalised configured keys, so a caller passing an alias such as go or rubygems skipped a scanner restricted to golang or gem. Normalise it first. NewGroup now also rejects unknown names the way Validate does, through a shared ScannerEcosystemKey, instead of building a scanner that can never match. --- internal/config/config.go | 16 +++++++++++++--- internal/scanner/group.go | 8 +++++++- internal/scanner/group_test.go | 15 ++++++++++++++- 3 files changed, 34 insertions(+), 5 deletions(-) diff --git a/internal/config/config.go b/internal/config/config.go index 7b20795..25536e7 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -439,9 +439,9 @@ func (s *ScannerConfig) Validate() error { return fmt.Errorf("invalid mode %q (must be block or monitor)", s.Mode) } for i, eco := range s.Ecosystems { - key := NormalizeScannerEcosystem(eco) - if !slices.Contains(scannerEcosystems, key) { - return fmt.Errorf("unknown ecosystem %q (valid: %s)", eco, strings.Join(scannerEcosystems, ", ")) + key, err := ScannerEcosystemKey(eco) + if err != nil { + return err } s.Ecosystems[i] = key } @@ -477,6 +477,16 @@ func NormalizeScannerEcosystem(eco string) string { return eco } +// ScannerEcosystemKey is NormalizeScannerEcosystem that also rejects +// names no handler uses. +func ScannerEcosystemKey(eco string) (string, error) { + key := NormalizeScannerEcosystem(eco) + if !slices.Contains(scannerEcosystems, key) { + return "", fmt.Errorf("unknown ecosystem %q (valid: %s)", eco, strings.Join(scannerEcosystems, ", ")) + } + return key, nil +} + // StorageConfig configures artifact storage. type StorageConfig struct { // URL is the storage backend URL. diff --git a/internal/scanner/group.go b/internal/scanner/group.go index 3066219..a4856e0 100644 --- a/internal/scanner/group.go +++ b/internal/scanner/group.go @@ -67,7 +67,11 @@ func NewGroup(cfg config.ScanningConfig, logger *slog.Logger) (*Group, error) { if len(sc.Ecosystems) > 0 { ecosystems = make(map[string]struct{}, len(sc.Ecosystems)) for _, eco := range sc.Ecosystems { - ecosystems[config.NormalizeScannerEcosystem(eco)] = struct{}{} + key, err := config.ScannerEcosystemKey(eco) + if err != nil { + return nil, fmt.Errorf("scanner %q: %w", sc.Name, err) + } + ecosystems[key] = struct{}{} } } @@ -112,6 +116,8 @@ func (g *Group) Timeout() time.Duration { } func (g *Group) applicable(ecosystem string) []entry { + // Callers may pass an alias such as go from a PURL type. + ecosystem = config.NormalizeScannerEcosystem(ecosystem) var out []entry for _, e := range g.entries { if len(e.ecosystems) == 0 { diff --git a/internal/scanner/group_test.go b/internal/scanner/group_test.go index d80166b..5dca201 100644 --- a/internal/scanner/group_test.go +++ b/internal/scanner/group_test.go @@ -306,9 +306,22 @@ func TestGroup_NormalisesEcosystemAliases(t *testing.T) { if err != nil { t.Fatalf("NewGroup() error: %v", err) } - for _, ecosystem := range []string{"golang", "oci", "gem"} { + for _, ecosystem := range []string{"golang", "oci", "gem", "go", "Docker", "rubygems"} { if !g.Blocks(ecosystem) { t.Errorf("Blocks(%s) = false, want true", ecosystem) } } } + +func TestNewGroup_RejectsUnknownEcosystem(t *testing.T) { + _, err := NewGroup(config.ScanningConfig{ + Enabled: true, + SigningKey: "test-signing-key", + Scanners: []config.ScannerConfig{ + {Name: "clamav", URL: "http://clamav.invalid", Ecosystems: []string{"npm", "pypy"}}, + }, + }, discardLogger()) + if err == nil || !strings.Contains(err.Error(), "pypy") { + t.Errorf("NewGroup() error = %v, want unknown ecosystem pypy", err) + } +} From c7031e4897e7fef6a826df37693be48597cc33b4 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 8 Nov 2026 10:09:23 +0100 Subject: [PATCH 612/633] Track mirrored manifests by sha256 whatever digest names them Mirroring img:v1@sha512: compared the sha512 digest with the sha256 digest the tag resolved to, so an unmoved tag was reported as moved and its manifest walked twice. Key seen manifests and the returned digest by sha256, as tags are. --- internal/handler/container_mirror.go | 5 ++- internal/handler/container_mirror_test.go | 39 +++++++++++++++++++++++ 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/internal/handler/container_mirror.go b/internal/handler/container_mirror.go index e17311c..f19ac73 100644 --- a/internal/handler/container_mirror.go +++ b/internal/handler/container_mirror.go @@ -332,7 +332,7 @@ func (m *imageMirror) requestTag(ctx context.Context, method, tag, accept string // mirrorManifest stores the manifest for reference through the handler's own // manifest route, then follows what it references. It returns the manifest -// digest. +// sha256 digest. func (m *imageMirror) mirrorManifest(ctx context.Context, reference string, depth int) (string, error) { cacheAccept := normalizeContainerManifestAccept(containerManifestAccept(&http.Request{Header: http.Header{}})) before, _ := m.h.loadContainerManifestForAccept(ctx, m.registryURL, m.upstreamName, reference, cacheAccept, cacheAccept) @@ -360,6 +360,9 @@ func (m *imageMirror) mirrorManifest(ctx context.Context, reference string, dept if before == nil || before.contentDigest != digest { m.result.Fetched++ } + // Tags resolve to sha256 digests, so a manifest asked for by another + // digest is tracked by its sha256 too. + digest = sha256Digest(response.body.Bytes()) if m.seen[digest] { return digest, nil } diff --git a/internal/handler/container_mirror_test.go b/internal/handler/container_mirror_test.go index f3a15ee..34af48c 100644 --- a/internal/handler/container_mirror_test.go +++ b/internal/handler/container_mirror_test.go @@ -10,6 +10,7 @@ import ( "net/http" "net/http/httptest" "net/url" + "slices" "strconv" "strings" "sync" @@ -36,6 +37,9 @@ type fakeImageRegistry struct { truncateManifests atomic.Bool // onBlob, if set, runs before a blob is served. onBlob func(path string) + // sha512Digests makes manifests requested by a sha512 digest name that + // digest in Docker-Content-Digest, as registries do. + sha512Digests bool } type fakeImage struct { @@ -129,6 +133,9 @@ func newFakeImageRegistry(t *testing.T) (*fakeImageRegistry, fakeImage) { if ok { w.Header().Set("Content-Type", doc.contentType) w.Header().Set("Docker-Content-Digest", sha256Digest([]byte(doc.body))) + if _, ref, _ := strings.Cut(req.URL.Path, "/manifests/"); r.sha512Digests && strings.HasPrefix(ref, "sha512:") { + w.Header().Set("Docker-Content-Digest", ref) + } if req.Method == http.MethodGet && !strings.Contains(req.URL.Path, "/manifests/sha256:") { r.tagGets.Add(1) } @@ -644,3 +651,35 @@ func TestMirrorImageAcceptsSHA512Blobs(t *testing.T) { _ = cached.Reader.Close() } } + +// TestMirrorImageSHA512ReferenceMatchesTag checks that mirroring a tag +// pinned by a sha512 digest compares it with the tag's manifest by sha512, +// so an unmoved tag is neither reported as moved nor counted twice. +func TestMirrorImageSHA512ReferenceMatchesTag(t *testing.T) { + mirror := func(t *testing.T, sha512Ref bool) ImageResult { + registry, image := newFakeImageRegistry(t) + reference := image.singleDigest + if sha512Ref { + body := registry.manifests["/v2/team/app/manifests/single"].body + sum := sha512.Sum512([]byte(body)) + reference = "sha512:" + hex.EncodeToString(sum[:]) + registry.manifests["/v2/team/app/manifests/"+reference] = registry.manifests["/v2/team/app/manifests/single"] + registry.sha512Digests = true + } + proxy := newMirrorTestProxy(t, registry) + h := NewContainerHandlerWithRegistry(proxy, "http://proxy.example", registry.server.URL) + result, err := h.MirrorImage(context.Background(), ImageRef{Name: "team/app", References: []string{reference, "single"}}, nil) + if err != nil { + t.Fatalf("MirrorImage: %v", err) + } + return result + } + want := mirror(t, false) + got := mirror(t, true) + if !slices.Equal(got.Warnings, want.Warnings) { + t.Errorf("warnings = %v, want %v as for a sha256 reference", got.Warnings, want.Warnings) + } + if got.Fetched != want.Fetched { + t.Errorf("fetched = %d, want %d as for a sha256 reference", got.Fetched, want.Fetched) + } +} From 989697596f9f4fa07e80aa74c198094d25e8bbba Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 8 Nov 2026 10:09:25 +0100 Subject: [PATCH 613/633] Accept uppercase commit IDs and explain short ones when mirroring A version that wasn't a lowercase full-length hex ID was taken for a tag, so an uppercase commit ID failed as a missing tag. Lowercase a full-length hex version first. A shorter hex version that names no tag now fails saying it is neither a full commit ID nor a tag. --- docs/configuration.md | 2 +- internal/gitcache/mirror.go | 7 +++++++ internal/gitcache/mirror_test.go | 24 ++++++++++++++++++++++++ 3 files changed, 32 insertions(+), 1 deletion(-) diff --git a/docs/configuration.md b/docs/configuration.md index a1ced4b..0d81646 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1920,7 +1920,7 @@ proxy mirror pkg:git/github.com/git-pkgs/purl - The repository is served by the `upstream.git` host whose URL its host and path start with, the longest one if several do, and has to match that host's `repos`. Others fail. - The repository's branches and tags are cached, as a clone through the proxy caches them. A commit on none of them is fetched by ID and kept, like one a client fetches by ID, until a branch or tag includes it, or for seven days from the last mirror run that named it, as long as `git.max_size` doesn't evict the repository. Where `upstream.auth` has credentials for the host, commits aren't fetched by ID, and one on no branch or tag that isn't cached yet fails. -- Without a version the repository's branches and tags are mirrored. A tag as the version has to be one of them, so a tag the upstream doesn't have fails. A branch name as the version fails too; give the commit ID. +- Without a version the repository's branches and tags are mirrored. A tag as the version has to be one of them, so a tag the upstream doesn't have fails. A branch name or an abbreviated commit ID as the version fails too; give the full commit ID. - With a version, the submodules that commit records are mirrored at the commits it pins, with their submodules in turn. A relative submodule URL is resolved against the repository's, and an `ssh://`, `git://` or `git@host:path` URL names the repository by its host and path. A submodule no `upstream.git` host serves is skipped with a warning in the log, as are all of a commit's submodules when git can't read its `.gitmodules`. - The git cache is used by one process. While the server runs, the command can't open `git.cache_dir`, so `pkg:git` items fail and the rest are mirrored; use the mirror API instead. diff --git a/internal/gitcache/mirror.go b/internal/gitcache/mirror.go index d6e07f9..f884d37 100644 --- a/internal/gitcache/mirror.go +++ b/internal/gitcache/mirror.go @@ -37,6 +37,9 @@ type Submodule struct { // OpenOptions.FetchWants), and is kept as long as a commit a client fetched // by ID. A tag is cached with the repository's tags. func (c *Cache) Mirror(ctx context.Context, key, repoURL, version string, fetchWants bool) (MirrorResult, error) { + if lower := strings.ToLower(version); isObjectID(lower) { + version = lower + } tag := version != "" && !isObjectID(version) if tag { if err := c.checkTagName(ctx, version); err != nil { @@ -78,6 +81,10 @@ func (c *Cache) Mirror(ctx context.Context, key, repoURL, version string, fetchW // The tag may be one the failed refresh would have brought. return MirrorResult{}, refreshErr } + if errors.Is(err, ErrNotFound) && strings.Trim(strings.ToLower(version), "0123456789abcdef") == "" { + // Likely an abbreviated commit ID. + return MirrorResult{}, fmt.Errorf("%s is not a full commit ID or tag: %w", version, ErrNotFound) + } return MirrorResult{}, err } case version != "": diff --git a/internal/gitcache/mirror_test.go b/internal/gitcache/mirror_test.go index e57e64f..b1e27f6 100644 --- a/internal/gitcache/mirror_test.go +++ b/internal/gitcache/mirror_test.go @@ -365,3 +365,27 @@ func TestMirrorSkipsBrokenGitmodules(t *testing.T) { } } } + +func TestMirrorAcceptsUppercaseCommit(t *testing.T) { + upstream := newFakeUpstream(t) + work := upstream.newRepo("repo") + commit := runGit(t, work, "rev-parse", "HEAD") + tc := newTestCache(t, upstream.server.URL, nil) + + got, err := tc.Mirror(context.Background(), "up/repo.git", upstream.server.URL+"/repo", strings.ToUpper(commit), true) + if err != nil || got.Commit != commit { + t.Errorf("mirror of the uppercase commit ID = %+v, %v, want commit %s", got, err, commit) + } +} + +func TestMirrorExplainsShortCommit(t *testing.T) { + upstream := newFakeUpstream(t) + work := upstream.newRepo("repo") + commit := runGit(t, work, "rev-parse", "HEAD") + tc := newTestCache(t, upstream.server.URL, nil) + + _, err := tc.Mirror(context.Background(), "up/repo.git", upstream.server.URL+"/repo", commit[:12], true) + if !errors.Is(err, ErrNotFound) || !strings.Contains(err.Error(), "full commit ID") { + t.Errorf("mirror of a short commit ID = %v, want ErrNotFound saying it is no full commit ID", err) + } +} From fe764ea6408dcc05c7b005534664664cdcde7152 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 8 Nov 2026 10:07:32 +0100 Subject: [PATCH 614/633] Clear a refusal once the upstream lists the refs A refusal was cleared only by a refresh that succeeded as a whole. When the upstream listed the refs again but the fetch that followed failed, say because the cache was full or the fetch timed out, the old refusal stayed and the repository was still refused, though the upstream had just confirmed access. Clear it as soon as the ref check succeeds. --- internal/gitcache/cache.go | 2 +- internal/gitcache/cache_test.go | 21 +++++++++++++ internal/gitcache/helpers_test.go | 50 ++++++++++++++++++++----------- internal/gitcache/refresh.go | 10 ++++--- internal/gitcache/serve.go | 2 +- 5 files changed, 61 insertions(+), 24 deletions(-) diff --git a/internal/gitcache/cache.go b/internal/gitcache/cache.go index 7901380..9a8628f 100644 --- a/internal/gitcache/cache.go +++ b/internal/gitcache/cache.go @@ -130,7 +130,7 @@ type repo struct { users int refreshing *refreshCall // refused is the error of the last refresh when the upstream reported - // the repository missing or refused it, until a refresh succeeds. + // the repository missing or refused it, until a ref check succeeds. refused error // writes serializes git fetches into the repository. writes chan struct{} diff --git a/internal/gitcache/cache_test.go b/internal/gitcache/cache_test.go index baf646a..931e6d5 100644 --- a/internal/gitcache/cache_test.go +++ b/internal/gitcache/cache_test.go @@ -922,3 +922,24 @@ func TestUploadPackClearsRefusal(t *testing.T) { t.Errorf("ls-refs after the upstream allowed the repository again: status %d", status) } } + +// TestRefCheckClearsRefusal has the upstream list the refs again after a +// refusal but fail the fetch that follows: the check alone clears the +// refusal, and the cached copy is served. +func TestRefCheckClearsRefusal(t *testing.T) { + upstream := newFakeUpstream(t) + work := upstream.newRepo("repo") + tc := newTestCache(t, upstream.server.URL, nil) + tc.clone(t, "repo") + tc.refuse(t, upstream) + + upstream.commit(work, "second") + runGit(t, work, "push", "--quiet", "origin", "main") + upstream.fetchStatus.Store(http.StatusInternalServerError) + if out, err := tryGit(t, "", "ls-remote", tc.url("repo")); err != nil { + t.Errorf("cached copy refused after the upstream listed its refs: %v\n%s", err, out) + } + if status := tc.lsRefs(t); status != http.StatusOK { + t.Errorf("ls-refs after the upstream listed its refs: status %d", status) + } +} diff --git a/internal/gitcache/helpers_test.go b/internal/gitcache/helpers_test.go index 55b7333..e5f1311 100644 --- a/internal/gitcache/helpers_test.go +++ b/internal/gitcache/helpers_test.go @@ -75,8 +75,10 @@ type fakeUpstream struct { root string server *httptest.Server - // status, when set, is returned instead of serving the request. - status atomic.Int32 + // status, when set, is returned instead of serving the request, and + // fetchStatus instead of serving git's own upload-pack requests. + status atomic.Int32 + fetchStatus atomic.Int32 // requireAuth, when set, is the Authorization every request must carry. requireAuth string // v0Only drops Git-Protocol, so only protocol v0 is spoken. @@ -178,22 +180,7 @@ func newFakeUpstream(t *testing.T) *fakeUpstream { if u.v0Only { r.Header.Del("Git-Protocol") } - if r.UserAgent() != gitUserAgent && r.Method == http.MethodPost { - u.holdMu.Lock() - fetchHold, fetchArrived, fetchOnce := u.fetchHold, u.fetchArrived, u.fetchOnce - u.holdMu.Unlock() - if fetchHold != nil { - fetchOnce.Do(func() { close(fetchArrived) }) - <-fetchHold - } - } - if delay := u.delayFetches.Load(); delay > 0 && r.UserAgent() != gitUserAgent && r.Method == http.MethodPost { - time.Sleep(time.Duration(delay)) - } - if u.stallFetches.Load() && r.UserAgent() != gitUserAgent && r.Method == http.MethodPost { - // The server notices the client leaving only once the body is read. - _, _ = io.Copy(io.Discard, r.Body) - <-r.Context().Done() + if r.UserAgent() != gitUserAgent && r.Method == http.MethodPost && u.interceptFetch(w, r) { return } if r.Method == http.MethodPost { @@ -205,6 +192,33 @@ func newFakeUpstream(t *testing.T) *fakeUpstream { return u } +// interceptFetch applies the fetch knobs to one of git's own upload-pack +// requests, and reports whether it answered the request. +func (u *fakeUpstream) interceptFetch(w http.ResponseWriter, r *http.Request) bool { + u.holdMu.Lock() + fetchHold, fetchArrived, fetchOnce := u.fetchHold, u.fetchArrived, u.fetchOnce + u.holdMu.Unlock() + if fetchHold != nil { + fetchOnce.Do(func() { close(fetchArrived) }) + <-fetchHold + } + if delay := u.delayFetches.Load(); delay > 0 { + time.Sleep(time.Duration(delay)) + } + if u.stallFetches.Load() { + // The server notices the client leaving only once the body is read. + _, _ = io.Copy(io.Discard, r.Body) + <-r.Context().Done() + return true + } + if status := u.fetchStatus.Load(); status != 0 { + w.Header().Set("Location", u.server.URL+"/elsewhere") + http.Error(w, "unavailable", int(status)) + return true + } + return false +} + // holdFetches holds git's own upload-pack requests until release is called. // arrived is closed when the first one reaches the upstream. func (u *fakeUpstream) holdFetches() (arrived <-chan struct{}, release func()) { diff --git a/internal/gitcache/refresh.go b/internal/gitcache/refresh.go index d2f8ee4..786603b 100644 --- a/internal/gitcache/refresh.go +++ b/internal/gitcache/refresh.go @@ -157,10 +157,7 @@ func (c *Cache) startRefresh(rp *repo, repoURL string) *refreshCall { } c.mu.Lock() rp.refreshing = nil - switch { - case call.err == nil: - rp.refused = nil - case !servesStale(call.err): + if call.err != nil && !servesStale(call.err) { rp.refused = call.err } c.mu.Unlock() @@ -186,6 +183,11 @@ func (c *Cache) refresh(ctx context.Context, rp *repo, repoURL string, checked f if err != nil { return err } + // The upstream lets us read the repository again, even if the fetch + // below fails. + c.mu.Lock() + rp.refused = nil + c.mu.Unlock() if rp.exists() { err = c.update(ctx, rp, repoURL, remote, checked) } else { diff --git a/internal/gitcache/serve.go b/internal/gitcache/serve.go index 5a9f1e0..bda671f 100644 --- a/internal/gitcache/serve.go +++ b/internal/gitcache/serve.go @@ -98,7 +98,7 @@ func (c *Cache) checkServable(ctx context.Context, rp *repo, repoURL string) err } // refusal returns the error of the last refresh that found the repository -// gone or refused upstream, until a refresh succeeds. +// gone or refused upstream, until a ref check succeeds. func (c *Cache) refusal(rp *repo) error { c.mu.Lock() defer c.mu.Unlock() From 806003e1c112fb7ecd9c41d4a38bf690363e00ec Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 8 Nov 2026 10:07:51 +0100 Subject: [PATCH 615/633] Saturate the upload-pack check age stale_max_age plus the upload-pack grace overflowed for a stale_max_age near the largest duration, which made every upload-pack request check the upstream. Cap the sum instead. --- internal/gitcache/cache_test.go | 21 +++++++++++++++++++++ internal/gitcache/serve.go | 8 +++++++- 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/internal/gitcache/cache_test.go b/internal/gitcache/cache_test.go index 931e6d5..dc70350 100644 --- a/internal/gitcache/cache_test.go +++ b/internal/gitcache/cache_test.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "maps" + "math" "net/http" "net/http/httptest" "net/http/httputil" @@ -943,3 +944,23 @@ func TestRefCheckClearsRefusal(t *testing.T) { t.Errorf("ls-refs after the upstream listed its refs: status %d", status) } } + +// TestHugeStaleMaxAgeUploadPackChecksNothing sets stale_max_age near the +// largest duration: upload-pack requests on a fresh copy do not check the +// upstream. +func TestHugeStaleMaxAgeUploadPackChecksNothing(t *testing.T) { + upstream := newFakeUpstream(t) + upstream.newRepo("repo") + tc := newTestCache(t, upstream.server.URL, func(o *Options) { o.StaleMaxAge = math.MaxInt64 - time.Second }) + tc.clone(t, "repo") + waitIdle(t, tc.Cache) + + before := upstream.requests.Load() + if status := tc.lsRefs(t); status != http.StatusOK { + t.Fatalf("ls-refs: status %d", status) + } + waitIdle(t, tc.Cache) + if got := upstream.requests.Load() - before; got != 0 { + t.Errorf("ls-refs made %d upstream requests, want 0", got) + } +} diff --git a/internal/gitcache/serve.go b/internal/gitcache/serve.go index bda671f..5d80938 100644 --- a/internal/gitcache/serve.go +++ b/internal/gitcache/serve.go @@ -6,6 +6,7 @@ import ( "context" "errors" "io" + "math" "net/http" "os" "path/filepath" @@ -88,7 +89,12 @@ const uploadPackGrace = time.Minute // found it gone or refused upstream, or it was not checked recently, and // refuses it while that refusal stands. func (c *Cache) checkServable(ctx context.Context, rp *repo, repoURL string) error { - if c.refusal(rp) == nil && time.Since(time.Unix(0, rp.lastChecked.Load())) <= c.opts.StaleMaxAge+uploadPackGrace { + // The sum saturates rather than overflow for a huge StaleMaxAge. + maxAge := c.opts.StaleMaxAge + uploadPackGrace + if maxAge < c.opts.StaleMaxAge { + maxAge = math.MaxInt64 + } + if c.refusal(rp) == nil && time.Since(time.Unix(0, rp.lastChecked.Load())) <= maxAge { return nil } if err := c.ensureFresh(ctx, rp, repoURL); err != nil { From 06aef28125d80a48607818d58ecca6b694a97756 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 8 Nov 2026 10:10:34 +0100 Subject: [PATCH 616/633] Refuse a repository whose fetch is redirected The relay turned a redirect into a failed fetch, which counted as the upstream failing rather than refusing, so the cached copy was still served and no refusal was recorded. Report it as a refusal, as for a redirected ref listing. --- internal/gitcache/cache_test.go | 20 ++++++++++++++++++++ internal/gitcache/refresh.go | 3 +++ internal/gitcache/relay.go | 4 ++++ 3 files changed, 27 insertions(+) diff --git a/internal/gitcache/cache_test.go b/internal/gitcache/cache_test.go index dc70350..9fd5449 100644 --- a/internal/gitcache/cache_test.go +++ b/internal/gitcache/cache_test.go @@ -964,3 +964,23 @@ func TestHugeStaleMaxAgeUploadPackChecksNothing(t *testing.T) { t.Errorf("ls-refs made %d upstream requests, want 0", got) } } + +// TestFetchRedirectRefusesRepository has the upstream list the refs but +// redirect the fetch that follows: the repository is refused, as when the +// ref listing is redirected. +func TestFetchRedirectRefusesRepository(t *testing.T) { + upstream := newFakeUpstream(t) + work := upstream.newRepo("repo") + tc := newTestCache(t, upstream.server.URL, nil) + tc.clone(t, "repo") + + upstream.commit(work, "second") + runGit(t, work, "push", "--quiet", "origin", "main") + upstream.fetchStatus.Store(http.StatusFound) + if out, err := tryGit(t, "", "ls-remote", tc.url("repo")); err == nil { + t.Errorf("served the cached copy after the fetch was redirected:\n%s", out) + } + if status := tc.lsRefs(t); status == http.StatusOK { + t.Errorf("ls-refs served after the fetch was redirected") + } +} diff --git a/internal/gitcache/refresh.go b/internal/gitcache/refresh.go index 786603b..f467a3d 100644 --- a/internal/gitcache/refresh.go +++ b/internal/gitcache/refresh.go @@ -326,6 +326,9 @@ func (c *Cache) fetch(ctx context.Context, dir, repoURL string, limit int64, res if route.full.Load() { return ErrCacheFull } + if status := route.redirected.Load(); status != 0 { + return upstreamStatusError(int(status)) + } if err != nil && !route.answered.Load() { return fmt.Errorf("%w: %w", errUpstreamUnanswered, err) } diff --git a/internal/gitcache/relay.go b/internal/gitcache/relay.go index ed42001..bd65b25 100644 --- a/internal/gitcache/relay.go +++ b/internal/gitcache/relay.go @@ -53,6 +53,9 @@ type relayRoute struct { // answered records that the upstream answered a git-upload-pack // request, so a failed fetch was not an outage. answered atomic.Bool + // redirected records the status of a redirect the upstream answered + // with, 0 for none. + redirected atomic.Int32 } func newRelay(client *http.Client, logger *slog.Logger) (*relay, error) { @@ -151,6 +154,7 @@ func (rl *relay) ServeHTTP(w http.ResponseWriter, r *http.Request) { defer func() { _ = resp.Body.Close() }() // A redirect passed on would send git to the Location itself. if resp.StatusCode >= http.StatusMultipleChoices && resp.StatusCode < http.StatusBadRequest { + route.redirected.Store(int32(resp.StatusCode)) http.Error(w, "upstream redirect not followed", http.StatusBadGateway) return } From 7fdfd9908fbc35af46b3d6f6795aa6dfacb48f63 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 8 Nov 2026 10:10:43 +0100 Subject: [PATCH 617/633] Refetch npm packuments only when a cached copy lacks the version Under cooldown, a tarball for a version the packument doesn't list is decided on a fresh fetch of the packument. That check had gaps: - A stale copy served because upstream failed counted as fetched, so the version went through with no publish time. It now triggers the refetch like a copy within its TTL. - A packument just stored from upstream counted as cached, so every request for an unlisted version fetched it twice. Only a stored row read within its TTL counts now. - A stored refetch that joined another caller's fetch could get that caller's stale fallback and take it as upstream's answer. The stored lookup now reports stale results, and a stale refetch is refused. - A refused refetch dropped the upstream error, so a rate-limit hold answered 502 instead of 503 with Retry-After, and a 404 was lost. Stale results now carry the upstream error, and the refusal wraps it. --- internal/handler/handler.go | 7 +- internal/handler/metadata_stored.go | 30 +++-- internal/handler/npm.go | 79 +++++++------ internal/handler/npm_tarball_test.go | 166 +++++++++++++++++++++++++++ 4 files changed, 230 insertions(+), 52 deletions(-) diff --git a/internal/handler/handler.go b/internal/handler/handler.go index d316aab..be818f2 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -1829,7 +1829,7 @@ func (p *Proxy) staleMetadata(ctx context.Context, ecosystem, cacheKey string, v p.Logger.Info("serving metadata from cache", "ecosystem", ecosystem, "key", cacheKey) - return metadataResult{body: data, contentType: ct, contentEncoding: entry.ContentEncoding.String, stale: true}.describedBy(entry) + return metadataResult{body: data, contentType: ct, contentEncoding: entry.ContentEncoding.String, stale: true, upstreamErr: err}.describedBy(entry) } // metadataResult is what a metadata lookup hands back. A shared fetch gives @@ -1843,8 +1843,9 @@ type metadataResult struct { digest string err error // stale is set when the body is the cached copy served because upstream - // failed. - stale bool + // failed, and upstreamErr is that failure. + stale bool + upstreamErr error // fromCache is set when the body is a stored copy within its TTL, read // without asking upstream. fromCache bool diff --git a/internal/handler/metadata_stored.go b/internal/handler/metadata_stored.go index ffe524d..fc16d43 100644 --- a/internal/handler/metadata_stored.go +++ b/internal/handler/metadata_stored.go @@ -45,30 +45,26 @@ var errStoredMetadataChanged = errors.New("stored metadata changed while it was // It returns errMetadataNotStored when the proxy doesn't cache metadata or // storing the document failed, and the caller reads it into memory instead. func (p *Proxy) storedMetadataDocument(ctx context.Context, ecosystem, cacheKey, upstreamURL, accept string, validate func(io.Reader) error) (*database.MetadataCacheEntry, error) { - return p.storedMetadata(ctx, ecosystem, cacheKey, upstreamURL, accept, validate, "") + res := p.storedMetadata(ctx, ecosystem, cacheKey, upstreamURL, accept, validate, "") + return res.entry, res.err } -// restoreStoredMetadata is storedMetadataDocument for a document whose row -// gone describes but whose stored copy was missing when read: the row is not -// served or revalidated, within its TTL or not, and the document is fetched -// again. -func (p *Proxy) restoreStoredMetadata(ctx context.Context, ecosystem, cacheKey, upstreamURL, accept string, validate func(io.Reader) error, gone *database.MetadataCacheEntry) (*database.MetadataCacheEntry, error) { - return p.storedMetadata(ctx, ecosystem, cacheKey, upstreamURL, accept, validate, gone.ContentDigest.String) -} - -// storedMetadata implements storedMetadataDocument. A row with the digest -// gone is treated as if there were none. -func (p *Proxy) storedMetadata(ctx context.Context, ecosystem, cacheKey, upstreamURL, accept string, validate func(io.Reader) error, gone string) (*database.MetadataCacheEntry, error) { +// storedMetadata implements storedMetadataDocument, and reports in the +// result whether the row was read from cache or served stale. A row with +// the digest gone, one whose stored copy was missing or out of date when +// read, is not served or revalidated, within its TTL or not, and the +// document is fetched again. +func (p *Proxy) storedMetadata(ctx context.Context, ecosystem, cacheKey, upstreamURL, accept string, validate func(io.Reader) error, gone string) metadataResult { if !p.CacheMetadata || p.DB == nil || p.Storage == nil || p.MetadataTTL <= 0 { - return nil, errMetadataNotStored + return metadataResult{err: errMetadataNotStored} } if containsPathTraversal(cacheKey) { - return nil, fmt.Errorf("invalid cache key: %q", cacheKey) + return metadataResult{err: fmt.Errorf("invalid cache key: %q", cacheKey)} } if entry, fresh := p.storedMetadataEntryState(ctx, ecosystem, cacheKey, gone); fresh { metrics.RecordMetadataCacheHit(ecosystem, int(entry.Size.Int64)) p.recordMetadataRead(entry) - return entry, nil + return metadataResult{entry: entry, fromCache: true} } p.recordMetadataCacheMiss(ecosystem) @@ -102,7 +98,7 @@ func (p *Proxy) storedMetadata(ctx context.Context, ecosystem, cacheKey, upstrea res = p.fetchStoredMetadata(fetchCtx, ecosystem, cacheKey, upstreamURL, accept, validate, entry) return res }) - return res.entry, res.err + return res } // storedMetadataEntryState reads the cache row of a document and reports @@ -157,7 +153,7 @@ func (p *Proxy) staleStoredMetadata(ecosystem, cacheKey string, entry *database. if !usableStoredMetadata(entry) { return metadataResult{err: fmt.Errorf("upstream failed and no usable cached metadata: %w", err)} } - return metadataResult{entry: entry, stale: true} + return metadataResult{entry: entry, stale: true, upstreamErr: err} } // storeUpstreamMetadata fetches a document and streams it into storage, and diff --git a/internal/handler/npm.go b/internal/handler/npm.go index 0e9f32a..7eef424 100644 --- a/internal/handler/npm.go +++ b/internal/handler/npm.go @@ -344,7 +344,7 @@ func (h *NPMHandler) serveStreamedPackument(w http.ResponseWriter, r *http.Reque fullRequired := h.packumentValidator() != nil options := h.npmRewriteOptions(packageName, nil) var rewritten rewrittenMetadata - err, fetchErr := h.readStoredPackument(r.Context(), upstreamURL, accept, metadataCacheKey, false, func(entry *database.MetadataCacheEntry) (err error) { + err, fetchErr := h.readStoredPackument(r.Context(), upstreamURL, accept, metadataCacheKey, func(entry *database.MetadataCacheEntry) (err error) { rewritten, err = h.proxy.streamedRewrite(r.Context(), "npm", h.proxyURL, metadataCacheKey, entry, func(open func() (io.ReadCloser, error), out io.Writer) error { var scan *npm.Scan err := readStored(open, func(in io.Reader) (err error) { @@ -409,27 +409,24 @@ func (h *NPMHandler) writeFetchError(w http.ResponseWriter, err error) { // document can't be read as a stream. var errInMemory = errors.New("npm packument needs the in-memory path") -// storedPackument returns the cache row of the packument stored for cacheKey, -// fetching it into storage first when needed. It returns errInMemory when +// storedPackument returns the lookup of the packument stored for cacheKey, +// fetching it into storage first when needed. Its err is errInMemory when // metadata isn't cached in storage, and ErrUpstreamNotFound and upstream // errors as they come. // With refresh, a stored copy within its TTL is fetched again rather than // served. -func (h *NPMHandler) storedPackument(ctx context.Context, packumentURL, accept, cacheKey string, refresh bool) (*database.MetadataCacheEntry, error) { +func (h *NPMHandler) storedPackument(ctx context.Context, packumentURL, accept, cacheKey string, refresh bool) metadataResult { + var gone string if refresh && h.proxy.DB != nil { if row, _ := h.proxy.DB.GetMetadataCache("npm", cacheKey); usableStoredMetadata(row) { - entry, err := h.proxy.restoreStoredMetadata(ctx, "npm", cacheKey, packumentURL, accept, h.packumentStreamValidator(), row) - if errors.Is(err, errMetadataNotStored) { - return nil, errInMemory - } - return entry, err + gone = row.ContentDigest.String } } - entry, err := h.proxy.storedMetadataDocument(ctx, "npm", cacheKey, packumentURL, accept, h.packumentStreamValidator()) - if errors.Is(err, errMetadataNotStored) { - return nil, errInMemory + res := h.proxy.storedMetadata(ctx, "npm", cacheKey, packumentURL, accept, h.packumentStreamValidator(), gone) + if errors.Is(res.err, errMetadataNotStored) { + res.err = errInMemory } - return entry, err + return res } // readStoredPackument reads the packument stored for cacheKey with read, @@ -437,23 +434,31 @@ func (h *NPMHandler) storedPackument(ctx context.Context, packumentURL, accept, // or no longer has the digest its row records, the packument is fetched // again and read once more. fetchErr is what storedPackument returned when the packument // couldn't be fetched, and err is what read returned. -func (h *NPMHandler) readStoredPackument(ctx context.Context, packumentURL, accept, cacheKey string, refresh bool, read func(*database.MetadataCacheEntry) error) (err, fetchErr error) { - entry, fetchErr := h.storedPackument(ctx, packumentURL, accept, cacheKey, refresh) - if fetchErr != nil { - return nil, fetchErr +func (h *NPMHandler) readStoredPackument(ctx context.Context, packumentURL, accept, cacheKey string, read func(*database.MetadataCacheEntry) error) (err, fetchErr error) { + fetched, err := h.lookupStoredPackument(ctx, packumentURL, accept, cacheKey, false, read) + return err, fetched.err +} + +// lookupStoredPackument implements readStoredPackument. fetched is the +// lookup of the packument read, whose err is set when it couldn't be +// fetched. +func (h *NPMHandler) lookupStoredPackument(ctx context.Context, packumentURL, accept, cacheKey string, refresh bool, read func(*database.MetadataCacheEntry) error) (fetched metadataResult, err error) { + fetched = h.storedPackument(ctx, packumentURL, accept, cacheKey, refresh) + if fetched.err != nil { + return fetched, nil } - if err = read(entry); !errors.Is(err, storage.ErrNotFound) && !errors.Is(err, errStoredMetadataChanged) { - return err, nil + if err = read(fetched.entry); !errors.Is(err, storage.ErrNotFound) && !errors.Is(err, errStoredMetadataChanged) { + return fetched, err } h.proxy.Logger.Warn("stored npm metadata is missing or changed, fetching it again", "url", packumentURL, "error", err) - entry, fetchErr = h.proxy.restoreStoredMetadata(ctx, "npm", cacheKey, packumentURL, accept, h.packumentStreamValidator(), entry) - if errors.Is(fetchErr, errMetadataNotStored) { - return nil, errInMemory + fetched = h.proxy.storedMetadata(ctx, "npm", cacheKey, packumentURL, accept, h.packumentStreamValidator(), fetched.entry.ContentDigest.String) + if errors.Is(fetched.err, errMetadataNotStored) { + fetched.err = errInMemory } - if fetchErr != nil { - return nil, fetchErr + if fetched.err != nil { + return fetched, nil } - return read(entry), nil + return fetched, read(fetched.entry) } // packumentStreamValidator is packumentValidator for a packument read from @@ -486,7 +491,7 @@ func (h *NPMHandler) streamedVersionMetadata(ctx context.Context, packageName, s fullRequired := h.packumentValidator() != nil options := h.npmRewriteOptions(packageName, nil) var m rewrittenMetadata - err, fetchErr := h.readStoredPackument(ctx, packumentURL, accept, cacheKey, false, func(entry *database.MetadataCacheEntry) (err error) { + err, fetchErr := h.readStoredPackument(ctx, packumentURL, accept, cacheKey, func(entry *database.MetadataCacheEntry) (err error) { m, err = h.proxy.streamedRewrite(ctx, "npm", h.proxyURL, metadataCacheKey, entry, func(open func() (io.ReadCloser, error), out io.Writer) error { var scan *npm.Scan err := readStored(open, func(in io.Reader) (err error) { @@ -910,19 +915,24 @@ func (h *NPMHandler) versionData(ctx context.Context, packageName, packumentURL, } // readVersionData implements versionData. fromCache reports whether the -// packument may have been a cached copy; refresh fetches it from upstream. +// packument was a cached copy, within its TTL or served stale; refresh +// fetches it from upstream. func (h *NPMHandler) readVersionData(ctx context.Context, packumentURL, cacheKey, version string, refresh bool) (data npmVersionData, fromCache bool, err error) { accept := h.packumentAccept() var found npm.VersionEntry - err, fetchErr := h.readStoredPackument(ctx, packumentURL, accept, cacheKey, refresh, func(entry *database.MetadataCacheEntry) error { + fetched, err := h.lookupStoredPackument(ctx, packumentURL, accept, cacheKey, refresh, func(entry *database.MetadataCacheEntry) error { return readStored(func() (io.ReadCloser, error) { return h.proxy.openStoredMetadata(ctx, entry) }, func(in io.Reader) (err error) { found, err = npm.LookupVersion(in, version) return err }) }) + fetchErr := fetched.err switch { case fetchErr == nil && err == nil && (!found.Abbreviated || h.packumentValidator() == nil): - return npmVersionData{entry: found.Entry, published: found.Published}, !refresh, nil + if refresh && fetched.stale { + return npmVersionData{}, false, errRefetchFailed(fetched) + } + return npmVersionData{entry: found.Entry, published: found.Published}, fetched.fromCache || fetched.stale, nil case fetchErr == nil && err != nil: h.proxy.Logger.Warn("failed to read stored npm metadata, reading it in memory", "url", packumentURL, "error", err) case fetchErr != nil && !errors.Is(fetchErr, errInMemory): @@ -938,16 +948,21 @@ func (h *NPMHandler) readVersionData(ctx context.Context, packumentURL, cacheKey if res.err != nil { return npmVersionData{}, false, res.err } - // The cached copy is what lacked the version, so it can't decide it. if refresh && res.stale { - return npmVersionData{}, false, errors.New("npm metadata could not be fetched again") + return npmVersionData{}, false, errRefetchFailed(res) } entry, _ := jsonscan.Lookup(res.body, "versions", version) published, _, err := jsonscan.LookupString(res.body, "time", version) if err != nil { return npmVersionData{}, false, fmt.Errorf("parsing npm metadata: %w", err) } - return npmVersionData{entry: entry, published: published}, res.fromCache, nil + return npmVersionData{entry: entry, published: published}, res.fromCache || res.stale, nil +} + +// errRefetchFailed is the error for a refetch that got only the stale copy +// back. That copy is what lacked the version, so it can't decide it. +func errRefetchFailed(res metadataResult) error { + return fmt.Errorf("npm metadata could not be fetched again: %w", res.upstreamErr) } func validateTarballURL(raw, registryURL string) (string, error) { diff --git a/internal/handler/npm_tarball_test.go b/internal/handler/npm_tarball_test.go index d844dd5..6bca0f6 100644 --- a/internal/handler/npm_tarball_test.go +++ b/internal/handler/npm_tarball_test.go @@ -1,6 +1,7 @@ package handler import ( + "context" "encoding/json" "fmt" "io" @@ -13,6 +14,7 @@ import ( "time" "github.com/git-pkgs/cooldown" + registryclient "github.com/git-pkgs/registries/client" "github.com/git-pkgs/registries/fetch" ) @@ -258,3 +260,167 @@ func TestNPMTarballCooldownRefetchesStalePackument(t *testing.T) { }) } } + +// widgetPackument is a packument listing only widget 1.0.0, published long +// ago. +func widgetPackument() string { + old := time.Now().Add(-60 * 24 * time.Hour).Format(time.RFC3339) + return fmt.Sprintf(`{"name":"widget","time":{"1.0.0":%q},"versions":{"1.0.0":{"dist":{}}}}`, old) +} + +// cooldownNPMHandler returns an npm handler for upstream with cooldown on +// and a tarball fetcher that would serve anything asked of it. +func cooldownNPMHandler(t *testing.T, upstream *httptest.Server, ttl time.Duration) *NPMHandler { + t.Helper() + p, _, _, fetcher := setupTestProxy(t) + p.CacheMetadata = true + p.MetadataTTL = ttl + p.Cooldown = &cooldown.Config{Default: "7d"} + p.HTTPClient = upstream.Client() + fetcher.artifact = &fetch.Artifact{Body: io.NopCloser(strings.NewReader("package"))} + return NewNPMHandler(p, "http://proxy.test", upstream.URL) +} + +// TestNPMTarballCooldownStalePackument checks that a stale packument served +// because upstream failed doesn't let through a version it doesn't list, +// and that the upstream failure decides the response. +func TestNPMTarballCooldownStalePackument(t *testing.T) { + for _, tc := range []struct { + name string + status int + limited bool + want int + }{ + {"upstream down", http.StatusServiceUnavailable, false, http.StatusBadGateway}, + {"not found", http.StatusNotFound, false, http.StatusNotFound}, + {"rate limited", http.StatusTooManyRequests, true, http.StatusServiceUnavailable}, + } { + t.Run(tc.name, func(t *testing.T) { + var down atomic.Bool + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + if down.Load() { + w.Header().Set("Retry-After", "60") + w.WriteHeader(tc.status) + return + } + _, _ = io.WriteString(w, widgetPackument()) + })) + t.Cleanup(upstream.Close) + // With no TTL the packument is read in memory and every read + // asks upstream. + h := cooldownNPMHandler(t, upstream, 0) + if tc.limited { + gate := registryclient.NewRateLimitGate(http.DefaultTransport) + h.proxy.HTTPClient = &http.Client{Transport: gate} + h.proxy.RateLimits = gate + } + + metadata := httptest.NewRecorder() + h.Routes().ServeHTTP(metadata, httptest.NewRequest(http.MethodGet, "/widget", nil)) + if metadata.Code != http.StatusOK { + t.Fatalf("metadata status = %d: %s", metadata.Code, metadata.Body.String()) + } + down.Store(true) + + response := httptest.NewRecorder() + h.Routes().ServeHTTP(response, httptest.NewRequest(http.MethodGet, "/widget/-/widget-2.0.0.tgz", nil)) + if response.Code != tc.want { + t.Fatalf("status = %d, want %d: %s", response.Code, tc.want, response.Body.String()) + } + if tc.limited && response.Header().Get("Retry-After") == "" { + t.Error("no Retry-After on a rate-limited response") + } + }) + } +} + +// TestNPMTarballCooldownFetchesMissingVersionOnce checks that a packument +// just fetched from upstream isn't fetched again because it doesn't list the +// requested version. +func TestNPMTarballCooldownFetchesMissingVersionOnce(t *testing.T) { + var requests atomic.Int32 + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + requests.Add(1) + _, _ = io.WriteString(w, widgetPackument()) + })) + t.Cleanup(upstream.Close) + h := cooldownNPMHandler(t, upstream, time.Hour) + + response := httptest.NewRecorder() + h.Routes().ServeHTTP(response, httptest.NewRequest(http.MethodGet, "/widget/-/widget-99.0.0.tgz", nil)) + if n := requests.Load(); n != 1 { + t.Errorf("upstream packument requests = %d, want 1", n) + } +} + +// TestNPMTarballCooldownRefreshIgnoresSharedStale checks that a refetch of a +// stored packument lacking the version, joining another caller's fetch that +// falls back to the stored copy, doesn't take that copy as upstream's answer. +func TestNPMTarballCooldownRefreshIgnoresSharedStale(t *testing.T) { + var blocked atomic.Bool + arrived := make(chan struct{}, 1) + release := make(chan struct{}) + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if !blocked.Load() { + _, _ = io.WriteString(w, widgetPackument()) + return + } + arrived <- struct{}{} + select { + case <-release: + case <-r.Context().Done(): + } + w.WriteHeader(http.StatusServiceUnavailable) + })) + t.Cleanup(upstream.Close) + h := cooldownNPMHandler(t, upstream, time.Hour) + p := h.proxy + + metadata := httptest.NewRecorder() + h.Routes().ServeHTTP(metadata, httptest.NewRequest(http.MethodGet, "/widget", nil)) + if metadata.Code != http.StatusOK { + t.Fatalf("metadata status = %d: %s", metadata.Code, metadata.Body.String()) + } + registryURL, cacheKey := h.sourceFor("widget") + packumentURL := registryURL + "/widget" + accept := h.packumentAccept() + row, err := p.DB.GetMetadataCache("npm", cacheKey) + if err != nil || row == nil { + t.Fatalf("no stored packument row: %v", err) + } + setFetchedAt := func(at time.Time) { + t.Helper() + if ok, err := p.DB.RefreshMetadataCache("npm", cacheKey, row.ContentDigest.String, at); err != nil || !ok { + t.Fatalf("setting fetched_at: %t, %v", ok, err) + } + } + + // Another caller revalidates the expired copy, and upstream holds it. + setFetchedAt(time.Now().Add(-2 * time.Hour)) + blocked.Store(true) + other := make(chan error, 1) + go func() { + _, err := p.storedMetadataDocument(context.Background(), "npm", cacheKey, packumentURL, accept, h.packumentStreamValidator()) + other <- err + }() + <-arrived + + // The tarball request reads the copy as within its TTL and refetches it, + // joining that fetch. + setFetchedAt(time.Now()) + response := httptest.NewRecorder() + done := make(chan struct{}) + go func() { + defer close(done) + h.Routes().ServeHTTP(response, httptest.NewRequest(http.MethodGet, "/widget/-/widget-2.0.0.tgz", nil)) + }() + waitForMetadataWaiters(t, p, metadataCoalesceKey("npm", cacheKey, packumentURL, accept, "", false)+"\x00stored", 1) + close(release) + <-done + if err := <-other; err != nil { + t.Fatalf("other caller: %v", err) + } + if response.Code != http.StatusBadGateway { + t.Fatalf("status = %d, want %d: %s", response.Code, http.StatusBadGateway, response.Body.String()) + } +} From fede2463f7a9fa54ae5419a7a34a989575f67c83 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 8 Nov 2026 10:10:43 +0100 Subject: [PATCH 618/633] Describe conda shard timestamps as condaTime reads them --- internal/handler/conda_shards.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/internal/handler/conda_shards.go b/internal/handler/conda_shards.go index 01434e4..8269f1d 100644 --- a/internal/handler/conda_shards.go +++ b/internal/handler/conda_shards.go @@ -378,7 +378,8 @@ func rewriteCondaShardIndex(body []byte, indexURL string, mask [condaShardHashLe } // condaShardRecord holds the fields cooldown needs from a shard record. -// Timestamps are milliseconds, encoded as an integer or a float. +// Timestamps are an integer or a float, in seconds or milliseconds as +// condaTime reads them. type condaShardRecord struct { Name string `msgpack:"name"` Version string `msgpack:"version"` From 6d80291d22334e14fb928d263c7848b56499e402 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 8 Nov 2026 10:11:36 +0100 Subject: [PATCH 619/633] Warn when mirroring a cached tag after a failed refresh A tag found in the cache after the refresh failed was reported as mirrored with no sign it might be stale, though tags can move. Keep serving it, as a cached commit is, but log a warning naming the tag, the commit and the refresh error. --- docs/configuration.md | 2 +- internal/gitcache/mirror.go | 4 ++++ internal/gitcache/mirror_test.go | 20 ++++++++++++++++++++ 3 files changed, 25 insertions(+), 1 deletion(-) diff --git a/docs/configuration.md b/docs/configuration.md index 0d81646..59e5960 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1920,7 +1920,7 @@ proxy mirror pkg:git/github.com/git-pkgs/purl - The repository is served by the `upstream.git` host whose URL its host and path start with, the longest one if several do, and has to match that host's `repos`. Others fail. - The repository's branches and tags are cached, as a clone through the proxy caches them. A commit on none of them is fetched by ID and kept, like one a client fetches by ID, until a branch or tag includes it, or for seven days from the last mirror run that named it, as long as `git.max_size` doesn't evict the repository. Where `upstream.auth` has credentials for the host, commits aren't fetched by ID, and one on no branch or tag that isn't cached yet fails. -- Without a version the repository's branches and tags are mirrored. A tag as the version has to be one of them, so a tag the upstream doesn't have fails. A branch name or an abbreviated commit ID as the version fails too; give the full commit ID. +- Without a version the repository's branches and tags are mirrored. A tag as the version has to be one of them, so a tag the upstream doesn't have fails. When the upstream can't be reached, a cached tag is mirrored at the commit it last pointed to, with a warning in the log. A branch name or an abbreviated commit ID as the version fails too; give the full commit ID. - With a version, the submodules that commit records are mirrored at the commits it pins, with their submodules in turn. A relative submodule URL is resolved against the repository's, and an `ssh://`, `git://` or `git@host:path` URL names the repository by its host and path. A submodule no `upstream.git` host serves is skipped with a warning in the log, as are all of a commit's submodules when git can't read its `.gitmodules`. - The git cache is used by one process. While the server runs, the command can't open `git.cache_dir`, so `pkg:git` items fail and the rest are mirrored; use the mirror API instead. diff --git a/internal/gitcache/mirror.go b/internal/gitcache/mirror.go index f884d37..ffcd0d9 100644 --- a/internal/gitcache/mirror.go +++ b/internal/gitcache/mirror.go @@ -87,6 +87,10 @@ func (c *Cache) Mirror(ctx context.Context, key, repoURL, version string, fetchW } return MirrorResult{}, err } + if refreshErr != nil { + // Tags can move; this is where it pointed when last fetched. + c.logger.Warn("git cache: refresh failed, mirroring the cached tag", "repository", key, "tag", version, "commit", commit, "error", refreshErr) + } case version != "": if err := c.ensureCommit(ctx, rp, repoURL, commit, fetchWants); err != nil { return MirrorResult{}, err diff --git a/internal/gitcache/mirror_test.go b/internal/gitcache/mirror_test.go index b1e27f6..17a19f0 100644 --- a/internal/gitcache/mirror_test.go +++ b/internal/gitcache/mirror_test.go @@ -3,6 +3,7 @@ package gitcache import ( "context" "errors" + "net/http" "os" "path/filepath" "slices" @@ -389,3 +390,22 @@ func TestMirrorExplainsShortCommit(t *testing.T) { t.Errorf("mirror of a short commit ID = %v, want ErrNotFound saying it is no full commit ID", err) } } + +// TestMirrorWarnsOfStaleTag checks a tag served from the cache when the +// refresh fails is mirrored with a warning, since the tag may have moved. +func TestMirrorWarnsOfStaleTag(t *testing.T) { + upstream := newFakeUpstream(t) + upstream.newRepo("repo") + tc := newTestCache(t, upstream.server.URL, nil) + if _, err := tc.Mirror(context.Background(), "up/repo.git", upstream.server.URL+"/repo", "v1", true); err != nil { + t.Fatalf("Mirror: %v", err) + } + upstream.status.Store(http.StatusServiceUnavailable) + + if _, err := tc.Mirror(context.Background(), "up/repo.git", upstream.server.URL+"/repo", "v1", true); err != nil { + t.Fatalf("mirror of a cached tag with the upstream down: %v", err) + } + if !strings.Contains(tc.logs.String(), "tag=v1") { + t.Errorf("no warning naming the tag in the log:\n%s", tc.logs.String()) + } +} From 0ff118b6420ee207c9bc8f8ce18090332635c649 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 8 Nov 2026 10:57:38 +0100 Subject: [PATCH 620/633] Try the Plugin Portal for Maven paths proxied uncached Paths the proxy doesn't cache, such as ones with a dotted group directory, went to the primary upstream only, so a file available only on the Gradle Plugin Portal returned 404. They now try the Plugin Portal on a 404, as cached files do. --- internal/handler/handler.go | 9 +++++++++ internal/handler/maven.go | 22 ++++++++++++++++++++- internal/handler/maven_repositories_test.go | 15 ++++++++++++++ 3 files changed, 45 insertions(+), 1 deletion(-) diff --git a/internal/handler/handler.go b/internal/handler/handler.go index d316aab..dc54269 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -1481,6 +1481,15 @@ func copyArtifactBody(w http.ResponseWriter, reader io.Reader, expectedSize int6 // It copies the request, forwards specified headers, and streams the response back. // forwardHeaders controls the request headers sent upstream. End-to-end response // headers and trailers are relayed independently of that list. +// upstreamRequest sends r's method to upstreamURL with no request headers. +func (p *Proxy) upstreamRequest(r *http.Request, upstreamURL string) (*http.Response, error) { + req, err := http.NewRequestWithContext(r.Context(), r.Method, upstreamURL, nil) + if err != nil { + return nil, err + } + return p.HTTPClient.Do(req) +} + func (p *Proxy) ProxyUpstream(w http.ResponseWriter, r *http.Request, upstreamURL string, forwardHeaders []string) { p.Logger.Debug("proxying to upstream", "url", upstreamURL) diff --git a/internal/handler/maven.go b/internal/handler/maven.go index 0fd81ec..ed417d7 100644 --- a/internal/handler/maven.go +++ b/internal/handler/maven.go @@ -343,5 +343,25 @@ func (h *MavenHandler) isChecksumFile(filename string) bool { // proxyUpstream forwards a request to the source's upstream without caching. func (h *MavenHandler) proxyUpstream(w http.ResponseWriter, r *http.Request, source mavenSource, urlPath string) { - h.proxy.ProxyUpstream(w, r, source.upstreamURL+"/"+urlPath, nil) + if source.fallbackURL == "" { + h.proxy.ProxyUpstream(w, r, source.upstreamURL+"/"+urlPath, nil) + return + } + // As for cached files, a file the primary upstream lacks is looked for + // on the Gradle Plugin Portal. + resp, err := h.proxy.upstreamRequest(r, source.upstreamURL+"/"+urlPath) + if err == nil && resp.StatusCode == http.StatusNotFound { + _ = resp.Body.Close() + resp, err = h.proxy.upstreamRequest(r, source.fallbackURL+"/"+urlPath) + } + if err != nil { + if writeUpstreamHeld(w, err) { + return + } + h.proxy.Logger.Error("upstream request failed", "error", err) + http.Error(w, "upstream request failed", http.StatusBadGateway) + return + } + defer func() { _ = resp.Body.Close() }() + h.proxy.relayResponse(w, r, resp, nil) } diff --git a/internal/handler/maven_repositories_test.go b/internal/handler/maven_repositories_test.go index 15db066..8fb6f27 100644 --- a/internal/handler/maven_repositories_test.go +++ b/internal/handler/maven_repositories_test.go @@ -156,3 +156,18 @@ func TestMavenDottedGroupDirectoryNotCachedAsGroup(t *testing.T) { } } } + +// TestMavenUncachedPathFallsBackToPluginPortal checks that a path proxied +// without caching, such as one with a dotted group directory, still tries +// the Gradle Plugin Portal when the primary upstream lacks it. +func TestMavenUncachedPathFallsBackToPluginPortal(t *testing.T) { + const dotted = "/org.foo/bar/1.0/bar-1.0.jar" + central, _ := mavenRepoUpstream(t, "central") + portal, _ := mavenRepoUpstream(t, "portal", dotted) + grails, _ := mavenRepoUpstream(t, "grails") + h := newMavenRepositoriesHandler(t, central, portal, grails) + + if w := getMaven(h, dotted); w.Code != http.StatusOK || w.Body.String() != "portal "+dotted { + t.Errorf("GET %s = %d %q, want the Plugin Portal's copy", dotted, w.Code, w.Body.String()) + } +} From da090443eceb2f99e50f85b33a9a660b1d7eb662 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 8 Nov 2026 12:19:19 +0100 Subject: [PATCH 621/633] Forward Range to upstream for Maven paths proxied uncached Paths proxied without caching sent no request headers, so a resumed download got the whole file back instead of the requested bytes. --- internal/handler/handler.go | 33 ++++++++------------- internal/handler/maven.go | 9 ++++-- internal/handler/maven_repositories_test.go | 25 ++++++++++++++++ 3 files changed, 44 insertions(+), 23 deletions(-) diff --git a/internal/handler/handler.go b/internal/handler/handler.go index dc54269..6c3fef8 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -1477,36 +1477,29 @@ func copyArtifactBody(w http.ResponseWriter, reader io.Reader, expectedSize int6 } } -// ProxyUpstream forwards a request to an upstream URL without caching. -// It copies the request, forwards specified headers, and streams the response back. -// forwardHeaders controls the request headers sent upstream. End-to-end response -// headers and trailers are relayed independently of that list. -// upstreamRequest sends r's method to upstreamURL with no request headers. -func (p *Proxy) upstreamRequest(r *http.Request, upstreamURL string) (*http.Response, error) { +// upstreamRequest sends r's method to upstreamURL, copying only the +// forwardHeaders that r sets. +func (p *Proxy) upstreamRequest(r *http.Request, upstreamURL string, forwardHeaders []string) (*http.Response, error) { req, err := http.NewRequestWithContext(r.Context(), r.Method, upstreamURL, nil) if err != nil { return nil, err } - return p.HTTPClient.Do(req) -} - -func (p *Proxy) ProxyUpstream(w http.ResponseWriter, r *http.Request, upstreamURL string, forwardHeaders []string) { - p.Logger.Debug("proxying to upstream", "url", upstreamURL) - - req, err := http.NewRequestWithContext(r.Context(), r.Method, upstreamURL, nil) - if err != nil { - http.Error(w, "failed to create request", http.StatusInternalServerError) - return - } - - // Copy request headers that affect content negotiation / caching for _, header := range forwardHeaders { if v := r.Header.Get(header); v != "" { req.Header.Set(header, v) } } + return p.HTTPClient.Do(req) +} - resp, err := p.HTTPClient.Do(req) +// ProxyUpstream forwards a request to an upstream URL without caching. +// It copies the request, forwards specified headers, and streams the response back. +// forwardHeaders controls the request headers sent upstream. End-to-end response +// headers and trailers are relayed independently of that list. +func (p *Proxy) ProxyUpstream(w http.ResponseWriter, r *http.Request, upstreamURL string, forwardHeaders []string) { + p.Logger.Debug("proxying to upstream", "url", upstreamURL) + + resp, err := p.upstreamRequest(r, upstreamURL, forwardHeaders) if err != nil { if writeUpstreamHeld(w, err) { return diff --git a/internal/handler/maven.go b/internal/handler/maven.go index ed417d7..4b170be 100644 --- a/internal/handler/maven.go +++ b/internal/handler/maven.go @@ -341,18 +341,21 @@ func (h *MavenHandler) isChecksumFile(filename string) bool { return false } +// mavenForwardHeaders lets clients resume artifacts proxied without caching. +var mavenForwardHeaders = []string{"Range", "If-Range"} + // proxyUpstream forwards a request to the source's upstream without caching. func (h *MavenHandler) proxyUpstream(w http.ResponseWriter, r *http.Request, source mavenSource, urlPath string) { if source.fallbackURL == "" { - h.proxy.ProxyUpstream(w, r, source.upstreamURL+"/"+urlPath, nil) + h.proxy.ProxyUpstream(w, r, source.upstreamURL+"/"+urlPath, mavenForwardHeaders) return } // As for cached files, a file the primary upstream lacks is looked for // on the Gradle Plugin Portal. - resp, err := h.proxy.upstreamRequest(r, source.upstreamURL+"/"+urlPath) + resp, err := h.proxy.upstreamRequest(r, source.upstreamURL+"/"+urlPath, mavenForwardHeaders) if err == nil && resp.StatusCode == http.StatusNotFound { _ = resp.Body.Close() - resp, err = h.proxy.upstreamRequest(r, source.fallbackURL+"/"+urlPath) + resp, err = h.proxy.upstreamRequest(r, source.fallbackURL+"/"+urlPath, mavenForwardHeaders) } if err != nil { if writeUpstreamHeld(w, err) { diff --git a/internal/handler/maven_repositories_test.go b/internal/handler/maven_repositories_test.go index 8fb6f27..260d3a8 100644 --- a/internal/handler/maven_repositories_test.go +++ b/internal/handler/maven_repositories_test.go @@ -7,6 +7,7 @@ import ( "strings" "sync/atomic" "testing" + "time" "github.com/git-pkgs/registries/fetch" ) @@ -171,3 +172,27 @@ func TestMavenUncachedPathFallsBackToPluginPortal(t *testing.T) { t.Errorf("GET %s = %d %q, want the Plugin Portal's copy", dotted, w.Code, w.Body.String()) } } + +func TestMavenUncachedPathKeepsByteRanges(t *testing.T) { + const dotted = "/org.foo/bar/1.0/bar-1.0.jar" + ranged := func() *httptest.Server { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.ServeContent(w, r, "bar.jar", time.Time{}, strings.NewReader("0123456789")) + })) + t.Cleanup(srv.Close) + return srv + } + central, grails := ranged(), ranged() + portal, _ := mavenRepoUpstream(t, "portal") + h := newMavenRepositoriesHandler(t, central, portal, grails) + + for _, p := range []string{dotted, "/grails" + dotted} { + req := httptest.NewRequest(http.MethodGet, p, nil) + req.Header.Set("Range", "bytes=2-4") + w := httptest.NewRecorder() + h.ServeHTTP(w, req) + if w.Code != http.StatusPartialContent || w.Body.String() != "234" { + t.Errorf("GET %s with Range = %d %q, want 206 %q", p, w.Code, w.Body.String(), "234") + } + } +} From 0829eae376be23dedc391d2f73f4eec2965b16d3 Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Sun, 8 Nov 2026 17:19:03 +0000 Subject: [PATCH 622/633] List only the gitlinks .gitmodules names ls-tree -r read every path in the commit to find a few submodules. --- internal/gitcache/mirror.go | 12 +++++++++++- internal/gitcache/mirror_test.go | 5 ++++- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/internal/gitcache/mirror.go b/internal/gitcache/mirror.go index ffcd0d9..0d76a46 100644 --- a/internal/gitcache/mirror.go +++ b/internal/gitcache/mirror.go @@ -172,7 +172,17 @@ func (c *Cache) submodules(ctx context.Context, key, dir, commit string) ([]Subm urls[name] = value } } - tree, err := c.run(ctx, dir, "ls-tree", "-r", "-z", "--full-tree", commit) + var pathspecs []string + for path, name := range paths { + if path != "" && urls[name] != "" { + pathspecs = append(pathspecs, path) + } + } + if len(pathspecs) == 0 { + return nil, nil + } + args := append([]string{"--literal-pathspecs", "ls-tree", "-z", "--full-tree", commit, "--"}, pathspecs...) + tree, err := c.run(ctx, dir, args...) if err != nil { return nil, err } diff --git a/internal/gitcache/mirror_test.go b/internal/gitcache/mirror_test.go index 17a19f0..ca734f1 100644 --- a/internal/gitcache/mirror_test.go +++ b/internal/gitcache/mirror_test.go @@ -261,7 +261,8 @@ func TestMirrorListsSubmodules(t *testing.T) { libCommit := runGit(t, libWork, "rev-parse", "HEAD") work := upstream.newRepo("repo") gitmodules := "[submodule \"vendor/lib\"]\n\tpath = vendor/lib\n\turl = ../lib.git\n" + - "[submodule \"other\"]\n\tpath = third_party/other\n\turl = https://example.com/other.git\n" + "[submodule \"other\"]\n\tpath = third_party/other\n\turl = https://example.com/other.git\n" + + "[submodule \"colon\"]\n\tpath = :colon\n\turl = ../lib.git\n" if err := os.WriteFile(filepath.Join(work, ".gitmodules"), []byte(gitmodules), 0o600); err != nil { t.Fatal(err) } @@ -269,6 +270,7 @@ func TestMirrorListsSubmodules(t *testing.T) { runGit(t, work, "add", ".gitmodules") runGit(t, work, "update-index", "--add", "--cacheinfo", "160000,"+libCommit+",vendor/lib") runGit(t, work, "update-index", "--add", "--cacheinfo", "160000,"+otherCommit+",third_party/other") + runGit(t, work, "update-index", "--add", "--cacheinfo", "160000,"+libCommit+",:colon") runGit(t, work, "commit", "--quiet", "-m", "submodules") runGit(t, work, "push", "--quiet", "origin", "main") commit := runGit(t, work, "rev-parse", "HEAD") @@ -279,6 +281,7 @@ func TestMirrorListsSubmodules(t *testing.T) { t.Fatalf("Mirror: %v", err) } want := []Submodule{ + {Path: ":colon", URL: "../lib.git", Commit: libCommit}, {Path: "third_party/other", URL: "https://example.com/other.git", Commit: otherCommit}, {Path: "vendor/lib", URL: "../lib.git", Commit: libCommit}, } From 1b22f79be2d0021d2bdb41faf0419f7511e57e34 Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Sun, 8 Nov 2026 17:23:11 +0000 Subject: [PATCH 623/633] Fail the conda timestamp test when the index is empty --- internal/handler/conda_test.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/internal/handler/conda_test.go b/internal/handler/conda_test.go index 63e518e..f6c37c6 100644 --- a/internal/handler/conda_test.go +++ b/internal/handler/conda_test.go @@ -166,6 +166,9 @@ func TestCondaCooldownSecondTimestamps(t *testing.T) { if err != nil { t.Fatal(err) } + if len(index) != 2 { + t.Fatalf("index has %d entries, want 2", len(index)) + } for filename, published := range index { if published.Unix() != recent.Unix() { t.Errorf("%s published at %v, want %v", filename, published, recent) From 2ecdcced4ca56de5f1e1287bd509d0e001fbb7b4 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 8 Nov 2026 18:51:00 +0100 Subject: [PATCH 624/633] Search again only for the replacement just made in the PyPI root index Each match searched the rest of the buffer for every replacement. The root index has no single-quoted links, so after every double-quoted one the other pattern scanned to the end again, making each chunk quadratic in its links. Each replacement's next match is now kept and only searched for again once a replacement passes it (50,000 links: 18.6 to 818 MB/s). --- internal/handler/pypi.go | 46 ++++++++++++++++--------- internal/handler/pypi_test.go | 63 +++++++++++++++++++++++++++++++++++ 2 files changed, 93 insertions(+), 16 deletions(-) diff --git a/internal/handler/pypi.go b/internal/handler/pypi.go index 50f9964..a1bee9d 100644 --- a/internal/handler/pypi.go +++ b/internal/handler/pypi.go @@ -1066,18 +1066,6 @@ type replacingReader struct { type replacement struct{ from, to []byte } -// next finds the earliest replacement in b, returning its index or -1. -func (r *replacingReader) next(b []byte) (int, *replacement) { - at, found := -1, (*replacement)(nil) - for i := range r.replacements { - rep := &r.replacements[i] - if j := bytes.Index(b, rep.from); j >= 0 && (at < 0 || j < at) { - at, found = j, rep - } - } - return at, found -} - func (r *replacingReader) Read(p []byte) (int, error) { for len(r.out) == 0 { if r.err != nil { @@ -1099,14 +1087,32 @@ func (r *replacingReader) Read(p []byte) (int, error) { scan := max(len(r.in)-keep, 0) var out []byte done := 0 + // at holds each replacement's next match at or after done, or -1. + // Only a match the last replacement consumed is searched for again, + // so each pass over r.in stays linear. + at := make([]int, len(r.replacements)) + for i := range at { + at[i] = indexFrom(r.in, r.replacements[i].from, 0) + } for { - i, rep := r.next(r.in[done:]) - if i < 0 || done+i >= scan { + k := -1 + for i, j := range at { + if j >= 0 && (k < 0 || j < at[k]) { + k = i + } + } + if k < 0 || at[k] >= scan { break } - out = append(out, r.in[done:done+i]...) + rep := &r.replacements[k] + out = append(out, r.in[done:at[k]]...) out = append(out, rep.to...) - done += i + len(rep.from) + done = at[k] + len(rep.from) + for i, j := range at { + if j >= 0 && j < done { + at[i] = indexFrom(r.in, r.replacements[i].from, done) + } + } } if done < scan { out = append(out, r.in[done:scan]...) @@ -1121,3 +1127,11 @@ func (r *replacingReader) Read(p []byte) (int, error) { } func (r *replacingReader) Close() error { return r.src.Close() } + +// indexFrom returns the index of the first sep in b at or after from, or -1. +func indexFrom(b, sep []byte, from int) int { + if i := bytes.Index(b[from:], sep); i >= 0 { + return from + i + } + return -1 +} diff --git a/internal/handler/pypi_test.go b/internal/handler/pypi_test.go index c34679d..1b5cbaa 100644 --- a/internal/handler/pypi_test.go +++ b/internal/handler/pypi_test.go @@ -5,6 +5,7 @@ import ( "fmt" "io" "log/slog" + "math/rand/v2" "net/http" "net/http/httptest" "strconv" @@ -678,3 +679,65 @@ func TestReplacingReaderSplitReads(t *testing.T) { t.Errorf("got %q, %v; want %q", got, err, want) } } + +// replaceReference makes the replacements the way replacingReader should: +// left to right, the earliest match first, and the first listed on a tie. +func replaceReference(in string, reps []replacement) string { + var out strings.Builder + for len(in) > 0 { + at, found := -1, -1 + for i, rep := range reps { + if j := strings.Index(in, string(rep.from)); j >= 0 && (at < 0 || j < at) { + at, found = j, i + } + } + if at < 0 { + out.WriteString(in) + break + } + out.WriteString(in[:at]) + out.Write(reps[found].to) + in = in[at+len(reps[found].from):] + } + return out.String() +} + +func TestReplacingReaderMatchesReference(t *testing.T) { + reps := []replacement{ + {from: []byte(`href="/s/`), to: []byte(`href="P/`)}, + {from: []byte(`href='/s/`), to: []byte(`href='Q/`)}, + } + pieces := []string{`href="/s/`, `href='/s/`, `href="/s`, `href=`, `x`, `/s/`, `'`, `"`} + rng := rand.New(rand.NewPCG(1, 2)) + for range 500 { + var b strings.Builder + for range rng.IntN(40) { + b.WriteString(pieces[rng.IntN(len(pieces))]) + } + in := b.String() + for _, src := range []io.Reader{strings.NewReader(in), iotest.OneByteReader(strings.NewReader(in))} { + got, err := io.ReadAll(&replacingReader{src: io.NopCloser(src), replacements: reps}) + if want := replaceReference(in, reps); err != nil || string(got) != want { + t.Fatalf("in %q: got %q, %v; want %q", in, got, err, want) + } + } + } +} + +// BenchmarkReplacingReaderRootIndex reads a root index of double-quoted +// links, where the single-quote pattern never matches. +func BenchmarkReplacingReaderRootIndex(b *testing.B) { + var page strings.Builder + for i := range 50000 { + fmt.Fprintf(&page, "project-%d\n", i, i) + } + in := page.String() + reps := []replacement{ + {from: []byte(`href="/simple/`), to: []byte(`href="http://proxy/pypi/simple/`)}, + {from: []byte(`href='/simple/`), to: []byte(`href='http://proxy/pypi/simple/`)}, + } + b.SetBytes(int64(len(in))) + for b.Loop() { + _, _ = io.Copy(io.Discard, &replacingReader{src: io.NopCloser(strings.NewReader(in)), replacements: reps}) + } +} From 140b70233a412fb54b3ddd0c84aa52966ff89fa3 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 8 Nov 2026 18:51:46 +0100 Subject: [PATCH 625/633] Move the PyPI root index replacement loop into its own method --- internal/handler/pypi.go | 63 ++++++++++++++++++++++------------------ 1 file changed, 34 insertions(+), 29 deletions(-) diff --git a/internal/handler/pypi.go b/internal/handler/pypi.go index a1bee9d..b6ffb50 100644 --- a/internal/handler/pypi.go +++ b/internal/handler/pypi.go @@ -1085,35 +1085,7 @@ func (r *replacingReader) Read(p []byte) (int, error) { } } scan := max(len(r.in)-keep, 0) - var out []byte - done := 0 - // at holds each replacement's next match at or after done, or -1. - // Only a match the last replacement consumed is searched for again, - // so each pass over r.in stays linear. - at := make([]int, len(r.replacements)) - for i := range at { - at[i] = indexFrom(r.in, r.replacements[i].from, 0) - } - for { - k := -1 - for i, j := range at { - if j >= 0 && (k < 0 || j < at[k]) { - k = i - } - } - if k < 0 || at[k] >= scan { - break - } - rep := &r.replacements[k] - out = append(out, r.in[done:at[k]]...) - out = append(out, rep.to...) - done = at[k] + len(rep.from) - for i, j := range at { - if j >= 0 && j < done { - at[i] = indexFrom(r.in, r.replacements[i].from, done) - } - } - } + out, done := r.replace(scan) if done < scan { out = append(out, r.in[done:scan]...) done = scan @@ -1126,6 +1098,39 @@ func (r *replacingReader) Read(p []byte) (int, error) { return n, nil } +// replace makes the replacements that start before scan in r.in, returning +// the output and how much of r.in it covers. +func (r *replacingReader) replace(scan int) (out []byte, done int) { + // at holds each replacement's next match at or after done, or -1. + // Only a match the last replacement consumed is searched for again, + // so each pass over r.in stays linear. + at := make([]int, len(r.replacements)) + for i := range at { + at[i] = indexFrom(r.in, r.replacements[i].from, 0) + } + for { + k := -1 + for i, j := range at { + if j >= 0 && (k < 0 || j < at[k]) { + k = i + } + } + if k < 0 || at[k] >= scan { + break + } + rep := &r.replacements[k] + out = append(out, r.in[done:at[k]]...) + out = append(out, rep.to...) + done = at[k] + len(rep.from) + for i, j := range at { + if j >= 0 && j < done { + at[i] = indexFrom(r.in, r.replacements[i].from, done) + } + } + } + return out, done +} + func (r *replacingReader) Close() error { return r.src.Close() } // indexFrom returns the index of the first sep in b at or after from, or -1. From a330d68564b6034aef3307ea9114e4eff412b7e8 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 8 Nov 2026 18:58:40 +0100 Subject: [PATCH 626/633] Use strings.CutLast for .gitmodules keys Go 1.27 is on main now. --- internal/gitcache/mirror.go | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/internal/gitcache/mirror.go b/internal/gitcache/mirror.go index c59e905..6e45d43 100644 --- a/internal/gitcache/mirror.go +++ b/internal/gitcache/mirror.go @@ -161,7 +161,7 @@ func (c *Cache) submodules(ctx context.Context, key, dir, commit string) ([]Subm for entry := range strings.SplitSeq(out, "\x00") { key, value, ok := strings.Cut(entry, "\n") section, ok2 := strings.CutPrefix(key, "submodule.") - name, field, dotted := cutLast(section, ".") + name, field, dotted := strings.CutLast(section, ".") if !ok || !ok2 || !dotted { continue } @@ -201,14 +201,6 @@ func (c *Cache) submodules(ctx context.Context, key, dir, commit string) ([]Subm return submodules, nil } -func cutLast(s, sep string) (before, after string, found bool) { - i := strings.LastIndex(s, sep) - if i < 0 { - return s, "", false - } - return s[:i], s[i+len(sep):], true -} - // ensureCommit makes sure the repository has commit, fetching it by ID when // it doesn't and fetchWants allows it. A commit on no branch or tag that is // cached already gets a new wanted ref, so it is kept for wantedMaxAge from From 44afe6997ab69ab70a53f3e85848e7e08bee1cf1 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 8 Nov 2026 19:03:21 +0100 Subject: [PATCH 627/633] Read jsonstream objects with jsontext CopyObject takes the reader and decodes it with jsontext, accepting repeated names and invalid UTF-8 as encoding/json did. Keys are written with the bytes they were read with instead of being re-encoded. Member gets Skip, which reads past a value without keeping it, and npm's RewriteStream skips dropped versions and duplicates with it. conda-forge noarch repodata.json (195 MB): filtering 0.89 to 0.62 s and 379 to 77 MB allocated, the timestamp index 0.84 to 0.61 s. The next packument (32 MB): RewriteStream 69 to 34 ms and 38 to 3.5 MB. --- internal/handler/conda.go | 4 +- internal/handler/conda_repodata_bench_test.go | 59 ++++++++++ internal/jsonstream/jsonstream.go | 103 ++++++++++-------- internal/jsonstream/jsonstream_test.go | 42 ++++++- internal/metadata/npm/stream.go | 16 ++- internal/metadata/npm/stream_bench_test.go | 28 +++++ 6 files changed, 195 insertions(+), 57 deletions(-) create mode 100644 internal/handler/conda_repodata_bench_test.go diff --git a/internal/handler/conda.go b/internal/handler/conda.go index f593820..c9db7a7 100644 --- a/internal/handler/conda.go +++ b/internal/handler/conda.go @@ -461,7 +461,7 @@ func (h *CondaHandler) filterRepodata(r io.Reader, w io.Writer) error { // filterRepodataWith is filterRepodata deciding by filter. func (h *CondaHandler) filterRepodataWith(r io.Reader, w io.Writer, filter *repodataFilter) error { - return jsonstream.CopyObject(json.NewDecoder(r), w, func(m *jsonstream.Member) error { + return jsonstream.CopyObject(r, w, func(m *jsonstream.Member) error { if m.Key != "packages" && m.Key != "packages.conda" { return m.Copy() } @@ -619,7 +619,7 @@ func (h *CondaHandler) streamRepodataIndex(r *http.Request, upstreamURL string) // is not limited by metadata_max_size. func readRepodataIndex(r io.Reader) (map[string]time.Time, error) { index := make(map[string]time.Time) - err := jsonstream.CopyObject(json.NewDecoder(io.LimitReader(r, maxRepodataSize)), io.Discard, func(m *jsonstream.Member) error { + err := jsonstream.CopyObject(io.LimitReader(r, maxRepodataSize), io.Discard, func(m *jsonstream.Member) error { if m.Key != "packages" && m.Key != "packages.conda" { return m.Copy() } diff --git a/internal/handler/conda_repodata_bench_test.go b/internal/handler/conda_repodata_bench_test.go new file mode 100644 index 0000000..61e68bd --- /dev/null +++ b/internal/handler/conda_repodata_bench_test.go @@ -0,0 +1,59 @@ +package handler + +import ( + "bytes" + "io" + "log/slog" + "os" + "testing" + "time" + + "github.com/git-pkgs/cooldown" +) + +// repodataBenchBody reads the repodata.json named by CONDA_REPODATA_BENCH, +// such as conda-forge's noarch one, and skips the benchmark without it. +func repodataBenchBody(b *testing.B) []byte { + b.Helper() + path := os.Getenv("CONDA_REPODATA_BENCH") + if path == "" { + b.Skip("CONDA_REPODATA_BENCH names no repodata.json") + } + body, err := os.ReadFile(path) + if err != nil { + b.Fatal(err) + } + return body +} + +func BenchmarkCondaFilterRepodata(b *testing.B) { + body := repodataBenchBody(b) + policy, err := cooldown.NewPolicy(&cooldown.Config{Default: "3d"}, nil) + if err != nil { + b.Fatal(err) + } + proxy := testProxy() + proxy.Logger = slog.New(slog.DiscardHandler) + proxy.Cooldown = policy + h := NewCondaHandler(proxy, "http://proxy.example") + now := time.Now() + b.SetBytes(int64(len(body))) + b.ReportAllocs() + for b.Loop() { + if err := h.filterRepodataWith(bytes.NewReader(body), io.Discard, h.newRepodataFilter(now)); err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkCondaReadRepodataIndex(b *testing.B) { + body := repodataBenchBody(b) + b.SetBytes(int64(len(body))) + b.ReportAllocs() + for b.Loop() { + index, err := readRepodataIndex(bytes.NewReader(body)) + if err != nil || len(index) == 0 { + b.Fatalf("index of %d files, %v", len(index), err) + } + } +} diff --git a/internal/jsonstream/jsonstream.go b/internal/jsonstream/jsonstream.go index 2d3d120..a515f11 100644 --- a/internal/jsonstream/jsonstream.go +++ b/internal/jsonstream/jsonstream.go @@ -5,7 +5,7 @@ package jsonstream import ( "bytes" - "encoding/json" + "encoding/json/jsontext" "errors" "fmt" "io" @@ -20,26 +20,32 @@ var ( ) // Member is one member of an object being copied. Its value must be consumed -// exactly once, with Raw, Copy or Object. +// exactly once, with Raw, Skip, Copy or Object. type Member struct { Key string - dec *json.Decoder + rawKey []byte + dec *jsontext.Decoder out *objectWriter consumed bool } // Raw reads the member's value without writing it. Follow it with Write to -// keep the member. -func (m *Member) Raw() (json.RawMessage, error) { +// keep the member. The bytes are only valid until the next member is read. +func (m *Member) Raw() ([]byte, error) { if m.consumed { return nil, errConsumed } m.consumed = true - var raw json.RawMessage - if err := m.dec.Decode(&raw); err != nil { - return nil, err + return m.dec.ReadValue() +} + +// Skip reads past the member's value, leaving the member out. +func (m *Member) Skip() error { + if m.consumed { + return errConsumed } - return raw, nil + m.consumed = true + return m.dec.SkipValue() } // Write writes the member with value as its raw JSON value. @@ -47,7 +53,7 @@ func (m *Member) Write(value []byte) error { if !m.consumed { return errUnconsumed } - if err := m.out.key(m.Key); err != nil { + if err := m.out.key(m.rawKey); err != nil { return err } _, err := m.out.w.Write(value) @@ -70,54 +76,66 @@ func (m *Member) Object(fn func(*Member) error) error { return errConsumed } m.consumed = true - if err := m.out.key(m.Key); err != nil { + if err := m.out.key(m.rawKey); err != nil { return err } return copyObject(m.dec, m.out.w, fn, true) } -// CopyObject copies the object read from dec to w, calling fn for each -// member. fn decides what is written by how it consumes the member's value; a -// member whose value fn leaves unconsumed is copied unchanged. -func CopyObject(dec *json.Decoder, w io.Writer, fn func(*Member) error) error { +// CopyObject copies the object read from r to w, calling fn for each member. +// fn decides what is written by how it consumes the member's value; a member +// whose value fn leaves unconsumed is copied unchanged. As with +// encoding/json, of repeated names each is passed to fn, and invalid UTF-8 +// is accepted. +func CopyObject(r io.Reader, w io.Writer, fn func(*Member) error) error { + dec := jsontext.NewDecoder(r, jsontext.AllowDuplicateNames(true), jsontext.AllowInvalidUTF8(true)) if err := copyObject(dec, w, fn, false); err != nil { return err } - if _, err := dec.Token(); err != io.EOF { - if err == nil { - err = errors.New("unexpected data after the top-level object") - } - return err + _, err := dec.ReadToken() + switch err { + case io.EOF: + return nil + case nil: + return errors.New("unexpected data after the top-level object") } - return nil + return err } -func copyObject(dec *json.Decoder, w io.Writer, fn func(*Member) error, allowNull bool) error { - tok, err := dec.Token() +func copyObject(dec *jsontext.Decoder, w io.Writer, fn func(*Member) error, allowNull bool) error { + tok, err := dec.ReadToken() if err != nil { return err } - if tok == nil && allowNull { + if tok.Kind() == 'n' && allowNull { _, err := io.WriteString(w, "null") return err } - if tok != json.Delim('{') { + if tok.Kind() != '{' { return ErrNotObject } if _, err := io.WriteString(w, "{"); err != nil { return err } out := &objectWriter{w: w} - for dec.More() { - tok, err := dec.Token() + m := &Member{dec: dec, out: out} + // keyBuf holds the current key; ReadValue's bytes don't outlast fn. + var keyBuf []byte + for dec.PeekKind() != '}' { + // The key is read raw, so it is written with its own bytes. + rawKey, err := dec.ReadValue() if err != nil { return err } - key, ok := tok.(string) - if !ok { - return fmt.Errorf("object key is %T, not a string", tok) + if rawKey.Kind() != '"' { + return fmt.Errorf("object key is %v, not a string", rawKey.Kind()) } - m := &Member{Key: key, dec: dec, out: out} + keyBuf = append(keyBuf[:0], rawKey...) + key, err := jsontext.AppendUnquote(nil, keyBuf) + if err != nil { + return err + } + *m = Member{Key: string(key), rawKey: keyBuf, dec: dec, out: out} if err := fn(m); err != nil { return err } @@ -127,7 +145,7 @@ func copyObject(dec *json.Decoder, w io.Writer, fn func(*Member) error, allowNul } } } - if _, err := dec.Token(); err != nil { + if _, err := dec.ReadToken(); err != nil { return err } _, err = io.WriteString(w, "}") @@ -137,23 +155,18 @@ func copyObject(dec *json.Decoder, w io.Writer, fn func(*Member) error, allowNul type objectWriter struct { w io.Writer wrote bool + buf bytes.Buffer } -// key writes the separator and key for the next member. Keys are re-encoded -// without HTML escaping, so plain keys keep their original bytes. -func (o *objectWriter) key(k string) error { - var buf bytes.Buffer +// key writes the separator and the key's bytes for the next member. +func (o *objectWriter) key(rawKey []byte) error { + o.buf.Reset() if o.wrote { - buf.WriteByte(',') + o.buf.WriteByte(',') } - enc := json.NewEncoder(&buf) - enc.SetEscapeHTML(false) - if err := enc.Encode(k); err != nil { - return err - } - buf.Truncate(buf.Len() - 1) // Encode appends a newline - buf.WriteByte(':') + o.buf.Write(rawKey) + o.buf.WriteByte(':') o.wrote = true - _, err := o.w.Write(buf.Bytes()) + _, err := o.w.Write(o.buf.Bytes()) return err } diff --git a/internal/jsonstream/jsonstream_test.go b/internal/jsonstream/jsonstream_test.go index 44a877e..e7bca69 100644 --- a/internal/jsonstream/jsonstream_test.go +++ b/internal/jsonstream/jsonstream_test.go @@ -4,6 +4,7 @@ import ( "bytes" "encoding/json" "errors" + "slices" "strings" "testing" ) @@ -11,7 +12,7 @@ import ( func copyString(t *testing.T, in string, fn func(*Member) error) (string, error) { t.Helper() var out bytes.Buffer - err := CopyObject(json.NewDecoder(strings.NewReader(in)), &out, fn) + err := CopyObject(strings.NewReader(in), &out, fn) return out.String(), err } @@ -140,3 +141,42 @@ func TestCopyObjectErrors(t *testing.T) { t.Errorf("err = %v, want ErrNotObject", err) } } + +// TestCopyObjectReadsJSONAsEncodingJSON checks CopyObject accepts what +// encoding/json does: repeated names, each passed to fn, and invalid UTF-8. +// Keys keep their bytes, escapes included, while Key is unquoted. +func TestCopyObjectReadsJSONAsEncodingJSON(t *testing.T) { + var keys []string + in := "{\"a\":1,\"a\":2,\"caf\\u00e9\":\"bad \xff byte\",\"skip\":{\"x\":[1,2]}}" + got, err := copyString(t, in, func(m *Member) error { + keys = append(keys, m.Key) + if m.Key == "skip" { + return m.Skip() + } + return nil + }) + if err != nil { + t.Fatal(err) + } + if want := "{\"a\":1,\"a\":2,\"caf\\u00e9\":\"bad \xff byte\"}"; got != want { + t.Errorf("got %q, want %q", got, want) + } + if want := []string{"a", "a", "café", "skip"}; !slices.Equal(keys, want) { + t.Errorf("keys %q, want %q", keys, want) + } +} + +func TestSkipThenWriteReplaces(t *testing.T) { + got, err := copyString(t, `{"a":{"big":[1,2,3]},"b":2}`, func(m *Member) error { + if m.Key != "a" { + return nil + } + if err := m.Skip(); err != nil { + return err + } + return m.Write([]byte(`"small"`)) + }) + if err != nil || got != `{"a":"small","b":2}` { + t.Errorf("got %s, %v", got, err) + } +} diff --git a/internal/metadata/npm/stream.go b/internal/metadata/npm/stream.go index b00f6a2..8788de1 100644 --- a/internal/metadata/npm/stream.go +++ b/internal/metadata/npm/stream.go @@ -102,7 +102,7 @@ func RewriteStream(packageName string, scan *Scan, r io.Reader, w io.Writer, opt filtered: f, seen: map[string]int{}, } - if err := jsonstream.CopyObject(json.NewDecoder(r), w, sw.member); err != nil { + if err := jsonstream.CopyObject(r, w, sw.member); err != nil { return nil, err } h.tarballsRewritten(sw.rewritten) @@ -162,8 +162,7 @@ func (s *streamWriter) member(m *jsonstream.Member) error { } s.seen[m.Key]++ if s.seen[m.Key] < total { - _, err := m.Raw() // an earlier duplicate, dropped - return err + return m.Skip() // an earlier duplicate, dropped } switch { case m.Key == "versions": @@ -171,7 +170,7 @@ func (s *streamWriter) member(m *jsonstream.Member) error { case m.Key == "time" && s.timeChanged: return m.Object(s.time) case m.Key == "dist-tags" && s.tagsChanged: - if _, err := m.Raw(); err != nil { + if err := m.Skip(); err != nil { return err } encoded, err := json.Marshal(s.distTags) @@ -185,13 +184,13 @@ func (s *streamWriter) member(m *jsonstream.Member) error { // version writes a kept version with its tarball pointing at the proxy. func (s *streamWriter) version(v *jsonstream.Member) error { + if _, keep := s.versions[v.Key]; !keep { + return v.Skip() + } raw, err := v.Raw() if err != nil { return err } - if _, keep := s.versions[v.Key]; !keep { - return nil - } s.entry.Reset() if s.writeNPMVersion(&s.entry, s.packageName, v.Key, raw) { s.rewritten++ @@ -204,8 +203,7 @@ func (s *streamWriter) time(t *jsonstream.Member) error { if _, keep := s.timeMap[t.Key]; keep { return t.Copy() } - _, err := t.Raw() - return err + return t.Skip() } // objectValue records value under name in metadata, as Rewrite's diff --git a/internal/metadata/npm/stream_bench_test.go b/internal/metadata/npm/stream_bench_test.go index 827cf94..12d04d8 100644 --- a/internal/metadata/npm/stream_bench_test.go +++ b/internal/metadata/npm/stream_bench_test.go @@ -2,6 +2,8 @@ package npm import ( "bytes" + "io" + "os" "testing" "github.com/git-pkgs/proxy/internal/metadata/npm/npmbench" @@ -29,3 +31,29 @@ func BenchmarkLookupVersion(b *testing.B) { } } } + +// BenchmarkRewriteStream times RewriteStream's second pass, the copy, over +// the 5,000-version packument, or the one NPM_PACKUMENT_BENCH names. +func BenchmarkRewriteStream(b *testing.B) { + body := npmbench.Packument(5000) + name := "big" + if path := os.Getenv("NPM_PACKUMENT_BENCH"); path != "" { + var err error + if body, err = os.ReadFile(path); err != nil { + b.Fatal(err) + } + name = os.Getenv("NPM_PACKUMENT_BENCH_NAME") + } + scan, err := ScanPackument(bytes.NewReader(body)) + if err != nil { + b.Fatal(err) + } + options := Options{ProxyURL: "http://proxy.example"} + b.SetBytes(int64(len(body))) + b.ReportAllocs() + for b.Loop() { + if _, err := RewriteStream(name, scan, bytes.NewReader(body), io.Discard, options); err != nil { + b.Fatal(err) + } + } +} From 0efe54fd73ff3c2aaf30372db48ee715b25d07f2 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 8 Nov 2026 19:11:36 +0100 Subject: [PATCH 628/633] Let the submodule test add a ':' path on Windows Git for Windows refuses it with core.protectNTFS on. --- internal/gitcache/mirror_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/internal/gitcache/mirror_test.go b/internal/gitcache/mirror_test.go index ca734f1..d016f2a 100644 --- a/internal/gitcache/mirror_test.go +++ b/internal/gitcache/mirror_test.go @@ -270,7 +270,8 @@ func TestMirrorListsSubmodules(t *testing.T) { runGit(t, work, "add", ".gitmodules") runGit(t, work, "update-index", "--add", "--cacheinfo", "160000,"+libCommit+",vendor/lib") runGit(t, work, "update-index", "--add", "--cacheinfo", "160000,"+otherCommit+",third_party/other") - runGit(t, work, "update-index", "--add", "--cacheinfo", "160000,"+libCommit+",:colon") + // Git for Windows refuses ':' in paths unless NTFS protection is off. + runGit(t, work, "-c", "core.protectNTFS=false", "update-index", "--add", "--cacheinfo", "160000,"+libCommit+",:colon") runGit(t, work, "commit", "--quiet", "-m", "submodules") runGit(t, work, "push", "--quiet", "origin", "main") commit := runGit(t, work, "rev-parse", "HEAD") From 23316c04d297c98e827c8cd6dc1a4121afc13210 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 8 Nov 2026 19:12:30 +0100 Subject: [PATCH 629/633] Accept object keys with invalid UTF-8 in jsonstream AppendUnquote reports invalid UTF-8 after replacing it, which failed the whole copy although the decoder allows it. Key now holds U+FFFD in its place, as with encoding/json and ScanPackument, and the key is still written with its original bytes. --- internal/jsonstream/jsonstream.go | 8 ++++---- internal/jsonstream/jsonstream_test.go | 20 ++++++++++++++++++++ 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/internal/jsonstream/jsonstream.go b/internal/jsonstream/jsonstream.go index a515f11..cf213ce 100644 --- a/internal/jsonstream/jsonstream.go +++ b/internal/jsonstream/jsonstream.go @@ -131,10 +131,10 @@ func copyObject(dec *jsontext.Decoder, w io.Writer, fn func(*Member) error, allo return fmt.Errorf("object key is %v, not a string", rawKey.Kind()) } keyBuf = append(keyBuf[:0], rawKey...) - key, err := jsontext.AppendUnquote(nil, keyBuf) - if err != nil { - return err - } + // The decoder has checked the key, so the only error left is invalid + // UTF-8, which AppendUnquote reports after replacing it with U+FFFD, + // as encoding/json does. + key, _ := jsontext.AppendUnquote(nil, keyBuf) *m = Member{Key: string(key), rawKey: keyBuf, dec: dec, out: out} if err := fn(m); err != nil { return err diff --git a/internal/jsonstream/jsonstream_test.go b/internal/jsonstream/jsonstream_test.go index e7bca69..cd201ae 100644 --- a/internal/jsonstream/jsonstream_test.go +++ b/internal/jsonstream/jsonstream_test.go @@ -180,3 +180,23 @@ func TestSkipThenWriteReplaces(t *testing.T) { t.Errorf("got %s, %v", got, err) } } + +// TestCopyObjectInvalidUTF8Key checks a key with invalid UTF-8 is accepted: +// Key has U+FFFD in its place, as with encoding/json, and the key is written +// with its original bytes. +func TestCopyObjectInvalidUTF8Key(t *testing.T) { + var keys []string + got, err := copyString(t, "{\"k\xff\":1}", func(m *Member) error { + keys = append(keys, m.Key) + return nil + }) + if err != nil { + t.Fatal(err) + } + if want := "{\"k\xff\":1}"; got != want { + t.Errorf("got %q, want %q", got, want) + } + if want := []string{"k�"}; !slices.Equal(keys, want) { + t.Errorf("keys %q, want %q", keys, want) + } +} From b61e35205c47238421c0c0b1b0c5a649d621e750 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 8 Nov 2026 21:49:48 +0100 Subject: [PATCH 630/633] Answer 404 for Maven files outside the layout A file whose group directories hold a dot or colon was proxied without caching, which skipped the denylist, scan and cooldown. Such paths aren't in the Maven layout, so they now get 404. Directory listings and catalogs still go through proxyUpstream, which shares its error handling with Proxy.ProxyUpstream in relayUpstream. --- internal/handler/handler.go | 6 +++ internal/handler/maven.go | 18 ++++---- internal/handler/maven_repositories_test.go | 51 +++++++++++++-------- 3 files changed, 46 insertions(+), 29 deletions(-) diff --git a/internal/handler/handler.go b/internal/handler/handler.go index b3ef666..9612e51 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -1500,6 +1500,12 @@ func (p *Proxy) ProxyUpstream(w http.ResponseWriter, r *http.Request, upstreamUR p.Logger.Debug("proxying to upstream", "url", upstreamURL) resp, err := p.upstreamRequest(r, upstreamURL, forwardHeaders) + p.relayUpstream(w, r, resp, err) +} + +// relayUpstream writes the result of an upstream request: the response, or +// a 502 when the request failed. +func (p *Proxy) relayUpstream(w http.ResponseWriter, r *http.Request, resp *http.Response, err error) { if err != nil { if writeUpstreamHeld(w, err) { return diff --git a/internal/handler/maven.go b/internal/handler/maven.go index 4b170be..5008e00 100644 --- a/internal/handler/maven.go +++ b/internal/handler/maven.go @@ -221,6 +221,13 @@ func (h *MavenHandler) handleDownload(w http.ResponseWriter, r *http.Request, so // e.g., com/google/guava/guava/32.1.3-jre/guava-32.1.3-jre.jar group, artifact, version, filename := h.parsePath(urlPath) if artifact == "" { + if strings.Count(urlPath, "/") >= minMavenParts-1 { + // A dot or colon in a group directory, or a colon in the + // artifact, which the Maven layout never has. Proxied, the + // file would skip the denylist, scan and cooldown. + http.NotFound(w, r) + return + } h.proxyUpstream(w, r, source, urlPath) return } @@ -357,14 +364,5 @@ func (h *MavenHandler) proxyUpstream(w http.ResponseWriter, r *http.Request, sou _ = resp.Body.Close() resp, err = h.proxy.upstreamRequest(r, source.fallbackURL+"/"+urlPath, mavenForwardHeaders) } - if err != nil { - if writeUpstreamHeld(w, err) { - return - } - h.proxy.Logger.Error("upstream request failed", "error", err) - http.Error(w, "upstream request failed", http.StatusBadGateway) - return - } - defer func() { _ = resp.Body.Close() }() - h.proxy.relayResponse(w, r, resp, nil) + h.proxy.relayUpstream(w, r, resp, err) } diff --git a/internal/handler/maven_repositories_test.go b/internal/handler/maven_repositories_test.go index 260d3a8..923af92 100644 --- a/internal/handler/maven_repositories_test.go +++ b/internal/handler/maven_repositories_test.go @@ -139,45 +139,58 @@ func TestMavenRepositoryRepointedServesTheNewUpstream(t *testing.T) { } } -// TestMavenDottedGroupDirectoryNotCachedAsGroup checks that a path whose -// group directories contain dots is not cached under the group the dots -// spell, where it would share an entry with the standard layout's path. +// TestMavenDottedGroupDirectoryNotCachedAsGroup checks that a file whose +// group directories contain dots or colons, which the Maven layout never +// has, is not served: it would either share the cache entry of the group +// the dots spell or be proxied past the denylist, scan and cooldown. func TestMavenDottedGroupDirectoryNotCachedAsGroup(t *testing.T) { const standard = "/org/foo/bar/1.0/bar-1.0.jar" - const dotted = "/org.foo/bar/1.0/bar-1.0.jar" - central, _ := mavenRepoUpstream(t, "central", standard, dotted) + odd := []string{"/org.foo/bar/1.0/bar-1.0.jar", "/org/foo:x/bar/1.0/bar-1.0.jar", "/org/foo/b:ar/1.0/b:ar-1.0.jar"} + central, requests := mavenRepoUpstream(t, "central", append([]string{standard}, odd...)...) portal, _ := mavenRepoUpstream(t, "portal") - grails, _ := mavenRepoUpstream(t, "grails") + var restricted []string + for _, p := range odd { + restricted = append(restricted, "/restricted"+p) + } + grails, grailsRequests := mavenRepoUpstream(t, "grails", restricted...) h := newMavenRepositoriesHandler(t, central, portal, grails) - for _, path := range []string{standard, dotted, standard} { - w := getMaven(h, path) - if want := "central " + path; w.Code != http.StatusOK || w.Body.String() != want { - t.Errorf("GET %s = %d %q, want 200 %q", path, w.Code, w.Body.String(), want) + if w := getMaven(h, standard); w.Code != http.StatusOK || w.Body.String() != "central "+standard { + t.Errorf("GET %s = %d %q", standard, w.Code, w.Body.String()) + } + before := requests.Load() + grailsRequests.Load() + for _, p := range odd { + for _, path := range []string{p, "/grails" + p} { + if w := getMaven(h, path); w.Code != http.StatusNotFound { + t.Errorf("GET %s = %d %q, want 404", path, w.Code, w.Body.String()) + } } } + if n := requests.Load() + grailsRequests.Load() - before; n != 0 { + t.Errorf("%d upstream requests for paths outside the Maven layout", n) + } } // TestMavenUncachedPathFallsBackToPluginPortal checks that a path proxied -// without caching, such as one with a dotted group directory, still tries -// the Gradle Plugin Portal when the primary upstream lacks it. +// without caching, such as a catalog, still tries the Gradle Plugin Portal +// when the primary upstream lacks it. func TestMavenUncachedPathFallsBackToPluginPortal(t *testing.T) { - const dotted = "/org.foo/bar/1.0/bar-1.0.jar" + const uncached = "/org/foo/archetype-catalog.xml" central, _ := mavenRepoUpstream(t, "central") - portal, _ := mavenRepoUpstream(t, "portal", dotted) + portal, _ := mavenRepoUpstream(t, "portal", uncached) grails, _ := mavenRepoUpstream(t, "grails") h := newMavenRepositoriesHandler(t, central, portal, grails) - if w := getMaven(h, dotted); w.Code != http.StatusOK || w.Body.String() != "portal "+dotted { - t.Errorf("GET %s = %d %q, want the Plugin Portal's copy", dotted, w.Code, w.Body.String()) + if w := getMaven(h, uncached); w.Code != http.StatusOK || w.Body.String() != "portal "+uncached { + t.Errorf("GET %s = %d %q, want the Plugin Portal's copy", uncached, w.Code, w.Body.String()) } } func TestMavenUncachedPathKeepsByteRanges(t *testing.T) { - const dotted = "/org.foo/bar/1.0/bar-1.0.jar" + const uncached = "/org/foo/archetype-catalog.xml" ranged := func() *httptest.Server { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - http.ServeContent(w, r, "bar.jar", time.Time{}, strings.NewReader("0123456789")) + http.ServeContent(w, r, "archetype-catalog.xml", time.Time{}, strings.NewReader("0123456789")) })) t.Cleanup(srv.Close) return srv @@ -186,7 +199,7 @@ func TestMavenUncachedPathKeepsByteRanges(t *testing.T) { portal, _ := mavenRepoUpstream(t, "portal") h := newMavenRepositoriesHandler(t, central, portal, grails) - for _, p := range []string{dotted, "/grails" + dotted} { + for _, p := range []string{uncached, "/grails" + uncached} { req := httptest.NewRequest(http.MethodGet, p, nil) req.Header.Set("Range", "bytes=2-4") w := httptest.NewRecorder() From 3776366ba22582ef2c5940686bf9072bf826491b Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 8 Nov 2026 21:58:47 +0100 Subject: [PATCH 631/633] Document the 404 for Maven files outside the layout --- docs/configuration.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index e61c7e9..4272a84 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1018,9 +1018,10 @@ one doesn't fall back to the Gradle Plugin Portal. Files are cached per repository, so the same coordinates in two repositories never share a cache entry. -In every repository, a path whose group directories contain a dot or are empty -(`org.foo/bar/1.0/bar-1.0.jar`) is not in the Maven layout and is proxied -without caching, since its group would read the same as `org/foo`'s. +In every repository, a file whose group directories contain a dot or colon or +are empty (`org.foo/bar/1.0/bar-1.0.jar`), or whose artifact contains a colon, +is not in the Maven layout and gets 404 without an upstream request. Its group +would read the same as `org/foo`'s. `upstream.maven_repositories` is additive. `/maven/…` keeps serving `upstream.maven` with unchanged cache identities. A repository name is matched From 4ae55605b14c52749aa780c673824d74e0140b7f Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 8 Nov 2026 22:18:14 +0100 Subject: [PATCH 632/633] Send Maven coordinate files under odd groups to the 404 A file named {artifact}-{version} under a dotted group directory, such as org.foo/bar/1.0/bar-1.0-linux.exe, was not recognised as a coordinate file and so was proxied past the checks. isCoordinateFile no longer checks the group, so handleDownload answers 404 for it. The docs say which files get the 404, and the Range test covers a stale If-Range. --- docs/configuration.md | 10 ++++++---- internal/handler/maven.go | 9 ++++++++- internal/handler/maven_repositories_test.go | 16 +++++++++++++++- 3 files changed, 29 insertions(+), 6 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 4272a84..9aa7d80 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1018,10 +1018,12 @@ one doesn't fall back to the Gradle Plugin Portal. Files are cached per repository, so the same coordinates in two repositories never share a cache entry. -In every repository, a file whose group directories contain a dot or colon or -are empty (`org.foo/bar/1.0/bar-1.0.jar`), or whose artifact contains a colon, -is not in the Maven layout and gets 404 without an upstream request. Its group -would read the same as `org/foo`'s. +In every repository, an artifact file (one with an artifact extension or named +`{artifact}-{version}…`) whose group directories contain a dot or colon or are +empty (`org.foo/bar/1.0/bar-1.0.jar`), or whose artifact contains a colon, is +not in the Maven layout and gets 404 without an upstream request. Its group +would read the same as `org/foo`'s. Other paths there, such as +`maven-metadata.xml` or a directory listing, are handled as anywhere else. `upstream.maven_repositories` is additive. `/maven/…` keeps serving `upstream.maven` with unchanged cache identities. A repository name is matched diff --git a/internal/handler/maven.go b/internal/handler/maven.go index 5008e00..e841d08 100644 --- a/internal/handler/maven.go +++ b/internal/handler/maven.go @@ -321,8 +321,15 @@ func (h *MavenHandler) isArtifactFile(filename string) bool { // artifact and version of its directory, {artifact}-{version}..., as Maven // names every file it publishes, whatever the extension. A SNAPSHOT's // files carry a timestamp in place of "SNAPSHOT". +// +// The group isn't checked, so a file under a group directory outside the +// Maven layout still goes to handleDownload, which answers 404. func (h *MavenHandler) isCoordinateFile(urlPath string) bool { - _, artifact, version, filename := h.parsePath(urlPath) + parts := strings.Split(urlPath, "/") + if len(parts) < minMavenParts { + return false + } + filename, version, artifact := parts[len(parts)-1], parts[len(parts)-2], parts[len(parts)-3] if artifact == "" || version == "" { return false } diff --git a/internal/handler/maven_repositories_test.go b/internal/handler/maven_repositories_test.go index 923af92..e9b2fcf 100644 --- a/internal/handler/maven_repositories_test.go +++ b/internal/handler/maven_repositories_test.go @@ -145,7 +145,10 @@ func TestMavenRepositoryRepointedServesTheNewUpstream(t *testing.T) { // the dots spell or be proxied past the denylist, scan and cooldown. func TestMavenDottedGroupDirectoryNotCachedAsGroup(t *testing.T) { const standard = "/org/foo/bar/1.0/bar-1.0.jar" - odd := []string{"/org.foo/bar/1.0/bar-1.0.jar", "/org/foo:x/bar/1.0/bar-1.0.jar", "/org/foo/b:ar/1.0/b:ar-1.0.jar"} + odd := []string{ + "/org.foo/bar/1.0/bar-1.0.jar", "/org.foo/bar/1.0/bar-1.0-linux.exe", + "/org/foo:x/bar/1.0/bar-1.0.jar", "/org/foo/b:ar/1.0/b:ar-1.0.jar", + } central, requests := mavenRepoUpstream(t, "central", append([]string{standard}, odd...)...) portal, _ := mavenRepoUpstream(t, "portal") var restricted []string @@ -190,6 +193,7 @@ func TestMavenUncachedPathKeepsByteRanges(t *testing.T) { const uncached = "/org/foo/archetype-catalog.xml" ranged := func() *httptest.Server { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("ETag", `"v2"`) http.ServeContent(w, r, "archetype-catalog.xml", time.Time{}, strings.NewReader("0123456789")) })) t.Cleanup(srv.Close) @@ -207,5 +211,15 @@ func TestMavenUncachedPathKeepsByteRanges(t *testing.T) { if w.Code != http.StatusPartialContent || w.Body.String() != "234" { t.Errorf("GET %s with Range = %d %q, want 206 %q", p, w.Code, w.Body.String(), "234") } + + // A Range for an older copy gets the whole current file. + req = httptest.NewRequest(http.MethodGet, p, nil) + req.Header.Set("Range", "bytes=2-4") + req.Header.Set("If-Range", `"v1"`) + w = httptest.NewRecorder() + h.ServeHTTP(w, req) + if w.Code != http.StatusOK || w.Body.String() != "0123456789" { + t.Errorf("GET %s with stale If-Range = %d %q, want 200 with the whole file", p, w.Code, w.Body.String()) + } } } From d7399b10435925d80815b6f726ae2e5dab8a2a62 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 8 Nov 2026 22:51:25 +0100 Subject: [PATCH 633/633] Say which Maven files get the 404 without listing name rules --- docs/configuration.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 9aa7d80..d887509 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1018,12 +1018,12 @@ one doesn't fall back to the Gradle Plugin Portal. Files are cached per repository, so the same coordinates in two repositories never share a cache entry. -In every repository, an artifact file (one with an artifact extension or named -`{artifact}-{version}…`) whose group directories contain a dot or colon or are -empty (`org.foo/bar/1.0/bar-1.0.jar`), or whose artifact contains a colon, is -not in the Maven layout and gets 404 without an upstream request. Its group -would read the same as `org/foo`'s. Other paths there, such as -`maven-metadata.xml` or a directory listing, are handled as anywhere else. +In every repository, a file the proxy would cache (an artifact, checksum or +signature) whose group directories contain a dot or colon +(`org.foo/bar/1.0/bar-1.0.jar`), or whose artifact contains a colon, is not in +the Maven layout and gets 404 without an upstream request. Its group would read +the same as `org/foo`'s. Other paths there, such as `maven-metadata.xml` or a +directory listing, are handled as anywhere else. `upstream.maven_repositories` is additive. `/maven/…` keeps serving `upstream.maven` with unchanged cache identities. A repository name is matched