From d9d6e8735afbf6b7688fa3e691b5e5b134bfa094 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 23 Apr 2026 15:21:18 +0000 Subject: [PATCH 1/9] Bump github.com/git-pkgs/registries from 0.4.0 to 0.4.1 Bumps [github.com/git-pkgs/registries](https://github.com/git-pkgs/registries) from 0.4.0 to 0.4.1. - [Commits](https://github.com/git-pkgs/registries/compare/v0.4.0...v0.4.1) --- updated-dependencies: - dependency-name: github.com/git-pkgs/registries dependency-version: 0.4.1 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 7ae8518..1067a3a 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ require ( github.com/git-pkgs/archives v0.2.2 github.com/git-pkgs/enrichment v0.2.2 github.com/git-pkgs/purl v0.1.10 - github.com/git-pkgs/registries v0.4.0 + github.com/git-pkgs/registries v0.4.1 github.com/git-pkgs/spdx v0.1.2 github.com/git-pkgs/vers v0.2.4 github.com/git-pkgs/vulns v0.1.4 diff --git a/go.sum b/go.sum index 9bdf6f9..49b11f9 100644 --- a/go.sum +++ b/go.sum @@ -238,8 +238,8 @@ github.com/git-pkgs/packageurl-go v0.3.1 h1:WM3RBABQZLaRBxgKyYughc3cVBE8KyQxbSC6 github.com/git-pkgs/packageurl-go v0.3.1/go.mod h1:rcIxiG37BlQLB6FZfgdj9Fm7yjhRQd3l+5o7J0QPAk4= github.com/git-pkgs/purl v0.1.10 h1:NMjeF10nzFn3tdQlz6rbmHB+i+YkyrFQxho3e33ePTQ= github.com/git-pkgs/purl v0.1.10/go.mod h1:C5Vp/kyZ/wGckCLexx4wPVfUxEiToRkdsOPh5Z7ig/I= -github.com/git-pkgs/registries v0.4.0 h1:GO7fQ8/jot0ulSQHBdxLSNSX/p8eB3gEXWO+98fmoEo= -github.com/git-pkgs/registries v0.4.0/go.mod h1:49UCPFWQmwNV7rBEr9TrTDWKR7vYxFcxp3VfdkeFbdE= +github.com/git-pkgs/registries v0.4.1 h1:4qlKVNhC/6x6Bt87t3wrGJtF3EFrUpHQt9/zKsa5IvU= +github.com/git-pkgs/registries v0.4.1/go.mod h1:49UCPFWQmwNV7rBEr9TrTDWKR7vYxFcxp3VfdkeFbdE= github.com/git-pkgs/spdx v0.1.2 h1:wHSK+CqFsO5N7yDTPvxDmer5LgNEa7vAsiZhi5Aci0A= github.com/git-pkgs/spdx v0.1.2/go.mod h1:V98MgZapNgYw54/pdGR82d7RU93qzJoybahbpZqTfw8= github.com/git-pkgs/vers v0.2.4 h1:Zr3jR/Xf1i/6cvBaJKPxhCwjzqz7uvYHE0Fhid/GPBk= From 34be35cafac00d016a9f3d28128f96dd85235961 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 23 Apr 2026 15:21:27 +0000 Subject: [PATCH 2/9] Bump zizmorcore/zizmor-action from 0.5.2 to 0.5.3 Bumps [zizmorcore/zizmor-action](https://github.com/zizmorcore/zizmor-action) from 0.5.2 to 0.5.3. - [Release notes](https://github.com/zizmorcore/zizmor-action/releases) - [Commits](https://github.com/zizmorcore/zizmor-action/compare/71321a20a9ded102f6e9ce5718a2fcec2c4f70d8...b1d7e1fb5de872772f31590499237e7cce841e8e) --- updated-dependencies: - dependency-name: zizmorcore/zizmor-action dependency-version: 0.5.3 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/zizmor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/zizmor.yml b/.github/workflows/zizmor.yml index 03ea882..4acd19b 100644 --- a/.github/workflows/zizmor.yml +++ b/.github/workflows/zizmor.yml @@ -26,4 +26,4 @@ jobs: persist-credentials: false - name: Run zizmor - uses: zizmorcore/zizmor-action@71321a20a9ded102f6e9ce5718a2fcec2c4f70d8 # v0.5.2 + uses: zizmorcore/zizmor-action@b1d7e1fb5de872772f31590499237e7cce841e8e # v0.5.3 From c73b0a35a1dfa9bdd51c2856dfa6f543def358b4 Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Mon, 27 Apr 2026 12:04:38 +0100 Subject: [PATCH 3/9] Add direct-serve via presigned storage URLs When storage.direct_serve is enabled and the backend supports it (S3, Azure), cached artifact downloads return a 302 redirect to a presigned URL instead of streaming bytes through the proxy. Falls back to streaming when the backend can't sign (fileblob, local filesystem) or signing fails. Adds the azureblob driver so azblob:// storage URLs work. Cache-hit accounting already happened before io.Copy so redirects are counted correctly; the metrics calls are pulled into a helper so both paths share them. Closes #96 --- config.example.yaml | 9 ++ go.mod | 16 +++ go.sum | 27 +++++ internal/config/config.go | 48 ++++++++- internal/config/config_test.go | 51 +++++++++ internal/handler/handler.go | 70 +++++++++---- internal/handler/handler_test.go | 157 +++++++++++++++++++++++++++- internal/server/server.go | 2 + internal/storage/blob.go | 17 +++ internal/storage/blob_test.go | 13 +++ internal/storage/filesystem.go | 5 + internal/storage/filesystem_test.go | 10 ++ internal/storage/storage.go | 9 ++ 13 files changed, 407 insertions(+), 27 deletions(-) diff --git a/config.example.yaml b/config.example.yaml index ea17d15..c272979 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -29,6 +29,15 @@ storage: # Empty or "0" means unlimited max_size: "" + # Redirect cached artifact downloads to presigned storage URLs (HTTP 302) + # instead of streaming through the proxy. Only effective for S3 and Azure. + # Leave disabled if clients reach the proxy through an authenticating gateway, + # since presigned URLs bypass it. + direct_serve: false + + # How long presigned URLs remain valid (e.g. "5m", "1h"). Default: "15m". + direct_serve_ttl: "15m" + # Database configuration database: # Database driver: "sqlite" (default) or "postgres" diff --git a/go.mod b/go.mod index 1067a3a..6226ab6 100644 --- a/go.mod +++ b/go.mod @@ -28,6 +28,8 @@ require ( require ( 4d63.com/gocheckcompilerdirectives v1.3.0 // indirect 4d63.com/gochecknoglobals v0.2.2 // indirect + cloud.google.com/go/auth v0.18.2 // indirect + cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect cloud.google.com/go/compute/metadata v0.9.0 // indirect codeberg.org/chavacava/garif v0.2.0 // indirect codeberg.org/polyfloyd/go-errorlint v1.9.0 // indirect @@ -40,6 +42,13 @@ require ( github.com/Antonboom/errname v1.1.1 // indirect github.com/Antonboom/nilnil v1.1.1 // indirect github.com/Antonboom/testifylint v1.6.4 // indirect + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.20.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1 // indirect + github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 // indirect + github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.3 // indirect + github.com/Azure/go-autorest v14.2.0+incompatible // indirect + github.com/Azure/go-autorest/autorest/to v0.4.1 // indirect + github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0 // indirect github.com/BurntSushi/toml v1.6.0 // indirect github.com/Djarvur/go-err113 v0.1.1 // indirect github.com/KyleBanks/depth v1.2.1 // indirect @@ -138,6 +147,7 @@ require ( github.com/gobwas/glob v0.2.3 // indirect github.com/godoc-lint/godoc-lint v0.11.2 // indirect github.com/gofrs/flock v0.13.0 // indirect + github.com/golang-jwt/jwt/v5 v5.3.0 // indirect github.com/golangci/asciicheck v0.5.0 // indirect github.com/golangci/dupl v0.0.0-20250308024227-f665c8d69b32 // indirect github.com/golangci/go-printf-func-name v0.1.1 // indirect @@ -150,8 +160,10 @@ require ( github.com/golangci/swaggoswag v0.0.0-20250504205917-77f2aca3143e // indirect github.com/golangci/unconvert v0.0.0-20250410112200-a129a6e6413e // indirect github.com/google/go-cmp v0.7.0 // indirect + github.com/google/s2a-go v0.1.9 // indirect github.com/google/uuid v1.6.0 // indirect github.com/google/wire v0.7.0 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.12 // indirect github.com/googleapis/gax-go/v2 v2.17.0 // indirect github.com/gordonklaus/ineffassign v0.2.0 // indirect github.com/gostaticanalysis/analysisutil v0.7.1 // indirect @@ -174,6 +186,7 @@ require ( github.com/kkHAIKE/contextcheck v1.1.6 // indirect github.com/kulti/thelper v0.7.1 // indirect github.com/kunwardeep/paralleltest v1.0.15 // indirect + github.com/kylelemons/godebug v1.1.0 // indirect github.com/lasiar/canonicalheader v1.1.2 // indirect github.com/ldez/exptostd v0.4.5 // indirect github.com/ldez/gomoddirectives v0.8.0 // indirect @@ -209,6 +222,7 @@ require ( github.com/pandatix/go-cvss v0.6.2 // indirect github.com/pelletier/go-toml v1.9.5 // indirect github.com/pelletier/go-toml/v2 v2.2.4 // indirect + github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/common v0.67.5 // indirect github.com/prometheus/procfs v0.20.1 // indirect @@ -277,10 +291,12 @@ require ( go.uber.org/zap v1.27.1 // indirect go.yaml.in/yaml/v2 v2.4.3 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect + golang.org/x/crypto v0.48.0 // indirect golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa // indirect golang.org/x/exp/typeparams v0.0.0-20260209203927-2842357ff358 // indirect golang.org/x/mod v0.33.0 // indirect golang.org/x/net v0.51.0 // indirect + golang.org/x/oauth2 v0.35.0 // indirect golang.org/x/sys v0.42.0 // indirect golang.org/x/text v0.34.0 // indirect golang.org/x/tools v0.42.0 // indirect diff --git a/go.sum b/go.sum index 49b11f9..26647ed 100644 --- a/go.sum +++ b/go.sum @@ -43,6 +43,26 @@ github.com/Antonboom/nilnil v1.1.1 h1:9Mdr6BYd8WHCDngQnNVV0b554xyisFioEKi30sksuf github.com/Antonboom/nilnil v1.1.1/go.mod h1:yCyAmSw3doopbOWhJlVci+HuyNRuHJKIv6V2oYQa8II= github.com/Antonboom/testifylint v1.6.4 h1:gs9fUEy+egzxkEbq9P4cpcMB6/G0DYdMeiFS87UiqmQ= github.com/Antonboom/testifylint v1.6.4/go.mod h1:YO33FROXX2OoUfwjz8g+gUxQXio5i9qpVy7nXGbxDD4= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.20.0 h1:JXg2dwJUmPB9JmtVmdEB16APJ7jurfbY5jnfXpJoRMc= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.20.0/go.mod h1:YD5h/ldMsG0XiIw7PdyNhLxaM317eFh5yNLccNfGdyw= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1 h1:Hk5QBxZQC1jb2Fwj6mpzme37xbCDdNTxU7O9eb5+LB4= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1/go.mod h1:IYus9qsFobWIc2YVwe/WPjcnyCkPKtnHAqUYeebc8z0= +github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.2 h1:yz1bePFlP5Vws5+8ez6T3HWXPmwOK7Yvq8QxDBD3SKY= +github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.2/go.mod h1:Pa9ZNPuoNu/GztvBSKk9J1cDJW6vk/n0zLtV4mgd8N8= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 h1:9iefClla7iYpfYWdzPCRDozdmndjTm8DXdpCzPajMgA= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2/go.mod h1:XtLgD3ZD34DAaVIIAyG3objl5DynM3CQ/vMcbBNJZGI= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1 h1:/Zt+cDPnpC3OVDm/JKLOs7M2DKmLRIIp3XIx9pHHiig= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1/go.mod h1:Ng3urmn6dYe8gnbCMoHHVl5APYz2txho3koEkV2o2HA= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.3 h1:ZJJNFaQ86GVKQ9ehwqyAFE6pIfyicpuJ8IkVaPBc6/4= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.3/go.mod h1:URuDvhmATVKqHBH9/0nOiNKk0+YcwfQ3WkK5PqHKxc8= +github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= +github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= +github.com/Azure/go-autorest/autorest/to v0.4.1 h1:CxNHBqdzTr7rLtdrtb5CMjJcDut+WNGCVv7OmS5+lTc= +github.com/Azure/go-autorest/autorest/to v0.4.1/go.mod h1:EtaofgU4zmtvn1zT2ARsjRFdq9vXx0YWtmElwL+GZ9M= +github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 h1:WJTmL004Abzc5wDB5VtZG2PJk5ndYDgVacGqfirKxjM= +github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mod h1:tCcJZ0uHAmvjsVYzEFivsRTN00oz5BEsRgQHu5JZ9WE= +github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0 h1:XRzhVemXdgvJqCH0sFfrBUTnUJSBrBf7++ypk+twtRs= +github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0/go.mod h1:HKpQxkWaGLJ+D/5H8QRpyQXA1eKjxkFlOMwck5+33Jk= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/toml v1.6.0 h1:dRaEfpa2VI55EwlIW72hMRHdWouJeRF7TPYhI+AUQjk= github.com/BurntSushi/toml v1.6.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= @@ -305,6 +325,8 @@ github.com/godoc-lint/godoc-lint v0.11.2 h1:Bp0FkJWoSdNsBikdNgIcgtaoo+xz6I/Y9s5W github.com/godoc-lint/godoc-lint v0.11.2/go.mod h1:iVpGdL1JCikNH2gGeAn3Hh+AgN5Gx/I/cxV+91L41jo= github.com/gofrs/flock v0.13.0 h1:95JolYOvGMqeH31+FC7D2+uULf6mG61mEZ/A8dRYMzw= github.com/gofrs/flock v0.13.0/go.mod h1:jxeyy9R1auM5S6JYDBhDt+E2TCo7DkratH4Pgi8P+Z0= +github.com/golang-jwt/jwt/v5 v5.3.0 h1:pv4AsKCKKZuqlgs5sUmn4x8UlGa0kEVt/puTpKx9vvo= +github.com/golang-jwt/jwt/v5 v5.3.0/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/golangci/asciicheck v0.5.0 h1:jczN/BorERZwK8oiFBOGvlGPknhvq0bjnysTj4nUfo0= @@ -396,6 +418,8 @@ github.com/julz/importas v0.2.0 h1:y+MJN/UdL63QbFJHws9BVC5RpA2iq0kpjrFajTGivjQ= github.com/julz/importas v0.2.0/go.mod h1:pThlt589EnCYtMnmhmRYY/qn9lCf/frPOK+WMx3xiJY= github.com/karamaru-alpha/copyloopvar v1.2.2 h1:yfNQvP9YaGQR7VaWLYcfZUlRP2eo2vhExWKxD/fP6q0= github.com/karamaru-alpha/copyloopvar v1.2.2/go.mod h1:oY4rGZqZ879JkJMtX3RRkcXRkmUvH0x35ykgaKgsgJY= +github.com/keybase/go-keychain v0.0.1 h1:way+bWYa6lDppZoZcgMbYsvC7GxljxrskdNInRtuthU= +github.com/keybase/go-keychain v0.0.1/go.mod h1:PdEILRW3i9D8JcdM+FmY6RwkHGnhHxXwkPPMeUgOK1k= github.com/kisielk/errcheck v1.9.0 h1:9xt1zI9EBfcYBvdU1nVrzMzzUPUtPKs9bVSIM3TAb3M= github.com/kisielk/errcheck v1.9.0/go.mod h1:kQxWMMVZgIkDq7U8xtG/n2juOjbLgZtedi0D+/VL/i8= github.com/kkHAIKE/contextcheck v1.1.6 h1:7HIyRcnyzxL9Lz06NGhiKvenXq7Zw6Q0UQu/ttjfJCE= @@ -509,6 +533,8 @@ github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0 github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= github.com/peterbourgon/g2s v0.0.0-20170223122336-d4e7ad98afea h1:sKwxy1H95npauwu8vtF95vG/syrL0p8fSZo/XlDg5gk= github.com/peterbourgon/g2s v0.0.0-20170223122336-d4e7ad98afea/go.mod h1:1VcHEd3ro4QMoHfiNl/j7Jkln9+KQuorp0PItHMJYNg= +github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= +github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU= github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo= github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -771,6 +797,7 @@ golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20211105183446-c75c47738b0c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/internal/config/config.go b/internal/config/config.go index ad0acc0..186b395 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -133,6 +133,15 @@ type StorageConfig struct { // When exceeded, least recently used artifacts are evicted. // Empty or "0" means unlimited. MaxSize string `json:"max_size" yaml:"max_size"` + + // DirectServe enables redirecting cached artifact downloads to presigned + // storage URLs (HTTP 302) instead of streaming bytes through the proxy. + // Only effective for backends that support URL signing (S3, Azure). + DirectServe bool `json:"direct_serve" yaml:"direct_serve"` + + // DirectServeTTL is how long presigned URLs remain valid. + // Uses Go duration syntax (e.g. "5m", "1h"). Default: "15m". + DirectServeTTL string `json:"direct_serve_ttl" yaml:"direct_serve_ttl"` } // DatabaseConfig configures the cache database. @@ -303,6 +312,12 @@ func (c *Config) LoadFromEnv() { if v := os.Getenv("PROXY_STORAGE_MAX_SIZE"); v != "" { c.Storage.MaxSize = v } + if v := os.Getenv("PROXY_STORAGE_DIRECT_SERVE"); v != "" { + c.Storage.DirectServe = envBool(v) + } + if v := os.Getenv("PROXY_STORAGE_DIRECT_SERVE_TTL"); v != "" { + c.Storage.DirectServeTTL = v + } if v := os.Getenv("PROXY_DATABASE_DRIVER"); v != "" { c.Database.Driver = v } @@ -322,10 +337,10 @@ func (c *Config) LoadFromEnv() { c.Cooldown.Default = v } if v := os.Getenv("PROXY_CACHE_METADATA"); v != "" { - c.CacheMetadata = v == "true" || v == "1" + c.CacheMetadata = envBool(v) } if v := os.Getenv("PROXY_MIRROR_API"); v != "" { - c.MirrorAPI = v == "true" || v == "1" + c.MirrorAPI = envBool(v) } if v := os.Getenv("PROXY_METADATA_TTL"); v != "" { c.MetadataTTL = v @@ -379,6 +394,13 @@ func (c *Config) Validate() error { } } + // Validate direct serve TTL if specified + if c.Storage.DirectServeTTL != "" { + if _, err := time.ParseDuration(c.Storage.DirectServeTTL); err != nil { + return fmt.Errorf("invalid storage.direct_serve_ttl %q: %w", c.Storage.DirectServeTTL, err) + } + } + // Validate metadata TTL if specified if c.MetadataTTL != "" && c.MetadataTTL != "0" { if _, err := time.ParseDuration(c.MetadataTTL); err != nil { @@ -389,7 +411,10 @@ func (c *Config) Validate() error { return nil } -const defaultMetadataTTL = 5 * time.Minute //nolint:mnd // sensible default +const ( + defaultMetadataTTL = 5 * time.Minute //nolint:mnd // sensible default + defaultDirectServeTTL = 15 * time.Minute //nolint:mnd // sensible default +) // ParseMetadataTTL returns the metadata TTL duration. // Returns 5 minutes if unset, 0 if explicitly disabled. @@ -407,6 +432,19 @@ func (c *Config) ParseMetadataTTL() time.Duration { return d } +// ParseDirectServeTTL returns the presigned URL expiry duration. +// Returns 15 minutes if unset. +func (c *Config) ParseDirectServeTTL() time.Duration { + if c.Storage.DirectServeTTL == "" { + return defaultDirectServeTTL + } + d, err := time.ParseDuration(c.Storage.DirectServeTTL) + if err != nil { + return defaultDirectServeTTL + } + return d +} + // ParseSize parses a human-readable size string (e.g., "10GB", "500MB"). // Returns the size in bytes. func ParseSize(s string) (int64, error) { @@ -487,3 +525,7 @@ func (a *AuthConfig) Header() (name, value string) { func expandEnv(s string) string { return os.Expand(s, os.Getenv) } + +func envBool(v string) bool { + return v == "true" || v == "1" +} diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 6e8c3a0..967a4e5 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -355,3 +355,54 @@ func TestLoadMetadataTTLFromEnv(t *testing.T) { t.Errorf("MetadataTTL = %q, want %q", cfg.MetadataTTL, "10m") } } + +func TestParseDirectServeTTL(t *testing.T) { + tests := []struct { + name string + ttl string + want time.Duration + }{ + {"empty defaults to 15m", "", 15 * time.Minute}, + {"5 minutes", "5m", 5 * time.Minute}, + {"1 hour", "1h", 1 * time.Hour}, + {"invalid defaults to 15m", "not-a-duration", 15 * time.Minute}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cfg := Default() + cfg.Storage.DirectServeTTL = tt.ttl + got := cfg.ParseDirectServeTTL() + if got != tt.want { + t.Errorf("ParseDirectServeTTL() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestValidateDirectServeTTL(t *testing.T) { + cfg := Default() + cfg.Storage.DirectServeTTL = "invalid" + if err := cfg.Validate(); err == nil { + t.Error("expected validation error for invalid storage.direct_serve_ttl") + } + + cfg.Storage.DirectServeTTL = "5m" + if err := cfg.Validate(); err != nil { + t.Errorf("unexpected error for valid storage.direct_serve_ttl: %v", err) + } +} + +func TestLoadDirectServeFromEnv(t *testing.T) { + cfg := Default() + t.Setenv("PROXY_STORAGE_DIRECT_SERVE", "true") + t.Setenv("PROXY_STORAGE_DIRECT_SERVE_TTL", "30m") + cfg.LoadFromEnv() + + if !cfg.Storage.DirectServe { + t.Error("Storage.DirectServe should be true") + } + if cfg.Storage.DirectServeTTL != "30m" { + t.Errorf("Storage.DirectServeTTL = %q, want %q", cfg.Storage.DirectServeTTL, "30m") + } +} diff --git a/internal/handler/handler.go b/internal/handler/handler.go index d7d79c9..d4609cb 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -61,15 +61,17 @@ func ReadMetadata(r io.Reader) ([]byte, error) { // Proxy provides shared functionality for protocol handlers. type Proxy struct { - DB *database.DB - Storage storage.Storage - Fetcher fetch.FetcherInterface - Resolver *fetch.Resolver - Logger *slog.Logger - Cooldown *cooldown.Config - CacheMetadata bool - MetadataTTL time.Duration - HTTPClient *http.Client + DB *database.DB + Storage storage.Storage + Fetcher fetch.FetcherInterface + Resolver *fetch.Resolver + Logger *slog.Logger + Cooldown *cooldown.Config + CacheMetadata bool + MetadataTTL time.Duration + DirectServe bool + DirectServeTTL time.Duration + HTTPClient *http.Client } // NewProxy creates a new Proxy with the given dependencies. @@ -92,6 +94,7 @@ func NewProxy(db *database.DB, store storage.Storage, fetcher fetch.FetcherInter // CacheResult contains information about a cached or fetched artifact. type CacheResult struct { Reader io.ReadCloser + RedirectURL string Size int64 ContentType string Hash string @@ -138,6 +141,26 @@ func (p *Proxy) checkCache(ctx context.Context, pkgPURL, versionPURL, filename s return nil, nil } + result := &CacheResult{ + Size: artifact.Size.Int64, + ContentType: artifact.ContentType.String, + Hash: artifact.ContentHash.String, + Cached: true, + } + + if p.DirectServe { + url, err := p.Storage.SignedURL(ctx, artifact.StoragePath.String, p.DirectServeTTL) + if err == nil { + result.RedirectURL = url + p.recordCacheHit(pkgPURL, versionPURL, filename) + return result, nil + } + if !errors.Is(err, storage.ErrSignedURLUnsupported) { + p.Logger.Warn("failed to sign storage URL, falling back to streaming", + "path", artifact.StoragePath.String, "error", err) + } + } + start := time.Now() reader, err := p.Storage.Open(ctx, artifact.StoragePath.String) metrics.RecordStorageOperation("read", time.Since(start)) @@ -148,20 +171,16 @@ func (p *Proxy) checkCache(ctx context.Context, pkgPURL, versionPURL, filename s return nil, nil } + result.Reader = reader + p.recordCacheHit(pkgPURL, versionPURL, filename) + return result, nil +} + +func (p *Proxy) recordCacheHit(pkgPURL, versionPURL, filename string) { _ = p.DB.RecordArtifactHit(versionPURL, filename) - - // Extract ecosystem from pkgPURL for metrics - if p, err := purl.Parse(pkgPURL); err == nil { - metrics.RecordCacheHit(purl.PURLTypeToEcosystem(p.Type)) + if parsed, err := purl.Parse(pkgPURL); err == nil { + metrics.RecordCacheHit(purl.PURLTypeToEcosystem(parsed.Type)) } - - return &CacheResult{ - Reader: reader, - Size: artifact.Size.Int64, - ContentType: artifact.ContentType.String, - Hash: artifact.ContentHash.String, - Cached: true, - }, nil } func (p *Proxy) fetchAndCache(ctx context.Context, ecosystem, name, version, filename, pkgPURL, versionPURL string) (*CacheResult, error) { @@ -276,6 +295,15 @@ func (p *Proxy) updateCacheDB(ecosystem, name, filename, pkgPURL, versionPURL, u // ServeArtifact writes a CacheResult to an HTTP response. func ServeArtifact(w http.ResponseWriter, result *CacheResult) { + if result.RedirectURL != "" { + if result.Hash != "" { + w.Header().Set("ETag", fmt.Sprintf(`"%s"`, result.Hash)) + } + w.Header().Set("Location", result.RedirectURL) + w.WriteHeader(http.StatusFound) + return + } + defer func() { _ = result.Reader.Close() }() if result.ContentType != "" { diff --git a/internal/handler/handler_test.go b/internal/handler/handler_test.go index 78ed415..b544c40 100644 --- a/internal/handler/handler_test.go +++ b/internal/handler/handler_test.go @@ -21,9 +21,11 @@ import ( // mockStorage implements storage.Storage for testing. type mockStorage struct { - files map[string][]byte - storeErr error - openErr error + files map[string][]byte + storeErr error + openErr error + signedURL string + signErr error } func newMockStorage() *mockStorage { @@ -79,6 +81,16 @@ func (s *mockStorage) UsedSpace(_ context.Context) (int64, error) { return total, nil } +func (s *mockStorage) SignedURL(_ context.Context, _ string, _ time.Duration) (string, error) { + if s.signErr != nil { + return "", s.signErr + } + if s.signedURL == "" { + return "", storage.ErrSignedURLUnsupported + } + return s.signedURL, nil +} + func (s *mockStorage) URL() string { return "mem://" } func (s *mockStorage) Close() error { return nil } @@ -311,6 +323,145 @@ func TestGetOrFetchArtifactFromURL_CacheMiss_StorageMissing(t *testing.T) { } } +func TestGetOrFetchArtifact_DirectServe_Redirect(t *testing.T) { + proxy, db, store, fetcher := setupTestProxy(t) + seedPackage(t, db, store, "npm", "lodash", "4.17.21", "lodash-4.17.21.tgz", "cached content") + + proxy.DirectServe = true + proxy.DirectServeTTL = 15 * time.Minute + store.signedURL = "https://bucket.s3.amazonaws.com/npm/lodash?X-Amz-Signature=abc" + + result, err := proxy.GetOrFetchArtifact(context.Background(), "npm", "lodash", "4.17.21", "lodash-4.17.21.tgz") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if !result.Cached { + t.Error("expected result to be cached") + } + if result.RedirectURL != store.signedURL { + t.Errorf("RedirectURL = %q, want %q", result.RedirectURL, store.signedURL) + } + if result.Reader != nil { + t.Error("Reader should be nil when redirecting") + } + if fetcher.fetchCalled { + t.Error("fetcher should not be called on cache hit") + } + + // Hit count should still be recorded on the redirect path. + art, _ := db.GetArtifact("pkg:npm/lodash@4.17.21", "lodash-4.17.21.tgz") + if art == nil || art.HitCount != 1 { + t.Errorf("artifact hit count not recorded: %+v", art) + } +} + +func TestGetOrFetchArtifact_DirectServe_FallbackOnUnsupported(t *testing.T) { + proxy, db, store, _ := setupTestProxy(t) + seedPackage(t, db, store, "npm", "lodash", "4.17.21", "lodash-4.17.21.tgz", "cached content") + + proxy.DirectServe = true + // store.signedURL is empty so SignedURL returns ErrSignedURLUnsupported. + + result, err := proxy.GetOrFetchArtifact(context.Background(), "npm", "lodash", "4.17.21", "lodash-4.17.21.tgz") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + defer func() { _ = result.Reader.Close() }() + + if result.RedirectURL != "" { + t.Errorf("RedirectURL should be empty, got %q", result.RedirectURL) + } + if result.Reader == nil { + t.Fatal("Reader should be set when signing is unsupported") + } + body, _ := io.ReadAll(result.Reader) + if string(body) != "cached content" { + t.Errorf("got body %q, want %q", body, "cached content") + } +} + +func TestGetOrFetchArtifact_DirectServe_FallbackOnError(t *testing.T) { + proxy, db, store, _ := setupTestProxy(t) + seedPackage(t, db, store, "npm", "lodash", "4.17.21", "lodash-4.17.21.tgz", "cached content") + + proxy.DirectServe = true + store.signErr = errors.New("signing failed") + + result, err := proxy.GetOrFetchArtifact(context.Background(), "npm", "lodash", "4.17.21", "lodash-4.17.21.tgz") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + defer func() { _ = result.Reader.Close() }() + + if result.RedirectURL != "" { + t.Errorf("RedirectURL should be empty on signing error, got %q", result.RedirectURL) + } + if result.Reader == nil { + t.Fatal("Reader should be set when signing fails") + } +} + +func TestGetOrFetchArtifact_DirectServe_DisabledIgnoresSigning(t *testing.T) { + proxy, db, store, _ := setupTestProxy(t) + seedPackage(t, db, store, "npm", "lodash", "4.17.21", "lodash-4.17.21.tgz", "cached content") + + proxy.DirectServe = false + store.signedURL = "https://bucket.example/should-not-be-used" + + result, err := proxy.GetOrFetchArtifact(context.Background(), "npm", "lodash", "4.17.21", "lodash-4.17.21.tgz") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + defer func() { _ = result.Reader.Close() }() + + if result.RedirectURL != "" { + t.Errorf("RedirectURL should be empty when DirectServe is off, got %q", result.RedirectURL) + } +} + +func TestServeArtifact_Redirect(t *testing.T) { + w := httptest.NewRecorder() + ServeArtifact(w, &CacheResult{ + RedirectURL: "https://bucket.s3.amazonaws.com/file?sig=abc", + Hash: "abc123", + Cached: true, + }) + + if w.Code != http.StatusFound { + t.Errorf("status = %d, want %d", w.Code, http.StatusFound) + } + if loc := w.Header().Get("Location"); loc != "https://bucket.s3.amazonaws.com/file?sig=abc" { + t.Errorf("Location = %q", loc) + } + if etag := w.Header().Get("ETag"); etag != `"abc123"` { + t.Errorf("ETag = %q, want %q", etag, `"abc123"`) + } + if cl := w.Header().Get("Content-Length"); cl != "" { + t.Errorf("Content-Length should not be set on redirect, got %q", cl) + } +} + +func TestServeArtifact_Stream(t *testing.T) { + w := httptest.NewRecorder() + ServeArtifact(w, &CacheResult{ + Reader: io.NopCloser(strings.NewReader("payload")), + Size: 7, + ContentType: "application/octet-stream", + Hash: "abc123", + }) + + if w.Code != http.StatusOK { + t.Errorf("status = %d, want %d", w.Code, http.StatusOK) + } + if w.Body.String() != "payload" { + t.Errorf("body = %q, want %q", w.Body.String(), "payload") + } + if ct := w.Header().Get("Content-Type"); ct != "application/octet-stream" { + t.Errorf("Content-Type = %q", ct) + } +} + func TestGetOrFetchArtifactFromURL_CacheHit(t *testing.T) { proxy, db, store, fetcher := setupTestProxy(t) seedPackage(t, db, store, "pypi", "requests", "2.28.0", "requests-2.28.0.tar.gz", "pypi content") diff --git a/internal/server/server.go b/internal/server/server.go index 5d544a2..445c0db 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -148,6 +148,8 @@ func (s *Server) Start() error { proxy.Cooldown = cd proxy.CacheMetadata = s.cfg.CacheMetadata proxy.MetadataTTL = s.cfg.ParseMetadataTTL() + proxy.DirectServe = s.cfg.Storage.DirectServe + proxy.DirectServeTTL = s.cfg.ParseDirectServeTTL() // Create router with Chi r := chi.NewRouter() diff --git a/internal/storage/blob.go b/internal/storage/blob.go index 2d6af46..dc41668 100644 --- a/internal/storage/blob.go +++ b/internal/storage/blob.go @@ -6,12 +6,15 @@ import ( "encoding/hex" "fmt" "io" + "net/http" "os" "path/filepath" "runtime" "strings" + "time" "gocloud.dev/blob" + _ "gocloud.dev/blob/azureblob" _ "gocloud.dev/blob/fileblob" _ "gocloud.dev/blob/s3blob" "gocloud.dev/gcerrors" @@ -138,6 +141,20 @@ func (b *Blob) Delete(ctx context.Context, path string) error { return nil } +func (b *Blob) SignedURL(ctx context.Context, path string, expiry time.Duration) (string, error) { + url, err := b.bucket.SignedURL(ctx, path, &blob.SignedURLOptions{ + Method: http.MethodGet, + Expiry: expiry, + }) + if err != nil { + if gcerrors.Code(err) == gcerrors.Unimplemented { + return "", ErrSignedURLUnsupported + } + return "", fmt.Errorf("signing URL: %w", err) + } + return url, nil +} + func (b *Blob) Size(ctx context.Context, path string) (int64, error) { attrs, err := b.bucket.Attributes(ctx, path) if err != nil { diff --git a/internal/storage/blob_test.go b/internal/storage/blob_test.go index bb2d089..d80290b 100644 --- a/internal/storage/blob_test.go +++ b/internal/storage/blob_test.go @@ -10,6 +10,7 @@ import ( "runtime" "strings" "testing" + "time" ) func TestOpenBucket(t *testing.T) { @@ -188,6 +189,18 @@ func TestBlobLargeFile(t *testing.T) { assertLargeFileRoundTrip(t, createTestBlob(t)) } +func TestBlobSignedURLUnsupported(t *testing.T) { + b := createTestBlob(t) + ctx := context.Background() + + // fileblob has no URL signer configured, so this must surface as + // ErrSignedURLUnsupported rather than a generic error. + _, err := b.SignedURL(ctx, "test/file.txt", time.Minute) + if !errors.Is(err, ErrSignedURLUnsupported) { + t.Errorf("SignedURL on fileblob = %v, want ErrSignedURLUnsupported", err) + } +} + func TestBlobOverwrite(t *testing.T) { b := createTestBlob(t) ctx := context.Background() diff --git a/internal/storage/filesystem.go b/internal/storage/filesystem.go index cf6a1fe..7a4147a 100644 --- a/internal/storage/filesystem.go +++ b/internal/storage/filesystem.go @@ -8,6 +8,7 @@ import ( "io" "os" "path/filepath" + "time" ) // Filesystem implements Storage using the local filesystem. @@ -129,6 +130,10 @@ func (fs *Filesystem) Delete(ctx context.Context, path string) error { return nil } +func (fs *Filesystem) SignedURL(_ context.Context, _ string, _ time.Duration) (string, error) { + return "", ErrSignedURLUnsupported +} + func (fs *Filesystem) Size(ctx context.Context, path string) (int64, error) { fullPath := fs.fullPath(path) diff --git a/internal/storage/filesystem_test.go b/internal/storage/filesystem_test.go index 7b7828d..cdd7f6b 100644 --- a/internal/storage/filesystem_test.go +++ b/internal/storage/filesystem_test.go @@ -10,6 +10,7 @@ import ( "path/filepath" "strings" "testing" + "time" ) func TestNewFilesystem(t *testing.T) { @@ -236,6 +237,15 @@ func TestFilesystemLargeFile(t *testing.T) { assertLargeFileRoundTrip(t, createTestFilesystem(t)) } +func TestFilesystemSignedURLUnsupported(t *testing.T) { + fs := createTestFilesystem(t) + + _, err := fs.SignedURL(context.Background(), "test/file.txt", time.Minute) + if !errors.Is(err, ErrSignedURLUnsupported) { + t.Errorf("SignedURL = %v, want ErrSignedURLUnsupported", err) + } +} + func createTestFilesystem(t *testing.T) *Filesystem { t.Helper() dir := t.TempDir() diff --git a/internal/storage/storage.go b/internal/storage/storage.go index 8a9026c..0dba46a 100644 --- a/internal/storage/storage.go +++ b/internal/storage/storage.go @@ -15,12 +15,17 @@ import ( "encoding/hex" "errors" "io" + "time" ) const dirPermissions = 0755 var ( ErrNotFound = errors.New("artifact not found") + + // ErrSignedURLUnsupported is returned by SignedURL when the backend + // cannot generate presigned URLs (e.g. local filesystem). + ErrSignedURLUnsupported = errors.New("signed URLs not supported by storage backend") ) // Storage defines the interface for artifact storage backends. @@ -45,6 +50,10 @@ type Storage interface { // Returns ErrNotFound if the path does not exist. Size(ctx context.Context, path string) (int64, error) + // SignedURL returns a presigned URL granting time-limited GET access to path. + // Returns ErrSignedURLUnsupported if the backend cannot generate presigned URLs. + SignedURL(ctx context.Context, path string, expiry time.Duration) (string, error) + // UsedSpace returns the total bytes used by all stored content. UsedSpace(ctx context.Context) (int64, error) From 1ad182782de398bbf621a81c9a0bbb5301482210 Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Mon, 27 Apr 2026 12:14:37 +0100 Subject: [PATCH 4/9] Add storage.direct_serve_base_url to override presigned URL host When the proxy reaches storage at an internal address (127.0.0.1, a Docker service name) the presigned URLs it generates point there too, which is useless to external clients. This adds an optional base URL that replaces the scheme and host of signed URLs before they're returned, keeping the signed path and query intact. --- config.example.yaml | 8 ++++ internal/config/config.go | 18 +++++++++ internal/config/config_test.go | 23 +++++++++++ internal/handler/handler.go | 31 +++++++++++++-- internal/handler/handler_test.go | 68 ++++++++++++++++++++++++++++++++ internal/server/server.go | 1 + 6 files changed, 146 insertions(+), 3 deletions(-) diff --git a/config.example.yaml b/config.example.yaml index c272979..8f37450 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -38,6 +38,14 @@ storage: # How long presigned URLs remain valid (e.g. "5m", "1h"). Default: "15m". direct_serve_ttl: "15m" + # Public base URL to substitute into presigned URLs. Set this when the + # proxy reaches storage at an internal address (127.0.0.1, a Docker + # service name) but clients must use a public hostname. Only scheme and + # host are used; the signed path and query are preserved. For S3/MinIO + # the reverse proxy at this address must forward requests with the + # internal Host header or the SigV4 signature will not validate. + # direct_serve_base_url: "https://minio.example.com" + # Database configuration database: # Database driver: "sqlite" (default) or "postgres" diff --git a/internal/config/config.go b/internal/config/config.go index 186b395..7dafab1 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -51,6 +51,7 @@ import ( "encoding/base64" "encoding/json" "fmt" + "net/url" "os" "path/filepath" "strconv" @@ -142,6 +143,12 @@ type StorageConfig struct { // DirectServeTTL is how long presigned URLs remain valid. // Uses Go duration syntax (e.g. "5m", "1h"). Default: "15m". DirectServeTTL string `json:"direct_serve_ttl" yaml:"direct_serve_ttl"` + + // DirectServeBaseURL overrides the scheme and host of presigned URLs + // before returning them to clients. Useful when the proxy reaches + // storage at an internal address (e.g. 127.0.0.1 or a Docker hostname) + // but clients must use a public one. + DirectServeBaseURL string `json:"direct_serve_base_url" yaml:"direct_serve_base_url"` } // DatabaseConfig configures the cache database. @@ -318,6 +325,9 @@ func (c *Config) LoadFromEnv() { if v := os.Getenv("PROXY_STORAGE_DIRECT_SERVE_TTL"); v != "" { c.Storage.DirectServeTTL = v } + if v := os.Getenv("PROXY_STORAGE_DIRECT_SERVE_BASE_URL"); v != "" { + c.Storage.DirectServeBaseURL = v + } if v := os.Getenv("PROXY_DATABASE_DRIVER"); v != "" { c.Database.Driver = v } @@ -401,6 +411,14 @@ func (c *Config) Validate() error { } } + // Validate direct serve base URL if specified + if c.Storage.DirectServeBaseURL != "" { + u, err := url.Parse(c.Storage.DirectServeBaseURL) + if err != nil || u.Scheme == "" || u.Host == "" { + return fmt.Errorf("invalid storage.direct_serve_base_url %q: must be an absolute URL", c.Storage.DirectServeBaseURL) + } + } + // Validate metadata TTL if specified if c.MetadataTTL != "" && c.MetadataTTL != "0" { if _, err := time.ParseDuration(c.MetadataTTL); err != nil { diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 967a4e5..2d85fd6 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -397,6 +397,7 @@ func TestLoadDirectServeFromEnv(t *testing.T) { cfg := Default() t.Setenv("PROXY_STORAGE_DIRECT_SERVE", "true") t.Setenv("PROXY_STORAGE_DIRECT_SERVE_TTL", "30m") + t.Setenv("PROXY_STORAGE_DIRECT_SERVE_BASE_URL", "https://cdn.example.com") cfg.LoadFromEnv() if !cfg.Storage.DirectServe { @@ -405,4 +406,26 @@ func TestLoadDirectServeFromEnv(t *testing.T) { if cfg.Storage.DirectServeTTL != "30m" { t.Errorf("Storage.DirectServeTTL = %q, want %q", cfg.Storage.DirectServeTTL, "30m") } + if cfg.Storage.DirectServeBaseURL != "https://cdn.example.com" { + t.Errorf("Storage.DirectServeBaseURL = %q, want %q", cfg.Storage.DirectServeBaseURL, "https://cdn.example.com") + } +} + +func TestValidateDirectServeBaseURL(t *testing.T) { + cfg := Default() + + cfg.Storage.DirectServeBaseURL = "not a url" + if err := cfg.Validate(); err == nil { + t.Error("expected validation error for relative direct_serve_base_url") + } + + cfg.Storage.DirectServeBaseURL = "://bad" + if err := cfg.Validate(); err == nil { + t.Error("expected validation error for unparseable direct_serve_base_url") + } + + cfg.Storage.DirectServeBaseURL = "https://cdn.example.com" + if err := cfg.Validate(); err != nil { + t.Errorf("unexpected error for valid direct_serve_base_url: %v", err) + } } diff --git a/internal/handler/handler.go b/internal/handler/handler.go index d4609cb..746b9e8 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -10,6 +10,7 @@ import ( "io" "log/slog" "net/http" + "net/url" "strconv" "strings" "time" @@ -71,7 +72,11 @@ type Proxy struct { MetadataTTL time.Duration DirectServe bool DirectServeTTL time.Duration - HTTPClient *http.Client + // DirectServeBaseURL, if set, replaces the scheme and host of presigned + // URLs so clients receive a public address even when the proxy reaches + // storage at an internal one. + DirectServeBaseURL string + HTTPClient *http.Client } // NewProxy creates a new Proxy with the given dependencies. @@ -149,9 +154,9 @@ func (p *Proxy) checkCache(ctx context.Context, pkgPURL, versionPURL, filename s } if p.DirectServe { - url, err := p.Storage.SignedURL(ctx, artifact.StoragePath.String, p.DirectServeTTL) + signed, err := p.Storage.SignedURL(ctx, artifact.StoragePath.String, p.DirectServeTTL) if err == nil { - result.RedirectURL = url + result.RedirectURL = rewriteSignedURLHost(signed, p.DirectServeBaseURL) p.recordCacheHit(pkgPURL, versionPURL, filename) return result, nil } @@ -176,6 +181,26 @@ func (p *Proxy) checkCache(ctx context.Context, pkgPURL, versionPURL, filename s return result, nil } +// rewriteSignedURLHost replaces the scheme and host of a signed URL with those +// from baseURL, preserving the path and query (which carry the signature). +// Returns signed unchanged if baseURL is empty or either URL fails to parse. +func rewriteSignedURLHost(signed, baseURL string) string { + if baseURL == "" { + return signed + } + s, err := url.Parse(signed) + if err != nil { + return signed + } + b, err := url.Parse(baseURL) + if err != nil || b.Scheme == "" || b.Host == "" { + return signed + } + s.Scheme = b.Scheme + s.Host = b.Host + return s.String() +} + func (p *Proxy) recordCacheHit(pkgPURL, versionPURL, filename string) { _ = p.DB.RecordArtifactHit(versionPURL, filename) if parsed, err := purl.Parse(pkgPURL); err == nil { diff --git a/internal/handler/handler_test.go b/internal/handler/handler_test.go index b544c40..3a1d2ab 100644 --- a/internal/handler/handler_test.go +++ b/internal/handler/handler_test.go @@ -356,6 +356,74 @@ func TestGetOrFetchArtifact_DirectServe_Redirect(t *testing.T) { } } +func TestGetOrFetchArtifact_DirectServe_BaseURLRewrite(t *testing.T) { + proxy, db, store, _ := setupTestProxy(t) + seedPackage(t, db, store, "npm", "lodash", "4.17.21", "lodash-4.17.21.tgz", "cached content") + + proxy.DirectServe = true + proxy.DirectServeBaseURL = "https://cdn.example.com" + store.signedURL = "http://127.0.0.1:9000/bucket/npm/lodash?X-Amz-Signature=abc&X-Amz-Expires=900" + + result, err := proxy.GetOrFetchArtifact(context.Background(), "npm", "lodash", "4.17.21", "lodash-4.17.21.tgz") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + want := "https://cdn.example.com/bucket/npm/lodash?X-Amz-Signature=abc&X-Amz-Expires=900" + if result.RedirectURL != want { + t.Errorf("RedirectURL = %q, want %q", result.RedirectURL, want) + } +} + +func TestRewriteSignedURLHost(t *testing.T) { + tests := []struct { + name string + signed string + baseURL string + want string + }{ + { + "empty base url is no-op", + "http://127.0.0.1:9000/bucket/key?sig=abc", + "", + "http://127.0.0.1:9000/bucket/key?sig=abc", + }, + { + "replaces scheme and host", + "http://127.0.0.1:9000/bucket/key?sig=abc", + "https://cdn.example.com", + "https://cdn.example.com/bucket/key?sig=abc", + }, + { + "preserves path and query", + "http://minio:9000/bucket/npm/lodash/4.17.21/lodash.tgz?X-Amz-Signature=abc&X-Amz-Date=20260101", + "https://files.example.com", + "https://files.example.com/bucket/npm/lodash/4.17.21/lodash.tgz?X-Amz-Signature=abc&X-Amz-Date=20260101", + }, + { + "ignores base url path", + "http://127.0.0.1:9000/bucket/key?sig=abc", + "https://cdn.example.com/ignored", + "https://cdn.example.com/bucket/key?sig=abc", + }, + { + "invalid base url is no-op", + "http://127.0.0.1:9000/bucket/key?sig=abc", + "://bad", + "http://127.0.0.1:9000/bucket/key?sig=abc", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := rewriteSignedURLHost(tt.signed, tt.baseURL) + if got != tt.want { + t.Errorf("rewriteSignedURLHost(%q, %q) = %q, want %q", tt.signed, tt.baseURL, got, tt.want) + } + }) + } +} + func TestGetOrFetchArtifact_DirectServe_FallbackOnUnsupported(t *testing.T) { proxy, db, store, _ := setupTestProxy(t) seedPackage(t, db, store, "npm", "lodash", "4.17.21", "lodash-4.17.21.tgz", "cached content") diff --git a/internal/server/server.go b/internal/server/server.go index 445c0db..fdfda20 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -150,6 +150,7 @@ func (s *Server) Start() error { proxy.MetadataTTL = s.cfg.ParseMetadataTTL() proxy.DirectServe = s.cfg.Storage.DirectServe proxy.DirectServeTTL = s.cfg.ParseDirectServeTTL() + proxy.DirectServeBaseURL = s.cfg.Storage.DirectServeBaseURL // Create router with Chi r := chi.NewRouter() From f9f2a6ecd4501c5670fbd0ba97188bd52bf2ccf4 Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Mon, 27 Apr 2026 21:26:23 +0100 Subject: [PATCH 5/9] Use cosign v4 bundle format for release signing --- .goreleaser.yaml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.goreleaser.yaml b/.goreleaser.yaml index b6256de..e5881db 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -36,11 +36,10 @@ checksum: signs: - cmd: cosign - certificate: "${artifact}.pem" + signature: "${artifact}.cosign.bundle" args: - sign-blob - - "--output-certificate=${certificate}" - - "--output-signature=${signature}" + - "--bundle=${signature}" - "${artifact}" - "--yes" artifacts: checksum From 3a68ccef3e534919de3a3350c484c08941e3ffcf Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 30 Apr 2026 15:18:04 +0000 Subject: [PATCH 6/9] Bump modernc.org/sqlite from 1.48.2 to 1.49.1 Bumps [modernc.org/sqlite](https://gitlab.com/cznic/sqlite) from 1.48.2 to 1.49.1. - [Changelog](https://gitlab.com/cznic/sqlite/blob/master/CHANGELOG.md) - [Commits](https://gitlab.com/cznic/sqlite/compare/v1.48.2...v1.49.1) --- updated-dependencies: - dependency-name: modernc.org/sqlite dependency-version: 1.49.1 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- go.mod | 4 ++-- go.sum | 16 ++++++++-------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/go.mod b/go.mod index 1067a3a..8e76200 100644 --- a/go.mod +++ b/go.mod @@ -22,7 +22,7 @@ require ( golang.org/x/sync v0.20.0 google.golang.org/protobuf v1.36.11 gopkg.in/yaml.v3 v3.0.1 - modernc.org/sqlite v1.48.2 + modernc.org/sqlite v1.49.1 ) require ( @@ -291,7 +291,7 @@ require ( gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect honnef.co/go/tools v0.7.0 // indirect - modernc.org/libc v1.70.0 // indirect + modernc.org/libc v1.72.0 // indirect modernc.org/mathutil v1.7.1 // indirect modernc.org/memory v1.11.0 // indirect mvdan.cc/gofumpt v0.9.2 // indirect diff --git a/go.sum b/go.sum index 49b11f9..19f8695 100644 --- a/go.sum +++ b/go.sum @@ -851,10 +851,10 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.7.0 h1:w6WUp1VbkqPEgLz4rkBzH/CSU6HkoqNLp6GstyTx3lU= honnef.co/go/tools v0.7.0/go.mod h1:pm29oPxeP3P82ISxZDgIYeOaf9ta6Pi0EWvCFoLG2vc= -modernc.org/cc/v4 v4.27.1 h1:9W30zRlYrefrDV2JE2O8VDtJ1yPGownxciz5rrbQZis= -modernc.org/cc/v4 v4.27.1/go.mod h1:uVtb5OGqUKpoLWhqwNQo/8LwvoiEBLvZXIQ/SmO6mL0= -modernc.org/ccgo/v4 v4.32.0 h1:hjG66bI/kqIPX1b2yT6fr/jt+QedtP2fqojG2VrFuVw= -modernc.org/ccgo/v4 v4.32.0/go.mod h1:6F08EBCx5uQc38kMGl+0Nm0oWczoo1c7cgpzEry7Uc0= +modernc.org/cc/v4 v4.27.3 h1:uNCgn37E5U09mTv1XgskEVUJ8ADKpmFMPxzGJ0TSo+U= +modernc.org/cc/v4 v4.27.3/go.mod h1:3YjcbCqhoTTHPycJDRl2WZKKFj0nwcOIPBfEZK0Hdk8= +modernc.org/ccgo/v4 v4.32.4 h1:L5OB8rpEX4ZsXEQwGozRfJyJSFHbbNVOoQ59DU9/KuU= +modernc.org/ccgo/v4 v4.32.4/go.mod h1:lY7f+fiTDHfcv6YlRgSkxYfhs+UvOEEzj49jAn2TOx0= modernc.org/fileutil v1.4.0 h1:j6ZzNTftVS054gi281TyLjHPp6CPHr2KCxEXjEbD6SM= modernc.org/fileutil v1.4.0/go.mod h1:EqdKFDxiByqxLk8ozOxObDSfcVOv/54xDs/DUHdvCUU= modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI= @@ -863,8 +863,8 @@ modernc.org/gc/v3 v3.1.2 h1:ZtDCnhonXSZexk/AYsegNRV1lJGgaNZJuKjJSWKyEqo= modernc.org/gc/v3 v3.1.2/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY= modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks= modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI= -modernc.org/libc v1.70.0 h1:U58NawXqXbgpZ/dcdS9kMshu08aiA6b7gusEusqzNkw= -modernc.org/libc v1.70.0/go.mod h1:OVmxFGP1CI/Z4L3E0Q3Mf1PDE0BucwMkcXjjLntvHJo= +modernc.org/libc v1.72.0 h1:IEu559v9a0XWjw0DPoVKtXpO2qt5NVLAnFaBbjq+n8c= +modernc.org/libc v1.72.0/go.mod h1:tTU8DL8A+XLVkEY3x5E/tO7s2Q/q42EtnNWda/L5QhQ= modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI= @@ -873,8 +873,8 @@ modernc.org/opt v0.1.4 h1:2kNGMRiUjrp4LcaPuLY2PzUfqM/w9N23quVwhKt5Qm8= modernc.org/opt v0.1.4/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns= modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w= modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE= -modernc.org/sqlite v1.48.2 h1:5CnW4uP8joZtA0LedVqLbZV5GD7F/0x91AXeSyjoh5c= -modernc.org/sqlite v1.48.2/go.mod h1:hWjRO6Tj/5Ik8ieqxQybiEOUXy0NJFNp2tpvVpKlvig= +modernc.org/sqlite v1.49.1 h1:dYGHTKcX1sJ+EQDnUzvz4TJ5GbuvhNJa8Fg6ElGx73U= +modernc.org/sqlite v1.49.1/go.mod h1:m0w8xhwYUVY3H6pSDwc3gkJ/irZT/0YEXwBlhaxQEew= modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0= modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A= modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= From 96049c1f883150db48be3feb9bcd7616ec4eca20 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 30 Apr 2026 15:18:13 +0000 Subject: [PATCH 7/9] Bump goreleaser/goreleaser-action from 7.0.0 to 7.1.0 Bumps [goreleaser/goreleaser-action](https://github.com/goreleaser/goreleaser-action) from 7.0.0 to 7.1.0. - [Release notes](https://github.com/goreleaser/goreleaser-action/releases) - [Commits](https://github.com/goreleaser/goreleaser-action/compare/ec59f474b9834571250b370d4735c50f8e2d1e29...e24998b8b67b290c2fa8b7c14fcfa7de2c5c9b8c) --- updated-dependencies: - dependency-name: goreleaser/goreleaser-action dependency-version: 7.1.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f787509..cc79d4a 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -27,7 +27,7 @@ jobs: go-version-file: go.mod cache: false - - uses: goreleaser/goreleaser-action@ec59f474b9834571250b370d4735c50f8e2d1e29 # v7.0.0 + - uses: goreleaser/goreleaser-action@e24998b8b67b290c2fa8b7c14fcfa7de2c5c9b8c # v7.1.0 with: version: "~> v2" args: release --clean From 461a95c5183bb9b96afcb9119f1fcc3179801a9d Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Thu, 30 Apr 2026 18:09:01 +0100 Subject: [PATCH 8/9] Enforce max_size config with LRU cache eviction Closes #99. The max_size storage config was parsed and validated but never enforced. This adds a background eviction loop that periodically checks total cache size and evicts least recently used artifacts when the limit is exceeded. --- internal/config/config.go | 13 ++ internal/config/config_test.go | 25 +++ internal/server/eviction.go | 105 +++++++++++ internal/server/eviction_test.go | 290 +++++++++++++++++++++++++++++++ internal/server/server.go | 1 + 5 files changed, 434 insertions(+) create mode 100644 internal/server/eviction.go create mode 100644 internal/server/eviction_test.go diff --git a/internal/config/config.go b/internal/config/config.go index ad0acc0..2984d5c 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -391,6 +391,19 @@ func (c *Config) Validate() error { const defaultMetadataTTL = 5 * time.Minute //nolint:mnd // sensible default +// ParseMaxSize returns the maximum cache size in bytes. +// Returns 0 if unset or explicitly disabled (meaning unlimited). +func (c *Config) ParseMaxSize() int64 { + if c.Storage.MaxSize == "" || c.Storage.MaxSize == "0" { + return 0 + } + size, err := ParseSize(c.Storage.MaxSize) + if err != nil { + return 0 + } + return size +} + // ParseMetadataTTL returns the metadata TTL duration. // Returns 5 minutes if unset, 0 if explicitly disabled. func (c *Config) ParseMetadataTTL() time.Duration { diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 6e8c3a0..25f3488 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -303,6 +303,31 @@ func TestLoadFileNotFound(t *testing.T) { } } +func TestParseMaxSize(t *testing.T) { + tests := []struct { + name string + maxSize string + want int64 + }{ + {"empty means unlimited", "", 0}, + {"zero means unlimited", "0", 0}, + {"10GB", "10GB", 10 * 1024 * 1024 * 1024}, + {"500MB", "500MB", 500 * 1024 * 1024}, + {"invalid returns 0", "invalid", 0}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cfg := Default() + cfg.Storage.MaxSize = tt.maxSize + got := cfg.ParseMaxSize() + if got != tt.want { + t.Errorf("ParseMaxSize() = %d, want %d", got, tt.want) + } + }) + } +} + func TestParseMetadataTTL(t *testing.T) { tests := []struct { name string diff --git a/internal/server/eviction.go b/internal/server/eviction.go new file mode 100644 index 0000000..4173bd5 --- /dev/null +++ b/internal/server/eviction.go @@ -0,0 +1,105 @@ +package server + +import ( + "context" + "log/slog" + "time" + + "github.com/git-pkgs/proxy/internal/database" + "github.com/git-pkgs/proxy/internal/storage" +) + +const ( + evictionInterval = 1 * time.Minute + evictionBatch = 50 +) + +func (s *Server) startEvictionLoop(ctx context.Context) { + maxSize := s.cfg.ParseMaxSize() + if maxSize <= 0 { + return + } + + s.logger.Info("cache eviction enabled", "max_size", s.cfg.Storage.MaxSize) + + ticker := time.NewTicker(evictionInterval) + defer ticker.Stop() + + s.runEviction(ctx, maxSize) + + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + s.runEviction(ctx, maxSize) + } + } +} + +func (s *Server) runEviction(ctx context.Context, maxSize int64) { + evictLRU(ctx, s.db, s.storage, s.logger, maxSize) +} + +func evictLRU(ctx context.Context, db *database.DB, store storage.Storage, logger *slog.Logger, maxSize int64) { + totalSize, err := db.GetTotalCacheSize() + if err != nil { + logger.Warn("eviction: failed to get cache size", "error", err) + return + } + + if totalSize <= maxSize { + return + } + + logger.Info("eviction: cache size exceeds limit, evicting", + "current_size", totalSize, "max_size", maxSize) + + evicted := 0 + freedBytes := int64(0) + + for totalSize-freedBytes > maxSize { + artifacts, err := db.GetLeastRecentlyUsedArtifacts(evictionBatch) + if err != nil { + logger.Warn("eviction: failed to get LRU artifacts", "error", err) + return + } + if len(artifacts) == 0 { + break + } + + for _, art := range artifacts { + if totalSize-freedBytes <= maxSize { + break + } + + if !art.StoragePath.Valid { + continue + } + + if err := store.Delete(ctx, art.StoragePath.String); err != nil { + logger.Warn("eviction: failed to delete from storage", + "path", art.StoragePath.String, "error", err) + continue + } + + if err := db.ClearArtifactCache(art.VersionPURL, art.Filename); err != nil { + logger.Warn("eviction: failed to clear artifact record", + "version_purl", art.VersionPURL, "filename", art.Filename, "error", err) + continue + } + + size := int64(0) + if art.Size.Valid { + size = art.Size.Int64 + } + freedBytes += size + evicted++ + } + } + + if evicted > 0 { + logger.Info("eviction: completed", + "evicted", evicted, "freed_bytes", freedBytes) + } +} diff --git a/internal/server/eviction_test.go b/internal/server/eviction_test.go new file mode 100644 index 0000000..9fa9e6b --- /dev/null +++ b/internal/server/eviction_test.go @@ -0,0 +1,290 @@ +package server + +import ( + "context" + "database/sql" + "io" + "log/slog" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/git-pkgs/proxy/internal/config" + "github.com/git-pkgs/proxy/internal/database" + "github.com/git-pkgs/proxy/internal/storage" +) + +func setupEvictionTest(t *testing.T) (*database.DB, *storage.Filesystem) { + t.Helper() + + tempDir := t.TempDir() + dbPath := filepath.Join(tempDir, "test.db") + storagePath := filepath.Join(tempDir, "artifacts") + + db, err := database.Create(dbPath) + if err != nil { + t.Fatalf("failed to create database: %v", err) + } + + store, err := storage.NewFilesystem(storagePath) + if err != nil { + _ = db.Close() + t.Fatalf("failed to create storage: %v", err) + } + + t.Cleanup(func() { + _ = db.Close() + }) + + return db, store +} + +func seedArtifact(t *testing.T, ctx context.Context, db *database.DB, store storage.Storage, name string, dataSize int, accessedAt time.Time) { + t.Helper() + + pkgPURL := "pkg:npm/" + name + versionPURL := pkgPURL + "@1.0.0" + filename := name + "-1.0.0.tgz" + + if err := db.UpsertPackage(&database.Package{ + PURL: pkgPURL, + Ecosystem: "npm", + Name: name, + }); err != nil { + t.Fatalf("failed to upsert package: %v", err) + } + + if err := db.UpsertVersion(&database.Version{ + PURL: versionPURL, + PackagePURL: pkgPURL, + }); err != nil { + t.Fatalf("failed to upsert version: %v", err) + } + + storagePath := storage.ArtifactPath("npm", "", name, "1.0.0", filename) + data := strings.NewReader(strings.Repeat("x", dataSize)) + size, hash, err := store.Store(ctx, storagePath, data) + if err != nil { + t.Fatalf("failed to store artifact: %v", err) + } + + if err := db.UpsertArtifact(&database.Artifact{ + VersionPURL: versionPURL, + Filename: filename, + UpstreamURL: "https://example.com/" + filename, + StoragePath: sql.NullString{String: storagePath, Valid: true}, + ContentHash: sql.NullString{String: hash, Valid: true}, + Size: sql.NullInt64{Int64: size, Valid: true}, + ContentType: sql.NullString{String: "application/gzip", Valid: true}, + FetchedAt: sql.NullTime{Time: time.Now(), Valid: true}, + LastAccessedAt: sql.NullTime{Time: accessedAt, Valid: true}, + }); err != nil { + t.Fatalf("failed to upsert artifact: %v", err) + } +} + +func TestEvictLRU_NoEvictionWhenUnderLimit(t *testing.T) { + db, store := setupEvictionTest(t) + ctx := context.Background() + logger := slog.New(slog.NewTextHandler(io.Discard, nil)) + + seedArtifact(t, ctx, db, store, "pkg-a", 100, time.Now()) + + evictLRU(ctx, db, store, logger, 1024) + + count, err := db.GetCachedArtifactCount() + if err != nil { + t.Fatalf("failed to get count: %v", err) + } + if count != 1 { + t.Errorf("expected 1 cached artifact, got %d", count) + } +} + +func TestEvictLRU_EvictsOldestFirst(t *testing.T) { + db, store := setupEvictionTest(t) + ctx := context.Background() + logger := slog.New(slog.NewTextHandler(io.Discard, nil)) + + now := time.Now() + seedArtifact(t, ctx, db, store, "old-pkg", 500, now.Add(-3*time.Hour)) + seedArtifact(t, ctx, db, store, "mid-pkg", 500, now.Add(-1*time.Hour)) + seedArtifact(t, ctx, db, store, "new-pkg", 500, now) + + // Total is 1500 bytes, limit to 1100 so only the oldest gets evicted + evictLRU(ctx, db, store, logger, 1100) + + // old-pkg should be evicted + art, err := db.GetArtifact("pkg:npm/old-pkg@1.0.0", "old-pkg-1.0.0.tgz") + if err != nil { + t.Fatalf("failed to get artifact: %v", err) + } + if art.StoragePath.Valid { + t.Error("expected old-pkg to be evicted (storage_path should be NULL)") + } + + // mid-pkg and new-pkg should remain + art, err = db.GetArtifact("pkg:npm/mid-pkg@1.0.0", "mid-pkg-1.0.0.tgz") + if err != nil { + t.Fatalf("failed to get artifact: %v", err) + } + if !art.StoragePath.Valid { + t.Error("expected mid-pkg to remain cached") + } + + art, err = db.GetArtifact("pkg:npm/new-pkg@1.0.0", "new-pkg-1.0.0.tgz") + if err != nil { + t.Fatalf("failed to get artifact: %v", err) + } + if !art.StoragePath.Valid { + t.Error("expected new-pkg to remain cached") + } + + // Storage file should be removed for old-pkg + storagePath := storage.ArtifactPath("npm", "", "old-pkg", "1.0.0", "old-pkg-1.0.0.tgz") + exists, _ := store.Exists(ctx, storagePath) + if exists { + t.Error("expected old-pkg file to be deleted from storage") + } +} + +func TestEvictLRU_EvictsMultipleToGetUnderLimit(t *testing.T) { + db, store := setupEvictionTest(t) + ctx := context.Background() + logger := slog.New(slog.NewTextHandler(io.Discard, nil)) + + now := time.Now() + seedArtifact(t, ctx, db, store, "pkg-1", 400, now.Add(-4*time.Hour)) + seedArtifact(t, ctx, db, store, "pkg-2", 400, now.Add(-3*time.Hour)) + seedArtifact(t, ctx, db, store, "pkg-3", 400, now.Add(-2*time.Hour)) + seedArtifact(t, ctx, db, store, "pkg-4", 400, now) + + // Total is 1600 bytes, limit to 900 so pkg-1 and pkg-2 get evicted + evictLRU(ctx, db, store, logger, 900) + + count, err := db.GetCachedArtifactCount() + if err != nil { + t.Fatalf("failed to get count: %v", err) + } + if count != 2 { + t.Errorf("expected 2 cached artifacts remaining, got %d", count) + } + + // Verify the right ones remain + for _, name := range []string{"pkg-3", "pkg-4"} { + art, err := db.GetArtifact("pkg:npm/"+name+"@1.0.0", name+"-1.0.0.tgz") + if err != nil { + t.Fatalf("failed to get artifact %s: %v", name, err) + } + if !art.StoragePath.Valid { + t.Errorf("expected %s to remain cached", name) + } + } +} + +func TestEvictLRU_NothingToEvictWhenEmpty(t *testing.T) { + db, store := setupEvictionTest(t) + ctx := context.Background() + logger := slog.New(slog.NewTextHandler(io.Discard, nil)) + + // Should not panic or error with no artifacts + evictLRU(ctx, db, store, logger, 1024) + + count, err := db.GetCachedArtifactCount() + if err != nil { + t.Fatalf("failed to get count: %v", err) + } + if count != 0 { + t.Errorf("expected 0 cached artifacts, got %d", count) + } +} + +func TestEvictLRU_StorageFileDeleted(t *testing.T) { + db, store := setupEvictionTest(t) + ctx := context.Background() + logger := slog.New(slog.NewTextHandler(io.Discard, nil)) + + seedArtifact(t, ctx, db, store, "delete-me", 1000, time.Now().Add(-1*time.Hour)) + + storagePath := storage.ArtifactPath("npm", "", "delete-me", "1.0.0", "delete-me-1.0.0.tgz") + exists, _ := store.Exists(ctx, storagePath) + if !exists { + t.Fatal("expected artifact file to exist before eviction") + } + + evictLRU(ctx, db, store, logger, 500) + + exists, _ = store.Exists(ctx, storagePath) + if exists { + t.Error("expected artifact file to be deleted after eviction") + } + + art, err := db.GetArtifact("pkg:npm/delete-me@1.0.0", "delete-me-1.0.0.tgz") + if err != nil { + t.Fatalf("failed to get artifact: %v", err) + } + if art.StoragePath.Valid { + t.Error("expected storage_path to be NULL after eviction") + } + if art.Size.Valid { + t.Error("expected size to be NULL after eviction") + } +} + +func TestStartEvictionLoop_UnlimitedSkips(t *testing.T) { + tempDir := t.TempDir() + dbPath := filepath.Join(tempDir, "test.db") + storagePath := filepath.Join(tempDir, "artifacts") + + db, err := database.Create(dbPath) + if err != nil { + t.Fatalf("failed to create database: %v", err) + } + defer func() { _ = db.Close() }() + + store, err := storage.NewFilesystem(storagePath) + if err != nil { + t.Fatalf("failed to create storage: %v", err) + } + + cfg := defaultTestConfig(storagePath, dbPath) + + s := &Server{ + cfg: cfg, + db: db, + storage: store, + logger: slog.New(slog.NewTextHandler(io.Discard, nil)), + } + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + // Should return immediately since max_size is empty (unlimited) + done := make(chan struct{}) + go func() { + s.startEvictionLoop(ctx) + close(done) + }() + + select { + case <-done: + // Good, returned immediately + case <-time.After(1 * time.Second): + t.Error("startEvictionLoop should return immediately when max_size is unlimited") + cancel() + } +} + +func defaultTestConfig(storagePath, dbPath string) *config.Config { + return &config.Config{ + Listen: ":8080", + BaseURL: "http://localhost:8080", + Storage: config.StorageConfig{Path: storagePath, MaxSize: ""}, + Database: config.DatabaseConfig{ + Driver: "sqlite", + Path: dbPath, + }, + Log: config.LogConfig{Level: "info", Format: "text"}, + } +} diff --git a/internal/server/server.go b/internal/server/server.go index 5d544a2..2ae0e69 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -261,6 +261,7 @@ func (s *Server) Start() error { "storage", s.storage.URL(), "database", s.cfg.Database.Path) go s.updateCacheStatsMetrics() + go s.startEvictionLoop(bgCtx) return s.http.ListenAndServe() } From 25d5d741c32d1308598ac8cd286c3f06db7ebd1f Mon Sep 17 00:00:00 2001 From: "Kevin P. Fleming" Date: Sun, 19 Apr 2026 07:27:30 -0400 Subject: [PATCH 9/9] WIP --- config.example.yaml | 139 ++++++++++++++++++++++++++++++ internal/config/cargo/cargo.go | 78 +++++++++++++++++ internal/config/config.go | 28 ++++++ internal/config/debian/debian.go | 72 ++++++++++++++++ internal/config/ecosystem.go | 12 +++ internal/handler/cargo.go | 18 ++-- internal/handler/cargo_test.go | 3 +- internal/handler/debian.go | 14 ++- internal/handler/debian_test.go | 3 +- internal/handler/download_test.go | 3 +- internal/server/server.go | 16 +++- internal/server/server_test.go | 5 +- 12 files changed, 371 insertions(+), 20 deletions(-) create mode 100644 internal/config/cargo/cargo.go create mode 100644 internal/config/debian/debian.go create mode 100644 internal/config/ecosystem.go diff --git a/config.example.yaml b/config.example.yaml index 8f37450..cb4881e 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -66,6 +66,145 @@ log: # Log format: "text" or "json" format: "text" +# Ecosystem support - routes and upstream repositories +# +# This section is optional, since 'include_default' in each section +# defaults to 'true' and the route map will be populated with all of +# the default routes if no configuration is provided. +ecosystem: + cargo: + include_default: true + # the default route for crates.io + # route: + # - path: /cargo + # upstream: + # - name: crates.io + # index: https://index.crates.io + # crates: https://static.crates.io/crates + composer: + include_default: true + # the default route for packagist.org + # route: + # - path: /composer + # upstream: + # - name: packagist.org + # upstream: https://packagist.org + # repository: https://repo.packagist.org + conan: + include_default: true + # the default route for conan.io + # route: + # - path: /conan + # upstream: + # - name: conan.io + # upstream: https://center.conan.io + conda: + include_default: true + # the default route for anaconda.org + # route: + # - path: /conda + # upstream: + # - name: anaconda.org + # upstream: https://conda.anaconda.org + cran: + include_default: true + # the default route for r-project.org + # route: + # - path: /cran + # upstream: + # - name: r-project.org + # upstream: https://cloud.r-project.org + debian: + include_default: true + # the default route for debian.org + # route: + # - path: /debian + # upstream: + # - name: debian.org + # upstream: http://deb.debian.org/debian + gem: + include_default: true + # the default route for rubygems.org + # route: + # - path: /gem + # upstream: + # - name: rubygems.org + # upstream: https://rubygems.org + go: + include_default: true + # the default route for golang.org + # route: + # - path: /go + # upstream: + # - name: golang.org + # upstream: https://proxy.golang.org + hex: + include_default: true + # the default route for hex.pm + # route: + # - path: /hex + # upstream: + # - name: hex.pm + # upstream: https://repo.hex.pm + maven: + include_default: true + # the default route for maven.org + # route: + # - path: /maven + # upstream: + # - name: maven.org + # upstream: https://repo1.maven.org/maven2 + npm: + include_default: true + # the default route for npmjs.org + # route: + # - path: /npm + # upstream: + # - name: npmjs.org + # upstream: https://registry.npmjs.org + nuget: + include_default: true + # the default route for nuget.org + # route: + # - path: /nuget + # upstream: + # - name: nuget.org + # upstream: https://api.nuget.org + oci: + include_default: true + # the default route for docker.io + # route: + # - path: /v2 + # upstream: + # - name: docker.io + # registry: https://registry-1.docker.io + # auth: https://auth.docker.io + pub: + include_default: true + # the default route for pub.dev + # route: + # - path: /pub + # upstream: + # - name: pub.dev + # upstream: https://pub.dev + pypi: + include_default: true + # the default route for pypi.org + # route: + # - path: /pypi + # upstream: + # - name: pypi.org + # index: https://pypi.org + # files_host: files.pythonhosted.org + rpm: + include_default: true + # the default route for fedoraproject.org + # route: + # - path: /rpm + # upstream: + # - name: fedoraproject.org + # upstream: https://dl.fedoraproject.org/pub/fedora/linux + # Upstream registry URLs and authentication upstream: # npm registry URL diff --git a/internal/config/cargo/cargo.go b/internal/config/cargo/cargo.go new file mode 100644 index 0000000..3e9c3cf --- /dev/null +++ b/internal/config/cargo/cargo.go @@ -0,0 +1,78 @@ +package cargo + +import ( + "fmt" + "net/url" +) + +// Config configures routes +type Config struct { + IncludeDefault bool `json:"include_default" yaml:"include_default"` + Route []RouteConfig `json:"route" yaml:"route"` +} + +// RouteConfig configures a route +type RouteConfig struct { + Path string `json:"path" yaml:"path"` + Upstream []UpstreamConfig `json:"upstream" yaml:"upstream"` +} + +// UpstreamConfig configures an upstream (source) +type UpstreamConfig struct { + Name string `json:"name" yaml:"name"` + Index string `json:"index" yaml:"index"` + Crates string `json:"crates" yaml:"crates"` +} + +// RouteDefault is the default route +var RouteDefault = RouteConfig{ + Path: "/cargo", + Upstream: []UpstreamConfig{ + { + Name: "crates.io", + Index: "https://index.crates.io", + Crates: "https://static.crates.io/crates", + }, + }, +} + +func (c *Config) Validate() error { + for _, route := range c.Route { + if err := route.Validate(); err != nil { + return err + } + } + + return nil +} + +func (r *RouteConfig) Validate() error { + // TODO: validate Path + + if len(r.Upstream) == 0 { + return fmt.Errorf("cargo route %q does not have any upstreams", r.Path) + } + if len(r.Upstream) > 1 { + return fmt.Errorf("cargo route %q has multiple upstreams; this is not yet supported", r.Path) + } + + for _, upstream := range r.Upstream { + if err := upstream.Validate(); err != nil { + return err + } + } + + return nil +} + +func (u *UpstreamConfig) Validate() error { + if _, err := url.Parse(u.Index); err != nil { + return fmt.Errorf("cargo upstream index %q is not a valid URL", u.Index) + } + + if _, err := url.Parse(u.Crates); err != nil { + return fmt.Errorf("cargo upstream crates %q is not a valid URL", u.Crates) + } + + return nil +} diff --git a/internal/config/config.go b/internal/config/config.go index c69e462..3a47347 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -58,6 +58,8 @@ import ( "strings" "time" + "github.com/git-pkgs/proxy/internal/config/cargo" + "github.com/git-pkgs/proxy/internal/config/debian" "gopkg.in/yaml.v3" ) @@ -83,6 +85,9 @@ type Config struct { // Upstream configures upstream registry URLs (optional overrides). Upstream UpstreamConfig `json:"upstream" yaml:"upstream"` + // Ecosystem configures ecosystem routes and upstreams + Ecosystem EcosystemConfig `json:"ecosystem" yaml:"ecosystem"` + // Cooldown configures version age filtering to mitigate supply chain attacks. Cooldown CooldownConfig `json:"cooldown" yaml:"cooldown"` @@ -255,6 +260,14 @@ func Default() *Config { Level: "info", Format: "text", }, + Ecosystem: EcosystemConfig{ + Cargo: cargo.Config{ + IncludeDefault: true, + }, + Debian: debian.Config{ + IncludeDefault: true, + }, + }, Upstream: UpstreamConfig{ NPM: "https://registry.npmjs.org", Cargo: "https://index.crates.io", @@ -359,6 +372,14 @@ func (c *Config) LoadFromEnv() { // Validate checks the configuration for errors. func (c *Config) Validate() error { + // finalize the configuration by injecting default routes if requested + if c.Ecosystem.Cargo.IncludeDefault { + c.Ecosystem.Cargo.Route = append(c.Ecosystem.Cargo.Route, cargo.RouteDefault) + } + if c.Ecosystem.Debian.IncludeDefault { + c.Ecosystem.Debian.Route = append(c.Ecosystem.Debian.Route, debian.RouteDefault) + } + if c.Listen == "" { return fmt.Errorf("listen address is required") } @@ -426,6 +447,13 @@ func (c *Config) Validate() error { } } + if err := c.Ecosystem.Cargo.Validate(); err != nil { + return err + } + if err := c.Ecosystem.Debian.Validate(); err != nil { + return err + } + return nil } diff --git a/internal/config/debian/debian.go b/internal/config/debian/debian.go new file mode 100644 index 0000000..06c5f97 --- /dev/null +++ b/internal/config/debian/debian.go @@ -0,0 +1,72 @@ +package debian + +import ( + "fmt" + "net/url" +) + +// Config configures routes +type Config struct { + IncludeDefault bool `json:"include_default" yaml:"include_default"` + Route []RouteConfig `json:"route" yaml:"route"` +} + +// RouteConfig configures a route +type RouteConfig struct { + Path string `json:"path" yaml:"path"` + Upstream []UpstreamConfig `json:"upstream" yaml:"upstream"` +} + +// UpstreamConfig configures an upstream (source) +type UpstreamConfig struct { + Name string `json:"name" yaml:"name"` + Upstream string `json:"upstream" yaml:"upstream"` +} + +// RouteDefault is the default route +var RouteDefault = RouteConfig{ + Path: "/debian", + Upstream: []UpstreamConfig{ + { + Name: "debian.org", + Upstream: "http://deb.debian.org/debian", + }, + }, +} + +func (c *Config) Validate() error { + for _, route := range c.Route { + if err := route.Validate(); err != nil { + return err + } + } + + return nil +} + +func (r *RouteConfig) Validate() error { + // TODO: validate Path + + if len(r.Upstream) == 0 { + return fmt.Errorf("debian route %q does not have any upstreams", r.Path) + } + if len(r.Upstream) > 1 { + return fmt.Errorf("debian route %q has multiple upstreams; this is not yet supported", r.Path) + } + + for _, upstream := range r.Upstream { + if err := upstream.Validate(); err != nil { + return err + } + } + + return nil +} + +func (u *UpstreamConfig) Validate() error { + if _, err := url.Parse(u.Upstream); err != nil { + return fmt.Errorf("debian upstream upstream %q is not a valid URL", u.Upstream) + } + + return nil +} diff --git a/internal/config/ecosystem.go b/internal/config/ecosystem.go new file mode 100644 index 0000000..bc0bc62 --- /dev/null +++ b/internal/config/ecosystem.go @@ -0,0 +1,12 @@ +package config + +import ( + "github.com/git-pkgs/proxy/internal/config/cargo" + "github.com/git-pkgs/proxy/internal/config/debian" +) + +// Ecosystem configuration (routes and upstreams) +type EcosystemConfig struct { + Cargo cargo.Config `json:"cargo" yaml:"cargo"` + Debian debian.Config `json:"debian" yaml:"debian"` +} diff --git a/internal/handler/cargo.go b/internal/handler/cargo.go index 5d7810c..4c6f195 100644 --- a/internal/handler/cargo.go +++ b/internal/handler/cargo.go @@ -9,13 +9,11 @@ import ( "strings" "time" + "github.com/git-pkgs/proxy/internal/config/cargo" "github.com/git-pkgs/purl" ) const ( - cargoUpstream = "https://index.crates.io" - cargoDownloadBase = "https://static.crates.io/crates" - cargoIndexLen1 = 1 cargoIndexLen2 = 2 cargoIndexLen3 = 3 @@ -24,21 +22,27 @@ const ( // CargoHandler handles cargo registry protocol requests. type CargoHandler struct { proxy *Proxy + path string indexURL string downloadURL string proxyURL string } // NewCargoHandler creates a new cargo protocol handler. -func NewCargoHandler(proxy *Proxy, proxyURL string) *CargoHandler { +func NewCargoHandler(proxy *Proxy, proxyURL string, cfg cargo.RouteConfig) *CargoHandler { return &CargoHandler{ proxy: proxy, - indexURL: cargoUpstream, - downloadURL: cargoDownloadBase, + path: cfg.Path, + indexURL: cfg.Upstream[0].Index, + downloadURL: cfg.Upstream[0].Crates, proxyURL: strings.TrimSuffix(proxyURL, "/"), } } +func (h *CargoHandler) Path() string { + return h.path +} + // Routes returns the HTTP handler for cargo requests. // Mount this at /cargo on your router. func (h *CargoHandler) Routes() http.Handler { @@ -71,7 +75,7 @@ type CargoConfig struct { // handleConfig returns the registry configuration. func (h *CargoHandler) handleConfig(w http.ResponseWriter, r *http.Request) { config := CargoConfig{ - DL: h.proxyURL + "/cargo/crates/{crate}/{version}/download", + DL: h.proxyURL + h.path + "/crates/{crate}/{version}/download", } w.Header().Set("Content-Type", "application/json") diff --git a/internal/handler/cargo_test.go b/internal/handler/cargo_test.go index 5ce81b6..9f76a9c 100644 --- a/internal/handler/cargo_test.go +++ b/internal/handler/cargo_test.go @@ -48,6 +48,7 @@ func TestCargoBuildIndexPath(t *testing.T) { func TestCargoConfigEndpoint(t *testing.T) { h := &CargoHandler{ proxyURL: "http://localhost:8080", + path: "/xyzzy", } req := httptest.NewRequest(http.MethodGet, "/config.json", nil) @@ -64,7 +65,7 @@ func TestCargoConfigEndpoint(t *testing.T) { t.Fatalf("failed to parse config: %v", err) } - expectedDL := "http://localhost:8080/cargo/crates/{crate}/{version}/download" + expectedDL := "http://localhost:8080/xyzzy/crates/{crate}/{version}/download" if config.DL != expectedDL { t.Errorf("DL = %q, want %q", config.DL, expectedDL) } diff --git a/internal/handler/debian.go b/internal/handler/debian.go index b767f6d..8b313dd 100644 --- a/internal/handler/debian.go +++ b/internal/handler/debian.go @@ -2,33 +2,39 @@ package handler import ( "fmt" + "github.com/git-pkgs/proxy/internal/config/debian" "net/http" "regexp" "strings" ) const ( - debianUpstream = "http://deb.debian.org/debian" - debMatchCount = 4 // full match + name + version + arch + debMatchCount = 4 // full match + name + version + arch ) // DebianHandler handles APT/Debian repository protocol requests. // It proxies requests to upstream Debian/Ubuntu repositories and caches .deb packages. type DebianHandler struct { proxy *Proxy + path string upstreamURL string proxyURL string } // NewDebianHandler creates a new Debian/APT protocol handler. -func NewDebianHandler(proxy *Proxy, proxyURL string) *DebianHandler { +func NewDebianHandler(proxy *Proxy, proxyURL string, cfg debian.RouteConfig) *DebianHandler { return &DebianHandler{ proxy: proxy, - upstreamURL: debianUpstream, + path: cfg.Path, + upstreamURL: cfg.Upstream[0].Upstream, proxyURL: strings.TrimSuffix(proxyURL, "/"), } } +func (h *DebianHandler) Path() string { + return h.path +} + // Routes returns the HTTP handler for Debian requests. // Mount this at /debian on your router. func (h *DebianHandler) Routes() http.Handler { diff --git a/internal/handler/debian_test.go b/internal/handler/debian_test.go index dfdd326..6a26573 100644 --- a/internal/handler/debian_test.go +++ b/internal/handler/debian_test.go @@ -2,6 +2,7 @@ package handler import ( "testing" + "github.com/git-pkgs/proxy/internal/config/debian" ) func TestDebianHandler_parsePoolPath(t *testing.T) { @@ -18,6 +19,6 @@ func TestDebianHandler_parsePoolPath(t *testing.T) { } func TestDebianHandler_Routes(t *testing.T) { - h := NewDebianHandler(nil, "http://localhost:8080") + h := NewDebianHandler(nil, "http://localhost:8080", debian.RouteDefault) assertRoutesBasics(t, h.Routes(), "/dists/stable/Release", "/pool/../../../etc/passwd") } diff --git a/internal/handler/download_test.go b/internal/handler/download_test.go index 639e976..620b022 100644 --- a/internal/handler/download_test.go +++ b/internal/handler/download_test.go @@ -10,6 +10,7 @@ import ( "testing" "time" + "github.com/git-pkgs/proxy/internal/config/debian" "github.com/git-pkgs/proxy/internal/database" "github.com/git-pkgs/proxy/internal/storage" "github.com/git-pkgs/purl" @@ -897,7 +898,7 @@ func TestDebianHandler_DownloadCacheMiss(t *testing.T) { ContentType: "application/vnd.debian.binary-package", } - h := NewDebianHandler(proxy, "http://localhost") + h := NewDebianHandler(proxy, "http://localhost", debian.RouteDefault) srv := httptest.NewServer(h.Routes()) defer srv.Close() diff --git a/internal/server/server.go b/internal/server/server.go index ebf9268..e3b13af 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -173,7 +173,6 @@ func (s *Server) Start() error { // Mount protocol handlers npmHandler := handler.NewNPMHandler(proxy, s.cfg.BaseURL) - cargoHandler := handler.NewCargoHandler(proxy, s.cfg.BaseURL) gemHandler := handler.NewGemHandler(proxy, s.cfg.BaseURL) goHandler := handler.NewGoHandler(proxy, s.cfg.BaseURL) hexHandler := handler.NewHexHandler(proxy, s.cfg.BaseURL) @@ -186,11 +185,21 @@ func (s *Server) Start() error { condaHandler := handler.NewCondaHandler(proxy, s.cfg.BaseURL) cranHandler := handler.NewCRANHandler(proxy, s.cfg.BaseURL) containerHandler := handler.NewContainerHandler(proxy, s.cfg.BaseURL) - debianHandler := handler.NewDebianHandler(proxy, s.cfg.BaseURL) rpmHandler := handler.NewRPMHandler(proxy, s.cfg.BaseURL) + for _, route := range s.cfg.Ecosystem.Cargo.Route { + routeHandler := handler.NewCargoHandler(proxy, s.cfg.BaseURL, route) + r.Mount(routeHandler.Path(), http.StripPrefix(routeHandler.Path(), routeHandler.Routes())) + s.logger.Info("mounted handler", "ecosystem", "cargo", "path", routeHandler.Path()) + } + + for _, route := range s.cfg.Ecosystem.Debian.Route { + routeHandler := handler.NewDebianHandler(proxy, s.cfg.BaseURL, route) + r.Mount(routeHandler.Path(), http.StripPrefix(routeHandler.Path(), routeHandler.Routes())) + s.logger.Info("mounted handler", "ecosystem", "debian", "path", routeHandler.Path()) + } + r.Mount("/npm", http.StripPrefix("/npm", npmHandler.Routes())) - r.Mount("/cargo", http.StripPrefix("/cargo", cargoHandler.Routes())) r.Mount("/gem", http.StripPrefix("/gem", gemHandler.Routes())) r.Mount("/go", http.StripPrefix("/go", goHandler.Routes())) r.Mount("/hex", http.StripPrefix("/hex", hexHandler.Routes())) @@ -203,7 +212,6 @@ func (s *Server) Start() error { r.Mount("/conda", http.StripPrefix("/conda", condaHandler.Routes())) r.Mount("/cran", http.StripPrefix("/cran", cranHandler.Routes())) r.Mount("/v2", http.StripPrefix("/v2", containerHandler.Routes())) - r.Mount("/debian", http.StripPrefix("/debian", debianHandler.Routes())) r.Mount("/rpm", http.StripPrefix("/rpm", rpmHandler.Routes())) // Health, stats, and static endpoints diff --git a/internal/server/server_test.go b/internal/server/server_test.go index be88bf6..0155437 100644 --- a/internal/server/server_test.go +++ b/internal/server/server_test.go @@ -15,6 +15,7 @@ import ( "time" "github.com/git-pkgs/proxy/internal/config" + "github.com/git-pkgs/proxy/internal/config/cargo" "github.com/git-pkgs/proxy/internal/database" "github.com/git-pkgs/proxy/internal/handler" "github.com/git-pkgs/proxy/internal/storage" @@ -68,13 +69,13 @@ func newTestServer(t *testing.T) *testServer { // Mount handlers npmHandler := handler.NewNPMHandler(proxy, cfg.BaseURL) - cargoHandler := handler.NewCargoHandler(proxy, cfg.BaseURL) + cargoHandler := handler.NewCargoHandler(proxy, cfg.BaseURL, cargo.RouteDefault) gemHandler := handler.NewGemHandler(proxy, cfg.BaseURL) goHandler := handler.NewGoHandler(proxy, cfg.BaseURL) pypiHandler := handler.NewPyPIHandler(proxy, cfg.BaseURL) r.Mount("/npm", http.StripPrefix("/npm", npmHandler.Routes())) - r.Mount("/cargo", http.StripPrefix("/cargo", cargoHandler.Routes())) + r.Mount(cargoHandler.Path(), http.StripPrefix(cargoHandler.Path(), cargoHandler.Routes())) r.Mount("/gem", http.StripPrefix("/gem", gemHandler.Routes())) r.Mount("/go", http.StripPrefix("/go", goHandler.Routes())) r.Mount("/pypi", http.StripPrefix("/pypi", pypiHandler.Routes()))